public function processRange(BucketRange $range)
{
Log::info('Processing range ' . $range->id());
$this->_statsReporter->nextRange($range);
$this->_processor->resetRangeData();
$listOpts = ['Bucket' => $this->_bucketName, 'MaxKeys' => $this->_batchSize, 'Prefix' => $range->prefix];
try {
if ($this->_processor->shouldSaveProgress() && !empty($range->lastObject)) {
$lastObject = $range->lastObject;
Log::info('Continuing from key ' . $lastObject);
} else {
$lastObject = "";
}
$gcs = $this->_getGCS();
$finished = false;
while (!$finished) {
if ($lastObject != "") {
$listOpts['Marker'] = $lastObject;
}
EventManager::trigger(Events::LIST_BUCKET_START);
$objects = $gcs->listObjects($listOpts);
EventManager::trigger(Events::LIST_BUCKET_END);
$numObjects = count($objects['Contents']);
if ($numObjects > 0) {
$items = [];
foreach ($objects['Contents'] as $object) {
$items[] = new ObjectInfo($object['Key'], $object['Size'], $object['ETag'], $object['LastModified']);
}
$batchProcessed = $this->_processor->processBatch($items);
$lastObjectObj = end($items);
$lastObject = $lastObjectObj->objectKey;
$range->lastObject = $lastObject;
$this->_statsReporter->addItems($numObjects, $batchProcessed);
if ($this->_processor->shouldSaveProgress()) {
$range->rangeData = $this->_processor->getRangeData();
$range->saveChanges();
}
} else {
Log::info('Finished processing range ' . $range->prefix . ' (' . $this->_statsReporter->rangeTotalItems . ' total items)');
$finished = true;
}
$this->_statsReporter->displayReport();
}
RangeManager::rangeProcessed($range, $this->_statsReporter->rangeStartTime, $this->_processor->getRangeData());
} catch (\Exception $e) {
$msg = "Code " . $e->getCode();
$exMsg = $e->getMessage();
if ($exMsg != "") {
$msg .= ": " . $exMsg;
}
Log::error('Error processing range: ' . $msg . "\n\nBacktrace:\n" . $e->getTraceAsString());
if ($this->_processor->isFatalException($e)) {
RangeManager::rangeFailed($range, $this->_statsReporter->rangeStartTime, $msg);
} else {
RangeManager::requeueRange($range, $this->_statsReporter->rangeStartTime, $msg);
}
}
}
public function init()
{
$this->_instanceName = $this->argumentValue('instance', '');
$this->_logger->setInstanceName($this->_instanceName);
if ($this->argumentIsSet('dry-run')) {
ScriptOptions::setDryRun(true);
}
BucketRange::setOverrideTableName($this->_getRangesTableName());
}
private static function _listRangesWhere($where, $limit = 0)
{
$query = "SELECT prefix, updatedAt, hostname, error FROM %T " . "WHERE " . $where;
if ($limit > 0) {
$query .= " LIMIT " . $limit;
}
$db = BucketRange::conn();
$ranges = $db->getRows(ParseQuery::parse($db, $query, BucketRange::tableName()));
$numRanges = count($ranges);
if ($numRanges > 0) {
$table = new TextTable();
$table->setColumnHeaders('prefix', 'updatedAt', 'hostname', 'error');
foreach ($ranges as $range) {
$table->appendRow([$range->prefix, $range->updatedAt, $range->hostname, $range->error]);
}
echo $table;
}
return $numRanges;
}
private function _generatePrettyReport(BucketRange $currentRange, $now, $rangeStartTime, $rangeTotal, $rangeProcessed, $currentRate, $rangeSkipped, $totalSkipped, $totalDuration, $averageRate, $lastObject)
{
$t = new TextTable(new ReportTableDecorator());
$t->appendSubHeading('Current Range ' . Shell::colourText("(" . $currentRange->id() . ")", Shell::COLOUR_FOREGROUND_LIGHT_GREY));
$t->appendRow(['Prefix', $currentRange->prefix]);
$t->appendSubHeading('Range statistics');
$t->appendRows([['Processing time', DateTimeHelper::secondsToTime($now - $rangeStartTime)], ['Total items', number_format($rangeTotal)], ['Processed items', number_format($rangeProcessed)], ['Skipped', number_format($rangeSkipped)], ['Processing rate', number_format($currentRate) . ' items/second']]);
$t->appendSubHeading('Total');
$t->appendRows([['Processing time', DateTimeHelper::secondsToTime($totalDuration)], ['Total items', number_format($this->totalItems)], ['Processed items', number_format($this->processedItems)], ['Skipped', number_format($totalSkipped)], ['Processing rate', number_format($averageRate) . ' items/second']]);
$t->appendSpacer();
$t->appendRow(['Last object seen', $lastObject]);
ob_start();
EventManager::trigger(Events::DISPLAY_REPORT_START);
echo $t;
EventManager::trigger(Events::DISPLAY_REPORT_END);
return ob_get_clean();
}
