public function processRange(BucketRange $range) { Log::info('Processing range ' . $range->id()); $this->_statsReporter->nextRange($range); $this->_processor->resetRangeData(); $listOpts = ['Bucket' => $this->_bucketName, 'MaxKeys' => $this->_batchSize, 'Prefix' => $range->prefix]; try { if ($this->_processor->shouldSaveProgress() && !empty($range->lastObject)) { $lastObject = $range->lastObject; Log::info('Continuing from key ' . $lastObject); } else { $lastObject = ""; } $gcs = $this->_getGCS(); $finished = false; while (!$finished) { if ($lastObject != "") { $listOpts['Marker'] = $lastObject; } EventManager::trigger(Events::LIST_BUCKET_START); $objects = $gcs->listObjects($listOpts); EventManager::trigger(Events::LIST_BUCKET_END); $numObjects = count($objects['Contents']); if ($numObjects > 0) { $items = []; foreach ($objects['Contents'] as $object) { $items[] = new ObjectInfo($object['Key'], $object['Size'], $object['ETag'], $object['LastModified']); } $batchProcessed = $this->_processor->processBatch($items); $lastObjectObj = end($items); $lastObject = $lastObjectObj->objectKey; $range->lastObject = $lastObject; $this->_statsReporter->addItems($numObjects, $batchProcessed); if ($this->_processor->shouldSaveProgress()) { $range->rangeData = $this->_processor->getRangeData(); $range->saveChanges(); } } else { Log::info('Finished processing range ' . $range->prefix . ' (' . $this->_statsReporter->rangeTotalItems . ' total items)'); $finished = true; } $this->_statsReporter->displayReport(); } RangeManager::rangeProcessed($range, $this->_statsReporter->rangeStartTime, $this->_processor->getRangeData()); } catch (\Exception $e) { $msg = "Code " . $e->getCode(); $exMsg = $e->getMessage(); if ($exMsg != "") { $msg .= ": " . $exMsg; } Log::error('Error processing range: ' . $msg . "\n\nBacktrace:\n" . $e->getTraceAsString()); if ($this->_processor->isFatalException($e)) { RangeManager::rangeFailed($range, $this->_statsReporter->rangeStartTime, $msg); } else { RangeManager::requeueRange($range, $this->_statsReporter->rangeStartTime, $msg); } } }
public function init() { $this->_instanceName = $this->argumentValue('instance', ''); $this->_logger->setInstanceName($this->_instanceName); if ($this->argumentIsSet('dry-run')) { ScriptOptions::setDryRun(true); } BucketRange::setOverrideTableName($this->_getRangesTableName()); }
private static function _listRangesWhere($where, $limit = 0) { $query = "SELECT prefix, updatedAt, hostname, error FROM %T " . "WHERE " . $where; if ($limit > 0) { $query .= " LIMIT " . $limit; } $db = BucketRange::conn(); $ranges = $db->getRows(ParseQuery::parse($db, $query, BucketRange::tableName())); $numRanges = count($ranges); if ($numRanges > 0) { $table = new TextTable(); $table->setColumnHeaders('prefix', 'updatedAt', 'hostname', 'error'); foreach ($ranges as $range) { $table->appendRow([$range->prefix, $range->updatedAt, $range->hostname, $range->error]); } echo $table; } return $numRanges; }
private function _generatePrettyReport(BucketRange $currentRange, $now, $rangeStartTime, $rangeTotal, $rangeProcessed, $currentRate, $rangeSkipped, $totalSkipped, $totalDuration, $averageRate, $lastObject) { $t = new TextTable(new ReportTableDecorator()); $t->appendSubHeading('Current Range ' . Shell::colourText("(" . $currentRange->id() . ")", Shell::COLOUR_FOREGROUND_LIGHT_GREY)); $t->appendRow(['Prefix', $currentRange->prefix]); $t->appendSubHeading('Range statistics'); $t->appendRows([['Processing time', DateTimeHelper::secondsToTime($now - $rangeStartTime)], ['Total items', number_format($rangeTotal)], ['Processed items', number_format($rangeProcessed)], ['Skipped', number_format($rangeSkipped)], ['Processing rate', number_format($currentRate) . ' items/second']]); $t->appendSubHeading('Total'); $t->appendRows([['Processing time', DateTimeHelper::secondsToTime($totalDuration)], ['Total items', number_format($this->totalItems)], ['Processed items', number_format($this->processedItems)], ['Skipped', number_format($totalSkipped)], ['Processing rate', number_format($averageRate) . ' items/second']]); $t->appendSpacer(); $t->appendRow(['Last object seen', $lastObject]); ob_start(); EventManager::trigger(Events::DISPLAY_REPORT_START); echo $t; EventManager::trigger(Events::DISPLAY_REPORT_END); return ob_get_clean(); }