private function _generatePrettyReport(TokenRange $currentRange, $now, $rangeStartTime, $rangeTotal, $rangeProcessed, $rangeErrors, $currentRate, $rangeSkipped, $totalSkipped, $totalDuration, $averageRate, $lastKey)
{
$t = new TextTable(new ReportTableDecorator());
$t->appendSubHeading('Current Range ' . Shell::colourText("(" . $currentRange->id() . ")", Shell::COLOUR_FOREGROUND_LIGHT_GREY));
$t->appendRows([['Start token', $currentRange->startToken], ['End token', $currentRange->endToken], ['First key', $currentRange->firstKey], ['Last key', $currentRange->lastKey]]);
$t->appendSubHeading('Range statistics');
$t->appendRows([['Processing time', DateTimeHelper::secondsToTime($now - $rangeStartTime)], ['Total items', number_format($rangeTotal)], ['Processed items', number_format($rangeProcessed)], ['Skipped', number_format($rangeSkipped)], ['Errors', number_format($rangeErrors)], ['Processing rate', number_format($currentRate) . ' items/second']]);
$t->appendSubHeading('Total');
$t->appendRows([['Processing time', DateTimeHelper::secondsToTime($totalDuration)], ['Total items', number_format($this->totalItems)], ['Processed items', number_format($this->processedItems)], ['Skipped', number_format($totalSkipped)], ['Errors', number_format($this->errors)], ['Processing rate', number_format($averageRate) . ' items/second']]);
$t->appendSpacer();
$t->appendRow(['Last key seen', $lastKey]);
ob_start();
echo $_REQUEST['__path__'] . "\n";
echo $t;
EventManager::trigger(Events::DISPLAY_REPORT_END);
return ob_get_clean();
}
public function processRange(TokenRange $range)
{
Log::info("Processing range ID " . $range->id() . " from '" . $range->firstKey . "' to '" . $range->lastKey . "'...");
$this->_batchSizeTuner->reset();
$totalItems = 0;
$processedItems = 0;
$errors = 0;
$rangeStartTime = microtime(true);
$this->_processor->resetRangeData();
try {
$cf = $this->_getCF();
$cols = $this->_processor->requiredColumns();
if (is_numeric($cols)) {
$cols = new SlicePredicate(['slice_range' => $cf->makeSlice('', '', false, $cols)]);
}
$lastKey = $range->firstKey ? $range->firstKey : "";
$rangeLastKey = $range->lastKey ? $range->lastKey : "";
$finished = false;
while (!$finished) {
$this->_batchSizeTuner->nextBatch();
$batchSize = $this->_batchSizeTuner->getBatchSize();
$items = $this->_getKeysWithRetry($cf, $lastKey, $rangeLastKey, $batchSize, $cols);
//$items = $cf->getKeys($lastKey, $rangeLastKey, $batchSize, $cols);
if (!$items) {
Log::info("Found no more items in range");
break;
}
// Skip the last item in the range because this will be the
// first item in the next range
if ($rangeLastKey != "" && last_key($items) == $rangeLastKey) {
array_pop($items);
}
if ($this->_processor->supportsBatchProcessing()) {
try {
EventManager::trigger(Events::PROCESS_BATCH_START);
$batchProcessed = $this->_processor->processBatch($items);
EventManager::trigger(Events::PROCESS_BATCH_END);
$processedItems += $batchProcessed;
$totalItems += count($items);
$this->_statsReporter->processedItems += $batchProcessed;
$this->_statsReporter->totalItems += count($items);
} catch (BatchException $e) {
$errors += $e->getErrorCount();
$this->_statsReporter->errors += $e->getErrorCount();
$msg = $e->getMessage();
if ($msg != "") {
$msg = 'Error processing batch: ' . $msg;
}
Log::error($msg);
if ($this->_processor->stopOnErrors()) {
die;
}
}
} else {
foreach ($items as $key => $itemData) {
try {
if ($this->_processor->processItem($key, $itemData)) {
$processedItems++;
$this->_statsReporter->processedItems++;
}
} catch (ItemException $e) {
$errors++;
$this->_statsReporter->errors++;
Log::error('Error processing item ' . $key . ' : ' . $e->getMessage());
if ($this->_processor->stopOnErrors()) {
die;
}
}
$totalItems++;
$this->_statsReporter->totalItems++;
}
}
$lastKey = last_key($items);
if ($this->_processor->shouldSaveProgress()) {
$this->_scriptProgress->save($range->firstKey, $lastKey);
}
if ($lastKey == $rangeLastKey || count($items) < $batchSize) {
$finished = true;
}
$this->_statsReporter->displayReport($finished, $range, $totalItems, $processedItems, $errors, $rangeStartTime, $lastKey);
if ($errors > 0) {
break;
}
}
$rangeData = $this->_processor->getRangeData();
$range->rangeData = $rangeData ? json_encode($rangeData) : '';
$range->failed = 0;
$range->error = "";
} catch (\Exception $e) {
$msg = "Code " . $e->getCode();
$exMsg = $e->getMessage();
if ($exMsg != "") {
$msg .= ": " . $exMsg;
}
Log::error('Error processing range: ' . $msg . "\n\nBacktrace:\n" . $e->getTraceAsString());
$range->error = $msg;
// Check for non-fatal errors (i.e. Cassandra timeouts)
if ($e instanceof CassandraException && ($e->getCode() == 408 || $e->getCode() == 503 || starts_with($e->getMessage(), 'TSocket: timed out'))) {
$errors++;
} else {
$range->failed = 1;
}
}
if ($errors > 0) {
$this->_requeueRange($range);
} else {
$range->processing = 0;
$range->processed = 1;
$range->processingTime = microtime(true) - $rangeStartTime;
$range->totalItems = $totalItems;
$range->processedItems = $processedItems;
$range->errorCount = $errors;
$range->saveChanges();
}
}
