private function _generatePrettyReport(TokenRange $currentRange, $now, $rangeStartTime, $rangeTotal, $rangeProcessed, $rangeErrors, $currentRate, $rangeSkipped, $totalSkipped, $totalDuration, $averageRate, $lastKey) { $t = new TextTable(new ReportTableDecorator()); $t->appendSubHeading('Current Range ' . Shell::colourText("(" . $currentRange->id() . ")", Shell::COLOUR_FOREGROUND_LIGHT_GREY)); $t->appendRows([['Start token', $currentRange->startToken], ['End token', $currentRange->endToken], ['First key', $currentRange->firstKey], ['Last key', $currentRange->lastKey]]); $t->appendSubHeading('Range statistics'); $t->appendRows([['Processing time', DateTimeHelper::secondsToTime($now - $rangeStartTime)], ['Total items', number_format($rangeTotal)], ['Processed items', number_format($rangeProcessed)], ['Skipped', number_format($rangeSkipped)], ['Errors', number_format($rangeErrors)], ['Processing rate', number_format($currentRate) . ' items/second']]); $t->appendSubHeading('Total'); $t->appendRows([['Processing time', DateTimeHelper::secondsToTime($totalDuration)], ['Total items', number_format($this->totalItems)], ['Processed items', number_format($this->processedItems)], ['Skipped', number_format($totalSkipped)], ['Errors', number_format($this->errors)], ['Processing rate', number_format($averageRate) . ' items/second']]); $t->appendSpacer(); $t->appendRow(['Last key seen', $lastKey]); ob_start(); echo $_REQUEST['__path__'] . "\n"; echo $t; EventManager::trigger(Events::DISPLAY_REPORT_END); return ob_get_clean(); }
public function processRange(TokenRange $range) { Log::info("Processing range ID " . $range->id() . " from '" . $range->firstKey . "' to '" . $range->lastKey . "'..."); $this->_batchSizeTuner->reset(); $totalItems = 0; $processedItems = 0; $errors = 0; $rangeStartTime = microtime(true); $this->_processor->resetRangeData(); try { $cf = $this->_getCF(); $cols = $this->_processor->requiredColumns(); if (is_numeric($cols)) { $cols = new SlicePredicate(['slice_range' => $cf->makeSlice('', '', false, $cols)]); } $lastKey = $range->firstKey ? $range->firstKey : ""; $rangeLastKey = $range->lastKey ? $range->lastKey : ""; $finished = false; while (!$finished) { $this->_batchSizeTuner->nextBatch(); $batchSize = $this->_batchSizeTuner->getBatchSize(); $items = $this->_getKeysWithRetry($cf, $lastKey, $rangeLastKey, $batchSize, $cols); //$items = $cf->getKeys($lastKey, $rangeLastKey, $batchSize, $cols); if (!$items) { Log::info("Found no more items in range"); break; } // Skip the last item in the range because this will be the // first item in the next range if ($rangeLastKey != "" && last_key($items) == $rangeLastKey) { array_pop($items); } if ($this->_processor->supportsBatchProcessing()) { try { EventManager::trigger(Events::PROCESS_BATCH_START); $batchProcessed = $this->_processor->processBatch($items); EventManager::trigger(Events::PROCESS_BATCH_END); $processedItems += $batchProcessed; $totalItems += count($items); $this->_statsReporter->processedItems += $batchProcessed; $this->_statsReporter->totalItems += count($items); } catch (BatchException $e) { $errors += $e->getErrorCount(); $this->_statsReporter->errors += $e->getErrorCount(); $msg = $e->getMessage(); if ($msg != "") { $msg = 'Error processing batch: ' . $msg; } Log::error($msg); if ($this->_processor->stopOnErrors()) { die; } } } else { foreach ($items as $key => $itemData) { try { if ($this->_processor->processItem($key, $itemData)) { $processedItems++; $this->_statsReporter->processedItems++; } } catch (ItemException $e) { $errors++; $this->_statsReporter->errors++; Log::error('Error processing item ' . $key . ' : ' . $e->getMessage()); if ($this->_processor->stopOnErrors()) { die; } } $totalItems++; $this->_statsReporter->totalItems++; } } $lastKey = last_key($items); if ($this->_processor->shouldSaveProgress()) { $this->_scriptProgress->save($range->firstKey, $lastKey); } if ($lastKey == $rangeLastKey || count($items) < $batchSize) { $finished = true; } $this->_statsReporter->displayReport($finished, $range, $totalItems, $processedItems, $errors, $rangeStartTime, $lastKey); if ($errors > 0) { break; } } $rangeData = $this->_processor->getRangeData(); $range->rangeData = $rangeData ? json_encode($rangeData) : ''; $range->failed = 0; $range->error = ""; } catch (\Exception $e) { $msg = "Code " . $e->getCode(); $exMsg = $e->getMessage(); if ($exMsg != "") { $msg .= ": " . $exMsg; } Log::error('Error processing range: ' . $msg . "\n\nBacktrace:\n" . $e->getTraceAsString()); $range->error = $msg; // Check for non-fatal errors (i.e. Cassandra timeouts) if ($e instanceof CassandraException && ($e->getCode() == 408 || $e->getCode() == 503 || starts_with($e->getMessage(), 'TSocket: timed out'))) { $errors++; } else { $range->failed = 1; } } if ($errors > 0) { $this->_requeueRange($range); } else { $range->processing = 0; $range->processed = 1; $range->processingTime = microtime(true) - $rangeStartTime; $range->totalItems = $totalItems; $range->processedItems = $processedItems; $range->errorCount = $errors; $range->saveChanges(); } }