/**
* @return int
*/
public function execute()
{
$this->_logger->setInstanceName($this->instanceName);
$this->_pidFile = new PidFile("", $this->instanceName);
echo Shell::colourText((new Figlet("speed"))->render("Defero"), Shell::COLOUR_FOREGROUND_GREEN);
echo "\n";
Log::debug("Setting Default Queue Provider to " . $this->queueService);
Queue::setDefaultQueueProvider($this->queueService);
$queue = Queue::getAccessor();
if ($queue instanceof DatabaseQueue) {
$instance = gethostname();
if ($this->instanceName) {
$instance .= ':' . $this->instanceName;
}
$queue->setOwnKey($instance);
}
$priority = (int) $this->priority;
if (in_array($priority, [1, 5, 10, 99])) {
$this->queueName .= $priority;
} else {
throw new \Exception("Invalid priority. Supported values: 1 , 5, 10, 99");
}
Log::info("Starting to consume queue " . $this->queueName);
$queue->consume(new StdQueue($this->queueName), new CampaignConsumer());
Log::info("Exiting Defero Processor");
}
/**
* Execute the process
*
* @return int|void
*/
public function execute()
{
$articles = Article::collection();
if ($articles->count() === 0) {
echo 'No articles to list.';
exit;
}
$platforms = Platform::collection();
foreach ($articles as $article) {
echo PHP_EOL;
echo Shell::colourText("Title: " . $article->title . PHP_EOL, true, Shell::COLOUR_FOREGROUND_BROWN, Shell::COLOUR_BACKGROUND_RED);
echo 'Article ID: ' . $article->id() . PHP_EOL;
echo 'Sub-Title: ' . $article->subTitle . PHP_EOL;
echo 'View: ' . $article . PHP_EOL;
foreach ($article->getBlockGroups() as $blockGroup) {
foreach ($blockGroup->getBlocks() as $block) {
/** @var Platform $platform */
$platform = $platforms->getById($block->platformId);
echo PHP_EOL;
echo 'Block Platform: ' . $platform->name . PHP_EOL;
echo 'Block Title: ' . $block->title . PHP_EOL;
echo 'Block Content: ' . $block->content . PHP_EOL;
}
}
echo PHP_EOL;
}
return $this;
}
/**
* @param int $prefixLength
*/
public static function buildRanges($prefixLength)
{
$db = BucketRange::conn();
if ((new BucketRange())->tableExists()) {
$db->query(ParseQuery::parse($db, 'DELETE FROM %T', BucketRange::tableName()));
}
$maxNum = pow(16, $prefixLength);
for ($i = 0; $i < $maxNum; $i++) {
$prefix = sprintf("%0" . $prefixLength . "x", $i);
$range = new BucketRange($prefix);
$range->processed = 0;
$range->processing = 0;
$range->randomKey = rand(1, 10000);
$range->saveChanges();
Shell::clearLine();
echo "Building ranges: " . number_format($i + 1) . "/" . number_format($maxNum);
}
echo "\n";
}
/**
* @return int
*/
public function execute()
{
$this->_logger->setInstanceName($this->instanceName);
$this->_pidFile = new PidFile("", $this->instanceName);
echo Shell::colourText((new Figlet("speed"))->render("Defero"), Shell::COLOUR_FOREGROUND_GREEN);
echo "\n";
Log::debug("Setting Default Queue Provider to " . $this->queueService);
Queue::setDefaultQueueProvider($this->queueService);
$queue = Queue::getAccessor();
if ($queue instanceof DatabaseQueue) {
$instance = gethostname();
if ($this->instanceName) {
$instance .= ':' . $this->instanceName;
}
$queue->setOwnKey($instance);
}
Log::info("Starting to consume queue " . $this->queueName);
$queue->consume(new StdQueue($this->queueName), new CampaignQueueConsumer());
Log::info("Exiting Defero Processor");
}
/**
* Perform a database reset
* --reset=true
*
* @return $this
*/
protected function _resetDatabase()
{
if (!$this->reset) {
return $this;
}
echo Shell::colourText("***** Resetting Database!!! *****", true, Shell::COLOUR_FOREGROUND_BROWN, Shell::COLOUR_BACKGROUND_RED);
echo PHP_EOL;
$reset = UserPrompt::confirm('Are you sure? You will lose ALL data?');
if (!$reset) {
exit;
}
/** @var RecordMapper[] $mappers */
$mappers = [new User(), new Platform(), new PlatformText(), new Category(), new CategoryText(), new Article(), new ArticleText(), new ArticleSectionBlock(), new ArticleSection(), new Video(), new VideoCaption(), new VideoText(), new Walkthrough(), new WalkthroughStep(), new WalkthroughText()];
foreach ($mappers as $mapper) {
$mapper->createTable(true);
echo Shell::colourText(sprintf("Dropped & Created `%s`", $mapper->tableName()), true, Shell::COLOUR_FOREGROUND_LIGHT_GREY, Shell::COLOUR_BACKGROUND_RED);
echo PHP_EOL;
}
echo 'Done, now run without flag...' . PHP_EOL;
exit;
}
private function _generatePrettyReport(TokenRange $currentRange, $now, $rangeStartTime, $rangeTotal, $rangeProcessed, $rangeErrors, $currentRate, $rangeSkipped, $totalSkipped, $totalDuration, $averageRate, $lastKey)
{
$t = new TextTable(new ReportTableDecorator());
$t->appendSubHeading('Current Range ' . Shell::colourText("(" . $currentRange->id() . ")", Shell::COLOUR_FOREGROUND_LIGHT_GREY));
$t->appendRows([['Start token', $currentRange->startToken], ['End token', $currentRange->endToken], ['First key', $currentRange->firstKey], ['Last key', $currentRange->lastKey]]);
$t->appendSubHeading('Range statistics');
$t->appendRows([['Processing time', DateTimeHelper::secondsToTime($now - $rangeStartTime)], ['Total items', number_format($rangeTotal)], ['Processed items', number_format($rangeProcessed)], ['Skipped', number_format($rangeSkipped)], ['Errors', number_format($rangeErrors)], ['Processing rate', number_format($currentRate) . ' items/second']]);
$t->appendSubHeading('Total');
$t->appendRows([['Processing time', DateTimeHelper::secondsToTime($totalDuration)], ['Total items', number_format($this->totalItems)], ['Processed items', number_format($this->processedItems)], ['Skipped', number_format($totalSkipped)], ['Errors', number_format($this->errors)], ['Processing rate', number_format($averageRate) . ' items/second']]);
$t->appendSpacer();
$t->appendRow(['Last key seen', $lastKey]);
ob_start();
echo $_REQUEST['__path__'] . "\n";
echo $t;
EventManager::trigger(Events::DISPLAY_REPORT_END);
return ob_get_clean();
}
/**
* @throws \Exception
*/
public function execute()
{
Shell::clear();
ob_start();
Shell::clearToEol();
$eolClear = ob_get_clean();
$eol = $eolClear . "\n";
$colSeparator = ' | ';
$lastNumProcessed = 0;
$firstNumProcessed = 0;
$lastRangeRateTime = $startTime = time();
$avgRangeRate = 0;
$curRangeRate = 0;
while (true) {
$loopStartTime = microtime(true);
$hostMaxLen = 0;
$hosts = $this->_rangeStats->getProcessingHosts();
foreach ($hosts as $hostInfo) {
$len = strlen(self::hostDisplayName($hostInfo['hostname'], $hostInfo['instance']));
if ($len > $hostMaxLen) {
$hostMaxLen = $len;
}
}
$colWidth = $hostMaxLen + 14 + strlen($colSeparator);
$winWidth = Shell::columns();
$numCols = floor($winWidth / $colWidth);
if ($numCols < 1) {
$numCols = 1;
}
$curCol = 1;
$allReports = StatsCollector::getReports($hosts);
Shell::home();
echo $eol;
echo " Stats for " . $_REQUEST['__path__'] . $eol . $eol;
$totalItems = 0;
$processed = 0;
$errors = 0;
$skipped = 0;
$totalRate = 0;
$numNodes = 0;
foreach ($hosts as $hostInfo) {
$hostname = $hostInfo['hostname'];
$instance = $hostInfo['instance'];
$reportId = $hostname . '|' . $instance;
$report = isset($allReports[$reportId]) ? $allReports[$reportId] : false;
echo " " . str_pad(self::hostDisplayName($hostname, $instance), $hostMaxLen, ' ', STR_PAD_RIGHT);
if ($report) {
$totalItems += $report->totalItems;
$processed += $report->totalProcessed;
$errors += $report->totalErrors;
$skipped += $report->totalSkipped;
$totalRate += $report->averageRate;
$numNodes++;
$rateNum = $report->averageRate;
printf(" %4s items/s", $rateNum);
} else {
echo " [0;31mDOWN[0m";
}
if ($curCol >= $numCols) {
echo $eol;
$curCol = 1;
} else {
echo $colSeparator;
$curCol++;
}
}
if ($curCol != 1) {
echo $eol;
}
echo $eol;
$avgRate = $numNodes > 0 ? round($totalRate / $numNodes) : 0;
echo " Active Nodes : " . $numNodes . $eol;
/*echo " Total Items : " . $totalItems . $eol;
echo " Processed : " . $processed . $eol;
echo " Skipped : " . $skipped . $eol;
echo " Errors : " . $errors . $eol;*/
echo " Total Rate : " . number_format($totalRate) . " items/second" . $eol;
echo " Average rate : " . number_format($avgRate) . " items/second" . $eol;
echo $eol . $eol . " Range Statistics:" . $eol;
$stats = $this->_rangeStats->getStats();
$processedStats = isset($stats['processed']) ? $stats['processed'] : array();
$numProcessed = empty($processedStats['Ranges']) ? 0 : $processedStats['Ranges'];
$numUnprocessed = empty($stats['unprocessed']['Ranges']) ? 0 : $stats['unprocessed']['Ranges'];
$numProcessing = empty($stats['processing']['Ranges']) ? 0 : $stats['processing']['Ranges'];
$numFailed = empty($stats['failed']['Ranges']) ? 0 : $stats['failed']['Ranges'];
$totalRanges = $numProcessed + $numUnprocessed + $numProcessing + $numFailed;
if ($totalRanges > 0) {
$processedPct = round($numProcessed * 100 / $totalRanges, 2);
$unprocessedPct = round($numUnprocessed * 100 / $totalRanges, 2);
$processingPct = round($numProcessing * 100 / $totalRanges, 2);
$failedPct = round($numFailed * 100 / $totalRanges, 2);
} else {
$processedPct = 0;
$unprocessedPct = 0;
$processingPct = 0;
$failedPct = 0;
}
$len = strlen(number_format($totalRanges));
$fmt = "%-" . $len . "s (%0.2f%%)" . $eol;
echo " Processed : " . sprintf($fmt, number_format($numProcessed), $processedPct);
echo " Unprocessed : " . sprintf($fmt, number_format($numUnprocessed), $unprocessedPct);
echo " Processing : " . sprintf($fmt, number_format($numProcessing), $processingPct);
$failedStr = " Failed : " . sprintf($fmt, number_format($numFailed), $failedPct);
if ($numFailed > 0) {
$failedStr = "[0;31m" . $failedStr . "[0m";
}
echo $failedStr;
echo " Requeued : " . $this->_rangeStats->getRequeuedRangesDelta() . $eol;
echo $eol;
$fields = array('Total Items' => 'num', 'Processed Items' => 'num', '' => 'blank', 'Avg Range Time' => 'time', 'Max Range Time' => 'time', 'Avg Proc Items' => 'num', 'Max Proc Items' => 'num', 'Avg Total Items' => 'num', 'Max Total Items' => 'num');
$labelWidth = 0;
foreach ($fields as $field => $type) {
if ($type != 'blank') {
$len = strlen($field);
if ($len > $labelWidth) {
$labelWidth = $len;
}
}
}
foreach ($fields as $field => $type) {
if ($type == 'blank') {
echo $eol;
} else {
$value = empty($processedStats[$field]) ? 0 : $processedStats[$field];
echo ' ' . str_pad($field, $labelWidth, ' ', STR_PAD_RIGHT) . ' : ';
switch ($type) {
case 'num':
echo number_format($value);
break;
case 'rate':
echo number_format($value) . ' items/second';
break;
case 'time':
echo number_format($value) . ' seconds';
break;
default:
echo $processedStats[$field];
break;
}
echo $eol;
}
}
// Calculate time remaining
$avgRangeTime = $this->_rangeStats->getAvgRangeTime(30);
$remainingMachineSeconds = $avgRangeTime * ($numUnprocessed + $numProcessing);
$remainingSeconds = $numNodes > 0 ? round($remainingMachineSeconds / $numNodes) : 0;
echo $eol;
echo " Remaining time: " . self::formatTime($remainingSeconds) . $eol;
// Calculate time remaining (method 2)
$statsTotalItems = empty($processedStats['Total Items']) ? 0 : $processedStats['Total Items'];
$estimatedTotalItems = $numProcessed == 0 || $totalRanges == 0 ? 0 : $statsTotalItems / ($numProcessed / $totalRanges);
$estimatedRemainingItems = $estimatedTotalItems - $statsTotalItems;
$remainingSeconds2 = $totalRate == 0 ? 0 : round($estimatedRemainingItems / $totalRate);
echo $eol;
echo " Estimated total items: " . number_format($estimatedTotalItems) . $eol;
echo " Estimated remaining items: " . number_format($estimatedRemainingItems) . $eol;
echo " Remaining time: " . self::formatTime($remainingSeconds2) . $eol;
// Estimate number of items still to process
$processedRatio = $statsTotalItems == 0 ? 0 : $processedStats['Processed Items'] / $statsTotalItems;
$estimatedProcessedItems = $estimatedRemainingItems * $processedRatio;
echo $eol;
echo " Estimated remaining items to process: " . number_format($estimatedProcessedItems) . $eol;
Shell::clearToEndOfScreen();
// make sure it takes at least 1 second to go round the loop
if (microtime(true) < $loopStartTime + 1) {
time_sleep_until($loopStartTime + 1);
}
}
}
private function _generatePrettyReport(BucketRange $currentRange, $now, $rangeStartTime, $rangeTotal, $rangeProcessed, $currentRate, $rangeSkipped, $totalSkipped, $totalDuration, $averageRate, $lastObject)
{
$t = new TextTable(new ReportTableDecorator());
$t->appendSubHeading('Current Range ' . Shell::colourText("(" . $currentRange->id() . ")", Shell::COLOUR_FOREGROUND_LIGHT_GREY));
$t->appendRow(['Prefix', $currentRange->prefix]);
$t->appendSubHeading('Range statistics');
$t->appendRows([['Processing time', DateTimeHelper::secondsToTime($now - $rangeStartTime)], ['Total items', number_format($rangeTotal)], ['Processed items', number_format($rangeProcessed)], ['Skipped', number_format($rangeSkipped)], ['Processing rate', number_format($currentRate) . ' items/second']]);
$t->appendSubHeading('Total');
$t->appendRows([['Processing time', DateTimeHelper::secondsToTime($totalDuration)], ['Total items', number_format($this->totalItems)], ['Processed items', number_format($this->processedItems)], ['Skipped', number_format($totalSkipped)], ['Processing rate', number_format($averageRate) . ' items/second']]);
$t->appendSpacer();
$t->appendRow(['Last object seen', $lastObject]);
ob_start();
EventManager::trigger(Events::DISPLAY_REPORT_START);
echo $t;
EventManager::trigger(Events::DISPLAY_REPORT_END);
return ob_get_clean();
}
private function _countRange($startKey, $endKey)
{
if ($startKey == "" && $endKey == "") {
echo "Start key and end key are both blank\n";
die;
}
$cass = Cassandra::getAccessor($this->_getCassServiceName());
$cf = $cass->cf($this->_getColumnFamilyName(), false);
echo "Counting range from '" . $startKey . "' to '" . $endKey . "'\n";
$batchSize = 1000;
$totalKeys = 1;
$finished = false;
$lastKey = $startKey;
while (!$finished) {
// ignore the duplicate key from each time around
$totalKeys--;
$items = $cf->getKeys($lastKey, $endKey, $batchSize, array());
$cnt = count($items);
$totalKeys += $cnt;
$lastKey = last_key($items);
if ($cnt < $batchSize || $lastKey == $endKey) {
$finished = true;
}
Shell::clearLine();
echo "Found " . number_format($totalKeys) . " keys";
}
echo "\n";
}
/**
* For testing only: refresh the keys for all ranges
*/
public function refreshKeysForAllRanges()
{
$result = $this->_multiGetRows(TokenRange::conn(), 'SELECT id FROM %T', $this->listAllRangeTables());
$ids = [];
foreach ($result as $row) {
$ids[] = $row->id;
}
$total = count($ids);
$processed = 0;
foreach ($ids as $id) {
$range = new TokenRange($id);
$this->refreshKeysForRange($range);
$processed++;
Shell::clearLine();
echo "Processed: " . number_format($processed) . ' / ' . number_format($total);
}
echo "\n";
}