예제 #1
0
 /**
  * @return int
  */
 public function execute()
 {
     $this->_logger->setInstanceName($this->instanceName);
     $this->_pidFile = new PidFile("", $this->instanceName);
     echo Shell::colourText((new Figlet("speed"))->render("Defero"), Shell::COLOUR_FOREGROUND_GREEN);
     echo "\n";
     Log::debug("Setting Default Queue Provider to " . $this->queueService);
     Queue::setDefaultQueueProvider($this->queueService);
     $queue = Queue::getAccessor();
     if ($queue instanceof DatabaseQueue) {
         $instance = gethostname();
         if ($this->instanceName) {
             $instance .= ':' . $this->instanceName;
         }
         $queue->setOwnKey($instance);
     }
     $priority = (int) $this->priority;
     if (in_array($priority, [1, 5, 10, 99])) {
         $this->queueName .= $priority;
     } else {
         throw new \Exception("Invalid priority. Supported values: 1 , 5, 10, 99");
     }
     Log::info("Starting to consume queue " . $this->queueName);
     $queue->consume(new StdQueue($this->queueName), new CampaignConsumer());
     Log::info("Exiting Defero Processor");
 }
예제 #2
0
 /**
  * Execute the process
  *
  * @return int|void
  */
 public function execute()
 {
     $articles = Article::collection();
     if ($articles->count() === 0) {
         echo 'No articles to list.';
         exit;
     }
     $platforms = Platform::collection();
     foreach ($articles as $article) {
         echo PHP_EOL;
         echo Shell::colourText("Title: " . $article->title . PHP_EOL, true, Shell::COLOUR_FOREGROUND_BROWN, Shell::COLOUR_BACKGROUND_RED);
         echo 'Article ID:  ' . $article->id() . PHP_EOL;
         echo 'Sub-Title:   ' . $article->subTitle . PHP_EOL;
         echo 'View:   ' . $article . PHP_EOL;
         foreach ($article->getBlockGroups() as $blockGroup) {
             foreach ($blockGroup->getBlocks() as $block) {
                 /** @var Platform $platform */
                 $platform = $platforms->getById($block->platformId);
                 echo PHP_EOL;
                 echo 'Block Platform: ' . $platform->name . PHP_EOL;
                 echo 'Block Title: ' . $block->title . PHP_EOL;
                 echo 'Block Content: ' . $block->content . PHP_EOL;
             }
         }
         echo PHP_EOL;
     }
     return $this;
 }
예제 #3
0
 /**
  * @param int $prefixLength
  */
 public static function buildRanges($prefixLength)
 {
     $db = BucketRange::conn();
     if ((new BucketRange())->tableExists()) {
         $db->query(ParseQuery::parse($db, 'DELETE FROM %T', BucketRange::tableName()));
     }
     $maxNum = pow(16, $prefixLength);
     for ($i = 0; $i < $maxNum; $i++) {
         $prefix = sprintf("%0" . $prefixLength . "x", $i);
         $range = new BucketRange($prefix);
         $range->processed = 0;
         $range->processing = 0;
         $range->randomKey = rand(1, 10000);
         $range->saveChanges();
         Shell::clearLine();
         echo "Building ranges: " . number_format($i + 1) . "/" . number_format($maxNum);
     }
     echo "\n";
 }
예제 #4
0
 /**
  * @return int
  */
 public function execute()
 {
     $this->_logger->setInstanceName($this->instanceName);
     $this->_pidFile = new PidFile("", $this->instanceName);
     echo Shell::colourText((new Figlet("speed"))->render("Defero"), Shell::COLOUR_FOREGROUND_GREEN);
     echo "\n";
     Log::debug("Setting Default Queue Provider to " . $this->queueService);
     Queue::setDefaultQueueProvider($this->queueService);
     $queue = Queue::getAccessor();
     if ($queue instanceof DatabaseQueue) {
         $instance = gethostname();
         if ($this->instanceName) {
             $instance .= ':' . $this->instanceName;
         }
         $queue->setOwnKey($instance);
     }
     Log::info("Starting to consume queue " . $this->queueName);
     $queue->consume(new StdQueue($this->queueName), new CampaignQueueConsumer());
     Log::info("Exiting Defero Processor");
 }
예제 #5
0
파일: Populate.php 프로젝트: qubes/support
 /**
  * Perform a database reset
  * --reset=true
  *
  * @return $this
  */
 protected function _resetDatabase()
 {
     if (!$this->reset) {
         return $this;
     }
     echo Shell::colourText("***** Resetting Database!!! *****", true, Shell::COLOUR_FOREGROUND_BROWN, Shell::COLOUR_BACKGROUND_RED);
     echo PHP_EOL;
     $reset = UserPrompt::confirm('Are you sure? You will lose ALL data?');
     if (!$reset) {
         exit;
     }
     /** @var RecordMapper[] $mappers */
     $mappers = [new User(), new Platform(), new PlatformText(), new Category(), new CategoryText(), new Article(), new ArticleText(), new ArticleSectionBlock(), new ArticleSection(), new Video(), new VideoCaption(), new VideoText(), new Walkthrough(), new WalkthroughStep(), new WalkthroughText()];
     foreach ($mappers as $mapper) {
         $mapper->createTable(true);
         echo Shell::colourText(sprintf("Dropped & Created `%s`", $mapper->tableName()), true, Shell::COLOUR_FOREGROUND_LIGHT_GREY, Shell::COLOUR_BACKGROUND_RED);
         echo PHP_EOL;
     }
     echo 'Done, now run without flag...' . PHP_EOL;
     exit;
 }
예제 #6
0
 private function _generatePrettyReport(TokenRange $currentRange, $now, $rangeStartTime, $rangeTotal, $rangeProcessed, $rangeErrors, $currentRate, $rangeSkipped, $totalSkipped, $totalDuration, $averageRate, $lastKey)
 {
     $t = new TextTable(new ReportTableDecorator());
     $t->appendSubHeading('Current Range ' . Shell::colourText("(" . $currentRange->id() . ")", Shell::COLOUR_FOREGROUND_LIGHT_GREY));
     $t->appendRows([['Start token', $currentRange->startToken], ['End token', $currentRange->endToken], ['First key', $currentRange->firstKey], ['Last key', $currentRange->lastKey]]);
     $t->appendSubHeading('Range statistics');
     $t->appendRows([['Processing time', DateTimeHelper::secondsToTime($now - $rangeStartTime)], ['Total items', number_format($rangeTotal)], ['Processed items', number_format($rangeProcessed)], ['Skipped', number_format($rangeSkipped)], ['Errors', number_format($rangeErrors)], ['Processing rate', number_format($currentRate) . ' items/second']]);
     $t->appendSubHeading('Total');
     $t->appendRows([['Processing time', DateTimeHelper::secondsToTime($totalDuration)], ['Total items', number_format($this->totalItems)], ['Processed items', number_format($this->processedItems)], ['Skipped', number_format($totalSkipped)], ['Errors', number_format($this->errors)], ['Processing rate', number_format($averageRate) . ' items/second']]);
     $t->appendSpacer();
     $t->appendRow(['Last key seen', $lastKey]);
     ob_start();
     echo $_REQUEST['__path__'] . "\n";
     echo $t;
     EventManager::trigger(Events::DISPLAY_REPORT_END);
     return ob_get_clean();
 }
예제 #7
0
 /**
  * @throws \Exception
  */
 public function execute()
 {
     Shell::clear();
     ob_start();
     Shell::clearToEol();
     $eolClear = ob_get_clean();
     $eol = $eolClear . "\n";
     $colSeparator = '  | ';
     $lastNumProcessed = 0;
     $firstNumProcessed = 0;
     $lastRangeRateTime = $startTime = time();
     $avgRangeRate = 0;
     $curRangeRate = 0;
     while (true) {
         $loopStartTime = microtime(true);
         $hostMaxLen = 0;
         $hosts = $this->_rangeStats->getProcessingHosts();
         foreach ($hosts as $hostInfo) {
             $len = strlen(self::hostDisplayName($hostInfo['hostname'], $hostInfo['instance']));
             if ($len > $hostMaxLen) {
                 $hostMaxLen = $len;
             }
         }
         $colWidth = $hostMaxLen + 14 + strlen($colSeparator);
         $winWidth = Shell::columns();
         $numCols = floor($winWidth / $colWidth);
         if ($numCols < 1) {
             $numCols = 1;
         }
         $curCol = 1;
         $allReports = StatsCollector::getReports($hosts);
         Shell::home();
         echo $eol;
         echo " Stats for " . $_REQUEST['__path__'] . $eol . $eol;
         $totalItems = 0;
         $processed = 0;
         $errors = 0;
         $skipped = 0;
         $totalRate = 0;
         $numNodes = 0;
         foreach ($hosts as $hostInfo) {
             $hostname = $hostInfo['hostname'];
             $instance = $hostInfo['instance'];
             $reportId = $hostname . '|' . $instance;
             $report = isset($allReports[$reportId]) ? $allReports[$reportId] : false;
             echo " " . str_pad(self::hostDisplayName($hostname, $instance), $hostMaxLen, ' ', STR_PAD_RIGHT);
             if ($report) {
                 $totalItems += $report->totalItems;
                 $processed += $report->totalProcessed;
                 $errors += $report->totalErrors;
                 $skipped += $report->totalSkipped;
                 $totalRate += $report->averageRate;
                 $numNodes++;
                 $rateNum = $report->averageRate;
                 printf(" %4s items/s", $rateNum);
             } else {
                 echo "         DOWN";
             }
             if ($curCol >= $numCols) {
                 echo $eol;
                 $curCol = 1;
             } else {
                 echo $colSeparator;
                 $curCol++;
             }
         }
         if ($curCol != 1) {
             echo $eol;
         }
         echo $eol;
         $avgRate = $numNodes > 0 ? round($totalRate / $numNodes) : 0;
         echo " Active Nodes : " . $numNodes . $eol;
         /*echo " Total Items  : " . $totalItems . $eol;
           echo " Processed    : " . $processed . $eol;
           echo " Skipped      : " . $skipped . $eol;
           echo " Errors       : " . $errors . $eol;*/
         echo " Total Rate   : " . number_format($totalRate) . " items/second" . $eol;
         echo " Average rate : " . number_format($avgRate) . " items/second" . $eol;
         echo $eol . $eol . " Range Statistics:" . $eol;
         $stats = $this->_rangeStats->getStats();
         $processedStats = isset($stats['processed']) ? $stats['processed'] : array();
         $numProcessed = empty($processedStats['Ranges']) ? 0 : $processedStats['Ranges'];
         $numUnprocessed = empty($stats['unprocessed']['Ranges']) ? 0 : $stats['unprocessed']['Ranges'];
         $numProcessing = empty($stats['processing']['Ranges']) ? 0 : $stats['processing']['Ranges'];
         $numFailed = empty($stats['failed']['Ranges']) ? 0 : $stats['failed']['Ranges'];
         $totalRanges = $numProcessed + $numUnprocessed + $numProcessing + $numFailed;
         if ($totalRanges > 0) {
             $processedPct = round($numProcessed * 100 / $totalRanges, 2);
             $unprocessedPct = round($numUnprocessed * 100 / $totalRanges, 2);
             $processingPct = round($numProcessing * 100 / $totalRanges, 2);
             $failedPct = round($numFailed * 100 / $totalRanges, 2);
         } else {
             $processedPct = 0;
             $unprocessedPct = 0;
             $processingPct = 0;
             $failedPct = 0;
         }
         $len = strlen(number_format($totalRanges));
         $fmt = "%-" . $len . "s (%0.2f%%)" . $eol;
         echo "  Processed   : " . sprintf($fmt, number_format($numProcessed), $processedPct);
         echo "  Unprocessed : " . sprintf($fmt, number_format($numUnprocessed), $unprocessedPct);
         echo "  Processing  : " . sprintf($fmt, number_format($numProcessing), $processingPct);
         $failedStr = "  Failed      : " . sprintf($fmt, number_format($numFailed), $failedPct);
         if ($numFailed > 0) {
             $failedStr = "" . $failedStr . "";
         }
         echo $failedStr;
         echo "  Requeued    : " . $this->_rangeStats->getRequeuedRangesDelta() . $eol;
         echo $eol;
         $fields = array('Total Items' => 'num', 'Processed Items' => 'num', '' => 'blank', 'Avg Range Time' => 'time', 'Max Range Time' => 'time', 'Avg Proc Items' => 'num', 'Max Proc Items' => 'num', 'Avg Total Items' => 'num', 'Max Total Items' => 'num');
         $labelWidth = 0;
         foreach ($fields as $field => $type) {
             if ($type != 'blank') {
                 $len = strlen($field);
                 if ($len > $labelWidth) {
                     $labelWidth = $len;
                 }
             }
         }
         foreach ($fields as $field => $type) {
             if ($type == 'blank') {
                 echo $eol;
             } else {
                 $value = empty($processedStats[$field]) ? 0 : $processedStats[$field];
                 echo '  ' . str_pad($field, $labelWidth, ' ', STR_PAD_RIGHT) . ' : ';
                 switch ($type) {
                     case 'num':
                         echo number_format($value);
                         break;
                     case 'rate':
                         echo number_format($value) . ' items/second';
                         break;
                     case 'time':
                         echo number_format($value) . ' seconds';
                         break;
                     default:
                         echo $processedStats[$field];
                         break;
                 }
                 echo $eol;
             }
         }
         // Calculate time remaining
         $avgRangeTime = $this->_rangeStats->getAvgRangeTime(30);
         $remainingMachineSeconds = $avgRangeTime * ($numUnprocessed + $numProcessing);
         $remainingSeconds = $numNodes > 0 ? round($remainingMachineSeconds / $numNodes) : 0;
         echo $eol;
         echo " Remaining time: " . self::formatTime($remainingSeconds) . $eol;
         // Calculate time remaining (method 2)
         $statsTotalItems = empty($processedStats['Total Items']) ? 0 : $processedStats['Total Items'];
         $estimatedTotalItems = $numProcessed == 0 || $totalRanges == 0 ? 0 : $statsTotalItems / ($numProcessed / $totalRanges);
         $estimatedRemainingItems = $estimatedTotalItems - $statsTotalItems;
         $remainingSeconds2 = $totalRate == 0 ? 0 : round($estimatedRemainingItems / $totalRate);
         echo $eol;
         echo " Estimated total items: " . number_format($estimatedTotalItems) . $eol;
         echo " Estimated remaining items: " . number_format($estimatedRemainingItems) . $eol;
         echo " Remaining time: " . self::formatTime($remainingSeconds2) . $eol;
         // Estimate number of items still to process
         $processedRatio = $statsTotalItems == 0 ? 0 : $processedStats['Processed Items'] / $statsTotalItems;
         $estimatedProcessedItems = $estimatedRemainingItems * $processedRatio;
         echo $eol;
         echo " Estimated remaining items to process: " . number_format($estimatedProcessedItems) . $eol;
         Shell::clearToEndOfScreen();
         // make sure it takes at least 1 second to go round the loop
         if (microtime(true) < $loopStartTime + 1) {
             time_sleep_until($loopStartTime + 1);
         }
     }
 }
예제 #8
0
 private function _generatePrettyReport(BucketRange $currentRange, $now, $rangeStartTime, $rangeTotal, $rangeProcessed, $currentRate, $rangeSkipped, $totalSkipped, $totalDuration, $averageRate, $lastObject)
 {
     $t = new TextTable(new ReportTableDecorator());
     $t->appendSubHeading('Current Range ' . Shell::colourText("(" . $currentRange->id() . ")", Shell::COLOUR_FOREGROUND_LIGHT_GREY));
     $t->appendRow(['Prefix', $currentRange->prefix]);
     $t->appendSubHeading('Range statistics');
     $t->appendRows([['Processing time', DateTimeHelper::secondsToTime($now - $rangeStartTime)], ['Total items', number_format($rangeTotal)], ['Processed items', number_format($rangeProcessed)], ['Skipped', number_format($rangeSkipped)], ['Processing rate', number_format($currentRate) . ' items/second']]);
     $t->appendSubHeading('Total');
     $t->appendRows([['Processing time', DateTimeHelper::secondsToTime($totalDuration)], ['Total items', number_format($this->totalItems)], ['Processed items', number_format($this->processedItems)], ['Skipped', number_format($totalSkipped)], ['Processing rate', number_format($averageRate) . ' items/second']]);
     $t->appendSpacer();
     $t->appendRow(['Last object seen', $lastObject]);
     ob_start();
     EventManager::trigger(Events::DISPLAY_REPORT_START);
     echo $t;
     EventManager::trigger(Events::DISPLAY_REPORT_END);
     return ob_get_clean();
 }
 private function _countRange($startKey, $endKey)
 {
     if ($startKey == "" && $endKey == "") {
         echo "Start key and end key are both blank\n";
         die;
     }
     $cass = Cassandra::getAccessor($this->_getCassServiceName());
     $cf = $cass->cf($this->_getColumnFamilyName(), false);
     echo "Counting range from '" . $startKey . "' to '" . $endKey . "'\n";
     $batchSize = 1000;
     $totalKeys = 1;
     $finished = false;
     $lastKey = $startKey;
     while (!$finished) {
         // ignore the duplicate key from each time around
         $totalKeys--;
         $items = $cf->getKeys($lastKey, $endKey, $batchSize, array());
         $cnt = count($items);
         $totalKeys += $cnt;
         $lastKey = last_key($items);
         if ($cnt < $batchSize || $lastKey == $endKey) {
             $finished = true;
         }
         Shell::clearLine();
         echo "Found " . number_format($totalKeys) . " keys";
     }
     echo "\n";
 }
예제 #10
0
 /**
  * For testing only: refresh the keys for all ranges
  */
 public function refreshKeysForAllRanges()
 {
     $result = $this->_multiGetRows(TokenRange::conn(), 'SELECT id FROM %T', $this->listAllRangeTables());
     $ids = [];
     foreach ($result as $row) {
         $ids[] = $row->id;
     }
     $total = count($ids);
     $processed = 0;
     foreach ($ids as $id) {
         $range = new TokenRange($id);
         $this->refreshKeysForRange($range);
         $processed++;
         Shell::clearLine();
         echo "Processed: " . number_format($processed) . ' / ' . number_format($total);
     }
     echo "\n";
 }