예제 #1
0
 protected function execute(InputInterface $input, OutputInterface $output)
 {
     $systemCheck = new SystemCheck();
     $systemCheck->checkRedisIsInstalled();
     $settings = Queue\Factory::getSettings();
     if ($settings->queueEnabled->getValue()) {
         $output->writeln('Queue is enabled');
     } else {
         $output->writeln('<comment>' . strtoupper('Queue is disabled: ') . 'No new requests will be written into the queue, processing the remaining requests is still possible.</comment>');
     }
     $backend = Queue\Factory::makeBackend();
     $manager = Queue\Factory::makeQueueManager($backend);
     $queues = $manager->getAllQueues();
     $lock = Queue\Factory::makeLock($backend);
     if ($settings->processDuringTrackingRequest->getValue()) {
         $output->writeln('Request sets in the queue will be processed automatically after a tracking request');
     } else {
         $output->writeln('The command <comment>./console queuedtracking:process</comment> has to be executed to process request sets within queue');
     }
     $output->writeln(sprintf('Up to %d workers will be used', $manager->getNumberOfAvailableQueues()));
     $output->writeln(sprintf('Processor will start once there are at least %s request sets in the queue', $manager->getNumberOfRequestsToProcessAtSameTime()));
     while (1) {
         $memory = $backend->getMemoryStats();
         // I know this will only work with redis currently as it is not defined in backend interface etc. needs to be refactored once we add another backend
         $numInQueue = array();
         foreach ($queues as $queue) {
             $numInQueue[] = $queue->getNumberOfRequestSetsInQueue();
         }
         $message = sprintf('%s (%s) request sets left in queue. %s used memory (%s peak). %d workers active.        ', array_sum($numInQueue), implode('+', $numInQueue), $memory['used_memory_human'], $memory['used_memory_peak_human'], $lock->getNumberOfAcquiredLocks());
         $output->write("\r");
         $output->write($message);
         sleep(2);
     }
 }
 public function test_makeQueueMananger_shouldConfigureTheNumberOfWorkers()
 {
     $redis = $this->createRedisBackend();
     Factory::getSettings()->numQueueWorkers->setValue(7);
     $queue = Factory::makeQueueManager($redis);
     $this->assertSame(7, $queue->getNumberOfAvailableQueues());
 }
 public function setUp()
 {
     parent::setUp();
     Fixture::createWebsite('2014-01-01 00:00:00');
     Fixture::createSuperUser();
     Tracker\Cache::deleteTrackerCache();
     $this->backend = $this->createRedisBackend();
     $this->queue = Queue\Factory::makeQueueManager($this->backend);
     $this->response = new Response();
     $this->handler = new Handler();
     $this->handler->setResponse($this->response);
     $this->tracker = new Tracker();
     $this->requestSet = new RequestSet();
 }
 public function setValue($value)
 {
     $newNumWorkers = $value;
     $oldNumWorkers = $this->getValue();
     parent::setValue($value);
     if ($newNumWorkers && $oldNumWorkers) {
         try {
             $manager = Factory::makeQueueManager(Factory::makeBackend());
             $manager->setNumberOfAvailableQueues($newNumWorkers);
             $manager->moveSomeQueuesIfNeeded($newNumWorkers, $oldNumWorkers);
         } catch (\Exception $e) {
             // it is ok if this fails. then it is most likely not enabled etc.
         }
     }
 }
예제 #5
0
 protected function execute(InputInterface $input, OutputInterface $output)
 {
     $systemCheck = new SystemCheck();
     $systemCheck->checkRedisIsInstalled();
     $trackerEnvironment = new Environment('tracker');
     $trackerEnvironment->init();
     Log::unsetInstance();
     $trackerEnvironment->getContainer()->get('Piwik\\Access')->setSuperUserAccess(false);
     $trackerEnvironment->getContainer()->get('Piwik\\Plugin\\Manager')->setTrackerPluginsNotToLoad(array('Provider'));
     Tracker::loadTrackerEnvironment();
     if (OutputInterface::VERBOSITY_VERY_VERBOSE <= $output->getVerbosity()) {
         $GLOBALS['PIWIK_TRACKER_DEBUG'] = true;
     }
     $backend = Queue\Factory::makeBackend();
     $queueManager = Queue\Factory::makeQueueManager($backend);
     if (!$queueManager->canAcquireMoreLocks()) {
         $trackerEnvironment->destroy();
         $this->writeSuccessMessage($output, array("Nothing to proccess. Already max number of workers in process."));
         return;
     }
     $shouldProcess = false;
     foreach ($queueManager->getAllQueues() as $queue) {
         if ($queue->shouldProcess()) {
             $shouldProcess = true;
             break;
         }
     }
     if (!$shouldProcess) {
         $trackerEnvironment->destroy();
         $this->writeSuccessMessage($output, array("No queue currently needs processing"));
         return;
     }
     $output->writeln("<info>Starting to process request sets, this can take a while</info>");
     register_shutdown_function(function () use($queueManager) {
         $queueManager->unlock();
     });
     $startTime = microtime(true);
     $processor = new Processor($queueManager);
     $processor->setNumberOfMaxBatchesToProcess(1000);
     $tracker = $processor->process();
     $neededTime = microtime(true) - $startTime;
     $numRequestsTracked = $tracker->getCountOfLoggedRequests();
     $requestsPerSecond = $this->getNumberOfRequestsPerSecond($numRequestsTracked, $neededTime);
     Piwik::postEvent('Tracker.end');
     $trackerEnvironment->destroy();
     $this->writeSuccessMessage($output, array(sprintf('This worker finished queue processing with %sreq/s (%s requests in %02.2f seconds)', $requestsPerSecond, $numRequestsTracked, $neededTime)));
 }
 protected function execute(InputInterface $input, OutputInterface $output)
 {
     $systemCheck = new SystemCheck();
     $systemCheck->checkRedisIsInstalled();
     $backend = Queue\Factory::makeBackend();
     $manager = Queue\Factory::makeQueueManager($backend);
     $queueId = $input->getOption('queue-id');
     foreach ($manager->getAllQueues() as $index => $queue) {
         $thisQueueId = $queue->getId();
         if (isset($queueId) && $queueId != $thisQueueId) {
             continue;
         }
         $output->writeln(sprintf('<info>Showing requests of queue %s. Use <comment>--queue-id=%s</comment> to print only information for this queue.</info>', $thisQueueId, $thisQueueId));
         $requests = $queue->getRequestSetsToProcess();
         $output->writeln(var_export($requests, 1));
         $output->writeln(sprintf('<info>These were the requests of queue %s. Use <comment>--queue-id=%s</comment> to print only information for this queue.</info>', $thisQueueId, $thisQueueId));
     }
 }
 public function test_numQueueWorkers_WhenChangingAValueItMovesRequestsIntoDifferentQueues()
 {
     $oldNumWorkers = 4;
     $newNumWorkers = 2;
     $this->settings->numQueueWorkers->setValue($oldNumWorkers);
     $manager = Factory::makeQueueManager(Factory::makeBackend());
     $requestSet = new RequestSet();
     $requestSet->setRequests(array('idsite' => '1', '_id' => 1));
     $queues = $manager->getAllQueues();
     foreach ($queues as $queue) {
         $queue->addRequestSet($requestSet);
     }
     $this->assertSame(4, $manager->getNumberOfRequestSetsInAllQueues());
     $this->assertSame(1, $queues[0]->getNumberOfRequestSetsInQueue());
     $this->assertSame(1, $queues[1]->getNumberOfRequestSetsInQueue());
     $this->assertSame(1, $queues[2]->getNumberOfRequestSetsInQueue());
     $this->assertSame(1, $queues[3]->getNumberOfRequestSetsInQueue());
     $this->settings->numQueueWorkers->setValue($newNumWorkers);
     $this->assertSame(4, $manager->getNumberOfRequestSetsInAllQueues());
     $this->assertGreaterThanOrEqual(1, $queues[0]->getNumberOfRequestSetsInQueue());
     $this->assertGreaterThanOrEqual(1, $queues[1]->getNumberOfRequestSetsInQueue());
     $this->assertSame(0, $queues[2]->getNumberOfRequestSetsInQueue());
     $this->assertSame(0, $queues[3]->getNumberOfRequestSetsInQueue());
 }
예제 #8
0
 private function getQueueManager()
 {
     $backend = $this->getBackend();
     $queue = Queue\Factory::makeQueueManager($backend);
     return $queue;
 }
 protected function createQueue()
 {
     return Queue\Factory::makeQueueManager($this->createRedisBackend());
 }