protected function execute(InputInterface $input, OutputInterface $output)
{
$systemCheck = new SystemCheck();
$systemCheck->checkRedisIsInstalled();
$settings = Queue\Factory::getSettings();
if ($settings->queueEnabled->getValue()) {
$output->writeln('Queue is enabled');
} else {
$output->writeln('<comment>' . strtoupper('Queue is disabled: ') . 'No new requests will be written into the queue, processing the remaining requests is still possible.</comment>');
}
$backend = Queue\Factory::makeBackend();
$manager = Queue\Factory::makeQueueManager($backend);
$queues = $manager->getAllQueues();
$lock = Queue\Factory::makeLock($backend);
if ($settings->processDuringTrackingRequest->getValue()) {
$output->writeln('Request sets in the queue will be processed automatically after a tracking request');
} else {
$output->writeln('The command <comment>./console queuedtracking:process</comment> has to be executed to process request sets within queue');
}
$output->writeln(sprintf('Up to %d workers will be used', $manager->getNumberOfAvailableQueues()));
$output->writeln(sprintf('Processor will start once there are at least %s request sets in the queue', $manager->getNumberOfRequestsToProcessAtSameTime()));
while (1) {
$memory = $backend->getMemoryStats();
// I know this will only work with redis currently as it is not defined in backend interface etc. needs to be refactored once we add another backend
$numInQueue = array();
foreach ($queues as $queue) {
$numInQueue[] = $queue->getNumberOfRequestSetsInQueue();
}
$message = sprintf('%s (%s) request sets left in queue. %s used memory (%s peak). %d workers active. ', array_sum($numInQueue), implode('+', $numInQueue), $memory['used_memory_human'], $memory['used_memory_peak_human'], $lock->getNumberOfAcquiredLocks());
$output->write("\r");
$output->write($message);
sleep(2);
}
}
public function test_makeQueueMananger_shouldConfigureTheNumberOfWorkers()
{
$redis = $this->createRedisBackend();
Factory::getSettings()->numQueueWorkers->setValue(7);
$queue = Factory::makeQueueManager($redis);
$this->assertSame(7, $queue->getNumberOfAvailableQueues());
}
public function setUp()
{
parent::setUp();
Fixture::createWebsite('2014-01-01 00:00:00');
Fixture::createSuperUser();
Tracker\Cache::deleteTrackerCache();
$this->backend = $this->createRedisBackend();
$this->queue = Queue\Factory::makeQueueManager($this->backend);
$this->response = new Response();
$this->handler = new Handler();
$this->handler->setResponse($this->response);
$this->tracker = new Tracker();
$this->requestSet = new RequestSet();
}
public function setValue($value)
{
$newNumWorkers = $value;
$oldNumWorkers = $this->getValue();
parent::setValue($value);
if ($newNumWorkers && $oldNumWorkers) {
try {
$manager = Factory::makeQueueManager(Factory::makeBackend());
$manager->setNumberOfAvailableQueues($newNumWorkers);
$manager->moveSomeQueuesIfNeeded($newNumWorkers, $oldNumWorkers);
} catch (\Exception $e) {
// it is ok if this fails. then it is most likely not enabled etc.
}
}
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$systemCheck = new SystemCheck();
$systemCheck->checkRedisIsInstalled();
$trackerEnvironment = new Environment('tracker');
$trackerEnvironment->init();
Log::unsetInstance();
$trackerEnvironment->getContainer()->get('Piwik\\Access')->setSuperUserAccess(false);
$trackerEnvironment->getContainer()->get('Piwik\\Plugin\\Manager')->setTrackerPluginsNotToLoad(array('Provider'));
Tracker::loadTrackerEnvironment();
if (OutputInterface::VERBOSITY_VERY_VERBOSE <= $output->getVerbosity()) {
$GLOBALS['PIWIK_TRACKER_DEBUG'] = true;
}
$backend = Queue\Factory::makeBackend();
$queueManager = Queue\Factory::makeQueueManager($backend);
if (!$queueManager->canAcquireMoreLocks()) {
$trackerEnvironment->destroy();
$this->writeSuccessMessage($output, array("Nothing to proccess. Already max number of workers in process."));
return;
}
$shouldProcess = false;
foreach ($queueManager->getAllQueues() as $queue) {
if ($queue->shouldProcess()) {
$shouldProcess = true;
break;
}
}
if (!$shouldProcess) {
$trackerEnvironment->destroy();
$this->writeSuccessMessage($output, array("No queue currently needs processing"));
return;
}
$output->writeln("<info>Starting to process request sets, this can take a while</info>");
register_shutdown_function(function () use($queueManager) {
$queueManager->unlock();
});
$startTime = microtime(true);
$processor = new Processor($queueManager);
$processor->setNumberOfMaxBatchesToProcess(1000);
$tracker = $processor->process();
$neededTime = microtime(true) - $startTime;
$numRequestsTracked = $tracker->getCountOfLoggedRequests();
$requestsPerSecond = $this->getNumberOfRequestsPerSecond($numRequestsTracked, $neededTime);
Piwik::postEvent('Tracker.end');
$trackerEnvironment->destroy();
$this->writeSuccessMessage($output, array(sprintf('This worker finished queue processing with %sreq/s (%s requests in %02.2f seconds)', $requestsPerSecond, $numRequestsTracked, $neededTime)));
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$systemCheck = new SystemCheck();
$systemCheck->checkRedisIsInstalled();
$backend = Queue\Factory::makeBackend();
$manager = Queue\Factory::makeQueueManager($backend);
$queueId = $input->getOption('queue-id');
foreach ($manager->getAllQueues() as $index => $queue) {
$thisQueueId = $queue->getId();
if (isset($queueId) && $queueId != $thisQueueId) {
continue;
}
$output->writeln(sprintf('<info>Showing requests of queue %s. Use <comment>--queue-id=%s</comment> to print only information for this queue.</info>', $thisQueueId, $thisQueueId));
$requests = $queue->getRequestSetsToProcess();
$output->writeln(var_export($requests, 1));
$output->writeln(sprintf('<info>These were the requests of queue %s. Use <comment>--queue-id=%s</comment> to print only information for this queue.</info>', $thisQueueId, $thisQueueId));
}
}
public function test_numQueueWorkers_WhenChangingAValueItMovesRequestsIntoDifferentQueues()
{
$oldNumWorkers = 4;
$newNumWorkers = 2;
$this->settings->numQueueWorkers->setValue($oldNumWorkers);
$manager = Factory::makeQueueManager(Factory::makeBackend());
$requestSet = new RequestSet();
$requestSet->setRequests(array('idsite' => '1', '_id' => 1));
$queues = $manager->getAllQueues();
foreach ($queues as $queue) {
$queue->addRequestSet($requestSet);
}
$this->assertSame(4, $manager->getNumberOfRequestSetsInAllQueues());
$this->assertSame(1, $queues[0]->getNumberOfRequestSetsInQueue());
$this->assertSame(1, $queues[1]->getNumberOfRequestSetsInQueue());
$this->assertSame(1, $queues[2]->getNumberOfRequestSetsInQueue());
$this->assertSame(1, $queues[3]->getNumberOfRequestSetsInQueue());
$this->settings->numQueueWorkers->setValue($newNumWorkers);
$this->assertSame(4, $manager->getNumberOfRequestSetsInAllQueues());
$this->assertGreaterThanOrEqual(1, $queues[0]->getNumberOfRequestSetsInQueue());
$this->assertGreaterThanOrEqual(1, $queues[1]->getNumberOfRequestSetsInQueue());
$this->assertSame(0, $queues[2]->getNumberOfRequestSetsInQueue());
$this->assertSame(0, $queues[3]->getNumberOfRequestSetsInQueue());
}
private function getQueueManager()
{
$backend = $this->getBackend();
$queue = Queue\Factory::makeQueueManager($backend);
return $queue;
}
protected function createQueue()
{
return Queue\Factory::makeQueueManager($this->createRedisBackend());
}