public function test_replaceHandler_ShouldNotEnableProcessingInTrackerModeIfDisabled() { Factory::getSettings()->processDuringTrackingRequest->setValue(false); $handler = null; $this->plugin->replaceHandlerIfQueueIsEnabled($handler); $this->assertFalse($handler->isAllowedToProcessInTrackerMode()); }
protected function execute(InputInterface $input, OutputInterface $output) { $systemCheck = new SystemCheck(); $systemCheck->checkRedisIsInstalled(); $settings = Queue\Factory::getSettings(); if ($settings->queueEnabled->getValue()) { $output->writeln('Queue is enabled'); } else { $output->writeln('<comment>' . strtoupper('Queue is disabled: ') . 'No new requests will be written into the queue, processing the remaining requests is still possible.</comment>'); } $backend = Queue\Factory::makeBackend(); $manager = Queue\Factory::makeQueueManager($backend); $queues = $manager->getAllQueues(); $lock = Queue\Factory::makeLock($backend); if ($settings->processDuringTrackingRequest->getValue()) { $output->writeln('Request sets in the queue will be processed automatically after a tracking request'); } else { $output->writeln('The command <comment>./console queuedtracking:process</comment> has to be executed to process request sets within queue'); } $output->writeln(sprintf('Up to %d workers will be used', $manager->getNumberOfAvailableQueues())); $output->writeln(sprintf('Processor will start once there are at least %s request sets in the queue', $manager->getNumberOfRequestsToProcessAtSameTime())); while (1) { $memory = $backend->getMemoryStats(); // I know this will only work with redis currently as it is not defined in backend interface etc. needs to be refactored once we add another backend $numInQueue = array(); foreach ($queues as $queue) { $numInQueue[] = $queue->getNumberOfRequestSetsInQueue(); } $message = sprintf('%s (%s) request sets left in queue. %s used memory (%s peak). %d workers active. ', array_sum($numInQueue), implode('+', $numInQueue), $memory['used_memory_human'], $memory['used_memory_peak_human'], $lock->getNumberOfAcquiredLocks()); $output->write("\r"); $output->write($message); sleep(2); } }
public function test_getSettings_shouldReturnASingleton() { $settings = Factory::getSettings(); $settings->redisTimeout->setValue(0.7); // it would not return the same value usually as $settings->save() is not called $settings = Factory::getSettings(); $this->assertEquals(0.7, $settings->redisTimeout->getValue()); }
public function replaceHandlerIfQueueIsEnabled(&$handler) { $settings = Queue\Factory::getSettings(); if ($settings->queueEnabled->getValue()) { $handler = new Handler(); if ($settings->processDuringTrackingRequest->getValue()) { $handler->enableProcessingInTrackerMode(); } } }
public function setValue($value) { $newNumWorkers = $value; $oldNumWorkers = $this->getValue(); parent::setValue($value); if ($newNumWorkers && $oldNumWorkers) { try { $manager = Factory::makeQueueManager(Factory::makeBackend()); $manager->setNumberOfAvailableQueues($newNumWorkers); $manager->moveSomeQueuesIfNeeded($newNumWorkers, $oldNumWorkers); } catch (\Exception $e) { // it is ok if this fails. then it is most likely not enabled etc. } } }
protected function execute(InputInterface $input, OutputInterface $output) { $systemCheck = new SystemCheck(); $systemCheck->checkRedisIsInstalled(); $trackerEnvironment = new Environment('tracker'); $trackerEnvironment->init(); Log::unsetInstance(); $trackerEnvironment->getContainer()->get('Piwik\\Access')->setSuperUserAccess(false); $trackerEnvironment->getContainer()->get('Piwik\\Plugin\\Manager')->setTrackerPluginsNotToLoad(array('Provider')); Tracker::loadTrackerEnvironment(); if (OutputInterface::VERBOSITY_VERY_VERBOSE <= $output->getVerbosity()) { $GLOBALS['PIWIK_TRACKER_DEBUG'] = true; } $backend = Queue\Factory::makeBackend(); $queueManager = Queue\Factory::makeQueueManager($backend); if (!$queueManager->canAcquireMoreLocks()) { $trackerEnvironment->destroy(); $this->writeSuccessMessage($output, array("Nothing to proccess. Already max number of workers in process.")); return; } $shouldProcess = false; foreach ($queueManager->getAllQueues() as $queue) { if ($queue->shouldProcess()) { $shouldProcess = true; break; } } if (!$shouldProcess) { $trackerEnvironment->destroy(); $this->writeSuccessMessage($output, array("No queue currently needs processing")); return; } $output->writeln("<info>Starting to process request sets, this can take a while</info>"); register_shutdown_function(function () use($queueManager) { $queueManager->unlock(); }); $startTime = microtime(true); $processor = new Processor($queueManager); $processor->setNumberOfMaxBatchesToProcess(1000); $tracker = $processor->process(); $neededTime = microtime(true) - $startTime; $numRequestsTracked = $tracker->getCountOfLoggedRequests(); $requestsPerSecond = $this->getNumberOfRequestsPerSecond($numRequestsTracked, $neededTime); Piwik::postEvent('Tracker.end'); $trackerEnvironment->destroy(); $this->writeSuccessMessage($output, array(sprintf('This worker finished queue processing with %sreq/s (%s requests in %02.2f seconds)', $requestsPerSecond, $numRequestsTracked, $neededTime))); }
protected function execute(InputInterface $input, OutputInterface $output) { $systemCheck = new SystemCheck(); $systemCheck->checkRedisIsInstalled(); $backend = Queue\Factory::makeBackend(); $manager = Queue\Factory::makeQueueManager($backend); $queueId = $input->getOption('queue-id'); foreach ($manager->getAllQueues() as $index => $queue) { $thisQueueId = $queue->getId(); if (isset($queueId) && $queueId != $thisQueueId) { continue; } $output->writeln(sprintf('<info>Showing requests of queue %s. Use <comment>--queue-id=%s</comment> to print only information for this queue.</info>', $thisQueueId, $thisQueueId)); $requests = $queue->getRequestSetsToProcess(); $output->writeln(var_export($requests, 1)); $output->writeln(sprintf('<info>These were the requests of queue %s. Use <comment>--queue-id=%s</comment> to print only information for this queue.</info>', $thisQueueId, $thisQueueId)); } }
protected function execute(InputInterface $input, OutputInterface $output) { $systemCheck = new SystemCheck(); $systemCheck->checkRedisIsInstalled(); $backend = Queue\Factory::makeBackend(); $lock = Queue\Factory::makeLock($backend); $keys = $lock->getAllAcquiredLockKeys(); $keyToUnlock = $input->getOption('unlock'); if ($keyToUnlock && in_array($keyToUnlock, $keys)) { $backend->delete($keyToUnlock); $this->writeSuccessMessage($output, array(sprintf('Key %s unlocked', $keyToUnlock))); } elseif ($keyToUnlock) { $output->writeln(sprintf('<error>%s is not or no longer locked</error>', $keyToUnlock)); $output->writeln(' '); } foreach ($keys as $lockKey) { $time = $backend->getTimeToLive($lockKey); $output->writeln(sprintf('"%s" is locked for <comment>%d ms</comment>', $lockKey, $time)); $output->writeln(sprintf('Set option <comment>--unlock=%s</comment> to unlock the queue.', $lockKey)); $output->writeln(' '); } }
public function test_numQueueWorkers_WhenChangingAValueItMovesRequestsIntoDifferentQueues() { $oldNumWorkers = 4; $newNumWorkers = 2; $this->settings->numQueueWorkers->setValue($oldNumWorkers); $manager = Factory::makeQueueManager(Factory::makeBackend()); $requestSet = new RequestSet(); $requestSet->setRequests(array('idsite' => '1', '_id' => 1)); $queues = $manager->getAllQueues(); foreach ($queues as $queue) { $queue->addRequestSet($requestSet); } $this->assertSame(4, $manager->getNumberOfRequestSetsInAllQueues()); $this->assertSame(1, $queues[0]->getNumberOfRequestSetsInQueue()); $this->assertSame(1, $queues[1]->getNumberOfRequestSetsInQueue()); $this->assertSame(1, $queues[2]->getNumberOfRequestSetsInQueue()); $this->assertSame(1, $queues[3]->getNumberOfRequestSetsInQueue()); $this->settings->numQueueWorkers->setValue($newNumWorkers); $this->assertSame(4, $manager->getNumberOfRequestSetsInAllQueues()); $this->assertGreaterThanOrEqual(1, $queues[0]->getNumberOfRequestSetsInQueue()); $this->assertGreaterThanOrEqual(1, $queues[1]->getNumberOfRequestSetsInQueue()); $this->assertSame(0, $queues[2]->getNumberOfRequestSetsInQueue()); $this->assertSame(0, $queues[3]->getNumberOfRequestSetsInQueue()); }
public function test_process_ShouldNotCreateADatabaseConnectionAtAnyTime() { $this->setDummyRequests(false); Queue\Factory::getSettings()->queueEnabled->getValue(); // this will cause a db query but will be cached afterwards Db::destroyDatabaseObject(); $this->handler->init($this->tracker, $this->requestSet); $this->assertNotDbConnectionCreated(); $this->handler->onStartTrackRequests($this->tracker, $this->requestSet); $this->assertNotDbConnectionCreated(); $this->handler->process($this->tracker, $this->requestSet); $this->assertNotDbConnectionCreated(); $this->handler->onAllRequestsTracked($this->tracker, $this->requestSet); $this->assertNotDbConnectionCreated(); $this->handler->finish($this->tracker, $this->requestSet); $this->assertNotDbConnectionCreated(); }
private function getQueueManager() { $backend = $this->getBackend(); $queue = Queue\Factory::makeQueueManager($backend); return $queue; }
protected function createRedisBackend() { return Queue\Factory::makeBackend(); }
/** * The actual task is defined in this method. Here you can access any option or argument that was defined on the * command line via $input and write anything to the console via $output argument. * In case anything went wrong during the execution you should throw an exception to make sure the user will get a * useful error message and to make sure the command does not exit with the status code 0. * * Ideally, the actual command is quite short as it acts like a controller. It should only receive the input values, * execute the task by calling a method of another class and output any useful information. * * Execute the command like: ./console queuedtracking:test --name="The Piwik Team" */ protected function execute(InputInterface $input, OutputInterface $output) { $systemCheck = new SystemCheck(); $systemCheck->checkRedisIsInstalled(); $trackerEnvironment = new Environment('tracker'); $trackerEnvironment->init(); Tracker::loadTrackerEnvironment(); $settings = Queue\Factory::getSettings(); $output->writeln('<comment>Settings that will be used:</comment>'); $output->writeln('Host: ' . $settings->redisHost->getValue()); $output->writeln('Port: ' . $settings->redisPort->getValue()); $output->writeln('Timeout: ' . $settings->redisTimeout->getValue()); $output->writeln('Password: '******'Database: ' . $settings->redisDatabase->getValue()); $output->writeln('NumQueueWorkers: ' . $settings->numQueueWorkers->getValue()); $output->writeln('NumRequestsToProcess: ' . $settings->numRequestsToProcess->getValue()); $output->writeln('ProcessDuringTrackingRequest: ' . (int) $settings->processDuringTrackingRequest->getValue()); $output->writeln('QueueEnabled: ' . (int) $settings->queueEnabled->getValue()); $output->writeln(''); $output->writeln('<comment>Version / stats:</comment>'); $output->writeln('PHP version: ' . phpversion()); $output->writeln('Uname: ' . php_uname()); $extension = new \ReflectionExtension('redis'); $output->writeln('PHPRedis version: ' . $extension->getVersion()); $backend = Queue\Factory::makeBackend(); $output->writeln('Redis version: ' . $backend->getServerVersion()); $output->writeln('Memory: ' . var_export($backend->getMemoryStats(), 1)); $redis = $backend->getConnection(); $evictionPolicy = $this->getRedisConfig($redis, 'maxmemory-policy'); $output->writeln('MaxMemory Eviction Policy config: ' . $evictionPolicy); if ($evictionPolicy !== 'allkeys-lru' && $evictionPolicy !== 'noeviction') { $output->writeln('<error>The eviction policy can likely lead to errors when memory is low. We recommend to use eviction policy <comment>allkeys-lru</comment> or alternatively <comment>noeviction</comment>. Read more here: http://redis.io/topics/lru-cache</error>'); } $evictionPolicy = $this->getRedisConfig($redis, 'maxmemory'); $output->writeln('MaxMemory config: ' . $evictionPolicy); $output->writeln(''); $output->writeln('<comment>Performing some tests:</comment>'); if (method_exists($redis, 'isConnected')) { $output->writeln('Redis is connected: ' . (int) $redis->isConnected()); } if ($backend->testConnection()) { $output->writeln('Connection works in general'); } else { $output->writeln('Connection does not actually work: ' . $redis->getLastError()); } $this->testRedis($redis, 'set', array('testKey', 'value'), 'testKey', $output); $this->testRedis($redis, 'setnx', array('testnxkey', 'value'), 'testnxkey', $output); $this->testRedis($redis, 'setex', array('testexkey', 5, 'value'), 'testexkey', $output); $this->testRedis($redis, 'set', array('testKeyWithNx', 'value', array('nx')), 'testKeyWithNx', $output); $this->testRedis($redis, 'set', array('testKeyWithEx', 'value', array('ex' => 5)), 'testKeyWithEx', $output); $backend->delete('foo'); if (!$backend->setIfNotExists('foo', 'bar', 5)) { $output->writeln("setIfNotExists(foo, bar, 1) does not work, most likely we won't be able to acquire a lock:" . $redis->getLastError()); } else { $initialTtl = $redis->ttl('foo'); if ($initialTtl > 3 && $initialTtl <= 5) { $output->writeln('Initial expire seems to be set correctly'); } else { $output->writeln('<error>Initial expire seems to be not set correctly: ' . $initialTtl . ' </error>'); } if ($backend->get('foo') == 'bar') { $output->writeln('setIfNotExists works fine'); } else { $output->writeln('There might be a problem with setIfNotExists'); } if ($backend->expireIfKeyHasValue('foo', 'bar', 10)) { $output->writeln('expireIfKeyHasValue seems to work fine'); } else { $output->writeln('<error>There might be a problem with expireIfKeyHasValue: ' . $redis->getLastError() . '</error>'); } $extendedTtl = $redis->ttl('foo'); if ($extendedTtl > 8 && $extendedTtl <= 10) { $output->writeln('Extending expire seems to be set correctly'); } else { $output->writeln('<error>Extending expire seems to be not set correctly: ' . $extendedTtl . ' </error>'); } if ($backend->expireIfKeyHasValue('foo', 'invalidValue', 10)) { $output->writeln('<error>expireIfKeyHasValue expired a key which it should not have since values does not match</error>'); } else { $output->writeln('expireIfKeyHasValue correctly expires only when the value is correct'); } $extendedTtl = $redis->ttl('foo'); if ($extendedTtl > 7 && $extendedTtl <= 10) { $output->writeln('Expire is still set which is correct'); } else { $output->writeln('<error>Expire missing after a wrong extendExpire: ' . $extendedTtl . ' </error>'); } if ($backend->deleteIfKeyHasValue('foo', 'bar')) { $output->writeln('deleteIfKeyHasValue seems to work fine'); } else { $output->writeln('<error>There might be a problem with deleteIfKeyHasValue: ' . $redis->getLastError() . '</error>'); } } $redis->delete('fooList'); $backend->appendValuesToList('fooList', array('value1', 'value2', 'value3')); $values = $backend->getFirstXValuesFromList('fooList', 2); if ($values == array('value1', 'value2')) { $backend->removeFirstXValuesFromList('fooList', 1); $backend->removeFirstXValuesFromList('fooList', 1); $values = $backend->getFirstXValuesFromList('fooList', 2); if ($values == array('value3')) { $output->writeln('List feature seems to work fine'); } else { $output->writeln('List feature seems to work only partially: ' . var_export($values, 1)); } } else { $output->writeln('<error>List feature seems to not work fine: ' . $redis->getLastError() . '</error>'); } $output->writeln(''); $output->writeln('<comment>Done</comment>'); }