function execute()
{
$totalOnly = $this->hasOption('totalonly');
$pendingDBs = JobQueueAggregator::singleton()->getAllReadyWikiQueues();
$sizeByWiki = array();
// (wiki => type => count) map
foreach ($pendingDBs as $type => $wikis) {
foreach ($wikis as $wiki) {
$sizeByWiki[$wiki][$type] = JobQueueGroup::singleton($wiki)->get($type)->getSize();
}
}
if ($this->hasOption('grouponly')) {
$this->output(FormatJSON::encode($sizeByWiki, true) . "\n");
} else {
$total = 0;
foreach ($sizeByWiki as $wiki => $counts) {
$count = array_sum($counts);
if ($count > 0) {
if (!$totalOnly) {
$this->output("{$wiki} {$count}\n");
}
$total += $count;
}
}
if (!$this->hasOption('nototal')) {
$this->output("Total {$total}\n");
}
}
}
/**
* @params include:
* - redisConfig : An array of parameters to RedisConnectionPool::__construct().
* - redisServers : Array of server entries, the first being the primary and the
* others being fallback servers. Each entry is either a hostname/port
* combination or the absolute path of a UNIX socket.
* If a hostname is specified but no port, the standard port number
* 6379 will be used. Required.
* @param array $params
*/
protected function __construct(array $params)
{
parent::__construct($params);
$this->servers = isset($params['redisServers']) ? $params['redisServers'] : array($params['redisServer']);
// b/c
$this->redisPool = RedisConnectionPool::singleton($params['redisConfig']);
}
/**
* Pop a job off of the queue.
* This requires $wgJobClasses to be set for the given job type.
* Outside callers should use JobQueueGroup::pop() instead of this function.
*
* @throws MWException
* @return Job|bool Returns false if there are no jobs
*/
public final function pop()
{
global $wgJobClasses;
if ($this->wiki !== wfWikiID()) {
throw new MWException("Cannot pop '{$this->type}' job off foreign wiki queue.");
} elseif (!isset($wgJobClasses[$this->type])) {
// Do not pop jobs if there is no class for the queue type
throw new MWException("Unrecognized job type '{$this->type}'.");
}
$job = $this->doPop();
if (!$job) {
$this->aggr->notifyQueueEmpty($this->wiki, $this->type);
}
// Flag this job as an old duplicate based on its "root" job...
try {
if ($job && $this->isRootJobOldDuplicate($job)) {
JobQueue::incrStats('dupe_pops', $this->type);
$job = DuplicateJob::newFromJob($job);
// convert to a no-op
}
} catch (Exception $e) {
// don't lose jobs over this
}
return $job;
}
/**
* @params include:
* - objectCache : Name of an object cache registered in $wgObjectCaches.
* This defaults to the one specified by $wgMainCacheType.
* - cacheTTL : Seconds to cache the aggregate data before regenerating.
* @param array $params
*/
protected function __construct(array $params)
{
parent::__construct($params);
$this->cache = isset($params['objectCache']) ? wfGetCache($params['objectCache']) : wfGetMainCache();
$this->cacheTTL = isset($params['cacheTTL']) ? $params['cacheTTL'] : 180;
// 3 min
}
/**
* @param array $params Possible keys:
* - redisConfig : An array of parameters to RedisConnectionPool::__construct().
* - redisServers : Array of server entries, the first being the primary and the
* others being fallback servers. Each entry is either a hostname/port
* combination or the absolute path of a UNIX socket.
* If a hostname is specified but no port, the standard port number
* 6379 will be used. Required.
*/
public function __construct(array $params)
{
parent::__construct($params);
$this->servers = isset($params['redisServers']) ? $params['redisServers'] : array($params['redisServer']);
// b/c
$params['redisConfig']['serializer'] = 'none';
$this->redisPool = RedisConnectionPool::singleton($params['redisConfig']);
}
/**
* @param array $params Possible keys:
* - redisConfig : An array of parameters to RedisConnectionPool::__construct().
* - redisServers : Array of server entries, the first being the primary and the
* others being fallback servers. Each entry is either a hostname/port
* combination or the absolute path of a UNIX socket.
* If a hostname is specified but no port, the standard port number
* 6379 will be used. Required.
*/
public function __construct(array $params)
{
parent::__construct($params);
$this->servers = isset($params['redisServers']) ? $params['redisServers'] : [$params['redisServer']];
// b/c
$params['redisConfig']['serializer'] = 'none';
$this->redisPool = RedisConnectionPool::singleton($params['redisConfig']);
$this->logger = \MediaWiki\Logger\LoggerFactory::getInstance('redis');
}
public function execute()
{
global $wgJobTypesExcludedFromDefaultQueue;
// job type required/picked
if ($this->hasOption('types')) {
$types = explode(' ', $this->getOption('types'));
} elseif ($this->hasOption('type')) {
$types = array($this->getOption('type'));
} else {
$types = false;
}
// Handle any required periodic queue maintenance
$this->executeReadyPeriodicTasks();
// Get all the queues with jobs in them
$pendingDBs = JobQueueAggregator::singleton()->getAllReadyWikiQueues();
if (!count($pendingDBs)) {
return;
// no DBs with jobs or cache is both empty and locked
}
do {
$again = false;
$candidates = array();
// list of (type, db)
// Flatten the tree of candidates into a flat list so that a random
// item can be selected, weighing each queue (type/db tuple) equally.
foreach ($pendingDBs as $type => $dbs) {
if (is_array($types) && in_array($type, $types) || $types === false && !in_array($type, $wgJobTypesExcludedFromDefaultQueue)) {
foreach ($dbs as $db) {
$candidates[] = array($type, $db);
}
}
}
if (!count($candidates)) {
return;
// no jobs for this type
}
list($type, $db) = $candidates[mt_rand(0, count($candidates) - 1)];
if (JobQueueGroup::singleton($db)->isQueueDeprioritized($type)) {
$pendingDBs[$type] = array_diff($pendingDBs[$type], array($db));
$again = true;
}
} while ($again);
if ($this->hasOption('types')) {
$this->output($db . " " . $type . "\n");
} else {
$this->output($db . "\n");
}
}
/**
* Get the job queue object for a given queue type
*
* @param string $type
* @return JobQueue
*/
public function get($type)
{
global $wgJobTypeConf;
$conf = array('wiki' => $this->wiki, 'type' => $type);
if (isset($wgJobTypeConf[$type])) {
$conf = $conf + $wgJobTypeConf[$type];
} else {
$conf = $conf + $wgJobTypeConf['default'];
}
$conf['aggregator'] = JobQueueAggregator::singleton();
return JobQueue::factory($conf);
}
/**
* Get the job queue object for a given queue type
*
* @param string $type
* @return JobQueue
*/
public function get($type)
{
global $wgJobTypeConf;
$conf = ['wiki' => $this->wiki, 'type' => $type];
if (isset($wgJobTypeConf[$type])) {
$conf = $conf + $wgJobTypeConf[$type];
} else {
$conf = $conf + $wgJobTypeConf['default'];
}
$conf['aggregator'] = JobQueueAggregator::singleton();
if ($this->readOnlyReason !== false) {
$conf['readOnlyReason'] = $this->readOnlyReason;
}
return JobQueue::factory($conf);
}
/**
* Pop a job off one of the job queues
*
* This pops a job off a queue as specified by $wgJobTypeConf and
* updates the aggregate job queue information cache as needed.
*
* @param $qtype integer|string JobQueueGroup::TYPE_DEFAULT or type string
* @param $flags integer Bitfield of JobQueueGroup::USE_* constants
* @return Job|bool Returns false on failure
*/
public function pop($qtype = self::TYPE_DEFAULT, $flags = 0)
{
if (is_string($qtype)) {
// specific job type
$job = $this->get($qtype)->pop();
if (!$job) {
JobQueueAggregator::singleton()->notifyQueueEmpty($this->wiki, $qtype);
}
return $job;
} else {
// any job in the "default" jobs types
if ($flags & self::USE_CACHE) {
if (!$this->cache->has('queues-ready', 'list', self::PROC_CACHE_TTL)) {
$this->cache->set('queues-ready', 'list', $this->getQueuesWithJobs());
}
$types = $this->cache->get('queues-ready', 'list');
} else {
$types = $this->getQueuesWithJobs();
}
if ($qtype == self::TYPE_DEFAULT) {
$types = array_intersect($types, $this->getDefaultQueueTypes());
}
shuffle($types);
// avoid starvation
foreach ($types as $type) {
// for each queue...
$job = $this->get($type)->pop();
if ($job) {
// found
return $job;
} else {
// not found
JobQueueAggregator::singleton()->notifyQueueEmpty($this->wiki, $type);
$this->cache->clear('queues-ready');
}
}
return false;
// no jobs found
}
}
/**
* Destroy the singleton instance
*
* @return void
*/
public static final function destroySingleton()
{
self::$instance = null;
}
/**
* @params include:
* - redisConfig : An array of parameters to RedisConnectionPool::__construct().
* - redisServer : A hostname/port combination or the absolute path of a UNIX socket.
* If a hostname is specified but no port, the standard port number
* 6379 will be used. Required.
* @param array $params
*/
protected function __construct(array $params)
{
parent::__construct($params);
$this->server = $params['redisServer'];
$this->redisPool = RedisConnectionPool::singleton($params['redisConfig']);
}
/**
* Execute any due periodic queue maintenance tasks for all queues.
*
* A task is "due" if the time ellapsed since the last run is greater than
* the defined run period. Concurrent calls to this function will cause tasks
* to be attempted twice, so they may need their own methods of mutual exclusion.
*
* @return int Number of tasks run
*/
public function executeReadyPeriodicTasks()
{
global $wgMemc;
list($db, $prefix) = wfSplitWikiID($this->wiki);
$key = wfForeignMemcKey($db, $prefix, 'jobqueuegroup', 'taskruns', 'v1');
$lastRuns = $wgMemc->get($key);
// (queue => task => UNIX timestamp)
$count = 0;
$tasksRun = array();
// (queue => task => UNIX timestamp)
foreach ($this->getQueueTypes() as $type) {
$queue = $this->get($type);
foreach ($queue->getPeriodicTasks() as $task => $definition) {
if ($definition['period'] <= 0) {
continue;
// disabled
} elseif (!isset($lastRuns[$type][$task]) || $lastRuns[$type][$task] < time() - $definition['period']) {
try {
if (call_user_func($definition['callback']) !== null) {
$tasksRun[$type][$task] = time();
++$count;
}
} catch (JobQueueError $e) {
MWExceptionHandler::logException($e);
}
}
}
// The tasks may have recycled jobs or release delayed jobs into the queue
if (isset($tasksRun[$type]) && !$queue->isEmpty()) {
JobQueueAggregator::singleton()->notifyQueueNonEmpty($this->wiki, $type);
}
}
$wgMemc->merge($key, function ($cache, $key, $lastRuns) use($tasksRun) {
if (is_array($lastRuns)) {
foreach ($tasksRun as $type => $tasks) {
foreach ($tasks as $task => $timestamp) {
if (!isset($lastRuns[$type][$task]) || $timestamp > $lastRuns[$type][$task]) {
$lastRuns[$type][$task] = $timestamp;
}
}
}
} else {
$lastRuns = $tasksRun;
}
return $lastRuns;
});
return $count;
}
