protected function setUp() { global $wgJobTypeConf; parent::setUp(); $this->setMwGlobals('wgMemc', new HashBagOStuff()); if ($this->getCliArg('use-jobqueue')) { $name = $this->getCliArg('use-jobqueue'); if (!isset($wgJobTypeConf[$name])) { throw new MWException("No \$wgJobTypeConf entry for '{$name}'."); } $baseConfig = $wgJobTypeConf[$name]; } else { $baseConfig = array('class' => 'JobQueueDB'); } $baseConfig['type'] = 'null'; $baseConfig['wiki'] = wfWikiID(); $variants = array('queueRand' => array('order' => 'random', 'claimTTL' => 0), 'queueRandTTL' => array('order' => 'random', 'claimTTL' => 10), 'queueTimestamp' => array('order' => 'timestamp', 'claimTTL' => 0), 'queueTimestampTTL' => array('order' => 'timestamp', 'claimTTL' => 10), 'queueFifo' => array('order' => 'fifo', 'claimTTL' => 0), 'queueFifoTTL' => array('order' => 'fifo', 'claimTTL' => 10)); foreach ($variants as $q => $settings) { try { $this->{$q} = JobQueue::factory($settings + $baseConfig); if (!$this->{$q} instanceof JobQueueDB) { $this->{$q}->setTestingPrefix('unittests-' . wfRandomString(32)); } } catch (MWException $e) { // unsupported? // @todo What if it was another error? } } }
protected function setUp() { global $wgJobTypeConf; parent::setUp(); if ($this->getCliArg('use-jobqueue')) { $name = $this->getCliArg('use-jobqueue'); if (!isset($wgJobTypeConf[$name])) { throw new MWException("No \$wgJobTypeConf entry for '{$name}'."); } $baseConfig = $wgJobTypeConf[$name]; } else { $baseConfig = ['class' => 'JobQueueDB']; } $baseConfig['type'] = 'null'; $baseConfig['wiki'] = wfWikiID(); $variants = ['queueRand' => ['order' => 'random', 'claimTTL' => 0], 'queueRandTTL' => ['order' => 'random', 'claimTTL' => 10], 'queueTimestamp' => ['order' => 'timestamp', 'claimTTL' => 0], 'queueTimestampTTL' => ['order' => 'timestamp', 'claimTTL' => 10], 'queueFifo' => ['order' => 'fifo', 'claimTTL' => 0], 'queueFifoTTL' => ['order' => 'fifo', 'claimTTL' => 10]]; foreach ($variants as $q => $settings) { try { $this->{$q} = JobQueue::factory($settings + $baseConfig); } catch (MWException $e) { // unsupported? // @todo What if it was another error? } } }
/** * @param $type string * @return JobQueue Job queue object for a given queue type */ public function get($type) { global $wgJobTypeConf; $conf = array('wiki' => $this->wiki, 'type' => $type); if (isset($wgJobTypeConf[$type])) { $conf = $conf + $wgJobTypeConf[$type]; } else { $conf = $conf + $wgJobTypeConf['default']; } return JobQueue::factory($conf); }
public function execute() { global $wgJobQueueMigrationConfig; $srcKey = $this->getOption('src'); $dstKey = $this->getOption('dst'); if (!isset($wgJobQueueMigrationConfig[$srcKey])) { $this->error("\$wgJobQueueMigrationConfig not set for '{$srcKey}'.", 1); } elseif (!isset($wgJobQueueMigrationConfig[$dstKey])) { $this->error("\$wgJobQueueMigrationConfig not set for '{$dstKey}'.", 1); } $types = $this->getOption('type') === 'all' ? JobQueueGroup::singleton()->getQueueTypes() : array($this->getOption('type')); foreach ($types as $type) { $baseConfig = array('type' => $type, 'wiki' => wfWikiID()); $src = JobQueue::factory($baseConfig + $wgJobQueueMigrationConfig[$srcKey]); $dst = JobQueue::factory($baseConfig + $wgJobQueueMigrationConfig[$dstKey]); list($total, $totalOK) = $this->copyJobs($src, $dst, $src->getAllQueuedJobs()); $this->output("Copied {$totalOK}/{$total} queued {$type} jobs.\n"); list($total, $totalOK) = $this->copyJobs($src, $dst, $src->getAllDelayedJobs()); $this->output("Copied {$totalOK}/{$total} delayed {$type} jobs.\n"); } }
/** * @return JobQueueMemory */ private function newJobQueue() { return JobQueue::factory(array('class' => 'JobQueueMemory', 'wiki' => wfWikiID(), 'type' => 'null')); }
/** * @return array */ protected function getCoalescedQueues() { global $wgJobTypeConf; if ($this->coalescedQueues === null) { $this->coalescedQueues = array(); foreach ($wgJobTypeConf as $type => $conf) { $queue = JobQueue::factory(array('wiki' => $this->wiki, 'type' => 'null') + $conf); $loc = $queue->getCoalesceLocationInternal(); if (!isset($this->coalescedQueues[$loc])) { $this->coalescedQueues[$loc]['queue'] = $queue; $this->coalescedQueues[$loc]['types'] = array(); } if ($type === 'default') { $this->coalescedQueues[$loc]['types'] = array_merge($this->coalescedQueues[$loc]['types'], array_diff($this->getQueueTypes(), array_keys($wgJobTypeConf))); } else { $this->coalescedQueues[$loc]['types'][] = $type; } } } return $this->coalescedQueues; }
/** * @params include: * - sectionsByWiki : A map of wiki IDs to section names. * Wikis will default to using the section "default". * - partitionsBySection : Map of section names to maps of (partition name => weight). * A section called 'default' must be defined if not all wikis * have explicitly defined sections. * - configByPartition : Map of queue partition names to configuration arrays. * These configuration arrays are passed to JobQueue::factory(). * The options set here are overriden by those passed to this * the federated queue itself (e.g. 'order' and 'claimTTL'). * - partitionsNoPush : List of partition names that can handle pop() but not push(). * This can be used to migrate away from a certain partition. * @param array $params */ protected function __construct( array $params ) { parent::__construct( $params ); $section = isset( $params['sectionsByWiki'][$this->wiki] ) ? $params['sectionsByWiki'][$this->wiki] : 'default'; if ( !isset( $params['partitionsBySection'][$section] ) ) { throw new MWException( "No configuration for section '$section'." ); } // Get the full partition map $this->partitionMap = $params['partitionsBySection'][$section]; arsort( $this->partitionMap, SORT_NUMERIC ); // Get the partitions jobs can actually be pushed to $partitionPushMap = $this->partitionMap; if ( isset( $params['partitionsNoPush'] ) ) { foreach ( $params['partitionsNoPush'] as $partition ) { unset( $partitionPushMap[$partition] ); } } // Get the config to pass to merge into each partition queue config $baseConfig = $params; foreach ( array( 'class', 'sectionsByWiki', 'partitionsBySection', 'configByPartition', 'partitionsNoPush' ) as $o ) { unset( $baseConfig[$o] ); } // Get the partition queue objects foreach ( $this->partitionMap as $partition => $w ) { if ( !isset( $params['configByPartition'][$partition] ) ) { throw new MWException( "No configuration for partition '$partition'." ); } $this->partitionQueues[$partition] = JobQueue::factory( $baseConfig + $params['configByPartition'][$partition] ); } // Get the ring of partitions to push jobs into $this->partitionPushRing = new HashRing( $partitionPushMap ); // Aggregate cache some per-queue values if there are multiple partition queues $this->cache = count( $this->partitionMap ) > 1 ? wfGetMainCache() : new EmptyBagOStuff(); }
/** * @param array $params Possible keys: * - sectionsByWiki : A map of wiki IDs to section names. * Wikis will default to using the section "default". * - partitionsBySection : Map of section names to maps of (partition name => weight). * A section called 'default' must be defined if not all wikis * have explicitly defined sections. * - configByPartition : Map of queue partition names to configuration arrays. * These configuration arrays are passed to JobQueue::factory(). * The options set here are overridden by those passed to this * the federated queue itself (e.g. 'order' and 'claimTTL'). * - maxPartitionsTry : Maximum number of times to attempt job insertion using * different partition queues. This improves availability * during failure, at the cost of added latency and somewhat * less reliable job de-duplication mechanisms. * @throws MWException */ protected function __construct(array $params) { parent::__construct($params); $section = isset($params['sectionsByWiki'][$this->wiki]) ? $params['sectionsByWiki'][$this->wiki] : 'default'; if (!isset($params['partitionsBySection'][$section])) { throw new MWException("No configuration for section '{$section}'."); } $this->maxPartitionsTry = isset($params['maxPartitionsTry']) ? $params['maxPartitionsTry'] : 2; // Get the full partition map $partitionMap = $params['partitionsBySection'][$section]; arsort($partitionMap, SORT_NUMERIC); // Get the config to pass to merge into each partition queue config $baseConfig = $params; foreach (array('class', 'sectionsByWiki', 'maxPartitionsTry', 'partitionsBySection', 'configByPartition') as $o) { unset($baseConfig[$o]); // partition queue doesn't care about this } // The class handles all aggregator calls already unset($baseConfig['aggregator']); // Get the partition queue objects foreach ($partitionMap as $partition => $w) { if (!isset($params['configByPartition'][$partition])) { throw new MWException("No configuration for partition '{$partition}'."); } $this->partitionQueues[$partition] = JobQueue::factory($baseConfig + $params['configByPartition'][$partition]); } // Ring of all partitions $this->partitionRing = new HashRing($partitionMap); }
/** * @params include: * - sectionsByWiki : A map of wiki IDs to section names. * Wikis will default to using the section "default". * - partitionsBySection : Map of section names to maps of (partition name => weight). * A section called 'default' must be defined if not all wikis * have explicitly defined sections. * - configByPartition : Map of queue partition names to configuration arrays. * These configuration arrays are passed to JobQueue::factory(). * The options set here are overriden by those passed to this * the federated queue itself (e.g. 'order' and 'claimTTL'). * - partitionsNoPush : List of partition names that can handle pop() but not push(). * This can be used to migrate away from a certain partition. * @param array $params */ protected function __construct(array $params) { parent::__construct($params); $this->sectionsByWiki = $params['sectionsByWiki']; $this->partitionsBySection = $params['partitionsBySection']; $this->configByPartition = $params['configByPartition']; if (isset($params['partitionsNoPush'])) { $this->partitionsNoPush = array_flip($params['partitionsNoPush']); } $baseConfig = $params; foreach (array('class', 'sectionsByWiki', 'partitionsBySection', 'configByPartition', 'partitionsNoPush') as $o) { unset($baseConfig[$o]); } foreach ($this->getPartitionMap() as $partition => $w) { if (!isset($this->configByPartition[$partition])) { throw new MWException("No configuration for partition '{$partition}'."); } $this->partitionQueues[$partition] = JobQueue::factory($baseConfig + $this->configByPartition[$partition]); } // Aggregate cache some per-queue values if there are multiple partition queues $this->cache = $this->isFederated() ? wfGetMainCache() : new EmptyBagOStuff(); }