protected function setUp()
{
global $wgJobTypeConf;
parent::setUp();
$this->setMwGlobals('wgMemc', new HashBagOStuff());
if ($this->getCliArg('use-jobqueue')) {
$name = $this->getCliArg('use-jobqueue');
if (!isset($wgJobTypeConf[$name])) {
throw new MWException("No \$wgJobTypeConf entry for '{$name}'.");
}
$baseConfig = $wgJobTypeConf[$name];
} else {
$baseConfig = array('class' => 'JobQueueDB');
}
$baseConfig['type'] = 'null';
$baseConfig['wiki'] = wfWikiID();
$variants = array('queueRand' => array('order' => 'random', 'claimTTL' => 0), 'queueRandTTL' => array('order' => 'random', 'claimTTL' => 10), 'queueTimestamp' => array('order' => 'timestamp', 'claimTTL' => 0), 'queueTimestampTTL' => array('order' => 'timestamp', 'claimTTL' => 10), 'queueFifo' => array('order' => 'fifo', 'claimTTL' => 0), 'queueFifoTTL' => array('order' => 'fifo', 'claimTTL' => 10));
foreach ($variants as $q => $settings) {
try {
$this->{$q} = JobQueue::factory($settings + $baseConfig);
if (!$this->{$q} instanceof JobQueueDB) {
$this->{$q}->setTestingPrefix('unittests-' . wfRandomString(32));
}
} catch (MWException $e) {
// unsupported?
// @todo What if it was another error?
}
}
}
protected function setUp()
{
global $wgJobTypeConf;
parent::setUp();
if ($this->getCliArg('use-jobqueue')) {
$name = $this->getCliArg('use-jobqueue');
if (!isset($wgJobTypeConf[$name])) {
throw new MWException("No \$wgJobTypeConf entry for '{$name}'.");
}
$baseConfig = $wgJobTypeConf[$name];
} else {
$baseConfig = ['class' => 'JobQueueDB'];
}
$baseConfig['type'] = 'null';
$baseConfig['wiki'] = wfWikiID();
$variants = ['queueRand' => ['order' => 'random', 'claimTTL' => 0], 'queueRandTTL' => ['order' => 'random', 'claimTTL' => 10], 'queueTimestamp' => ['order' => 'timestamp', 'claimTTL' => 0], 'queueTimestampTTL' => ['order' => 'timestamp', 'claimTTL' => 10], 'queueFifo' => ['order' => 'fifo', 'claimTTL' => 0], 'queueFifoTTL' => ['order' => 'fifo', 'claimTTL' => 10]];
foreach ($variants as $q => $settings) {
try {
$this->{$q} = JobQueue::factory($settings + $baseConfig);
} catch (MWException $e) {
// unsupported?
// @todo What if it was another error?
}
}
}
/**
* @param $type string
* @return JobQueue Job queue object for a given queue type
*/
public function get($type)
{
global $wgJobTypeConf;
$conf = array('wiki' => $this->wiki, 'type' => $type);
if (isset($wgJobTypeConf[$type])) {
$conf = $conf + $wgJobTypeConf[$type];
} else {
$conf = $conf + $wgJobTypeConf['default'];
}
return JobQueue::factory($conf);
}
public function execute()
{
global $wgJobQueueMigrationConfig;
$srcKey = $this->getOption('src');
$dstKey = $this->getOption('dst');
if (!isset($wgJobQueueMigrationConfig[$srcKey])) {
$this->error("\$wgJobQueueMigrationConfig not set for '{$srcKey}'.", 1);
} elseif (!isset($wgJobQueueMigrationConfig[$dstKey])) {
$this->error("\$wgJobQueueMigrationConfig not set for '{$dstKey}'.", 1);
}
$types = $this->getOption('type') === 'all' ? JobQueueGroup::singleton()->getQueueTypes() : array($this->getOption('type'));
foreach ($types as $type) {
$baseConfig = array('type' => $type, 'wiki' => wfWikiID());
$src = JobQueue::factory($baseConfig + $wgJobQueueMigrationConfig[$srcKey]);
$dst = JobQueue::factory($baseConfig + $wgJobQueueMigrationConfig[$dstKey]);
list($total, $totalOK) = $this->copyJobs($src, $dst, $src->getAllQueuedJobs());
$this->output("Copied {$totalOK}/{$total} queued {$type} jobs.\n");
list($total, $totalOK) = $this->copyJobs($src, $dst, $src->getAllDelayedJobs());
$this->output("Copied {$totalOK}/{$total} delayed {$type} jobs.\n");
}
}
/**
* @return JobQueueMemory
*/
private function newJobQueue()
{
return JobQueue::factory(array('class' => 'JobQueueMemory', 'wiki' => wfWikiID(), 'type' => 'null'));
}
/**
* @return array
*/
protected function getCoalescedQueues()
{
global $wgJobTypeConf;
if ($this->coalescedQueues === null) {
$this->coalescedQueues = array();
foreach ($wgJobTypeConf as $type => $conf) {
$queue = JobQueue::factory(array('wiki' => $this->wiki, 'type' => 'null') + $conf);
$loc = $queue->getCoalesceLocationInternal();
if (!isset($this->coalescedQueues[$loc])) {
$this->coalescedQueues[$loc]['queue'] = $queue;
$this->coalescedQueues[$loc]['types'] = array();
}
if ($type === 'default') {
$this->coalescedQueues[$loc]['types'] = array_merge($this->coalescedQueues[$loc]['types'], array_diff($this->getQueueTypes(), array_keys($wgJobTypeConf)));
} else {
$this->coalescedQueues[$loc]['types'][] = $type;
}
}
}
return $this->coalescedQueues;
}
/**
* @params include:
* - sectionsByWiki : A map of wiki IDs to section names.
* Wikis will default to using the section "default".
* - partitionsBySection : Map of section names to maps of (partition name => weight).
* A section called 'default' must be defined if not all wikis
* have explicitly defined sections.
* - configByPartition : Map of queue partition names to configuration arrays.
* These configuration arrays are passed to JobQueue::factory().
* The options set here are overriden by those passed to this
* the federated queue itself (e.g. 'order' and 'claimTTL').
* - partitionsNoPush : List of partition names that can handle pop() but not push().
* This can be used to migrate away from a certain partition.
* @param array $params
*/
protected function __construct( array $params ) {
parent::__construct( $params );
$section = isset( $params['sectionsByWiki'][$this->wiki] )
? $params['sectionsByWiki'][$this->wiki]
: 'default';
if ( !isset( $params['partitionsBySection'][$section] ) ) {
throw new MWException( "No configuration for section '$section'." );
}
// Get the full partition map
$this->partitionMap = $params['partitionsBySection'][$section];
arsort( $this->partitionMap, SORT_NUMERIC );
// Get the partitions jobs can actually be pushed to
$partitionPushMap = $this->partitionMap;
if ( isset( $params['partitionsNoPush'] ) ) {
foreach ( $params['partitionsNoPush'] as $partition ) {
unset( $partitionPushMap[$partition] );
}
}
// Get the config to pass to merge into each partition queue config
$baseConfig = $params;
foreach ( array( 'class', 'sectionsByWiki',
'partitionsBySection', 'configByPartition', 'partitionsNoPush' ) as $o )
{
unset( $baseConfig[$o] );
}
// Get the partition queue objects
foreach ( $this->partitionMap as $partition => $w ) {
if ( !isset( $params['configByPartition'][$partition] ) ) {
throw new MWException( "No configuration for partition '$partition'." );
}
$this->partitionQueues[$partition] = JobQueue::factory(
$baseConfig + $params['configByPartition'][$partition] );
}
// Get the ring of partitions to push jobs into
$this->partitionPushRing = new HashRing( $partitionPushMap );
// Aggregate cache some per-queue values if there are multiple partition queues
$this->cache = count( $this->partitionMap ) > 1 ? wfGetMainCache() : new EmptyBagOStuff();
}
/**
* @param array $params Possible keys:
* - sectionsByWiki : A map of wiki IDs to section names.
* Wikis will default to using the section "default".
* - partitionsBySection : Map of section names to maps of (partition name => weight).
* A section called 'default' must be defined if not all wikis
* have explicitly defined sections.
* - configByPartition : Map of queue partition names to configuration arrays.
* These configuration arrays are passed to JobQueue::factory().
* The options set here are overridden by those passed to this
* the federated queue itself (e.g. 'order' and 'claimTTL').
* - maxPartitionsTry : Maximum number of times to attempt job insertion using
* different partition queues. This improves availability
* during failure, at the cost of added latency and somewhat
* less reliable job de-duplication mechanisms.
* @throws MWException
*/
protected function __construct(array $params)
{
parent::__construct($params);
$section = isset($params['sectionsByWiki'][$this->wiki]) ? $params['sectionsByWiki'][$this->wiki] : 'default';
if (!isset($params['partitionsBySection'][$section])) {
throw new MWException("No configuration for section '{$section}'.");
}
$this->maxPartitionsTry = isset($params['maxPartitionsTry']) ? $params['maxPartitionsTry'] : 2;
// Get the full partition map
$partitionMap = $params['partitionsBySection'][$section];
arsort($partitionMap, SORT_NUMERIC);
// Get the config to pass to merge into each partition queue config
$baseConfig = $params;
foreach (array('class', 'sectionsByWiki', 'maxPartitionsTry', 'partitionsBySection', 'configByPartition') as $o) {
unset($baseConfig[$o]);
// partition queue doesn't care about this
}
// The class handles all aggregator calls already
unset($baseConfig['aggregator']);
// Get the partition queue objects
foreach ($partitionMap as $partition => $w) {
if (!isset($params['configByPartition'][$partition])) {
throw new MWException("No configuration for partition '{$partition}'.");
}
$this->partitionQueues[$partition] = JobQueue::factory($baseConfig + $params['configByPartition'][$partition]);
}
// Ring of all partitions
$this->partitionRing = new HashRing($partitionMap);
}
/**
* @params include:
* - sectionsByWiki : A map of wiki IDs to section names.
* Wikis will default to using the section "default".
* - partitionsBySection : Map of section names to maps of (partition name => weight).
* A section called 'default' must be defined if not all wikis
* have explicitly defined sections.
* - configByPartition : Map of queue partition names to configuration arrays.
* These configuration arrays are passed to JobQueue::factory().
* The options set here are overriden by those passed to this
* the federated queue itself (e.g. 'order' and 'claimTTL').
* - partitionsNoPush : List of partition names that can handle pop() but not push().
* This can be used to migrate away from a certain partition.
* @param array $params
*/
protected function __construct(array $params)
{
parent::__construct($params);
$this->sectionsByWiki = $params['sectionsByWiki'];
$this->partitionsBySection = $params['partitionsBySection'];
$this->configByPartition = $params['configByPartition'];
if (isset($params['partitionsNoPush'])) {
$this->partitionsNoPush = array_flip($params['partitionsNoPush']);
}
$baseConfig = $params;
foreach (array('class', 'sectionsByWiki', 'partitionsBySection', 'configByPartition', 'partitionsNoPush') as $o) {
unset($baseConfig[$o]);
}
foreach ($this->getPartitionMap() as $partition => $w) {
if (!isset($this->configByPartition[$partition])) {
throw new MWException("No configuration for partition '{$partition}'.");
}
$this->partitionQueues[$partition] = JobQueue::factory($baseConfig + $this->configByPartition[$partition]);
}
// Aggregate cache some per-queue values if there are multiple partition queues
$this->cache = $this->isFederated() ? wfGetMainCache() : new EmptyBagOStuff();
}
