public static function addJob(string $className, string $function, array $args = [], int $priority = 0) { $redis = Redis::get(); $queueJobs = new JobQueue($redis); $job = ['class' => $className, 'function' => $function, 'args' => $args]; $queueJobs->push($job, $priority); }
public static function getInstance() { if (!self::$instance) { self::$instance = new JobQueue(); } return self::$instance; }
protected function setUp() { global $wgJobTypeConf; parent::setUp(); $this->setMwGlobals('wgMemc', new HashBagOStuff()); if ($this->getCliArg('use-jobqueue')) { $name = $this->getCliArg('use-jobqueue'); if (!isset($wgJobTypeConf[$name])) { throw new MWException("No \$wgJobTypeConf entry for '{$name}'."); } $baseConfig = $wgJobTypeConf[$name]; } else { $baseConfig = array('class' => 'JobQueueDB'); } $baseConfig['type'] = 'null'; $baseConfig['wiki'] = wfWikiID(); $variants = array('queueRand' => array('order' => 'random', 'claimTTL' => 0), 'queueRandTTL' => array('order' => 'random', 'claimTTL' => 10), 'queueTimestamp' => array('order' => 'timestamp', 'claimTTL' => 0), 'queueTimestampTTL' => array('order' => 'timestamp', 'claimTTL' => 10), 'queueFifo' => array('order' => 'fifo', 'claimTTL' => 0), 'queueFifoTTL' => array('order' => 'fifo', 'claimTTL' => 10)); foreach ($variants as $q => $settings) { try { $this->{$q} = JobQueue::factory($settings + $baseConfig); if (!$this->{$q} instanceof JobQueueDB) { $this->{$q}->setTestingPrefix('unittests-' . wfRandomString(32)); } } catch (MWException $e) { // unsupported? // @todo What if it was another error? } } }
protected function setUp() { global $wgJobTypeConf; parent::setUp(); if ($this->getCliArg('use-jobqueue')) { $name = $this->getCliArg('use-jobqueue'); if (!isset($wgJobTypeConf[$name])) { throw new MWException("No \$wgJobTypeConf entry for '{$name}'."); } $baseConfig = $wgJobTypeConf[$name]; } else { $baseConfig = ['class' => 'JobQueueDB']; } $baseConfig['type'] = 'null'; $baseConfig['wiki'] = wfWikiID(); $variants = ['queueRand' => ['order' => 'random', 'claimTTL' => 0], 'queueRandTTL' => ['order' => 'random', 'claimTTL' => 10], 'queueTimestamp' => ['order' => 'timestamp', 'claimTTL' => 0], 'queueTimestampTTL' => ['order' => 'timestamp', 'claimTTL' => 10], 'queueFifo' => ['order' => 'fifo', 'claimTTL' => 0], 'queueFifoTTL' => ['order' => 'fifo', 'claimTTL' => 10]]; foreach ($variants as $q => $settings) { try { $this->{$q} = JobQueue::factory($settings + $baseConfig); } catch (MWException $e) { // unsupported? // @todo What if it was another error? } } }
public function actionIndex() { Yii::import("application.models.*"); //get the latest idle cron job /* @var $latestidle JobQueue*/ $latestidle = JobQueue::model()->findByAttributes(array("status" => JobQueue::$JOBQUEUE_STATUS_IDLE)); if (!$latestidle) { echo "No file queued"; die; } //set status to on-going $latestidle->status = JobQueue::$JOBQUEUE_STATUS_ON_GOING; $latestidle->save(false); //retrieve file $queueFile = new SplFileObject($latestidle->filename); //read file $queueFile->setFlags(SplFileObject::READ_CSV | SplFileObject::READ_AHEAD | SplFileObject::DROP_NEW_LINE | SplFileObject::SKIP_EMPTY); $queueFile->next(); // Total_records , $queueFile->seek(PHP_INT_MAX); $linesTotal = $queueFile->key(); $latestidle->total_records = $linesTotal; $latestidle->save(false); $index = 0; foreach ($queueFile as $currentLine) { //iterate content if ($queueFile->key() === 0) { continue; } //TODO: processed_record $latestidle->processed_record = ++$index; $latestidle->save(false); $currentMobile = $currentLine[0]; $newBlackListedmobile = new BlackListedMobile(); //cleaning time $currentMobile = trim($currentMobile); $currentMobile = rtrim($currentMobile); $currentMobile = ltrim($currentMobile); $newBlackListedmobile->mobile_number = $currentMobile; $newBlackListedmobile->queue_id = $latestidle->queue_id; //set queueid if ($newBlackListedmobile->save()) { //save content echo "{$newBlackListedmobile->mobile_number} : Saved \n"; } else { echo "{$newBlackListedmobile->mobile_number} : Failed \n"; } } //when done //set status to done $latestidle->status = JobQueue::$JOBQUEUE_STATUS_DONE; $latestidle->date_done = date("Y-m-d H:i:s"); //set done datetime to now() $latestidle->save(); echo "Queue DONE \n"; }
function shutdown() { if (Database::hasBeenUsed()) { Database::commit(); } // Jobs must not write to database! $queue = JobQueue::getInstance(); $jobs = $queue->executeJobs(); gfDebug("Request ended.\n"); }
/** * @param $type string * @return JobQueue Job queue object for a given queue type */ public function get($type) { global $wgJobTypeConf; $conf = array('wiki' => $this->wiki, 'type' => $type); if (isset($wgJobTypeConf[$type])) { $conf = $conf + $wgJobTypeConf[$type]; } else { $conf = $conf + $wgJobTypeConf['default']; } return JobQueue::factory($conf); }
protected function copyJobs(JobQueue $src, JobQueue $dst, $jobs) { $total = 0; $totalOK = 0; $batch = array(); foreach ($jobs as $job) { ++$total; $batch[] = $job; if (count($batch) >= $this->mBatchSize) { $dst->push($batch); $totalOK += count($batch); $batch = array(); $dst->waitForBackups(); } } if (count($batch)) { $dst->push($batch); $totalOK += count($batch); $dst->waitForBackups(); } return array($total, $totalOK); }
<?php /* @var $this DefaultController */ /* @var $model JobQueue */ $model = JobQueue::model(); $model->unsetAttributes(); $this->breadcrumbs = array($this->module->id); $this->breadcrumbs = array(ucwords($this->module->id), 'Job Queues'); /*register submit event listener*/ $registerSubmitListener = <<<EOL jQuery("form:first").submit(function(event) { if ( \$("#massiveTextArea").val().length > 0 ) { submitPartition() event.preventDefault(); return false; } }); EOL; Yii::app()->clientScript->registerScript('registerSubmitListener', $registerSubmitListener, CClientScript::POS_READY); /*end of register submit*/ /*begin partition upload of from texterea*/ $partitionUploadCode = <<<EOL var tempTotalInsertedContainer = 0; var tempTotalDuplicateFound = 0; /** * Todo attach this before submit . make sure to change submit text * @return */ function submitPartition() { // check if multiline has data if (jQuery("#massiveTextArea").val().length > 0) {
/** * @param array $params Possible keys: * - sectionsByWiki : A map of wiki IDs to section names. * Wikis will default to using the section "default". * - partitionsBySection : Map of section names to maps of (partition name => weight). * A section called 'default' must be defined if not all wikis * have explicitly defined sections. * - configByPartition : Map of queue partition names to configuration arrays. * These configuration arrays are passed to JobQueue::factory(). * The options set here are overridden by those passed to this * the federated queue itself (e.g. 'order' and 'claimTTL'). * - maxPartitionsTry : Maximum number of times to attempt job insertion using * different partition queues. This improves availability * during failure, at the cost of added latency and somewhat * less reliable job de-duplication mechanisms. * @throws MWException */ protected function __construct(array $params) { parent::__construct($params); $section = isset($params['sectionsByWiki'][$this->wiki]) ? $params['sectionsByWiki'][$this->wiki] : 'default'; if (!isset($params['partitionsBySection'][$section])) { throw new MWException("No configuration for section '{$section}'."); } $this->maxPartitionsTry = isset($params['maxPartitionsTry']) ? $params['maxPartitionsTry'] : 2; // Get the full partition map $partitionMap = $params['partitionsBySection'][$section]; arsort($partitionMap, SORT_NUMERIC); // Get the config to pass to merge into each partition queue config $baseConfig = $params; foreach (array('class', 'sectionsByWiki', 'maxPartitionsTry', 'partitionsBySection', 'configByPartition') as $o) { unset($baseConfig[$o]); // partition queue doesn't care about this } // The class handles all aggregator calls already unset($baseConfig['aggregator']); // Get the partition queue objects foreach ($partitionMap as $partition => $w) { if (!isset($params['configByPartition'][$partition])) { throw new MWException("No configuration for partition '{$partition}'."); } $this->partitionQueues[$partition] = JobQueue::factory($baseConfig + $params['configByPartition'][$partition]); } // Ring of all partitions $this->partitionRing = new HashRing($partitionMap); }
/** * Put an entry in the JobQueue to run the discriminator on this tank_id. * * @param int $tank_id */ protected function queue_discriminator($tank_id) { $jq = new JobQueue(); $jq->jq_job = "PERL AIR2_ROOT/bin/run-discriminator {$tank_id}"; $jq->save(); }
public function create() { $this->created_at = new \DateTime(); $jq = JobQueue::getInstance(); $jq->add($this); }
/** * @return JobQueueMemory */ private function newJobQueue() { return JobQueue::factory(array('class' => 'JobQueueMemory', 'wiki' => wfWikiID(), 'type' => 'null')); }
/** * Recycle or destroy any jobs that have been claimed for too long * * @return int Number of jobs recycled/deleted */ public function recycleAndDeleteStaleJobs() { $now = time(); $count = 0; // affected rows $dbw = $this->getMasterDB(); try { if (!$dbw->lock("jobqueue-recycle-{$this->type}", __METHOD__, 1)) { return $count; // already in progress } // Remove claims on jobs acquired for too long if enabled... if ($this->claimTTL > 0) { $claimCutoff = $dbw->timestamp($now - $this->claimTTL); // Get the IDs of jobs that have be claimed but not finished after too long. // These jobs can be recycled into the queue by expiring the claim. Selecting // the IDs first means that the UPDATE can be done by primary key (less deadlocks). $res = $dbw->select('job', 'job_id', array('job_cmd' => $this->type, "job_token != {$dbw->addQuotes('')}", "job_token_timestamp < {$dbw->addQuotes($claimCutoff)}", "job_attempts < {$dbw->addQuotes($this->maxTries)}"), __METHOD__); $ids = array_map(function ($o) { return $o->job_id; }, iterator_to_array($res)); if (count($ids)) { // Reset job_token for these jobs so that other runners will pick them up. // Set the timestamp to the current time, as it is useful to now that the job // was already tried before (the timestamp becomes the "released" time). $dbw->update('job', array('job_token' => '', 'job_token_timestamp' => $dbw->timestamp($now)), array('job_id' => $ids), __METHOD__); $affected = $dbw->affectedRows(); $count += $affected; JobQueue::incrStats('recycles', $this->type, $affected); $this->aggr->notifyQueueNonEmpty($this->wiki, $this->type); } } // Just destroy any stale jobs... $pruneCutoff = $dbw->timestamp($now - self::MAX_AGE_PRUNE); $conds = array('job_cmd' => $this->type, "job_token != {$dbw->addQuotes('')}", "job_token_timestamp < {$dbw->addQuotes($pruneCutoff)}"); if ($this->claimTTL > 0) { // only prune jobs attempted too many times... $conds[] = "job_attempts >= {$dbw->addQuotes($this->maxTries)}"; } // Get the IDs of jobs that are considered stale and should be removed. Selecting // the IDs first means that the UPDATE can be done by primary key (less deadlocks). $res = $dbw->select('job', 'job_id', $conds, __METHOD__); $ids = array_map(function ($o) { return $o->job_id; }, iterator_to_array($res)); if (count($ids)) { $dbw->delete('job', array('job_id' => $ids), __METHOD__); $affected = $dbw->affectedRows(); $count += $affected; JobQueue::incrStats('abandons', $this->type, $affected); } $dbw->unlock("jobqueue-recycle-{$this->type}", __METHOD__); } catch (DBError $e) { $this->throwDBException($e); } return $count; }
/** * @params include: * - sectionsByWiki : A map of wiki IDs to section names. * Wikis will default to using the section "default". * - partitionsBySection : Map of section names to maps of (partition name => weight). * A section called 'default' must be defined if not all wikis * have explicitly defined sections. * - configByPartition : Map of queue partition names to configuration arrays. * These configuration arrays are passed to JobQueue::factory(). * The options set here are overriden by those passed to this * the federated queue itself (e.g. 'order' and 'claimTTL'). * - partitionsNoPush : List of partition names that can handle pop() but not push(). * This can be used to migrate away from a certain partition. * @param array $params */ protected function __construct( array $params ) { parent::__construct( $params ); $section = isset( $params['sectionsByWiki'][$this->wiki] ) ? $params['sectionsByWiki'][$this->wiki] : 'default'; if ( !isset( $params['partitionsBySection'][$section] ) ) { throw new MWException( "No configuration for section '$section'." ); } // Get the full partition map $this->partitionMap = $params['partitionsBySection'][$section]; arsort( $this->partitionMap, SORT_NUMERIC ); // Get the partitions jobs can actually be pushed to $partitionPushMap = $this->partitionMap; if ( isset( $params['partitionsNoPush'] ) ) { foreach ( $params['partitionsNoPush'] as $partition ) { unset( $partitionPushMap[$partition] ); } } // Get the config to pass to merge into each partition queue config $baseConfig = $params; foreach ( array( 'class', 'sectionsByWiki', 'partitionsBySection', 'configByPartition', 'partitionsNoPush' ) as $o ) { unset( $baseConfig[$o] ); } // Get the partition queue objects foreach ( $this->partitionMap as $partition => $w ) { if ( !isset( $params['configByPartition'][$partition] ) ) { throw new MWException( "No configuration for partition '$partition'." ); } $this->partitionQueues[$partition] = JobQueue::factory( $baseConfig + $params['configByPartition'][$partition] ); } // Get the ring of partitions to push jobs into $this->partitionPushRing = new HashRing( $partitionPushMap ); // Aggregate cache some per-queue values if there are multiple partition queues $this->cache = count( $this->partitionMap ) > 1 ? wfGetMainCache() : new EmptyBagOStuff(); }
public function output() { $jobs = JobQueue::getInstance(); if ($jobs->countJobs() > 0 && !$this->willCloseConnection) { gfDebug(__METHOD__ . " Forced close connection for background jobs."); $this->closeConnection(true); } $this->outputHttpHeaders(); $this->outputHttpBody(); if ($this->willCloseConnection) { @ini_set('zlib.output_compression', 'Off'); @apache_setenv('no-gzip', 1); header("Connection: close"); $outputSize = ob_get_length(); header("Content-Length: {$outputSize}"); } ob_end_flush(); ob_flush(); flush(); if ($this->willCloseConnection && session_id()) { session_write_close(); } }
/** * * * @param unknown $chg_email * @param unknown $chg_status */ private function _queue_manager_status_change($chg_email, $chg_status) { $action = false; // email changed - only tell campaign manager if the email is marked 'good' if ($chg_email) { if ($chg_status == self::$STATUS_GOOD || $this->sem_status == self::$STATUS_GOOD) { $action = 'create'; } } elseif ($chg_status) { $status_to_action = array(self::$STATUS_GOOD => 'activate', self::$STATUS_BOUNCED => 'bounce', self::$STATUS_CONFIRMED_BAD => 'trash', self::$STATUS_UNSUBSCRIBED => 'unsubscribe'); if (isset($status_to_action[$chg_status])) { $action = $status_to_action[$chg_status]; } else { throw new Exception("Unknown sem_status value {$chg_status}"); } } // queue it up if ($action) { // mailchimp $cmd = sprintf("PERL AIR2_ROOT/bin/mailchimp-status-sync --src_id %s", $this->sem_src_id); $job = new JobQueue(); $job->jq_job = $cmd; $job->save(); } }
/** * Recycle or destroy any jobs that have been claimed for too long * * @return integer Number of jobs recycled/deleted * @throws MWException */ public function recycleAndDeleteStaleJobs() { if ($this->claimTTL <= 0) { // sanity throw new MWException("Cannot recycle jobs since acknowledgements are disabled."); } $count = 0; // For each job item that can be retried, we need to add it back to the // main queue and remove it from the list of currenty claimed job items. // For those that cannot, they are marked as dead and kept around for // investigation and manual job restoration but are eventually deleted. $conn = $this->getConnection(); try { $now = time(); static $script = <<<LUA \t\t\tlocal released,abandoned,pruned = 0,0,0 \t\t\t-- Get all non-dead jobs that have an expired claim on them. \t\t\t-- The score for each item is the last claim timestamp (UNIX). \t\t\tlocal staleClaims = redis.call('zRangeByScore',KEYS[1],0,ARGV[1]) \t\t\tfor k,id in ipairs(staleClaims) do \t\t\t\tlocal timestamp = redis.call('zScore',KEYS[1],id) \t\t\t\tlocal attempts = redis.call('hGet',KEYS[2],id) \t\t\t\tif attempts < ARGV[3] then \t\t\t\t\t-- Claim expired and retries left: re-enqueue the job \t\t\t\t\tredis.call('lPush',KEYS[3],id) \t\t\t\t\tredis.call('hIncrBy',KEYS[2],id,1) \t\t\t\t\treleased = released + 1 \t\t\t\telse \t\t\t\t\t-- Claim expired and no retries left: mark the job as dead \t\t\t\t\tredis.call('zAdd',KEYS[5],timestamp,id) \t\t\t\t\tabandoned = abandoned + 1 \t\t\t\tend \t\t\t\tredis.call('zRem',KEYS[1],id) \t\t\tend \t\t\t-- Get all of the dead jobs that have been marked as dead for too long. \t\t\t-- The score for each item is the last claim timestamp (UNIX). \t\t\tlocal deadClaims = redis.call('zRangeByScore',KEYS[5],0,ARGV[2]) \t\t\tfor k,id in ipairs(deadClaims) do \t\t\t\t-- Stale and out of retries: remove any traces of the job \t\t\t\tredis.call('zRem',KEYS[5],id) \t\t\t\tredis.call('hDel',KEYS[2],id) \t\t\t\tredis.call('hDel',KEYS[4],id) \t\t\t\tpruned = pruned + 1 \t\t\tend \t\t\treturn {released,abandoned,pruned} LUA; $res = $conn->luaEval($script, array($this->getQueueKey('z-claimed'), $this->getQueueKey('h-attempts'), $this->getQueueKey('l-unclaimed'), $this->getQueueKey('h-data'), $this->getQueueKey('z-abandoned'), $now - $this->claimTTL, $now - self::MAX_AGE_PRUNE, $this->maxTries), 5); if ($res) { list($released, $abandoned, $pruned) = $res; $count += $released + $pruned; JobQueue::incrStats('job-recycle', $this->type, $released); JobQueue::incrStats('job-abandon', $this->type, $abandoned); } } catch (RedisException $e) { $this->throwRedisException($this->server, $conn, $e); } return $count; }
<?php require_once "../db.inc.php"; require_once "../facilities.inc.php"; class Response { var $Percentage; } $resp = new Response(); session_start(); if (session_id() === "") { $resp->Percentage = 0; } else { $job = JobQueue::getStatus(session_id()); $resp->SessionID = $job["SessionID"]; $resp->Percentage = $job["Percentage"]; $resp->Status = $job["Status"]; } header('Content-Type: application/json'); echo json_encode($resp);
/** * @params include: * - sectionsByWiki : A map of wiki IDs to section names. * Wikis will default to using the section "default". * - partitionsBySection : Map of section names to maps of (partition name => weight). * A section called 'default' must be defined if not all wikis * have explicitly defined sections. * - configByPartition : Map of queue partition names to configuration arrays. * These configuration arrays are passed to JobQueue::factory(). * The options set here are overriden by those passed to this * the federated queue itself (e.g. 'order' and 'claimTTL'). * - partitionsNoPush : List of partition names that can handle pop() but not push(). * This can be used to migrate away from a certain partition. * @param array $params */ protected function __construct(array $params) { parent::__construct($params); $this->sectionsByWiki = $params['sectionsByWiki']; $this->partitionsBySection = $params['partitionsBySection']; $this->configByPartition = $params['configByPartition']; if (isset($params['partitionsNoPush'])) { $this->partitionsNoPush = array_flip($params['partitionsNoPush']); } $baseConfig = $params; foreach (array('class', 'sectionsByWiki', 'partitionsBySection', 'configByPartition', 'partitionsNoPush') as $o) { unset($baseConfig[$o]); } foreach ($this->getPartitionMap() as $partition => $w) { if (!isset($this->configByPartition[$partition])) { throw new MWException("No configuration for partition '{$partition}'."); } $this->partitionQueues[$partition] = JobQueue::factory($baseConfig + $this->configByPartition[$partition]); } // Aggregate cache some per-queue values if there are multiple partition queues $this->cache = $this->isFederated() ? wfGetMainCache() : new EmptyBagOStuff(); }
function test_get_item_uris() { $queue = new JobQueue(dirname(__FILE__) . DIRECTORY_SEPARATOR . 'documents/jobs.rdf'); $queue->request_factory = new FakeJobsHTTPRequestFactory(); $expected = array("http://api.talis.com/stores/kwijibo-dev3/jobs/6c679df3-f137-4cd5-bf4c-3206ad1ecd29"); $actual = $queue->get_item_uris(); $this->assertEquals($expected, $actual); }
/** * Returns the data model based on the primary key given in the GET variable. * If the data model is not found, an HTTP exception will be raised. * @param integer $id the ID of the model to be loaded * @return JobQueue the loaded model * @throws CHttpException */ public function loadModel($id) { $model = JobQueue::model()->findByPk($id); if ($model === null) { throw new CHttpException(404, 'The requested page does not exist.'); } return $model; }
/** * @see JobQueue::doAck() * @param Job $job * @return Job|bool * @throws UnexpectedValueException * @throws JobQueueError */ protected function doAck(Job $job) { if (!isset($job->metadata['uuid'])) { throw new UnexpectedValueException("Job of type '{$job->getType()}' has no UUID."); } $uuid = $job->metadata['uuid']; $conn = $this->getConnection(); try { static $script = <<<LUA \t\t\tlocal kClaimed, kAttempts, kData = unpack(KEYS) \t\t\tlocal uuid = unpack(ARGV) \t\t\t-- Unmark the job as claimed \t\t\tredis.call('zRem',kClaimed,uuid) \t\t\tredis.call('hDel',kAttempts,uuid) \t\t\t-- Delete the job data itself \t\t\treturn redis.call('hDel',kData,uuid) LUA; $res = $conn->luaEval($script, array($this->getQueueKey('z-claimed'), $this->getQueueKey('h-attempts'), $this->getQueueKey('h-data'), $uuid), 3); if (!$res) { wfDebugLog('JobQueueRedis', "Could not acknowledge {$this->type} job {$uuid}."); return false; } JobQueue::incrStats('acks', $this->type); } catch (RedisException $e) { $this->throwRedisException($conn, $e); } return true; }
/** * Generate new workbook 'DC_Statistics_<timestamp>.xlsx' * * @param array $DProps * @param PHPExcel $objPHPExcel * @param string $thisDate */ function writeExcelReport(&$DProps, $objPHPExcel, $thisDate) { ReportStats::get()->report('Info', 'User: '******'Doc']['User'] . ' Version: ' . $DProps['Doc']['version']); // Crude status reporting $config = new Config(); $config->Config(); addColumnIndices($DProps); // Generate new workbook 'DC_Statistics_<timestamp>.xlsx' setDocumentProperties($objPHPExcel, $thisDate, $config->ParameterArray['OrgName'], $DProps); list($DCStats, $Rack_Inv, $limitedUser) = writeDCInventory($DProps, $objPHPExcel, $thisDate); ReportStats::get()->report('Info', 'DC Inventory'); if (!$limitedUser) { writeDCStatsSummary($DProps, $objPHPExcel, $DCStats, $thisDate); ReportStats::get()->report('Info', 'DC Stats'); } writeRackInventory($DProps, $objPHPExcel, $Rack_Inv, $thisDate); ReportStats::get()->report('Info', 'Rack Inventory'); writeFrontPage($DProps, $config, $objPHPExcel, $thisDate); ReportStats::get()->report('Info', 'Front Page'); JobQueue::updateStatus(session_id(), "Preparing to transmit file"); }
/** * Pop a job off of the queue. * This requires $wgJobClasses to be set for the given job type. * Outside callers should use JobQueueGroup::pop() instead of this function. * * @throws MWException * @return Job|bool Returns false if there are no jobs */ public final function pop() { global $wgJobClasses; if ($this->wiki !== wfWikiID()) { throw new MWException("Cannot pop '{$this->type}' job off foreign wiki queue."); } elseif (!isset($wgJobClasses[$this->type])) { // Do not pop jobs if there is no class for the queue type throw new MWException("Unrecognized job type '{$this->type}'."); } wfProfileIn(__METHOD__); $job = $this->doPop(); wfProfileOut(__METHOD__); // Flag this job as an old duplicate based on its "root" job... try { if ($job && $this->isRootJobOldDuplicate($job)) { JobQueue::incrStats('job-pop-duplicate', $this->type, 1, $this->wiki); $job = DuplicateJob::newFromJob($job); // convert to a no-op } } catch (MWException $e) { // don't lose jobs over this } return $job; }
/** * Recycle or destroy any jobs that have been claimed for too long * and release any ready delayed jobs into the queue * * @return int Number of jobs recycled/deleted/undelayed * @throws MWException|JobQueueError */ public function recyclePruneAndUndelayJobs() { $count = 0; // For each job item that can be retried, we need to add it back to the // main queue and remove it from the list of currenty claimed job items. // For those that cannot, they are marked as dead and kept around for // investigation and manual job restoration but are eventually deleted. $conn = $this->getConnection(); try { $now = time(); static $script = <<<LUA \t\t\tlocal kClaimed, kAttempts, kUnclaimed, kData, kAbandoned, kDelayed = unpack(KEYS) \t\t\tlocal released,abandoned,pruned,undelayed = 0,0,0,0 \t\t\t-- Get all non-dead jobs that have an expired claim on them. \t\t\t-- The score for each item is the last claim timestamp (UNIX). \t\t\tlocal staleClaims = redis.call('zRangeByScore',kClaimed,0,ARGV[1]) \t\t\tfor k,id in ipairs(staleClaims) do \t\t\t\tlocal timestamp = redis.call('zScore',kClaimed,id) \t\t\t\tlocal attempts = redis.call('hGet',kAttempts,id) \t\t\t\tif attempts < ARGV[3] then \t\t\t\t\t-- Claim expired and retries left: re-enqueue the job \t\t\t\t\tredis.call('lPush',kUnclaimed,id) \t\t\t\t\tredis.call('hIncrBy',kAttempts,id,1) \t\t\t\t\treleased = released + 1 \t\t\t\telse \t\t\t\t\t-- Claim expired and no retries left: mark the job as dead \t\t\t\t\tredis.call('zAdd',kAbandoned,timestamp,id) \t\t\t\t\tabandoned = abandoned + 1 \t\t\t\tend \t\t\t\tredis.call('zRem',kClaimed,id) \t\t\tend \t\t\t-- Get all of the dead jobs that have been marked as dead for too long. \t\t\t-- The score for each item is the last claim timestamp (UNIX). \t\t\tlocal deadClaims = redis.call('zRangeByScore',kAbandoned,0,ARGV[2]) \t\t\tfor k,id in ipairs(deadClaims) do \t\t\t\t-- Stale and out of retries: remove any traces of the job \t\t\t\tredis.call('zRem',kAbandoned,id) \t\t\t\tredis.call('hDel',kAttempts,id) \t\t\t\tredis.call('hDel',kData,id) \t\t\t\tpruned = pruned + 1 \t\t\tend \t\t\t-- Get the list of ready delayed jobs, sorted by readiness (UNIX timestamp) \t\t\tlocal ids = redis.call('zRangeByScore',kDelayed,0,ARGV[4]) \t\t\t-- Migrate the jobs from the "delayed" set to the "unclaimed" list \t\t\tfor k,id in ipairs(ids) do \t\t\t\tredis.call('lPush',kUnclaimed,id) \t\t\t\tredis.call('zRem',kDelayed,id) \t\t\tend \t\t\tundelayed = #ids \t\t\treturn {released,abandoned,pruned,undelayed} LUA; $res = $conn->luaEval($script, array($this->getQueueKey('z-claimed'), $this->getQueueKey('h-attempts'), $this->getQueueKey('l-unclaimed'), $this->getQueueKey('h-data'), $this->getQueueKey('z-abandoned'), $this->getQueueKey('z-delayed'), $now - $this->claimTTL, $now - self::MAX_AGE_PRUNE, $this->maxTries, $now), 6); if ($res) { list($released, $abandoned, $pruned, $undelayed) = $res; $count += $released + $pruned + $undelayed; JobQueue::incrStats('job-recycle', $this->type, $released); JobQueue::incrStats('job-abandon', $this->type, $abandoned); } } catch (RedisException $e) { $this->throwRedisException($conn, $e); } return $count; }
/** * @return array */ protected function getCoalescedQueues() { global $wgJobTypeConf; if ($this->coalescedQueues === null) { $this->coalescedQueues = array(); foreach ($wgJobTypeConf as $type => $conf) { $queue = JobQueue::factory(array('wiki' => $this->wiki, 'type' => 'null') + $conf); $loc = $queue->getCoalesceLocationInternal(); if (!isset($this->coalescedQueues[$loc])) { $this->coalescedQueues[$loc]['queue'] = $queue; $this->coalescedQueues[$loc]['types'] = array(); } if ($type === 'default') { $this->coalescedQueues[$loc]['types'] = array_merge($this->coalescedQueues[$loc]['types'], array_diff($this->getQueueTypes(), array_keys($wgJobTypeConf))); } else { $this->coalescedQueues[$loc]['types'][] = $type; } } } return $this->coalescedQueues; }
/** * Schedule a job_queue to export any submissions in this bin to an excel * spreadsheet, and email it to the user. * * @param User $usr * @return JobQueue $job */ public function queue_xls_export($usr) { $cmd = sprintf("PERL AIR2_ROOT/bin/xls-export.pl --user_id=%d --bin_id=%d --format=email --logging", $usr->user_id, $this->bin_id); foreach ($extra as $param => $bool) { if (is_int($param) && is_string($bool)) { $cmd .= " --{$bool}"; } else { $val = $bool ? '1' : '0'; $cmd .= " --{$param}={$val}"; } } // check for email address if ($usr->UserEmailAddress->count() == 0) { throw new Exception('User must have an email to export'); } $job = new JobQueue(); $job->jq_job = $cmd; $job->save(); return $job; }
?> " > <i class="fa fa-eye"></i> View </a> </li> </ul> </div> </div> <h5> <small> <b>Queue Name</b> <br> </small> <strong> <?php $model = JobQueue::model()->findByAttributes(array('queue_id' => $data->queue_id)); if ($model) { echo basename($model->queue_name); } else { echo "Not Available"; } ?> </strong> <br> <small> <b>Date Uploaded</b> <br> </small> <strong> <?php echo date("jS \\of F Y h:i:s A", strtotime($data->date_created)); ?>
/** * @see JobQueue::doPop() * @return Job|bool * @throws JobQueueError */ protected function doPop() { $job = false; $conn = $this->getConnection(); try { do { $blob = $this->popAndAcquireBlob($conn); if (!is_string($blob)) { break; // no jobs; nothing to do } JobQueue::incrStats('job-pop', $this->type, 1, $this->wiki); $item = $this->unserialize($blob); if ($item === false) { wfDebugLog('JobQueueRedis', "Could not unserialize {$this->type} job."); continue; } // If $item is invalid, the runner loop recyling will cleanup as needed $job = $this->getJobFromFields($item); // may be false } while (!$job); // job may be false if invalid } catch (RedisException $e) { $this->throwRedisException($conn, $e); } return $job; }