/** * Pop a job off of the queue. * This requires $wgJobClasses to be set for the given job type. * Outside callers should use JobQueueGroup::pop() instead of this function. * * @throws MWException * @return Job|bool Returns false if there are no jobs */ public final function pop() { global $wgJobClasses; if ($this->wiki !== wfWikiID()) { throw new MWException("Cannot pop '{$this->type}' job off foreign wiki queue."); } elseif (!isset($wgJobClasses[$this->type])) { // Do not pop jobs if there is no class for the queue type throw new MWException("Unrecognized job type '{$this->type}'."); } wfProfileIn(__METHOD__); $job = $this->doPop(); wfProfileOut(__METHOD__); // Flag this job as an old duplicate based on its "root" job... try { if ($job && $this->isRootJobOldDuplicate($job)) { JobQueue::incrStats('job-pop-duplicate', $this->type, 1, $this->wiki); $job = DuplicateJob::newFromJob($job); // convert to a no-op } } catch (MWException $e) { // don't lose jobs over this } return $job; }
/** * Recycle or destroy any jobs that have been claimed for too long * * @return int Number of jobs recycled/deleted */ public function recycleAndDeleteStaleJobs() { $now = time(); $count = 0; // affected rows $dbw = $this->getMasterDB(); try { if (!$dbw->lock("jobqueue-recycle-{$this->type}", __METHOD__, 1)) { return $count; // already in progress } // Remove claims on jobs acquired for too long if enabled... if ($this->claimTTL > 0) { $claimCutoff = $dbw->timestamp($now - $this->claimTTL); // Get the IDs of jobs that have be claimed but not finished after too long. // These jobs can be recycled into the queue by expiring the claim. Selecting // the IDs first means that the UPDATE can be done by primary key (less deadlocks). $res = $dbw->select('job', 'job_id', array('job_cmd' => $this->type, "job_token != {$dbw->addQuotes('')}", "job_token_timestamp < {$dbw->addQuotes($claimCutoff)}", "job_attempts < {$dbw->addQuotes($this->maxTries)}"), __METHOD__); $ids = array_map(function ($o) { return $o->job_id; }, iterator_to_array($res)); if (count($ids)) { // Reset job_token for these jobs so that other runners will pick them up. // Set the timestamp to the current time, as it is useful to now that the job // was already tried before (the timestamp becomes the "released" time). $dbw->update('job', array('job_token' => '', 'job_token_timestamp' => $dbw->timestamp($now)), array('job_id' => $ids), __METHOD__); $affected = $dbw->affectedRows(); $count += $affected; JobQueue::incrStats('recycles', $this->type, $affected); $this->aggr->notifyQueueNonEmpty($this->wiki, $this->type); } } // Just destroy any stale jobs... $pruneCutoff = $dbw->timestamp($now - self::MAX_AGE_PRUNE); $conds = array('job_cmd' => $this->type, "job_token != {$dbw->addQuotes('')}", "job_token_timestamp < {$dbw->addQuotes($pruneCutoff)}"); if ($this->claimTTL > 0) { // only prune jobs attempted too many times... $conds[] = "job_attempts >= {$dbw->addQuotes($this->maxTries)}"; } // Get the IDs of jobs that are considered stale and should be removed. Selecting // the IDs first means that the UPDATE can be done by primary key (less deadlocks). $res = $dbw->select('job', 'job_id', $conds, __METHOD__); $ids = array_map(function ($o) { return $o->job_id; }, iterator_to_array($res)); if (count($ids)) { $dbw->delete('job', array('job_id' => $ids), __METHOD__); $affected = $dbw->affectedRows(); $count += $affected; JobQueue::incrStats('abandons', $this->type, $affected); } $dbw->unlock("jobqueue-recycle-{$this->type}", __METHOD__); } catch (DBError $e) { $this->throwDBException($e); } return $count; }
/** * @see JobQueue::doAck() * @param Job $job * @return Job|bool * @throws UnexpectedValueException * @throws JobQueueError */ protected function doAck(Job $job) { if (!isset($job->metadata['uuid'])) { throw new UnexpectedValueException("Job of type '{$job->getType()}' has no UUID."); } $uuid = $job->metadata['uuid']; $conn = $this->getConnection(); try { static $script = <<<LUA \t\t\tlocal kClaimed, kAttempts, kData = unpack(KEYS) \t\t\tlocal uuid = unpack(ARGV) \t\t\t-- Unmark the job as claimed \t\t\tredis.call('zRem',kClaimed,uuid) \t\t\tredis.call('hDel',kAttempts,uuid) \t\t\t-- Delete the job data itself \t\t\treturn redis.call('hDel',kData,uuid) LUA; $res = $conn->luaEval($script, array($this->getQueueKey('z-claimed'), $this->getQueueKey('h-attempts'), $this->getQueueKey('h-data'), $uuid), 3); if (!$res) { wfDebugLog('JobQueueRedis', "Could not acknowledge {$this->type} job {$uuid}."); return false; } JobQueue::incrStats('acks', $this->type); } catch (RedisException $e) { $this->throwRedisException($conn, $e); } return true; }
/** * Recycle or destroy any jobs that have been claimed for too long * and release any ready delayed jobs into the queue * * @return int Number of jobs recycled/deleted/undelayed * @throws MWException|JobQueueError */ public function recyclePruneAndUndelayJobs() { $count = 0; // For each job item that can be retried, we need to add it back to the // main queue and remove it from the list of currenty claimed job items. // For those that cannot, they are marked as dead and kept around for // investigation and manual job restoration but are eventually deleted. $conn = $this->getConnection(); try { $now = time(); static $script = <<<LUA \t\t\tlocal kClaimed, kAttempts, kUnclaimed, kData, kAbandoned, kDelayed = unpack(KEYS) \t\t\tlocal released,abandoned,pruned,undelayed = 0,0,0,0 \t\t\t-- Get all non-dead jobs that have an expired claim on them. \t\t\t-- The score for each item is the last claim timestamp (UNIX). \t\t\tlocal staleClaims = redis.call('zRangeByScore',kClaimed,0,ARGV[1]) \t\t\tfor k,id in ipairs(staleClaims) do \t\t\t\tlocal timestamp = redis.call('zScore',kClaimed,id) \t\t\t\tlocal attempts = redis.call('hGet',kAttempts,id) \t\t\t\tif attempts < ARGV[3] then \t\t\t\t\t-- Claim expired and retries left: re-enqueue the job \t\t\t\t\tredis.call('lPush',kUnclaimed,id) \t\t\t\t\tredis.call('hIncrBy',kAttempts,id,1) \t\t\t\t\treleased = released + 1 \t\t\t\telse \t\t\t\t\t-- Claim expired and no retries left: mark the job as dead \t\t\t\t\tredis.call('zAdd',kAbandoned,timestamp,id) \t\t\t\t\tabandoned = abandoned + 1 \t\t\t\tend \t\t\t\tredis.call('zRem',kClaimed,id) \t\t\tend \t\t\t-- Get all of the dead jobs that have been marked as dead for too long. \t\t\t-- The score for each item is the last claim timestamp (UNIX). \t\t\tlocal deadClaims = redis.call('zRangeByScore',kAbandoned,0,ARGV[2]) \t\t\tfor k,id in ipairs(deadClaims) do \t\t\t\t-- Stale and out of retries: remove any traces of the job \t\t\t\tredis.call('zRem',kAbandoned,id) \t\t\t\tredis.call('hDel',kAttempts,id) \t\t\t\tredis.call('hDel',kData,id) \t\t\t\tpruned = pruned + 1 \t\t\tend \t\t\t-- Get the list of ready delayed jobs, sorted by readiness (UNIX timestamp) \t\t\tlocal ids = redis.call('zRangeByScore',kDelayed,0,ARGV[4]) \t\t\t-- Migrate the jobs from the "delayed" set to the "unclaimed" list \t\t\tfor k,id in ipairs(ids) do \t\t\t\tredis.call('lPush',kUnclaimed,id) \t\t\t\tredis.call('zRem',kDelayed,id) \t\t\tend \t\t\tundelayed = #ids \t\t\treturn {released,abandoned,pruned,undelayed} LUA; $res = $conn->luaEval($script, array($this->getQueueKey('z-claimed'), $this->getQueueKey('h-attempts'), $this->getQueueKey('l-unclaimed'), $this->getQueueKey('h-data'), $this->getQueueKey('z-abandoned'), $this->getQueueKey('z-delayed'), $now - $this->claimTTL, $now - self::MAX_AGE_PRUNE, $this->maxTries, $now), 6); if ($res) { list($released, $abandoned, $pruned, $undelayed) = $res; $count += $released + $pruned + $undelayed; JobQueue::incrStats('job-recycle', $this->type, $released); JobQueue::incrStats('job-abandon', $this->type, $abandoned); } } catch (RedisException $e) { $this->throwRedisException($conn, $e); } return $count; }
/** * Recycle or destroy any jobs that have been claimed for too long * * @return integer Number of jobs recycled/deleted * @throws MWException */ public function recycleAndDeleteStaleJobs() { if ($this->claimTTL <= 0) { // sanity throw new MWException("Cannot recycle jobs since acknowledgements are disabled."); } $count = 0; // For each job item that can be retried, we need to add it back to the // main queue and remove it from the list of currenty claimed job items. // For those that cannot, they are marked as dead and kept around for // investigation and manual job restoration but are eventually deleted. $conn = $this->getConnection(); try { $now = time(); static $script = <<<LUA \t\t\tlocal released,abandoned,pruned = 0,0,0 \t\t\t-- Get all non-dead jobs that have an expired claim on them. \t\t\t-- The score for each item is the last claim timestamp (UNIX). \t\t\tlocal staleClaims = redis.call('zRangeByScore',KEYS[1],0,ARGV[1]) \t\t\tfor k,id in ipairs(staleClaims) do \t\t\t\tlocal timestamp = redis.call('zScore',KEYS[1],id) \t\t\t\tlocal attempts = redis.call('hGet',KEYS[2],id) \t\t\t\tif attempts < ARGV[3] then \t\t\t\t\t-- Claim expired and retries left: re-enqueue the job \t\t\t\t\tredis.call('lPush',KEYS[3],id) \t\t\t\t\tredis.call('hIncrBy',KEYS[2],id,1) \t\t\t\t\treleased = released + 1 \t\t\t\telse \t\t\t\t\t-- Claim expired and no retries left: mark the job as dead \t\t\t\t\tredis.call('zAdd',KEYS[5],timestamp,id) \t\t\t\t\tabandoned = abandoned + 1 \t\t\t\tend \t\t\t\tredis.call('zRem',KEYS[1],id) \t\t\tend \t\t\t-- Get all of the dead jobs that have been marked as dead for too long. \t\t\t-- The score for each item is the last claim timestamp (UNIX). \t\t\tlocal deadClaims = redis.call('zRangeByScore',KEYS[5],0,ARGV[2]) \t\t\tfor k,id in ipairs(deadClaims) do \t\t\t\t-- Stale and out of retries: remove any traces of the job \t\t\t\tredis.call('zRem',KEYS[5],id) \t\t\t\tredis.call('hDel',KEYS[2],id) \t\t\t\tredis.call('hDel',KEYS[4],id) \t\t\t\tpruned = pruned + 1 \t\t\tend \t\t\treturn {released,abandoned,pruned} LUA; $res = $conn->luaEval($script, array($this->getQueueKey('z-claimed'), $this->getQueueKey('h-attempts'), $this->getQueueKey('l-unclaimed'), $this->getQueueKey('h-data'), $this->getQueueKey('z-abandoned'), $now - $this->claimTTL, $now - self::MAX_AGE_PRUNE, $this->maxTries), 5); if ($res) { list($released, $abandoned, $pruned) = $res; $count += $released + $pruned; JobQueue::incrStats('job-recycle', $this->type, $released); JobQueue::incrStats('job-abandon', $this->type, $abandoned); } } catch (RedisException $e) { $this->throwRedisException($this->server, $conn, $e); } return $count; }
/** * @see JobQueue::doPop() * @return Job|bool * @throws JobQueueError */ protected function doPop() { $job = false; $conn = $this->getConnection(); try { do { $blob = $this->popAndAcquireBlob($conn); if (!is_string($blob)) { break; // no jobs; nothing to do } JobQueue::incrStats('job-pop', $this->type, 1, $this->wiki); $item = $this->unserialize($blob); if ($item === false) { wfDebugLog('JobQueueRedis', "Could not unserialize {$this->type} job."); continue; } // If $item is invalid, the runner loop recyling will cleanup as needed $job = $this->getJobFromFields($item); // may be false } while (!$job); // job may be false if invalid } catch (RedisException $e) { $this->throwRedisException($conn, $e); } return $job; }
/** * Pop a job off of the queue. * This requires $wgJobClasses to be set for the given job type. * Outside callers should use JobQueueGroup::pop() instead of this function. * * @throws MWException * @return Job|bool Returns false if there are no jobs */ public final function pop() { global $wgJobClasses; $this->assertNotReadOnly(); if ($this->wiki !== wfWikiID()) { throw new MWException("Cannot pop '{$this->type}' job off foreign wiki queue."); } elseif (!isset($wgJobClasses[$this->type])) { // Do not pop jobs if there is no class for the queue type throw new MWException("Unrecognized job type '{$this->type}'."); } $job = $this->doPop(); if (!$job) { $this->aggr->notifyQueueEmpty($this->wiki, $this->type); } // Flag this job as an old duplicate based on its "root" job... try { if ($job && $this->isRootJobOldDuplicate($job)) { JobQueue::incrStats('dupe_pops', $this->type); $job = DuplicateJob::newFromJob($job); // convert to a no-op } } catch (Exception $e) { // don't lose jobs over this } return $job; }