/**
* Pop a job off of the queue.
* This requires $wgJobClasses to be set for the given job type.
* Outside callers should use JobQueueGroup::pop() instead of this function.
*
* @throws MWException
* @return Job|bool Returns false if there are no jobs
*/
public final function pop()
{
global $wgJobClasses;
if ($this->wiki !== wfWikiID()) {
throw new MWException("Cannot pop '{$this->type}' job off foreign wiki queue.");
} elseif (!isset($wgJobClasses[$this->type])) {
// Do not pop jobs if there is no class for the queue type
throw new MWException("Unrecognized job type '{$this->type}'.");
}
wfProfileIn(__METHOD__);
$job = $this->doPop();
wfProfileOut(__METHOD__);
// Flag this job as an old duplicate based on its "root" job...
try {
if ($job && $this->isRootJobOldDuplicate($job)) {
JobQueue::incrStats('job-pop-duplicate', $this->type, 1, $this->wiki);
$job = DuplicateJob::newFromJob($job);
// convert to a no-op
}
} catch (MWException $e) {
// don't lose jobs over this
}
return $job;
}
/**
* Recycle or destroy any jobs that have been claimed for too long
*
* @return int Number of jobs recycled/deleted
*/
public function recycleAndDeleteStaleJobs()
{
$now = time();
$count = 0;
// affected rows
$dbw = $this->getMasterDB();
try {
if (!$dbw->lock("jobqueue-recycle-{$this->type}", __METHOD__, 1)) {
return $count;
// already in progress
}
// Remove claims on jobs acquired for too long if enabled...
if ($this->claimTTL > 0) {
$claimCutoff = $dbw->timestamp($now - $this->claimTTL);
// Get the IDs of jobs that have be claimed but not finished after too long.
// These jobs can be recycled into the queue by expiring the claim. Selecting
// the IDs first means that the UPDATE can be done by primary key (less deadlocks).
$res = $dbw->select('job', 'job_id', array('job_cmd' => $this->type, "job_token != {$dbw->addQuotes('')}", "job_token_timestamp < {$dbw->addQuotes($claimCutoff)}", "job_attempts < {$dbw->addQuotes($this->maxTries)}"), __METHOD__);
$ids = array_map(function ($o) {
return $o->job_id;
}, iterator_to_array($res));
if (count($ids)) {
// Reset job_token for these jobs so that other runners will pick them up.
// Set the timestamp to the current time, as it is useful to now that the job
// was already tried before (the timestamp becomes the "released" time).
$dbw->update('job', array('job_token' => '', 'job_token_timestamp' => $dbw->timestamp($now)), array('job_id' => $ids), __METHOD__);
$affected = $dbw->affectedRows();
$count += $affected;
JobQueue::incrStats('recycles', $this->type, $affected);
$this->aggr->notifyQueueNonEmpty($this->wiki, $this->type);
}
}
// Just destroy any stale jobs...
$pruneCutoff = $dbw->timestamp($now - self::MAX_AGE_PRUNE);
$conds = array('job_cmd' => $this->type, "job_token != {$dbw->addQuotes('')}", "job_token_timestamp < {$dbw->addQuotes($pruneCutoff)}");
if ($this->claimTTL > 0) {
// only prune jobs attempted too many times...
$conds[] = "job_attempts >= {$dbw->addQuotes($this->maxTries)}";
}
// Get the IDs of jobs that are considered stale and should be removed. Selecting
// the IDs first means that the UPDATE can be done by primary key (less deadlocks).
$res = $dbw->select('job', 'job_id', $conds, __METHOD__);
$ids = array_map(function ($o) {
return $o->job_id;
}, iterator_to_array($res));
if (count($ids)) {
$dbw->delete('job', array('job_id' => $ids), __METHOD__);
$affected = $dbw->affectedRows();
$count += $affected;
JobQueue::incrStats('abandons', $this->type, $affected);
}
$dbw->unlock("jobqueue-recycle-{$this->type}", __METHOD__);
} catch (DBError $e) {
$this->throwDBException($e);
}
return $count;
}
/**
* @see JobQueue::doAck()
* @param Job $job
* @return Job|bool
* @throws UnexpectedValueException
* @throws JobQueueError
*/
protected function doAck(Job $job)
{
if (!isset($job->metadata['uuid'])) {
throw new UnexpectedValueException("Job of type '{$job->getType()}' has no UUID.");
}
$uuid = $job->metadata['uuid'];
$conn = $this->getConnection();
try {
static $script = <<<LUA
\t\t\tlocal kClaimed, kAttempts, kData = unpack(KEYS)
\t\t\tlocal uuid = unpack(ARGV)
\t\t\t-- Unmark the job as claimed
\t\t\tredis.call('zRem',kClaimed,uuid)
\t\t\tredis.call('hDel',kAttempts,uuid)
\t\t\t-- Delete the job data itself
\t\t\treturn redis.call('hDel',kData,uuid)
LUA;
$res = $conn->luaEval($script, array($this->getQueueKey('z-claimed'), $this->getQueueKey('h-attempts'), $this->getQueueKey('h-data'), $uuid), 3);
if (!$res) {
wfDebugLog('JobQueueRedis', "Could not acknowledge {$this->type} job {$uuid}.");
return false;
}
JobQueue::incrStats('acks', $this->type);
} catch (RedisException $e) {
$this->throwRedisException($conn, $e);
}
return true;
}
/**
* Recycle or destroy any jobs that have been claimed for too long
* and release any ready delayed jobs into the queue
*
* @return int Number of jobs recycled/deleted/undelayed
* @throws MWException|JobQueueError
*/
public function recyclePruneAndUndelayJobs()
{
$count = 0;
// For each job item that can be retried, we need to add it back to the
// main queue and remove it from the list of currenty claimed job items.
// For those that cannot, they are marked as dead and kept around for
// investigation and manual job restoration but are eventually deleted.
$conn = $this->getConnection();
try {
$now = time();
static $script = <<<LUA
\t\t\tlocal kClaimed, kAttempts, kUnclaimed, kData, kAbandoned, kDelayed = unpack(KEYS)
\t\t\tlocal released,abandoned,pruned,undelayed = 0,0,0,0
\t\t\t-- Get all non-dead jobs that have an expired claim on them.
\t\t\t-- The score for each item is the last claim timestamp (UNIX).
\t\t\tlocal staleClaims = redis.call('zRangeByScore',kClaimed,0,ARGV[1])
\t\t\tfor k,id in ipairs(staleClaims) do
\t\t\t\tlocal timestamp = redis.call('zScore',kClaimed,id)
\t\t\t\tlocal attempts = redis.call('hGet',kAttempts,id)
\t\t\t\tif attempts < ARGV[3] then
\t\t\t\t\t-- Claim expired and retries left: re-enqueue the job
\t\t\t\t\tredis.call('lPush',kUnclaimed,id)
\t\t\t\t\tredis.call('hIncrBy',kAttempts,id,1)
\t\t\t\t\treleased = released + 1
\t\t\t\telse
\t\t\t\t\t-- Claim expired and no retries left: mark the job as dead
\t\t\t\t\tredis.call('zAdd',kAbandoned,timestamp,id)
\t\t\t\t\tabandoned = abandoned + 1
\t\t\t\tend
\t\t\t\tredis.call('zRem',kClaimed,id)
\t\t\tend
\t\t\t-- Get all of the dead jobs that have been marked as dead for too long.
\t\t\t-- The score for each item is the last claim timestamp (UNIX).
\t\t\tlocal deadClaims = redis.call('zRangeByScore',kAbandoned,0,ARGV[2])
\t\t\tfor k,id in ipairs(deadClaims) do
\t\t\t\t-- Stale and out of retries: remove any traces of the job
\t\t\t\tredis.call('zRem',kAbandoned,id)
\t\t\t\tredis.call('hDel',kAttempts,id)
\t\t\t\tredis.call('hDel',kData,id)
\t\t\t\tpruned = pruned + 1
\t\t\tend
\t\t\t-- Get the list of ready delayed jobs, sorted by readiness (UNIX timestamp)
\t\t\tlocal ids = redis.call('zRangeByScore',kDelayed,0,ARGV[4])
\t\t\t-- Migrate the jobs from the "delayed" set to the "unclaimed" list
\t\t\tfor k,id in ipairs(ids) do
\t\t\t\tredis.call('lPush',kUnclaimed,id)
\t\t\t\tredis.call('zRem',kDelayed,id)
\t\t\tend
\t\t\tundelayed = #ids
\t\t\treturn {released,abandoned,pruned,undelayed}
LUA;
$res = $conn->luaEval($script, array($this->getQueueKey('z-claimed'), $this->getQueueKey('h-attempts'), $this->getQueueKey('l-unclaimed'), $this->getQueueKey('h-data'), $this->getQueueKey('z-abandoned'), $this->getQueueKey('z-delayed'), $now - $this->claimTTL, $now - self::MAX_AGE_PRUNE, $this->maxTries, $now), 6);
if ($res) {
list($released, $abandoned, $pruned, $undelayed) = $res;
$count += $released + $pruned + $undelayed;
JobQueue::incrStats('job-recycle', $this->type, $released);
JobQueue::incrStats('job-abandon', $this->type, $abandoned);
}
} catch (RedisException $e) {
$this->throwRedisException($conn, $e);
}
return $count;
}
/**
* Recycle or destroy any jobs that have been claimed for too long
*
* @return integer Number of jobs recycled/deleted
* @throws MWException
*/
public function recycleAndDeleteStaleJobs()
{
if ($this->claimTTL <= 0) {
// sanity
throw new MWException("Cannot recycle jobs since acknowledgements are disabled.");
}
$count = 0;
// For each job item that can be retried, we need to add it back to the
// main queue and remove it from the list of currenty claimed job items.
// For those that cannot, they are marked as dead and kept around for
// investigation and manual job restoration but are eventually deleted.
$conn = $this->getConnection();
try {
$now = time();
static $script = <<<LUA
\t\t\tlocal released,abandoned,pruned = 0,0,0
\t\t\t-- Get all non-dead jobs that have an expired claim on them.
\t\t\t-- The score for each item is the last claim timestamp (UNIX).
\t\t\tlocal staleClaims = redis.call('zRangeByScore',KEYS[1],0,ARGV[1])
\t\t\tfor k,id in ipairs(staleClaims) do
\t\t\t\tlocal timestamp = redis.call('zScore',KEYS[1],id)
\t\t\t\tlocal attempts = redis.call('hGet',KEYS[2],id)
\t\t\t\tif attempts < ARGV[3] then
\t\t\t\t\t-- Claim expired and retries left: re-enqueue the job
\t\t\t\t\tredis.call('lPush',KEYS[3],id)
\t\t\t\t\tredis.call('hIncrBy',KEYS[2],id,1)
\t\t\t\t\treleased = released + 1
\t\t\t\telse
\t\t\t\t\t-- Claim expired and no retries left: mark the job as dead
\t\t\t\t\tredis.call('zAdd',KEYS[5],timestamp,id)
\t\t\t\t\tabandoned = abandoned + 1
\t\t\t\tend
\t\t\t\tredis.call('zRem',KEYS[1],id)
\t\t\tend
\t\t\t-- Get all of the dead jobs that have been marked as dead for too long.
\t\t\t-- The score for each item is the last claim timestamp (UNIX).
\t\t\tlocal deadClaims = redis.call('zRangeByScore',KEYS[5],0,ARGV[2])
\t\t\tfor k,id in ipairs(deadClaims) do
\t\t\t\t-- Stale and out of retries: remove any traces of the job
\t\t\t\tredis.call('zRem',KEYS[5],id)
\t\t\t\tredis.call('hDel',KEYS[2],id)
\t\t\t\tredis.call('hDel',KEYS[4],id)
\t\t\t\tpruned = pruned + 1
\t\t\tend
\t\t\treturn {released,abandoned,pruned}
LUA;
$res = $conn->luaEval($script, array($this->getQueueKey('z-claimed'), $this->getQueueKey('h-attempts'), $this->getQueueKey('l-unclaimed'), $this->getQueueKey('h-data'), $this->getQueueKey('z-abandoned'), $now - $this->claimTTL, $now - self::MAX_AGE_PRUNE, $this->maxTries), 5);
if ($res) {
list($released, $abandoned, $pruned) = $res;
$count += $released + $pruned;
JobQueue::incrStats('job-recycle', $this->type, $released);
JobQueue::incrStats('job-abandon', $this->type, $abandoned);
}
} catch (RedisException $e) {
$this->throwRedisException($this->server, $conn, $e);
}
return $count;
}
/**
* @see JobQueue::doPop()
* @return Job|bool
* @throws JobQueueError
*/
protected function doPop()
{
$job = false;
$conn = $this->getConnection();
try {
do {
$blob = $this->popAndAcquireBlob($conn);
if (!is_string($blob)) {
break;
// no jobs; nothing to do
}
JobQueue::incrStats('job-pop', $this->type, 1, $this->wiki);
$item = $this->unserialize($blob);
if ($item === false) {
wfDebugLog('JobQueueRedis', "Could not unserialize {$this->type} job.");
continue;
}
// If $item is invalid, the runner loop recyling will cleanup as needed
$job = $this->getJobFromFields($item);
// may be false
} while (!$job);
// job may be false if invalid
} catch (RedisException $e) {
$this->throwRedisException($conn, $e);
}
return $job;
}
/**
* Pop a job off of the queue.
* This requires $wgJobClasses to be set for the given job type.
* Outside callers should use JobQueueGroup::pop() instead of this function.
*
* @throws MWException
* @return Job|bool Returns false if there are no jobs
*/
public final function pop()
{
global $wgJobClasses;
$this->assertNotReadOnly();
if ($this->wiki !== wfWikiID()) {
throw new MWException("Cannot pop '{$this->type}' job off foreign wiki queue.");
} elseif (!isset($wgJobClasses[$this->type])) {
// Do not pop jobs if there is no class for the queue type
throw new MWException("Unrecognized job type '{$this->type}'.");
}
$job = $this->doPop();
if (!$job) {
$this->aggr->notifyQueueEmpty($this->wiki, $this->type);
}
// Flag this job as an old duplicate based on its "root" job...
try {
if ($job && $this->isRootJobOldDuplicate($job)) {
JobQueue::incrStats('dupe_pops', $this->type);
$job = DuplicateJob::newFromJob($job);
// convert to a no-op
}
} catch (Exception $e) {
// don't lose jobs over this
}
return $job;
}
