/**
* @param string $queue
* @return null|QueuedJobDescriptor
*/
protected function getNextJobDescriptorWithoutMutex($queue)
{
$list = QueuedJobDescriptor::get()->filter('JobType', $queue)->sort('ID', 'ASC');
$descriptor = $list->filter('JobStatus', QueuedJob::STATUS_WAIT)->first();
if ($descriptor) {
return $descriptor;
}
return $list->filter('JobStatus', QueuedJob::STATUS_NEW)->where(sprintf('"StartAfter" < \'%s\' OR "StartAfter" IS NULL', SS_DateTime::now()->getValue()))->first();
}
/**
* @inheritdoc
*
* @throws InvalidArgumentException
* @param string
*/
public function unserialize($serialized)
{
$data = unserialize($serialized);
if (!isset($data['descriptor'])) {
throw new InvalidArgumentException('Malformed data');
}
$descriptor = QueuedJobDescriptor::get()->filter('ID', $data['descriptor'])->first();
if (!$descriptor) {
throw new InvalidArgumentException('Descriptor not found');
}
$this->descriptor = $descriptor;
}
/**
*
* Create a new {@link GenerateGoogleSitemapJob} after each CMS write manipulation.
*
* @param {@inheritdoc}
* @return mixed void | null
*/
public function onAfterPublish(&$original)
{
if (!class_exists('AbstractQueuedJob')) {
return;
}
// Get all "running" GenerateGoogleSitemapJob's
$list = QueuedJobDescriptor::get()->filter(array('Implementation' => 'GenerateGoogleSitemapJob', 'JobStatus' => array(QueuedJob::STATUS_INIT, QueuedJob::STATUS_RUN)));
$existingJob = $list ? $list->first() : null;
if ($existingJob && $existingJob->exists()) {
// Do nothing. There's a job for generating the sitemap already running
} else {
$where = '"StartAfter" > \'' . date('Y-m-d H:i:s') . '\'';
$list = QueuedJobDescriptor::get()->where($where);
$list = $list->filter(array('Implementation' => 'GenerateGoogleSitemapJob', 'JobStatus' => array(QueuedJob::STATUS_NEW)));
$list = $list->sort('ID', 'ASC');
if ($list && $list->count()) {
// Execute immediately
$existingJob = $list->first();
$existingJob->StartAfter = date('Y-m-d H:i:s');
$existingJob->write();
return;
}
// If no such a job existing, create a new one for the first time, and run immediately
// But first remove all legacy jobs which might be of the following statuses:
/**
* New (but Start data somehow is less than now)
* Waiting
* Completed
* Paused
* Cancelled
* Broken
*/
$list = QueuedJobDescriptor::get()->filter(array('Implementation' => 'GenerateGoogleSitemapJob'));
if ($list && $list->count()) {
$list->removeAll();
}
$job = new GenerateGoogleSitemapJob();
singleton('QueuedJobService')->queueJob($job);
}
}
/**
* Runs an explicit check on all currently running jobs to make sure their "processed" count is incrementing
* between each run. If it's not, then we need to flag it as paused due to an error.
*
* This typically happens when a PHP fatal error is thrown, which can't be picked up by the error
* handler or exception checker; in this case, we detect these stalled jobs later and fix (try) to
* fix them
*
* @param int $queue The queue to check against
*/
public function checkJobHealth($queue = null)
{
$queue = $queue ?: QueuedJob::QUEUED;
// Select all jobs currently marked as running
$runningJobs = QueuedJobDescriptor::get()->filter(array('JobStatus' => array(QueuedJob::STATUS_RUN, QueuedJob::STATUS_INIT), 'JobType' => $queue));
// If no steps have been processed since the last run, consider it a broken job
// Only check jobs that have been viewed before. LastProcessedCount defaults to -1 on new jobs.
$stalledJobs = $runningJobs->filter('LastProcessedCount:GreaterThanOrEqual', 0)->where('"StepsProcessed" = "LastProcessedCount"');
foreach ($stalledJobs as $stalledJob) {
$this->restartStalledJob($stalledJob);
}
// now, find those that need to be marked before the next check
// foreach job, mark it as having been incremented
foreach ($runningJobs as $job) {
$job->LastProcessedCount = $job->StepsProcessed;
$job->write();
}
// finally, find the list of broken jobs and send an email if there's some found
$brokenJobs = QueuedJobDescriptor::get()->filter('JobStatus', QueuedJob::STATUS_BROKEN);
if ($brokenJobs && $brokenJobs->count()) {
SS_Log::log(array('errno' => 0, 'errstr' => 'Broken jobs were found in the job queue', 'errfile' => __FILE__, 'errline' => __LINE__, 'errcontext' => array()), SS_Log::ERR);
}
}
/**
* Runs an explicit check on all currently running jobs to make sure their "processed" count is incrementing
* between each run. If it's not, then we need to flag it as paused due to an error.
*
* This typically happens when a PHP fatal error is thrown, which can't be picked up by the error
* handler or exception checker; in this case, we detect these stalled jobs later and fix (try) to
* fix them
*/
public function checkJobHealth()
{
// first off, we want to find jobs that haven't changed since they were last checked (assuming they've actually
// processed a few steps...)
$filter = singleton('QJUtils')->dbQuote(array('JobStatus =' => QueuedJob::STATUS_RUN, 'StepsProcessed >' => 0));
$filter = $filter . ' AND "StepsProcessed"="LastProcessedCount"';
$stalledJobs = QueuedJobDescriptor::get()->filter(array('JobStatus' => QueuedJob::STATUS_RUN, 'StepsProcessed:GreaterThan' => 0));
$stalledJobs = $stalledJobs->where('"StepsProcessed"="LastProcessedCount"');
if ($stalledJobs) {
foreach ($stalledJobs as $stalledJob) {
if ($stalledJob->ResumeCount <= self::$stall_threshold) {
$stalledJob->ResumeCount++;
$stalledJob->pause();
$stalledJob->resume();
$msg = sprintf(_t('QueuedJobs.STALLED_JOB_MSG', 'A job named %s appears to have stalled. It will be stopped and restarted, please login to make sure it has continued'), $stalledJob->JobTitle);
} else {
$stalledJob->pause();
$msg = sprintf(_t('QueuedJobs.STALLED_JOB_MSG', 'A job named %s appears to have stalled. It has been paused, please login to check it'), $stalledJob->JobTitle);
}
singleton('QJUtils')->log($msg);
$mail = new Email(Email::getAdminEmail(), Email::getAdminEmail(), _t('QueuedJobs.STALLED_JOB', 'Stalled job'), $msg);
$mail->send();
}
}
// now, find those that need to be marked before the next check
$runningJobs = QueuedJobDescriptor::get()->filter('JobStatus', QueuedJob::STATUS_RUN);
if ($runningJobs) {
// foreach job, mark it as having been incremented
foreach ($runningJobs as $job) {
$job->LastProcessedCount = $job->StepsProcessed;
$job->write();
}
}
// finally, find the list of broken jobs and send an email if there's some found
$min = date('i');
if ($min == '42' || true) {
$brokenJobs = QueuedJobDescriptor::get()->filter('JobStatus', QueuedJob::STATUS_BROKEN);
if ($brokenJobs && $brokenJobs->count()) {
SS_Log::log(array('errno' => 0, 'errstr' => 'Broken jobs were found in the job queue', 'errfile' => __FILE__, 'errline' => __LINE__, 'errcontext' => ''), SS_Log::ERR);
}
}
}
/**
* Verify that broken jobs are correctly verified for health and restarted as necessary
*
* Order of checkJobHealth() and getNextPendingJob() is important
*
* Execution of this job is broken into several "loops", each of which represents one invocation
* of ProcessJobQueueTask
*/
public function testJobHealthCheck()
{
// Create a job and add it to the queue
$svc = $this->getService();
$job = new TestQueuedJob(QueuedJob::IMMEDIATE);
$job->firstJob = true;
$id = $svc->queueJob($job);
$descriptor = QueuedJobDescriptor::get()->byID($id);
// Verify initial state is new and LastProcessedCount is not marked yet
$this->assertEquals(QueuedJob::STATUS_NEW, $descriptor->JobStatus);
$this->assertEquals(0, $descriptor->StepsProcessed);
$this->assertEquals(-1, $descriptor->LastProcessedCount);
$this->assertEquals(0, $descriptor->ResumeCounts);
// Loop 1 - Pick up new job and attempt to run it
// Job health should not attempt to cleanup unstarted jobs
$svc->checkJobHealth();
$nextJob = $svc->getNextPendingJob(QueuedJob::IMMEDIATE);
// Ensure that this is the next job ready to go
$descriptor = QueuedJobDescriptor::get()->byID($id);
$this->assertEquals($nextJob->ID, $descriptor->ID);
$this->assertEquals(QueuedJob::STATUS_NEW, $descriptor->JobStatus);
$this->assertEquals(0, $descriptor->StepsProcessed);
$this->assertEquals(-1, $descriptor->LastProcessedCount);
$this->assertEquals(0, $descriptor->ResumeCounts);
// Run 1 - Start the job (no work is done)
$descriptor->JobStatus = QueuedJob::STATUS_INIT;
$descriptor->write();
// Assume that something bad happens at this point, the process dies during execution, and
// the task is re-initiated somewhere down the track
// Loop 2 - Detect broken job, and mark it for future checking.
$svc->checkJobHealth();
$nextJob = $svc->getNextPendingJob(QueuedJob::IMMEDIATE);
// Note that we don't immediately try to restart it until StepsProcessed = LastProcessedCount
$descriptor = QueuedJobDescriptor::get()->byID($id);
$this->assertFalse($nextJob);
// Don't run it this round please!
$this->assertEquals(QueuedJob::STATUS_INIT, $descriptor->JobStatus);
$this->assertEquals(0, $descriptor->StepsProcessed);
$this->assertEquals(0, $descriptor->LastProcessedCount);
$this->assertEquals(0, $descriptor->ResumeCounts);
// Loop 3 - We've previously marked this job as broken, so restart it this round
// If no more work has been done on the job at this point, assume that we are able to
// restart it
$svc->checkJobHealth();
$nextJob = $svc->getNextPendingJob(QueuedJob::IMMEDIATE);
// This job is resumed and exeuction is attempted this round
$descriptor = QueuedJobDescriptor::get()->byID($id);
$this->assertEquals($nextJob->ID, $descriptor->ID);
$this->assertEquals(QueuedJob::STATUS_WAIT, $descriptor->JobStatus);
$this->assertEquals(0, $descriptor->StepsProcessed);
$this->assertEquals(0, $descriptor->LastProcessedCount);
$this->assertEquals(1, $descriptor->ResumeCounts);
// Run 2 - First restart (work is done)
$descriptor->JobStatus = QueuedJob::STATUS_RUN;
$descriptor->StepsProcessed++;
// Essentially delays the next restart by 1 loop
$descriptor->write();
// Once again, at this point, assume the job fails and crashes
// Loop 4 - Assuming a job has LastProcessedCount < StepsProcessed we are in the same
// situation as step 2.
// Because the last time the loop ran, StepsProcessed was incremented,
// this indicates that it's likely that another task could be working on this job, so
// don't run this.
$svc->checkJobHealth();
$nextJob = $svc->getNextPendingJob(QueuedJob::IMMEDIATE);
$descriptor = QueuedJobDescriptor::get()->byID($id);
$this->assertFalse($nextJob);
// Don't run jobs we aren't sure should be restarted
$this->assertEquals(QueuedJob::STATUS_RUN, $descriptor->JobStatus);
$this->assertEquals(1, $descriptor->StepsProcessed);
$this->assertEquals(1, $descriptor->LastProcessedCount);
$this->assertEquals(1, $descriptor->ResumeCounts);
// Loop 5 - Job is again found to not have been restarted since last iteration, so perform second
// restart. The job should be attempted to run this loop
$svc->checkJobHealth();
$nextJob = $svc->getNextPendingJob(QueuedJob::IMMEDIATE);
// This job is resumed and exeuction is attempted this round
$descriptor = QueuedJobDescriptor::get()->byID($id);
$this->assertEquals($nextJob->ID, $descriptor->ID);
$this->assertEquals(QueuedJob::STATUS_WAIT, $descriptor->JobStatus);
$this->assertEquals(1, $descriptor->StepsProcessed);
$this->assertEquals(1, $descriptor->LastProcessedCount);
$this->assertEquals(2, $descriptor->ResumeCounts);
// Run 3 - Second and last restart (no work is done)
$descriptor->JobStatus = QueuedJob::STATUS_RUN;
$descriptor->write();
// Loop 6 - As no progress has been made since loop 3, we can mark this as dead
$svc->checkJobHealth();
$nextJob = $svc->getNextPendingJob(QueuedJob::IMMEDIATE);
// Since no StepsProcessed has been done, don't wait another loop to mark this as dead
$descriptor = QueuedJobDescriptor::get()->byID($id);
$this->assertEquals(QueuedJob::STATUS_PAUSED, $descriptor->JobStatus);
$this->assertEmpty($nextJob);
}
/**
* If the queued jobs module is installed, queue up the first job for 9am tomorrow morning
* (by default).
*/
public function requireDefaultRecords()
{
if (class_exists("ContentReviewNotificationJob")) {
// Ensure there is not already a job queued
if (QueuedJobDescriptor::get()->filter("Implementation", "ContentReviewNotificationJob")->first()) {
return;
}
$nextRun = new ContentReviewNotificationJob();
$runHour = Config::inst()->get("ContentReviewNotificationJob", "first_run_hour");
$firstRunTime = date("Y-m-d H:i:s", mktime($runHour, 0, 0, date("m"), date("d") + 1, date("y")));
singleton("QueuedJobService")->queueJob($nextRun, $firstRunTime);
DB::alteration_message(sprintf("Added ContentReviewNotificationJob to run at %s", $firstRunTime));
}
}
