public function run()
{
while (true) {
$this->scaleAllQueues();
// @todo does detectHungJobs() need a different frequency?
$this->detectHungJobs();
JQWorker::sleep($this->scalerPollingInterval);
}
print "Autoscaler exiting.\n";
}
/**
* OK here's the situation
*
* Our signal processing stuff is supposed to prevent jobs from being hung in a RUNNING state.
*
* The way this works is that essentially anytime a job is INTERRUPTED un-gracefully, we will just pretend that the job never ran.
* Since it's marked as RUNNING in the DB, this means that we need to update the DB to show it as QUEUED (mulligan retry).
*
* Here's where it gets really ugly. If the signal fires after the data has been applied to the Propel instance, but before it's committed to the DB,
* then it's possible for the software model to be differnt from the DB model in a way that makes the graceful signal handler think that the job's status
* was successfully reported to the DB when it wasn't.
*
* So the graceful signal handler needs to be sure to know what the DB thinks, and to re-queue the job if it's checked out as running.
*
* If propel's instance cache is hit for the job, then you risk the problem that the data from the "get job from DB" will be the not-yet-persisted status
* which would foil the check described above from working. This test just ensures that this isn't broken.
*
*/
function testGracefulRetryDuringSplitBrain()
{
$q = $this->jqStore;
// exit our transaction insanity since abort() inside of gracefullyRetryCurrentJob() needs to actually work
$q->abort();
$this->assertEquals(0, $q->count('test'), "Test database not empty; you should re-initialize the test db.");
$mJob = $q->enqueue(new SampleFailJob());
// mark job as running in DB to simulate a job that's running when interrupted
$mJob->markJobStarted();
// will save to DB
$this->assertEquals(0, $q->count('test', JQManagedJob::STATUS_QUEUED));
// mark job as FAILED on the PROPEL object to create split-brain (DB thinks RUNNING, MEMORY thinks FAILED)
$dbJob = JQStoreManagedJobPeer::retrieveByPK($mJob->getJobId());
$mJob->setStatus(JQManagedJob::STATUS_FAILED);
$dbJob->setStatus(JQManagedJob::STATUS_FAILED);
// attempt to gracefully retry; should mark as queued
$w = new JQWorker($q, array('queueName' => 'test', 'exitIfNoJobs' => true, 'silent' => true, 'enableJitter' => false));
$w->gracefullyRetryCurrentJob($mJob);
$this->assertEquals(1, $q->count('test', JQManagedJob::STATUS_QUEUED));
// cleanup
$q->delete($mJob);
}
public static function handleShutdown()
{
$w = new JQWorker(self::$jqStore, array('exitIfNoJobs' => true, 'silent' => true));
$w->start();
}
/**
* @testdox Test Basic JQJobs Processing using JQStore_Propel
*/
function testJQJobs()
{
$q = $this->jqStore;
$this->assertEquals(0, $q->count('test'));
// Add jobs
foreach (range(1, 10) as $i) {
$q->enqueue(new QuietSimpleJob($i));
}
$this->assertEquals(10, $q->count());
$this->assertEquals(10, $q->count('test'));
$this->assertEquals(10, $q->count('test', JQManagedJob::STATUS_QUEUED));
// Start a worker to run the jobs.
$w = new JQWorker($q, array('queueName' => 'test', 'exitIfNoJobs' => true, 'silent' => true, 'enableJitter' => false));
$w->start();
$this->assertEquals(10, $w->jobsProcessed());
$this->assertEquals(0, $q->count('test'));
}
print <<<EXPLAIN
***********************************************************************************************************
This test makes sure that a fatal error during job execution will result in the job being marked failed.
This exercises our shutdown error detection.
This test works if you see output indicating:
- a job starts running
- then a fatal error
- finally "Status change: running => failed"
***********************************************************************************************************
EXPLAIN;
class SampleFatalJob extends SampleLoggingJob
{
function run(JQManagedJob $mJob)
{
// causes a FATAL error
$foo->bar();
}
}
// run a job that will FATAL
$q = new JQStore_Array();
$goodJob = $q->enqueue(new SampleFatalJob());
if ($q->count('test') !== 1) {
throw new Exception("assert failed");
}
SampleJobCounter::reset();
// Start a worker to run the job.
$w = new JQWorker($q, array('queueName' => 'test', 'exitIfNoJobs' => true, 'verbose' => true, 'enableJitter' => false));
$w->start();
/**
* Starts the worker process.
*
* Blocks until the worker exists.
*/
public function start()
{
$this->log("Starting worker process on queue: " . ($this->options['queueName'] === NULL ? '(any)' : $this->options['queueName']));
if ($this->options['enableJitter']) {
$ms = rand(0, 999);
$this->log("Startup jitter: 0.{$ms} seconds...");
JQWorker::sleep(0, $ms * 1000000);
}
if (isset($this->options['adjustPriority'])) {
$this->adjustPriority($this->options['adjustPriority']);
}
$this->okToRun = true;
try {
while ($this->okToRun) {
$this->memCheck();
$this->codeCheck();
$this->currentJob = $this->jqStore->next($this->options['queueName']);
if ($this->currentJob) {
try {
$this->logJobStatus($this->currentJob, "Job checked out.");
// attempt to un-serialize the job
if ($this->currentJob->getJob() instanceof JQJob) {
$this->logJobStatus($this->currentJob, $this->currentJob->description());
$result = $this->currentJob->run($this->currentJob);
} else {
$this->currentJob->markJobFailed("JQManagedJob.job is not a JQJob instance.");
$result = "No JQJob found.";
}
if ($result === NULL) {
$this->logJobStatus($this->currentJob, "Done!");
} else {
$this->logJobStatus($this->currentJob, $result);
}
$this->currentJob = NULL;
} catch (Exception $e) {
if ($this->currentSignalNo === NULL) {
throw $e;
}
// we only handle signals here
// This block helps JQJobs minimize the incidence of jobs getting stuck in the "running" state
// It is designed to catch exceptions from JQJob->run() trying to record a finished ActualJob->run() disposition to JQStore
// although the job might've finished, we couldn't tell JQStore, thus the loop can't be closed
// Therefore, we will gracefullyRetryCurrentJob() so that it can run again another day and close out the loop gracefully
$result = $e->getMessage();
$this->logJobStatus($this->currentJob, "signal raised during job->run()", true);
$this->gracefullyRetryCurrentJob($this->currentJob);
$this->currentJob = NULL;
// now that we've cleaned up, the run loop will gracefully exit
}
$this->jobsProcessed++;
if ($this->options['exitAfterNJobs'] && $this->jobsProcessed >= $this->options['exitAfterNJobs']) {
break;
}
} else {
$this->log("No jobs available.");
if ($this->options['exitIfNoJobs']) {
$this->log("Exiting since exitIfNoJobs=true");
break;
} else {
$s = $this->options['wakeupEvery'];
$ms = 0;
if ($this->options['enableJitter']) {
$ms = rand(0, 999);
}
$this->log("Sleeping for {$s}.{$ms} seconds...");
JQWorker::sleep($s, $ms * 1000000);
}
}
}
} catch (Exception $e) {
if ($this->currentSignalNo === NULL) {
throw $e;
}
// we only handle signals here
// This block helps JQJobs minimize the incidence of jobs getting stuck in the "running" state
// It is designed to catch jobs that have been checked out but not yet run when a signal fires
// Therefore, we will gracefullyRetryCurrentJob() so that it can run again another day and close out the loop gracefully
$result = $e->getMessage();
$this->log("signal raised during run()");
$this->gracefullyRetryCurrentJob($this->currentJob);
exit(self::EXIT_CODE_SIGNAL);
}
$this->log("Stopping worker process on queue: " . ($this->options['queueName'] === NULL ? '(any)' : $this->options['queueName']));
}
function testJobsAutoRetryOnFailure()
{
// create a queuestore
$maxAttempts = 5;
$q = new JQStore_Array();
$mJob = $q->enqueue(new SampleFailJob(array('maxAttempts' => $maxAttempts)));
// Start a worker to run the jobs.
$w = new JQWorker($q, array('queueName' => 'test', 'exitIfNoJobs' => true, 'silent' => true, 'enableJitter' => false));
$w->start();
$this->assertEquals(0, $q->count('test', 'queued'));
$this->assertEquals(1, $q->count('test', 'failed'));
$this->assertEquals($maxAttempts, $mJob->getAttemptNumber());
}
