public function run() { while (true) { $this->scaleAllQueues(); // @todo does detectHungJobs() need a different frequency? $this->detectHungJobs(); JQWorker::sleep($this->scalerPollingInterval); } print "Autoscaler exiting.\n"; }
/** * OK here's the situation * * Our signal processing stuff is supposed to prevent jobs from being hung in a RUNNING state. * * The way this works is that essentially anytime a job is INTERRUPTED un-gracefully, we will just pretend that the job never ran. * Since it's marked as RUNNING in the DB, this means that we need to update the DB to show it as QUEUED (mulligan retry). * * Here's where it gets really ugly. If the signal fires after the data has been applied to the Propel instance, but before it's committed to the DB, * then it's possible for the software model to be differnt from the DB model in a way that makes the graceful signal handler think that the job's status * was successfully reported to the DB when it wasn't. * * So the graceful signal handler needs to be sure to know what the DB thinks, and to re-queue the job if it's checked out as running. * * If propel's instance cache is hit for the job, then you risk the problem that the data from the "get job from DB" will be the not-yet-persisted status * which would foil the check described above from working. This test just ensures that this isn't broken. * */ function testGracefulRetryDuringSplitBrain() { $q = $this->jqStore; // exit our transaction insanity since abort() inside of gracefullyRetryCurrentJob() needs to actually work $q->abort(); $this->assertEquals(0, $q->count('test'), "Test database not empty; you should re-initialize the test db."); $mJob = $q->enqueue(new SampleFailJob()); // mark job as running in DB to simulate a job that's running when interrupted $mJob->markJobStarted(); // will save to DB $this->assertEquals(0, $q->count('test', JQManagedJob::STATUS_QUEUED)); // mark job as FAILED on the PROPEL object to create split-brain (DB thinks RUNNING, MEMORY thinks FAILED) $dbJob = JQStoreManagedJobPeer::retrieveByPK($mJob->getJobId()); $mJob->setStatus(JQManagedJob::STATUS_FAILED); $dbJob->setStatus(JQManagedJob::STATUS_FAILED); // attempt to gracefully retry; should mark as queued $w = new JQWorker($q, array('queueName' => 'test', 'exitIfNoJobs' => true, 'silent' => true, 'enableJitter' => false)); $w->gracefullyRetryCurrentJob($mJob); $this->assertEquals(1, $q->count('test', JQManagedJob::STATUS_QUEUED)); // cleanup $q->delete($mJob); }
public static function handleShutdown() { $w = new JQWorker(self::$jqStore, array('exitIfNoJobs' => true, 'silent' => true)); $w->start(); }
/** * @testdox Test Basic JQJobs Processing using JQStore_Propel */ function testJQJobs() { $q = $this->jqStore; $this->assertEquals(0, $q->count('test')); // Add jobs foreach (range(1, 10) as $i) { $q->enqueue(new QuietSimpleJob($i)); } $this->assertEquals(10, $q->count()); $this->assertEquals(10, $q->count('test')); $this->assertEquals(10, $q->count('test', JQManagedJob::STATUS_QUEUED)); // Start a worker to run the jobs. $w = new JQWorker($q, array('queueName' => 'test', 'exitIfNoJobs' => true, 'silent' => true, 'enableJitter' => false)); $w->start(); $this->assertEquals(10, $w->jobsProcessed()); $this->assertEquals(0, $q->count('test')); }
print <<<EXPLAIN *********************************************************************************************************** This test makes sure that a fatal error during job execution will result in the job being marked failed. This exercises our shutdown error detection. This test works if you see output indicating: - a job starts running - then a fatal error - finally "Status change: running => failed" *********************************************************************************************************** EXPLAIN; class SampleFatalJob extends SampleLoggingJob { function run(JQManagedJob $mJob) { // causes a FATAL error $foo->bar(); } } // run a job that will FATAL $q = new JQStore_Array(); $goodJob = $q->enqueue(new SampleFatalJob()); if ($q->count('test') !== 1) { throw new Exception("assert failed"); } SampleJobCounter::reset(); // Start a worker to run the job. $w = new JQWorker($q, array('queueName' => 'test', 'exitIfNoJobs' => true, 'verbose' => true, 'enableJitter' => false)); $w->start();
/** * Starts the worker process. * * Blocks until the worker exists. */ public function start() { $this->log("Starting worker process on queue: " . ($this->options['queueName'] === NULL ? '(any)' : $this->options['queueName'])); if ($this->options['enableJitter']) { $ms = rand(0, 999); $this->log("Startup jitter: 0.{$ms} seconds..."); JQWorker::sleep(0, $ms * 1000000); } if (isset($this->options['adjustPriority'])) { $this->adjustPriority($this->options['adjustPriority']); } $this->okToRun = true; try { while ($this->okToRun) { $this->memCheck(); $this->codeCheck(); $this->currentJob = $this->jqStore->next($this->options['queueName']); if ($this->currentJob) { try { $this->logJobStatus($this->currentJob, "Job checked out."); // attempt to un-serialize the job if ($this->currentJob->getJob() instanceof JQJob) { $this->logJobStatus($this->currentJob, $this->currentJob->description()); $result = $this->currentJob->run($this->currentJob); } else { $this->currentJob->markJobFailed("JQManagedJob.job is not a JQJob instance."); $result = "No JQJob found."; } if ($result === NULL) { $this->logJobStatus($this->currentJob, "Done!"); } else { $this->logJobStatus($this->currentJob, $result); } $this->currentJob = NULL; } catch (Exception $e) { if ($this->currentSignalNo === NULL) { throw $e; } // we only handle signals here // This block helps JQJobs minimize the incidence of jobs getting stuck in the "running" state // It is designed to catch exceptions from JQJob->run() trying to record a finished ActualJob->run() disposition to JQStore // although the job might've finished, we couldn't tell JQStore, thus the loop can't be closed // Therefore, we will gracefullyRetryCurrentJob() so that it can run again another day and close out the loop gracefully $result = $e->getMessage(); $this->logJobStatus($this->currentJob, "signal raised during job->run()", true); $this->gracefullyRetryCurrentJob($this->currentJob); $this->currentJob = NULL; // now that we've cleaned up, the run loop will gracefully exit } $this->jobsProcessed++; if ($this->options['exitAfterNJobs'] && $this->jobsProcessed >= $this->options['exitAfterNJobs']) { break; } } else { $this->log("No jobs available."); if ($this->options['exitIfNoJobs']) { $this->log("Exiting since exitIfNoJobs=true"); break; } else { $s = $this->options['wakeupEvery']; $ms = 0; if ($this->options['enableJitter']) { $ms = rand(0, 999); } $this->log("Sleeping for {$s}.{$ms} seconds..."); JQWorker::sleep($s, $ms * 1000000); } } } } catch (Exception $e) { if ($this->currentSignalNo === NULL) { throw $e; } // we only handle signals here // This block helps JQJobs minimize the incidence of jobs getting stuck in the "running" state // It is designed to catch jobs that have been checked out but not yet run when a signal fires // Therefore, we will gracefullyRetryCurrentJob() so that it can run again another day and close out the loop gracefully $result = $e->getMessage(); $this->log("signal raised during run()"); $this->gracefullyRetryCurrentJob($this->currentJob); exit(self::EXIT_CODE_SIGNAL); } $this->log("Stopping worker process on queue: " . ($this->options['queueName'] === NULL ? '(any)' : $this->options['queueName'])); }
function testJobsAutoRetryOnFailure() { // create a queuestore $maxAttempts = 5; $q = new JQStore_Array(); $mJob = $q->enqueue(new SampleFailJob(array('maxAttempts' => $maxAttempts))); // Start a worker to run the jobs. $w = new JQWorker($q, array('queueName' => 'test', 'exitIfNoJobs' => true, 'silent' => true, 'enableJitter' => false)); $w->start(); $this->assertEquals(0, $q->count('test', 'queued')); $this->assertEquals(1, $q->count('test', 'failed')); $this->assertEquals($maxAttempts, $mJob->getAttemptNumber()); }