protected function execute(InputInterface $input, OutputInterface $output)
{
$jobId = $input->getArgument('jobId');
$this->init($jobId);
// Ensure that job status is 'terminating'
if ($this->job->getStatus() != Job::STATUS_TERMINATING) {
$this->job->setStatus(Job::STATUS_TERMINATING);
$this->jobMapper->update($this->job);
}
/** @var ExecutorInterface $jobExecutor */
$jobExecutor = $this->getContainer()->get('syrup.job_executor_factory')->create();
// run cleanup
$jobExecutor->cleanup($this->job);
// Update job
$endTime = time();
$duration = $endTime - strtotime($this->job->getStartTime());
$this->job->setStatus(Job::STATUS_TERMINATED);
$this->job->setResult(['message' => 'Job has been terminated']);
$this->job->setEndTime(date('c', $endTime));
$this->job->setDurationSeconds($duration);
$this->jobMapper->update($this->job);
// run post-cleanup
$jobExecutor->postCleanup($this->job);
// DB unlock
$this->lock->unlock();
}
public static function create(Lock $lock, callable $retryCallback, callable $killCallback, callable $successCallback, callable $errorCallback)
{
return function ($command, QueueMessage $message, Job $job) use($lock, $retryCallback, $killCallback, $successCallback, $errorCallback) {
if (!$lock->lock()) {
$retryCallback($message, "DB is locked");
}
$process = new Process($command);
$process->setTimeout(86400);
$process->setIdleTimeout(null);
try {
$process->run();
} catch (ProcessTimedOutException $e) {
$errorCallback($job, "Process timed out");
} catch (RuntimeException $e) {
// process has been signaled or some other error occurred, handled down in code
}
switch ($process->getExitCode()) {
case JobCommand::STATUS_RETRY:
$message->incrementRetries();
if ($message->getRetryCount() > self::MAX_EXECUTION_RETRIES) {
$errorCallback($job, "Retries exceeded");
break;
}
$retryCallback($message);
break;
case 137:
case 143:
// process has been signaled
$killCallback($job);
break;
case JobCommand::STATUS_SUCCESS:
$successCallback();
break;
case JobCommand::STATUS_ERROR:
default:
$errorCallback($job, $process->getOutput());
break;
}
return $process;
};
}
/**
* @covers \Keboola\Syrup\Service\Db\Lock::lock
* @covers \Keboola\Syrup\Service\Db\Lock::isFree
* @covers \Keboola\Syrup\Service\Db\Lock::unlock
*/
public function testLocks()
{
$connectionParams = ['host' => SYRUP_DATABASE_HOST, 'dbname' => SYRUP_DATABASE_NAME, 'user' => SYRUP_DATABASE_USER, 'password' => SYRUP_DATABASE_PASSWORD, 'port' => SYRUP_DATABASE_PORT, 'driver' => 'pdo_mysql', 'charset' => 'utf8'];
$db1 = \Doctrine\DBAL\DriverManager::getConnection($connectionParams);
$db1->exec('SET wait_timeout = 31536000;');
$db2 = \Doctrine\DBAL\DriverManager::getConnection($connectionParams);
$db2->exec('SET wait_timeout = 31536000;');
$lockName = uniqid();
$lock1 = new Lock($db1, $lockName);
$lock2 = new Lock($db2, $lockName);
$this->assertTrue($lock1->lock(), 'Should successfully lock');
$this->assertFalse($lock1->isFree(), 'Should tell lock not free');
$this->assertFalse($lock2->isFree(), 'Should tell lock not free');
$this->assertFalse($lock2->lock(), 'Should fail locking');
$this->assertTrue($lock1->unlock(), 'Should successfully unlock');
$this->assertTrue($lock2->lock(), 'Should successfully lock');
$this->assertFalse($lock2->isFree(), 'Should tell lock not free');
$this->assertFalse($lock1->isFree(), 'Should tell lock not free');
$this->assertFalse($lock1->lock(), 'Should fail locking');
$this->assertTrue($lock2->unlock(), 'Should successfully unlock');
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$jobId = $input->getArgument('jobId');
$this->logger = $this->getContainer()->get('logger');
try {
$this->init($jobId);
} catch (\Exception $e) {
// Initialization error -> job will be requeued
$this->logException('error', $e);
// Don't update job status or result -> error could be related to ES
// Don't unlock DB, error happened either before lock creation or when creating the lock,
// so the DB isn't locked
return self::STATUS_RETRY;
}
if (is_null($jobId)) {
throw new UserException("Missing jobId argument.");
}
if (!$this->lock->lock()) {
return self::STATUS_LOCK;
}
/** @var Connection $checkConn */
$checkConn = null;
/** @var Lock $validationLock */
$validationLock = null;
if (StorageApiLimits::hasParallelLimit($this->storageApiService->getTokenData())) {
try {
$checkConn = $this->getContainer()->get('doctrine.dbal.limit_lock_connection');
$checkConn->exec('SET wait_timeout = 31536000;');
$validationLock = new Lock($checkConn, sprintf('syrup-%s-job-limit-check', $this->job->getProject()['id']));
if (!$validationLock->lock(self::PARALLEL_LIMIT_LOCK_TIMEOUT)) {
throw new \RuntimeException('Could not lock for parallel validation');
}
if ($this->isParallelLimitExceeded()) {
throw new \RuntimeException('Exceeded parallel processing limit');
}
} catch (\RuntimeException $e) {
return self::STATUS_LOCK;
}
}
// check job status
$this->job = $this->jobMapper->get($jobId);
if (!in_array($this->job->getStatus(), [Job::STATUS_WAITING, Job::STATUS_PROCESSING])) {
// job is not waiting or processing
return self::STATUS_LOCK;
}
$startTime = time();
// Update job status to 'processing'
$this->job->setStatus(Job::STATUS_PROCESSING);
$this->job->setStartTime(date('c', $startTime));
$this->job->setEndTime(null);
$this->job->setDurationSeconds(null);
$this->job->setResult(null);
$this->job->setProcess(['host' => gethostname(), 'pid' => getmypid()]);
/** @var ExecutorInterface $jobExecutor */
$jobExecutor = $this->getContainer()->get('syrup.job_executor_factory')->create();
// update the job status after jobExecutor was created, so the signal handler is properly registered
$this->jobMapper->update($this->job);
// Execute job
try {
if (StorageApiLimits::hasParallelLimit($this->storageApiService->getTokenData())) {
$validationLock->unlock();
$checkConn->close();
}
$jobResult = $jobExecutor->execute($this->job);
$jobStatus = Job::STATUS_SUCCESS;
$status = self::STATUS_SUCCESS;
} catch (InitializationException $e) {
// job will be requeued
$exceptionId = $this->logException('error', $e);
$jobResult = ['message' => $e->getMessage(), 'exceptionId' => $exceptionId];
$jobStatus = Job::STATUS_PROCESSING;
$status = self::STATUS_RETRY;
} catch (MaintenanceException $e) {
$jobResult = [];
$jobStatus = Job::STATUS_WAITING;
$status = self::STATUS_LOCK;
} catch (UserException $e) {
$exceptionId = $this->logException('error', $e);
$jobResult = ['message' => $e->getMessage(), 'exceptionId' => $exceptionId];
$jobStatus = Job::STATUS_ERROR;
$this->job->setError(Job::ERROR_USER);
$status = self::STATUS_SUCCESS;
} catch (JobException $e) {
$logLevel = 'error';
if ($e->getStatus() === Job::STATUS_WARNING) {
$logLevel = Job::STATUS_WARNING;
}
$exceptionId = $this->logException($logLevel, $e);
$jobResult = ['message' => $e->getMessage(), 'exceptionId' => $exceptionId];
if ($e->getResult()) {
$jobResult += $e->getResult();
}
$jobStatus = $e->getStatus();
$status = self::STATUS_SUCCESS;
} catch (\Exception $e) {
// make sure that the job is recorded as failed
$jobStatus = Job::STATUS_ERROR;
$jobResult = ['message' => 'Internal error occurred, evaluating details'];
$this->job->setStatus($jobStatus);
$this->job->setResult($jobResult);
$this->job->setError(Job::ERROR_APPLICATION);
// try to log the exception
$exceptionId = $this->logException('critical', $e);
$jobResult = ['message' => 'Internal error occurred, please contact support@keboola.com', 'exceptionId' => $exceptionId];
$status = self::STATUS_ERROR;
}
// Update job with results
$endTime = time();
$createdTime = $this->job->getCreatedTime();
$waitSeconds = is_null($createdTime) ?: $startTime - strtotime($createdTime);
$this->job->setStatus($jobStatus);
$this->job->setResult($jobResult);
$this->job->setEndTime(date('c', $endTime));
$this->job->setDurationSeconds($endTime - $startTime);
$this->job->setWaitSeconds($waitSeconds);
$this->jobMapper->update($this->job);
// postExecution action
try {
$jobExecutor->postExecute($this->job);
//@todo: remove call to this deprecated interface method
if ($jobExecutor instanceof HookExecutorInterface) {
/** @var HookExecutorInterface $jobExecutor */
$jobExecutor->postExecution($this->job);
}
} catch (\Exception $e) {
$this->logException('critical', $e);
}
// DB unlock
$this->lock->unlock();
return $status;
}
