protected function execute(InputInterface $input, OutputInterface $output) { $jobId = $input->getArgument('jobId'); $this->init($jobId); // Ensure that job status is 'terminating' if ($this->job->getStatus() != Job::STATUS_TERMINATING) { $this->job->setStatus(Job::STATUS_TERMINATING); $this->jobMapper->update($this->job); } /** @var ExecutorInterface $jobExecutor */ $jobExecutor = $this->getContainer()->get('syrup.job_executor_factory')->create(); // run cleanup $jobExecutor->cleanup($this->job); // Update job $endTime = time(); $duration = $endTime - strtotime($this->job->getStartTime()); $this->job->setStatus(Job::STATUS_TERMINATED); $this->job->setResult(['message' => 'Job has been terminated']); $this->job->setEndTime(date('c', $endTime)); $this->job->setDurationSeconds($duration); $this->jobMapper->update($this->job); // run post-cleanup $jobExecutor->postCleanup($this->job); // DB unlock $this->lock->unlock(); }
public function execute(Job $job) { $params = $job->getParams(); // simulate long running job for ($i = 0; $i < 120; $i++) { $this->logger->debug(sprintf("Job '%s' is '%s'", $job->getId(), $job->getStatus())); sleep(1); } if (isset($params['returnStatus']) && $params['returnStatus'] == 'error') { throw new ApplicationException("Application error occured"); } }
private function assertJob(Job $job, $resJob) { $this->assertEquals($job->getId(), $resJob['id']); $this->assertEquals($job->getRunId(), $resJob['runId']); $this->assertEquals($job->getLockName(), $resJob['lockName']); $this->assertEquals($job->getProject()['id'], $resJob['project']['id']); $this->assertEquals($job->getProject()['name'], $resJob['project']['name']); $this->assertEquals($job->getToken()['id'], $resJob['token']['id']); $this->assertEquals($job->getToken()['description'], $resJob['token']['description']); $this->assertEquals($job->getToken()['token'], $resJob['token']['token']); $this->assertEquals($job->getComponent(), $resJob['component']); $this->assertEquals($job->getStatus(), $resJob['status']); }
protected function execute(InputInterface $input, OutputInterface $output) { $jobId = $input->getArgument('jobId'); $this->logger = $this->getContainer()->get('logger'); try { $this->init($jobId); } catch (\Exception $e) { // Initialization error -> job will be requeued $this->logException('error', $e); // Don't update job status or result -> error could be related to ES // Don't unlock DB, error happened either before lock creation or when creating the lock, // so the DB isn't locked return self::STATUS_RETRY; } if (is_null($jobId)) { throw new UserException("Missing jobId argument."); } if (!$this->lock->lock()) { return self::STATUS_LOCK; } /** @var Connection $checkConn */ $checkConn = null; /** @var Lock $validationLock */ $validationLock = null; if (StorageApiLimits::hasParallelLimit($this->storageApiService->getTokenData())) { try { $checkConn = $this->getContainer()->get('doctrine.dbal.limit_lock_connection'); $checkConn->exec('SET wait_timeout = 31536000;'); $validationLock = new Lock($checkConn, sprintf('syrup-%s-job-limit-check', $this->job->getProject()['id'])); if (!$validationLock->lock(self::PARALLEL_LIMIT_LOCK_TIMEOUT)) { throw new \RuntimeException('Could not lock for parallel validation'); } if ($this->isParallelLimitExceeded()) { throw new \RuntimeException('Exceeded parallel processing limit'); } } catch (\RuntimeException $e) { return self::STATUS_LOCK; } } // check job status $this->job = $this->jobMapper->get($jobId); if (!in_array($this->job->getStatus(), [Job::STATUS_WAITING, Job::STATUS_PROCESSING])) { // job is not waiting or processing return self::STATUS_LOCK; } $startTime = time(); // Update job status to 'processing' $this->job->setStatus(Job::STATUS_PROCESSING); $this->job->setStartTime(date('c', $startTime)); $this->job->setEndTime(null); $this->job->setDurationSeconds(null); $this->job->setResult(null); $this->job->setProcess(['host' => gethostname(), 'pid' => getmypid()]); /** @var ExecutorInterface $jobExecutor */ $jobExecutor = $this->getContainer()->get('syrup.job_executor_factory')->create(); // update the job status after jobExecutor was created, so the signal handler is properly registered $this->jobMapper->update($this->job); // Execute job try { if (StorageApiLimits::hasParallelLimit($this->storageApiService->getTokenData())) { $validationLock->unlock(); $checkConn->close(); } $jobResult = $jobExecutor->execute($this->job); $jobStatus = Job::STATUS_SUCCESS; $status = self::STATUS_SUCCESS; } catch (InitializationException $e) { // job will be requeued $exceptionId = $this->logException('error', $e); $jobResult = ['message' => $e->getMessage(), 'exceptionId' => $exceptionId]; $jobStatus = Job::STATUS_PROCESSING; $status = self::STATUS_RETRY; } catch (MaintenanceException $e) { $jobResult = []; $jobStatus = Job::STATUS_WAITING; $status = self::STATUS_LOCK; } catch (UserException $e) { $exceptionId = $this->logException('error', $e); $jobResult = ['message' => $e->getMessage(), 'exceptionId' => $exceptionId]; $jobStatus = Job::STATUS_ERROR; $this->job->setError(Job::ERROR_USER); $status = self::STATUS_SUCCESS; } catch (JobException $e) { $logLevel = 'error'; if ($e->getStatus() === Job::STATUS_WARNING) { $logLevel = Job::STATUS_WARNING; } $exceptionId = $this->logException($logLevel, $e); $jobResult = ['message' => $e->getMessage(), 'exceptionId' => $exceptionId]; if ($e->getResult()) { $jobResult += $e->getResult(); } $jobStatus = $e->getStatus(); $status = self::STATUS_SUCCESS; } catch (\Exception $e) { // make sure that the job is recorded as failed $jobStatus = Job::STATUS_ERROR; $jobResult = ['message' => 'Internal error occurred, evaluating details']; $this->job->setStatus($jobStatus); $this->job->setResult($jobResult); $this->job->setError(Job::ERROR_APPLICATION); // try to log the exception $exceptionId = $this->logException('critical', $e); $jobResult = ['message' => 'Internal error occurred, please contact support@keboola.com', 'exceptionId' => $exceptionId]; $status = self::STATUS_ERROR; } // Update job with results $endTime = time(); $createdTime = $this->job->getCreatedTime(); $waitSeconds = is_null($createdTime) ?: $startTime - strtotime($createdTime); $this->job->setStatus($jobStatus); $this->job->setResult($jobResult); $this->job->setEndTime(date('c', $endTime)); $this->job->setDurationSeconds($endTime - $startTime); $this->job->setWaitSeconds($waitSeconds); $this->jobMapper->update($this->job); // postExecution action try { $jobExecutor->postExecute($this->job); //@todo: remove call to this deprecated interface method if ($jobExecutor instanceof HookExecutorInterface) { /** @var HookExecutorInterface $jobExecutor */ $jobExecutor->postExecution($this->job); } } catch (\Exception $e) { $this->logException('critical', $e); } // DB unlock $this->lock->unlock(); return $status; }
public function build(\Keboola\Syrup\Job\Metadata\Job $job) { $params = $job->getRawParams(); $process = $job->getProcess(); $token = $job->getToken(); $project = $job->getProject(); $this->setId($job->getId()); if (!empty($params['config'])) { $this->setConfig($params['config']); } $this->setProjectId($project['id']); $this->setOrchestrationId($params['orchestration']['id']); $this->setToken($token['token']); $this->setTokenId($token['id']); $this->setTokenDesc($token['description']); $this->setTokenOwnerName($project['name']); $this->setStatus($job->getStatus()); $this->setStartTime($job->getStartTime()); $this->setEndTime($job->getEndTime()); $this->setResults($job->getResult()); if (!empty($params['tasks'])) { $this->setTaks($params['tasks']); } $this->setCreatedTime($job->getCreatedTime()); $this->setOrchestrationName($params['orchestration']['name']); $this->setInitializedBy($params['initializedBy']); $this->setInitiatorTokenId($params['initiator']['id']); $this->setInitiatorTokenDesc($params['initiator']['description']); $this->setInitiatorUserAgent($params['initiator']['userAgent']); $this->setNotificationsEmails($params['notificationsEmails']); $this->setRunId($job->getRunId()); $this->setPid($process['pid']); $this->setWorkerAddress($process['host']); }