/**
* Update last executed job in orchestration
*
* If job failed, send error notification
*
* @param SyrupJob $job
*/
public final function postExecute(SyrupJob $job)
{
$this->validateJob($job);
// not executed or different job - do nothing
if (!$this->job || !$this->orchestration) {
return;
}
if ($this->job->getId() !== $job->getId()) {
return;
}
$this->job = new Job();
$this->job->build($job);
$this->orchestrationManager->updateLastExecutedJob($this->orchestration, $this->job);
if ($job->getStatus() === SyrupJob::STATUS_ERROR) {
$this->mailer->sendJobErrorMessage($this->job, $this->orchestration, $this->jobEsManager, $this->componentsList);
}
if ($job->getStatus() === SyrupJob::STATUS_WARNING) {
$this->mailer->sendJobWarningMessage($this->job, $this->orchestration, $this->jobEsManager, $this->componentsList);
}
}
public function execute(SyrupJob $job)
{
parent::execute($job);
$command = $job->getCommand();
try {
// run jobu
if ($command === 'run') {
$esJob = new Job();
$esJob->build($job);
if (!$this->usePhases($esJob)) {
$executor = new Run($this->orchestrationManager, $this->jobManagerFactory, $this->mailer, $this->encryptor, $this->logger, $this->queueFactory);
} else {
$executor = new RunParallel($this->orchestrationManager, $this->jobManagerFactory, $this->mailer, $this->encryptor, $this->logger, $this->queueFactory);
}
$executor->setStorageApi($this->storageApi);
$this->cache[$job->getId()] = $executor;
return $executor->execute($job);
}
} catch (\Exception $e) {
throw $e;
}
throw new \InvalidArgumentException(sprintf('Executor cannot process "%s" jobs.', $job->getCommand()));
}
public function postCleanup(Metadata\Job $job)
{
$esJob = new Job();
$esJob->build($job);
try {
$token = new Token($this->storageApi);
} catch (StorageApiException $e) {
$this->logger->error('Cleanup error - invalid token', array('jobId' => $esJob->getId()));
throw new UserException(sprintf("Invalid token for job %d", $esJob->getId()), $e);
}
$orchestration = $this->orchestrationManager->findOrchestrationById($esJob->getOrchestrationId(), $token);
if (!$orchestration) {
$this->logger->error('PostCleanup error - orchestration not found', array('jobId' => $esJob->getId()));
throw new UserException(sprintf("Orchestration %s not found. Could not update last job", $esJob->getOrchestrationId()));
}
$this->logger->debug('PostCleanup job', array('jobId' => $esJob->getId()));
$this->orchestrationManager->updateLastExecutedJob($orchestration, $esJob);
}
/**
* Testing watchdog on waiting jobs from manual run
*
*/
public function testManuallyLongProcessingWatchdog()
{
$token1 = $this->createNewToken($this->storageApi, 'manage');
$notification1 = new StorageApi\Notification();
$notification1->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_WAITING)->setParameters(array('timeout' => 1));
$notification2 = new StorageApi\Notification();
$notification2->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10));
$notification3 = new StorageApi\Notification();
$notification3->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_2)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10));
$orchestrationId = $this->createOrchestrationTest($token1, array($notification1, $notification2, $notification3));
// first job fake processing
$jobId = $this->enqueueJobTest($orchestrationId);
$syrupJob = $this->syrupJobMapper->get($jobId);
$this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob);
$syrupJob->setStartTime((new \DateTime())->format('c'));
$syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING);
$syrupJob->setResult(array('tasks' => array()));
$this->syrupJobMapper->update($syrupJob);
sleep(3);
$syrupJob->setEndTime((new \DateTime())->format('c'));
$syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS);
$syrupJob->setResult(array('tasks' => array()));
$this->syrupJobMapper->update($syrupJob);
// second job fake processing
$jobId = $this->enqueueJobTest($orchestrationId);
$syrupJob = $this->syrupJobMapper->get($jobId);
$this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob);
$syrupJob->setStartTime((new \DateTime())->format('c'));
$syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING);
$syrupJob->setResult(array('tasks' => array()));
$this->syrupJobMapper->update($syrupJob);
sleep(120);
// check jobs
$this->runWatchdogCommandTest();
// second job finishing
$syrupJob->setEndTime((new \DateTime())->format('c'));
$syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS);
$syrupJob->setResult(array('tasks' => array()));
$this->syrupJobMapper->update($syrupJob);
$esJob = new Job();
$esJob->build($syrupJob);
// check sended notification - manual run - only one must be sended
$events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification2, $this->storageApi);
$this->assertCount(1, $events);
$events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification3, $this->storageApi);
$this->assertCount(0, $events);
}
/**
* @param array $criteria
* @return Job[]
* @throws \Keboola\StorageApi\Exception
*/
public function findJobs(array $criteria, $offset = 0, $limit = JobManager::PAGING)
{
$exportOptions = $this->buildExportOptions($criteria);
$data = $this->syrupJobSearch->getJobs(array('projectId' => $this->token->getProjectId(), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, 'offset' => $offset, 'limit' => $limit, 'query' => $exportOptions));
$manager = $this;
return array_map(function ($line) use($manager) {
$esJob = new Elasticsearch\Job($this->objectEncryptor, $line, $line['_index'], $line['_type']);
$job = new Job();
$job->build($esJob);
$job->setUrl($manager->generateJobUri($job));
$job->setToken($this->decryptToken($job->getToken()));
return $job;
}, $data);
}
