/** * Update last executed job in orchestration * * If job failed, send error notification * * @param SyrupJob $job */ public final function postExecute(SyrupJob $job) { $this->validateJob($job); // not executed or different job - do nothing if (!$this->job || !$this->orchestration) { return; } if ($this->job->getId() !== $job->getId()) { return; } $this->job = new Job(); $this->job->build($job); $this->orchestrationManager->updateLastExecutedJob($this->orchestration, $this->job); if ($job->getStatus() === SyrupJob::STATUS_ERROR) { $this->mailer->sendJobErrorMessage($this->job, $this->orchestration, $this->jobEsManager, $this->componentsList); } if ($job->getStatus() === SyrupJob::STATUS_WARNING) { $this->mailer->sendJobWarningMessage($this->job, $this->orchestration, $this->jobEsManager, $this->componentsList); } }
public function execute(SyrupJob $job) { parent::execute($job); $command = $job->getCommand(); try { // run jobu if ($command === 'run') { $esJob = new Job(); $esJob->build($job); if (!$this->usePhases($esJob)) { $executor = new Run($this->orchestrationManager, $this->jobManagerFactory, $this->mailer, $this->encryptor, $this->logger, $this->queueFactory); } else { $executor = new RunParallel($this->orchestrationManager, $this->jobManagerFactory, $this->mailer, $this->encryptor, $this->logger, $this->queueFactory); } $executor->setStorageApi($this->storageApi); $this->cache[$job->getId()] = $executor; return $executor->execute($job); } } catch (\Exception $e) { throw $e; } throw new \InvalidArgumentException(sprintf('Executor cannot process "%s" jobs.', $job->getCommand())); }
public function postCleanup(Metadata\Job $job) { $esJob = new Job(); $esJob->build($job); try { $token = new Token($this->storageApi); } catch (StorageApiException $e) { $this->logger->error('Cleanup error - invalid token', array('jobId' => $esJob->getId())); throw new UserException(sprintf("Invalid token for job %d", $esJob->getId()), $e); } $orchestration = $this->orchestrationManager->findOrchestrationById($esJob->getOrchestrationId(), $token); if (!$orchestration) { $this->logger->error('PostCleanup error - orchestration not found', array('jobId' => $esJob->getId())); throw new UserException(sprintf("Orchestration %s not found. Could not update last job", $esJob->getOrchestrationId())); } $this->logger->debug('PostCleanup job', array('jobId' => $esJob->getId())); $this->orchestrationManager->updateLastExecutedJob($orchestration, $esJob); }
/** * Testing watchdog on waiting jobs from manual run * */ public function testManuallyLongProcessingWatchdog() { $token1 = $this->createNewToken($this->storageApi, 'manage'); $notification1 = new StorageApi\Notification(); $notification1->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_WAITING)->setParameters(array('timeout' => 1)); $notification2 = new StorageApi\Notification(); $notification2->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10)); $notification3 = new StorageApi\Notification(); $notification3->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_2)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10)); $orchestrationId = $this->createOrchestrationTest($token1, array($notification1, $notification2, $notification3)); // first job fake processing $jobId = $this->enqueueJobTest($orchestrationId); $syrupJob = $this->syrupJobMapper->get($jobId); $this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob); $syrupJob->setStartTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); sleep(3); $syrupJob->setEndTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); // second job fake processing $jobId = $this->enqueueJobTest($orchestrationId); $syrupJob = $this->syrupJobMapper->get($jobId); $this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob); $syrupJob->setStartTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); sleep(120); // check jobs $this->runWatchdogCommandTest(); // second job finishing $syrupJob->setEndTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); $esJob = new Job(); $esJob->build($syrupJob); // check sended notification - manual run - only one must be sended $events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification2, $this->storageApi); $this->assertCount(1, $events); $events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification3, $this->storageApi); $this->assertCount(0, $events); }
/** * @param array $criteria * @return Job[] * @throws \Keboola\StorageApi\Exception */ public function findJobs(array $criteria, $offset = 0, $limit = JobManager::PAGING) { $exportOptions = $this->buildExportOptions($criteria); $data = $this->syrupJobSearch->getJobs(array('projectId' => $this->token->getProjectId(), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, 'offset' => $offset, 'limit' => $limit, 'query' => $exportOptions)); $manager = $this; return array_map(function ($line) use($manager) { $esJob = new Elasticsearch\Job($this->objectEncryptor, $line, $line['_index'], $line['_type']); $job = new Job(); $job->build($esJob); $job->setUrl($manager->generateJobUri($job)); $job->setToken($this->decryptToken($job->getToken())); return $job; }, $data); }