/**
* Setup executor properties and validate job
*
* @param SyrupJob $job
*/
public final function build(SyrupJob $job)
{
$this->validateJob($job);
$this->job = new Job();
$this->job->build($job);
$this->jobResult = new JobResult($this->job);
$this->stopwatch->start(self::STOPWATCH_ORCHESTRATION_EVENT);
//@TODO new feature - better handling with sapi errors - send errorNotification
try {
$this->token = new StorageApi\Token($this->storageApi);
} catch (StorageApiException $e) {
throw new UserException(sprintf("Invalid token for job %d", $this->job->getId()), $e);
}
$this->jobEsManager = $this->jobManagerFactory->createJobManager($this->token);
$this->componentsList = new KbcComponentsList($this->storageApi);
$orchestration = $this->orchestrationManager->findOrchestrationById($this->job->getOrchestrationId(), $this->token);
if (!$orchestration) {
throw new UserException(sprintf("Orchestration %s not found", $this->job->getOrchestrationId()));
} else {
$this->orchestration = $orchestration;
}
$this->orchestrationManager->updateLastExecutedJob($this->orchestration, $this->job);
// validation for empty orchestrations
$hasActiveTasks = false;
if ($this->job->getTasks()) {
foreach ($this->job->getTasks() as $task) {
if (!empty($task->active)) {
$hasActiveTasks = true;
}
}
}
if ($hasActiveTasks === false) {
throw new UserException(self::ERROR_NO_TASK);
}
}
/**
* @param StorageApiEvent $event
* @return mixed|string Event ID
*/
private function save(StorageApiEvent $event)
{
$event->setParams(array_merge($event->getParams(), array('jobId' => $this->job->getId(), 'orchestrationId' => $this->job->getOrchestrationId(), 'pid' => getmypid())));
if ($event->getType() == StorageApiEvent::TYPE_ERROR) {
$event->setParams(array_merge($event->getParams(), array('notificationEmails' => $this->job->getNotificationsEmails())));
}
return EventLogger::create($event, $this->client);
}
/**
* @param Elasticsearch\Job $job
* @param Token $token
* @return OrchestrationTask[]
*/
public function findTasksByJob(Elasticsearch\Job $job, Token $token)
{
$data = $this->components->getConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, (new Entity\Orchestration())->setId($job->getOrchestrationId())->getId());
if (empty($data['configuration']['tasks'])) {
return array();
}
$data = $data['configuration']['tasks'];
/** @var OrchestrationTask[] $rows */
$rows = array_map(function ($line) use($token) {
$task = new OrchestrationTask($token);
$task->fromArray($line);
return $task;
}, $data['configuration']['tasks']);
return $this->sortTasks($rows);
}
/**
* @param Elasticsearch\Job $job
* @param Orchestration $orchestration
* @param Notification $notification
* @param KbcComponentsList $components
* @param null $averageDuration
*/
public function sendLongProcessingMessage(Elasticsearch\Job $job, Orchestration $orchestration, Notification $notification, KbcComponentsList $components, $averageDuration = null)
{
$notificationsEmails = array($notification->getEmail());
// validating emails
foreach ($notificationsEmails as $key => $notificationsEmail) {
if (!\Swift_Validate::email($notificationsEmail)) {
unset($notificationsEmails[$key]);
}
}
if (!count($notificationsEmails)) {
return;
}
$message = \Swift_Message::newInstance();
$message->setSubject(sprintf("[KBC] %s orchestrator %s is still processing", $job->getTokenOwnerName(), $job->getOrchestrationName()));
$message->setFrom(self::MAIL_SENDER);
foreach ($notificationsEmails as $notificationsEmail) {
$message->addTo($notificationsEmail);
}
$schedule = null;
try {
$cronSchedule = CronSchedule::fromCronString($orchestration->getCrontabRecord(), 'en');
$schedule = $cronSchedule->asNaturalLanguage();
} catch (\Exception $e) {
}
$results = $job->getResults();
$tasks = array();
if (!empty($results->tasks)) {
$tasks = $results->tasks;
}
$jobUrl = $components->getJobUriTemplate(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME);
$jobUrl = str_replace('&&projectId&&', $job->getProjectId(), $jobUrl);
$jobUrl = str_replace('&&orchestrationId&&', $job->getOrchestrationId(), $jobUrl);
$jobUrl = str_replace('&&jobId&&', $job->getId(), $jobUrl);
$message->setBody($this->templating->render('KeboolaOrchestratorBundle:Email:jobLongProcessing.email.html.twig', array('schedule' => $schedule, 'tasks' => $tasks, 'componentsIcons' => $components->getComponentsIcons(), 'componentsNames' => $components->getComponentsNames(), 'componentsTypes' => $this->filterComponentsTypes($components->getComponentsTypes()), 'job' => $job, 'avgDurationString' => Utils::convertDurationToString($averageDuration), 'durationString' => Utils::convertDurationToString(Utils::dateDiffInMinutes($job->getCreatedTime(), new \DateTime()) * 60), 'durationMinutes' => Utils::dateDiffInMinutes($job->getCreatedTime(), new \DateTime()), 'jobUrl' => $jobUrl)), 'text/html');
$this->mailer->send($message);
/**
* @var \Swift_Spool $spool
*/
$spool = $this->mailer->getTransport()->getSpool();
$spool->flushQueue($this->mailerTransport);
}
public function postCleanup(Metadata\Job $job)
{
$esJob = new Job();
$esJob->build($job);
try {
$token = new Token($this->storageApi);
} catch (StorageApiException $e) {
$this->logger->error('Cleanup error - invalid token', array('jobId' => $esJob->getId()));
throw new UserException(sprintf("Invalid token for job %d", $esJob->getId()), $e);
}
$orchestration = $this->orchestrationManager->findOrchestrationById($esJob->getOrchestrationId(), $token);
if (!$orchestration) {
$this->logger->error('PostCleanup error - orchestration not found', array('jobId' => $esJob->getId()));
throw new UserException(sprintf("Orchestration %s not found. Could not update last job", $esJob->getOrchestrationId()));
}
$this->logger->debug('PostCleanup job', array('jobId' => $esJob->getId()));
$this->orchestrationManager->updateLastExecutedJob($orchestration, $esJob);
}
/**
* @param Elasticsearch\Job $job
* @param Client $sapi
* @param StorageApi\Notification $notification
* @param null $averageDuration
* @throws Exception
*/
private function logLongProcessing(Elasticsearch\Job $job, Client $sapi, StorageApi\Notification $notification, $averageDuration = null)
{
$event = new Event();
$event->setComponent(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME)->setType(Event::TYPE_WARN)->setMessage(sprintf(self::EVENT_MESSAGE_LONG_PROCESSING, $job->getId()))->setParams(array('jobId' => $job->getId(), 'orchestrationId' => $job->getOrchestrationId(), 'createdTime' => $job->getCreatedTime()->format('c'), 'notificationEmail' => $notification->getEmail(), 'tolerance' => $notification->getParameter('tolerance'), 'averageDuration' => $averageDuration));
StorageApi\EventLogger::create($event, $sapi);
$this->logger->error(sprintf(self::EVENT_MESSAGE_LONG_PROCESSING, $job->getId()), array('job' => $job->getId(), 'orchestrationId' => $job->getOrchestrationId(), 'projectId' => $job->getProjectId(), 'project' => $job->getTokenOwnerName(), 'averageDuration' => $averageDuration));
}
/**
* @param Job $job
* @param StorageApi\UniqueManager $uniqueGenerator
* @return Elasticsearch\Job
*/
private function buildEsJob(Job $job, StorageApi\UniqueManager $uniqueGenerator)
{
//@FIXME replace with jobfactory https://github.com/keboola/syrup/wiki/Upgrade
$jobId = $uniqueGenerator->generateId();
$runId = $uniqueGenerator->generateRunId($uniqueGenerator->getRunId());
$job->setId($jobId);
$job->setRunId($runId);
return new Elasticsearch\Job($this->objectEncryptor, ['id' => $jobId, 'lockName' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME . '-' . $job->getProjectId() . '-' . $job->getOrchestrationId(), 'runId' => $runId, 'nestingLevel' => 0, 'project' => ['id' => $job->getProjectId(), 'name' => $job->getTokenOwnerName()], 'token' => ['id' => $job->getTokenId(), 'description' => $job->getTokenDesc(), 'token' => $this->encryptToken($job->getToken())], 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, 'command' => 'run', 'params' => array('config' => $job->getConfig(), 'orchestration' => array('id' => $job->getOrchestrationId(), 'name' => $job->getOrchestrationName()), 'initializedBy' => $job->getInitializedBy(), 'initiator' => array('id' => $job->getInitiatorTokenId(), 'description' => $job->getInitiatorTokenDesc(), 'userAgent' => $job->getInitiatorUserAgent()), 'notificationsEmails' => $job->getNotificationsEmails(), 'tasks' => $job->getTasks()), 'process' => ['host' => gethostname(), 'pid' => posix_getpid()], 'createdTime' => $job->getCreatedTime()->format('c')]);
}
