/**
* @param Elasticsearch\Job $job
* @param Notification $notification
* @param Client $sapi
* @return array
*/
public static function longProcessingEvents(Elasticsearch\Job $job, Notification $notification, Client $sapi)
{
$query = array(sprintf("type:%s", Event::TYPE_WARN), sprintf("message:'%s'", sprintf(CronWatchdogCommand::EVENT_MESSAGE_LONG_PROCESSING, $job->getId())));
$params = array('q' => implode(' ', $query), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, 'limit' => 1000);
return array_filter($sapi->listEvents($params), function ($row) use($notification) {
if (empty($row['params'])) {
return false;
}
if (empty($row['params']['notificationEmail'])) {
return false;
}
return $row['params']['notificationEmail'] === $notification->getEmail();
});
}
/**
* @param Orchestration $entity
* @param \Keboola\StorageApi\Client $storageApi
* @return Orchestration
*/
public function fillOrchestrationEntity(Orchestration $entity, Client $storageApi)
{
$oldTokenId = $entity->getTokenId();
$oldName = $entity->getName();
/** @var $entity Orchestration */
$entity = parent::fillEntity($entity);
$name = $entity->getName();
if (empty($name)) {
$entity->setName($oldName);
}
// token change
if ($oldTokenId !== $entity->getTokenId()) {
$token = $storageApi->getToken($entity->getTokenId());
$token = new Token(new Client(array('token' => $token['token'], 'url' => $storageApi->getApiUrl())));
$entity->setToken($token)->setTokenId($token->getId())->setTokenDesc($token->getDescription())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId());
}
// notifications
$data = $this->getPostJsonData();
if (!empty($data['notifications'])) {
$data['notifications'] = array_map(function ($row) {
$notification = new Notification();
$notification->fromArray($row);
return $notification;
}, $data['notifications']);
$entity->setNotifications($data['notifications']);
}
// tasks
$data = $this->getPostJsonData();
if (array_key_exists('tasks', $data)) {
$entity->removeTasks();
$data['tasks'] = array_map(function ($row) {
$notification = new OrchestrationTask();
$notification->fromArray($row);
return $notification;
}, $data['tasks']);
foreach ($data['tasks'] as $task) {
$entity->addTask($task);
}
}
//@FIXME catch and throw exception
return $entity;
}
/**
* @param Elasticsearch\Job $job
* @param Orchestration $orchestration
* @param Notification $notification
* @param KbcComponentsList $components
* @param null $averageDuration
*/
public function sendLongProcessingMessage(Elasticsearch\Job $job, Orchestration $orchestration, Notification $notification, KbcComponentsList $components, $averageDuration = null)
{
$notificationsEmails = array($notification->getEmail());
// validating emails
foreach ($notificationsEmails as $key => $notificationsEmail) {
if (!\Swift_Validate::email($notificationsEmail)) {
unset($notificationsEmails[$key]);
}
}
if (!count($notificationsEmails)) {
return;
}
$message = \Swift_Message::newInstance();
$message->setSubject(sprintf("[KBC] %s orchestrator %s is still processing", $job->getTokenOwnerName(), $job->getOrchestrationName()));
$message->setFrom(self::MAIL_SENDER);
foreach ($notificationsEmails as $notificationsEmail) {
$message->addTo($notificationsEmail);
}
$schedule = null;
try {
$cronSchedule = CronSchedule::fromCronString($orchestration->getCrontabRecord(), 'en');
$schedule = $cronSchedule->asNaturalLanguage();
} catch (\Exception $e) {
}
$results = $job->getResults();
$tasks = array();
if (!empty($results->tasks)) {
$tasks = $results->tasks;
}
$jobUrl = $components->getJobUriTemplate(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME);
$jobUrl = str_replace('&&projectId&&', $job->getProjectId(), $jobUrl);
$jobUrl = str_replace('&&orchestrationId&&', $job->getOrchestrationId(), $jobUrl);
$jobUrl = str_replace('&&jobId&&', $job->getId(), $jobUrl);
$message->setBody($this->templating->render('KeboolaOrchestratorBundle:Email:jobLongProcessing.email.html.twig', array('schedule' => $schedule, 'tasks' => $tasks, 'componentsIcons' => $components->getComponentsIcons(), 'componentsNames' => $components->getComponentsNames(), 'componentsTypes' => $this->filterComponentsTypes($components->getComponentsTypes()), 'job' => $job, 'avgDurationString' => Utils::convertDurationToString($averageDuration), 'durationString' => Utils::convertDurationToString(Utils::dateDiffInMinutes($job->getCreatedTime(), new \DateTime()) * 60), 'durationMinutes' => Utils::dateDiffInMinutes($job->getCreatedTime(), new \DateTime()), 'jobUrl' => $jobUrl)), 'text/html');
$this->mailer->send($message);
/**
* @var \Swift_Spool $spool
*/
$spool = $this->mailer->getTransport()->getSpool();
$spool->flushQueue($this->mailerTransport);
}
/**
* @param Orchestration $entity
* @param \Keboola\StorageApi\Client $storageApi
* @return Orchestration
*/
public function fillOrchestrationEntity(Orchestration $entity, Client $storageApi)
{
/** @var $entity Orchestration */
$entity = parent::fillEntity($entity);
if (!$entity->getTokenId()) {
$entity->setTokenId($storageApi->createToken('manage', sprintf('Orchestrator %s', $entity->getName()), null, true));
}
// fill token data
$token = $storageApi->getToken($entity->getTokenId());
$token = new Token(new Client(array('token' => $token['token'], 'url' => $storageApi->getApiUrl())));
$entity->setToken($token)->setTokenId($token->getId())->setTokenDesc($token->getDescription())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId());
// notifications
$data = $this->getPostJsonData();
if (!empty($data['notifications'])) {
$data['notifications'] = array_map(function ($row) {
$notification = new Notification();
$notification->fromArray($row);
return $notification;
}, $data['notifications']);
$entity->setNotifications($data['notifications']);
}
// tasks
$data = $this->getPostJsonData();
if (!empty($data['tasks'])) {
$data['tasks'] = array_map(function ($row) {
$notification = new OrchestrationTask();
$notification->fromArray($row);
return $notification;
}, $data['tasks']);
foreach ($data['tasks'] as $task) {
$entity->addTask($task);
}
}
//@FIXME catch and throw exception
return $entity;
}
public function postPersist(LifecycleEventArgs $event)
{
$entity = $event->getEntity();
// token encryption on save
if ($entity instanceof Orchestration) {
$entity->setToken($this->decryptToken($entity->getToken()));
$entity->setLastExecutedJob(json_decode($entity->getLastExecutedJob(), true));
if ($entity->getNotifications() && $entity->getNotifications() !== 'null') {
$notificationsList = array_map(function ($row) {
$notification = new Notification();
$notification->fromArray($row);
return $notification;
}, json_decode($entity->getNotifications()));
$entity->setNotifications($notificationsList);
} else {
$entity->setNotifications(array());
}
}
}
/**
* @param Elasticsearch\Job $job
* @param Client $sapi
* @param StorageApi\Notification $notification
* @param null $averageDuration
* @throws Exception
*/
private function logLongProcessing(Elasticsearch\Job $job, Client $sapi, StorageApi\Notification $notification, $averageDuration = null)
{
$event = new Event();
$event->setComponent(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME)->setType(Event::TYPE_WARN)->setMessage(sprintf(self::EVENT_MESSAGE_LONG_PROCESSING, $job->getId()))->setParams(array('jobId' => $job->getId(), 'orchestrationId' => $job->getOrchestrationId(), 'createdTime' => $job->getCreatedTime()->format('c'), 'notificationEmail' => $notification->getEmail(), 'tolerance' => $notification->getParameter('tolerance'), 'averageDuration' => $averageDuration));
StorageApi\EventLogger::create($event, $sapi);
$this->logger->error(sprintf(self::EVENT_MESSAGE_LONG_PROCESSING, $job->getId()), array('job' => $job->getId(), 'orchestrationId' => $job->getOrchestrationId(), 'projectId' => $job->getProjectId(), 'project' => $job->getTokenOwnerName(), 'averageDuration' => $averageDuration));
}
/**
* Testing watchdog on waiting jobs from manual run
*
*/
public function testManuallyLongProcessingWatchdog()
{
$token1 = $this->createNewToken($this->storageApi, 'manage');
$notification1 = new StorageApi\Notification();
$notification1->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_WAITING)->setParameters(array('timeout' => 1));
$notification2 = new StorageApi\Notification();
$notification2->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10));
$notification3 = new StorageApi\Notification();
$notification3->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_2)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10));
$orchestrationId = $this->createOrchestrationTest($token1, array($notification1, $notification2, $notification3));
// first job fake processing
$jobId = $this->enqueueJobTest($orchestrationId);
$syrupJob = $this->syrupJobMapper->get($jobId);
$this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob);
$syrupJob->setStartTime((new \DateTime())->format('c'));
$syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING);
$syrupJob->setResult(array('tasks' => array()));
$this->syrupJobMapper->update($syrupJob);
sleep(3);
$syrupJob->setEndTime((new \DateTime())->format('c'));
$syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS);
$syrupJob->setResult(array('tasks' => array()));
$this->syrupJobMapper->update($syrupJob);
// second job fake processing
$jobId = $this->enqueueJobTest($orchestrationId);
$syrupJob = $this->syrupJobMapper->get($jobId);
$this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob);
$syrupJob->setStartTime((new \DateTime())->format('c'));
$syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING);
$syrupJob->setResult(array('tasks' => array()));
$this->syrupJobMapper->update($syrupJob);
sleep(120);
// check jobs
$this->runWatchdogCommandTest();
// second job finishing
$syrupJob->setEndTime((new \DateTime())->format('c'));
$syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS);
$syrupJob->setResult(array('tasks' => array()));
$this->syrupJobMapper->update($syrupJob);
$esJob = new Job();
$esJob->build($syrupJob);
// check sended notification - manual run - only one must be sended
$events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification2, $this->storageApi);
$this->assertCount(1, $events);
$events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification3, $this->storageApi);
$this->assertCount(0, $events);
}
