/** * @param Elasticsearch\Job $job * @param Notification $notification * @param Client $sapi * @return array */ public static function longProcessingEvents(Elasticsearch\Job $job, Notification $notification, Client $sapi) { $query = array(sprintf("type:%s", Event::TYPE_WARN), sprintf("message:'%s'", sprintf(CronWatchdogCommand::EVENT_MESSAGE_LONG_PROCESSING, $job->getId()))); $params = array('q' => implode(' ', $query), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, 'limit' => 1000); return array_filter($sapi->listEvents($params), function ($row) use($notification) { if (empty($row['params'])) { return false; } if (empty($row['params']['notificationEmail'])) { return false; } return $row['params']['notificationEmail'] === $notification->getEmail(); }); }
/** * @param Orchestration $entity * @param \Keboola\StorageApi\Client $storageApi * @return Orchestration */ public function fillOrchestrationEntity(Orchestration $entity, Client $storageApi) { $oldTokenId = $entity->getTokenId(); $oldName = $entity->getName(); /** @var $entity Orchestration */ $entity = parent::fillEntity($entity); $name = $entity->getName(); if (empty($name)) { $entity->setName($oldName); } // token change if ($oldTokenId !== $entity->getTokenId()) { $token = $storageApi->getToken($entity->getTokenId()); $token = new Token(new Client(array('token' => $token['token'], 'url' => $storageApi->getApiUrl()))); $entity->setToken($token)->setTokenId($token->getId())->setTokenDesc($token->getDescription())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId()); } // notifications $data = $this->getPostJsonData(); if (!empty($data['notifications'])) { $data['notifications'] = array_map(function ($row) { $notification = new Notification(); $notification->fromArray($row); return $notification; }, $data['notifications']); $entity->setNotifications($data['notifications']); } // tasks $data = $this->getPostJsonData(); if (array_key_exists('tasks', $data)) { $entity->removeTasks(); $data['tasks'] = array_map(function ($row) { $notification = new OrchestrationTask(); $notification->fromArray($row); return $notification; }, $data['tasks']); foreach ($data['tasks'] as $task) { $entity->addTask($task); } } //@FIXME catch and throw exception return $entity; }
/** * @param Elasticsearch\Job $job * @param Orchestration $orchestration * @param Notification $notification * @param KbcComponentsList $components * @param null $averageDuration */ public function sendLongProcessingMessage(Elasticsearch\Job $job, Orchestration $orchestration, Notification $notification, KbcComponentsList $components, $averageDuration = null) { $notificationsEmails = array($notification->getEmail()); // validating emails foreach ($notificationsEmails as $key => $notificationsEmail) { if (!\Swift_Validate::email($notificationsEmail)) { unset($notificationsEmails[$key]); } } if (!count($notificationsEmails)) { return; } $message = \Swift_Message::newInstance(); $message->setSubject(sprintf("[KBC] %s orchestrator %s is still processing", $job->getTokenOwnerName(), $job->getOrchestrationName())); $message->setFrom(self::MAIL_SENDER); foreach ($notificationsEmails as $notificationsEmail) { $message->addTo($notificationsEmail); } $schedule = null; try { $cronSchedule = CronSchedule::fromCronString($orchestration->getCrontabRecord(), 'en'); $schedule = $cronSchedule->asNaturalLanguage(); } catch (\Exception $e) { } $results = $job->getResults(); $tasks = array(); if (!empty($results->tasks)) { $tasks = $results->tasks; } $jobUrl = $components->getJobUriTemplate(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME); $jobUrl = str_replace('&&projectId&&', $job->getProjectId(), $jobUrl); $jobUrl = str_replace('&&orchestrationId&&', $job->getOrchestrationId(), $jobUrl); $jobUrl = str_replace('&&jobId&&', $job->getId(), $jobUrl); $message->setBody($this->templating->render('KeboolaOrchestratorBundle:Email:jobLongProcessing.email.html.twig', array('schedule' => $schedule, 'tasks' => $tasks, 'componentsIcons' => $components->getComponentsIcons(), 'componentsNames' => $components->getComponentsNames(), 'componentsTypes' => $this->filterComponentsTypes($components->getComponentsTypes()), 'job' => $job, 'avgDurationString' => Utils::convertDurationToString($averageDuration), 'durationString' => Utils::convertDurationToString(Utils::dateDiffInMinutes($job->getCreatedTime(), new \DateTime()) * 60), 'durationMinutes' => Utils::dateDiffInMinutes($job->getCreatedTime(), new \DateTime()), 'jobUrl' => $jobUrl)), 'text/html'); $this->mailer->send($message); /** * @var \Swift_Spool $spool */ $spool = $this->mailer->getTransport()->getSpool(); $spool->flushQueue($this->mailerTransport); }
/** * @param Orchestration $entity * @param \Keboola\StorageApi\Client $storageApi * @return Orchestration */ public function fillOrchestrationEntity(Orchestration $entity, Client $storageApi) { /** @var $entity Orchestration */ $entity = parent::fillEntity($entity); if (!$entity->getTokenId()) { $entity->setTokenId($storageApi->createToken('manage', sprintf('Orchestrator %s', $entity->getName()), null, true)); } // fill token data $token = $storageApi->getToken($entity->getTokenId()); $token = new Token(new Client(array('token' => $token['token'], 'url' => $storageApi->getApiUrl()))); $entity->setToken($token)->setTokenId($token->getId())->setTokenDesc($token->getDescription())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId()); // notifications $data = $this->getPostJsonData(); if (!empty($data['notifications'])) { $data['notifications'] = array_map(function ($row) { $notification = new Notification(); $notification->fromArray($row); return $notification; }, $data['notifications']); $entity->setNotifications($data['notifications']); } // tasks $data = $this->getPostJsonData(); if (!empty($data['tasks'])) { $data['tasks'] = array_map(function ($row) { $notification = new OrchestrationTask(); $notification->fromArray($row); return $notification; }, $data['tasks']); foreach ($data['tasks'] as $task) { $entity->addTask($task); } } //@FIXME catch and throw exception return $entity; }
public function postPersist(LifecycleEventArgs $event) { $entity = $event->getEntity(); // token encryption on save if ($entity instanceof Orchestration) { $entity->setToken($this->decryptToken($entity->getToken())); $entity->setLastExecutedJob(json_decode($entity->getLastExecutedJob(), true)); if ($entity->getNotifications() && $entity->getNotifications() !== 'null') { $notificationsList = array_map(function ($row) { $notification = new Notification(); $notification->fromArray($row); return $notification; }, json_decode($entity->getNotifications())); $entity->setNotifications($notificationsList); } else { $entity->setNotifications(array()); } } }
/** * @param Elasticsearch\Job $job * @param Client $sapi * @param StorageApi\Notification $notification * @param null $averageDuration * @throws Exception */ private function logLongProcessing(Elasticsearch\Job $job, Client $sapi, StorageApi\Notification $notification, $averageDuration = null) { $event = new Event(); $event->setComponent(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME)->setType(Event::TYPE_WARN)->setMessage(sprintf(self::EVENT_MESSAGE_LONG_PROCESSING, $job->getId()))->setParams(array('jobId' => $job->getId(), 'orchestrationId' => $job->getOrchestrationId(), 'createdTime' => $job->getCreatedTime()->format('c'), 'notificationEmail' => $notification->getEmail(), 'tolerance' => $notification->getParameter('tolerance'), 'averageDuration' => $averageDuration)); StorageApi\EventLogger::create($event, $sapi); $this->logger->error(sprintf(self::EVENT_MESSAGE_LONG_PROCESSING, $job->getId()), array('job' => $job->getId(), 'orchestrationId' => $job->getOrchestrationId(), 'projectId' => $job->getProjectId(), 'project' => $job->getTokenOwnerName(), 'averageDuration' => $averageDuration)); }
/** * Testing watchdog on waiting jobs from manual run * */ public function testManuallyLongProcessingWatchdog() { $token1 = $this->createNewToken($this->storageApi, 'manage'); $notification1 = new StorageApi\Notification(); $notification1->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_WAITING)->setParameters(array('timeout' => 1)); $notification2 = new StorageApi\Notification(); $notification2->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10)); $notification3 = new StorageApi\Notification(); $notification3->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_2)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10)); $orchestrationId = $this->createOrchestrationTest($token1, array($notification1, $notification2, $notification3)); // first job fake processing $jobId = $this->enqueueJobTest($orchestrationId); $syrupJob = $this->syrupJobMapper->get($jobId); $this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob); $syrupJob->setStartTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); sleep(3); $syrupJob->setEndTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); // second job fake processing $jobId = $this->enqueueJobTest($orchestrationId); $syrupJob = $this->syrupJobMapper->get($jobId); $this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob); $syrupJob->setStartTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); sleep(120); // check jobs $this->runWatchdogCommandTest(); // second job finishing $syrupJob->setEndTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); $esJob = new Job(); $esJob->build($syrupJob); // check sended notification - manual run - only one must be sended $events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification2, $this->storageApi); $this->assertCount(1, $events); $events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification3, $this->storageApi); $this->assertCount(0, $events); }