private function watchProcessing(InputInterface $input, OutputInterface $output)
{
$output->writeln(sprintf('<info>%s</info>', 'Check processing jobs'));
$orchestrations = array();
$i = 0;
while ($jobs = $this->loadProcessingJobs($i)) {
foreach ($jobs as $job) {
$i++;
// check sapi connection
if (!$this->sapiPing($job, $this->encryptor)) {
$message = 'SAPI connect failed';
$this->logger->info($message, array('job' => $job->getId(), 'project' => $job->getProject(), 'params' => $job->getRawParams()));
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
continue;
}
$sapiClient = $this->createJobSapiClient($job);
$jobManager = $this->jobManagerFactory->createJobManager(new StorageApi\Token($sapiClient));
try {
$esJob = $jobManager->findJobByIdForWatchdog($job->getId());
if (!$esJob) {
$message = 'Job load from ES failed';
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
continue;
}
} catch (\InvalidArgumentException $e) {
$message = 'Job load from ES failed';
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
continue;
}
$orchestrations[$esJob->getOrchestrationId()] = $esJob->getOrchestrationId();
$orchestration = $this->orchestrationManager->findDeletedOrchestrations(array('id' => $esJob->getOrchestrationId()), false);
$orchestration = reset($orchestration);
if ($orchestration) {
$message = 'Orchestration deleted but job is still running';
$this->logger->critical($message, array('job' => $job->getId(), 'project' => $job->getProject(), 'params' => $job->getRawParams()));
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[deleted orchestration]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
continue;
}
$orchestration = $this->orchestrationManager->findOrchestrations(array('id' => $esJob->getOrchestrationId()), false);
$orchestration = reset($orchestration);
if (!$orchestration) {
$message = 'Orchestration load failed';
$this->logger->critical($message, array('job' => $job->getId(), 'orchestrationId' => $esJob->getOrchestrationId(), 'projectId' => $esJob->getProjectId(), 'project' => $esJob->getTokenOwnerName()));
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
continue;
}
$componentsList = new KbcComponentsList($sapiClient);
// processing duration check
if ($esJob->getInitializedBy() === 'manually') {
/**
* @var StorageApi\Notification[] $notifications
*/
$notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) {
return $row->getChannel() === Metadata\Job::STATUS_PROCESSING;
});
if ($notifications) {
$durations = array();
foreach ($this->loadOrchestrationSuccessJobs($orchestration, 0, 20) as $computedJob) {
$durations[] = (int) $computedJob->getDurationSeconds();
}
if (count($durations) < 1) {
continue;
}
foreach ($notifications as $notification) {
if ($notification->getEmail() !== $esJob->getInitiatorTokenDesc()) {
continue;
}
$averageDuration = round(array_sum($durations) / count($durations), 2);
$diff = $this->startTime->getTimestamp() - $esJob->getCreatedTime()->getTimestamp();
// tolerance AVG
if ($diff <= $averageDuration * ((100 + $notification->getParameter('tolerance', 0)) / 100)) {
continue;
}
$events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification, $sapiClient);
if ($events) {
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
} else {
$this->mailer->sendLongProcessingMessage($esJob, $orchestration, $notification, $componentsList, $averageDuration);
$this->logLongProcessing($esJob, $sapiClient, $notification, $averageDuration);
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
}
}
}
} else {
/**
* @var StorageApi\Notification[] $notifications
*/
$notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) {
return $row->getChannel() === Metadata\Job::STATUS_PROCESSING;
});
if ($notifications) {
$durations = array();
foreach ($this->loadOrchestrationSuccessJobs($orchestration, 0, 20) as $computedJob) {
$durations[] = $computedJob->getDurationSeconds();
}
if (count($durations) < 1) {
continue;
}
foreach ($notifications as $notification) {
$averageDuration = round(array_sum($durations) / count($durations), 2);
$diff = $this->startTime->getTimestamp() - $esJob->getCreatedTime()->getTimestamp();
// tolerance AVG
if ($diff <= $averageDuration * ((100 + $notification->getParameter('tolerance', 0)) / 100)) {
continue;
}
$events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification, $sapiClient);
if ($events) {
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
} else {
$this->mailer->sendLongProcessingMessage($esJob, $orchestration, $notification, $componentsList, $averageDuration);
$this->logLongProcessing($esJob, $sapiClient, $notification, $averageDuration);
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
}
}
}
}
}
}
}
/**
* Testing watchdog on waiting jobs from manual run
*
*/
public function testManuallyLongProcessingWatchdog()
{
$token1 = $this->createNewToken($this->storageApi, 'manage');
$notification1 = new StorageApi\Notification();
$notification1->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_WAITING)->setParameters(array('timeout' => 1));
$notification2 = new StorageApi\Notification();
$notification2->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10));
$notification3 = new StorageApi\Notification();
$notification3->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_2)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10));
$orchestrationId = $this->createOrchestrationTest($token1, array($notification1, $notification2, $notification3));
// first job fake processing
$jobId = $this->enqueueJobTest($orchestrationId);
$syrupJob = $this->syrupJobMapper->get($jobId);
$this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob);
$syrupJob->setStartTime((new \DateTime())->format('c'));
$syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING);
$syrupJob->setResult(array('tasks' => array()));
$this->syrupJobMapper->update($syrupJob);
sleep(3);
$syrupJob->setEndTime((new \DateTime())->format('c'));
$syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS);
$syrupJob->setResult(array('tasks' => array()));
$this->syrupJobMapper->update($syrupJob);
// second job fake processing
$jobId = $this->enqueueJobTest($orchestrationId);
$syrupJob = $this->syrupJobMapper->get($jobId);
$this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob);
$syrupJob->setStartTime((new \DateTime())->format('c'));
$syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING);
$syrupJob->setResult(array('tasks' => array()));
$this->syrupJobMapper->update($syrupJob);
sleep(120);
// check jobs
$this->runWatchdogCommandTest();
// second job finishing
$syrupJob->setEndTime((new \DateTime())->format('c'));
$syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS);
$syrupJob->setResult(array('tasks' => array()));
$this->syrupJobMapper->update($syrupJob);
$esJob = new Job();
$esJob->build($syrupJob);
// check sended notification - manual run - only one must be sended
$events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification2, $this->storageApi);
$this->assertCount(1, $events);
$events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification3, $this->storageApi);
$this->assertCount(0, $events);
}
