private function watchWaiting(InputInterface $input, OutputInterface $output)
{
$now = new \DateTime();
$output->writeln(sprintf('<info>%s</info>', 'Check waiting jobs'));
$i = 0;
while ($jobs = $this->loadWaitingJobs($i)) {
foreach ($jobs as $job) {
$i++;
// check sapi connection
if (!$this->sapiPing($job, $this->encryptor)) {
$message = 'SAPI connect failed';
$this->logger->info($message, array('job' => $job->getId(), 'project' => $job->getProject(), 'params' => $job->getRawParams()));
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
continue;
}
$sapiClient = $this->createJobSapiClient($job);
$jobManager = $this->jobManagerFactory->createJobManager(new StorageApi\Token($sapiClient));
try {
$esJob = $jobManager->findJobByIdForWatchdog($job->getId());
if (!$esJob) {
$message = 'Job load from ES failed';
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
continue;
}
} catch (\InvalidArgumentException $e) {
$message = 'Job load from ES failed';
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
continue;
}
// clean jobs of deleted orchestrations
$orchestration = $this->orchestrationManager->findDeletedOrchestrations(array('id' => $esJob->getOrchestrationId()), false);
$orchestration = reset($orchestration);
if ($orchestration) {
$updateJob = $this->syrupJobMapper->get($esJob->getId());
if ($updateJob->getComponent() !== KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME) {
continue;
}
$updateJob->setStatus(Metadata\Job::STATUS_CANCELLED);
$this->syrupJobMapper->update($updateJob);
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[deleted orchestration]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
continue;
}
$orchestrations = $this->orchestrationManager->findOrchestrations(array('id' => $esJob->getOrchestrationId()), false);
/**
* @var Orchestration $orchestration
*/
$orchestration = reset($orchestrations);
if (!$orchestration) {
$message = 'Orchestration load failed';
$this->logger->critical($message, array('job' => $job->getId(), 'orchestrationId' => $esJob->getOrchestrationId(), 'projectId' => $esJob->getProjectId(), 'project' => $esJob->getTokenOwnerName()));
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
continue;
}
$componentsList = new KbcComponentsList($sapiClient);
// waiting duration check
if ($esJob->getInitializedBy() === 'manually') {
/**
* @var StorageApi\Notification[] $notifications
*/
$notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) {
return $row->getChannel() === Metadata\Job::STATUS_WAITING;
});
if ($notifications) {
foreach ($notifications as $notification) {
if ($notification->getEmail() !== $esJob->getInitiatorTokenDesc()) {
continue;
}
$date = new \DateTime($job->getCreatedTime());
$date->modify(sprintf('+%d minutes', $notification->getParameter('timeout', 0)));
if ($date > $now) {
continue;
}
$events = StorageApi\EventLoader::longWaitingEvents($esJob, $notification, $sapiClient);
if ($events) {
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
} else {
$this->mailer->sendLongWaitingMessage($esJob, $orchestration, $notification, $componentsList);
$this->logLongWaiting($esJob, $sapiClient, $notification);
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
}
}
} else {
// log waiting for default timeout
$date = new \DateTime($job->getCreatedTime());
$date->modify(sprintf('+%d minutes', self::WAIT_LIMIT_MINUTES));
if ($date <= $now) {
$events = StorageApi\EventLoader::longWaitingEventsAll($esJob, $sapiClient);
if ($events) {
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[system already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
} else {
$this->logLongWaiting($esJob, $sapiClient, null);
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[system notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
}
}
}
} else {
/**
* @var StorageApi\Notification[] $notifications
*/
$notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) {
return $row->getChannel() === Metadata\Job::STATUS_WAITING;
});
if ($notifications) {
foreach ($notifications as $notification) {
$date = new \DateTime($job->getCreatedTime());
$date->modify(sprintf('+%d minutes', $notification->getParameter('timeout', 0)));
if ($date > $now) {
continue;
}
$events = StorageApi\EventLoader::longWaitingEvents($esJob, $notification, $sapiClient);
if ($events) {
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
} else {
$this->mailer->sendLongWaitingMessage($esJob, $orchestration, $notification, $componentsList);
$this->logLongWaiting($esJob, $sapiClient, $notification);
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
}
}
} else {
// log waiting for default timeout
$date = new \DateTime($job->getCreatedTime());
$date->modify(sprintf('+%d minutes', self::WAIT_LIMIT_MINUTES));
if ($date <= $now) {
$events = StorageApi\EventLoader::longWaitingEventsAll($esJob, $sapiClient);
if ($events) {
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[system already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
} else {
$this->logLongWaiting($esJob, $sapiClient, null);
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[system notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
}
}
}
}
}
}
}
public function testManuallyLongWaitingJobs()
{
$token1 = $this->createNewToken($this->storageApi, 'manage');
$notification1 = new StorageApi\Notification();
$notification1->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_WAITING)->setParameters(array('timeout' => 1));
$notification2 = new StorageApi\Notification();
$notification2->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_2)->setChannel(Metadata\Job::STATUS_WAITING)->setParameters(array('timeout' => 1));
$orchestrationId = $this->createCrontabOrchestrationTest($token1, array($notification1, $notification2));
$jobId = $this->enqueueJobTest($orchestrationId);
$syrupJob = $this->syrupJobMapper->get($jobId);
$this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob);
sleep(120);
// check jobs
$this->runWatchdogCommandTest();
$syrupJob = $this->syrupJobMapper->get($jobId);
$this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob);
$esJob = new Job();
$esJob->build($syrupJob);
// check sended notification - manual run - only one must be sended
$events = StorageApi\EventLoader::longWaitingEvents($esJob, $notification1, $this->storageApi);
$this->assertCount(1, $events);
$events = StorageApi\EventLoader::longWaitingEvents($esJob, $notification2, $this->storageApi);
$this->assertCount(0, $events);
}
