private function watchWaiting(InputInterface $input, OutputInterface $output) { $now = new \DateTime(); $output->writeln(sprintf('<info>%s</info>', 'Check waiting jobs')); $i = 0; while ($jobs = $this->loadWaitingJobs($i)) { foreach ($jobs as $job) { $i++; // check sapi connection if (!$this->sapiPing($job, $this->encryptor)) { $message = 'SAPI connect failed'; $this->logger->info($message, array('job' => $job->getId(), 'project' => $job->getProject(), 'params' => $job->getRawParams())); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } $sapiClient = $this->createJobSapiClient($job); $jobManager = $this->jobManagerFactory->createJobManager(new StorageApi\Token($sapiClient)); try { $esJob = $jobManager->findJobByIdForWatchdog($job->getId()); if (!$esJob) { $message = 'Job load from ES failed'; $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } } catch (\InvalidArgumentException $e) { $message = 'Job load from ES failed'; $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } // clean jobs of deleted orchestrations $orchestration = $this->orchestrationManager->findDeletedOrchestrations(array('id' => $esJob->getOrchestrationId()), false); $orchestration = reset($orchestration); if ($orchestration) { $updateJob = $this->syrupJobMapper->get($esJob->getId()); if ($updateJob->getComponent() !== KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME) { continue; } $updateJob->setStatus(Metadata\Job::STATUS_CANCELLED); $this->syrupJobMapper->update($updateJob); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[deleted orchestration]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); continue; } $orchestrations = $this->orchestrationManager->findOrchestrations(array('id' => $esJob->getOrchestrationId()), false); /** * @var Orchestration $orchestration */ $orchestration = reset($orchestrations); if (!$orchestration) { $message = 'Orchestration load failed'; $this->logger->critical($message, array('job' => $job->getId(), 'orchestrationId' => $esJob->getOrchestrationId(), 'projectId' => $esJob->getProjectId(), 'project' => $esJob->getTokenOwnerName())); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } $componentsList = new KbcComponentsList($sapiClient); // waiting duration check if ($esJob->getInitializedBy() === 'manually') { /** * @var StorageApi\Notification[] $notifications */ $notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) { return $row->getChannel() === Metadata\Job::STATUS_WAITING; }); if ($notifications) { foreach ($notifications as $notification) { if ($notification->getEmail() !== $esJob->getInitiatorTokenDesc()) { continue; } $date = new \DateTime($job->getCreatedTime()); $date->modify(sprintf('+%d minutes', $notification->getParameter('timeout', 0))); if ($date > $now) { continue; } $events = StorageApi\EventLoader::longWaitingEvents($esJob, $notification, $sapiClient); if ($events) { $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } else { $this->mailer->sendLongWaitingMessage($esJob, $orchestration, $notification, $componentsList); $this->logLongWaiting($esJob, $sapiClient, $notification); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } } } else { // log waiting for default timeout $date = new \DateTime($job->getCreatedTime()); $date->modify(sprintf('+%d minutes', self::WAIT_LIMIT_MINUTES)); if ($date <= $now) { $events = StorageApi\EventLoader::longWaitingEventsAll($esJob, $sapiClient); if ($events) { $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[system already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } else { $this->logLongWaiting($esJob, $sapiClient, null); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[system notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } } } } else { /** * @var StorageApi\Notification[] $notifications */ $notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) { return $row->getChannel() === Metadata\Job::STATUS_WAITING; }); if ($notifications) { foreach ($notifications as $notification) { $date = new \DateTime($job->getCreatedTime()); $date->modify(sprintf('+%d minutes', $notification->getParameter('timeout', 0))); if ($date > $now) { continue; } $events = StorageApi\EventLoader::longWaitingEvents($esJob, $notification, $sapiClient); if ($events) { $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } else { $this->mailer->sendLongWaitingMessage($esJob, $orchestration, $notification, $componentsList); $this->logLongWaiting($esJob, $sapiClient, $notification); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } } } else { // log waiting for default timeout $date = new \DateTime($job->getCreatedTime()); $date->modify(sprintf('+%d minutes', self::WAIT_LIMIT_MINUTES)); if ($date <= $now) { $events = StorageApi\EventLoader::longWaitingEventsAll($esJob, $sapiClient); if ($events) { $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[system already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } else { $this->logLongWaiting($esJob, $sapiClient, null); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[system notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } } } } } } }
public function testManuallyLongWaitingJobs() { $token1 = $this->createNewToken($this->storageApi, 'manage'); $notification1 = new StorageApi\Notification(); $notification1->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_WAITING)->setParameters(array('timeout' => 1)); $notification2 = new StorageApi\Notification(); $notification2->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_2)->setChannel(Metadata\Job::STATUS_WAITING)->setParameters(array('timeout' => 1)); $orchestrationId = $this->createCrontabOrchestrationTest($token1, array($notification1, $notification2)); $jobId = $this->enqueueJobTest($orchestrationId); $syrupJob = $this->syrupJobMapper->get($jobId); $this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob); sleep(120); // check jobs $this->runWatchdogCommandTest(); $syrupJob = $this->syrupJobMapper->get($jobId); $this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob); $esJob = new Job(); $esJob->build($syrupJob); // check sended notification - manual run - only one must be sended $events = StorageApi\EventLoader::longWaitingEvents($esJob, $notification1, $this->storageApi); $this->assertCount(1, $events); $events = StorageApi\EventLoader::longWaitingEvents($esJob, $notification2, $this->storageApi); $this->assertCount(0, $events); }