private function watchProcessing(InputInterface $input, OutputInterface $output) { $output->writeln(sprintf('<info>%s</info>', 'Check processing jobs')); $orchestrations = array(); $i = 0; while ($jobs = $this->loadProcessingJobs($i)) { foreach ($jobs as $job) { $i++; // check sapi connection if (!$this->sapiPing($job, $this->encryptor)) { $message = 'SAPI connect failed'; $this->logger->info($message, array('job' => $job->getId(), 'project' => $job->getProject(), 'params' => $job->getRawParams())); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } $sapiClient = $this->createJobSapiClient($job); $jobManager = $this->jobManagerFactory->createJobManager(new StorageApi\Token($sapiClient)); try { $esJob = $jobManager->findJobByIdForWatchdog($job->getId()); if (!$esJob) { $message = 'Job load from ES failed'; $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } } catch (\InvalidArgumentException $e) { $message = 'Job load from ES failed'; $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } $orchestrations[$esJob->getOrchestrationId()] = $esJob->getOrchestrationId(); $orchestration = $this->orchestrationManager->findDeletedOrchestrations(array('id' => $esJob->getOrchestrationId()), false); $orchestration = reset($orchestration); if ($orchestration) { $message = 'Orchestration deleted but job is still running'; $this->logger->critical($message, array('job' => $job->getId(), 'project' => $job->getProject(), 'params' => $job->getRawParams())); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[deleted orchestration]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); continue; } $orchestration = $this->orchestrationManager->findOrchestrations(array('id' => $esJob->getOrchestrationId()), false); $orchestration = reset($orchestration); if (!$orchestration) { $message = 'Orchestration load failed'; $this->logger->critical($message, array('job' => $job->getId(), 'orchestrationId' => $esJob->getOrchestrationId(), 'projectId' => $esJob->getProjectId(), 'project' => $esJob->getTokenOwnerName())); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } $componentsList = new KbcComponentsList($sapiClient); // processing duration check if ($esJob->getInitializedBy() === 'manually') { /** * @var StorageApi\Notification[] $notifications */ $notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) { return $row->getChannel() === Metadata\Job::STATUS_PROCESSING; }); if ($notifications) { $durations = array(); foreach ($this->loadOrchestrationSuccessJobs($orchestration, 0, 20) as $computedJob) { $durations[] = (int) $computedJob->getDurationSeconds(); } if (count($durations) < 1) { continue; } foreach ($notifications as $notification) { if ($notification->getEmail() !== $esJob->getInitiatorTokenDesc()) { continue; } $averageDuration = round(array_sum($durations) / count($durations), 2); $diff = $this->startTime->getTimestamp() - $esJob->getCreatedTime()->getTimestamp(); // tolerance AVG if ($diff <= $averageDuration * ((100 + $notification->getParameter('tolerance', 0)) / 100)) { continue; } $events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification, $sapiClient); if ($events) { $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } else { $this->mailer->sendLongProcessingMessage($esJob, $orchestration, $notification, $componentsList, $averageDuration); $this->logLongProcessing($esJob, $sapiClient, $notification, $averageDuration); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } } } } else { /** * @var StorageApi\Notification[] $notifications */ $notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) { return $row->getChannel() === Metadata\Job::STATUS_PROCESSING; }); if ($notifications) { $durations = array(); foreach ($this->loadOrchestrationSuccessJobs($orchestration, 0, 20) as $computedJob) { $durations[] = $computedJob->getDurationSeconds(); } if (count($durations) < 1) { continue; } foreach ($notifications as $notification) { $averageDuration = round(array_sum($durations) / count($durations), 2); $diff = $this->startTime->getTimestamp() - $esJob->getCreatedTime()->getTimestamp(); // tolerance AVG if ($diff <= $averageDuration * ((100 + $notification->getParameter('tolerance', 0)) / 100)) { continue; } $events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification, $sapiClient); if ($events) { $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } else { $this->mailer->sendLongProcessingMessage($esJob, $orchestration, $notification, $componentsList, $averageDuration); $this->logLongProcessing($esJob, $sapiClient, $notification, $averageDuration); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } } } } } } }
/** * Testing watchdog on waiting jobs from manual run * */ public function testManuallyLongProcessingWatchdog() { $token1 = $this->createNewToken($this->storageApi, 'manage'); $notification1 = new StorageApi\Notification(); $notification1->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_WAITING)->setParameters(array('timeout' => 1)); $notification2 = new StorageApi\Notification(); $notification2->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10)); $notification3 = new StorageApi\Notification(); $notification3->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_2)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10)); $orchestrationId = $this->createOrchestrationTest($token1, array($notification1, $notification2, $notification3)); // first job fake processing $jobId = $this->enqueueJobTest($orchestrationId); $syrupJob = $this->syrupJobMapper->get($jobId); $this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob); $syrupJob->setStartTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); sleep(3); $syrupJob->setEndTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); // second job fake processing $jobId = $this->enqueueJobTest($orchestrationId); $syrupJob = $this->syrupJobMapper->get($jobId); $this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob); $syrupJob->setStartTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); sleep(120); // check jobs $this->runWatchdogCommandTest(); // second job finishing $syrupJob->setEndTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); $esJob = new Job(); $esJob->build($syrupJob); // check sended notification - manual run - only one must be sended $events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification2, $this->storageApi); $this->assertCount(1, $events); $events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification3, $this->storageApi); $this->assertCount(0, $events); }