/**
  * @param $projectId
  * @return \Keboola\OrchestratorBundle\Entity\Orchestration[]
  */
 private function loadProjectDeletedOrchestrations($projectId)
 {
     return $this->dbOrchestrationManager->findDeletedOrchestrations(array('projectId' => $projectId));
 }
 private function watchProcessing(InputInterface $input, OutputInterface $output)
 {
     $output->writeln(sprintf('<info>%s</info>', 'Check processing jobs'));
     $orchestrations = array();
     $i = 0;
     while ($jobs = $this->loadProcessingJobs($i)) {
         foreach ($jobs as $job) {
             $i++;
             // check sapi connection
             if (!$this->sapiPing($job, $this->encryptor)) {
                 $message = 'SAPI connect failed';
                 $this->logger->info($message, array('job' => $job->getId(), 'project' => $job->getProject(), 'params' => $job->getRawParams()));
                 $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
                 continue;
             }
             $sapiClient = $this->createJobSapiClient($job);
             $jobManager = $this->jobManagerFactory->createJobManager(new StorageApi\Token($sapiClient));
             try {
                 $esJob = $jobManager->findJobByIdForWatchdog($job->getId());
                 if (!$esJob) {
                     $message = 'Job load from ES failed';
                     $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
                     continue;
                 }
             } catch (\InvalidArgumentException $e) {
                 $message = 'Job load from ES failed';
                 $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
                 continue;
             }
             $orchestrations[$esJob->getOrchestrationId()] = $esJob->getOrchestrationId();
             $orchestration = $this->orchestrationManager->findDeletedOrchestrations(array('id' => $esJob->getOrchestrationId()), false);
             $orchestration = reset($orchestration);
             if ($orchestration) {
                 $message = 'Orchestration deleted but job is still running';
                 $this->logger->critical($message, array('job' => $job->getId(), 'project' => $job->getProject(), 'params' => $job->getRawParams()));
                 $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[deleted orchestration]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
                 continue;
             }
             $orchestration = $this->orchestrationManager->findOrchestrations(array('id' => $esJob->getOrchestrationId()), false);
             $orchestration = reset($orchestration);
             if (!$orchestration) {
                 $message = 'Orchestration load failed';
                 $this->logger->critical($message, array('job' => $job->getId(), 'orchestrationId' => $esJob->getOrchestrationId(), 'projectId' => $esJob->getProjectId(), 'project' => $esJob->getTokenOwnerName()));
                 $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
                 continue;
             }
             $componentsList = new KbcComponentsList($sapiClient);
             // processing duration check
             if ($esJob->getInitializedBy() === 'manually') {
                 /**
                  * @var StorageApi\Notification[] $notifications
                  */
                 $notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) {
                     return $row->getChannel() === Metadata\Job::STATUS_PROCESSING;
                 });
                 if ($notifications) {
                     $durations = array();
                     foreach ($this->loadOrchestrationSuccessJobs($orchestration, 0, 20) as $computedJob) {
                         $durations[] = (int) $computedJob->getDurationSeconds();
                     }
                     if (count($durations) < 1) {
                         continue;
                     }
                     foreach ($notifications as $notification) {
                         if ($notification->getEmail() !== $esJob->getInitiatorTokenDesc()) {
                             continue;
                         }
                         $averageDuration = round(array_sum($durations) / count($durations), 2);
                         $diff = $this->startTime->getTimestamp() - $esJob->getCreatedTime()->getTimestamp();
                         // tolerance AVG
                         if ($diff <= $averageDuration * ((100 + $notification->getParameter('tolerance', 0)) / 100)) {
                             continue;
                         }
                         $events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification, $sapiClient);
                         if ($events) {
                             $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
                         } else {
                             $this->mailer->sendLongProcessingMessage($esJob, $orchestration, $notification, $componentsList, $averageDuration);
                             $this->logLongProcessing($esJob, $sapiClient, $notification, $averageDuration);
                             $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
                         }
                     }
                 }
             } else {
                 /**
                  * @var StorageApi\Notification[] $notifications
                  */
                 $notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) {
                     return $row->getChannel() === Metadata\Job::STATUS_PROCESSING;
                 });
                 if ($notifications) {
                     $durations = array();
                     foreach ($this->loadOrchestrationSuccessJobs($orchestration, 0, 20) as $computedJob) {
                         $durations[] = $computedJob->getDurationSeconds();
                     }
                     if (count($durations) < 1) {
                         continue;
                     }
                     foreach ($notifications as $notification) {
                         $averageDuration = round(array_sum($durations) / count($durations), 2);
                         $diff = $this->startTime->getTimestamp() - $esJob->getCreatedTime()->getTimestamp();
                         // tolerance AVG
                         if ($diff <= $averageDuration * ((100 + $notification->getParameter('tolerance', 0)) / 100)) {
                             continue;
                         }
                         $events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification, $sapiClient);
                         if ($events) {
                             $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
                         } else {
                             $this->mailer->sendLongProcessingMessage($esJob, $orchestration, $notification, $componentsList, $averageDuration);
                             $this->logLongProcessing($esJob, $sapiClient, $notification, $averageDuration);
                             $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
                         }
                     }
                 }
             }
         }
     }
 }