/**
* Get all orchestrations
*
* @return Response
*/
public function getAllAction()
{
$orchestrations = $this->dbOrchestrationManager->findOrchestrations(array('projectId' => $this->token->getProjectId()));
$orchestrations = array_map(function (Orchestration $row) {
return $row->toApiSimpleArray();
}, $orchestrations);
return $this->createJsonResponse($orchestrations);
}
protected function execute(InputInterface $input, OutputInterface $output)
{
parent::execute($input, $output);
$this->dbOrchestrationManager = $this->getContainer()->get('orchestrator.doctrine.orchestration_manager');
$projects = array();
if ($input->getOption('projectId')) {
$projects[(int) $input->getOption('projectId')] = (int) $input->getOption('projectId');
} else {
foreach ($this->dbOrchestrationManager->findOrchestrations(array()) as $orchestration) {
$projects[$orchestration->getProjectId()] = (int) $orchestration->getProjectId();
}
}
foreach ($projects as $projectId) {
$output->writeln(sprintf('<info>[%s] Check start</info>', $projectId));
$filter = array('projectId' => $projectId);
foreach ($this->dbOrchestrationManager->findOrchestrations($filter) as $orchestration) {
try {
$jobClient = $this->createProjectSapi($orchestration);
$token = new StorageApi\Token($jobClient);
if ($token->hasEnabledFeature(StorageApi\Token::ORCHESTRATOR_NEW_CONFIGURATION)) {
$output->writeln("\t" . sprintf('<comment>[%s] (%s) %s - New configuration already enabled</comment>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName()));
continue;
}
} catch (DisabledException $e) {
$message = $e->getMessage();
if ($e->getPrevious()) {
$message = sprintf('%s: %s', $message, $e->getPrevious()->getMessage());
}
$output->writeln("\t" . sprintf('<error>[%s] (%s) %s - ' . $message . '</error>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName()));
continue;
}
$orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestration->getId(), $token, false);
// validate configuration
try {
$this->validateToken($jobClient);
if (!$input->getOption('onlyErrors')) {
$output->writeln("\t" . sprintf('<info>[%s] (%s) %s - Checked</info>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName()));
}
} catch (StorageApi\InvalidStateException $e) {
$output->writeln("\t" . sprintf('<error>[%s] (%s) %s - %s</error>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName(), $e->getMessage()));
}
}
$output->writeln(sprintf('<info>[%s] Check finished</info>', $projectId));
}
}
/**
* @param $projectId
* @return \Keboola\OrchestratorBundle\Entity\Orchestration[]
*/
private function loadProjectOrchestrations($projectId)
{
return $this->dbOrchestrationManager->findOrchestrations(array('projectId' => $projectId));
}
protected function execute(InputInterface $input, OutputInterface $output)
{
parent::execute($input, $output);
$this->logger->info('scheduler.start', array());
$output->writeln('Current time: ' . $this->getStartTime()->format('Y-m-d H:i:s'));
$this->lock->setLockName('scheduler-' . $this->getStartTime()->format('Y-m-d-H-i'));
// skip locked
try {
if (!$this->lock->lock()) {
$this->logger->info('scheduler.skipped', array('lockName' => $this->lock->getLockName()));
return;
}
} catch (\Exception $e) {
$this->logger->info('scheduler.error', array('message' => $e->getMessage(), 'lockName' => $this->lock->getLockName()));
return;
}
$this->logger->info('scheduler.locked', array('lockName' => $this->lock->getLockName()));
/**
* @var Elasticsearch\JobManagerFactory $jobManagerFactory
*/
$jobManagerFactory = $this->getContainer()->get('orchestrator.job_manager.factory');
$this->queue = $this->getContainer()->get('syrup.queue_factory')->get(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME);
$this->dbOrchestrationManager = $this->getContainer()->get('orchestrator.doctrine.orchestration_manager');
$newJobs = array();
$orchestrations = $this->dbOrchestrationManager->findOrchestrations(array());
foreach ($orchestrations as $orchestration) {
// skip deleted and non-active orchestrations
if ($orchestration->getDeleted() || !$orchestration->getActive() || !$orchestration->getCrontabRecord()) {
continue;
}
$cronExpression = CronExpression::factory($orchestration->getCrontabRecord());
if (!$cronExpression->isDue($this->getStartTime())) {
continue;
}
$jobClient = new Client(array('token' => $orchestration->getToken(), 'url' => $this->getConfiguration()->getStorageApiUrl(), 'userAgent' => $this->appName));
try {
$pingClient = new StorageApi\PingClient($jobClient, $this->logger);
if (!$pingClient->ping()) {
$this->logger->info('scheduler.orchestration.skipped.disabled', array('lockName' => $this->lock->getLockName(), 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $orchestration->getProjectId(), 'projectName' => $orchestration->getTokenOwnerName()));
continue;
}
} catch (\Exception $e) {
$this->logger->info('scheduler.orchestration.skipped.error', array('lockName' => $this->lock->getLockName(), 'message' => $e->getMessage(), 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $orchestration->getProjectId(), 'projectName' => $orchestration->getTokenOwnerName()));
continue;
}
$token = new StorageApi\Token($jobClient);
$jobEsManager = $jobManagerFactory->createJobManager($token);
$jobsStats = $jobEsManager->getWaitingStats();
// skip waiting
if (array_key_exists($orchestration->getId(), $jobsStats) && $jobsStats[$orchestration->getId()] >= AppConfiguration::MAX_WAITING_JOBS_COUNT) {
$this->logger->info('scheduler.orchestration.skipped', array('lockName' => $this->lock->getLockName(), 'waitingCount' => $jobsStats[$orchestration->getId()], 'limit' => AppConfiguration::MAX_WAITING_JOBS_COUNT, 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $token->getProjectId(), 'projectName' => $token->getOwnerName()));
continue;
}
$orchestrationWithTasks = $this->dbOrchestrationManager->findOrchestrationById($orchestration->getId(), new StorageApi\Token($jobClient), true);
$tasks = array_map(function ($task) {
/** @var StorageApi\OrchestrationTask $task */
return $task->toApiArray();
}, $orchestrationWithTasks->getTasks());
$orchestration = $this->dbOrchestrationManager->updateLastScheduledTime($orchestration, new \DateTime());
// save job to queue
$job = new Elasticsearch\Job();
$job->setOrchestrationId($orchestration->getId())->setConfig($orchestration->getId())->setOrchestrationName($orchestration->getName())->setToken($orchestration->getToken())->setTokenId($orchestration->getTokenId())->setTokenDesc($orchestration->getTokenDesc())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId())->setInitializedBy('scheduler')->setTaks($tasks)->setInitiatorTokenId($orchestration->getTokenId())->setInitiatorTokenDesc($orchestration->getTokenDesc())->setInitiatorUserAgent($this->appName . ' Scheduler');
$job = $jobEsManager->saveJob($job, new StorageApi\UniqueManager($jobClient));
$newJobs[] = $job;
// add job to queue
try {
if (is_null($job)) {
throw new ApplicationException('Unable to save or retrieve job from ES', null, array('orchestrationId' => $orchestration->getId()));
}
$this->queue->enqueue($job->getId(), array('jobId' => $job->getId(), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME));
$output->writeln(sprintf('Orchestration %d with cron %s scheduled. Job id: %s', $orchestration->getId(), $orchestration->getCrontabRecord(), $job->getId()));
} catch (\Exception $e) {
$this->logger->error('Error during adding job to sqs', array('lockName' => $this->lock->getLockName(), 'jobId' => $job->getId(), 'exception' => $e));
continue;
}
// log event
try {
$event = new Event();
$event->setComponent($this->appName)->setRunId($job->getRunId())->setParams(array('orchestrationId' => $job->getOrchestrationId()))->setResults(array('jobId' => $job->getId()))->setMessage(sprintf('Orchestration job %s scheduled', $job->getId()));
StorageApi\EventLogger::create($event, $jobClient);
$this->logger->info($event->getMessage(), array('lockName' => $this->lock->getLockName(), 'configurationId' => $event->getConfigurationId(), 'runId' => $event->getRunId(), 'description' => $event->getDescription(), 'params' => $event->getParams(), 'results' => $event->getResults()));
} catch (Exception $e) {
//@TODO user notification when notification API will be available
$this->logger->warning('Invalid token', array('lockName' => $this->lock->getLockName(), 'jobId' => $job->getId(), 'exception' => $e));
}
}
$this->logger->info('scheduler.end', array('lockName' => $this->lock->getLockName(), 'duration' => $this->getDuration(), 'orchestrationsCheckedCount' => count($orchestrations), 'scheduledJobsCount' => count($newJobs)));
sleep(90);
// trying fix double run
// unlock
$this->lock->unlock();
}
private function watchProcessing(InputInterface $input, OutputInterface $output)
{
$output->writeln(sprintf('<info>%s</info>', 'Check processing jobs'));
$orchestrations = array();
$i = 0;
while ($jobs = $this->loadProcessingJobs($i)) {
foreach ($jobs as $job) {
$i++;
// check sapi connection
if (!$this->sapiPing($job, $this->encryptor)) {
$message = 'SAPI connect failed';
$this->logger->info($message, array('job' => $job->getId(), 'project' => $job->getProject(), 'params' => $job->getRawParams()));
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
continue;
}
$sapiClient = $this->createJobSapiClient($job);
$jobManager = $this->jobManagerFactory->createJobManager(new StorageApi\Token($sapiClient));
try {
$esJob = $jobManager->findJobByIdForWatchdog($job->getId());
if (!$esJob) {
$message = 'Job load from ES failed';
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
continue;
}
} catch (\InvalidArgumentException $e) {
$message = 'Job load from ES failed';
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
continue;
}
$orchestrations[$esJob->getOrchestrationId()] = $esJob->getOrchestrationId();
$orchestration = $this->orchestrationManager->findDeletedOrchestrations(array('id' => $esJob->getOrchestrationId()), false);
$orchestration = reset($orchestration);
if ($orchestration) {
$message = 'Orchestration deleted but job is still running';
$this->logger->critical($message, array('job' => $job->getId(), 'project' => $job->getProject(), 'params' => $job->getRawParams()));
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[deleted orchestration]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
continue;
}
$orchestration = $this->orchestrationManager->findOrchestrations(array('id' => $esJob->getOrchestrationId()), false);
$orchestration = reset($orchestration);
if (!$orchestration) {
$message = 'Orchestration load failed';
$this->logger->critical($message, array('job' => $job->getId(), 'orchestrationId' => $esJob->getOrchestrationId(), 'projectId' => $esJob->getProjectId(), 'project' => $esJob->getTokenOwnerName()));
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
continue;
}
$componentsList = new KbcComponentsList($sapiClient);
// processing duration check
if ($esJob->getInitializedBy() === 'manually') {
/**
* @var StorageApi\Notification[] $notifications
*/
$notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) {
return $row->getChannel() === Metadata\Job::STATUS_PROCESSING;
});
if ($notifications) {
$durations = array();
foreach ($this->loadOrchestrationSuccessJobs($orchestration, 0, 20) as $computedJob) {
$durations[] = (int) $computedJob->getDurationSeconds();
}
if (count($durations) < 1) {
continue;
}
foreach ($notifications as $notification) {
if ($notification->getEmail() !== $esJob->getInitiatorTokenDesc()) {
continue;
}
$averageDuration = round(array_sum($durations) / count($durations), 2);
$diff = $this->startTime->getTimestamp() - $esJob->getCreatedTime()->getTimestamp();
// tolerance AVG
if ($diff <= $averageDuration * ((100 + $notification->getParameter('tolerance', 0)) / 100)) {
continue;
}
$events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification, $sapiClient);
if ($events) {
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
} else {
$this->mailer->sendLongProcessingMessage($esJob, $orchestration, $notification, $componentsList, $averageDuration);
$this->logLongProcessing($esJob, $sapiClient, $notification, $averageDuration);
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
}
}
}
} else {
/**
* @var StorageApi\Notification[] $notifications
*/
$notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) {
return $row->getChannel() === Metadata\Job::STATUS_PROCESSING;
});
if ($notifications) {
$durations = array();
foreach ($this->loadOrchestrationSuccessJobs($orchestration, 0, 20) as $computedJob) {
$durations[] = $computedJob->getDurationSeconds();
}
if (count($durations) < 1) {
continue;
}
foreach ($notifications as $notification) {
$averageDuration = round(array_sum($durations) / count($durations), 2);
$diff = $this->startTime->getTimestamp() - $esJob->getCreatedTime()->getTimestamp();
// tolerance AVG
if ($diff <= $averageDuration * ((100 + $notification->getParameter('tolerance', 0)) / 100)) {
continue;
}
$events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification, $sapiClient);
if ($events) {
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
} else {
$this->mailer->sendLongProcessingMessage($esJob, $orchestration, $notification, $componentsList, $averageDuration);
$this->logLongProcessing($esJob, $sapiClient, $notification, $averageDuration);
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
}
}
}
}
}
}
}
