protected function execute(InputInterface $input, OutputInterface $output) { parent::execute($input, $output); $this->dbOrchestrationManager = $this->getContainer()->get('orchestrator.doctrine.orchestration_manager'); $projects = array(); if ($input->getOption('projectId')) { $projects[(int) $input->getOption('projectId')] = (int) $input->getOption('projectId'); } else { foreach ($this->dbOrchestrationManager->findOrchestrations(array()) as $orchestration) { $projects[$orchestration->getProjectId()] = (int) $orchestration->getProjectId(); } } foreach ($projects as $projectId) { $output->writeln(sprintf('<info>[%s] Check start</info>', $projectId)); $filter = array('projectId' => $projectId); foreach ($this->dbOrchestrationManager->findOrchestrations($filter) as $orchestration) { try { $jobClient = $this->createProjectSapi($orchestration); $token = new StorageApi\Token($jobClient); if ($token->hasEnabledFeature(StorageApi\Token::ORCHESTRATOR_NEW_CONFIGURATION)) { $output->writeln("\t" . sprintf('<comment>[%s] (%s) %s - New configuration already enabled</comment>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName())); continue; } } catch (DisabledException $e) { $message = $e->getMessage(); if ($e->getPrevious()) { $message = sprintf('%s: %s', $message, $e->getPrevious()->getMessage()); } $output->writeln("\t" . sprintf('<error>[%s] (%s) %s - ' . $message . '</error>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName())); continue; } $orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestration->getId(), $token, false); // validate configuration try { $this->validateToken($jobClient); if (!$input->getOption('onlyErrors')) { $output->writeln("\t" . sprintf('<info>[%s] (%s) %s - Checked</info>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName())); } } catch (StorageApi\InvalidStateException $e) { $output->writeln("\t" . sprintf('<error>[%s] (%s) %s - %s</error>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName(), $e->getMessage())); } } $output->writeln(sprintf('<info>[%s] Check finished</info>', $projectId)); } }
public function postCreateTokenAction($orchestrationId) { $orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestrationId, $this->token); if (!$orchestration) { $exception = new OrchestratorException(404, sprintf('Orchestration %s not found', $orchestrationId)); $exception->setExceptionCode('ORCHESTRATION_NOT_FOUND'); throw $exception; } try { $tokenId = $this->storageApi->createToken('manage', sprintf('Orchestrator %s', $orchestration->getName()), null, true); // fill token data $token = $this->storageApi->getToken($tokenId); $token = new StorageApi\Token(new Client(array('token' => $token['token'], 'url' => $this->storageApi->getApiUrl()))); $orchestration->setToken((string) $token)->setTokenId($token->getId())->setTokenDesc($token->getDescription())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId()); } catch (\Exception $e) { $exception = new OrchestratorException(500, sprintf('Could not create new token'), $e); $exception->setExceptionCode('TOKEN_VALIDATION'); throw $exception; } $orchestration = $this->dbOrchestrationManager->updateOrchestration($orchestration); $this->logger->info('Orchestration token changed'); $data = $orchestration->toApiArray(); return $this->createJsonResponse($data['token'], 201); }
/** * @param $projectId * @return \Keboola\OrchestratorBundle\Entity\Orchestration[] */ private function loadProjectDeletedOrchestrations($projectId) { return $this->dbOrchestrationManager->findDeletedOrchestrations(array('projectId' => $projectId)); }
/** * @param $jobId * @param Request $request * @return \Symfony\Component\HttpFoundation\JsonResponse */ public function postRetryAction($jobId, Request $request) { $job = $this->jobEsManager->findJobById($jobId); if (!$job) { $exception = new OrchestratorException(404, sprintf('Job %s not found', $jobId)); $exception->setExceptionCode('JOB_NOT_FOUND'); throw $exception; } if (!StatusConverter::isFinishedStatus($job->getStatus())) { $exception = new OrchestratorException(400, sprintf('You can retry only finished jobs', $jobId)); $exception->setExceptionCode('JOB_VALIDATION'); throw $exception; } $orchestrationId = $job->getOrchestrationId(); $orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestrationId, $this->token, true); if (!$orchestration) { $exception = new OrchestratorException(404, sprintf('Orchestration %s not found', $orchestrationId)); $exception->setExceptionCode('ORCHESTRATION_NOT_FOUND'); throw $exception; } try { $form = $this->createForm(new OrchestrationRunType($this->storageApi, $this->dbOrchestrationManager, $this->token, $orchestration)); $handler = $this->createRunFormHandler($form, $request, $orchestration); if ($handler->process()) { if ($handler->getTaskList()) { $tasks = array_map(function ($task) { /** @var StorageApi\OrchestrationTask $task */ $task->setId($this->storageApi->generateId()); return $task->toApiArray(); }, $handler->getTaskList()); } else { $tasks = $job->getTasks(); } } } catch (HttpException $e) { $exception = new OrchestratorException($e->getStatusCode(), $e->getMessage()); $exception->setExceptionCode('JOB_VALIDATION'); throw $exception; } // waiting jobs limit $jobsStats = $this->jobEsManager->getWaitingStats(); // skip waiting if (array_key_exists($orchestration->getId(), $jobsStats) && $jobsStats[$orchestration->getId()] >= AppConfiguration::MAX_WAITING_JOBS_COUNT) { $count = $jobsStats[$orchestration->getId()]; $this->logger->info('scheduler.orchestration.skipped', array('waitingCount' => $count, 'limit' => AppConfiguration::MAX_WAITING_JOBS_COUNT, 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $this->token->getProjectId(), 'projectName' => $this->token->getOwnerName())); if ($count > 1) { $exception = new OrchestratorException(409, sprintf('Orchestration %s has %d waiting jobs. Current limit is %d.', $orchestrationId, $count, AppConfiguration::MAX_WAITING_JOBS_COUNT)); } else { $exception = new OrchestratorException(409, sprintf('Orchestration %s has %d waiting job. Current limit is %d.', $orchestrationId, $count, AppConfiguration::MAX_WAITING_JOBS_COUNT)); } $exception->setExceptionCode('ORCHESTRATION_VALIDATION'); throw $exception; } $form = $this->createForm(new ScheduleJobType($this->storageApi, $this->dbOrchestrationManager, $this->token)); $handler = parent::createFormHandler($form, $request); try { $notificationsEmails = array(); if ($handler->process()) { $notificationsEmails = $handler->getPost('notificationsEmails', array()); } if (!$notificationsEmails && \Swift_Validate::email($this->token->getDescription())) { $notificationsEmails = array($this->token->getDescription()); } } catch (HttpException $e) { $exception = new OrchestratorException(400, $e->getMessage()); $exception->setExceptionCode('JOB_VALIDATION'); throw $exception; } $this->initSqsQueue(); $job = new Elasticsearch\Job(); $job->setConfig($orchestration->getId())->setOrchestrationId($orchestration->getId())->setOrchestrationName($orchestration->getName())->setToken($orchestration->getToken())->setTokenId($orchestration->getTokenId())->setTokenDesc($orchestration->getTokenDesc())->setTokenOwnerName($this->token->getOwnerName())->setProjectId($this->token->getProjectId())->setInitializedBy('manually')->setInitiatorTokenId($this->token->getId())->setInitiatorTokenDesc($this->token->getDescription())->setInitiatorUserAgent($this->getRequestUserAgent($request))->setNotificationsEmails($notificationsEmails)->setTaks($tasks); $job = $this->jobEsManager->saveJob($job, new StorageApi\UniqueManager($this->storageApi)); $this->queue->enqueue($job->getId(), array('jobId' => $job->getId(), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME)); $this->logger->info(sprintf('Orchestration job %s created manually', $job->getId())); return $this->createJsonResponse($job->toOldApiArray(), 201); }
protected function execute(InputInterface $input, OutputInterface $output) { parent::execute($input, $output); $this->logger->info('scheduler.start', array()); $output->writeln('Current time: ' . $this->getStartTime()->format('Y-m-d H:i:s')); $this->lock->setLockName('scheduler-' . $this->getStartTime()->format('Y-m-d-H-i')); // skip locked try { if (!$this->lock->lock()) { $this->logger->info('scheduler.skipped', array('lockName' => $this->lock->getLockName())); return; } } catch (\Exception $e) { $this->logger->info('scheduler.error', array('message' => $e->getMessage(), 'lockName' => $this->lock->getLockName())); return; } $this->logger->info('scheduler.locked', array('lockName' => $this->lock->getLockName())); /** * @var Elasticsearch\JobManagerFactory $jobManagerFactory */ $jobManagerFactory = $this->getContainer()->get('orchestrator.job_manager.factory'); $this->queue = $this->getContainer()->get('syrup.queue_factory')->get(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME); $this->dbOrchestrationManager = $this->getContainer()->get('orchestrator.doctrine.orchestration_manager'); $newJobs = array(); $orchestrations = $this->dbOrchestrationManager->findOrchestrations(array()); foreach ($orchestrations as $orchestration) { // skip deleted and non-active orchestrations if ($orchestration->getDeleted() || !$orchestration->getActive() || !$orchestration->getCrontabRecord()) { continue; } $cronExpression = CronExpression::factory($orchestration->getCrontabRecord()); if (!$cronExpression->isDue($this->getStartTime())) { continue; } $jobClient = new Client(array('token' => $orchestration->getToken(), 'url' => $this->getConfiguration()->getStorageApiUrl(), 'userAgent' => $this->appName)); try { $pingClient = new StorageApi\PingClient($jobClient, $this->logger); if (!$pingClient->ping()) { $this->logger->info('scheduler.orchestration.skipped.disabled', array('lockName' => $this->lock->getLockName(), 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $orchestration->getProjectId(), 'projectName' => $orchestration->getTokenOwnerName())); continue; } } catch (\Exception $e) { $this->logger->info('scheduler.orchestration.skipped.error', array('lockName' => $this->lock->getLockName(), 'message' => $e->getMessage(), 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $orchestration->getProjectId(), 'projectName' => $orchestration->getTokenOwnerName())); continue; } $token = new StorageApi\Token($jobClient); $jobEsManager = $jobManagerFactory->createJobManager($token); $jobsStats = $jobEsManager->getWaitingStats(); // skip waiting if (array_key_exists($orchestration->getId(), $jobsStats) && $jobsStats[$orchestration->getId()] >= AppConfiguration::MAX_WAITING_JOBS_COUNT) { $this->logger->info('scheduler.orchestration.skipped', array('lockName' => $this->lock->getLockName(), 'waitingCount' => $jobsStats[$orchestration->getId()], 'limit' => AppConfiguration::MAX_WAITING_JOBS_COUNT, 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $token->getProjectId(), 'projectName' => $token->getOwnerName())); continue; } $orchestrationWithTasks = $this->dbOrchestrationManager->findOrchestrationById($orchestration->getId(), new StorageApi\Token($jobClient), true); $tasks = array_map(function ($task) { /** @var StorageApi\OrchestrationTask $task */ return $task->toApiArray(); }, $orchestrationWithTasks->getTasks()); $orchestration = $this->dbOrchestrationManager->updateLastScheduledTime($orchestration, new \DateTime()); // save job to queue $job = new Elasticsearch\Job(); $job->setOrchestrationId($orchestration->getId())->setConfig($orchestration->getId())->setOrchestrationName($orchestration->getName())->setToken($orchestration->getToken())->setTokenId($orchestration->getTokenId())->setTokenDesc($orchestration->getTokenDesc())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId())->setInitializedBy('scheduler')->setTaks($tasks)->setInitiatorTokenId($orchestration->getTokenId())->setInitiatorTokenDesc($orchestration->getTokenDesc())->setInitiatorUserAgent($this->appName . ' Scheduler'); $job = $jobEsManager->saveJob($job, new StorageApi\UniqueManager($jobClient)); $newJobs[] = $job; // add job to queue try { if (is_null($job)) { throw new ApplicationException('Unable to save or retrieve job from ES', null, array('orchestrationId' => $orchestration->getId())); } $this->queue->enqueue($job->getId(), array('jobId' => $job->getId(), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME)); $output->writeln(sprintf('Orchestration %d with cron %s scheduled. Job id: %s', $orchestration->getId(), $orchestration->getCrontabRecord(), $job->getId())); } catch (\Exception $e) { $this->logger->error('Error during adding job to sqs', array('lockName' => $this->lock->getLockName(), 'jobId' => $job->getId(), 'exception' => $e)); continue; } // log event try { $event = new Event(); $event->setComponent($this->appName)->setRunId($job->getRunId())->setParams(array('orchestrationId' => $job->getOrchestrationId()))->setResults(array('jobId' => $job->getId()))->setMessage(sprintf('Orchestration job %s scheduled', $job->getId())); StorageApi\EventLogger::create($event, $jobClient); $this->logger->info($event->getMessage(), array('lockName' => $this->lock->getLockName(), 'configurationId' => $event->getConfigurationId(), 'runId' => $event->getRunId(), 'description' => $event->getDescription(), 'params' => $event->getParams(), 'results' => $event->getResults())); } catch (Exception $e) { //@TODO user notification when notification API will be available $this->logger->warning('Invalid token', array('lockName' => $this->lock->getLockName(), 'jobId' => $job->getId(), 'exception' => $e)); } } $this->logger->info('scheduler.end', array('lockName' => $this->lock->getLockName(), 'duration' => $this->getDuration(), 'orchestrationsCheckedCount' => count($orchestrations), 'scheduledJobsCount' => count($newJobs))); sleep(90); // trying fix double run // unlock $this->lock->unlock(); }
private function watchProcessing(InputInterface $input, OutputInterface $output) { $output->writeln(sprintf('<info>%s</info>', 'Check processing jobs')); $orchestrations = array(); $i = 0; while ($jobs = $this->loadProcessingJobs($i)) { foreach ($jobs as $job) { $i++; // check sapi connection if (!$this->sapiPing($job, $this->encryptor)) { $message = 'SAPI connect failed'; $this->logger->info($message, array('job' => $job->getId(), 'project' => $job->getProject(), 'params' => $job->getRawParams())); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } $sapiClient = $this->createJobSapiClient($job); $jobManager = $this->jobManagerFactory->createJobManager(new StorageApi\Token($sapiClient)); try { $esJob = $jobManager->findJobByIdForWatchdog($job->getId()); if (!$esJob) { $message = 'Job load from ES failed'; $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } } catch (\InvalidArgumentException $e) { $message = 'Job load from ES failed'; $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } $orchestrations[$esJob->getOrchestrationId()] = $esJob->getOrchestrationId(); $orchestration = $this->orchestrationManager->findDeletedOrchestrations(array('id' => $esJob->getOrchestrationId()), false); $orchestration = reset($orchestration); if ($orchestration) { $message = 'Orchestration deleted but job is still running'; $this->logger->critical($message, array('job' => $job->getId(), 'project' => $job->getProject(), 'params' => $job->getRawParams())); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[deleted orchestration]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); continue; } $orchestration = $this->orchestrationManager->findOrchestrations(array('id' => $esJob->getOrchestrationId()), false); $orchestration = reset($orchestration); if (!$orchestration) { $message = 'Orchestration load failed'; $this->logger->critical($message, array('job' => $job->getId(), 'orchestrationId' => $esJob->getOrchestrationId(), 'projectId' => $esJob->getProjectId(), 'project' => $esJob->getTokenOwnerName())); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } $componentsList = new KbcComponentsList($sapiClient); // processing duration check if ($esJob->getInitializedBy() === 'manually') { /** * @var StorageApi\Notification[] $notifications */ $notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) { return $row->getChannel() === Metadata\Job::STATUS_PROCESSING; }); if ($notifications) { $durations = array(); foreach ($this->loadOrchestrationSuccessJobs($orchestration, 0, 20) as $computedJob) { $durations[] = (int) $computedJob->getDurationSeconds(); } if (count($durations) < 1) { continue; } foreach ($notifications as $notification) { if ($notification->getEmail() !== $esJob->getInitiatorTokenDesc()) { continue; } $averageDuration = round(array_sum($durations) / count($durations), 2); $diff = $this->startTime->getTimestamp() - $esJob->getCreatedTime()->getTimestamp(); // tolerance AVG if ($diff <= $averageDuration * ((100 + $notification->getParameter('tolerance', 0)) / 100)) { continue; } $events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification, $sapiClient); if ($events) { $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } else { $this->mailer->sendLongProcessingMessage($esJob, $orchestration, $notification, $componentsList, $averageDuration); $this->logLongProcessing($esJob, $sapiClient, $notification, $averageDuration); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } } } } else { /** * @var StorageApi\Notification[] $notifications */ $notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) { return $row->getChannel() === Metadata\Job::STATUS_PROCESSING; }); if ($notifications) { $durations = array(); foreach ($this->loadOrchestrationSuccessJobs($orchestration, 0, 20) as $computedJob) { $durations[] = $computedJob->getDurationSeconds(); } if (count($durations) < 1) { continue; } foreach ($notifications as $notification) { $averageDuration = round(array_sum($durations) / count($durations), 2); $diff = $this->startTime->getTimestamp() - $esJob->getCreatedTime()->getTimestamp(); // tolerance AVG if ($diff <= $averageDuration * ((100 + $notification->getParameter('tolerance', 0)) / 100)) { continue; } $events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification, $sapiClient); if ($events) { $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } else { $this->mailer->sendLongProcessingMessage($esJob, $orchestration, $notification, $componentsList, $averageDuration); $this->logLongProcessing($esJob, $sapiClient, $notification, $averageDuration); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } } } } } } }
protected function execute(InputInterface $input, OutputInterface $output) { parent::execute($input, $output); $this->dbOrchestrationManager = $this->getContainer()->get('orchestrator.doctrine.orchestration_manager'); $skippedCount = 0; $successCount = 0; $errorCount = 0; $force = $input->getOption('force'); $projectId = $input->getArgument('projectId'); $this->logger->info('configuration-migration.start ' . $projectId, array('test' => $force ? true : false)); foreach ($this->loadProjectOrchestrations($projectId) as $orchestration) { try { $jobClient = $this->createProjectSapi($orchestration); $token = new StorageApi\Token($jobClient); if ($token->hasEnabledFeature(StorageApi\Token::ORCHESTRATOR_NEW_CONFIGURATION)) { throw new DisabledException('Migration skipped: new configuration already enabled'); } } catch (DisabledException $e) { $message = $e->getMessage(); if ($e->getPrevious()) { $message = sprintf('%s: %s', $message, $e->getMessage()); } $output->writeln(sprintf('<error>' . $message . '</error>')); $this->logger->info('configuration-migration.failed ' . $projectId, array('test' => $force ? true : false, 'reason' => $message)); $output->writeln(sprintf('<error>Finished with errors</error>')); return JobCommand::STATUS_ERROR; } $orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestration->getId(), $token, true); $components = new Components($jobClient); // remove old configuration $oldConfiguration = null; try { $oldConfiguration = $components->getConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, sprintf("%s-%s", KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $orchestration->getId())); } catch (ClientException $e) { if ($e->getCode() !== 404) { throw $e; } } if ($force) { // create or update configuration $orchestrationTaskManager = new StorageApi\OrchestrationTaskManager($jobClient); $orchestrationTaskManager->updateTasks($orchestration); // validate configuration try { $this->validateConfiguration($orchestration, $jobClient); $output->writeln(sprintf('<info>(%s) %s - Migrated</info>', $orchestration->getId(), $orchestration->getName())); if ($oldConfiguration) { $components->deleteConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $oldConfiguration['id']); $output->writeln(sprintf('<comment>(%s) - Old configuration deleted</comment>', $oldConfiguration['id'])); } $successCount++; } catch (StorageApi\InvalidStateException $e) { $output->writeln(sprintf('<error>' . $e->getMessage() . '</error>')); $errorCount++; } } else { $output->writeln(sprintf('<comment>(%s) %s - Will be migrated</comment>', $orchestration->getId(), $orchestration->getName())); if ($oldConfiguration) { $output->writeln(sprintf('<comment>(%s) - Old configuration will be deleted</comment>', $oldConfiguration['id'])); } $skippedCount++; } } if ($errorCount) { $output->writeln(sprintf('<error>Finished with errors</error>')); } else { $output->writeln(sprintf('<info>Finished</info>')); } $this->logger->info('configuration-migration.migrated ' . $projectId, array('test' => $force ? true : false, 'successCount' => $successCount, 'errorCount' => $errorCount, 'skippedCount' => $skippedCount)); if ($errorCount) { return JobCommand::STATUS_ERROR; } }