/** * @param Entity\Orchestration $orchestration * @param Token $token * @return OrchestrationTask[] * @throws \InvalidArgumentException */ public function findTasks(Entity\Orchestration $orchestration, Token $token) { $clientToken = new Token($this->storageApi); if ($clientToken->getProjectId() !== $orchestration->getProjectId()) { throw new \InvalidArgumentException('Given orchestration does not belong to requested tasks'); } if ($clientToken->getProjectId() !== $token->getProjectId()) { throw new \InvalidArgumentException('Given orchestration does not belong to requested tasks'); } $data = $this->components->getConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $orchestration->getId()); if (empty($data['configuration']['tasks'])) { return array(); } /** @var OrchestrationTask[] $rows */ $rows = array_map(function ($line) use($token) { $task = new OrchestrationTask($token); $task->fromArray($line); return $task; }, $data['configuration']['tasks']); return $this->sortTasks($rows); }
public function jobsCount(array $criteria, ElasticsearchClient $client, array $config) { $exportOptions = $this->buildExportOptions($criteria); $component = 'orchestrator'; $filter = []; $filter[] = ['term' => ['project.id' => $this->token->getProjectId()]]; $query = ['match_all' => []]; if ($exportOptions != null) { $query = ['query_string' => ['allow_leading_wildcard' => 'false', 'default_operator' => 'AND', 'query' => $exportOptions]]; } $params = []; $params['index'] = $config['index_prefix'] . '_syrup_*'; if (!is_null($component)) { $params['index'] = $config['index_prefix'] . '_syrup_' . $component; } $params['body'] = ['query' => ['filtered' => ['filter' => ['bool' => ['must' => $filter]], 'query' => $query]]]; $count = $client->count($params); return $count['count']; }
public function postCreateTokenAction($orchestrationId) { $orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestrationId, $this->token); if (!$orchestration) { $exception = new OrchestratorException(404, sprintf('Orchestration %s not found', $orchestrationId)); $exception->setExceptionCode('ORCHESTRATION_NOT_FOUND'); throw $exception; } try { $tokenId = $this->storageApi->createToken('manage', sprintf('Orchestrator %s', $orchestration->getName()), null, true); // fill token data $token = $this->storageApi->getToken($tokenId); $token = new StorageApi\Token(new Client(array('token' => $token['token'], 'url' => $this->storageApi->getApiUrl()))); $orchestration->setToken((string) $token)->setTokenId($token->getId())->setTokenDesc($token->getDescription())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId()); } catch (\Exception $e) { $exception = new OrchestratorException(500, sprintf('Could not create new token'), $e); $exception->setExceptionCode('TOKEN_VALIDATION'); throw $exception; } $orchestration = $this->dbOrchestrationManager->updateOrchestration($orchestration); $this->logger->info('Orchestration token changed'); $data = $orchestration->toApiArray(); return $this->createJsonResponse($data['token'], 201); }
/** * @param $jobId * @param Request $request * @return \Symfony\Component\HttpFoundation\JsonResponse */ public function postRetryAction($jobId, Request $request) { $job = $this->jobEsManager->findJobById($jobId); if (!$job) { $exception = new OrchestratorException(404, sprintf('Job %s not found', $jobId)); $exception->setExceptionCode('JOB_NOT_FOUND'); throw $exception; } if (!StatusConverter::isFinishedStatus($job->getStatus())) { $exception = new OrchestratorException(400, sprintf('You can retry only finished jobs', $jobId)); $exception->setExceptionCode('JOB_VALIDATION'); throw $exception; } $orchestrationId = $job->getOrchestrationId(); $orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestrationId, $this->token, true); if (!$orchestration) { $exception = new OrchestratorException(404, sprintf('Orchestration %s not found', $orchestrationId)); $exception->setExceptionCode('ORCHESTRATION_NOT_FOUND'); throw $exception; } try { $form = $this->createForm(new OrchestrationRunType($this->storageApi, $this->dbOrchestrationManager, $this->token, $orchestration)); $handler = $this->createRunFormHandler($form, $request, $orchestration); if ($handler->process()) { if ($handler->getTaskList()) { $tasks = array_map(function ($task) { /** @var StorageApi\OrchestrationTask $task */ $task->setId($this->storageApi->generateId()); return $task->toApiArray(); }, $handler->getTaskList()); } else { $tasks = $job->getTasks(); } } } catch (HttpException $e) { $exception = new OrchestratorException($e->getStatusCode(), $e->getMessage()); $exception->setExceptionCode('JOB_VALIDATION'); throw $exception; } // waiting jobs limit $jobsStats = $this->jobEsManager->getWaitingStats(); // skip waiting if (array_key_exists($orchestration->getId(), $jobsStats) && $jobsStats[$orchestration->getId()] >= AppConfiguration::MAX_WAITING_JOBS_COUNT) { $count = $jobsStats[$orchestration->getId()]; $this->logger->info('scheduler.orchestration.skipped', array('waitingCount' => $count, 'limit' => AppConfiguration::MAX_WAITING_JOBS_COUNT, 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $this->token->getProjectId(), 'projectName' => $this->token->getOwnerName())); if ($count > 1) { $exception = new OrchestratorException(409, sprintf('Orchestration %s has %d waiting jobs. Current limit is %d.', $orchestrationId, $count, AppConfiguration::MAX_WAITING_JOBS_COUNT)); } else { $exception = new OrchestratorException(409, sprintf('Orchestration %s has %d waiting job. Current limit is %d.', $orchestrationId, $count, AppConfiguration::MAX_WAITING_JOBS_COUNT)); } $exception->setExceptionCode('ORCHESTRATION_VALIDATION'); throw $exception; } $form = $this->createForm(new ScheduleJobType($this->storageApi, $this->dbOrchestrationManager, $this->token)); $handler = parent::createFormHandler($form, $request); try { $notificationsEmails = array(); if ($handler->process()) { $notificationsEmails = $handler->getPost('notificationsEmails', array()); } if (!$notificationsEmails && \Swift_Validate::email($this->token->getDescription())) { $notificationsEmails = array($this->token->getDescription()); } } catch (HttpException $e) { $exception = new OrchestratorException(400, $e->getMessage()); $exception->setExceptionCode('JOB_VALIDATION'); throw $exception; } $this->initSqsQueue(); $job = new Elasticsearch\Job(); $job->setConfig($orchestration->getId())->setOrchestrationId($orchestration->getId())->setOrchestrationName($orchestration->getName())->setToken($orchestration->getToken())->setTokenId($orchestration->getTokenId())->setTokenDesc($orchestration->getTokenDesc())->setTokenOwnerName($this->token->getOwnerName())->setProjectId($this->token->getProjectId())->setInitializedBy('manually')->setInitiatorTokenId($this->token->getId())->setInitiatorTokenDesc($this->token->getDescription())->setInitiatorUserAgent($this->getRequestUserAgent($request))->setNotificationsEmails($notificationsEmails)->setTaks($tasks); $job = $this->jobEsManager->saveJob($job, new StorageApi\UniqueManager($this->storageApi)); $this->queue->enqueue($job->getId(), array('jobId' => $job->getId(), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME)); $this->logger->info(sprintf('Orchestration job %s created manually', $job->getId())); return $this->createJsonResponse($job->toOldApiArray(), 201); }