/**
* @param Entity\Orchestration $orchestration
* @param Token $token
* @return OrchestrationTask[]
* @throws \InvalidArgumentException
*/
public function findTasks(Entity\Orchestration $orchestration, Token $token)
{
$clientToken = new Token($this->storageApi);
if ($clientToken->getProjectId() !== $orchestration->getProjectId()) {
throw new \InvalidArgumentException('Given orchestration does not belong to requested tasks');
}
if ($clientToken->getProjectId() !== $token->getProjectId()) {
throw new \InvalidArgumentException('Given orchestration does not belong to requested tasks');
}
$data = $this->components->getConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $orchestration->getId());
if (empty($data['configuration']['tasks'])) {
return array();
}
/** @var OrchestrationTask[] $rows */
$rows = array_map(function ($line) use($token) {
$task = new OrchestrationTask($token);
$task->fromArray($line);
return $task;
}, $data['configuration']['tasks']);
return $this->sortTasks($rows);
}
public function jobsCount(array $criteria, ElasticsearchClient $client, array $config)
{
$exportOptions = $this->buildExportOptions($criteria);
$component = 'orchestrator';
$filter = [];
$filter[] = ['term' => ['project.id' => $this->token->getProjectId()]];
$query = ['match_all' => []];
if ($exportOptions != null) {
$query = ['query_string' => ['allow_leading_wildcard' => 'false', 'default_operator' => 'AND', 'query' => $exportOptions]];
}
$params = [];
$params['index'] = $config['index_prefix'] . '_syrup_*';
if (!is_null($component)) {
$params['index'] = $config['index_prefix'] . '_syrup_' . $component;
}
$params['body'] = ['query' => ['filtered' => ['filter' => ['bool' => ['must' => $filter]], 'query' => $query]]];
$count = $client->count($params);
return $count['count'];
}
public function postCreateTokenAction($orchestrationId)
{
$orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestrationId, $this->token);
if (!$orchestration) {
$exception = new OrchestratorException(404, sprintf('Orchestration %s not found', $orchestrationId));
$exception->setExceptionCode('ORCHESTRATION_NOT_FOUND');
throw $exception;
}
try {
$tokenId = $this->storageApi->createToken('manage', sprintf('Orchestrator %s', $orchestration->getName()), null, true);
// fill token data
$token = $this->storageApi->getToken($tokenId);
$token = new StorageApi\Token(new Client(array('token' => $token['token'], 'url' => $this->storageApi->getApiUrl())));
$orchestration->setToken((string) $token)->setTokenId($token->getId())->setTokenDesc($token->getDescription())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId());
} catch (\Exception $e) {
$exception = new OrchestratorException(500, sprintf('Could not create new token'), $e);
$exception->setExceptionCode('TOKEN_VALIDATION');
throw $exception;
}
$orchestration = $this->dbOrchestrationManager->updateOrchestration($orchestration);
$this->logger->info('Orchestration token changed');
$data = $orchestration->toApiArray();
return $this->createJsonResponse($data['token'], 201);
}
/**
* @param $jobId
* @param Request $request
* @return \Symfony\Component\HttpFoundation\JsonResponse
*/
public function postRetryAction($jobId, Request $request)
{
$job = $this->jobEsManager->findJobById($jobId);
if (!$job) {
$exception = new OrchestratorException(404, sprintf('Job %s not found', $jobId));
$exception->setExceptionCode('JOB_NOT_FOUND');
throw $exception;
}
if (!StatusConverter::isFinishedStatus($job->getStatus())) {
$exception = new OrchestratorException(400, sprintf('You can retry only finished jobs', $jobId));
$exception->setExceptionCode('JOB_VALIDATION');
throw $exception;
}
$orchestrationId = $job->getOrchestrationId();
$orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestrationId, $this->token, true);
if (!$orchestration) {
$exception = new OrchestratorException(404, sprintf('Orchestration %s not found', $orchestrationId));
$exception->setExceptionCode('ORCHESTRATION_NOT_FOUND');
throw $exception;
}
try {
$form = $this->createForm(new OrchestrationRunType($this->storageApi, $this->dbOrchestrationManager, $this->token, $orchestration));
$handler = $this->createRunFormHandler($form, $request, $orchestration);
if ($handler->process()) {
if ($handler->getTaskList()) {
$tasks = array_map(function ($task) {
/** @var StorageApi\OrchestrationTask $task */
$task->setId($this->storageApi->generateId());
return $task->toApiArray();
}, $handler->getTaskList());
} else {
$tasks = $job->getTasks();
}
}
} catch (HttpException $e) {
$exception = new OrchestratorException($e->getStatusCode(), $e->getMessage());
$exception->setExceptionCode('JOB_VALIDATION');
throw $exception;
}
// waiting jobs limit
$jobsStats = $this->jobEsManager->getWaitingStats();
// skip waiting
if (array_key_exists($orchestration->getId(), $jobsStats) && $jobsStats[$orchestration->getId()] >= AppConfiguration::MAX_WAITING_JOBS_COUNT) {
$count = $jobsStats[$orchestration->getId()];
$this->logger->info('scheduler.orchestration.skipped', array('waitingCount' => $count, 'limit' => AppConfiguration::MAX_WAITING_JOBS_COUNT, 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $this->token->getProjectId(), 'projectName' => $this->token->getOwnerName()));
if ($count > 1) {
$exception = new OrchestratorException(409, sprintf('Orchestration %s has %d waiting jobs. Current limit is %d.', $orchestrationId, $count, AppConfiguration::MAX_WAITING_JOBS_COUNT));
} else {
$exception = new OrchestratorException(409, sprintf('Orchestration %s has %d waiting job. Current limit is %d.', $orchestrationId, $count, AppConfiguration::MAX_WAITING_JOBS_COUNT));
}
$exception->setExceptionCode('ORCHESTRATION_VALIDATION');
throw $exception;
}
$form = $this->createForm(new ScheduleJobType($this->storageApi, $this->dbOrchestrationManager, $this->token));
$handler = parent::createFormHandler($form, $request);
try {
$notificationsEmails = array();
if ($handler->process()) {
$notificationsEmails = $handler->getPost('notificationsEmails', array());
}
if (!$notificationsEmails && \Swift_Validate::email($this->token->getDescription())) {
$notificationsEmails = array($this->token->getDescription());
}
} catch (HttpException $e) {
$exception = new OrchestratorException(400, $e->getMessage());
$exception->setExceptionCode('JOB_VALIDATION');
throw $exception;
}
$this->initSqsQueue();
$job = new Elasticsearch\Job();
$job->setConfig($orchestration->getId())->setOrchestrationId($orchestration->getId())->setOrchestrationName($orchestration->getName())->setToken($orchestration->getToken())->setTokenId($orchestration->getTokenId())->setTokenDesc($orchestration->getTokenDesc())->setTokenOwnerName($this->token->getOwnerName())->setProjectId($this->token->getProjectId())->setInitializedBy('manually')->setInitiatorTokenId($this->token->getId())->setInitiatorTokenDesc($this->token->getDescription())->setInitiatorUserAgent($this->getRequestUserAgent($request))->setNotificationsEmails($notificationsEmails)->setTaks($tasks);
$job = $this->jobEsManager->saveJob($job, new StorageApi\UniqueManager($this->storageApi));
$this->queue->enqueue($job->getId(), array('jobId' => $job->getId(), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME));
$this->logger->info(sprintf('Orchestration job %s created manually', $job->getId()));
return $this->createJsonResponse($job->toOldApiArray(), 201);
}
