/** * Check job in syrup queue * * @param $orchestrationId */ private function sqsScheduledJobTest($orchestrationId) { $assertCount = 1; $requiredStatus = StatusConverter::syrupToOrchestrator(Metadata\Job::STATUS_WAITING); $url = sprintf('/orchestrator/orchestrations/%d/jobs', $orchestrationId); $response = $this->callApiGet($url); $errorMessage = sprintf("Response for call '%s %s' should return only %d jobs.", 'GET', $url, $assertCount); $this->assertCount($assertCount, $response, $errorMessage); $errorMessage = sprintf("Response for call '%s %s' should return list of scheduled jobs.", 'GET', $url); $this->assertArrayHasKey('id', $response[0], $errorMessage); $this->assertArrayHasKey('initializedBy', $response[0], $errorMessage); $this->assertArrayHasKey('status', $response[0], $errorMessage); $this->assertEquals('scheduler', $response[0]['initializedBy'], $errorMessage); $this->assertEquals($requiredStatus, $response[0]['status'], $errorMessage); $this->assertArrayHasKey('isFinished', $response[0]); $this->assertEquals(false, $response[0]['isFinished']); $jobId = $response[0]['id']; $maxTime = 60; $startTime = time(); $errorMessage = sprintf("SQS Message for job '%s' not found.", $jobId); do { foreach ($this->queue->receive() as $message) { /** @var Queue\QueueMessage $message */ $body = $message->getBody(); $errorMessage = 'SQS Message should contains jobId and component name'; $this->assertObjectHasAttribute('jobId', $body, $errorMessage); $this->assertObjectHasAttribute('component', $body, $errorMessage); $errorMessage = 'SQS Message is not from orchestrator'; $this->assertEquals(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $body->component, $errorMessage); $this->queue->deleteMessage($message); if ($body->jobId == $jobId) { $jobId = null; } } } while ($jobId && time() - $startTime < $maxTime); $this->assertEmpty($jobId, $errorMessage); }
public function execute(SyrupJob $job) { parent::build($job); $jobLogger = new JobLogger($this->job, $this->storageApi); try { $jobLogger->start($this->stopwatch->getEvent(self::STOPWATCH_ORCHESTRATION_EVENT)); if ($this->job->getTasks()) { /** @var StorageApi\OrchestrationTask[] $rows */ $orchestrationTasks = array_map(function ($line) { $task = new StorageApi\OrchestrationTask(); $task->fromArray(json_decode(json_encode($line), true)); return $task; }, $this->job->getTasks()); } else { $orchestrationTasks = $this->loadTasks($this->job, $this->token); if (!$this->job->getTasks() && $orchestrationTasks) { $this->logger->info('orchestration.job.emptyTasks', array('jobId' => $this->job->getId(), 'orchestrationId' => $this->job->getOrchestrationId(), 'orchestrationName' => $this->job->getOrchestrationName(), 'projectId' => $this->job->getProjectId())); } } // fill result foreach ($orchestrationTasks as $task) { $this->jobResult->addTaskResult(new TaskResult($task)); } $this->saveJobResult($this->jobResult); foreach ($orchestrationTasks as $task) { // skip inactive taks if (!$task->getActive()) { continue; } try { $taskResult = (new TaskResult($task))->setProcessingStatus(); $this->jobResult->addTaskResult($taskResult); $this->saveJobResult($this->jobResult); $result = $this->executeJobTask($this->job, $taskResult, $this->storageApi); $this->jobResult->addTaskResult($result); $this->saveJobResult($this->jobResult); if (!$task->getContinueOnFailure() && $result->getError()) { break; } if ($task->getContinueOnFailure() && $result->getError()) { continue; } } catch (\Exception $taskE) { //@FIXME better handling = not log to sapi throw Exception\JobRuntimeException::factory($taskE); } } $jobLogger->end($this->jobResult); $status = StatusConverter::orchestratorToSyrup($this->jobResult->getEventType()); $result = array('tasks' => $this->jobResult->toArray()); if ($status === SyrupJob::STATUS_SUCCESS) { return $result; } if ($status === SyrupJob::STATUS_WARNING) { $eData = array('jobId' => $this->job->getId(), 'orchestrationId' => $this->orchestration->getId(), 'projectId' => $this->orchestration->getProjectId()); $e = new JobException(500, sprintf('Some of tasks failed'), null, $eData); $e->setStatus($status)->setResult($result); throw $e; } if ($status === SyrupJob::STATUS_ERROR) { $eData = array('jobId' => $this->job->getId(), 'orchestrationId' => $this->orchestration->getId(), 'projectId' => $this->orchestration->getProjectId()); $e = new JobException(500, sprintf('Job failed', $job->getId()), null, $eData); $e->setStatus($status)->setResult($result); throw $e; } } catch (Exception\JobInvalidStateException $e) { $jobLogger->stateError($e); // any change of job status, job is probably processing } catch (Exception\JobRuntimeException $e) { throw $e->getPrevious(); } return array(); }
/** * @param $jobId * @param Request $request * @return \Symfony\Component\HttpFoundation\JsonResponse */ public function postRetryAction($jobId, Request $request) { $job = $this->jobEsManager->findJobById($jobId); if (!$job) { $exception = new OrchestratorException(404, sprintf('Job %s not found', $jobId)); $exception->setExceptionCode('JOB_NOT_FOUND'); throw $exception; } if (!StatusConverter::isFinishedStatus($job->getStatus())) { $exception = new OrchestratorException(400, sprintf('You can retry only finished jobs', $jobId)); $exception->setExceptionCode('JOB_VALIDATION'); throw $exception; } $orchestrationId = $job->getOrchestrationId(); $orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestrationId, $this->token, true); if (!$orchestration) { $exception = new OrchestratorException(404, sprintf('Orchestration %s not found', $orchestrationId)); $exception->setExceptionCode('ORCHESTRATION_NOT_FOUND'); throw $exception; } try { $form = $this->createForm(new OrchestrationRunType($this->storageApi, $this->dbOrchestrationManager, $this->token, $orchestration)); $handler = $this->createRunFormHandler($form, $request, $orchestration); if ($handler->process()) { if ($handler->getTaskList()) { $tasks = array_map(function ($task) { /** @var StorageApi\OrchestrationTask $task */ $task->setId($this->storageApi->generateId()); return $task->toApiArray(); }, $handler->getTaskList()); } else { $tasks = $job->getTasks(); } } } catch (HttpException $e) { $exception = new OrchestratorException($e->getStatusCode(), $e->getMessage()); $exception->setExceptionCode('JOB_VALIDATION'); throw $exception; } // waiting jobs limit $jobsStats = $this->jobEsManager->getWaitingStats(); // skip waiting if (array_key_exists($orchestration->getId(), $jobsStats) && $jobsStats[$orchestration->getId()] >= AppConfiguration::MAX_WAITING_JOBS_COUNT) { $count = $jobsStats[$orchestration->getId()]; $this->logger->info('scheduler.orchestration.skipped', array('waitingCount' => $count, 'limit' => AppConfiguration::MAX_WAITING_JOBS_COUNT, 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $this->token->getProjectId(), 'projectName' => $this->token->getOwnerName())); if ($count > 1) { $exception = new OrchestratorException(409, sprintf('Orchestration %s has %d waiting jobs. Current limit is %d.', $orchestrationId, $count, AppConfiguration::MAX_WAITING_JOBS_COUNT)); } else { $exception = new OrchestratorException(409, sprintf('Orchestration %s has %d waiting job. Current limit is %d.', $orchestrationId, $count, AppConfiguration::MAX_WAITING_JOBS_COUNT)); } $exception->setExceptionCode('ORCHESTRATION_VALIDATION'); throw $exception; } $form = $this->createForm(new ScheduleJobType($this->storageApi, $this->dbOrchestrationManager, $this->token)); $handler = parent::createFormHandler($form, $request); try { $notificationsEmails = array(); if ($handler->process()) { $notificationsEmails = $handler->getPost('notificationsEmails', array()); } if (!$notificationsEmails && \Swift_Validate::email($this->token->getDescription())) { $notificationsEmails = array($this->token->getDescription()); } } catch (HttpException $e) { $exception = new OrchestratorException(400, $e->getMessage()); $exception->setExceptionCode('JOB_VALIDATION'); throw $exception; } $this->initSqsQueue(); $job = new Elasticsearch\Job(); $job->setConfig($orchestration->getId())->setOrchestrationId($orchestration->getId())->setOrchestrationName($orchestration->getName())->setToken($orchestration->getToken())->setTokenId($orchestration->getTokenId())->setTokenDesc($orchestration->getTokenDesc())->setTokenOwnerName($this->token->getOwnerName())->setProjectId($this->token->getProjectId())->setInitializedBy('manually')->setInitiatorTokenId($this->token->getId())->setInitiatorTokenDesc($this->token->getDescription())->setInitiatorUserAgent($this->getRequestUserAgent($request))->setNotificationsEmails($notificationsEmails)->setTaks($tasks); $job = $this->jobEsManager->saveJob($job, new StorageApi\UniqueManager($this->storageApi)); $this->queue->enqueue($job->getId(), array('jobId' => $job->getId(), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME)); $this->logger->info(sprintf('Orchestration job %s created manually', $job->getId())); return $this->createJsonResponse($job->toOldApiArray(), 201); }
public function testExecutionError() { $token1 = $this->createNewToken($this->storageApi, 'manage'); $orchestrationId = $this->createOrchestrationTest($token1); $actionParameters = array(array(), array('sleep' => 3)); $this->putOrchestrationErrorTaskTest($orchestrationId, $actionParameters); $jobId = $this->runOrchestrationTest($orchestrationId); $this->runCommandTest($jobId); $url = '/orchestrator/jobs/' . $jobId; $response = $this->callApiGet($url); $this->assertArrayHasKey('id', $response); $this->assertEquals($jobId, $response['id']); $this->assertArrayHasKey('status', $response); $this->assertEquals(StatusConverter::syrupToOrchestrator(Metadata\Job::STATUS_ERROR), $response['status']); $this->assertArrayHasKey('isFinished', $response); $this->assertEquals(true, $response['isFinished']); $this->assertArrayHasKey('initializedBy', $response); $this->assertEquals('manually', $response['initializedBy']); $this->assertArrayHasKey('results', $response); $this->assertArrayHasKey('tasks', $response['results']); $tasksResult = $response['results']['tasks']; // task 1 validation $this->assertArrayHasKey('response', $tasksResult[0]); $this->assertArrayHasKey('responseCode', $tasksResult[0]); $this->assertArrayHasKey('actionParameters', $tasksResult[0]); $this->assertEquals(400, $tasksResult[0]['responseCode']); $this->assertEquals($actionParameters[0], $tasksResult[0]['actionParameters']); $this->assertArrayHasKey('code', $tasksResult[0]['response']); $this->assertArrayHasKey('status', $tasksResult[0]); $this->assertEquals(StatusConverter::syrupToOrchestrator(Metadata\Job::STATUS_ERROR), $tasksResult[0]['status']); // task 2 validation $this->assertArrayHasKey('startTime', $tasksResult[1]); $this->assertArrayHasKey('actionParameters', $tasksResult[1]); $this->assertEquals($actionParameters[1], $tasksResult[1]['actionParameters']); $this->assertArrayHasKey('startTime', $tasksResult[1]); $this->assertArrayHasKey('status', $tasksResult[1]); $this->assertEmpty($tasksResult[1]['startTime']); $this->assertEmpty($tasksResult[1]['status']); }
public function toApiArray() { return array('id' => $this->getId(), 'orchestrationId' => $this->getOrchestrationId(), 'createdTime' => $this->getCreatedTime() ? $this->getCreatedTime()->format('c') : null, 'startTime' => $this->getStartTime() ? $this->getStartTime()->format('c') : null, 'endTime' => $this->getEndTime() ? $this->getEndTime()->format('c') : null, 'status' => StatusConverter::syrupToOrchestrator($this->getStatus()), 'isFinished' => StatusConverter::isFinishedStatus(StatusConverter::syrupToOrchestrator($this->getStatus())), 'initializedBy' => $this->getInitializedBy(), 'token' => array('id' => $this->getTokenId(), 'description' => $this->getTokenDesc()), 'initiatorToken' => array('id' => $this->getInitiatorTokenId(), 'description' => $this->getInitiatorTokenDesc(), 'userAgent' => $this->getInitiatorUserAgent()), 'tasks' => $this->getTasks(), 'results' => $this->getResults(), 'notificationsEmails' => $this->getNotificationsEmails(), 'runId' => $this->getRunId(), 'url' => $this->getUrl()); }
public function execute(SyrupJob $job) { parent::build($job); //@FIXME remove debug logging from info channel $jobLogger = new JobLogger($this->job, $this->storageApi); try { $jobLogger->start($this->stopwatch->getEvent(self::STOPWATCH_ORCHESTRATION_EVENT)); $phases = $this->createPhases($this->job); $this->saveJobResult($this->jobResult); $httpClient = new Client(array(), $this->logger); foreach ($phases as $phase) { $phaseLogger = new PhaseLogger($phase, $this->storageApi); $startTime = time(); $jobResponses = $this->startPhase($phase, $phaseLogger, $httpClient); if ($jobResponses === false) { break; } $pollResponses = $this->pollPhase($phase, $phaseLogger, $httpClient, $jobResponses); if ($pollResponses === false) { break; } } $endEvent = $jobLogger->end($this->jobResult); $status = StatusConverter::orchestratorToSyrup($this->jobResult->getEventType()); $result = array('tasks' => $this->jobResult->toArray()); if ($status === SyrupJob::STATUS_SUCCESS) { return $result; } if ($status === SyrupJob::STATUS_WARNING) { $eData = array('jobId' => $this->job->getId(), 'orchestrationId' => $this->orchestration->getId(), 'projectId' => $this->orchestration->getProjectId()); $e = new JobException(500, sprintf('Some of tasks failed'), null, $eData); $e->setStatus($status)->setResult($result); throw $e; } if ($status === SyrupJob::STATUS_ERROR) { $eData = array('jobId' => $this->job->getId(), 'orchestrationId' => $this->orchestration->getId(), 'projectId' => $this->orchestration->getProjectId()); $e = new JobException(500, sprintf('Job failed', $job->getId()), null, $eData); $e->setStatus($status)->setResult($result); throw $e; } } catch (Exception\JobInvalidStateException $e) { $endEvent = $jobLogger->stateError($e); // any change of job status, job is probably processing } catch (Exception\JobRuntimeException $e) { throw $e->getPrevious(); } return array(); }