/**
* Check job in syrup queue
*
* @param $orchestrationId
*/
private function sqsScheduledJobTest($orchestrationId)
{
$assertCount = 1;
$requiredStatus = StatusConverter::syrupToOrchestrator(Metadata\Job::STATUS_WAITING);
$url = sprintf('/orchestrator/orchestrations/%d/jobs', $orchestrationId);
$response = $this->callApiGet($url);
$errorMessage = sprintf("Response for call '%s %s' should return only %d jobs.", 'GET', $url, $assertCount);
$this->assertCount($assertCount, $response, $errorMessage);
$errorMessage = sprintf("Response for call '%s %s' should return list of scheduled jobs.", 'GET', $url);
$this->assertArrayHasKey('id', $response[0], $errorMessage);
$this->assertArrayHasKey('initializedBy', $response[0], $errorMessage);
$this->assertArrayHasKey('status', $response[0], $errorMessage);
$this->assertEquals('scheduler', $response[0]['initializedBy'], $errorMessage);
$this->assertEquals($requiredStatus, $response[0]['status'], $errorMessage);
$this->assertArrayHasKey('isFinished', $response[0]);
$this->assertEquals(false, $response[0]['isFinished']);
$jobId = $response[0]['id'];
$maxTime = 60;
$startTime = time();
$errorMessage = sprintf("SQS Message for job '%s' not found.", $jobId);
do {
foreach ($this->queue->receive() as $message) {
/** @var Queue\QueueMessage $message */
$body = $message->getBody();
$errorMessage = 'SQS Message should contains jobId and component name';
$this->assertObjectHasAttribute('jobId', $body, $errorMessage);
$this->assertObjectHasAttribute('component', $body, $errorMessage);
$errorMessage = 'SQS Message is not from orchestrator';
$this->assertEquals(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $body->component, $errorMessage);
$this->queue->deleteMessage($message);
if ($body->jobId == $jobId) {
$jobId = null;
}
}
} while ($jobId && time() - $startTime < $maxTime);
$this->assertEmpty($jobId, $errorMessage);
}
public function execute(SyrupJob $job)
{
parent::build($job);
$jobLogger = new JobLogger($this->job, $this->storageApi);
try {
$jobLogger->start($this->stopwatch->getEvent(self::STOPWATCH_ORCHESTRATION_EVENT));
if ($this->job->getTasks()) {
/** @var StorageApi\OrchestrationTask[] $rows */
$orchestrationTasks = array_map(function ($line) {
$task = new StorageApi\OrchestrationTask();
$task->fromArray(json_decode(json_encode($line), true));
return $task;
}, $this->job->getTasks());
} else {
$orchestrationTasks = $this->loadTasks($this->job, $this->token);
if (!$this->job->getTasks() && $orchestrationTasks) {
$this->logger->info('orchestration.job.emptyTasks', array('jobId' => $this->job->getId(), 'orchestrationId' => $this->job->getOrchestrationId(), 'orchestrationName' => $this->job->getOrchestrationName(), 'projectId' => $this->job->getProjectId()));
}
}
// fill result
foreach ($orchestrationTasks as $task) {
$this->jobResult->addTaskResult(new TaskResult($task));
}
$this->saveJobResult($this->jobResult);
foreach ($orchestrationTasks as $task) {
// skip inactive taks
if (!$task->getActive()) {
continue;
}
try {
$taskResult = (new TaskResult($task))->setProcessingStatus();
$this->jobResult->addTaskResult($taskResult);
$this->saveJobResult($this->jobResult);
$result = $this->executeJobTask($this->job, $taskResult, $this->storageApi);
$this->jobResult->addTaskResult($result);
$this->saveJobResult($this->jobResult);
if (!$task->getContinueOnFailure() && $result->getError()) {
break;
}
if ($task->getContinueOnFailure() && $result->getError()) {
continue;
}
} catch (\Exception $taskE) {
//@FIXME better handling = not log to sapi
throw Exception\JobRuntimeException::factory($taskE);
}
}
$jobLogger->end($this->jobResult);
$status = StatusConverter::orchestratorToSyrup($this->jobResult->getEventType());
$result = array('tasks' => $this->jobResult->toArray());
if ($status === SyrupJob::STATUS_SUCCESS) {
return $result;
}
if ($status === SyrupJob::STATUS_WARNING) {
$eData = array('jobId' => $this->job->getId(), 'orchestrationId' => $this->orchestration->getId(), 'projectId' => $this->orchestration->getProjectId());
$e = new JobException(500, sprintf('Some of tasks failed'), null, $eData);
$e->setStatus($status)->setResult($result);
throw $e;
}
if ($status === SyrupJob::STATUS_ERROR) {
$eData = array('jobId' => $this->job->getId(), 'orchestrationId' => $this->orchestration->getId(), 'projectId' => $this->orchestration->getProjectId());
$e = new JobException(500, sprintf('Job failed', $job->getId()), null, $eData);
$e->setStatus($status)->setResult($result);
throw $e;
}
} catch (Exception\JobInvalidStateException $e) {
$jobLogger->stateError($e);
// any change of job status, job is probably processing
} catch (Exception\JobRuntimeException $e) {
throw $e->getPrevious();
}
return array();
}
/**
* @param $jobId
* @param Request $request
* @return \Symfony\Component\HttpFoundation\JsonResponse
*/
public function postRetryAction($jobId, Request $request)
{
$job = $this->jobEsManager->findJobById($jobId);
if (!$job) {
$exception = new OrchestratorException(404, sprintf('Job %s not found', $jobId));
$exception->setExceptionCode('JOB_NOT_FOUND');
throw $exception;
}
if (!StatusConverter::isFinishedStatus($job->getStatus())) {
$exception = new OrchestratorException(400, sprintf('You can retry only finished jobs', $jobId));
$exception->setExceptionCode('JOB_VALIDATION');
throw $exception;
}
$orchestrationId = $job->getOrchestrationId();
$orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestrationId, $this->token, true);
if (!$orchestration) {
$exception = new OrchestratorException(404, sprintf('Orchestration %s not found', $orchestrationId));
$exception->setExceptionCode('ORCHESTRATION_NOT_FOUND');
throw $exception;
}
try {
$form = $this->createForm(new OrchestrationRunType($this->storageApi, $this->dbOrchestrationManager, $this->token, $orchestration));
$handler = $this->createRunFormHandler($form, $request, $orchestration);
if ($handler->process()) {
if ($handler->getTaskList()) {
$tasks = array_map(function ($task) {
/** @var StorageApi\OrchestrationTask $task */
$task->setId($this->storageApi->generateId());
return $task->toApiArray();
}, $handler->getTaskList());
} else {
$tasks = $job->getTasks();
}
}
} catch (HttpException $e) {
$exception = new OrchestratorException($e->getStatusCode(), $e->getMessage());
$exception->setExceptionCode('JOB_VALIDATION');
throw $exception;
}
// waiting jobs limit
$jobsStats = $this->jobEsManager->getWaitingStats();
// skip waiting
if (array_key_exists($orchestration->getId(), $jobsStats) && $jobsStats[$orchestration->getId()] >= AppConfiguration::MAX_WAITING_JOBS_COUNT) {
$count = $jobsStats[$orchestration->getId()];
$this->logger->info('scheduler.orchestration.skipped', array('waitingCount' => $count, 'limit' => AppConfiguration::MAX_WAITING_JOBS_COUNT, 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $this->token->getProjectId(), 'projectName' => $this->token->getOwnerName()));
if ($count > 1) {
$exception = new OrchestratorException(409, sprintf('Orchestration %s has %d waiting jobs. Current limit is %d.', $orchestrationId, $count, AppConfiguration::MAX_WAITING_JOBS_COUNT));
} else {
$exception = new OrchestratorException(409, sprintf('Orchestration %s has %d waiting job. Current limit is %d.', $orchestrationId, $count, AppConfiguration::MAX_WAITING_JOBS_COUNT));
}
$exception->setExceptionCode('ORCHESTRATION_VALIDATION');
throw $exception;
}
$form = $this->createForm(new ScheduleJobType($this->storageApi, $this->dbOrchestrationManager, $this->token));
$handler = parent::createFormHandler($form, $request);
try {
$notificationsEmails = array();
if ($handler->process()) {
$notificationsEmails = $handler->getPost('notificationsEmails', array());
}
if (!$notificationsEmails && \Swift_Validate::email($this->token->getDescription())) {
$notificationsEmails = array($this->token->getDescription());
}
} catch (HttpException $e) {
$exception = new OrchestratorException(400, $e->getMessage());
$exception->setExceptionCode('JOB_VALIDATION');
throw $exception;
}
$this->initSqsQueue();
$job = new Elasticsearch\Job();
$job->setConfig($orchestration->getId())->setOrchestrationId($orchestration->getId())->setOrchestrationName($orchestration->getName())->setToken($orchestration->getToken())->setTokenId($orchestration->getTokenId())->setTokenDesc($orchestration->getTokenDesc())->setTokenOwnerName($this->token->getOwnerName())->setProjectId($this->token->getProjectId())->setInitializedBy('manually')->setInitiatorTokenId($this->token->getId())->setInitiatorTokenDesc($this->token->getDescription())->setInitiatorUserAgent($this->getRequestUserAgent($request))->setNotificationsEmails($notificationsEmails)->setTaks($tasks);
$job = $this->jobEsManager->saveJob($job, new StorageApi\UniqueManager($this->storageApi));
$this->queue->enqueue($job->getId(), array('jobId' => $job->getId(), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME));
$this->logger->info(sprintf('Orchestration job %s created manually', $job->getId()));
return $this->createJsonResponse($job->toOldApiArray(), 201);
}
public function testExecutionError()
{
$token1 = $this->createNewToken($this->storageApi, 'manage');
$orchestrationId = $this->createOrchestrationTest($token1);
$actionParameters = array(array(), array('sleep' => 3));
$this->putOrchestrationErrorTaskTest($orchestrationId, $actionParameters);
$jobId = $this->runOrchestrationTest($orchestrationId);
$this->runCommandTest($jobId);
$url = '/orchestrator/jobs/' . $jobId;
$response = $this->callApiGet($url);
$this->assertArrayHasKey('id', $response);
$this->assertEquals($jobId, $response['id']);
$this->assertArrayHasKey('status', $response);
$this->assertEquals(StatusConverter::syrupToOrchestrator(Metadata\Job::STATUS_ERROR), $response['status']);
$this->assertArrayHasKey('isFinished', $response);
$this->assertEquals(true, $response['isFinished']);
$this->assertArrayHasKey('initializedBy', $response);
$this->assertEquals('manually', $response['initializedBy']);
$this->assertArrayHasKey('results', $response);
$this->assertArrayHasKey('tasks', $response['results']);
$tasksResult = $response['results']['tasks'];
// task 1 validation
$this->assertArrayHasKey('response', $tasksResult[0]);
$this->assertArrayHasKey('responseCode', $tasksResult[0]);
$this->assertArrayHasKey('actionParameters', $tasksResult[0]);
$this->assertEquals(400, $tasksResult[0]['responseCode']);
$this->assertEquals($actionParameters[0], $tasksResult[0]['actionParameters']);
$this->assertArrayHasKey('code', $tasksResult[0]['response']);
$this->assertArrayHasKey('status', $tasksResult[0]);
$this->assertEquals(StatusConverter::syrupToOrchestrator(Metadata\Job::STATUS_ERROR), $tasksResult[0]['status']);
// task 2 validation
$this->assertArrayHasKey('startTime', $tasksResult[1]);
$this->assertArrayHasKey('actionParameters', $tasksResult[1]);
$this->assertEquals($actionParameters[1], $tasksResult[1]['actionParameters']);
$this->assertArrayHasKey('startTime', $tasksResult[1]);
$this->assertArrayHasKey('status', $tasksResult[1]);
$this->assertEmpty($tasksResult[1]['startTime']);
$this->assertEmpty($tasksResult[1]['status']);
}
public function toApiArray()
{
return array('id' => $this->getId(), 'orchestrationId' => $this->getOrchestrationId(), 'createdTime' => $this->getCreatedTime() ? $this->getCreatedTime()->format('c') : null, 'startTime' => $this->getStartTime() ? $this->getStartTime()->format('c') : null, 'endTime' => $this->getEndTime() ? $this->getEndTime()->format('c') : null, 'status' => StatusConverter::syrupToOrchestrator($this->getStatus()), 'isFinished' => StatusConverter::isFinishedStatus(StatusConverter::syrupToOrchestrator($this->getStatus())), 'initializedBy' => $this->getInitializedBy(), 'token' => array('id' => $this->getTokenId(), 'description' => $this->getTokenDesc()), 'initiatorToken' => array('id' => $this->getInitiatorTokenId(), 'description' => $this->getInitiatorTokenDesc(), 'userAgent' => $this->getInitiatorUserAgent()), 'tasks' => $this->getTasks(), 'results' => $this->getResults(), 'notificationsEmails' => $this->getNotificationsEmails(), 'runId' => $this->getRunId(), 'url' => $this->getUrl());
}
public function execute(SyrupJob $job)
{
parent::build($job);
//@FIXME remove debug logging from info channel
$jobLogger = new JobLogger($this->job, $this->storageApi);
try {
$jobLogger->start($this->stopwatch->getEvent(self::STOPWATCH_ORCHESTRATION_EVENT));
$phases = $this->createPhases($this->job);
$this->saveJobResult($this->jobResult);
$httpClient = new Client(array(), $this->logger);
foreach ($phases as $phase) {
$phaseLogger = new PhaseLogger($phase, $this->storageApi);
$startTime = time();
$jobResponses = $this->startPhase($phase, $phaseLogger, $httpClient);
if ($jobResponses === false) {
break;
}
$pollResponses = $this->pollPhase($phase, $phaseLogger, $httpClient, $jobResponses);
if ($pollResponses === false) {
break;
}
}
$endEvent = $jobLogger->end($this->jobResult);
$status = StatusConverter::orchestratorToSyrup($this->jobResult->getEventType());
$result = array('tasks' => $this->jobResult->toArray());
if ($status === SyrupJob::STATUS_SUCCESS) {
return $result;
}
if ($status === SyrupJob::STATUS_WARNING) {
$eData = array('jobId' => $this->job->getId(), 'orchestrationId' => $this->orchestration->getId(), 'projectId' => $this->orchestration->getProjectId());
$e = new JobException(500, sprintf('Some of tasks failed'), null, $eData);
$e->setStatus($status)->setResult($result);
throw $e;
}
if ($status === SyrupJob::STATUS_ERROR) {
$eData = array('jobId' => $this->job->getId(), 'orchestrationId' => $this->orchestration->getId(), 'projectId' => $this->orchestration->getProjectId());
$e = new JobException(500, sprintf('Job failed', $job->getId()), null, $eData);
$e->setStatus($status)->setResult($result);
throw $e;
}
} catch (Exception\JobInvalidStateException $e) {
$endEvent = $jobLogger->stateError($e);
// any change of job status, job is probably processing
} catch (Exception\JobRuntimeException $e) {
throw $e->getPrevious();
}
return array();
}