/**
* @param Elasticsearch\Job $job
* @param Token $token
* @return OrchestrationTask[]
*/
public function findTasksByJob(Elasticsearch\Job $job, Token $token)
{
$data = $this->components->getConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, (new Entity\Orchestration())->setId($job->getOrchestrationId())->getId());
if (empty($data['configuration']['tasks'])) {
return array();
}
$data = $data['configuration']['tasks'];
/** @var OrchestrationTask[] $rows */
$rows = array_map(function ($line) use($token) {
$task = new OrchestrationTask($token);
$task->fromArray($line);
return $task;
}, $data['configuration']['tasks']);
return $this->sortTasks($rows);
}
/**
* @param Elasticsearch\Job $job
* @param StorageApi\OrchestrationTask $task
* @param Encryptor $encryptor
* @return \GuzzleHttp\Psr7\Response
*/
public function sendOrchestratorRequest(Elasticsearch\Job $job, StorageApi\OrchestrationTask $task, Encryptor $encryptor)
{
$timeout = $task->getTimeoutMinutes() ? $task->getTimeoutMinutes() : KeboolaOrchestratorBundle::TIMEOUT_MINUTES;
try {
return $this->post($task->getComponentUrl(), array('config' => array('curl' => array(CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_0)), 'body' => json_encode($task->getActionParameters()), 'headers' => array('X-StorageApi-Token' => $encryptor->decrypt($job->getToken()), 'X-KBC-RunId' => $job->getRunId(), 'X-User-Agent', KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME . " - JobExecutor"), 'timeout' => $timeout * 60));
} catch (RequestException $e) {
$handlerContext = $e->getHandlerContext();
if (is_array($handlerContext)) {
if (array_key_exists('errno', $handlerContext)) {
if ($handlerContext['errno'] == CURLE_OPERATION_TIMEOUTED) {
$this->logger->debug('curl.debug', array('handlerContext' => $e->getHandlerContext()));
// throw new Exception\JobRuntimeException(sprintf('Task processing timeout after %d minutes', $timeout));
}
}
}
throw $e;
}
}
/**
* @param \Exception $e
* @param \Monolog\Logger $logger
* @return StorageApiEvent
* @throws \Keboola\StorageApi\Exception
*/
public function someError(\Exception $e, \Monolog\Logger $logger)
{
$exceptionId = KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME . '-' . md5(microtime());
$code = $e->getCode() < 300 || $e->getCode() >= 600 ? 500 : $e->getCode();
if ($e instanceof HttpExceptionInterface) {
$code = $e->getStatusCode();
}
$content = array('status' => 'error', 'error' => 'Application error', 'code' => $code, 'message' => 'Contact support@keboola.com and attach this exception id.', 'exceptionId' => $exceptionId);
$logger->error($e->getMessage(), $content);
$event = $this->prepareEvent();
$event->setMessage(sprintf(self::MESSAGE_END, $this->task->getRunUrl()))->setDescription(sprintf('%s %s', $content['message'], $content['exceptionId']))->setType(StorageApiEvent::TYPE_WARN)->setResults(array('exceptionId' => $exceptionId));
$this->save($event);
return $event;
}
/**
* @param Orchestration $entity
* @param \Keboola\StorageApi\Client $storageApi
* @return Orchestration
*/
public function fillOrchestrationEntity(Orchestration $entity, Client $storageApi)
{
/** @var $entity Orchestration */
$entity = parent::fillEntity($entity);
if (!$entity->getTokenId()) {
$entity->setTokenId($storageApi->createToken('manage', sprintf('Orchestrator %s', $entity->getName()), null, true));
}
// fill token data
$token = $storageApi->getToken($entity->getTokenId());
$token = new Token(new Client(array('token' => $token['token'], 'url' => $storageApi->getApiUrl())));
$entity->setToken($token)->setTokenId($token->getId())->setTokenDesc($token->getDescription())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId());
// notifications
$data = $this->getPostJsonData();
if (!empty($data['notifications'])) {
$data['notifications'] = array_map(function ($row) {
$notification = new Notification();
$notification->fromArray($row);
return $notification;
}, $data['notifications']);
$entity->setNotifications($data['notifications']);
}
// tasks
$data = $this->getPostJsonData();
if (!empty($data['tasks'])) {
$data['tasks'] = array_map(function ($row) {
$notification = new OrchestrationTask();
$notification->fromArray($row);
return $notification;
}, $data['tasks']);
foreach ($data['tasks'] as $task) {
$entity->addTask($task);
}
}
//@FIXME catch and throw exception
return $entity;
}
public function execute(SyrupJob $job)
{
parent::build($job);
$jobLogger = new JobLogger($this->job, $this->storageApi);
try {
$jobLogger->start($this->stopwatch->getEvent(self::STOPWATCH_ORCHESTRATION_EVENT));
if ($this->job->getTasks()) {
/** @var StorageApi\OrchestrationTask[] $rows */
$orchestrationTasks = array_map(function ($line) {
$task = new StorageApi\OrchestrationTask();
$task->fromArray(json_decode(json_encode($line), true));
return $task;
}, $this->job->getTasks());
} else {
$orchestrationTasks = $this->loadTasks($this->job, $this->token);
if (!$this->job->getTasks() && $orchestrationTasks) {
$this->logger->info('orchestration.job.emptyTasks', array('jobId' => $this->job->getId(), 'orchestrationId' => $this->job->getOrchestrationId(), 'orchestrationName' => $this->job->getOrchestrationName(), 'projectId' => $this->job->getProjectId()));
}
}
// fill result
foreach ($orchestrationTasks as $task) {
$this->jobResult->addTaskResult(new TaskResult($task));
}
$this->saveJobResult($this->jobResult);
foreach ($orchestrationTasks as $task) {
// skip inactive taks
if (!$task->getActive()) {
continue;
}
try {
$taskResult = (new TaskResult($task))->setProcessingStatus();
$this->jobResult->addTaskResult($taskResult);
$this->saveJobResult($this->jobResult);
$result = $this->executeJobTask($this->job, $taskResult, $this->storageApi);
$this->jobResult->addTaskResult($result);
$this->saveJobResult($this->jobResult);
if (!$task->getContinueOnFailure() && $result->getError()) {
break;
}
if ($task->getContinueOnFailure() && $result->getError()) {
continue;
}
} catch (\Exception $taskE) {
//@FIXME better handling = not log to sapi
throw Exception\JobRuntimeException::factory($taskE);
}
}
$jobLogger->end($this->jobResult);
$status = StatusConverter::orchestratorToSyrup($this->jobResult->getEventType());
$result = array('tasks' => $this->jobResult->toArray());
if ($status === SyrupJob::STATUS_SUCCESS) {
return $result;
}
if ($status === SyrupJob::STATUS_WARNING) {
$eData = array('jobId' => $this->job->getId(), 'orchestrationId' => $this->orchestration->getId(), 'projectId' => $this->orchestration->getProjectId());
$e = new JobException(500, sprintf('Some of tasks failed'), null, $eData);
$e->setStatus($status)->setResult($result);
throw $e;
}
if ($status === SyrupJob::STATUS_ERROR) {
$eData = array('jobId' => $this->job->getId(), 'orchestrationId' => $this->orchestration->getId(), 'projectId' => $this->orchestration->getProjectId());
$e = new JobException(500, sprintf('Job failed', $job->getId()), null, $eData);
$e->setStatus($status)->setResult($result);
throw $e;
}
} catch (Exception\JobInvalidStateException $e) {
$jobLogger->stateError($e);
// any change of job status, job is probably processing
} catch (Exception\JobRuntimeException $e) {
throw $e->getPrevious();
}
return array();
}
/**
* @return boolean
*/
public function getActive()
{
return $this->task->getActive();
}
/**
* @param Orchestration $entity
* @param \Keboola\StorageApi\Client $storageApi
* @return Orchestration
*/
public function fillOrchestrationEntity(Orchestration $entity, Client $storageApi)
{
$oldTokenId = $entity->getTokenId();
$oldName = $entity->getName();
/** @var $entity Orchestration */
$entity = parent::fillEntity($entity);
$name = $entity->getName();
if (empty($name)) {
$entity->setName($oldName);
}
// token change
if ($oldTokenId !== $entity->getTokenId()) {
$token = $storageApi->getToken($entity->getTokenId());
$token = new Token(new Client(array('token' => $token['token'], 'url' => $storageApi->getApiUrl())));
$entity->setToken($token)->setTokenId($token->getId())->setTokenDesc($token->getDescription())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId());
}
// notifications
$data = $this->getPostJsonData();
if (!empty($data['notifications'])) {
$data['notifications'] = array_map(function ($row) {
$notification = new Notification();
$notification->fromArray($row);
return $notification;
}, $data['notifications']);
$entity->setNotifications($data['notifications']);
}
// tasks
$data = $this->getPostJsonData();
if (array_key_exists('tasks', $data)) {
$entity->removeTasks();
$data['tasks'] = array_map(function ($row) {
$notification = new OrchestrationTask();
$notification->fromArray($row);
return $notification;
}, $data['tasks']);
foreach ($data['tasks'] as $task) {
$entity->addTask($task);
}
}
//@FIXME catch and throw exception
return $entity;
}
/**
* @return OrchestrationTask[]
*/
public function getTaskList()
{
$return = array();
// tasks
$data = $this->getPostJsonData();
if (array_key_exists('tasks', $data)) {
$return = array_map(function ($row) {
$notification = new OrchestrationTask();
$notification->fromArray($row);
return $notification;
}, $data['tasks']);
}
return $return;
}
/**
* @param Job $job
* @return Phase[]
*/
private function createPhases(Job $job)
{
/** @var StorageApi\OrchestrationTask[] $orchestrationTasks */
$orchestrationTasks = array();
if ($job->getTasks()) {
$orchestrationTasks = array_map(function ($line) {
$task = new StorageApi\OrchestrationTask();
$task->fromArray(json_decode(json_encode($line), true));
return $task;
}, $job->getTasks());
}
/**
* @var Phase[] $phases
*/
$phases = array();
$prevPhase = false;
foreach ($orchestrationTasks as $orchestrationTask) {
if ($orchestrationTask->getPhase() !== $prevPhase || $orchestrationTask->getPhase() == null) {
$phase = new Phase($job);
$phases[count($phases)] = $phase;
$this->jobResult->addPhase($phase);
}
$taskResult = new TaskResult($orchestrationTask);
$this->jobResult->addTaskResult($taskResult);
$phases[count($phases) - 1]->addTaskResult($taskResult);
$prevPhase = $orchestrationTask->getPhase();
}
return $phases;
}
