/**
* @param Job $job
* @param TaskResult $taskResult
* @return TaskResult
*/
private function executeJobTask(Job $job, TaskResult $taskResult)
{
$stopwatchEvent = $this->stopwatch->start($taskResult->getId(), 'task');
$logger = new TaskLogger($job, $taskResult->getTask(), $this->storageApi);
$logger->start($stopwatchEvent);
$startTime = time();
try {
$this->fillComponentData($taskResult);
$httpClient = new Client(array(), $this->logger);
$response = $httpClient->sendOrchestratorRequest($job, $taskResult->getTask(), $this->encryptor);
$pollResponse = null;
// handling async task
if (ResponseValidator::isAsyncResponse($response)) {
$taskJobUrl = null;
try {
$data = ResponseDecoder::decode($response);
if (!empty($data['url'])) {
$taskJobUrl = $data['url'];
}
} catch (\Exception $e) {
}
if ($taskJobUrl) {
$taskResult->setJobUrl($taskJobUrl);
$this->saveJobResult($this->jobResult);
}
$this->logTaskUsability($taskResult, $response);
$retriesCount = 1;
$timeout = $taskResult->getTimeoutMinutes() ? $taskResult->getTimeoutMinutes() : KeboolaOrchestratorBundle::TIMEOUT_MINUTES;
// polling job
do {
$data = array('status' => 'unknown');
if (time() >= $startTime + $timeout * 60) {
throw new Exception\JobRuntimeException(sprintf('Asynchronous task processing timeout after %d minutes', $timeout));
}
$waitSeconds = min(pow(2, $retriesCount), 20);
sleep($waitSeconds);
try {
$pollResponse = $httpClient->sendOrchestratorPollRequest($job, $response, $this->encryptor, $retriesCount);
if ($pollResponse) {
$data = ResponseDecoder::decode($pollResponse);
}
} catch (GuzzleException\RequestException $e) {
if ($e instanceof Exception\CurlException) {
} else {
$prevE = $e->getPrevious();
if (!$prevE || !$prevE instanceof Exception\JobRuntimeException) {
throw $e;
}
}
} catch (\Exception $e) {
// maybe json decode error, do nothing
}
$retriesCount++;
} while (array_key_exists('status', $data) && !StatusConverter::isFinishedStatus($data['status']));
} else {
// stats log
if (ResponseValidator::isSuccessfulResponse($response) || ResponseValidator::isInformationalResponse($response)) {
$this->logTaskUsability($taskResult, $response);
}
}
if ($pollResponse) {
$response = $pollResponse;
if (empty($data) || !in_array($data['status'], array('success', 'ok'))) {
//@TODO logovat odpovedi, ktere vraci nesmyslne OK status
$fakeRequest = new Request($pollResponse ? 'GET' : 'POST', $response->getHeaderLine(Client::EFFECTIVE_URL_HEADER_NAME));
$clientExc = new GuzzleException\ClientException('Error response from component', $fakeRequest, $response);
throw $clientExc;
}
}
if (!ResponseValidator::isSuccessfulResponse($response) && !ResponseValidator::isInformationalResponse($response)) {
$fakeRequest = new Request($pollResponse ? 'GET' : 'POST', $response->getHeaderLine(Client::EFFECTIVE_URL_HEADER_NAME));
$clientExc = new GuzzleException\ClientException('Response with error HTTP code', $fakeRequest, $response);
throw $clientExc;
}
$endEvent = $logger->end();
} catch (GuzzleException\ClientException $e) {
$endEvent = $logger->clientError($e);
if ($e->getResponse()) {
$response = $e->getResponse();
}
} catch (GuzzleException\BadResponseException $e) {
$endEvent = $logger->responseError($e);
if ($e->getResponse()) {
$response = $e->getResponse();
}
} catch (\Exception $e) {
$endEvent = $logger->someError($e, $this->logger);
$response = null;
}
$taskResult->setResult($endEvent, empty($response) ? null : $response);
return $taskResult;
}
public function toApiArray()
{
return array('id' => $this->getId(), 'orchestrationId' => $this->getOrchestrationId(), 'createdTime' => $this->getCreatedTime() ? $this->getCreatedTime()->format('c') : null, 'startTime' => $this->getStartTime() ? $this->getStartTime()->format('c') : null, 'endTime' => $this->getEndTime() ? $this->getEndTime()->format('c') : null, 'status' => StatusConverter::syrupToOrchestrator($this->getStatus()), 'isFinished' => StatusConverter::isFinishedStatus(StatusConverter::syrupToOrchestrator($this->getStatus())), 'initializedBy' => $this->getInitializedBy(), 'token' => array('id' => $this->getTokenId(), 'description' => $this->getTokenDesc()), 'initiatorToken' => array('id' => $this->getInitiatorTokenId(), 'description' => $this->getInitiatorTokenDesc(), 'userAgent' => $this->getInitiatorUserAgent()), 'tasks' => $this->getTasks(), 'results' => $this->getResults(), 'notificationsEmails' => $this->getNotificationsEmails(), 'runId' => $this->getRunId(), 'url' => $this->getUrl());
}
/**
* @param $jobId
* @param Request $request
* @return \Symfony\Component\HttpFoundation\JsonResponse
*/
public function postRetryAction($jobId, Request $request)
{
$job = $this->jobEsManager->findJobById($jobId);
if (!$job) {
$exception = new OrchestratorException(404, sprintf('Job %s not found', $jobId));
$exception->setExceptionCode('JOB_NOT_FOUND');
throw $exception;
}
if (!StatusConverter::isFinishedStatus($job->getStatus())) {
$exception = new OrchestratorException(400, sprintf('You can retry only finished jobs', $jobId));
$exception->setExceptionCode('JOB_VALIDATION');
throw $exception;
}
$orchestrationId = $job->getOrchestrationId();
$orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestrationId, $this->token, true);
if (!$orchestration) {
$exception = new OrchestratorException(404, sprintf('Orchestration %s not found', $orchestrationId));
$exception->setExceptionCode('ORCHESTRATION_NOT_FOUND');
throw $exception;
}
try {
$form = $this->createForm(new OrchestrationRunType($this->storageApi, $this->dbOrchestrationManager, $this->token, $orchestration));
$handler = $this->createRunFormHandler($form, $request, $orchestration);
if ($handler->process()) {
if ($handler->getTaskList()) {
$tasks = array_map(function ($task) {
/** @var StorageApi\OrchestrationTask $task */
$task->setId($this->storageApi->generateId());
return $task->toApiArray();
}, $handler->getTaskList());
} else {
$tasks = $job->getTasks();
}
}
} catch (HttpException $e) {
$exception = new OrchestratorException($e->getStatusCode(), $e->getMessage());
$exception->setExceptionCode('JOB_VALIDATION');
throw $exception;
}
// waiting jobs limit
$jobsStats = $this->jobEsManager->getWaitingStats();
// skip waiting
if (array_key_exists($orchestration->getId(), $jobsStats) && $jobsStats[$orchestration->getId()] >= AppConfiguration::MAX_WAITING_JOBS_COUNT) {
$count = $jobsStats[$orchestration->getId()];
$this->logger->info('scheduler.orchestration.skipped', array('waitingCount' => $count, 'limit' => AppConfiguration::MAX_WAITING_JOBS_COUNT, 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $this->token->getProjectId(), 'projectName' => $this->token->getOwnerName()));
if ($count > 1) {
$exception = new OrchestratorException(409, sprintf('Orchestration %s has %d waiting jobs. Current limit is %d.', $orchestrationId, $count, AppConfiguration::MAX_WAITING_JOBS_COUNT));
} else {
$exception = new OrchestratorException(409, sprintf('Orchestration %s has %d waiting job. Current limit is %d.', $orchestrationId, $count, AppConfiguration::MAX_WAITING_JOBS_COUNT));
}
$exception->setExceptionCode('ORCHESTRATION_VALIDATION');
throw $exception;
}
$form = $this->createForm(new ScheduleJobType($this->storageApi, $this->dbOrchestrationManager, $this->token));
$handler = parent::createFormHandler($form, $request);
try {
$notificationsEmails = array();
if ($handler->process()) {
$notificationsEmails = $handler->getPost('notificationsEmails', array());
}
if (!$notificationsEmails && \Swift_Validate::email($this->token->getDescription())) {
$notificationsEmails = array($this->token->getDescription());
}
} catch (HttpException $e) {
$exception = new OrchestratorException(400, $e->getMessage());
$exception->setExceptionCode('JOB_VALIDATION');
throw $exception;
}
$this->initSqsQueue();
$job = new Elasticsearch\Job();
$job->setConfig($orchestration->getId())->setOrchestrationId($orchestration->getId())->setOrchestrationName($orchestration->getName())->setToken($orchestration->getToken())->setTokenId($orchestration->getTokenId())->setTokenDesc($orchestration->getTokenDesc())->setTokenOwnerName($this->token->getOwnerName())->setProjectId($this->token->getProjectId())->setInitializedBy('manually')->setInitiatorTokenId($this->token->getId())->setInitiatorTokenDesc($this->token->getDescription())->setInitiatorUserAgent($this->getRequestUserAgent($request))->setNotificationsEmails($notificationsEmails)->setTaks($tasks);
$job = $this->jobEsManager->saveJob($job, new StorageApi\UniqueManager($this->storageApi));
$this->queue->enqueue($job->getId(), array('jobId' => $job->getId(), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME));
$this->logger->info(sprintf('Orchestration job %s created manually', $job->getId()));
return $this->createJsonResponse($job->toOldApiArray(), 201);
}
/**
* @param Phase $phase
* @param PhaseLogger $logger
* @param Client $httpClient
* @param Response[] $jobResponses
* @return array|bool
*/
public function pollPhase(Phase $phase, PhaseLogger $logger, Client $httpClient, $jobResponses = array())
{
/**
* @var Response[] $pollResponses
*/
$pollResponses = array();
/**
* @var TaskResult[] $taskResults
*/
$taskResults = array();
foreach ($phase->getTasks() as $taskResult) {
$taskResults[$taskResult->getId()] = $taskResult;
}
// all jobs polling
$retriesCount = 1;
$errorsCount = 0;
do {
$waitSeconds = min(pow(2, $retriesCount), 20);
sleep($waitSeconds);
foreach ($jobResponses as $key => $jobResponse) {
$taskResult = $taskResults[$key];
$startTime = $taskResult->getStartTime()->getTimestamp();
try {
$pollResponse = null;
$timeout = $taskResult->getTimeoutMinutes() ? $taskResult->getTimeoutMinutes() : KeboolaOrchestratorBundle::TIMEOUT_MINUTES;
$data = array('status' => 'unknown');
if (time() >= $startTime + $timeout * 60) {
throw new Exception\JobRuntimeException(sprintf('Asynchronous task processing timeout after %d minutes', $timeout));
}
try {
$pollResponse = $httpClient->sendOrchestratorPollRequest($phase->getJob(), $jobResponse, $this->encryptor, $retriesCount);
if ($pollResponse) {
$pollResponses[$key] = $pollResponse;
$data = ResponseDecoder::decode($pollResponse);
}
} catch (GuzzleException\RequestException $e) {
if ($e instanceof Exception\CurlException) {
} else {
$prevE = $e->getPrevious();
if (!$prevE || !$prevE instanceof Exception\JobRuntimeException) {
throw $e;
}
}
} catch (\Exception $e) {
// maybe json decode error, do nothing
}
if (!array_key_exists('status', $data)) {
continue;
}
if (!StatusConverter::isFinishedStatus($data['status'])) {
continue;
}
if ($pollResponse) {
$response = $pollResponse;
if (empty($data) || !in_array($data['status'], array('success', 'ok'))) {
//@TODO logovat odpovedi, ktere vraci nesmyslne OK status
$fakeRequest = new Request('GET', $response->getHeaderLine(Client::EFFECTIVE_URL_HEADER_NAME));
$clientExc = new GuzzleException\ClientException('Error response from component', $fakeRequest, $response);
throw $clientExc;
}
}
if (!ResponseValidator::isSuccessfulResponse($pollResponse) && !ResponseValidator::isInformationalResponse($pollResponse)) {
$fakeRequest = new Request('GET', $pollResponse->getHeaderLine(Client::EFFECTIVE_URL_HEADER_NAME));
$clientExc = new GuzzleException\ClientException('Response with error HTTP code', $fakeRequest, $pollResponse);
throw $clientExc;
}
$endEvent = $logger->getLogger($taskResult)->end();
} catch (GuzzleException\ClientException $e) {
$endEvent = $logger->getLogger($taskResult)->clientError($e);
if ($e->getResponse()) {
$response = $e->getResponse();
}
} catch (GuzzleException\BadResponseException $e) {
$endEvent = $logger->getLogger($taskResult)->responseError($e);
if ($e->getResponse()) {
$response = $e->getResponse();
}
} catch (\Exception $e) {
$endEvent = $logger->getLogger($taskResult)->someError($e, $this->logger);
$response = null;
}
$taskResult->setResult($endEvent, empty($response) ? null : $response);
if ($taskResult->getError() && !$taskResult->getTask()->getContinueOnFailure()) {
$errorsCount++;
}
unset($jobResponses[$key]);
}
$this->saveJobResult($this->jobResult);
$retriesCount++;
} while (count($jobResponses));
if ($errorsCount) {
return false;
}
return array();
}