Beispiel #1
0
 /**
  * @param Job $job
  * @param TaskResult $taskResult
  * @return TaskResult
  */
 private function executeJobTask(Job $job, TaskResult $taskResult)
 {
     $stopwatchEvent = $this->stopwatch->start($taskResult->getId(), 'task');
     $logger = new TaskLogger($job, $taskResult->getTask(), $this->storageApi);
     $logger->start($stopwatchEvent);
     $startTime = time();
     try {
         $this->fillComponentData($taskResult);
         $httpClient = new Client(array(), $this->logger);
         $response = $httpClient->sendOrchestratorRequest($job, $taskResult->getTask(), $this->encryptor);
         $pollResponse = null;
         // handling async task
         if (ResponseValidator::isAsyncResponse($response)) {
             $taskJobUrl = null;
             try {
                 $data = ResponseDecoder::decode($response);
                 if (!empty($data['url'])) {
                     $taskJobUrl = $data['url'];
                 }
             } catch (\Exception $e) {
             }
             if ($taskJobUrl) {
                 $taskResult->setJobUrl($taskJobUrl);
                 $this->saveJobResult($this->jobResult);
             }
             $this->logTaskUsability($taskResult, $response);
             $retriesCount = 1;
             $timeout = $taskResult->getTimeoutMinutes() ? $taskResult->getTimeoutMinutes() : KeboolaOrchestratorBundle::TIMEOUT_MINUTES;
             // polling job
             do {
                 $data = array('status' => 'unknown');
                 if (time() >= $startTime + $timeout * 60) {
                     throw new Exception\JobRuntimeException(sprintf('Asynchronous task processing timeout after %d minutes', $timeout));
                 }
                 $waitSeconds = min(pow(2, $retriesCount), 20);
                 sleep($waitSeconds);
                 try {
                     $pollResponse = $httpClient->sendOrchestratorPollRequest($job, $response, $this->encryptor, $retriesCount);
                     if ($pollResponse) {
                         $data = ResponseDecoder::decode($pollResponse);
                     }
                 } catch (GuzzleException\RequestException $e) {
                     if ($e instanceof Exception\CurlException) {
                     } else {
                         $prevE = $e->getPrevious();
                         if (!$prevE || !$prevE instanceof Exception\JobRuntimeException) {
                             throw $e;
                         }
                     }
                 } catch (\Exception $e) {
                     // maybe json decode error, do nothing
                 }
                 $retriesCount++;
             } while (array_key_exists('status', $data) && !StatusConverter::isFinishedStatus($data['status']));
         } else {
             // stats log
             if (ResponseValidator::isSuccessfulResponse($response) || ResponseValidator::isInformationalResponse($response)) {
                 $this->logTaskUsability($taskResult, $response);
             }
         }
         if ($pollResponse) {
             $response = $pollResponse;
             if (empty($data) || !in_array($data['status'], array('success', 'ok'))) {
                 //@TODO logovat odpovedi, ktere vraci nesmyslne OK status
                 $fakeRequest = new Request($pollResponse ? 'GET' : 'POST', $response->getHeaderLine(Client::EFFECTIVE_URL_HEADER_NAME));
                 $clientExc = new GuzzleException\ClientException('Error response from component', $fakeRequest, $response);
                 throw $clientExc;
             }
         }
         if (!ResponseValidator::isSuccessfulResponse($response) && !ResponseValidator::isInformationalResponse($response)) {
             $fakeRequest = new Request($pollResponse ? 'GET' : 'POST', $response->getHeaderLine(Client::EFFECTIVE_URL_HEADER_NAME));
             $clientExc = new GuzzleException\ClientException('Response with error HTTP code', $fakeRequest, $response);
             throw $clientExc;
         }
         $endEvent = $logger->end();
     } catch (GuzzleException\ClientException $e) {
         $endEvent = $logger->clientError($e);
         if ($e->getResponse()) {
             $response = $e->getResponse();
         }
     } catch (GuzzleException\BadResponseException $e) {
         $endEvent = $logger->responseError($e);
         if ($e->getResponse()) {
             $response = $e->getResponse();
         }
     } catch (\Exception $e) {
         $endEvent = $logger->someError($e, $this->logger);
         $response = null;
     }
     $taskResult->setResult($endEvent, empty($response) ? null : $response);
     return $taskResult;
 }
Beispiel #2
0
 public function toApiArray()
 {
     return array('id' => $this->getId(), 'orchestrationId' => $this->getOrchestrationId(), 'createdTime' => $this->getCreatedTime() ? $this->getCreatedTime()->format('c') : null, 'startTime' => $this->getStartTime() ? $this->getStartTime()->format('c') : null, 'endTime' => $this->getEndTime() ? $this->getEndTime()->format('c') : null, 'status' => StatusConverter::syrupToOrchestrator($this->getStatus()), 'isFinished' => StatusConverter::isFinishedStatus(StatusConverter::syrupToOrchestrator($this->getStatus())), 'initializedBy' => $this->getInitializedBy(), 'token' => array('id' => $this->getTokenId(), 'description' => $this->getTokenDesc()), 'initiatorToken' => array('id' => $this->getInitiatorTokenId(), 'description' => $this->getInitiatorTokenDesc(), 'userAgent' => $this->getInitiatorUserAgent()), 'tasks' => $this->getTasks(), 'results' => $this->getResults(), 'notificationsEmails' => $this->getNotificationsEmails(), 'runId' => $this->getRunId(), 'url' => $this->getUrl());
 }
 /**
  * @param $jobId
  * @param Request $request
  * @return \Symfony\Component\HttpFoundation\JsonResponse
  */
 public function postRetryAction($jobId, Request $request)
 {
     $job = $this->jobEsManager->findJobById($jobId);
     if (!$job) {
         $exception = new OrchestratorException(404, sprintf('Job %s not found', $jobId));
         $exception->setExceptionCode('JOB_NOT_FOUND');
         throw $exception;
     }
     if (!StatusConverter::isFinishedStatus($job->getStatus())) {
         $exception = new OrchestratorException(400, sprintf('You can retry only finished jobs', $jobId));
         $exception->setExceptionCode('JOB_VALIDATION');
         throw $exception;
     }
     $orchestrationId = $job->getOrchestrationId();
     $orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestrationId, $this->token, true);
     if (!$orchestration) {
         $exception = new OrchestratorException(404, sprintf('Orchestration %s not found', $orchestrationId));
         $exception->setExceptionCode('ORCHESTRATION_NOT_FOUND');
         throw $exception;
     }
     try {
         $form = $this->createForm(new OrchestrationRunType($this->storageApi, $this->dbOrchestrationManager, $this->token, $orchestration));
         $handler = $this->createRunFormHandler($form, $request, $orchestration);
         if ($handler->process()) {
             if ($handler->getTaskList()) {
                 $tasks = array_map(function ($task) {
                     /** @var StorageApi\OrchestrationTask $task */
                     $task->setId($this->storageApi->generateId());
                     return $task->toApiArray();
                 }, $handler->getTaskList());
             } else {
                 $tasks = $job->getTasks();
             }
         }
     } catch (HttpException $e) {
         $exception = new OrchestratorException($e->getStatusCode(), $e->getMessage());
         $exception->setExceptionCode('JOB_VALIDATION');
         throw $exception;
     }
     // waiting jobs limit
     $jobsStats = $this->jobEsManager->getWaitingStats();
     // skip waiting
     if (array_key_exists($orchestration->getId(), $jobsStats) && $jobsStats[$orchestration->getId()] >= AppConfiguration::MAX_WAITING_JOBS_COUNT) {
         $count = $jobsStats[$orchestration->getId()];
         $this->logger->info('scheduler.orchestration.skipped', array('waitingCount' => $count, 'limit' => AppConfiguration::MAX_WAITING_JOBS_COUNT, 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $this->token->getProjectId(), 'projectName' => $this->token->getOwnerName()));
         if ($count > 1) {
             $exception = new OrchestratorException(409, sprintf('Orchestration %s has %d waiting jobs. Current limit is %d.', $orchestrationId, $count, AppConfiguration::MAX_WAITING_JOBS_COUNT));
         } else {
             $exception = new OrchestratorException(409, sprintf('Orchestration %s has %d waiting job. Current limit is %d.', $orchestrationId, $count, AppConfiguration::MAX_WAITING_JOBS_COUNT));
         }
         $exception->setExceptionCode('ORCHESTRATION_VALIDATION');
         throw $exception;
     }
     $form = $this->createForm(new ScheduleJobType($this->storageApi, $this->dbOrchestrationManager, $this->token));
     $handler = parent::createFormHandler($form, $request);
     try {
         $notificationsEmails = array();
         if ($handler->process()) {
             $notificationsEmails = $handler->getPost('notificationsEmails', array());
         }
         if (!$notificationsEmails && \Swift_Validate::email($this->token->getDescription())) {
             $notificationsEmails = array($this->token->getDescription());
         }
     } catch (HttpException $e) {
         $exception = new OrchestratorException(400, $e->getMessage());
         $exception->setExceptionCode('JOB_VALIDATION');
         throw $exception;
     }
     $this->initSqsQueue();
     $job = new Elasticsearch\Job();
     $job->setConfig($orchestration->getId())->setOrchestrationId($orchestration->getId())->setOrchestrationName($orchestration->getName())->setToken($orchestration->getToken())->setTokenId($orchestration->getTokenId())->setTokenDesc($orchestration->getTokenDesc())->setTokenOwnerName($this->token->getOwnerName())->setProjectId($this->token->getProjectId())->setInitializedBy('manually')->setInitiatorTokenId($this->token->getId())->setInitiatorTokenDesc($this->token->getDescription())->setInitiatorUserAgent($this->getRequestUserAgent($request))->setNotificationsEmails($notificationsEmails)->setTaks($tasks);
     $job = $this->jobEsManager->saveJob($job, new StorageApi\UniqueManager($this->storageApi));
     $this->queue->enqueue($job->getId(), array('jobId' => $job->getId(), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME));
     $this->logger->info(sprintf('Orchestration job %s created manually', $job->getId()));
     return $this->createJsonResponse($job->toOldApiArray(), 201);
 }
 /**
  * @param Phase $phase
  * @param PhaseLogger $logger
  * @param Client $httpClient
  * @param Response[] $jobResponses
  * @return array|bool
  */
 public function pollPhase(Phase $phase, PhaseLogger $logger, Client $httpClient, $jobResponses = array())
 {
     /**
      * @var Response[] $pollResponses
      */
     $pollResponses = array();
     /**
      * @var TaskResult[] $taskResults
      */
     $taskResults = array();
     foreach ($phase->getTasks() as $taskResult) {
         $taskResults[$taskResult->getId()] = $taskResult;
     }
     // all jobs polling
     $retriesCount = 1;
     $errorsCount = 0;
     do {
         $waitSeconds = min(pow(2, $retriesCount), 20);
         sleep($waitSeconds);
         foreach ($jobResponses as $key => $jobResponse) {
             $taskResult = $taskResults[$key];
             $startTime = $taskResult->getStartTime()->getTimestamp();
             try {
                 $pollResponse = null;
                 $timeout = $taskResult->getTimeoutMinutes() ? $taskResult->getTimeoutMinutes() : KeboolaOrchestratorBundle::TIMEOUT_MINUTES;
                 $data = array('status' => 'unknown');
                 if (time() >= $startTime + $timeout * 60) {
                     throw new Exception\JobRuntimeException(sprintf('Asynchronous task processing timeout after %d minutes', $timeout));
                 }
                 try {
                     $pollResponse = $httpClient->sendOrchestratorPollRequest($phase->getJob(), $jobResponse, $this->encryptor, $retriesCount);
                     if ($pollResponse) {
                         $pollResponses[$key] = $pollResponse;
                         $data = ResponseDecoder::decode($pollResponse);
                     }
                 } catch (GuzzleException\RequestException $e) {
                     if ($e instanceof Exception\CurlException) {
                     } else {
                         $prevE = $e->getPrevious();
                         if (!$prevE || !$prevE instanceof Exception\JobRuntimeException) {
                             throw $e;
                         }
                     }
                 } catch (\Exception $e) {
                     // maybe json decode error, do nothing
                 }
                 if (!array_key_exists('status', $data)) {
                     continue;
                 }
                 if (!StatusConverter::isFinishedStatus($data['status'])) {
                     continue;
                 }
                 if ($pollResponse) {
                     $response = $pollResponse;
                     if (empty($data) || !in_array($data['status'], array('success', 'ok'))) {
                         //@TODO logovat odpovedi, ktere vraci nesmyslne OK status
                         $fakeRequest = new Request('GET', $response->getHeaderLine(Client::EFFECTIVE_URL_HEADER_NAME));
                         $clientExc = new GuzzleException\ClientException('Error response from component', $fakeRequest, $response);
                         throw $clientExc;
                     }
                 }
                 if (!ResponseValidator::isSuccessfulResponse($pollResponse) && !ResponseValidator::isInformationalResponse($pollResponse)) {
                     $fakeRequest = new Request('GET', $pollResponse->getHeaderLine(Client::EFFECTIVE_URL_HEADER_NAME));
                     $clientExc = new GuzzleException\ClientException('Response with error HTTP code', $fakeRequest, $pollResponse);
                     throw $clientExc;
                 }
                 $endEvent = $logger->getLogger($taskResult)->end();
             } catch (GuzzleException\ClientException $e) {
                 $endEvent = $logger->getLogger($taskResult)->clientError($e);
                 if ($e->getResponse()) {
                     $response = $e->getResponse();
                 }
             } catch (GuzzleException\BadResponseException $e) {
                 $endEvent = $logger->getLogger($taskResult)->responseError($e);
                 if ($e->getResponse()) {
                     $response = $e->getResponse();
                 }
             } catch (\Exception $e) {
                 $endEvent = $logger->getLogger($taskResult)->someError($e, $this->logger);
                 $response = null;
             }
             $taskResult->setResult($endEvent, empty($response) ? null : $response);
             if ($taskResult->getError() && !$taskResult->getTask()->getContinueOnFailure()) {
                 $errorsCount++;
             }
             unset($jobResponses[$key]);
         }
         $this->saveJobResult($this->jobResult);
         $retriesCount++;
     } while (count($jobResponses));
     if ($errorsCount) {
         return false;
     }
     return array();
 }