/**
* @param Response $response
* @return bool
*/
public static function isAsyncResponse(Response $response)
{
if ($response->getStatusCode() != 202) {
return false;
}
try {
$data = ResponseDecoder::decode($response);
if (empty($data['url'])) {
return false;
}
} catch (\Exception $e) {
return false;
}
return true;
}
public function ping()
{
$options = array('timeout' => 7200, 'headers' => array('X-StorageApi-Token' => $this->storageApi->getTokenString(), 'Accept-Encoding' => 'gzip', 'User-Agent' => $this->storageApi->getUserAgent()));
if ($this->storageApi->getRunId()) {
$options['headers']['X-KBC-RunId'] = $this->storageApi->getRunId();
}
try {
$response = $this->client->get('storage/tokens/verify', $options);
} catch (RequestException $e) {
$response = $e->getResponse();
if ($response && $response->getStatusCode() == 503) {
return false;
} else {
throw $e;
}
}
if ($response && $response->getStatusCode() == 200) {
$body = ResponseDecoder::decode($response);
return array_key_exists('token', $body);
}
return false;
}
/**
* @param Job $job
* @param TaskResult $taskResult
* @return TaskResult
*/
private function executeJobTask(Job $job, TaskResult $taskResult)
{
$stopwatchEvent = $this->stopwatch->start($taskResult->getId(), 'task');
$logger = new TaskLogger($job, $taskResult->getTask(), $this->storageApi);
$logger->start($stopwatchEvent);
$startTime = time();
try {
$this->fillComponentData($taskResult);
$httpClient = new Client(array(), $this->logger);
$response = $httpClient->sendOrchestratorRequest($job, $taskResult->getTask(), $this->encryptor);
$pollResponse = null;
// handling async task
if (ResponseValidator::isAsyncResponse($response)) {
$taskJobUrl = null;
try {
$data = ResponseDecoder::decode($response);
if (!empty($data['url'])) {
$taskJobUrl = $data['url'];
}
} catch (\Exception $e) {
}
if ($taskJobUrl) {
$taskResult->setJobUrl($taskJobUrl);
$this->saveJobResult($this->jobResult);
}
$this->logTaskUsability($taskResult, $response);
$retriesCount = 1;
$timeout = $taskResult->getTimeoutMinutes() ? $taskResult->getTimeoutMinutes() : KeboolaOrchestratorBundle::TIMEOUT_MINUTES;
// polling job
do {
$data = array('status' => 'unknown');
if (time() >= $startTime + $timeout * 60) {
throw new Exception\JobRuntimeException(sprintf('Asynchronous task processing timeout after %d minutes', $timeout));
}
$waitSeconds = min(pow(2, $retriesCount), 20);
sleep($waitSeconds);
try {
$pollResponse = $httpClient->sendOrchestratorPollRequest($job, $response, $this->encryptor, $retriesCount);
if ($pollResponse) {
$data = ResponseDecoder::decode($pollResponse);
}
} catch (GuzzleException\RequestException $e) {
if ($e instanceof Exception\CurlException) {
} else {
$prevE = $e->getPrevious();
if (!$prevE || !$prevE instanceof Exception\JobRuntimeException) {
throw $e;
}
}
} catch (\Exception $e) {
// maybe json decode error, do nothing
}
$retriesCount++;
} while (array_key_exists('status', $data) && !StatusConverter::isFinishedStatus($data['status']));
} else {
// stats log
if (ResponseValidator::isSuccessfulResponse($response) || ResponseValidator::isInformationalResponse($response)) {
$this->logTaskUsability($taskResult, $response);
}
}
if ($pollResponse) {
$response = $pollResponse;
if (empty($data) || !in_array($data['status'], array('success', 'ok'))) {
//@TODO logovat odpovedi, ktere vraci nesmyslne OK status
$fakeRequest = new Request($pollResponse ? 'GET' : 'POST', $response->getHeaderLine(Client::EFFECTIVE_URL_HEADER_NAME));
$clientExc = new GuzzleException\ClientException('Error response from component', $fakeRequest, $response);
throw $clientExc;
}
}
if (!ResponseValidator::isSuccessfulResponse($response) && !ResponseValidator::isInformationalResponse($response)) {
$fakeRequest = new Request($pollResponse ? 'GET' : 'POST', $response->getHeaderLine(Client::EFFECTIVE_URL_HEADER_NAME));
$clientExc = new GuzzleException\ClientException('Response with error HTTP code', $fakeRequest, $response);
throw $clientExc;
}
$endEvent = $logger->end();
} catch (GuzzleException\ClientException $e) {
$endEvent = $logger->clientError($e);
if ($e->getResponse()) {
$response = $e->getResponse();
}
} catch (GuzzleException\BadResponseException $e) {
$endEvent = $logger->responseError($e);
if ($e->getResponse()) {
$response = $e->getResponse();
}
} catch (\Exception $e) {
$endEvent = $logger->someError($e, $this->logger);
$response = null;
}
$taskResult->setResult($endEvent, empty($response) ? null : $response);
return $taskResult;
}
public function cleanup(Metadata\Job $job)
{
$result = $job->getResult();
if (!$result) {
$result = array();
}
$esJob = new Job();
$esJob->build($job);
try {
$token = new Token($this->storageApi);
} catch (StorageApiException $e) {
$this->logger->error('Cleanup error - invalid token', array('jobId' => $esJob->getId()));
throw new UserException(sprintf("Invalid token for job %d", $esJob->getId()), $e);
}
$orchestration = $this->orchestrationManager->findOrchestrationById($esJob->getOrchestrationId(), $token);
if (!$orchestration) {
$this->logger->error('Cleanup error - orchestration not found', array('jobId' => $esJob->getId()));
throw new UserException(sprintf("Orchestration %s not found. Could not update last job", $esJob->getOrchestrationId()));
}
if (!empty($result['tasks'])) {
foreach ($result['tasks'] as $key => $task) {
// skip non processing tasks
if ($task['status'] !== Metadata\Job::STATUS_PROCESSING) {
continue;
}
$httpClient = new Client(array(), $this->logger);
// skip tasks without URL
if (empty($task['jobUrl'])) {
$retriesCount = 1;
$jobsCount = 0;
do {
$waitSeconds = min(pow(2, $retriesCount), 20);
$this->logger->debug('Poll jobs count', array('sleep' => $waitSeconds));
sleep($waitSeconds);
try {
$jobsCount = $httpClient->getNonFinishedChildJobsCount($esJob, $this->encryptor);
} catch (\GuzzleHttp\Exception\RequestException $e) {
$this->logger->error('Cleanup jobs count error', array('sleep' => $waitSeconds, 'exception' => $e));
} catch (\Exception $e) {
$this->logger->error('Cleanup jobs count error', array('sleep' => $waitSeconds, 'exception' => $e));
}
$retriesCount++;
} while ($jobsCount != 0);
$result['tasks'][$key]['status'] = Metadata\Job::STATUS_TERMINATED;
$result['tasks'][$key]['endTime'] = (new \DateTime())->format('c');
} else {
$this->logger->debug('Check task status', array('task' => $task));
$retriesCount = 1;
do {
$data = array('status' => 'unknown');
$waitSeconds = min(pow(2, $retriesCount), 20);
$this->logger->debug('Poll task status', array('task' => $task, 'sleep' => $waitSeconds));
sleep($waitSeconds);
try {
$response = $httpClient->getJobStatus($task['jobUrl'], $esJob, $this->encryptor);
if ($response) {
$data = ResponseDecoder::decode($response);
$this->logger->debug('Poll response', array('task' => $task, 'reponse' => $data));
} else {
$this->logger->error('Any poll response', array('task' => $task, 'sleep' => $waitSeconds));
}
} catch (\GuzzleHttp\Exception\RequestException $e) {
$this->logger->error('Cleanup poll job error', array('task' => $task, 'sleep' => $waitSeconds, 'exception' => $e));
} catch (\Exception $e) {
$this->logger->error('Cleanup poll job error', array('task' => $task, 'sleep' => $waitSeconds, 'exception' => $e));
}
$retriesCount++;
} while (!array_key_exists('isFinished', $data) || $data['isFinished'] != 1);
$result['tasks'][$key]['status'] = $data['status'];
if (!empty($data['endTime'])) {
$result['tasks'][$key]['endTime'] = $data['endTime'];
}
if (!empty($data['durationSeconds'])) {
$result['tasks'][$key]['duration'] = $data['durationSeconds'];
}
}
}
}
$jobEsManager = $this->jobManagerFactory->createJobManager($token);
$this->logger->debug('Cleanup job', array('jobId' => $esJob->getId()));
$job->setResult($result);
$esJob->setResults($result);
$jobEsManager->updateResult($esJob, $result);
}
/**
* Cancel waiting job
*
* @param $jobId
* @throws \Keboola\OrchestratorBundle\Exception\OrchestratorException
* @return Response
*/
public function deleteAction($jobId)
{
$job = $this->jobEsManager->findJobById($jobId);
if (!$job) {
$exception = new OrchestratorException(404, sprintf('Job %s not found', $jobId));
$exception->setExceptionCode('JOB_NOT_FOUND');
throw $exception;
}
if (!$job->isWaiting()) {
$exception = new OrchestratorException(403, 'Only waiting job can be cancelled');
$exception->setExceptionCode('INVALID_JOB_STATUS');
throw $exception;
}
$client = new GuzzleHttp\Client([]);
try {
$response = $client->post(sprintf($this->killUrl, $job->getId()), array('config' => array('curl' => array(CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_0)), 'headers' => array('X-StorageApi-Token' => $this->token, 'X-KBC-RunId' => $this->storageApi->getRunId(), 'X-User-Agent', KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME . " - API cancel"), 'timeout' => 60));
if ($response->getStatusCode() == 202) {
$this->logger->info(sprintf('Orchestration job %s cancelled', $job->getId()));
$this->logger->debug(sprintf('Orchestration job %s cancelled - old api', $job->getId()), array('orchestrationId' => $job->getOrchestrationId(), 'orchestration' => $job->getOrchestrationName(), 'projectId' => $job->getProjectId(), 'project' => $job->getTokenOwnerName(), 'tokenId' => $this->token->getId(), 'token' => $this->token->getDescription()));
} else {
throw new ApplicationException('Job cancel via syrup api failed', null, array('statusCode' => $response->getStatusCode(), 'body' => ResponseDecoder::decode($response)));
}
} catch (GuzzleHttp\Exception\RequestException $e) {
throw new ApplicationException('Job cancel via syrup api failed', $e);
}
$jsonResponse = $this->createJsonResponse(array(), 204);
return $jsonResponse;
}
/**
* @return array
*/
public function toArray()
{
$return = array('id' => $this->getId(), 'component' => $this->getComponent(), 'componentUrl' => $this->getComponentUrl(), 'action' => $this->getAction(), 'actionParameters' => $this->getActionParameters(), 'timeoutMinutes' => $this->getTimeoutMinutes(), 'continueOnFailure' => $this->getContinueOnFailure(), 'active' => $this->getActive(), 'duration' => $this->getDuration(), 'status' => $this->getStatus(), 'jobUrl' => $this->getJobUrl(), 'phase' => $this->getPhase(), 'startTime' => $this->startTime ? $this->startTime->format('c') : null, 'endTime' => $this->endTime ? $this->endTime->format('c') : null);
if ($this->getError()) {
$return['error'] = $this->getError();
}
// response data
if ($this->response) {
try {
$return['response'] = ResponseDecoder::decode($this->response);
if (!is_array($return['response'])) {
throw new \Exception('Not json reponse');
}
} catch (\Exception $jsonE) {
$return['responseRaw'] = $this->response->getBody();
}
$return['responseCode'] = $this->response->getStatusCode();
}
return $return;
}
/**
* @FIXME replace with ping client
*
* @param Metadata\Job $job
* @param Encryptor $encryptor
* @return bool
*/
private function sapiPing(Metadata\Job $job, Encryptor $encryptor)
{
$maxRetries = 4;
$handlerStack = HandlerStack::create();
$handlerStack->push(MiddlewareBuilder::factoryForWatchdog($this->logger, $maxRetries));
try {
$guzzle = new \GuzzleHttp\Client(['handler' => $handlerStack]);
$response = $guzzle->get(sprintf('%s/storage/tokens/verify', $this->getConfiguration()->getStorageApiUrl()), array('config' => array('curl' => array(CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_0)), 'headers' => array('X-StorageApi-Token' => $encryptor->decrypt($job->getToken()["token"]), 'X-User-Agent', KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME . " - Watchdog")));
if ($response->getStatusCode() == 200) {
$data = ResponseDecoder::decode($response);
if (array_key_exists('id', $data) && array_key_exists('token', $data)) {
return true;
}
}
} catch (\Exception $e) {
}
return false;
}
/**
* @param \GuzzleHttp\Exception\ClientException $e
* @return StorageApiEvent
*/
public function clientError(\GuzzleHttp\Exception\ClientException $e)
{
$response = null;
try {
$response = ResponseDecoder::decode($e->getResponse());
} catch (\Exception $jsonE) {
$response = $e->getResponse()->getBody();
}
$event = $this->prepareEvent();
$event->setMessage(sprintf(self::MESSAGE_END, $this->task->getRunUrl()))->setDescription('Error response from component')->setType(StorageApiEvent::TYPE_WARN)->setResults(array('response' => $response, 'code' => $e->getResponse()->getStatusCode()));
$this->save($event);
return $event;
}
public function getNonFinishedChildJobsCount(Elasticsearch\Job $job, Encryptor $encryptor)
{
$timeout = 2;
//@FIXME loop for paging
$url = sprintf("https://syrup.keboola.com/queue/jobs?q=runId:%s.*", $job->getRunId());
$response = $this->get($url, array('config' => array('curl' => array(CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_0)), 'headers' => array('X-StorageApi-Token' => $encryptor->decrypt($job->getToken()), 'X-KBC-RunId' => $job->getRunId(), 'X-User-Agent', KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME . " - JobExecutor"), 'timeout' => 60 * $timeout));
// handling curl errors
// $request->getEmitter()->on('error', function (ErrorEvent $e) use ($timeout) {
// $curlError = $e->getTransferInfo('curl_result');
// if ($curlError == 28) {
// throw new Exception\JobRuntimeException(sprintf('Task polling timeout after %d minutes', $timeout));
// }
// });
$data = ResponseDecoder::decode($response);
$count = 0;
foreach ($data as $job) {
if (!array_key_exists('isFinished', $job)) {
$count++;
continue;
}
if ($job['isFinished'] != 1) {
$count++;
continue;
}
}
return $count;
}
/**
* @param Phase $phase
* @param PhaseLogger $logger
* @param Client $httpClient
* @param Response[] $jobResponses
* @return array|bool
*/
public function pollPhase(Phase $phase, PhaseLogger $logger, Client $httpClient, $jobResponses = array())
{
/**
* @var Response[] $pollResponses
*/
$pollResponses = array();
/**
* @var TaskResult[] $taskResults
*/
$taskResults = array();
foreach ($phase->getTasks() as $taskResult) {
$taskResults[$taskResult->getId()] = $taskResult;
}
// all jobs polling
$retriesCount = 1;
$errorsCount = 0;
do {
$waitSeconds = min(pow(2, $retriesCount), 20);
sleep($waitSeconds);
foreach ($jobResponses as $key => $jobResponse) {
$taskResult = $taskResults[$key];
$startTime = $taskResult->getStartTime()->getTimestamp();
try {
$pollResponse = null;
$timeout = $taskResult->getTimeoutMinutes() ? $taskResult->getTimeoutMinutes() : KeboolaOrchestratorBundle::TIMEOUT_MINUTES;
$data = array('status' => 'unknown');
if (time() >= $startTime + $timeout * 60) {
throw new Exception\JobRuntimeException(sprintf('Asynchronous task processing timeout after %d minutes', $timeout));
}
try {
$pollResponse = $httpClient->sendOrchestratorPollRequest($phase->getJob(), $jobResponse, $this->encryptor, $retriesCount);
if ($pollResponse) {
$pollResponses[$key] = $pollResponse;
$data = ResponseDecoder::decode($pollResponse);
}
} catch (GuzzleException\RequestException $e) {
if ($e instanceof Exception\CurlException) {
} else {
$prevE = $e->getPrevious();
if (!$prevE || !$prevE instanceof Exception\JobRuntimeException) {
throw $e;
}
}
} catch (\Exception $e) {
// maybe json decode error, do nothing
}
if (!array_key_exists('status', $data)) {
continue;
}
if (!StatusConverter::isFinishedStatus($data['status'])) {
continue;
}
if ($pollResponse) {
$response = $pollResponse;
if (empty($data) || !in_array($data['status'], array('success', 'ok'))) {
//@TODO logovat odpovedi, ktere vraci nesmyslne OK status
$fakeRequest = new Request('GET', $response->getHeaderLine(Client::EFFECTIVE_URL_HEADER_NAME));
$clientExc = new GuzzleException\ClientException('Error response from component', $fakeRequest, $response);
throw $clientExc;
}
}
if (!ResponseValidator::isSuccessfulResponse($pollResponse) && !ResponseValidator::isInformationalResponse($pollResponse)) {
$fakeRequest = new Request('GET', $pollResponse->getHeaderLine(Client::EFFECTIVE_URL_HEADER_NAME));
$clientExc = new GuzzleException\ClientException('Response with error HTTP code', $fakeRequest, $pollResponse);
throw $clientExc;
}
$endEvent = $logger->getLogger($taskResult)->end();
} catch (GuzzleException\ClientException $e) {
$endEvent = $logger->getLogger($taskResult)->clientError($e);
if ($e->getResponse()) {
$response = $e->getResponse();
}
} catch (GuzzleException\BadResponseException $e) {
$endEvent = $logger->getLogger($taskResult)->responseError($e);
if ($e->getResponse()) {
$response = $e->getResponse();
}
} catch (\Exception $e) {
$endEvent = $logger->getLogger($taskResult)->someError($e, $this->logger);
$response = null;
}
$taskResult->setResult($endEvent, empty($response) ? null : $response);
if ($taskResult->getError() && !$taskResult->getTask()->getContinueOnFailure()) {
$errorsCount++;
}
unset($jobResponses[$key]);
}
$this->saveJobResult($this->jobResult);
$retriesCount++;
} while (count($jobResponses));
if ($errorsCount) {
return false;
}
return array();
}
