public function cleanup(Metadata\Job $job) { $result = $job->getResult(); if (!$result) { $result = array(); } $esJob = new Job(); $esJob->build($job); try { $token = new Token($this->storageApi); } catch (StorageApiException $e) { $this->logger->error('Cleanup error - invalid token', array('jobId' => $esJob->getId())); throw new UserException(sprintf("Invalid token for job %d", $esJob->getId()), $e); } $orchestration = $this->orchestrationManager->findOrchestrationById($esJob->getOrchestrationId(), $token); if (!$orchestration) { $this->logger->error('Cleanup error - orchestration not found', array('jobId' => $esJob->getId())); throw new UserException(sprintf("Orchestration %s not found. Could not update last job", $esJob->getOrchestrationId())); } if (!empty($result['tasks'])) { foreach ($result['tasks'] as $key => $task) { // skip non processing tasks if ($task['status'] !== Metadata\Job::STATUS_PROCESSING) { continue; } $httpClient = new Client(array(), $this->logger); // skip tasks without URL if (empty($task['jobUrl'])) { $retriesCount = 1; $jobsCount = 0; do { $waitSeconds = min(pow(2, $retriesCount), 20); $this->logger->debug('Poll jobs count', array('sleep' => $waitSeconds)); sleep($waitSeconds); try { $jobsCount = $httpClient->getNonFinishedChildJobsCount($esJob, $this->encryptor); } catch (\GuzzleHttp\Exception\RequestException $e) { $this->logger->error('Cleanup jobs count error', array('sleep' => $waitSeconds, 'exception' => $e)); } catch (\Exception $e) { $this->logger->error('Cleanup jobs count error', array('sleep' => $waitSeconds, 'exception' => $e)); } $retriesCount++; } while ($jobsCount != 0); $result['tasks'][$key]['status'] = Metadata\Job::STATUS_TERMINATED; $result['tasks'][$key]['endTime'] = (new \DateTime())->format('c'); } else { $this->logger->debug('Check task status', array('task' => $task)); $retriesCount = 1; do { $data = array('status' => 'unknown'); $waitSeconds = min(pow(2, $retriesCount), 20); $this->logger->debug('Poll task status', array('task' => $task, 'sleep' => $waitSeconds)); sleep($waitSeconds); try { $response = $httpClient->getJobStatus($task['jobUrl'], $esJob, $this->encryptor); if ($response) { $data = ResponseDecoder::decode($response); $this->logger->debug('Poll response', array('task' => $task, 'reponse' => $data)); } else { $this->logger->error('Any poll response', array('task' => $task, 'sleep' => $waitSeconds)); } } catch (\GuzzleHttp\Exception\RequestException $e) { $this->logger->error('Cleanup poll job error', array('task' => $task, 'sleep' => $waitSeconds, 'exception' => $e)); } catch (\Exception $e) { $this->logger->error('Cleanup poll job error', array('task' => $task, 'sleep' => $waitSeconds, 'exception' => $e)); } $retriesCount++; } while (!array_key_exists('isFinished', $data) || $data['isFinished'] != 1); $result['tasks'][$key]['status'] = $data['status']; if (!empty($data['endTime'])) { $result['tasks'][$key]['endTime'] = $data['endTime']; } if (!empty($data['durationSeconds'])) { $result['tasks'][$key]['duration'] = $data['durationSeconds']; } } } } $jobEsManager = $this->jobManagerFactory->createJobManager($token); $this->logger->debug('Cleanup job', array('jobId' => $esJob->getId())); $job->setResult($result); $esJob->setResults($result); $jobEsManager->updateResult($esJob, $result); }