예제 #1
0
 public function execute(Job $job)
 {
     $command = $job->getCommand();
     $params = $job->getParams();
     if (!method_exists($this, $command)) {
         throw new ApplicationException(sprintf("Unknown command '%s'", $command));
     }
     return $this->{$command}($params);
 }
예제 #2
0
 public function execute(Job $job)
 {
     $params = $job->getParams();
     // simulate long running job
     for ($i = 0; $i < 120; $i++) {
         $this->logger->debug(sprintf("Job '%s' is '%s'", $job->getId(), $job->getStatus()));
         sleep(1);
     }
     if (isset($params['returnStatus']) && $params['returnStatus'] == 'error') {
         throw new ApplicationException("Application error occured");
     }
 }
예제 #3
0
 protected function execute(InputInterface $input, OutputInterface $output)
 {
     $jobId = $input->getArgument('jobId');
     $this->init($jobId);
     // Ensure that job status is 'terminating'
     if ($this->job->getStatus() != Job::STATUS_TERMINATING) {
         $this->job->setStatus(Job::STATUS_TERMINATING);
         $this->jobMapper->update($this->job);
     }
     /** @var ExecutorInterface $jobExecutor */
     $jobExecutor = $this->getContainer()->get('syrup.job_executor_factory')->create();
     // run cleanup
     $jobExecutor->cleanup($this->job);
     // Update job
     $endTime = time();
     $duration = $endTime - strtotime($this->job->getStartTime());
     $this->job->setStatus(Job::STATUS_TERMINATED);
     $this->job->setResult(['message' => 'Job has been terminated']);
     $this->job->setEndTime(date('c', $endTime));
     $this->job->setDurationSeconds($duration);
     $this->jobMapper->update($this->job);
     // run post-cleanup
     $jobExecutor->postCleanup($this->job);
     // DB unlock
     $this->lock->unlock();
 }
예제 #4
0
 public function create($command, array $params = [], $lockName = null)
 {
     $this->storageApiClient = $this->storageApiService->getClient();
     $tokenData = $this->storageApiService->getTokenData();
     $job = new Job($this->objectEncryptor, ['id' => $this->storageApiClient->generateId(), 'runId' => $this->storageApiClient->generateRunId($this->storageApiClient->getRunId()), 'project' => ['id' => $tokenData['owner']['id'], 'name' => $tokenData['owner']['name']], 'token' => ['id' => $tokenData['id'], 'description' => $tokenData['description'], 'token' => $this->objectEncryptor->encrypt($this->storageApiClient->getTokenString())], 'component' => $this->componentName, 'command' => $command, 'params' => $params, 'process' => ['host' => gethostname(), 'pid' => getmypid()], 'nestingLevel' => 0, 'createdTime' => date('c')], null, null, null);
     if ($lockName) {
         $job->setLockName($lockName);
     }
     $componentConfiguration = $this->getComponentConfiguration();
     if (isset($componentConfiguration['flags']) && in_array('encrypt', $componentConfiguration['flags'])) {
         $job->setEncrypted(true);
     }
     return $job;
 }
예제 #5
0
 public function create(Job $job)
 {
     /** @var StorageApiService $storageApiService */
     $storageApiService = $this->container->get('syrup.storage_api');
     $jobExecutorName = str_replace('-', '_', $job->getComponent()) . '.job_executor';
     /** @var ExecutorInterface $jobExecutor */
     try {
         $jobExecutor = $this->container->get($jobExecutorName);
     } catch (ServiceNotFoundException $e) {
         $jobExecutor = $this->container->get('syrup.job_executor');
     }
     $jobExecutor->setStorageApi($storageApiService->getClient());
     $jobExecutor->setJob($job);
     return $jobExecutor;
 }
예제 #6
0
 private function isParallelLimitExceeded()
 {
     // skip validation for components without limit
     if (in_array($this->job->getComponent(), Limits::unlimitedComponents())) {
         $this->logger->debug('isParallelLimitExceeded - NO - unlimited component');
         return false;
     }
     $maxLimit = Limits::getParallelLimit($this->storageApiService->getTokenData());
     /** @var Search $elasticSearch */
     $elasticSearch = $this->getContainer()->get('syrup.elasticsearch.search');
     $jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) {
         return '-component:' . $name;
     }, Limits::unlimitedComponents())))));
     if (count($jobs) < $maxLimit) {
         $this->logger->debug('isParallelLimitExceeded - NO - free workers ' . ($maxLimit - count($jobs)));
         return false;
     }
     $this->logger->debug('isParallelLimitExceeded - full workers ' . ($maxLimit - count($jobs)));
     if ($this->job->getNestingLevel() >= 1) {
         $runIds = explode('.', $this->job->getRunId());
         unset($runIds[count($runIds) - 1]);
         $jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s) AND (%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) {
             return '-component:' . $name;
         }, Limits::unlimitedComponents())), sprintf('runId:%s.*', implode('.', $runIds)), sprintf('nestingLevel:%s', $this->job->getNestingLevel()))));
         if (!count($jobs)) {
             $this->logger->debug('isParallelLimitExceeded - NO - free at nesting level');
             return false;
         }
     }
     $this->logger->debug('isParallelLimitExceeded - YES - any free worker');
     return true;
 }
예제 #7
0
 /**
  * Hook for modify job after job execution
  *
  * @param Job $job
  * @return void
  */
 public function postExecution(Job $job)
 {
     if ($job->getId() !== $this->job->getId()) {
         throw new \InvalidArgumentException('Given job must be same as previous executed');
     }
     if ($job->getComponent() !== $this->job->getComponent()) {
         throw new \InvalidArgumentException('Given job must be same as previous executed');
     }
     $job->setResult($job->getResult() + array(self::HOOK_RESULT_KEY => self::HOOK_RESULT_VALUE));
     $this->jobMapper->update($job);
 }
예제 #8
0
 public function execute(Job $job)
 {
     $this->extractor->setConfiguration($this->initConfiguration());
     return $this->extractor->run($job->getParams());
 }
예제 #9
0
 /**
  * @param Job|SyrupJob $job
  */
 public function postExecute(SyrupJob $job)
 {
     if ($this->cache[$job->getId()]) {
         $this->cache[$job->getId()]->postExecute($job);
         unset($this->cache[$job->getId()]);
     }
 }
예제 #10
0
 private function assertJob(Job $job, $resJob)
 {
     $this->assertEquals($job->getId(), $resJob['id']);
     $this->assertEquals($job->getRunId(), $resJob['runId']);
     $this->assertEquals($job->getLockName(), $resJob['lockName']);
     $this->assertEquals($job->getProject()['id'], $resJob['project']['id']);
     $this->assertEquals($job->getProject()['name'], $resJob['project']['name']);
     $this->assertEquals($job->getToken()['id'], $resJob['token']['id']);
     $this->assertEquals($job->getToken()['description'], $resJob['token']['description']);
     $this->assertEquals($job->getToken()['token'], $resJob['token']['token']);
     $this->assertEquals($job->getComponent(), $resJob['component']);
     $this->assertEquals($job->getStatus(), $resJob['status']);
 }
 /**
  * @param Metadata\Job $job
  * @return Client
  */
 private function createJobSapiClient(Metadata\Job $job)
 {
     return new Client(array('token' => $this->encryptor->decrypt($job->getToken()['token']), 'url' => $this->getConfiguration()->getStorageApiUrl()));
 }
예제 #12
0
 /**
  * Hook for modify job after job execution
  *
  * @param Job $job
  * @return void
  */
 public function postExecution(Job $job)
 {
     $job->setResult($job->getResult() + array(self::HOOK_RESULT_KEY => self::HOOK_RESULT_VALUE));
     $this->jobMapper->update($job);
 }
예제 #13
0
 /**
  * @param Job $job
  * @return array
  * @throws ApplicationException
  */
 public function execute(Job $job)
 {
     $params = $job->getParams();
     $command = $job->getCommand();
     switch ($command) {
         case 'create':
             $response = $this->createWorkspace($params);
             break;
         case 'delete':
             $response = $this->deleteWorkspace($params);
             break;
         default:
             throw new ApplicationException("Unknown command {$command}");
     }
     return $response;
 }
예제 #14
0
 protected function validateJob(Metadata\Job $job)
 {
     if ($job->getComponent() !== KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME) {
         throw new \InvalidArgumentException(sprintf('Executor can process only jobs from "%s". "%s" job given', KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $job->getComponent()));
     }
 }
예제 #15
0
 public function cleanup(Metadata\Job $job)
 {
     $result = $job->getResult();
     if (!$result) {
         $result = array();
     }
     $esJob = new Job();
     $esJob->build($job);
     try {
         $token = new Token($this->storageApi);
     } catch (StorageApiException $e) {
         $this->logger->error('Cleanup error - invalid token', array('jobId' => $esJob->getId()));
         throw new UserException(sprintf("Invalid token for job %d", $esJob->getId()), $e);
     }
     $orchestration = $this->orchestrationManager->findOrchestrationById($esJob->getOrchestrationId(), $token);
     if (!$orchestration) {
         $this->logger->error('Cleanup error - orchestration not found', array('jobId' => $esJob->getId()));
         throw new UserException(sprintf("Orchestration %s not found. Could not update last job", $esJob->getOrchestrationId()));
     }
     if (!empty($result['tasks'])) {
         foreach ($result['tasks'] as $key => $task) {
             // skip non processing tasks
             if ($task['status'] !== Metadata\Job::STATUS_PROCESSING) {
                 continue;
             }
             $httpClient = new Client(array(), $this->logger);
             // skip tasks without URL
             if (empty($task['jobUrl'])) {
                 $retriesCount = 1;
                 $jobsCount = 0;
                 do {
                     $waitSeconds = min(pow(2, $retriesCount), 20);
                     $this->logger->debug('Poll jobs count', array('sleep' => $waitSeconds));
                     sleep($waitSeconds);
                     try {
                         $jobsCount = $httpClient->getNonFinishedChildJobsCount($esJob, $this->encryptor);
                     } catch (\GuzzleHttp\Exception\RequestException $e) {
                         $this->logger->error('Cleanup jobs count error', array('sleep' => $waitSeconds, 'exception' => $e));
                     } catch (\Exception $e) {
                         $this->logger->error('Cleanup jobs count error', array('sleep' => $waitSeconds, 'exception' => $e));
                     }
                     $retriesCount++;
                 } while ($jobsCount != 0);
                 $result['tasks'][$key]['status'] = Metadata\Job::STATUS_TERMINATED;
                 $result['tasks'][$key]['endTime'] = (new \DateTime())->format('c');
             } else {
                 $this->logger->debug('Check task status', array('task' => $task));
                 $retriesCount = 1;
                 do {
                     $data = array('status' => 'unknown');
                     $waitSeconds = min(pow(2, $retriesCount), 20);
                     $this->logger->debug('Poll task status', array('task' => $task, 'sleep' => $waitSeconds));
                     sleep($waitSeconds);
                     try {
                         $response = $httpClient->getJobStatus($task['jobUrl'], $esJob, $this->encryptor);
                         if ($response) {
                             $data = ResponseDecoder::decode($response);
                             $this->logger->debug('Poll response', array('task' => $task, 'reponse' => $data));
                         } else {
                             $this->logger->error('Any poll response', array('task' => $task, 'sleep' => $waitSeconds));
                         }
                     } catch (\GuzzleHttp\Exception\RequestException $e) {
                         $this->logger->error('Cleanup poll job error', array('task' => $task, 'sleep' => $waitSeconds, 'exception' => $e));
                     } catch (\Exception $e) {
                         $this->logger->error('Cleanup poll job error', array('task' => $task, 'sleep' => $waitSeconds, 'exception' => $e));
                     }
                     $retriesCount++;
                 } while (!array_key_exists('isFinished', $data) || $data['isFinished'] != 1);
                 $result['tasks'][$key]['status'] = $data['status'];
                 if (!empty($data['endTime'])) {
                     $result['tasks'][$key]['endTime'] = $data['endTime'];
                 }
                 if (!empty($data['durationSeconds'])) {
                     $result['tasks'][$key]['duration'] = $data['durationSeconds'];
                 }
             }
         }
     }
     $jobEsManager = $this->jobManagerFactory->createJobManager($token);
     $this->logger->debug('Cleanup job', array('jobId' => $esJob->getId()));
     $job->setResult($result);
     $esJob->setResults($result);
     $jobEsManager->updateResult($esJob, $result);
 }
 public function execute(Job $job)
 {
     $this->configuration->setStorageApi($this->storageApi);
     $accounts = $this->configuration->getAccounts();
     $options = $job->getParams();
     $status = [];
     if (isset($options['external'])) {
         // load files by tag from SAPI
         if (!isset($options['external']['account'])) {
             throw new UserException("Missing field 'account'");
         }
         try {
             $this->configuration->create();
         } catch (\Exception $e) {
             // create configuration if not exists
         }
         $account = new Account($this->configuration, uniqid('external'));
         $account->fromArray($options['external']['account']);
         $writer = $this->writerFactory->create($account);
         if (!isset($options['external']['query'])) {
             throw new UserException("Missing field 'query'");
         }
         $queryString = $options['external']['query'];
         $queryString .= ' -tags:wr-google-drive-processed';
         if (isset($options['external']['filterByRunId']) && $options['external']['filterByRunId']) {
             $parentRunId = $this->getParentRunId($job->getRunId());
             if ($parentRunId) {
                 $queryString .= ' +tags:runId-' . $parentRunId;
             }
         }
         $uploadedFiles = $this->storageApi->listFiles((new ListFilesOptions())->setQuery($queryString));
         if (empty($uploadedFiles)) {
             throw new UserException("No file matches your query '" . $queryString . "'");
         }
         foreach ($uploadedFiles as $uploadedFile) {
             $tmpFile = $this->temp->createTmpFile('wr-gd');
             file_put_contents($tmpFile->getPathname(), fopen($uploadedFile['url'], 'r'));
             $file = new File(['id' => $uploadedFile['id'], 'title' => $uploadedFile['name'], 'targetFolder' => isset($options['external']['targetFolder']) ? $options['external']['targetFolder'] : null, 'type' => File::TYPE_FILE, 'pathname' => $tmpFile, 'size' => $uploadedFile['sizeBytes']]);
             $gdFiles = $writer->listFiles(['q' => "trashed=false and name='" . $uploadedFile['name'] . "'"]);
             if (!empty($gdFiles['files'])) {
                 $lastGdFile = array_shift($gdFiles['files']);
                 $file->setGoogleId($lastGdFile['id']);
             }
             $file = $writer->process($file);
             // tag file 'wr-google-drive-processed'
             $this->storageApi->addFileTag($uploadedFile['id'], 'wr-google-drive-processed');
         }
         // delete temporary account
         $this->configuration->removeAccount($account->getAccountId());
     } else {
         $fileFilter = null;
         if (isset($options['config'])) {
             if (!isset($accounts[$options['config']])) {
                 throw new UserException("Config '" . $options['config'] . "' does not exist.");
             }
             $accounts = [$options['config'] => $accounts[$options['config']]];
             if (isset($options['file'])) {
                 /** @var Account $account */
                 $account = $accounts[$options['config']];
                 if (null == $account->getFile($options['file'])) {
                     throw new UserException("File '" . $options['file'] . "' not found");
                 }
                 $fileFilter = $options['file'];
             }
         }
         /** @var Account $account */
         foreach ($accounts as $accountId => $account) {
             $writer = $this->writerFactory->create($account);
             $files = $account->getFiles();
             /** @var File $file */
             foreach ($files as $file) {
                 if ($fileFilter != null && $file->getId() != $fileFilter) {
                     continue;
                 }
                 $file->setPathname($this->temp->createTmpFile()->getPathname());
                 try {
                     $tableExporter = new TableExporter($this->storageApi);
                     $tableExporter->exportTable($file->getTableId(), $file->getPathname(), []);
                 } catch (ClientException $e) {
                     throw new UserException($e->getMessage(), $e, ['file' => $file->toArray()]);
                 }
                 $file = $writer->process($file);
                 $status[$account->getAccountName()][$file->getTitle()] = 'ok';
             }
             // updated changes to files
             $account->save();
         }
     }
     return $status;
 }
예제 #17
0
 public function build(\Keboola\Syrup\Job\Metadata\Job $job)
 {
     $params = $job->getRawParams();
     $process = $job->getProcess();
     $token = $job->getToken();
     $project = $job->getProject();
     $this->setId($job->getId());
     if (!empty($params['config'])) {
         $this->setConfig($params['config']);
     }
     $this->setProjectId($project['id']);
     $this->setOrchestrationId($params['orchestration']['id']);
     $this->setToken($token['token']);
     $this->setTokenId($token['id']);
     $this->setTokenDesc($token['description']);
     $this->setTokenOwnerName($project['name']);
     $this->setStatus($job->getStatus());
     $this->setStartTime($job->getStartTime());
     $this->setEndTime($job->getEndTime());
     $this->setResults($job->getResult());
     if (!empty($params['tasks'])) {
         $this->setTaks($params['tasks']);
     }
     $this->setCreatedTime($job->getCreatedTime());
     $this->setOrchestrationName($params['orchestration']['name']);
     $this->setInitializedBy($params['initializedBy']);
     $this->setInitiatorTokenId($params['initiator']['id']);
     $this->setInitiatorTokenDesc($params['initiator']['description']);
     $this->setInitiatorUserAgent($params['initiator']['userAgent']);
     $this->setNotificationsEmails($params['notificationsEmails']);
     $this->setRunId($job->getRunId());
     $this->setPid($process['pid']);
     $this->setWorkerAddress($process['host']);
 }
예제 #18
0
 private function executeJob($command, $data)
 {
     /** @var StorageApiService $storageApi */
     $storageApi = $this->getSapiServiceStub();
     $logger = new Logger("null");
     $logger->pushHandler(new NullHandler());
     $serverFactory = new ServerFactory($this->doctrine, $storageApi, $logger);
     $jobExecutor = new Executor($logger, $serverFactory);
     $jobExecutor->setStorageApi($storageApi->getClient());
     $encryptor = new ObjectEncryptor();
     $job = new Job($encryptor, $data);
     $job->setCommand($command);
     $job->setId(123456);
     $jobExecutor->setStorageApi($this->client);
     return $jobExecutor->execute($job);
 }
예제 #19
0
파일: Executor.php 프로젝트: keboola/syrup
 public function postCleanup(Job $job)
 {
     $oldRes = $job->getResult();
     $job->setResult(['message' => $oldRes['message'] . '&cleared']);
     $this->jobMapper->update($job);
 }
예제 #20
0
 protected function makeJob($parentRunId = null, $isV2 = false)
 {
     /** @var Encryptor $encryptor */
     $encryptor = $this->httpClient->getContainer()->get('syrup.encryptor');
     /** @var ObjectEncryptor $objectEncryptor */
     $objectEncryptor = $this->httpClient->getContainer()->get('syrup.object_encryptor');
     // create job
     $tokenData = $this->storageApiClient->verifyToken();
     $componentName = 'queue';
     $jobId = $this->storageApiClient->generateId();
     $job = new Job($objectEncryptor);
     $job->setId($jobId);
     $job->setRunId(is_null($parentRunId) ? $this->storageApiClient->generateId() : $parentRunId . '.' . $this->storageApiClient->generateId());
     $job->setCommand('run');
     $job->setComponent($componentName);
     $job->setLockName('syrup-queue-test');
     if ($parentRunId) {
         $job->setLockName($job->getLockName() . '-child');
     }
     $job->setToken(['id' => $tokenData['id'], 'description' => $tokenData['description'], 'token' => $encryptor->encrypt($this->storageApiClient->getTokenString())]);
     $job->setStatus(Job::STATUS_WAITING);
     $job->setCreatedTime(date('c'));
     $job->setProject(['id' => $tokenData['owner']['id'], 'name' => $tokenData['owner']['name']]);
     if ($isV2) {
         $job->setAttribute('protocol', 'v2');
     }
     return $job;
 }
 /**
  * @param $writerId
  * @return JsonResponse
  */
 public function cancelWaitingJobsAction($writerId)
 {
     $sapiData = $this->storageApi->getLogData();
     $projectId = $sapiData['owner']['id'];
     $query = "(status:waiting)AND(writer:{$writerId})";
     $jobs = $this->getElasticSearch()->getJobs(['projectId' => $projectId, 'component' => $this->getParameter("app_name"), 'query' => $query]);
     $jobMapper = $this->getJobMapper();
     foreach ($jobs as $item) {
         $job = new Job($item);
         $job->setStatus(Job::STATUS_CANCELLED);
         $jobMapper->update($job);
     }
     return $this->createJsonResponse([]);
 }
예제 #22
0
 public function execute(Job $job)
 {
     $this->extractor->extract($this->getConfiguration(), $job->getParams());
 }