protected function init($jobId) { $this->jobMapper = $this->getContainer()->get('syrup.elasticsearch.current_component_job_mapper'); // Get job from ES $this->job = $this->jobMapper->get($jobId); if ($this->job == null) { throw new ApplicationException(sprintf("Job id '%s' not found", $jobId)); } // SAPI init /** @var ObjectEncryptor $encryptor */ $encryptor = $this->getContainer()->get('syrup.object_encryptor'); $this->sapiClient = new SapiClient(['token' => $encryptor->decrypt($this->job->getToken()['token']), 'url' => $this->getContainer()->getParameter('storage_api.url'), 'userAgent' => $this->job->getComponent()]); $this->sapiClient->setRunId($this->job->getRunId()); /** @var \Keboola\Syrup\Service\StorageApi\StorageApiService $storageApiService */ $storageApiService = $this->getContainer()->get('syrup.storage_api'); $storageApiService->setClient($this->sapiClient); /** @var \Keboola\Syrup\Monolog\Processor\JobProcessor $logProcessor */ $logProcessor = $this->getContainer()->get('syrup.monolog.job_processor'); $logProcessor->setJob($this->job); /** @var \Keboola\Syrup\Monolog\Processor\SyslogProcessor $logProcessor */ $logProcessor = $this->getContainer()->get('syrup.monolog.syslog_processor'); $logProcessor->setRunId($this->job->getRunId()); $logProcessor->setTokenData($storageApiService->getTokenData()); // Lock DB /** @var Connection $conn */ $conn = $this->getContainer()->get('doctrine.dbal.lock_connection'); $conn->exec('SET wait_timeout = 31536000;'); $this->lock = new Lock($conn, $this->job->getLockName()); }
private function isParallelLimitExceeded() { // skip validation for components without limit if (in_array($this->job->getComponent(), Limits::unlimitedComponents())) { $this->logger->debug('isParallelLimitExceeded - NO - unlimited component'); return false; } $maxLimit = Limits::getParallelLimit($this->storageApiService->getTokenData()); /** @var Search $elasticSearch */ $elasticSearch = $this->getContainer()->get('syrup.elasticsearch.search'); $jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) { return '-component:' . $name; }, Limits::unlimitedComponents()))))); if (count($jobs) < $maxLimit) { $this->logger->debug('isParallelLimitExceeded - NO - free workers ' . ($maxLimit - count($jobs))); return false; } $this->logger->debug('isParallelLimitExceeded - full workers ' . ($maxLimit - count($jobs))); if ($this->job->getNestingLevel() >= 1) { $runIds = explode('.', $this->job->getRunId()); unset($runIds[count($runIds) - 1]); $jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s) AND (%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) { return '-component:' . $name; }, Limits::unlimitedComponents())), sprintf('runId:%s.*', implode('.', $runIds)), sprintf('nestingLevel:%s', $this->job->getNestingLevel())))); if (!count($jobs)) { $this->logger->debug('isParallelLimitExceeded - NO - free at nesting level'); return false; } } $this->logger->debug('isParallelLimitExceeded - YES - any free worker'); return true; }
private function assertJob(Job $job, $resJob) { $this->assertEquals($job->getId(), $resJob['id']); $this->assertEquals($job->getRunId(), $resJob['runId']); $this->assertEquals($job->getLockName(), $resJob['lockName']); $this->assertEquals($job->getProject()['id'], $resJob['project']['id']); $this->assertEquals($job->getProject()['name'], $resJob['project']['name']); $this->assertEquals($job->getToken()['id'], $resJob['token']['id']); $this->assertEquals($job->getToken()['description'], $resJob['token']['description']); $this->assertEquals($job->getToken()['token'], $resJob['token']['token']); $this->assertEquals($job->getComponent(), $resJob['component']); $this->assertEquals($job->getStatus(), $resJob['status']); }
public function build(\Keboola\Syrup\Job\Metadata\Job $job) { $params = $job->getRawParams(); $process = $job->getProcess(); $token = $job->getToken(); $project = $job->getProject(); $this->setId($job->getId()); if (!empty($params['config'])) { $this->setConfig($params['config']); } $this->setProjectId($project['id']); $this->setOrchestrationId($params['orchestration']['id']); $this->setToken($token['token']); $this->setTokenId($token['id']); $this->setTokenDesc($token['description']); $this->setTokenOwnerName($project['name']); $this->setStatus($job->getStatus()); $this->setStartTime($job->getStartTime()); $this->setEndTime($job->getEndTime()); $this->setResults($job->getResult()); if (!empty($params['tasks'])) { $this->setTaks($params['tasks']); } $this->setCreatedTime($job->getCreatedTime()); $this->setOrchestrationName($params['orchestration']['name']); $this->setInitializedBy($params['initializedBy']); $this->setInitiatorTokenId($params['initiator']['id']); $this->setInitiatorTokenDesc($params['initiator']['description']); $this->setInitiatorUserAgent($params['initiator']['userAgent']); $this->setNotificationsEmails($params['notificationsEmails']); $this->setRunId($job->getRunId()); $this->setPid($process['pid']); $this->setWorkerAddress($process['host']); }
public function execute(Job $job) { $this->configuration->setStorageApi($this->storageApi); $accounts = $this->configuration->getAccounts(); $options = $job->getParams(); $status = []; if (isset($options['external'])) { // load files by tag from SAPI if (!isset($options['external']['account'])) { throw new UserException("Missing field 'account'"); } try { $this->configuration->create(); } catch (\Exception $e) { // create configuration if not exists } $account = new Account($this->configuration, uniqid('external')); $account->fromArray($options['external']['account']); $writer = $this->writerFactory->create($account); if (!isset($options['external']['query'])) { throw new UserException("Missing field 'query'"); } $queryString = $options['external']['query']; $queryString .= ' -tags:wr-google-drive-processed'; if (isset($options['external']['filterByRunId']) && $options['external']['filterByRunId']) { $parentRunId = $this->getParentRunId($job->getRunId()); if ($parentRunId) { $queryString .= ' +tags:runId-' . $parentRunId; } } $uploadedFiles = $this->storageApi->listFiles((new ListFilesOptions())->setQuery($queryString)); if (empty($uploadedFiles)) { throw new UserException("No file matches your query '" . $queryString . "'"); } foreach ($uploadedFiles as $uploadedFile) { $tmpFile = $this->temp->createTmpFile('wr-gd'); file_put_contents($tmpFile->getPathname(), fopen($uploadedFile['url'], 'r')); $file = new File(['id' => $uploadedFile['id'], 'title' => $uploadedFile['name'], 'targetFolder' => isset($options['external']['targetFolder']) ? $options['external']['targetFolder'] : null, 'type' => File::TYPE_FILE, 'pathname' => $tmpFile, 'size' => $uploadedFile['sizeBytes']]); $gdFiles = $writer->listFiles(['q' => "trashed=false and name='" . $uploadedFile['name'] . "'"]); if (!empty($gdFiles['files'])) { $lastGdFile = array_shift($gdFiles['files']); $file->setGoogleId($lastGdFile['id']); } $file = $writer->process($file); // tag file 'wr-google-drive-processed' $this->storageApi->addFileTag($uploadedFile['id'], 'wr-google-drive-processed'); } // delete temporary account $this->configuration->removeAccount($account->getAccountId()); } else { $fileFilter = null; if (isset($options['config'])) { if (!isset($accounts[$options['config']])) { throw new UserException("Config '" . $options['config'] . "' does not exist."); } $accounts = [$options['config'] => $accounts[$options['config']]]; if (isset($options['file'])) { /** @var Account $account */ $account = $accounts[$options['config']]; if (null == $account->getFile($options['file'])) { throw new UserException("File '" . $options['file'] . "' not found"); } $fileFilter = $options['file']; } } /** @var Account $account */ foreach ($accounts as $accountId => $account) { $writer = $this->writerFactory->create($account); $files = $account->getFiles(); /** @var File $file */ foreach ($files as $file) { if ($fileFilter != null && $file->getId() != $fileFilter) { continue; } $file->setPathname($this->temp->createTmpFile()->getPathname()); try { $tableExporter = new TableExporter($this->storageApi); $tableExporter->exportTable($file->getTableId(), $file->getPathname(), []); } catch (ClientException $e) { throw new UserException($e->getMessage(), $e, ['file' => $file->toArray()]); } $file = $writer->process($file); $status[$account->getAccountName()][$file->getTitle()] = 'ok'; } // updated changes to files $account->save(); } } return $status; }