protected function init($jobId)
{
$this->jobMapper = $this->getContainer()->get('syrup.elasticsearch.current_component_job_mapper');
// Get job from ES
$this->job = $this->jobMapper->get($jobId);
if ($this->job == null) {
throw new ApplicationException(sprintf("Job id '%s' not found", $jobId));
}
// SAPI init
/** @var ObjectEncryptor $encryptor */
$encryptor = $this->getContainer()->get('syrup.object_encryptor');
$this->sapiClient = new SapiClient(['token' => $encryptor->decrypt($this->job->getToken()['token']), 'url' => $this->getContainer()->getParameter('storage_api.url'), 'userAgent' => $this->job->getComponent()]);
$this->sapiClient->setRunId($this->job->getRunId());
/** @var \Keboola\Syrup\Service\StorageApi\StorageApiService $storageApiService */
$storageApiService = $this->getContainer()->get('syrup.storage_api');
$storageApiService->setClient($this->sapiClient);
/** @var \Keboola\Syrup\Monolog\Processor\JobProcessor $logProcessor */
$logProcessor = $this->getContainer()->get('syrup.monolog.job_processor');
$logProcessor->setJob($this->job);
/** @var \Keboola\Syrup\Monolog\Processor\SyslogProcessor $logProcessor */
$logProcessor = $this->getContainer()->get('syrup.monolog.syslog_processor');
$logProcessor->setRunId($this->job->getRunId());
$logProcessor->setTokenData($storageApiService->getTokenData());
// Lock DB
/** @var Connection $conn */
$conn = $this->getContainer()->get('doctrine.dbal.lock_connection');
$conn->exec('SET wait_timeout = 31536000;');
$this->lock = new Lock($conn, $this->job->getLockName());
}
private function isParallelLimitExceeded()
{
// skip validation for components without limit
if (in_array($this->job->getComponent(), Limits::unlimitedComponents())) {
$this->logger->debug('isParallelLimitExceeded - NO - unlimited component');
return false;
}
$maxLimit = Limits::getParallelLimit($this->storageApiService->getTokenData());
/** @var Search $elasticSearch */
$elasticSearch = $this->getContainer()->get('syrup.elasticsearch.search');
$jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) {
return '-component:' . $name;
}, Limits::unlimitedComponents())))));
if (count($jobs) < $maxLimit) {
$this->logger->debug('isParallelLimitExceeded - NO - free workers ' . ($maxLimit - count($jobs)));
return false;
}
$this->logger->debug('isParallelLimitExceeded - full workers ' . ($maxLimit - count($jobs)));
if ($this->job->getNestingLevel() >= 1) {
$runIds = explode('.', $this->job->getRunId());
unset($runIds[count($runIds) - 1]);
$jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s) AND (%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) {
return '-component:' . $name;
}, Limits::unlimitedComponents())), sprintf('runId:%s.*', implode('.', $runIds)), sprintf('nestingLevel:%s', $this->job->getNestingLevel()))));
if (!count($jobs)) {
$this->logger->debug('isParallelLimitExceeded - NO - free at nesting level');
return false;
}
}
$this->logger->debug('isParallelLimitExceeded - YES - any free worker');
return true;
}
private function assertJob(Job $job, $resJob)
{
$this->assertEquals($job->getId(), $resJob['id']);
$this->assertEquals($job->getRunId(), $resJob['runId']);
$this->assertEquals($job->getLockName(), $resJob['lockName']);
$this->assertEquals($job->getProject()['id'], $resJob['project']['id']);
$this->assertEquals($job->getProject()['name'], $resJob['project']['name']);
$this->assertEquals($job->getToken()['id'], $resJob['token']['id']);
$this->assertEquals($job->getToken()['description'], $resJob['token']['description']);
$this->assertEquals($job->getToken()['token'], $resJob['token']['token']);
$this->assertEquals($job->getComponent(), $resJob['component']);
$this->assertEquals($job->getStatus(), $resJob['status']);
}
public function build(\Keboola\Syrup\Job\Metadata\Job $job)
{
$params = $job->getRawParams();
$process = $job->getProcess();
$token = $job->getToken();
$project = $job->getProject();
$this->setId($job->getId());
if (!empty($params['config'])) {
$this->setConfig($params['config']);
}
$this->setProjectId($project['id']);
$this->setOrchestrationId($params['orchestration']['id']);
$this->setToken($token['token']);
$this->setTokenId($token['id']);
$this->setTokenDesc($token['description']);
$this->setTokenOwnerName($project['name']);
$this->setStatus($job->getStatus());
$this->setStartTime($job->getStartTime());
$this->setEndTime($job->getEndTime());
$this->setResults($job->getResult());
if (!empty($params['tasks'])) {
$this->setTaks($params['tasks']);
}
$this->setCreatedTime($job->getCreatedTime());
$this->setOrchestrationName($params['orchestration']['name']);
$this->setInitializedBy($params['initializedBy']);
$this->setInitiatorTokenId($params['initiator']['id']);
$this->setInitiatorTokenDesc($params['initiator']['description']);
$this->setInitiatorUserAgent($params['initiator']['userAgent']);
$this->setNotificationsEmails($params['notificationsEmails']);
$this->setRunId($job->getRunId());
$this->setPid($process['pid']);
$this->setWorkerAddress($process['host']);
}
public function execute(Job $job)
{
$this->configuration->setStorageApi($this->storageApi);
$accounts = $this->configuration->getAccounts();
$options = $job->getParams();
$status = [];
if (isset($options['external'])) {
// load files by tag from SAPI
if (!isset($options['external']['account'])) {
throw new UserException("Missing field 'account'");
}
try {
$this->configuration->create();
} catch (\Exception $e) {
// create configuration if not exists
}
$account = new Account($this->configuration, uniqid('external'));
$account->fromArray($options['external']['account']);
$writer = $this->writerFactory->create($account);
if (!isset($options['external']['query'])) {
throw new UserException("Missing field 'query'");
}
$queryString = $options['external']['query'];
$queryString .= ' -tags:wr-google-drive-processed';
if (isset($options['external']['filterByRunId']) && $options['external']['filterByRunId']) {
$parentRunId = $this->getParentRunId($job->getRunId());
if ($parentRunId) {
$queryString .= ' +tags:runId-' . $parentRunId;
}
}
$uploadedFiles = $this->storageApi->listFiles((new ListFilesOptions())->setQuery($queryString));
if (empty($uploadedFiles)) {
throw new UserException("No file matches your query '" . $queryString . "'");
}
foreach ($uploadedFiles as $uploadedFile) {
$tmpFile = $this->temp->createTmpFile('wr-gd');
file_put_contents($tmpFile->getPathname(), fopen($uploadedFile['url'], 'r'));
$file = new File(['id' => $uploadedFile['id'], 'title' => $uploadedFile['name'], 'targetFolder' => isset($options['external']['targetFolder']) ? $options['external']['targetFolder'] : null, 'type' => File::TYPE_FILE, 'pathname' => $tmpFile, 'size' => $uploadedFile['sizeBytes']]);
$gdFiles = $writer->listFiles(['q' => "trashed=false and name='" . $uploadedFile['name'] . "'"]);
if (!empty($gdFiles['files'])) {
$lastGdFile = array_shift($gdFiles['files']);
$file->setGoogleId($lastGdFile['id']);
}
$file = $writer->process($file);
// tag file 'wr-google-drive-processed'
$this->storageApi->addFileTag($uploadedFile['id'], 'wr-google-drive-processed');
}
// delete temporary account
$this->configuration->removeAccount($account->getAccountId());
} else {
$fileFilter = null;
if (isset($options['config'])) {
if (!isset($accounts[$options['config']])) {
throw new UserException("Config '" . $options['config'] . "' does not exist.");
}
$accounts = [$options['config'] => $accounts[$options['config']]];
if (isset($options['file'])) {
/** @var Account $account */
$account = $accounts[$options['config']];
if (null == $account->getFile($options['file'])) {
throw new UserException("File '" . $options['file'] . "' not found");
}
$fileFilter = $options['file'];
}
}
/** @var Account $account */
foreach ($accounts as $accountId => $account) {
$writer = $this->writerFactory->create($account);
$files = $account->getFiles();
/** @var File $file */
foreach ($files as $file) {
if ($fileFilter != null && $file->getId() != $fileFilter) {
continue;
}
$file->setPathname($this->temp->createTmpFile()->getPathname());
try {
$tableExporter = new TableExporter($this->storageApi);
$tableExporter->exportTable($file->getTableId(), $file->getPathname(), []);
} catch (ClientException $e) {
throw new UserException($e->getMessage(), $e, ['file' => $file->toArray()]);
}
$file = $writer->process($file);
$status[$account->getAccountName()][$file->getTitle()] = 'ok';
}
// updated changes to files
$account->save();
}
}
return $status;
}
