protected function init($jobId)
{
$this->jobMapper = $this->getContainer()->get('syrup.elasticsearch.current_component_job_mapper');
// Get job from ES
$this->job = $this->jobMapper->get($jobId);
if ($this->job == null) {
throw new ApplicationException(sprintf("Job id '%s' not found", $jobId));
}
// SAPI init
/** @var ObjectEncryptor $encryptor */
$encryptor = $this->getContainer()->get('syrup.object_encryptor');
$this->sapiClient = new SapiClient(['token' => $encryptor->decrypt($this->job->getToken()['token']), 'url' => $this->getContainer()->getParameter('storage_api.url'), 'userAgent' => $this->job->getComponent()]);
$this->sapiClient->setRunId($this->job->getRunId());
/** @var \Keboola\Syrup\Service\StorageApi\StorageApiService $storageApiService */
$storageApiService = $this->getContainer()->get('syrup.storage_api');
$storageApiService->setClient($this->sapiClient);
/** @var \Keboola\Syrup\Monolog\Processor\JobProcessor $logProcessor */
$logProcessor = $this->getContainer()->get('syrup.monolog.job_processor');
$logProcessor->setJob($this->job);
/** @var \Keboola\Syrup\Monolog\Processor\SyslogProcessor $logProcessor */
$logProcessor = $this->getContainer()->get('syrup.monolog.syslog_processor');
$logProcessor->setRunId($this->job->getRunId());
$logProcessor->setTokenData($storageApiService->getTokenData());
// Lock DB
/** @var Connection $conn */
$conn = $this->getContainer()->get('doctrine.dbal.lock_connection');
$conn->exec('SET wait_timeout = 31536000;');
$this->lock = new Lock($conn, $this->job->getLockName());
}
private function isParallelLimitExceeded()
{
// skip validation for components without limit
if (in_array($this->job->getComponent(), Limits::unlimitedComponents())) {
$this->logger->debug('isParallelLimitExceeded - NO - unlimited component');
return false;
}
$maxLimit = Limits::getParallelLimit($this->storageApiService->getTokenData());
/** @var Search $elasticSearch */
$elasticSearch = $this->getContainer()->get('syrup.elasticsearch.search');
$jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) {
return '-component:' . $name;
}, Limits::unlimitedComponents())))));
if (count($jobs) < $maxLimit) {
$this->logger->debug('isParallelLimitExceeded - NO - free workers ' . ($maxLimit - count($jobs)));
return false;
}
$this->logger->debug('isParallelLimitExceeded - full workers ' . ($maxLimit - count($jobs)));
if ($this->job->getNestingLevel() >= 1) {
$runIds = explode('.', $this->job->getRunId());
unset($runIds[count($runIds) - 1]);
$jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s) AND (%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) {
return '-component:' . $name;
}, Limits::unlimitedComponents())), sprintf('runId:%s.*', implode('.', $runIds)), sprintf('nestingLevel:%s', $this->job->getNestingLevel()))));
if (!count($jobs)) {
$this->logger->debug('isParallelLimitExceeded - NO - free at nesting level');
return false;
}
}
$this->logger->debug('isParallelLimitExceeded - YES - any free worker');
return true;
}
/**
* Hook for modify job after job execution
*
* @param Job $job
* @return void
*/
public function postExecution(Job $job)
{
if ($job->getId() !== $this->job->getId()) {
throw new \InvalidArgumentException('Given job must be same as previous executed');
}
if ($job->getComponent() !== $this->job->getComponent()) {
throw new \InvalidArgumentException('Given job must be same as previous executed');
}
$job->setResult($job->getResult() + array(self::HOOK_RESULT_KEY => self::HOOK_RESULT_VALUE));
$this->jobMapper->update($job);
}
private function assertJob(Job $job, $resJob)
{
$this->assertEquals($job->getId(), $resJob['id']);
$this->assertEquals($job->getRunId(), $resJob['runId']);
$this->assertEquals($job->getLockName(), $resJob['lockName']);
$this->assertEquals($job->getProject()['id'], $resJob['project']['id']);
$this->assertEquals($job->getProject()['name'], $resJob['project']['name']);
$this->assertEquals($job->getToken()['id'], $resJob['token']['id']);
$this->assertEquals($job->getToken()['description'], $resJob['token']['description']);
$this->assertEquals($job->getToken()['token'], $resJob['token']['token']);
$this->assertEquals($job->getComponent(), $resJob['component']);
$this->assertEquals($job->getStatus(), $resJob['status']);
}
public function create(Job $job)
{
/** @var StorageApiService $storageApiService */
$storageApiService = $this->container->get('syrup.storage_api');
$jobExecutorName = str_replace('-', '_', $job->getComponent()) . '.job_executor';
/** @var ExecutorInterface $jobExecutor */
try {
$jobExecutor = $this->container->get($jobExecutorName);
} catch (ServiceNotFoundException $e) {
$jobExecutor = $this->container->get('syrup.job_executor');
}
$jobExecutor->setStorageApi($storageApiService->getClient());
$jobExecutor->setJob($job);
return $jobExecutor;
}
protected function validateJob(Metadata\Job $job)
{
if ($job->getComponent() !== KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME) {
throw new \InvalidArgumentException(sprintf('Executor can process only jobs from "%s". "%s" job given', KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $job->getComponent()));
}
}
