protected function init($jobId) { $this->jobMapper = $this->getContainer()->get('syrup.elasticsearch.current_component_job_mapper'); // Get job from ES $this->job = $this->jobMapper->get($jobId); if ($this->job == null) { throw new ApplicationException(sprintf("Job id '%s' not found", $jobId)); } // SAPI init /** @var ObjectEncryptor $encryptor */ $encryptor = $this->getContainer()->get('syrup.object_encryptor'); $this->sapiClient = new SapiClient(['token' => $encryptor->decrypt($this->job->getToken()['token']), 'url' => $this->getContainer()->getParameter('storage_api.url'), 'userAgent' => $this->job->getComponent()]); $this->sapiClient->setRunId($this->job->getRunId()); /** @var \Keboola\Syrup\Service\StorageApi\StorageApiService $storageApiService */ $storageApiService = $this->getContainer()->get('syrup.storage_api'); $storageApiService->setClient($this->sapiClient); /** @var \Keboola\Syrup\Monolog\Processor\JobProcessor $logProcessor */ $logProcessor = $this->getContainer()->get('syrup.monolog.job_processor'); $logProcessor->setJob($this->job); /** @var \Keboola\Syrup\Monolog\Processor\SyslogProcessor $logProcessor */ $logProcessor = $this->getContainer()->get('syrup.monolog.syslog_processor'); $logProcessor->setRunId($this->job->getRunId()); $logProcessor->setTokenData($storageApiService->getTokenData()); // Lock DB /** @var Connection $conn */ $conn = $this->getContainer()->get('doctrine.dbal.lock_connection'); $conn->exec('SET wait_timeout = 31536000;'); $this->lock = new Lock($conn, $this->job->getLockName()); }
private function isParallelLimitExceeded() { // skip validation for components without limit if (in_array($this->job->getComponent(), Limits::unlimitedComponents())) { $this->logger->debug('isParallelLimitExceeded - NO - unlimited component'); return false; } $maxLimit = Limits::getParallelLimit($this->storageApiService->getTokenData()); /** @var Search $elasticSearch */ $elasticSearch = $this->getContainer()->get('syrup.elasticsearch.search'); $jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) { return '-component:' . $name; }, Limits::unlimitedComponents()))))); if (count($jobs) < $maxLimit) { $this->logger->debug('isParallelLimitExceeded - NO - free workers ' . ($maxLimit - count($jobs))); return false; } $this->logger->debug('isParallelLimitExceeded - full workers ' . ($maxLimit - count($jobs))); if ($this->job->getNestingLevel() >= 1) { $runIds = explode('.', $this->job->getRunId()); unset($runIds[count($runIds) - 1]); $jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s) AND (%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) { return '-component:' . $name; }, Limits::unlimitedComponents())), sprintf('runId:%s.*', implode('.', $runIds)), sprintf('nestingLevel:%s', $this->job->getNestingLevel())))); if (!count($jobs)) { $this->logger->debug('isParallelLimitExceeded - NO - free at nesting level'); return false; } } $this->logger->debug('isParallelLimitExceeded - YES - any free worker'); return true; }
/** * Hook for modify job after job execution * * @param Job $job * @return void */ public function postExecution(Job $job) { if ($job->getId() !== $this->job->getId()) { throw new \InvalidArgumentException('Given job must be same as previous executed'); } if ($job->getComponent() !== $this->job->getComponent()) { throw new \InvalidArgumentException('Given job must be same as previous executed'); } $job->setResult($job->getResult() + array(self::HOOK_RESULT_KEY => self::HOOK_RESULT_VALUE)); $this->jobMapper->update($job); }
private function assertJob(Job $job, $resJob) { $this->assertEquals($job->getId(), $resJob['id']); $this->assertEquals($job->getRunId(), $resJob['runId']); $this->assertEquals($job->getLockName(), $resJob['lockName']); $this->assertEquals($job->getProject()['id'], $resJob['project']['id']); $this->assertEquals($job->getProject()['name'], $resJob['project']['name']); $this->assertEquals($job->getToken()['id'], $resJob['token']['id']); $this->assertEquals($job->getToken()['description'], $resJob['token']['description']); $this->assertEquals($job->getToken()['token'], $resJob['token']['token']); $this->assertEquals($job->getComponent(), $resJob['component']); $this->assertEquals($job->getStatus(), $resJob['status']); }
public function create(Job $job) { /** @var StorageApiService $storageApiService */ $storageApiService = $this->container->get('syrup.storage_api'); $jobExecutorName = str_replace('-', '_', $job->getComponent()) . '.job_executor'; /** @var ExecutorInterface $jobExecutor */ try { $jobExecutor = $this->container->get($jobExecutorName); } catch (ServiceNotFoundException $e) { $jobExecutor = $this->container->get('syrup.job_executor'); } $jobExecutor->setStorageApi($storageApiService->getClient()); $jobExecutor->setJob($job); return $jobExecutor; }
protected function validateJob(Metadata\Job $job) { if ($job->getComponent() !== KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME) { throw new \InvalidArgumentException(sprintf('Executor can process only jobs from "%s". "%s" job given', KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $job->getComponent())); } }