private function assertJob(Job $job, $resJob)
{
$this->assertEquals($job->getId(), $resJob['id']);
$this->assertEquals($job->getRunId(), $resJob['runId']);
$this->assertEquals($job->getLockName(), $resJob['lockName']);
$this->assertEquals($job->getProject()['id'], $resJob['project']['id']);
$this->assertEquals($job->getProject()['name'], $resJob['project']['name']);
$this->assertEquals($job->getToken()['id'], $resJob['token']['id']);
$this->assertEquals($job->getToken()['description'], $resJob['token']['description']);
$this->assertEquals($job->getToken()['token'], $resJob['token']['token']);
$this->assertEquals($job->getComponent(), $resJob['component']);
$this->assertEquals($job->getStatus(), $resJob['status']);
}
private function isParallelLimitExceeded()
{
// skip validation for components without limit
if (in_array($this->job->getComponent(), Limits::unlimitedComponents())) {
$this->logger->debug('isParallelLimitExceeded - NO - unlimited component');
return false;
}
$maxLimit = Limits::getParallelLimit($this->storageApiService->getTokenData());
/** @var Search $elasticSearch */
$elasticSearch = $this->getContainer()->get('syrup.elasticsearch.search');
$jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) {
return '-component:' . $name;
}, Limits::unlimitedComponents())))));
if (count($jobs) < $maxLimit) {
$this->logger->debug('isParallelLimitExceeded - NO - free workers ' . ($maxLimit - count($jobs)));
return false;
}
$this->logger->debug('isParallelLimitExceeded - full workers ' . ($maxLimit - count($jobs)));
if ($this->job->getNestingLevel() >= 1) {
$runIds = explode('.', $this->job->getRunId());
unset($runIds[count($runIds) - 1]);
$jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s) AND (%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) {
return '-component:' . $name;
}, Limits::unlimitedComponents())), sprintf('runId:%s.*', implode('.', $runIds)), sprintf('nestingLevel:%s', $this->job->getNestingLevel()))));
if (!count($jobs)) {
$this->logger->debug('isParallelLimitExceeded - NO - free at nesting level');
return false;
}
}
$this->logger->debug('isParallelLimitExceeded - YES - any free worker');
return true;
}
public function build(\Keboola\Syrup\Job\Metadata\Job $job)
{
$params = $job->getRawParams();
$process = $job->getProcess();
$token = $job->getToken();
$project = $job->getProject();
$this->setId($job->getId());
if (!empty($params['config'])) {
$this->setConfig($params['config']);
}
$this->setProjectId($project['id']);
$this->setOrchestrationId($params['orchestration']['id']);
$this->setToken($token['token']);
$this->setTokenId($token['id']);
$this->setTokenDesc($token['description']);
$this->setTokenOwnerName($project['name']);
$this->setStatus($job->getStatus());
$this->setStartTime($job->getStartTime());
$this->setEndTime($job->getEndTime());
$this->setResults($job->getResult());
if (!empty($params['tasks'])) {
$this->setTaks($params['tasks']);
}
$this->setCreatedTime($job->getCreatedTime());
$this->setOrchestrationName($params['orchestration']['name']);
$this->setInitializedBy($params['initializedBy']);
$this->setInitiatorTokenId($params['initiator']['id']);
$this->setInitiatorTokenDesc($params['initiator']['description']);
$this->setInitiatorUserAgent($params['initiator']['userAgent']);
$this->setNotificationsEmails($params['notificationsEmails']);
$this->setRunId($job->getRunId());
$this->setPid($process['pid']);
$this->setWorkerAddress($process['host']);
}
