private function assertJob(Job $job, $resJob) { $this->assertEquals($job->getId(), $resJob['id']); $this->assertEquals($job->getRunId(), $resJob['runId']); $this->assertEquals($job->getLockName(), $resJob['lockName']); $this->assertEquals($job->getProject()['id'], $resJob['project']['id']); $this->assertEquals($job->getProject()['name'], $resJob['project']['name']); $this->assertEquals($job->getToken()['id'], $resJob['token']['id']); $this->assertEquals($job->getToken()['description'], $resJob['token']['description']); $this->assertEquals($job->getToken()['token'], $resJob['token']['token']); $this->assertEquals($job->getComponent(), $resJob['component']); $this->assertEquals($job->getStatus(), $resJob['status']); }
private function isParallelLimitExceeded() { // skip validation for components without limit if (in_array($this->job->getComponent(), Limits::unlimitedComponents())) { $this->logger->debug('isParallelLimitExceeded - NO - unlimited component'); return false; } $maxLimit = Limits::getParallelLimit($this->storageApiService->getTokenData()); /** @var Search $elasticSearch */ $elasticSearch = $this->getContainer()->get('syrup.elasticsearch.search'); $jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) { return '-component:' . $name; }, Limits::unlimitedComponents()))))); if (count($jobs) < $maxLimit) { $this->logger->debug('isParallelLimitExceeded - NO - free workers ' . ($maxLimit - count($jobs))); return false; } $this->logger->debug('isParallelLimitExceeded - full workers ' . ($maxLimit - count($jobs))); if ($this->job->getNestingLevel() >= 1) { $runIds = explode('.', $this->job->getRunId()); unset($runIds[count($runIds) - 1]); $jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s) AND (%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) { return '-component:' . $name; }, Limits::unlimitedComponents())), sprintf('runId:%s.*', implode('.', $runIds)), sprintf('nestingLevel:%s', $this->job->getNestingLevel())))); if (!count($jobs)) { $this->logger->debug('isParallelLimitExceeded - NO - free at nesting level'); return false; } } $this->logger->debug('isParallelLimitExceeded - YES - any free worker'); return true; }
public function build(\Keboola\Syrup\Job\Metadata\Job $job) { $params = $job->getRawParams(); $process = $job->getProcess(); $token = $job->getToken(); $project = $job->getProject(); $this->setId($job->getId()); if (!empty($params['config'])) { $this->setConfig($params['config']); } $this->setProjectId($project['id']); $this->setOrchestrationId($params['orchestration']['id']); $this->setToken($token['token']); $this->setTokenId($token['id']); $this->setTokenDesc($token['description']); $this->setTokenOwnerName($project['name']); $this->setStatus($job->getStatus()); $this->setStartTime($job->getStartTime()); $this->setEndTime($job->getEndTime()); $this->setResults($job->getResult()); if (!empty($params['tasks'])) { $this->setTaks($params['tasks']); } $this->setCreatedTime($job->getCreatedTime()); $this->setOrchestrationName($params['orchestration']['name']); $this->setInitializedBy($params['initializedBy']); $this->setInitiatorTokenId($params['initiator']['id']); $this->setInitiatorTokenDesc($params['initiator']['description']); $this->setInitiatorUserAgent($params['initiator']['userAgent']); $this->setNotificationsEmails($params['notificationsEmails']); $this->setRunId($job->getRunId()); $this->setPid($process['pid']); $this->setWorkerAddress($process['host']); }