public function execute(Job $job)
{
$command = $job->getCommand();
$params = $job->getParams();
if (!method_exists($this, $command)) {
throw new ApplicationException(sprintf("Unknown command '%s'", $command));
}
return $this->{$command}($params);
}
public function execute(Job $job)
{
$params = $job->getParams();
// simulate long running job
for ($i = 0; $i < 120; $i++) {
$this->logger->debug(sprintf("Job '%s' is '%s'", $job->getId(), $job->getStatus()));
sleep(1);
}
if (isset($params['returnStatus']) && $params['returnStatus'] == 'error') {
throw new ApplicationException("Application error occured");
}
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$jobId = $input->getArgument('jobId');
$this->init($jobId);
// Ensure that job status is 'terminating'
if ($this->job->getStatus() != Job::STATUS_TERMINATING) {
$this->job->setStatus(Job::STATUS_TERMINATING);
$this->jobMapper->update($this->job);
}
/** @var ExecutorInterface $jobExecutor */
$jobExecutor = $this->getContainer()->get('syrup.job_executor_factory')->create();
// run cleanup
$jobExecutor->cleanup($this->job);
// Update job
$endTime = time();
$duration = $endTime - strtotime($this->job->getStartTime());
$this->job->setStatus(Job::STATUS_TERMINATED);
$this->job->setResult(['message' => 'Job has been terminated']);
$this->job->setEndTime(date('c', $endTime));
$this->job->setDurationSeconds($duration);
$this->jobMapper->update($this->job);
// run post-cleanup
$jobExecutor->postCleanup($this->job);
// DB unlock
$this->lock->unlock();
}
public function create($command, array $params = [], $lockName = null)
{
$this->storageApiClient = $this->storageApiService->getClient();
$tokenData = $this->storageApiService->getTokenData();
$job = new Job($this->objectEncryptor, ['id' => $this->storageApiClient->generateId(), 'runId' => $this->storageApiClient->generateRunId($this->storageApiClient->getRunId()), 'project' => ['id' => $tokenData['owner']['id'], 'name' => $tokenData['owner']['name']], 'token' => ['id' => $tokenData['id'], 'description' => $tokenData['description'], 'token' => $this->objectEncryptor->encrypt($this->storageApiClient->getTokenString())], 'component' => $this->componentName, 'command' => $command, 'params' => $params, 'process' => ['host' => gethostname(), 'pid' => getmypid()], 'nestingLevel' => 0, 'createdTime' => date('c')], null, null, null);
if ($lockName) {
$job->setLockName($lockName);
}
$componentConfiguration = $this->getComponentConfiguration();
if (isset($componentConfiguration['flags']) && in_array('encrypt', $componentConfiguration['flags'])) {
$job->setEncrypted(true);
}
return $job;
}
public function create(Job $job)
{
/** @var StorageApiService $storageApiService */
$storageApiService = $this->container->get('syrup.storage_api');
$jobExecutorName = str_replace('-', '_', $job->getComponent()) . '.job_executor';
/** @var ExecutorInterface $jobExecutor */
try {
$jobExecutor = $this->container->get($jobExecutorName);
} catch (ServiceNotFoundException $e) {
$jobExecutor = $this->container->get('syrup.job_executor');
}
$jobExecutor->setStorageApi($storageApiService->getClient());
$jobExecutor->setJob($job);
return $jobExecutor;
}
private function isParallelLimitExceeded()
{
// skip validation for components without limit
if (in_array($this->job->getComponent(), Limits::unlimitedComponents())) {
$this->logger->debug('isParallelLimitExceeded - NO - unlimited component');
return false;
}
$maxLimit = Limits::getParallelLimit($this->storageApiService->getTokenData());
/** @var Search $elasticSearch */
$elasticSearch = $this->getContainer()->get('syrup.elasticsearch.search');
$jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) {
return '-component:' . $name;
}, Limits::unlimitedComponents())))));
if (count($jobs) < $maxLimit) {
$this->logger->debug('isParallelLimitExceeded - NO - free workers ' . ($maxLimit - count($jobs)));
return false;
}
$this->logger->debug('isParallelLimitExceeded - full workers ' . ($maxLimit - count($jobs)));
if ($this->job->getNestingLevel() >= 1) {
$runIds = explode('.', $this->job->getRunId());
unset($runIds[count($runIds) - 1]);
$jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s) AND (%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) {
return '-component:' . $name;
}, Limits::unlimitedComponents())), sprintf('runId:%s.*', implode('.', $runIds)), sprintf('nestingLevel:%s', $this->job->getNestingLevel()))));
if (!count($jobs)) {
$this->logger->debug('isParallelLimitExceeded - NO - free at nesting level');
return false;
}
}
$this->logger->debug('isParallelLimitExceeded - YES - any free worker');
return true;
}
/**
* Hook for modify job after job execution
*
* @param Job $job
* @return void
*/
public function postExecution(Job $job)
{
if ($job->getId() !== $this->job->getId()) {
throw new \InvalidArgumentException('Given job must be same as previous executed');
}
if ($job->getComponent() !== $this->job->getComponent()) {
throw new \InvalidArgumentException('Given job must be same as previous executed');
}
$job->setResult($job->getResult() + array(self::HOOK_RESULT_KEY => self::HOOK_RESULT_VALUE));
$this->jobMapper->update($job);
}
public function execute(Job $job)
{
$this->extractor->setConfiguration($this->initConfiguration());
return $this->extractor->run($job->getParams());
}
/**
* @param Job|SyrupJob $job
*/
public function postExecute(SyrupJob $job)
{
if ($this->cache[$job->getId()]) {
$this->cache[$job->getId()]->postExecute($job);
unset($this->cache[$job->getId()]);
}
}
private function assertJob(Job $job, $resJob)
{
$this->assertEquals($job->getId(), $resJob['id']);
$this->assertEquals($job->getRunId(), $resJob['runId']);
$this->assertEquals($job->getLockName(), $resJob['lockName']);
$this->assertEquals($job->getProject()['id'], $resJob['project']['id']);
$this->assertEquals($job->getProject()['name'], $resJob['project']['name']);
$this->assertEquals($job->getToken()['id'], $resJob['token']['id']);
$this->assertEquals($job->getToken()['description'], $resJob['token']['description']);
$this->assertEquals($job->getToken()['token'], $resJob['token']['token']);
$this->assertEquals($job->getComponent(), $resJob['component']);
$this->assertEquals($job->getStatus(), $resJob['status']);
}
/**
* @param Metadata\Job $job
* @return Client
*/
private function createJobSapiClient(Metadata\Job $job)
{
return new Client(array('token' => $this->encryptor->decrypt($job->getToken()['token']), 'url' => $this->getConfiguration()->getStorageApiUrl()));
}
/**
* Hook for modify job after job execution
*
* @param Job $job
* @return void
*/
public function postExecution(Job $job)
{
$job->setResult($job->getResult() + array(self::HOOK_RESULT_KEY => self::HOOK_RESULT_VALUE));
$this->jobMapper->update($job);
}
/**
* @param Job $job
* @return array
* @throws ApplicationException
*/
public function execute(Job $job)
{
$params = $job->getParams();
$command = $job->getCommand();
switch ($command) {
case 'create':
$response = $this->createWorkspace($params);
break;
case 'delete':
$response = $this->deleteWorkspace($params);
break;
default:
throw new ApplicationException("Unknown command {$command}");
}
return $response;
}
protected function validateJob(Metadata\Job $job)
{
if ($job->getComponent() !== KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME) {
throw new \InvalidArgumentException(sprintf('Executor can process only jobs from "%s". "%s" job given', KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $job->getComponent()));
}
}
public function cleanup(Metadata\Job $job)
{
$result = $job->getResult();
if (!$result) {
$result = array();
}
$esJob = new Job();
$esJob->build($job);
try {
$token = new Token($this->storageApi);
} catch (StorageApiException $e) {
$this->logger->error('Cleanup error - invalid token', array('jobId' => $esJob->getId()));
throw new UserException(sprintf("Invalid token for job %d", $esJob->getId()), $e);
}
$orchestration = $this->orchestrationManager->findOrchestrationById($esJob->getOrchestrationId(), $token);
if (!$orchestration) {
$this->logger->error('Cleanup error - orchestration not found', array('jobId' => $esJob->getId()));
throw new UserException(sprintf("Orchestration %s not found. Could not update last job", $esJob->getOrchestrationId()));
}
if (!empty($result['tasks'])) {
foreach ($result['tasks'] as $key => $task) {
// skip non processing tasks
if ($task['status'] !== Metadata\Job::STATUS_PROCESSING) {
continue;
}
$httpClient = new Client(array(), $this->logger);
// skip tasks without URL
if (empty($task['jobUrl'])) {
$retriesCount = 1;
$jobsCount = 0;
do {
$waitSeconds = min(pow(2, $retriesCount), 20);
$this->logger->debug('Poll jobs count', array('sleep' => $waitSeconds));
sleep($waitSeconds);
try {
$jobsCount = $httpClient->getNonFinishedChildJobsCount($esJob, $this->encryptor);
} catch (\GuzzleHttp\Exception\RequestException $e) {
$this->logger->error('Cleanup jobs count error', array('sleep' => $waitSeconds, 'exception' => $e));
} catch (\Exception $e) {
$this->logger->error('Cleanup jobs count error', array('sleep' => $waitSeconds, 'exception' => $e));
}
$retriesCount++;
} while ($jobsCount != 0);
$result['tasks'][$key]['status'] = Metadata\Job::STATUS_TERMINATED;
$result['tasks'][$key]['endTime'] = (new \DateTime())->format('c');
} else {
$this->logger->debug('Check task status', array('task' => $task));
$retriesCount = 1;
do {
$data = array('status' => 'unknown');
$waitSeconds = min(pow(2, $retriesCount), 20);
$this->logger->debug('Poll task status', array('task' => $task, 'sleep' => $waitSeconds));
sleep($waitSeconds);
try {
$response = $httpClient->getJobStatus($task['jobUrl'], $esJob, $this->encryptor);
if ($response) {
$data = ResponseDecoder::decode($response);
$this->logger->debug('Poll response', array('task' => $task, 'reponse' => $data));
} else {
$this->logger->error('Any poll response', array('task' => $task, 'sleep' => $waitSeconds));
}
} catch (\GuzzleHttp\Exception\RequestException $e) {
$this->logger->error('Cleanup poll job error', array('task' => $task, 'sleep' => $waitSeconds, 'exception' => $e));
} catch (\Exception $e) {
$this->logger->error('Cleanup poll job error', array('task' => $task, 'sleep' => $waitSeconds, 'exception' => $e));
}
$retriesCount++;
} while (!array_key_exists('isFinished', $data) || $data['isFinished'] != 1);
$result['tasks'][$key]['status'] = $data['status'];
if (!empty($data['endTime'])) {
$result['tasks'][$key]['endTime'] = $data['endTime'];
}
if (!empty($data['durationSeconds'])) {
$result['tasks'][$key]['duration'] = $data['durationSeconds'];
}
}
}
}
$jobEsManager = $this->jobManagerFactory->createJobManager($token);
$this->logger->debug('Cleanup job', array('jobId' => $esJob->getId()));
$job->setResult($result);
$esJob->setResults($result);
$jobEsManager->updateResult($esJob, $result);
}
public function execute(Job $job)
{
$this->configuration->setStorageApi($this->storageApi);
$accounts = $this->configuration->getAccounts();
$options = $job->getParams();
$status = [];
if (isset($options['external'])) {
// load files by tag from SAPI
if (!isset($options['external']['account'])) {
throw new UserException("Missing field 'account'");
}
try {
$this->configuration->create();
} catch (\Exception $e) {
// create configuration if not exists
}
$account = new Account($this->configuration, uniqid('external'));
$account->fromArray($options['external']['account']);
$writer = $this->writerFactory->create($account);
if (!isset($options['external']['query'])) {
throw new UserException("Missing field 'query'");
}
$queryString = $options['external']['query'];
$queryString .= ' -tags:wr-google-drive-processed';
if (isset($options['external']['filterByRunId']) && $options['external']['filterByRunId']) {
$parentRunId = $this->getParentRunId($job->getRunId());
if ($parentRunId) {
$queryString .= ' +tags:runId-' . $parentRunId;
}
}
$uploadedFiles = $this->storageApi->listFiles((new ListFilesOptions())->setQuery($queryString));
if (empty($uploadedFiles)) {
throw new UserException("No file matches your query '" . $queryString . "'");
}
foreach ($uploadedFiles as $uploadedFile) {
$tmpFile = $this->temp->createTmpFile('wr-gd');
file_put_contents($tmpFile->getPathname(), fopen($uploadedFile['url'], 'r'));
$file = new File(['id' => $uploadedFile['id'], 'title' => $uploadedFile['name'], 'targetFolder' => isset($options['external']['targetFolder']) ? $options['external']['targetFolder'] : null, 'type' => File::TYPE_FILE, 'pathname' => $tmpFile, 'size' => $uploadedFile['sizeBytes']]);
$gdFiles = $writer->listFiles(['q' => "trashed=false and name='" . $uploadedFile['name'] . "'"]);
if (!empty($gdFiles['files'])) {
$lastGdFile = array_shift($gdFiles['files']);
$file->setGoogleId($lastGdFile['id']);
}
$file = $writer->process($file);
// tag file 'wr-google-drive-processed'
$this->storageApi->addFileTag($uploadedFile['id'], 'wr-google-drive-processed');
}
// delete temporary account
$this->configuration->removeAccount($account->getAccountId());
} else {
$fileFilter = null;
if (isset($options['config'])) {
if (!isset($accounts[$options['config']])) {
throw new UserException("Config '" . $options['config'] . "' does not exist.");
}
$accounts = [$options['config'] => $accounts[$options['config']]];
if (isset($options['file'])) {
/** @var Account $account */
$account = $accounts[$options['config']];
if (null == $account->getFile($options['file'])) {
throw new UserException("File '" . $options['file'] . "' not found");
}
$fileFilter = $options['file'];
}
}
/** @var Account $account */
foreach ($accounts as $accountId => $account) {
$writer = $this->writerFactory->create($account);
$files = $account->getFiles();
/** @var File $file */
foreach ($files as $file) {
if ($fileFilter != null && $file->getId() != $fileFilter) {
continue;
}
$file->setPathname($this->temp->createTmpFile()->getPathname());
try {
$tableExporter = new TableExporter($this->storageApi);
$tableExporter->exportTable($file->getTableId(), $file->getPathname(), []);
} catch (ClientException $e) {
throw new UserException($e->getMessage(), $e, ['file' => $file->toArray()]);
}
$file = $writer->process($file);
$status[$account->getAccountName()][$file->getTitle()] = 'ok';
}
// updated changes to files
$account->save();
}
}
return $status;
}
public function build(\Keboola\Syrup\Job\Metadata\Job $job)
{
$params = $job->getRawParams();
$process = $job->getProcess();
$token = $job->getToken();
$project = $job->getProject();
$this->setId($job->getId());
if (!empty($params['config'])) {
$this->setConfig($params['config']);
}
$this->setProjectId($project['id']);
$this->setOrchestrationId($params['orchestration']['id']);
$this->setToken($token['token']);
$this->setTokenId($token['id']);
$this->setTokenDesc($token['description']);
$this->setTokenOwnerName($project['name']);
$this->setStatus($job->getStatus());
$this->setStartTime($job->getStartTime());
$this->setEndTime($job->getEndTime());
$this->setResults($job->getResult());
if (!empty($params['tasks'])) {
$this->setTaks($params['tasks']);
}
$this->setCreatedTime($job->getCreatedTime());
$this->setOrchestrationName($params['orchestration']['name']);
$this->setInitializedBy($params['initializedBy']);
$this->setInitiatorTokenId($params['initiator']['id']);
$this->setInitiatorTokenDesc($params['initiator']['description']);
$this->setInitiatorUserAgent($params['initiator']['userAgent']);
$this->setNotificationsEmails($params['notificationsEmails']);
$this->setRunId($job->getRunId());
$this->setPid($process['pid']);
$this->setWorkerAddress($process['host']);
}
private function executeJob($command, $data)
{
/** @var StorageApiService $storageApi */
$storageApi = $this->getSapiServiceStub();
$logger = new Logger("null");
$logger->pushHandler(new NullHandler());
$serverFactory = new ServerFactory($this->doctrine, $storageApi, $logger);
$jobExecutor = new Executor($logger, $serverFactory);
$jobExecutor->setStorageApi($storageApi->getClient());
$encryptor = new ObjectEncryptor();
$job = new Job($encryptor, $data);
$job->setCommand($command);
$job->setId(123456);
$jobExecutor->setStorageApi($this->client);
return $jobExecutor->execute($job);
}
public function postCleanup(Job $job)
{
$oldRes = $job->getResult();
$job->setResult(['message' => $oldRes['message'] . '&cleared']);
$this->jobMapper->update($job);
}
protected function makeJob($parentRunId = null, $isV2 = false)
{
/** @var Encryptor $encryptor */
$encryptor = $this->httpClient->getContainer()->get('syrup.encryptor');
/** @var ObjectEncryptor $objectEncryptor */
$objectEncryptor = $this->httpClient->getContainer()->get('syrup.object_encryptor');
// create job
$tokenData = $this->storageApiClient->verifyToken();
$componentName = 'queue';
$jobId = $this->storageApiClient->generateId();
$job = new Job($objectEncryptor);
$job->setId($jobId);
$job->setRunId(is_null($parentRunId) ? $this->storageApiClient->generateId() : $parentRunId . '.' . $this->storageApiClient->generateId());
$job->setCommand('run');
$job->setComponent($componentName);
$job->setLockName('syrup-queue-test');
if ($parentRunId) {
$job->setLockName($job->getLockName() . '-child');
}
$job->setToken(['id' => $tokenData['id'], 'description' => $tokenData['description'], 'token' => $encryptor->encrypt($this->storageApiClient->getTokenString())]);
$job->setStatus(Job::STATUS_WAITING);
$job->setCreatedTime(date('c'));
$job->setProject(['id' => $tokenData['owner']['id'], 'name' => $tokenData['owner']['name']]);
if ($isV2) {
$job->setAttribute('protocol', 'v2');
}
return $job;
}
/**
* @param $writerId
* @return JsonResponse
*/
public function cancelWaitingJobsAction($writerId)
{
$sapiData = $this->storageApi->getLogData();
$projectId = $sapiData['owner']['id'];
$query = "(status:waiting)AND(writer:{$writerId})";
$jobs = $this->getElasticSearch()->getJobs(['projectId' => $projectId, 'component' => $this->getParameter("app_name"), 'query' => $query]);
$jobMapper = $this->getJobMapper();
foreach ($jobs as $item) {
$job = new Job($item);
$job->setStatus(Job::STATUS_CANCELLED);
$jobMapper->update($job);
}
return $this->createJsonResponse([]);
}
public function execute(Job $job)
{
$this->extractor->extract($this->getConfiguration(), $job->getParams());
}