public function __invoke(Job $job)
{
$job = $this->elasticsearch->getJob($job->getId());
// find and kill child jobs
$childJobs = $this->elasticsearch->getJobs(['query' => 'runId:' . $job->getRunId() . '.*', 'project.id' => $job->getProject()['id']]);
$this->logger->debug("Child jobs found", ['jobs' => $childJobs]);
$snsClient = new SnsClient(['credentials' => ['key' => $this->snsConfig['key'], 'secret' => $this->snsConfig['secret']], 'region' => isset($this->snsConfig['region']) ? $this->snsConfig['region'] : 'us-east-1', 'version' => '2010-03-31']);
// kill them all
if (!empty($childJobs)) {
foreach ($childJobs as $childJob) {
$snsClient->publish(['TopicArn' => $this->snsConfig['topic_arn'], 'Message' => json_encode(['jobId' => $childJob['id']])]);
}
}
$callback = $this->cleanupCallback;
$callback($job);
$endTime = time();
$duration = $endTime - strtotime($job->getStartTime());
$job->setStatus(Job::STATUS_TERMINATED);
$job->setResult(['message' => 'Job has been terminated']);
$job->setEndTime(date('c', $endTime));
$job->setDurationSeconds($duration);
$this->jobMapper->update($job);
}
public function testGetJobs()
{
$job = $this->createJob();
self::$jobMapper->create($job);
$job2 = $this->createJob();
self::$jobMapper->create($job2);
$retries = 0;
$res = [];
while ($retries < 7) {
$delaySecs = 2 * pow(2, $retries);
sleep($delaySecs);
$retries++;
$projectId = $job->getProject()['id'];
$res = self::$search->getJobs(['projectId' => $projectId, 'component' => SYRUP_APP_NAME, 'since' => '-1 day', 'until' => 'now']);
if (count($res) >= 2) {
break;
}
}
$job1Asserted = false;
$job2Asserted = false;
foreach ($res as $r) {
if ($r['id'] == $job->getId()) {
$this->assertJob($job, $r);
$job1Asserted = true;
}
if ($r['id'] == $job2->getId()) {
$this->assertJob($job2, $r);
$job2Asserted = true;
}
}
$this->assertTrue($job1Asserted);
$this->assertTrue($job2Asserted);
}
/**
* Call API and process job immediately, return job info
* @param string $url URL of API call
* @param array $params parameters of POST call
* @param string $method HTTP method of API call
* @return Job
*/
protected function processJob($url, $params = [], $method = 'POST')
{
$responseJson = $this->callApi($url, $method, $params);
$this->assertArrayHasKey('id', $responseJson, sprintf("Response of API call '%s' should contain 'id' key.", $url));
$this->commandTester->execute(['command' => 'syrup:run-job', 'jobId' => $responseJson['id']]);
return $this->jobMapper->get($responseJson['id']);
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$jobId = $input->getArgument('jobId');
$this->init($jobId);
// Ensure that job status is 'terminating'
if ($this->job->getStatus() != Job::STATUS_TERMINATING) {
$this->job->setStatus(Job::STATUS_TERMINATING);
$this->jobMapper->update($this->job);
}
/** @var ExecutorInterface $jobExecutor */
$jobExecutor = $this->getContainer()->get('syrup.job_executor_factory')->create();
// run cleanup
$jobExecutor->cleanup($this->job);
// Update job
$endTime = time();
$duration = $endTime - strtotime($this->job->getStartTime());
$this->job->setStatus(Job::STATUS_TERMINATED);
$this->job->setResult(['message' => 'Job has been terminated']);
$this->job->setEndTime(date('c', $endTime));
$this->job->setDurationSeconds($duration);
$this->jobMapper->update($this->job);
// run post-cleanup
$jobExecutor->postCleanup($this->job);
// DB unlock
$this->lock->unlock();
}
/**
* Hook for modify job after job execution
*
* @param Job $job
* @return void
*/
public function postExecution(Job $job)
{
if ($job->getId() !== $this->job->getId()) {
throw new \InvalidArgumentException('Given job must be same as previous executed');
}
if ($job->getComponent() !== $this->job->getComponent()) {
throw new \InvalidArgumentException('Given job must be same as previous executed');
}
$job->setResult($job->getResult() + array(self::HOOK_RESULT_KEY => self::HOOK_RESULT_VALUE));
$this->jobMapper->update($job);
}
public function __invoke(Job $job, $errorMessage)
{
$job = $this->jobMapper->get($job->getId());
if (!in_array($job->getStatus(), [Job::STATUS_ERROR, Job::STATUS_CANCELLED, Job::STATUS_TERMINATING, Job::STATUS_TERMINATED])) {
$endTime = time();
$duration = $endTime - strtotime($job->getStartTime());
$job->setStatus(Job::STATUS_ERROR);
$job->setResult(['message' => $errorMessage]);
$job->setEndTime(date('c', $endTime));
$job->setDurationSeconds($duration);
$this->jobMapper->update($job);
}
}
public function testUpdateJob()
{
$job = self::$jobFactory->create(uniqid());
$id = self::$jobMapper->create($job);
$job = self::$jobMapper->get($id);
$job->setStatus(Job::STATUS_CANCELLED);
self::$jobMapper->update($job);
$job = self::$jobMapper->get($id);
$this->assertEquals($job->getStatus(), Job::STATUS_CANCELLED);
$job->setStatus(Job::STATUS_WARNING);
self::$jobMapper->update($job);
$job = self::$jobMapper->get($id);
$this->assertEquals($job->getStatus(), Job::STATUS_WARNING);
}
/**
* @param Job $job
* @param StorageApi\UniqueManager $uniqueGenerator
* @return Job|null
*/
public function saveJob(Job $job, StorageApi\UniqueManager $uniqueGenerator)
{
$job->setStatus(Elasticsearch\Job::STATUS_WAITING);
$job->setResults(null);
$job->setCreatedTime(new \DateTime());
$id = $this->syrupJobMapper->create($this->buildEsJob($job, $uniqueGenerator));
return $this->findJobById($id);
}
public function testConfigId()
{
$token1 = $this->createNewToken($this->storageApi, 'manage');
$orchestrationId = $this->createOrchestrationTest($token1);
$jobId = $this->runOrchestrationTest($orchestrationId);
$job = $this->syrupJobMapper->get($jobId);
$params = $job->getRawParams();
$this->assertArrayHasKey('config', $params);
$this->assertEquals($orchestrationId, $params['config']);
$components = new Components($this->storageApi);
$configuration = $components->getConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $orchestrationId);
$this->assertArrayHasKey('name', $configuration);
$this->assertArrayHasKey('id', $configuration);
}
/**
* Testing watchdog on waiting jobs from manual run
*
*/
public function testManuallyLongProcessingWatchdog()
{
$token1 = $this->createNewToken($this->storageApi, 'manage');
$notification1 = new StorageApi\Notification();
$notification1->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_WAITING)->setParameters(array('timeout' => 1));
$notification2 = new StorageApi\Notification();
$notification2->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10));
$notification3 = new StorageApi\Notification();
$notification3->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_2)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10));
$orchestrationId = $this->createOrchestrationTest($token1, array($notification1, $notification2, $notification3));
// first job fake processing
$jobId = $this->enqueueJobTest($orchestrationId);
$syrupJob = $this->syrupJobMapper->get($jobId);
$this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob);
$syrupJob->setStartTime((new \DateTime())->format('c'));
$syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING);
$syrupJob->setResult(array('tasks' => array()));
$this->syrupJobMapper->update($syrupJob);
sleep(3);
$syrupJob->setEndTime((new \DateTime())->format('c'));
$syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS);
$syrupJob->setResult(array('tasks' => array()));
$this->syrupJobMapper->update($syrupJob);
// second job fake processing
$jobId = $this->enqueueJobTest($orchestrationId);
$syrupJob = $this->syrupJobMapper->get($jobId);
$this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob);
$syrupJob->setStartTime((new \DateTime())->format('c'));
$syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING);
$syrupJob->setResult(array('tasks' => array()));
$this->syrupJobMapper->update($syrupJob);
sleep(120);
// check jobs
$this->runWatchdogCommandTest();
// second job finishing
$syrupJob->setEndTime((new \DateTime())->format('c'));
$syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS);
$syrupJob->setResult(array('tasks' => array()));
$this->syrupJobMapper->update($syrupJob);
$esJob = new Job();
$esJob->build($syrupJob);
// check sended notification - manual run - only one must be sended
$events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification2, $this->storageApi);
$this->assertCount(1, $events);
$events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification3, $this->storageApi);
$this->assertCount(0, $events);
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$jobId = $input->getArgument('jobId');
$this->logger = $this->getContainer()->get('logger');
try {
$this->init($jobId);
} catch (\Exception $e) {
// Initialization error -> job will be requeued
$this->logException('error', $e);
// Don't update job status or result -> error could be related to ES
// Don't unlock DB, error happened either before lock creation or when creating the lock,
// so the DB isn't locked
return self::STATUS_RETRY;
}
if (is_null($jobId)) {
throw new UserException("Missing jobId argument.");
}
if (!$this->lock->lock()) {
return self::STATUS_LOCK;
}
/** @var Connection $checkConn */
$checkConn = null;
/** @var Lock $validationLock */
$validationLock = null;
if (StorageApiLimits::hasParallelLimit($this->storageApiService->getTokenData())) {
try {
$checkConn = $this->getContainer()->get('doctrine.dbal.limit_lock_connection');
$checkConn->exec('SET wait_timeout = 31536000;');
$validationLock = new Lock($checkConn, sprintf('syrup-%s-job-limit-check', $this->job->getProject()['id']));
if (!$validationLock->lock(self::PARALLEL_LIMIT_LOCK_TIMEOUT)) {
throw new \RuntimeException('Could not lock for parallel validation');
}
if ($this->isParallelLimitExceeded()) {
throw new \RuntimeException('Exceeded parallel processing limit');
}
} catch (\RuntimeException $e) {
return self::STATUS_LOCK;
}
}
// check job status
$this->job = $this->jobMapper->get($jobId);
if (!in_array($this->job->getStatus(), [Job::STATUS_WAITING, Job::STATUS_PROCESSING])) {
// job is not waiting or processing
return self::STATUS_LOCK;
}
$startTime = time();
// Update job status to 'processing'
$this->job->setStatus(Job::STATUS_PROCESSING);
$this->job->setStartTime(date('c', $startTime));
$this->job->setEndTime(null);
$this->job->setDurationSeconds(null);
$this->job->setResult(null);
$this->job->setProcess(['host' => gethostname(), 'pid' => getmypid()]);
/** @var ExecutorInterface $jobExecutor */
$jobExecutor = $this->getContainer()->get('syrup.job_executor_factory')->create();
// update the job status after jobExecutor was created, so the signal handler is properly registered
$this->jobMapper->update($this->job);
// Execute job
try {
if (StorageApiLimits::hasParallelLimit($this->storageApiService->getTokenData())) {
$validationLock->unlock();
$checkConn->close();
}
$jobResult = $jobExecutor->execute($this->job);
$jobStatus = Job::STATUS_SUCCESS;
$status = self::STATUS_SUCCESS;
} catch (InitializationException $e) {
// job will be requeued
$exceptionId = $this->logException('error', $e);
$jobResult = ['message' => $e->getMessage(), 'exceptionId' => $exceptionId];
$jobStatus = Job::STATUS_PROCESSING;
$status = self::STATUS_RETRY;
} catch (MaintenanceException $e) {
$jobResult = [];
$jobStatus = Job::STATUS_WAITING;
$status = self::STATUS_LOCK;
} catch (UserException $e) {
$exceptionId = $this->logException('error', $e);
$jobResult = ['message' => $e->getMessage(), 'exceptionId' => $exceptionId];
$jobStatus = Job::STATUS_ERROR;
$this->job->setError(Job::ERROR_USER);
$status = self::STATUS_SUCCESS;
} catch (JobException $e) {
$logLevel = 'error';
if ($e->getStatus() === Job::STATUS_WARNING) {
$logLevel = Job::STATUS_WARNING;
}
$exceptionId = $this->logException($logLevel, $e);
$jobResult = ['message' => $e->getMessage(), 'exceptionId' => $exceptionId];
if ($e->getResult()) {
$jobResult += $e->getResult();
}
$jobStatus = $e->getStatus();
$status = self::STATUS_SUCCESS;
} catch (\Exception $e) {
// make sure that the job is recorded as failed
$jobStatus = Job::STATUS_ERROR;
$jobResult = ['message' => 'Internal error occurred, evaluating details'];
$this->job->setStatus($jobStatus);
$this->job->setResult($jobResult);
$this->job->setError(Job::ERROR_APPLICATION);
// try to log the exception
$exceptionId = $this->logException('critical', $e);
$jobResult = ['message' => 'Internal error occurred, please contact support@keboola.com', 'exceptionId' => $exceptionId];
$status = self::STATUS_ERROR;
}
// Update job with results
$endTime = time();
$createdTime = $this->job->getCreatedTime();
$waitSeconds = is_null($createdTime) ?: $startTime - strtotime($createdTime);
$this->job->setStatus($jobStatus);
$this->job->setResult($jobResult);
$this->job->setEndTime(date('c', $endTime));
$this->job->setDurationSeconds($endTime - $startTime);
$this->job->setWaitSeconds($waitSeconds);
$this->jobMapper->update($this->job);
// postExecution action
try {
$jobExecutor->postExecute($this->job);
//@todo: remove call to this deprecated interface method
if ($jobExecutor instanceof HookExecutorInterface) {
/** @var HookExecutorInterface $jobExecutor */
$jobExecutor->postExecution($this->job);
}
} catch (\Exception $e) {
$this->logException('critical', $e);
}
// DB unlock
$this->lock->unlock();
return $status;
}
public function postCleanup(Job $job)
{
$oldRes = $job->getResult();
$job->setResult(['message' => $oldRes['message'] . '&cleared']);
$this->jobMapper->update($job);
}
private function watchWaiting(InputInterface $input, OutputInterface $output)
{
$now = new \DateTime();
$output->writeln(sprintf('<info>%s</info>', 'Check waiting jobs'));
$i = 0;
while ($jobs = $this->loadWaitingJobs($i)) {
foreach ($jobs as $job) {
$i++;
// check sapi connection
if (!$this->sapiPing($job, $this->encryptor)) {
$message = 'SAPI connect failed';
$this->logger->info($message, array('job' => $job->getId(), 'project' => $job->getProject(), 'params' => $job->getRawParams()));
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
continue;
}
$sapiClient = $this->createJobSapiClient($job);
$jobManager = $this->jobManagerFactory->createJobManager(new StorageApi\Token($sapiClient));
try {
$esJob = $jobManager->findJobByIdForWatchdog($job->getId());
if (!$esJob) {
$message = 'Job load from ES failed';
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
continue;
}
} catch (\InvalidArgumentException $e) {
$message = 'Job load from ES failed';
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
continue;
}
// clean jobs of deleted orchestrations
$orchestration = $this->orchestrationManager->findDeletedOrchestrations(array('id' => $esJob->getOrchestrationId()), false);
$orchestration = reset($orchestration);
if ($orchestration) {
$updateJob = $this->syrupJobMapper->get($esJob->getId());
if ($updateJob->getComponent() !== KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME) {
continue;
}
$updateJob->setStatus(Metadata\Job::STATUS_CANCELLED);
$this->syrupJobMapper->update($updateJob);
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[deleted orchestration]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
continue;
}
$orchestrations = $this->orchestrationManager->findOrchestrations(array('id' => $esJob->getOrchestrationId()), false);
/**
* @var Orchestration $orchestration
*/
$orchestration = reset($orchestrations);
if (!$orchestration) {
$message = 'Orchestration load failed';
$this->logger->critical($message, array('job' => $job->getId(), 'orchestrationId' => $esJob->getOrchestrationId(), 'projectId' => $esJob->getProjectId(), 'project' => $esJob->getTokenOwnerName()));
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message));
continue;
}
$componentsList = new KbcComponentsList($sapiClient);
// waiting duration check
if ($esJob->getInitializedBy() === 'manually') {
/**
* @var StorageApi\Notification[] $notifications
*/
$notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) {
return $row->getChannel() === Metadata\Job::STATUS_WAITING;
});
if ($notifications) {
foreach ($notifications as $notification) {
if ($notification->getEmail() !== $esJob->getInitiatorTokenDesc()) {
continue;
}
$date = new \DateTime($job->getCreatedTime());
$date->modify(sprintf('+%d minutes', $notification->getParameter('timeout', 0)));
if ($date > $now) {
continue;
}
$events = StorageApi\EventLoader::longWaitingEvents($esJob, $notification, $sapiClient);
if ($events) {
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
} else {
$this->mailer->sendLongWaitingMessage($esJob, $orchestration, $notification, $componentsList);
$this->logLongWaiting($esJob, $sapiClient, $notification);
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
}
}
} else {
// log waiting for default timeout
$date = new \DateTime($job->getCreatedTime());
$date->modify(sprintf('+%d minutes', self::WAIT_LIMIT_MINUTES));
if ($date <= $now) {
$events = StorageApi\EventLoader::longWaitingEventsAll($esJob, $sapiClient);
if ($events) {
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[system already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
} else {
$this->logLongWaiting($esJob, $sapiClient, null);
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[system notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
}
}
}
} else {
/**
* @var StorageApi\Notification[] $notifications
*/
$notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) {
return $row->getChannel() === Metadata\Job::STATUS_WAITING;
});
if ($notifications) {
foreach ($notifications as $notification) {
$date = new \DateTime($job->getCreatedTime());
$date->modify(sprintf('+%d minutes', $notification->getParameter('timeout', 0)));
if ($date > $now) {
continue;
}
$events = StorageApi\EventLoader::longWaitingEvents($esJob, $notification, $sapiClient);
if ($events) {
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
} else {
$this->mailer->sendLongWaitingMessage($esJob, $orchestration, $notification, $componentsList);
$this->logLongWaiting($esJob, $sapiClient, $notification);
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
}
}
} else {
// log waiting for default timeout
$date = new \DateTime($job->getCreatedTime());
$date->modify(sprintf('+%d minutes', self::WAIT_LIMIT_MINUTES));
if ($date <= $now) {
$events = StorageApi\EventLoader::longWaitingEventsAll($esJob, $sapiClient);
if ($events) {
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[system already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
} else {
$this->logLongWaiting($esJob, $sapiClient, null);
$output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[system notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName()));
}
}
}
}
}
}
}
/**
* Hook for modify job after job execution
*
* @param Job $job
* @return void
*/
public function postExecution(Job $job)
{
$job->setResult($job->getResult() + array(self::HOOK_RESULT_KEY => self::HOOK_RESULT_VALUE));
$this->jobMapper->update($job);
}
