public function __invoke(Job $job) { $job = $this->elasticsearch->getJob($job->getId()); // find and kill child jobs $childJobs = $this->elasticsearch->getJobs(['query' => 'runId:' . $job->getRunId() . '.*', 'project.id' => $job->getProject()['id']]); $this->logger->debug("Child jobs found", ['jobs' => $childJobs]); $snsClient = new SnsClient(['credentials' => ['key' => $this->snsConfig['key'], 'secret' => $this->snsConfig['secret']], 'region' => isset($this->snsConfig['region']) ? $this->snsConfig['region'] : 'us-east-1', 'version' => '2010-03-31']); // kill them all if (!empty($childJobs)) { foreach ($childJobs as $childJob) { $snsClient->publish(['TopicArn' => $this->snsConfig['topic_arn'], 'Message' => json_encode(['jobId' => $childJob['id']])]); } } $callback = $this->cleanupCallback; $callback($job); $endTime = time(); $duration = $endTime - strtotime($job->getStartTime()); $job->setStatus(Job::STATUS_TERMINATED); $job->setResult(['message' => 'Job has been terminated']); $job->setEndTime(date('c', $endTime)); $job->setDurationSeconds($duration); $this->jobMapper->update($job); }
protected function execute(InputInterface $input, OutputInterface $output) { $jobId = $input->getArgument('jobId'); $this->init($jobId); // Ensure that job status is 'terminating' if ($this->job->getStatus() != Job::STATUS_TERMINATING) { $this->job->setStatus(Job::STATUS_TERMINATING); $this->jobMapper->update($this->job); } /** @var ExecutorInterface $jobExecutor */ $jobExecutor = $this->getContainer()->get('syrup.job_executor_factory')->create(); // run cleanup $jobExecutor->cleanup($this->job); // Update job $endTime = time(); $duration = $endTime - strtotime($this->job->getStartTime()); $this->job->setStatus(Job::STATUS_TERMINATED); $this->job->setResult(['message' => 'Job has been terminated']); $this->job->setEndTime(date('c', $endTime)); $this->job->setDurationSeconds($duration); $this->jobMapper->update($this->job); // run post-cleanup $jobExecutor->postCleanup($this->job); // DB unlock $this->lock->unlock(); }
/** * Hook for modify job after job execution * * @param Job $job * @return void */ public function postExecution(Job $job) { if ($job->getId() !== $this->job->getId()) { throw new \InvalidArgumentException('Given job must be same as previous executed'); } if ($job->getComponent() !== $this->job->getComponent()) { throw new \InvalidArgumentException('Given job must be same as previous executed'); } $job->setResult($job->getResult() + array(self::HOOK_RESULT_KEY => self::HOOK_RESULT_VALUE)); $this->jobMapper->update($job); }
public function __invoke(Job $job, $errorMessage) { $job = $this->jobMapper->get($job->getId()); if (!in_array($job->getStatus(), [Job::STATUS_ERROR, Job::STATUS_CANCELLED, Job::STATUS_TERMINATING, Job::STATUS_TERMINATED])) { $endTime = time(); $duration = $endTime - strtotime($job->getStartTime()); $job->setStatus(Job::STATUS_ERROR); $job->setResult(['message' => $errorMessage]); $job->setEndTime(date('c', $endTime)); $job->setDurationSeconds($duration); $this->jobMapper->update($job); } }
public function updateResult(Job $job, array $result) { // token decryption/encryption not required $syrupJob = $this->syrupJobMapper->get($job->getId()); $job->setResults($result); $syrupJob->setResult($job->getResults()); $this->syrupJobMapper->update($syrupJob); return true; }
public function testUpdateJob() { $job = self::$jobFactory->create(uniqid()); $id = self::$jobMapper->create($job); $job = self::$jobMapper->get($id); $job->setStatus(Job::STATUS_CANCELLED); self::$jobMapper->update($job); $job = self::$jobMapper->get($id); $this->assertEquals($job->getStatus(), Job::STATUS_CANCELLED); $job->setStatus(Job::STATUS_WARNING); self::$jobMapper->update($job); $job = self::$jobMapper->get($id); $this->assertEquals($job->getStatus(), Job::STATUS_WARNING); }
/** * Testing watchdog on waiting jobs from manual run * */ public function testManuallyLongProcessingWatchdog() { $token1 = $this->createNewToken($this->storageApi, 'manage'); $notification1 = new StorageApi\Notification(); $notification1->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_WAITING)->setParameters(array('timeout' => 1)); $notification2 = new StorageApi\Notification(); $notification2->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10)); $notification3 = new StorageApi\Notification(); $notification3->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_2)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10)); $orchestrationId = $this->createOrchestrationTest($token1, array($notification1, $notification2, $notification3)); // first job fake processing $jobId = $this->enqueueJobTest($orchestrationId); $syrupJob = $this->syrupJobMapper->get($jobId); $this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob); $syrupJob->setStartTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); sleep(3); $syrupJob->setEndTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); // second job fake processing $jobId = $this->enqueueJobTest($orchestrationId); $syrupJob = $this->syrupJobMapper->get($jobId); $this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob); $syrupJob->setStartTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); sleep(120); // check jobs $this->runWatchdogCommandTest(); // second job finishing $syrupJob->setEndTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); $esJob = new Job(); $esJob->build($syrupJob); // check sended notification - manual run - only one must be sended $events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification2, $this->storageApi); $this->assertCount(1, $events); $events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification3, $this->storageApi); $this->assertCount(0, $events); }
protected function execute(InputInterface $input, OutputInterface $output) { $jobId = $input->getArgument('jobId'); $this->logger = $this->getContainer()->get('logger'); try { $this->init($jobId); } catch (\Exception $e) { // Initialization error -> job will be requeued $this->logException('error', $e); // Don't update job status or result -> error could be related to ES // Don't unlock DB, error happened either before lock creation or when creating the lock, // so the DB isn't locked return self::STATUS_RETRY; } if (is_null($jobId)) { throw new UserException("Missing jobId argument."); } if (!$this->lock->lock()) { return self::STATUS_LOCK; } /** @var Connection $checkConn */ $checkConn = null; /** @var Lock $validationLock */ $validationLock = null; if (StorageApiLimits::hasParallelLimit($this->storageApiService->getTokenData())) { try { $checkConn = $this->getContainer()->get('doctrine.dbal.limit_lock_connection'); $checkConn->exec('SET wait_timeout = 31536000;'); $validationLock = new Lock($checkConn, sprintf('syrup-%s-job-limit-check', $this->job->getProject()['id'])); if (!$validationLock->lock(self::PARALLEL_LIMIT_LOCK_TIMEOUT)) { throw new \RuntimeException('Could not lock for parallel validation'); } if ($this->isParallelLimitExceeded()) { throw new \RuntimeException('Exceeded parallel processing limit'); } } catch (\RuntimeException $e) { return self::STATUS_LOCK; } } // check job status $this->job = $this->jobMapper->get($jobId); if (!in_array($this->job->getStatus(), [Job::STATUS_WAITING, Job::STATUS_PROCESSING])) { // job is not waiting or processing return self::STATUS_LOCK; } $startTime = time(); // Update job status to 'processing' $this->job->setStatus(Job::STATUS_PROCESSING); $this->job->setStartTime(date('c', $startTime)); $this->job->setEndTime(null); $this->job->setDurationSeconds(null); $this->job->setResult(null); $this->job->setProcess(['host' => gethostname(), 'pid' => getmypid()]); /** @var ExecutorInterface $jobExecutor */ $jobExecutor = $this->getContainer()->get('syrup.job_executor_factory')->create(); // update the job status after jobExecutor was created, so the signal handler is properly registered $this->jobMapper->update($this->job); // Execute job try { if (StorageApiLimits::hasParallelLimit($this->storageApiService->getTokenData())) { $validationLock->unlock(); $checkConn->close(); } $jobResult = $jobExecutor->execute($this->job); $jobStatus = Job::STATUS_SUCCESS; $status = self::STATUS_SUCCESS; } catch (InitializationException $e) { // job will be requeued $exceptionId = $this->logException('error', $e); $jobResult = ['message' => $e->getMessage(), 'exceptionId' => $exceptionId]; $jobStatus = Job::STATUS_PROCESSING; $status = self::STATUS_RETRY; } catch (MaintenanceException $e) { $jobResult = []; $jobStatus = Job::STATUS_WAITING; $status = self::STATUS_LOCK; } catch (UserException $e) { $exceptionId = $this->logException('error', $e); $jobResult = ['message' => $e->getMessage(), 'exceptionId' => $exceptionId]; $jobStatus = Job::STATUS_ERROR; $this->job->setError(Job::ERROR_USER); $status = self::STATUS_SUCCESS; } catch (JobException $e) { $logLevel = 'error'; if ($e->getStatus() === Job::STATUS_WARNING) { $logLevel = Job::STATUS_WARNING; } $exceptionId = $this->logException($logLevel, $e); $jobResult = ['message' => $e->getMessage(), 'exceptionId' => $exceptionId]; if ($e->getResult()) { $jobResult += $e->getResult(); } $jobStatus = $e->getStatus(); $status = self::STATUS_SUCCESS; } catch (\Exception $e) { // make sure that the job is recorded as failed $jobStatus = Job::STATUS_ERROR; $jobResult = ['message' => 'Internal error occurred, evaluating details']; $this->job->setStatus($jobStatus); $this->job->setResult($jobResult); $this->job->setError(Job::ERROR_APPLICATION); // try to log the exception $exceptionId = $this->logException('critical', $e); $jobResult = ['message' => 'Internal error occurred, please contact support@keboola.com', 'exceptionId' => $exceptionId]; $status = self::STATUS_ERROR; } // Update job with results $endTime = time(); $createdTime = $this->job->getCreatedTime(); $waitSeconds = is_null($createdTime) ?: $startTime - strtotime($createdTime); $this->job->setStatus($jobStatus); $this->job->setResult($jobResult); $this->job->setEndTime(date('c', $endTime)); $this->job->setDurationSeconds($endTime - $startTime); $this->job->setWaitSeconds($waitSeconds); $this->jobMapper->update($this->job); // postExecution action try { $jobExecutor->postExecute($this->job); //@todo: remove call to this deprecated interface method if ($jobExecutor instanceof HookExecutorInterface) { /** @var HookExecutorInterface $jobExecutor */ $jobExecutor->postExecution($this->job); } } catch (\Exception $e) { $this->logException('critical', $e); } // DB unlock $this->lock->unlock(); return $status; }
public function postCleanup(Job $job) { $oldRes = $job->getResult(); $job->setResult(['message' => $oldRes['message'] . '&cleared']); $this->jobMapper->update($job); }
private function watchWaiting(InputInterface $input, OutputInterface $output) { $now = new \DateTime(); $output->writeln(sprintf('<info>%s</info>', 'Check waiting jobs')); $i = 0; while ($jobs = $this->loadWaitingJobs($i)) { foreach ($jobs as $job) { $i++; // check sapi connection if (!$this->sapiPing($job, $this->encryptor)) { $message = 'SAPI connect failed'; $this->logger->info($message, array('job' => $job->getId(), 'project' => $job->getProject(), 'params' => $job->getRawParams())); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } $sapiClient = $this->createJobSapiClient($job); $jobManager = $this->jobManagerFactory->createJobManager(new StorageApi\Token($sapiClient)); try { $esJob = $jobManager->findJobByIdForWatchdog($job->getId()); if (!$esJob) { $message = 'Job load from ES failed'; $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } } catch (\InvalidArgumentException $e) { $message = 'Job load from ES failed'; $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } // clean jobs of deleted orchestrations $orchestration = $this->orchestrationManager->findDeletedOrchestrations(array('id' => $esJob->getOrchestrationId()), false); $orchestration = reset($orchestration); if ($orchestration) { $updateJob = $this->syrupJobMapper->get($esJob->getId()); if ($updateJob->getComponent() !== KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME) { continue; } $updateJob->setStatus(Metadata\Job::STATUS_CANCELLED); $this->syrupJobMapper->update($updateJob); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[deleted orchestration]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); continue; } $orchestrations = $this->orchestrationManager->findOrchestrations(array('id' => $esJob->getOrchestrationId()), false); /** * @var Orchestration $orchestration */ $orchestration = reset($orchestrations); if (!$orchestration) { $message = 'Orchestration load failed'; $this->logger->critical($message, array('job' => $job->getId(), 'orchestrationId' => $esJob->getOrchestrationId(), 'projectId' => $esJob->getProjectId(), 'project' => $esJob->getTokenOwnerName())); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } $componentsList = new KbcComponentsList($sapiClient); // waiting duration check if ($esJob->getInitializedBy() === 'manually') { /** * @var StorageApi\Notification[] $notifications */ $notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) { return $row->getChannel() === Metadata\Job::STATUS_WAITING; }); if ($notifications) { foreach ($notifications as $notification) { if ($notification->getEmail() !== $esJob->getInitiatorTokenDesc()) { continue; } $date = new \DateTime($job->getCreatedTime()); $date->modify(sprintf('+%d minutes', $notification->getParameter('timeout', 0))); if ($date > $now) { continue; } $events = StorageApi\EventLoader::longWaitingEvents($esJob, $notification, $sapiClient); if ($events) { $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } else { $this->mailer->sendLongWaitingMessage($esJob, $orchestration, $notification, $componentsList); $this->logLongWaiting($esJob, $sapiClient, $notification); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } } } else { // log waiting for default timeout $date = new \DateTime($job->getCreatedTime()); $date->modify(sprintf('+%d minutes', self::WAIT_LIMIT_MINUTES)); if ($date <= $now) { $events = StorageApi\EventLoader::longWaitingEventsAll($esJob, $sapiClient); if ($events) { $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[system already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } else { $this->logLongWaiting($esJob, $sapiClient, null); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[system notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } } } } else { /** * @var StorageApi\Notification[] $notifications */ $notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) { return $row->getChannel() === Metadata\Job::STATUS_WAITING; }); if ($notifications) { foreach ($notifications as $notification) { $date = new \DateTime($job->getCreatedTime()); $date->modify(sprintf('+%d minutes', $notification->getParameter('timeout', 0))); if ($date > $now) { continue; } $events = StorageApi\EventLoader::longWaitingEvents($esJob, $notification, $sapiClient); if ($events) { $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } else { $this->mailer->sendLongWaitingMessage($esJob, $orchestration, $notification, $componentsList); $this->logLongWaiting($esJob, $sapiClient, $notification); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } } } else { // log waiting for default timeout $date = new \DateTime($job->getCreatedTime()); $date->modify(sprintf('+%d minutes', self::WAIT_LIMIT_MINUTES)); if ($date <= $now) { $events = StorageApi\EventLoader::longWaitingEventsAll($esJob, $sapiClient); if ($events) { $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[system already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } else { $this->logLongWaiting($esJob, $sapiClient, null); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[system notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } } } } } } }
/** * Hook for modify job after job execution * * @param Job $job * @return void */ public function postExecution(Job $job) { $job->setResult($job->getResult() + array(self::HOOK_RESULT_KEY => self::HOOK_RESULT_VALUE)); $this->jobMapper->update($job); }