/** * Call API and process job immediately, return job info * @param string $url URL of API call * @param array $params parameters of POST call * @param string $method HTTP method of API call * @return Job */ protected function processJob($url, $params = [], $method = 'POST') { $responseJson = $this->callApi($url, $method, $params); $this->assertArrayHasKey('id', $responseJson, sprintf("Response of API call '%s' should contain 'id' key.", $url)); $this->commandTester->execute(['command' => 'syrup:run-job', 'jobId' => $responseJson['id']]); return $this->jobMapper->get($responseJson['id']); }
protected function init($jobId) { $this->jobMapper = $this->getContainer()->get('syrup.elasticsearch.current_component_job_mapper'); // Get job from ES $this->job = $this->jobMapper->get($jobId); if ($this->job == null) { throw new ApplicationException(sprintf("Job id '%s' not found", $jobId)); } // SAPI init /** @var ObjectEncryptor $encryptor */ $encryptor = $this->getContainer()->get('syrup.object_encryptor'); $this->sapiClient = new SapiClient(['token' => $encryptor->decrypt($this->job->getToken()['token']), 'url' => $this->getContainer()->getParameter('storage_api.url'), 'userAgent' => $this->job->getComponent()]); $this->sapiClient->setRunId($this->job->getRunId()); /** @var \Keboola\Syrup\Service\StorageApi\StorageApiService $storageApiService */ $storageApiService = $this->getContainer()->get('syrup.storage_api'); $storageApiService->setClient($this->sapiClient); /** @var \Keboola\Syrup\Monolog\Processor\JobProcessor $logProcessor */ $logProcessor = $this->getContainer()->get('syrup.monolog.job_processor'); $logProcessor->setJob($this->job); /** @var \Keboola\Syrup\Monolog\Processor\SyslogProcessor $logProcessor */ $logProcessor = $this->getContainer()->get('syrup.monolog.syslog_processor'); $logProcessor->setRunId($this->job->getRunId()); $logProcessor->setTokenData($storageApiService->getTokenData()); // Lock DB /** @var Connection $conn */ $conn = $this->getContainer()->get('doctrine.dbal.lock_connection'); $conn->exec('SET wait_timeout = 31536000;'); $this->lock = new Lock($conn, $this->job->getLockName()); }
public function __invoke(Job $job, $errorMessage) { $job = $this->jobMapper->get($job->getId()); if (!in_array($job->getStatus(), [Job::STATUS_ERROR, Job::STATUS_CANCELLED, Job::STATUS_TERMINATING, Job::STATUS_TERMINATED])) { $endTime = time(); $duration = $endTime - strtotime($job->getStartTime()); $job->setStatus(Job::STATUS_ERROR); $job->setResult(['message' => $errorMessage]); $job->setEndTime(date('c', $endTime)); $job->setDurationSeconds($duration); $this->jobMapper->update($job); } }
public function updateResult(Job $job, array $result) { // token decryption/encryption not required $syrupJob = $this->syrupJobMapper->get($job->getId()); $job->setResults($result); $syrupJob->setResult($job->getResults()); $this->syrupJobMapper->update($syrupJob); return true; }
public function testConfigId() { $token1 = $this->createNewToken($this->storageApi, 'manage'); $orchestrationId = $this->createOrchestrationTest($token1); $jobId = $this->runOrchestrationTest($orchestrationId); $job = $this->syrupJobMapper->get($jobId); $params = $job->getRawParams(); $this->assertArrayHasKey('config', $params); $this->assertEquals($orchestrationId, $params['config']); $components = new Components($this->storageApi); $configuration = $components->getConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $orchestrationId); $this->assertArrayHasKey('name', $configuration); $this->assertArrayHasKey('id', $configuration); }
public function testUpdateJob() { $job = self::$jobFactory->create(uniqid()); $id = self::$jobMapper->create($job); $job = self::$jobMapper->get($id); $job->setStatus(Job::STATUS_CANCELLED); self::$jobMapper->update($job); $job = self::$jobMapper->get($id); $this->assertEquals($job->getStatus(), Job::STATUS_CANCELLED); $job->setStatus(Job::STATUS_WARNING); self::$jobMapper->update($job); $job = self::$jobMapper->get($id); $this->assertEquals($job->getStatus(), Job::STATUS_WARNING); }
/** * Testing watchdog on waiting jobs from manual run * */ public function testManuallyLongProcessingWatchdog() { $token1 = $this->createNewToken($this->storageApi, 'manage'); $notification1 = new StorageApi\Notification(); $notification1->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_WAITING)->setParameters(array('timeout' => 1)); $notification2 = new StorageApi\Notification(); $notification2->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_1)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10)); $notification3 = new StorageApi\Notification(); $notification3->setEmail(TEST_ERROR_NOTIFICATION_EMAIL_2)->setChannel(Metadata\Job::STATUS_PROCESSING)->setParameters(array('tolerance' => 10)); $orchestrationId = $this->createOrchestrationTest($token1, array($notification1, $notification2, $notification3)); // first job fake processing $jobId = $this->enqueueJobTest($orchestrationId); $syrupJob = $this->syrupJobMapper->get($jobId); $this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob); $syrupJob->setStartTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); sleep(3); $syrupJob->setEndTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); // second job fake processing $jobId = $this->enqueueJobTest($orchestrationId); $syrupJob = $this->syrupJobMapper->get($jobId); $this->assertInstanceOf(get_class(new Metadata\Job($this->objectEncryptor)), $syrupJob); $syrupJob->setStartTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_PROCESSING); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); sleep(120); // check jobs $this->runWatchdogCommandTest(); // second job finishing $syrupJob->setEndTime((new \DateTime())->format('c')); $syrupJob->setStatus(Metadata\Job::STATUS_SUCCESS); $syrupJob->setResult(array('tasks' => array())); $this->syrupJobMapper->update($syrupJob); $esJob = new Job(); $esJob->build($syrupJob); // check sended notification - manual run - only one must be sended $events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification2, $this->storageApi); $this->assertCount(1, $events); $events = StorageApi\EventLoader::longProcessingEvents($esJob, $notification3, $this->storageApi); $this->assertCount(0, $events); }
protected function execute(InputInterface $input, OutputInterface $output) { $jobId = $input->getArgument('jobId'); $this->logger = $this->getContainer()->get('logger'); try { $this->init($jobId); } catch (\Exception $e) { // Initialization error -> job will be requeued $this->logException('error', $e); // Don't update job status or result -> error could be related to ES // Don't unlock DB, error happened either before lock creation or when creating the lock, // so the DB isn't locked return self::STATUS_RETRY; } if (is_null($jobId)) { throw new UserException("Missing jobId argument."); } if (!$this->lock->lock()) { return self::STATUS_LOCK; } /** @var Connection $checkConn */ $checkConn = null; /** @var Lock $validationLock */ $validationLock = null; if (StorageApiLimits::hasParallelLimit($this->storageApiService->getTokenData())) { try { $checkConn = $this->getContainer()->get('doctrine.dbal.limit_lock_connection'); $checkConn->exec('SET wait_timeout = 31536000;'); $validationLock = new Lock($checkConn, sprintf('syrup-%s-job-limit-check', $this->job->getProject()['id'])); if (!$validationLock->lock(self::PARALLEL_LIMIT_LOCK_TIMEOUT)) { throw new \RuntimeException('Could not lock for parallel validation'); } if ($this->isParallelLimitExceeded()) { throw new \RuntimeException('Exceeded parallel processing limit'); } } catch (\RuntimeException $e) { return self::STATUS_LOCK; } } // check job status $this->job = $this->jobMapper->get($jobId); if (!in_array($this->job->getStatus(), [Job::STATUS_WAITING, Job::STATUS_PROCESSING])) { // job is not waiting or processing return self::STATUS_LOCK; } $startTime = time(); // Update job status to 'processing' $this->job->setStatus(Job::STATUS_PROCESSING); $this->job->setStartTime(date('c', $startTime)); $this->job->setEndTime(null); $this->job->setDurationSeconds(null); $this->job->setResult(null); $this->job->setProcess(['host' => gethostname(), 'pid' => getmypid()]); /** @var ExecutorInterface $jobExecutor */ $jobExecutor = $this->getContainer()->get('syrup.job_executor_factory')->create(); // update the job status after jobExecutor was created, so the signal handler is properly registered $this->jobMapper->update($this->job); // Execute job try { if (StorageApiLimits::hasParallelLimit($this->storageApiService->getTokenData())) { $validationLock->unlock(); $checkConn->close(); } $jobResult = $jobExecutor->execute($this->job); $jobStatus = Job::STATUS_SUCCESS; $status = self::STATUS_SUCCESS; } catch (InitializationException $e) { // job will be requeued $exceptionId = $this->logException('error', $e); $jobResult = ['message' => $e->getMessage(), 'exceptionId' => $exceptionId]; $jobStatus = Job::STATUS_PROCESSING; $status = self::STATUS_RETRY; } catch (MaintenanceException $e) { $jobResult = []; $jobStatus = Job::STATUS_WAITING; $status = self::STATUS_LOCK; } catch (UserException $e) { $exceptionId = $this->logException('error', $e); $jobResult = ['message' => $e->getMessage(), 'exceptionId' => $exceptionId]; $jobStatus = Job::STATUS_ERROR; $this->job->setError(Job::ERROR_USER); $status = self::STATUS_SUCCESS; } catch (JobException $e) { $logLevel = 'error'; if ($e->getStatus() === Job::STATUS_WARNING) { $logLevel = Job::STATUS_WARNING; } $exceptionId = $this->logException($logLevel, $e); $jobResult = ['message' => $e->getMessage(), 'exceptionId' => $exceptionId]; if ($e->getResult()) { $jobResult += $e->getResult(); } $jobStatus = $e->getStatus(); $status = self::STATUS_SUCCESS; } catch (\Exception $e) { // make sure that the job is recorded as failed $jobStatus = Job::STATUS_ERROR; $jobResult = ['message' => 'Internal error occurred, evaluating details']; $this->job->setStatus($jobStatus); $this->job->setResult($jobResult); $this->job->setError(Job::ERROR_APPLICATION); // try to log the exception $exceptionId = $this->logException('critical', $e); $jobResult = ['message' => 'Internal error occurred, please contact support@keboola.com', 'exceptionId' => $exceptionId]; $status = self::STATUS_ERROR; } // Update job with results $endTime = time(); $createdTime = $this->job->getCreatedTime(); $waitSeconds = is_null($createdTime) ?: $startTime - strtotime($createdTime); $this->job->setStatus($jobStatus); $this->job->setResult($jobResult); $this->job->setEndTime(date('c', $endTime)); $this->job->setDurationSeconds($endTime - $startTime); $this->job->setWaitSeconds($waitSeconds); $this->jobMapper->update($this->job); // postExecution action try { $jobExecutor->postExecute($this->job); //@todo: remove call to this deprecated interface method if ($jobExecutor instanceof HookExecutorInterface) { /** @var HookExecutorInterface $jobExecutor */ $jobExecutor->postExecution($this->job); } } catch (\Exception $e) { $this->logException('critical', $e); } // DB unlock $this->lock->unlock(); return $status; }
private function watchWaiting(InputInterface $input, OutputInterface $output) { $now = new \DateTime(); $output->writeln(sprintf('<info>%s</info>', 'Check waiting jobs')); $i = 0; while ($jobs = $this->loadWaitingJobs($i)) { foreach ($jobs as $job) { $i++; // check sapi connection if (!$this->sapiPing($job, $this->encryptor)) { $message = 'SAPI connect failed'; $this->logger->info($message, array('job' => $job->getId(), 'project' => $job->getProject(), 'params' => $job->getRawParams())); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } $sapiClient = $this->createJobSapiClient($job); $jobManager = $this->jobManagerFactory->createJobManager(new StorageApi\Token($sapiClient)); try { $esJob = $jobManager->findJobByIdForWatchdog($job->getId()); if (!$esJob) { $message = 'Job load from ES failed'; $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } } catch (\InvalidArgumentException $e) { $message = 'Job load from ES failed'; $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } // clean jobs of deleted orchestrations $orchestration = $this->orchestrationManager->findDeletedOrchestrations(array('id' => $esJob->getOrchestrationId()), false); $orchestration = reset($orchestration); if ($orchestration) { $updateJob = $this->syrupJobMapper->get($esJob->getId()); if ($updateJob->getComponent() !== KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME) { continue; } $updateJob->setStatus(Metadata\Job::STATUS_CANCELLED); $this->syrupJobMapper->update($updateJob); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[deleted orchestration]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); continue; } $orchestrations = $this->orchestrationManager->findOrchestrations(array('id' => $esJob->getOrchestrationId()), false); /** * @var Orchestration $orchestration */ $orchestration = reset($orchestrations); if (!$orchestration) { $message = 'Orchestration load failed'; $this->logger->critical($message, array('job' => $job->getId(), 'orchestrationId' => $esJob->getOrchestrationId(), 'projectId' => $esJob->getProjectId(), 'project' => $esJob->getTokenOwnerName())); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=red>[ERROR] %s</fg=red>', $job->getId(), $message)); continue; } $componentsList = new KbcComponentsList($sapiClient); // waiting duration check if ($esJob->getInitializedBy() === 'manually') { /** * @var StorageApi\Notification[] $notifications */ $notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) { return $row->getChannel() === Metadata\Job::STATUS_WAITING; }); if ($notifications) { foreach ($notifications as $notification) { if ($notification->getEmail() !== $esJob->getInitiatorTokenDesc()) { continue; } $date = new \DateTime($job->getCreatedTime()); $date->modify(sprintf('+%d minutes', $notification->getParameter('timeout', 0))); if ($date > $now) { continue; } $events = StorageApi\EventLoader::longWaitingEvents($esJob, $notification, $sapiClient); if ($events) { $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } else { $this->mailer->sendLongWaitingMessage($esJob, $orchestration, $notification, $componentsList); $this->logLongWaiting($esJob, $sapiClient, $notification); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } } } else { // log waiting for default timeout $date = new \DateTime($job->getCreatedTime()); $date->modify(sprintf('+%d minutes', self::WAIT_LIMIT_MINUTES)); if ($date <= $now) { $events = StorageApi\EventLoader::longWaitingEventsAll($esJob, $sapiClient); if ($events) { $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[system already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } else { $this->logLongWaiting($esJob, $sapiClient, null); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[system notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } } } } else { /** * @var StorageApi\Notification[] $notifications */ $notifications = array_filter($orchestration->getNotifications(), function (StorageApi\Notification $row) { return $row->getChannel() === Metadata\Job::STATUS_WAITING; }); if ($notifications) { foreach ($notifications as $notification) { $date = new \DateTime($job->getCreatedTime()); $date->modify(sprintf('+%d minutes', $notification->getParameter('timeout', 0))); if ($date > $now) { continue; } $events = StorageApi\EventLoader::longWaitingEvents($esJob, $notification, $sapiClient); if ($events) { $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } else { $this->mailer->sendLongWaitingMessage($esJob, $orchestration, $notification, $componentsList); $this->logLongWaiting($esJob, $sapiClient, $notification); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } } } else { // log waiting for default timeout $date = new \DateTime($job->getCreatedTime()); $date->modify(sprintf('+%d minutes', self::WAIT_LIMIT_MINUTES)); if ($date <= $now) { $events = StorageApi\EventLoader::longWaitingEventsAll($esJob, $sapiClient); if ($events) { $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=yellow>[system already notified]</fg=yellow> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } else { $this->logLongWaiting($esJob, $sapiClient, null); $output->writeln(sprintf('<fg=white>%s</fg=white> <fg=green>[system notified]</fg=green> <fg=white>%s %s</fg=white>', $esJob->getId(), $esJob->getOrchestrationId(), $esJob->getOrchestrationName())); } } } } } } }