protected function processMessage(QueueMessage $message) { $msg = null; if (isset($message->getBody()->Message)) { $msg = json_decode($message->getBody()->Message); } $this->logger->debug("Processing kill message", ['message' => $msg]); if ($msg != null) { $jobId = $msg->jobId; /** @var Job $job */ $job = $this->elasticsearch->getJob($jobId); if (!is_null($job)) { if ($job->getProcess()['host'] == gethostname()) { if ($job->getStatus() == Job::STATUS_WAITING) { $job->setStatus(Job::STATUS_CANCELLED); $job->setResult(['message' => 'Job cancelled by user']); $this->getComponentJobMapper($job->getComponent())->update($job); $this->logger->info("receive-kill: Job '{$jobId}' cancelled", ['job' => $job->getData()]); $this->requeue($job->getId()); } else { if ($job->getStatus() == Job::STATUS_PROCESSING) { $job->setStatus(Job::STATUS_TERMINATING); $this->getComponentJobMapper($job->getComponent())->update($job); $pid = $job->getProcess()['pid']; // kill child processes $process = new Process(' function getcpid() { cpids=`pgrep -P $1|xargs` for cpid in $cpids; do echo "$cpid" getcpid $cpid done } getcpid ' . $pid); $process->run(); $processes = $process->getOutput(); if ($processes && count(explode("\n", $processes))) { foreach (explode("\n", $processes) as $child) { if ($child != '') { (new Process("sudo kill -KILL {$child}"))->run(); } } } // kill parent process posix_kill($pid, SIGKILL); $this->logger->info("receive-kill: Job '{$jobId}' killed", ['job' => $job->getData()]); } else { $this->logger->info("receive-kill: Job is not in waiting or processing state", ['job' => $job->getData()]); } } } } } else { $this->logger->warn("Corrupted message received", ['message' => $message]); } $this->queue->deleteMessage($message); }
protected function execute(InputInterface $input, OutputInterface $output) { $this->logger = $this->getContainer()->get('logger'); $this->dynamoDbClient = $this->getContainer()->get('queue.dynamodb_client'); $this->tableName = $this->getContainer()->getParameter('container_stats_dynamodb.table_name'); try { $this->dynamoDbClient->describeTable(['TableName' => $this->tableName]); } catch (DynamoDbException $e) { if (strpos($e->getMessage(), 'ResourceNotFoundException') !== false) { throw new \Exception('Table ' . $this->tableName . ' doesn\'t exist.'); } else { throw $e; } } $queueId = $input->getArgument('queueId'); $this->queue = $this->getContainer()->get('syrup.queue_factory')->get($queueId); $startTime = time(); do { foreach ($this->queue->receive(self::MAX_NUMBER_OF_MESSAGES) as $message) { $this->processMessage($message); $this->queue->deleteMessage($message); } } while (time() - $startTime < self::MAX_RUN_TIME); }
/** * Check job in syrup queue * * @param $orchestrationId */ private function sqsScheduledJobTest($orchestrationId) { $assertCount = 1; $requiredStatus = StatusConverter::syrupToOrchestrator(Metadata\Job::STATUS_WAITING); $url = sprintf('/orchestrator/orchestrations/%d/jobs', $orchestrationId); $response = $this->callApiGet($url); $errorMessage = sprintf("Response for call '%s %s' should return only %d jobs.", 'GET', $url, $assertCount); $this->assertCount($assertCount, $response, $errorMessage); $errorMessage = sprintf("Response for call '%s %s' should return list of scheduled jobs.", 'GET', $url); $this->assertArrayHasKey('id', $response[0], $errorMessage); $this->assertArrayHasKey('initializedBy', $response[0], $errorMessage); $this->assertArrayHasKey('status', $response[0], $errorMessage); $this->assertEquals('scheduler', $response[0]['initializedBy'], $errorMessage); $this->assertEquals($requiredStatus, $response[0]['status'], $errorMessage); $this->assertArrayHasKey('isFinished', $response[0]); $this->assertEquals(false, $response[0]['isFinished']); $jobId = $response[0]['id']; $maxTime = 60; $startTime = time(); $errorMessage = sprintf("SQS Message for job '%s' not found.", $jobId); do { foreach ($this->queue->receive() as $message) { /** @var Queue\QueueMessage $message */ $body = $message->getBody(); $errorMessage = 'SQS Message should contains jobId and component name'; $this->assertObjectHasAttribute('jobId', $body, $errorMessage); $this->assertObjectHasAttribute('component', $body, $errorMessage); $errorMessage = 'SQS Message is not from orchestrator'; $this->assertEquals(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $body->component, $errorMessage); $this->queue->deleteMessage($message); if ($body->jobId == $jobId) { $jobId = null; } } } while ($jobId && time() - $startTime < $maxTime); $this->assertEmpty($jobId, $errorMessage); }
protected function execute(InputInterface $input, OutputInterface $output) { parent::execute($input, $output); $this->logger->info('scheduler.start', array()); $output->writeln('Current time: ' . $this->getStartTime()->format('Y-m-d H:i:s')); $this->lock->setLockName('scheduler-' . $this->getStartTime()->format('Y-m-d-H-i')); // skip locked try { if (!$this->lock->lock()) { $this->logger->info('scheduler.skipped', array('lockName' => $this->lock->getLockName())); return; } } catch (\Exception $e) { $this->logger->info('scheduler.error', array('message' => $e->getMessage(), 'lockName' => $this->lock->getLockName())); return; } $this->logger->info('scheduler.locked', array('lockName' => $this->lock->getLockName())); /** * @var Elasticsearch\JobManagerFactory $jobManagerFactory */ $jobManagerFactory = $this->getContainer()->get('orchestrator.job_manager.factory'); $this->queue = $this->getContainer()->get('syrup.queue_factory')->get(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME); $this->dbOrchestrationManager = $this->getContainer()->get('orchestrator.doctrine.orchestration_manager'); $newJobs = array(); $orchestrations = $this->dbOrchestrationManager->findOrchestrations(array()); foreach ($orchestrations as $orchestration) { // skip deleted and non-active orchestrations if ($orchestration->getDeleted() || !$orchestration->getActive() || !$orchestration->getCrontabRecord()) { continue; } $cronExpression = CronExpression::factory($orchestration->getCrontabRecord()); if (!$cronExpression->isDue($this->getStartTime())) { continue; } $jobClient = new Client(array('token' => $orchestration->getToken(), 'url' => $this->getConfiguration()->getStorageApiUrl(), 'userAgent' => $this->appName)); try { $pingClient = new StorageApi\PingClient($jobClient, $this->logger); if (!$pingClient->ping()) { $this->logger->info('scheduler.orchestration.skipped.disabled', array('lockName' => $this->lock->getLockName(), 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $orchestration->getProjectId(), 'projectName' => $orchestration->getTokenOwnerName())); continue; } } catch (\Exception $e) { $this->logger->info('scheduler.orchestration.skipped.error', array('lockName' => $this->lock->getLockName(), 'message' => $e->getMessage(), 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $orchestration->getProjectId(), 'projectName' => $orchestration->getTokenOwnerName())); continue; } $token = new StorageApi\Token($jobClient); $jobEsManager = $jobManagerFactory->createJobManager($token); $jobsStats = $jobEsManager->getWaitingStats(); // skip waiting if (array_key_exists($orchestration->getId(), $jobsStats) && $jobsStats[$orchestration->getId()] >= AppConfiguration::MAX_WAITING_JOBS_COUNT) { $this->logger->info('scheduler.orchestration.skipped', array('lockName' => $this->lock->getLockName(), 'waitingCount' => $jobsStats[$orchestration->getId()], 'limit' => AppConfiguration::MAX_WAITING_JOBS_COUNT, 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $token->getProjectId(), 'projectName' => $token->getOwnerName())); continue; } $orchestrationWithTasks = $this->dbOrchestrationManager->findOrchestrationById($orchestration->getId(), new StorageApi\Token($jobClient), true); $tasks = array_map(function ($task) { /** @var StorageApi\OrchestrationTask $task */ return $task->toApiArray(); }, $orchestrationWithTasks->getTasks()); $orchestration = $this->dbOrchestrationManager->updateLastScheduledTime($orchestration, new \DateTime()); // save job to queue $job = new Elasticsearch\Job(); $job->setOrchestrationId($orchestration->getId())->setConfig($orchestration->getId())->setOrchestrationName($orchestration->getName())->setToken($orchestration->getToken())->setTokenId($orchestration->getTokenId())->setTokenDesc($orchestration->getTokenDesc())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId())->setInitializedBy('scheduler')->setTaks($tasks)->setInitiatorTokenId($orchestration->getTokenId())->setInitiatorTokenDesc($orchestration->getTokenDesc())->setInitiatorUserAgent($this->appName . ' Scheduler'); $job = $jobEsManager->saveJob($job, new StorageApi\UniqueManager($jobClient)); $newJobs[] = $job; // add job to queue try { if (is_null($job)) { throw new ApplicationException('Unable to save or retrieve job from ES', null, array('orchestrationId' => $orchestration->getId())); } $this->queue->enqueue($job->getId(), array('jobId' => $job->getId(), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME)); $output->writeln(sprintf('Orchestration %d with cron %s scheduled. Job id: %s', $orchestration->getId(), $orchestration->getCrontabRecord(), $job->getId())); } catch (\Exception $e) { $this->logger->error('Error during adding job to sqs', array('lockName' => $this->lock->getLockName(), 'jobId' => $job->getId(), 'exception' => $e)); continue; } // log event try { $event = new Event(); $event->setComponent($this->appName)->setRunId($job->getRunId())->setParams(array('orchestrationId' => $job->getOrchestrationId()))->setResults(array('jobId' => $job->getId()))->setMessage(sprintf('Orchestration job %s scheduled', $job->getId())); StorageApi\EventLogger::create($event, $jobClient); $this->logger->info($event->getMessage(), array('lockName' => $this->lock->getLockName(), 'configurationId' => $event->getConfigurationId(), 'runId' => $event->getRunId(), 'description' => $event->getDescription(), 'params' => $event->getParams(), 'results' => $event->getResults())); } catch (Exception $e) { //@TODO user notification when notification API will be available $this->logger->warning('Invalid token', array('lockName' => $this->lock->getLockName(), 'jobId' => $job->getId(), 'exception' => $e)); } } $this->logger->info('scheduler.end', array('lockName' => $this->lock->getLockName(), 'duration' => $this->getDuration(), 'orchestrationsCheckedCount' => count($orchestrations), 'scheduledJobsCount' => count($newJobs))); sleep(90); // trying fix double run // unlock $this->lock->unlock(); }
protected function requeue(QueueMessage $message) { $delaySecs = 30 + 30 * pow(2, $message->getRetryCount()); $delaySecs = $delaySecs > 600 ? 600 : $delaySecs; return $this->queue->enqueue($message->getBody()->jobId, ['component' => $message->getBody()->component], $delaySecs); }