public function __invoke(Job $job) { $job = $this->elasticsearch->getJob($job->getId()); // find and kill child jobs $childJobs = $this->elasticsearch->getJobs(['query' => 'runId:' . $job->getRunId() . '.*', 'project.id' => $job->getProject()['id']]); $this->logger->debug("Child jobs found", ['jobs' => $childJobs]); $snsClient = new SnsClient(['credentials' => ['key' => $this->snsConfig['key'], 'secret' => $this->snsConfig['secret']], 'region' => isset($this->snsConfig['region']) ? $this->snsConfig['region'] : 'us-east-1', 'version' => '2010-03-31']); // kill them all if (!empty($childJobs)) { foreach ($childJobs as $childJob) { $snsClient->publish(['TopicArn' => $this->snsConfig['topic_arn'], 'Message' => json_encode(['jobId' => $childJob['id']])]); } } $callback = $this->cleanupCallback; $callback($job); $endTime = time(); $duration = $endTime - strtotime($job->getStartTime()); $job->setStatus(Job::STATUS_TERMINATED); $job->setResult(['message' => 'Job has been terminated']); $job->setEndTime(date('c', $endTime)); $job->setDurationSeconds($duration); $this->jobMapper->update($job); }
protected function processMessage(QueueMessage $message) { $msg = null; if (isset($message->getBody()->Message)) { $msg = json_decode($message->getBody()->Message); } $this->logger->debug("Processing kill message", ['message' => $msg]); if ($msg != null) { $jobId = $msg->jobId; /** @var Job $job */ $job = $this->elasticsearch->getJob($jobId); if (!is_null($job)) { if ($job->getProcess()['host'] == gethostname()) { if ($job->getStatus() == Job::STATUS_WAITING) { $job->setStatus(Job::STATUS_CANCELLED); $job->setResult(['message' => 'Job cancelled by user']); $this->getComponentJobMapper($job->getComponent())->update($job); $this->logger->info("receive-kill: Job '{$jobId}' cancelled", ['job' => $job->getData()]); $this->requeue($job->getId()); } else { if ($job->getStatus() == Job::STATUS_PROCESSING) { $job->setStatus(Job::STATUS_TERMINATING); $this->getComponentJobMapper($job->getComponent())->update($job); $pid = $job->getProcess()['pid']; // kill child processes $process = new Process(' function getcpid() { cpids=`pgrep -P $1|xargs` for cpid in $cpids; do echo "$cpid" getcpid $cpid done } getcpid ' . $pid); $process->run(); $processes = $process->getOutput(); if ($processes && count(explode("\n", $processes))) { foreach (explode("\n", $processes) as $child) { if ($child != '') { (new Process("sudo kill -KILL {$child}"))->run(); } } } // kill parent process posix_kill($pid, SIGKILL); $this->logger->info("receive-kill: Job '{$jobId}' killed", ['job' => $job->getData()]); } else { $this->logger->info("receive-kill: Job is not in waiting or processing state", ['job' => $job->getData()]); } } } } } else { $this->logger->warn("Corrupted message received", ['message' => $message]); } $this->queue->deleteMessage($message); }
/** * Fetches job details for specified id * @param $runId * @return array|null */ private function fetchStatsForJob($runId) { // a bit of hack, since there's no method for getting job by runId $jobs = $this->elasticsearch->getJobs(['runId' => $runId]); if (!empty($jobs)) { $job = reset($jobs); return $job; } else { return null; } }
protected function execute(InputInterface $input, OutputInterface $output) { $jobId = $input->getArgument('jobId'); $this->elasticsearch = $this->getContainer()->get('syrup.elasticsearch.search'); /** @var Job $job */ $job = $this->elasticsearch->getJob($jobId); if ($job->getStatus() != Job::STATUS_PROCESSING && $job->getStatus() != Job::STATUS_WAITING) { $output->writeln("<error>Job {$jobId} is not in 'waiting' or 'processing' state</error>"); } else { // update job status and possibly kill it if ($job->getStatus() == Job::STATUS_WAITING) { $job->setStatus(Job::STATUS_CANCELLED); $job->setResult(['message' => 'Job cancelled by user']); $output->writeln("<info>Job {$jobId} cancelled</info>"); $this->elasticsearch->updateJob($job); } else { $job->setStatus(Job::STATUS_TERMINATING); $this->elasticsearch->updateJob($job); // @TODO: send message to SNS } } }
/** * @param array $criteria * @return Job[] * @throws \Keboola\StorageApi\Exception */ public function findJobs(array $criteria, $offset = 0, $limit = JobManager::PAGING) { $exportOptions = $this->buildExportOptions($criteria); $data = $this->syrupJobSearch->getJobs(array('projectId' => $this->token->getProjectId(), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, 'offset' => $offset, 'limit' => $limit, 'query' => $exportOptions)); $manager = $this; return array_map(function ($line) use($manager) { $esJob = new Elasticsearch\Job($this->objectEncryptor, $line, $line['_index'], $line['_type']); $job = new Job(); $job->build($esJob); $job->setUrl($manager->generateJobUri($job)); $job->setToken($this->decryptToken($job->getToken())); return $job; }, $data); }
public function testGetIndices() { $indices = self::$search->getIndices(); $this->assertNotEmpty($indices); }