Ejemplo n.º 1
2
 public function __invoke(Job $job)
 {
     $job = $this->elasticsearch->getJob($job->getId());
     // find and kill child jobs
     $childJobs = $this->elasticsearch->getJobs(['query' => 'runId:' . $job->getRunId() . '.*', 'project.id' => $job->getProject()['id']]);
     $this->logger->debug("Child jobs found", ['jobs' => $childJobs]);
     $snsClient = new SnsClient(['credentials' => ['key' => $this->snsConfig['key'], 'secret' => $this->snsConfig['secret']], 'region' => isset($this->snsConfig['region']) ? $this->snsConfig['region'] : 'us-east-1', 'version' => '2010-03-31']);
     // kill them all
     if (!empty($childJobs)) {
         foreach ($childJobs as $childJob) {
             $snsClient->publish(['TopicArn' => $this->snsConfig['topic_arn'], 'Message' => json_encode(['jobId' => $childJob['id']])]);
         }
     }
     $callback = $this->cleanupCallback;
     $callback($job);
     $endTime = time();
     $duration = $endTime - strtotime($job->getStartTime());
     $job->setStatus(Job::STATUS_TERMINATED);
     $job->setResult(['message' => 'Job has been terminated']);
     $job->setEndTime(date('c', $endTime));
     $job->setDurationSeconds($duration);
     $this->jobMapper->update($job);
 }
Ejemplo n.º 2
0
 protected function processMessage(QueueMessage $message)
 {
     $msg = null;
     if (isset($message->getBody()->Message)) {
         $msg = json_decode($message->getBody()->Message);
     }
     $this->logger->debug("Processing kill message", ['message' => $msg]);
     if ($msg != null) {
         $jobId = $msg->jobId;
         /** @var Job $job */
         $job = $this->elasticsearch->getJob($jobId);
         if (!is_null($job)) {
             if ($job->getProcess()['host'] == gethostname()) {
                 if ($job->getStatus() == Job::STATUS_WAITING) {
                     $job->setStatus(Job::STATUS_CANCELLED);
                     $job->setResult(['message' => 'Job cancelled by user']);
                     $this->getComponentJobMapper($job->getComponent())->update($job);
                     $this->logger->info("receive-kill: Job '{$jobId}' cancelled", ['job' => $job->getData()]);
                     $this->requeue($job->getId());
                 } else {
                     if ($job->getStatus() == Job::STATUS_PROCESSING) {
                         $job->setStatus(Job::STATUS_TERMINATING);
                         $this->getComponentJobMapper($job->getComponent())->update($job);
                         $pid = $job->getProcess()['pid'];
                         // kill child processes
                         $process = new Process('
                         function getcpid() {
                                     cpids=`pgrep -P $1|xargs`
                             for cpid in $cpids;
                             do
                                 echo "$cpid"
                                 getcpid $cpid
                             done
                         }
                         getcpid ' . $pid);
                         $process->run();
                         $processes = $process->getOutput();
                         if ($processes && count(explode("\n", $processes))) {
                             foreach (explode("\n", $processes) as $child) {
                                 if ($child != '') {
                                     (new Process("sudo kill -KILL {$child}"))->run();
                                 }
                             }
                         }
                         // kill parent process
                         posix_kill($pid, SIGKILL);
                         $this->logger->info("receive-kill: Job '{$jobId}' killed", ['job' => $job->getData()]);
                     } else {
                         $this->logger->info("receive-kill: Job is not in waiting or processing state", ['job' => $job->getData()]);
                     }
                 }
             }
         }
     } else {
         $this->logger->warn("Corrupted message received", ['message' => $message]);
     }
     $this->queue->deleteMessage($message);
 }
 /**
  * Fetches job details for specified id
  * @param $runId
  * @return array|null
  */
 private function fetchStatsForJob($runId)
 {
     // a bit of hack, since there's no method for getting job by runId
     $jobs = $this->elasticsearch->getJobs(['runId' => $runId]);
     if (!empty($jobs)) {
         $job = reset($jobs);
         return $job;
     } else {
         return null;
     }
 }
Ejemplo n.º 4
0
 protected function execute(InputInterface $input, OutputInterface $output)
 {
     $jobId = $input->getArgument('jobId');
     $this->elasticsearch = $this->getContainer()->get('syrup.elasticsearch.search');
     /** @var Job $job */
     $job = $this->elasticsearch->getJob($jobId);
     if ($job->getStatus() != Job::STATUS_PROCESSING && $job->getStatus() != Job::STATUS_WAITING) {
         $output->writeln("<error>Job {$jobId} is not in 'waiting' or 'processing' state</error>");
     } else {
         // update job status and possibly kill it
         if ($job->getStatus() == Job::STATUS_WAITING) {
             $job->setStatus(Job::STATUS_CANCELLED);
             $job->setResult(['message' => 'Job cancelled by user']);
             $output->writeln("<info>Job {$jobId} cancelled</info>");
             $this->elasticsearch->updateJob($job);
         } else {
             $job->setStatus(Job::STATUS_TERMINATING);
             $this->elasticsearch->updateJob($job);
             // @TODO: send message to SNS
         }
     }
 }
Ejemplo n.º 5
0
 /**
  * @param array $criteria
  * @return Job[]
  * @throws \Keboola\StorageApi\Exception
  */
 public function findJobs(array $criteria, $offset = 0, $limit = JobManager::PAGING)
 {
     $exportOptions = $this->buildExportOptions($criteria);
     $data = $this->syrupJobSearch->getJobs(array('projectId' => $this->token->getProjectId(), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, 'offset' => $offset, 'limit' => $limit, 'query' => $exportOptions));
     $manager = $this;
     return array_map(function ($line) use($manager) {
         $esJob = new Elasticsearch\Job($this->objectEncryptor, $line, $line['_index'], $line['_type']);
         $job = new Job();
         $job->build($esJob);
         $job->setUrl($manager->generateJobUri($job));
         $job->setToken($this->decryptToken($job->getToken()));
         return $job;
     }, $data);
 }
Ejemplo n.º 6
0
 public function testGetIndices()
 {
     $indices = self::$search->getIndices();
     $this->assertNotEmpty($indices);
 }