public function __invoke(Job $job)
{
$job = $this->elasticsearch->getJob($job->getId());
// find and kill child jobs
$childJobs = $this->elasticsearch->getJobs(['query' => 'runId:' . $job->getRunId() . '.*', 'project.id' => $job->getProject()['id']]);
$this->logger->debug("Child jobs found", ['jobs' => $childJobs]);
$snsClient = new SnsClient(['credentials' => ['key' => $this->snsConfig['key'], 'secret' => $this->snsConfig['secret']], 'region' => isset($this->snsConfig['region']) ? $this->snsConfig['region'] : 'us-east-1', 'version' => '2010-03-31']);
// kill them all
if (!empty($childJobs)) {
foreach ($childJobs as $childJob) {
$snsClient->publish(['TopicArn' => $this->snsConfig['topic_arn'], 'Message' => json_encode(['jobId' => $childJob['id']])]);
}
}
$callback = $this->cleanupCallback;
$callback($job);
$endTime = time();
$duration = $endTime - strtotime($job->getStartTime());
$job->setStatus(Job::STATUS_TERMINATED);
$job->setResult(['message' => 'Job has been terminated']);
$job->setEndTime(date('c', $endTime));
$job->setDurationSeconds($duration);
$this->jobMapper->update($job);
}
public function testGetJob()
{
$job = $this->createJob();
$id = self::$jobMapper->create($job);
$resJob = self::$search->getJob($id);
$this->assertJob($job, $resJob->getData());
}
protected function processMessage(QueueMessage $message)
{
$msg = null;
if (isset($message->getBody()->Message)) {
$msg = json_decode($message->getBody()->Message);
}
$this->logger->debug("Processing kill message", ['message' => $msg]);
if ($msg != null) {
$jobId = $msg->jobId;
/** @var Job $job */
$job = $this->elasticsearch->getJob($jobId);
if (!is_null($job)) {
if ($job->getProcess()['host'] == gethostname()) {
if ($job->getStatus() == Job::STATUS_WAITING) {
$job->setStatus(Job::STATUS_CANCELLED);
$job->setResult(['message' => 'Job cancelled by user']);
$this->getComponentJobMapper($job->getComponent())->update($job);
$this->logger->info("receive-kill: Job '{$jobId}' cancelled", ['job' => $job->getData()]);
$this->requeue($job->getId());
} else {
if ($job->getStatus() == Job::STATUS_PROCESSING) {
$job->setStatus(Job::STATUS_TERMINATING);
$this->getComponentJobMapper($job->getComponent())->update($job);
$pid = $job->getProcess()['pid'];
// kill child processes
$process = new Process('
function getcpid() {
cpids=`pgrep -P $1|xargs`
for cpid in $cpids;
do
echo "$cpid"
getcpid $cpid
done
}
getcpid ' . $pid);
$process->run();
$processes = $process->getOutput();
if ($processes && count(explode("\n", $processes))) {
foreach (explode("\n", $processes) as $child) {
if ($child != '') {
(new Process("sudo kill -KILL {$child}"))->run();
}
}
}
// kill parent process
posix_kill($pid, SIGKILL);
$this->logger->info("receive-kill: Job '{$jobId}' killed", ['job' => $job->getData()]);
} else {
$this->logger->info("receive-kill: Job is not in waiting or processing state", ['job' => $job->getData()]);
}
}
}
}
} else {
$this->logger->warn("Corrupted message received", ['message' => $message]);
}
$this->queue->deleteMessage($message);
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$jobId = $input->getArgument('jobId');
$this->elasticsearch = $this->getContainer()->get('syrup.elasticsearch.search');
/** @var Job $job */
$job = $this->elasticsearch->getJob($jobId);
if ($job->getStatus() != Job::STATUS_PROCESSING && $job->getStatus() != Job::STATUS_WAITING) {
$output->writeln("<error>Job {$jobId} is not in 'waiting' or 'processing' state</error>");
} else {
// update job status and possibly kill it
if ($job->getStatus() == Job::STATUS_WAITING) {
$job->setStatus(Job::STATUS_CANCELLED);
$job->setResult(['message' => 'Job cancelled by user']);
$output->writeln("<info>Job {$jobId} cancelled</info>");
$this->elasticsearch->updateJob($job);
} else {
$job->setStatus(Job::STATUS_TERMINATING);
$this->elasticsearch->updateJob($job);
// @TODO: send message to SNS
}
}
}