protected function processMessage(QueueMessage $message)
{
$msg = null;
if (isset($message->getBody()->Message)) {
$msg = json_decode($message->getBody()->Message);
}
$this->logger->debug("Processing kill message", ['message' => $msg]);
if ($msg != null) {
$jobId = $msg->jobId;
/** @var Job $job */
$job = $this->elasticsearch->getJob($jobId);
if (!is_null($job)) {
if ($job->getProcess()['host'] == gethostname()) {
if ($job->getStatus() == Job::STATUS_WAITING) {
$job->setStatus(Job::STATUS_CANCELLED);
$job->setResult(['message' => 'Job cancelled by user']);
$this->getComponentJobMapper($job->getComponent())->update($job);
$this->logger->info("receive-kill: Job '{$jobId}' cancelled", ['job' => $job->getData()]);
$this->requeue($job->getId());
} else {
if ($job->getStatus() == Job::STATUS_PROCESSING) {
$job->setStatus(Job::STATUS_TERMINATING);
$this->getComponentJobMapper($job->getComponent())->update($job);
$pid = $job->getProcess()['pid'];
// kill child processes
$process = new Process('
function getcpid() {
cpids=`pgrep -P $1|xargs`
for cpid in $cpids;
do
echo "$cpid"
getcpid $cpid
done
}
getcpid ' . $pid);
$process->run();
$processes = $process->getOutput();
if ($processes && count(explode("\n", $processes))) {
foreach (explode("\n", $processes) as $child) {
if ($child != '') {
(new Process("sudo kill -KILL {$child}"))->run();
}
}
}
// kill parent process
posix_kill($pid, SIGKILL);
$this->logger->info("receive-kill: Job '{$jobId}' killed", ['job' => $job->getData()]);
} else {
$this->logger->info("receive-kill: Job is not in waiting or processing state", ['job' => $job->getData()]);
}
}
}
}
} else {
$this->logger->warn("Corrupted message received", ['message' => $message]);
}
$this->queue->deleteMessage($message);
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$this->logger = $this->getContainer()->get('logger');
$this->dynamoDbClient = $this->getContainer()->get('queue.dynamodb_client');
$this->tableName = $this->getContainer()->getParameter('container_stats_dynamodb.table_name');
try {
$this->dynamoDbClient->describeTable(['TableName' => $this->tableName]);
} catch (DynamoDbException $e) {
if (strpos($e->getMessage(), 'ResourceNotFoundException') !== false) {
throw new \Exception('Table ' . $this->tableName . ' doesn\'t exist.');
} else {
throw $e;
}
}
$queueId = $input->getArgument('queueId');
$this->queue = $this->getContainer()->get('syrup.queue_factory')->get($queueId);
$startTime = time();
do {
foreach ($this->queue->receive(self::MAX_NUMBER_OF_MESSAGES) as $message) {
$this->processMessage($message);
$this->queue->deleteMessage($message);
}
} while (time() - $startTime < self::MAX_RUN_TIME);
}
/**
* Check job in syrup queue
*
* @param $orchestrationId
*/
private function sqsScheduledJobTest($orchestrationId)
{
$assertCount = 1;
$requiredStatus = StatusConverter::syrupToOrchestrator(Metadata\Job::STATUS_WAITING);
$url = sprintf('/orchestrator/orchestrations/%d/jobs', $orchestrationId);
$response = $this->callApiGet($url);
$errorMessage = sprintf("Response for call '%s %s' should return only %d jobs.", 'GET', $url, $assertCount);
$this->assertCount($assertCount, $response, $errorMessage);
$errorMessage = sprintf("Response for call '%s %s' should return list of scheduled jobs.", 'GET', $url);
$this->assertArrayHasKey('id', $response[0], $errorMessage);
$this->assertArrayHasKey('initializedBy', $response[0], $errorMessage);
$this->assertArrayHasKey('status', $response[0], $errorMessage);
$this->assertEquals('scheduler', $response[0]['initializedBy'], $errorMessage);
$this->assertEquals($requiredStatus, $response[0]['status'], $errorMessage);
$this->assertArrayHasKey('isFinished', $response[0]);
$this->assertEquals(false, $response[0]['isFinished']);
$jobId = $response[0]['id'];
$maxTime = 60;
$startTime = time();
$errorMessage = sprintf("SQS Message for job '%s' not found.", $jobId);
do {
foreach ($this->queue->receive() as $message) {
/** @var Queue\QueueMessage $message */
$body = $message->getBody();
$errorMessage = 'SQS Message should contains jobId and component name';
$this->assertObjectHasAttribute('jobId', $body, $errorMessage);
$this->assertObjectHasAttribute('component', $body, $errorMessage);
$errorMessage = 'SQS Message is not from orchestrator';
$this->assertEquals(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $body->component, $errorMessage);
$this->queue->deleteMessage($message);
if ($body->jobId == $jobId) {
$jobId = null;
}
}
} while ($jobId && time() - $startTime < $maxTime);
$this->assertEmpty($jobId, $errorMessage);
}
