/**
* Processes message from SQS Queue
* @param QueueMessage $message
*/
private function processMessage(QueueMessage $message)
{
$body = \Keboola\Utils\objectToArray($message->getBody());
if (isset($body['container_id'], $body['container_name'], $body['container_stats'])) {
$containerName = $body['container_name'];
$jobIdAndRunId = $this->getJobIdAndRunIdFromContainerName($containerName);
if (!empty($jobIdAndRunId)) {
$containerStats = $body['container_stats'];
$networkRxBytes = strval(isset($containerStats['network']['rx_bytes']) ? $containerStats['network']['rx_bytes'] : 0);
$networkTxBytes = strval(isset($containerStats['network']['tx_bytes']) ? $containerStats['network']['tx_bytes'] : 0);
$updateExpression = 'SET dockerNetwork = :dockerNetwork';
$expressionAttributeValues = [':dockerNetwork' => ['M' => ['rxBytes' => ['N' => $networkRxBytes], 'txBytes' => ['N' => $networkTxBytes]]], ':newRxBytes' => ['N' => $networkRxBytes], ':newTxBytes' => ['N' => $networkTxBytes]];
$updateExpression .= ', syrupJobId = if_not_exists(syrupJobId, :syrupJobId)';
$expressionAttributeValues[':syrupJobId'] = ['S' => $jobIdAndRunId['jobId']];
$params = ['TableName' => $this->tableName, 'Key' => ['runId' => ['S' => $jobIdAndRunId['runId']]], 'UpdateExpression' => $updateExpression, 'ConditionExpression' => <<<EXPR
attribute_not_exists(dockerNetwork)
or (dockerNetwork.rxBytes < :newRxBytes or dockerNetwork.txBytes < :newTxBytes)
EXPR
, 'ExpressionAttributeValues' => $expressionAttributeValues, 'ReturnValues' => 'UPDATED_NEW'];
try {
$this->dynamoDbClient->updateItem($params);
} catch (DynamoDbException $e) {
if (strpos($e->getMessage(), 'ConditionalCheckFailedException') === false) {
throw $e;
}
}
}
}
}
protected function processMessage(QueueMessage $message)
{
$msg = null;
if (isset($message->getBody()->Message)) {
$msg = json_decode($message->getBody()->Message);
}
$this->logger->debug("Processing kill message", ['message' => $msg]);
if ($msg != null) {
$jobId = $msg->jobId;
/** @var Job $job */
$job = $this->elasticsearch->getJob($jobId);
if (!is_null($job)) {
if ($job->getProcess()['host'] == gethostname()) {
if ($job->getStatus() == Job::STATUS_WAITING) {
$job->setStatus(Job::STATUS_CANCELLED);
$job->setResult(['message' => 'Job cancelled by user']);
$this->getComponentJobMapper($job->getComponent())->update($job);
$this->logger->info("receive-kill: Job '{$jobId}' cancelled", ['job' => $job->getData()]);
$this->requeue($job->getId());
} else {
if ($job->getStatus() == Job::STATUS_PROCESSING) {
$job->setStatus(Job::STATUS_TERMINATING);
$this->getComponentJobMapper($job->getComponent())->update($job);
$pid = $job->getProcess()['pid'];
// kill child processes
$process = new Process('
function getcpid() {
cpids=`pgrep -P $1|xargs`
for cpid in $cpids;
do
echo "$cpid"
getcpid $cpid
done
}
getcpid ' . $pid);
$process->run();
$processes = $process->getOutput();
if ($processes && count(explode("\n", $processes))) {
foreach (explode("\n", $processes) as $child) {
if ($child != '') {
(new Process("sudo kill -KILL {$child}"))->run();
}
}
}
// kill parent process
posix_kill($pid, SIGKILL);
$this->logger->info("receive-kill: Job '{$jobId}' killed", ['job' => $job->getData()]);
} else {
$this->logger->info("receive-kill: Job is not in waiting or processing state", ['job' => $job->getData()]);
}
}
}
}
} else {
$this->logger->warn("Corrupted message received", ['message' => $message]);
}
$this->queue->deleteMessage($message);
}
/**
* Processes message from SQS Queue
* @param QueueMessage $message
*/
private function processMessage(QueueMessage $message)
{
$body = \Keboola\Utils\objectToArray($message->getBody());
// sync actions has empty runId
if (isset($body['runId'], $body['projectId']) && !empty($body['runId'])) {
$this->initStorageStats($body);
$this->updateStorageStats($body);
}
}
public function deleteMessage(QueueMessage $message)
{
$this->client->deleteMessage(['QueueUrl' => $message->getQueueUrl(), 'ReceiptHandle' => $message->getReceiptHandle()]);
}
protected function requeue(QueueMessage $message)
{
$delaySecs = 30 + 30 * pow(2, $message->getRetryCount());
$delaySecs = $delaySecs > 600 ? 600 : $delaySecs;
return $this->queue->enqueue($message->getBody()->jobId, ['component' => $message->getBody()->component], $delaySecs);
}
