/**
* Processes message from SQS Queue
* @param QueueMessage $message
*/
private function processMessage(QueueMessage $message)
{
$body = \Keboola\Utils\objectToArray($message->getBody());
if (isset($body['container_id'], $body['container_name'], $body['container_stats'])) {
$containerName = $body['container_name'];
$jobIdAndRunId = $this->getJobIdAndRunIdFromContainerName($containerName);
if (!empty($jobIdAndRunId)) {
$containerStats = $body['container_stats'];
$networkRxBytes = strval(isset($containerStats['network']['rx_bytes']) ? $containerStats['network']['rx_bytes'] : 0);
$networkTxBytes = strval(isset($containerStats['network']['tx_bytes']) ? $containerStats['network']['tx_bytes'] : 0);
$updateExpression = 'SET dockerNetwork = :dockerNetwork';
$expressionAttributeValues = [':dockerNetwork' => ['M' => ['rxBytes' => ['N' => $networkRxBytes], 'txBytes' => ['N' => $networkTxBytes]]], ':newRxBytes' => ['N' => $networkRxBytes], ':newTxBytes' => ['N' => $networkTxBytes]];
$updateExpression .= ', syrupJobId = if_not_exists(syrupJobId, :syrupJobId)';
$expressionAttributeValues[':syrupJobId'] = ['S' => $jobIdAndRunId['jobId']];
$params = ['TableName' => $this->tableName, 'Key' => ['runId' => ['S' => $jobIdAndRunId['runId']]], 'UpdateExpression' => $updateExpression, 'ConditionExpression' => <<<EXPR
attribute_not_exists(dockerNetwork)
or (dockerNetwork.rxBytes < :newRxBytes or dockerNetwork.txBytes < :newTxBytes)
EXPR
, 'ExpressionAttributeValues' => $expressionAttributeValues, 'ReturnValues' => 'UPDATED_NEW'];
try {
$this->dynamoDbClient->updateItem($params);
} catch (DynamoDbException $e) {
if (strpos($e->getMessage(), 'ConditionalCheckFailedException') === false) {
throw $e;
}
}
}
}
}
/**
* {@inheritDoc}
*/
public function update($storageName, $key, array $data)
{
$this->prepareData($key, $data);
unset($data['id']);
foreach ($data as $k => $v) {
$data[$k] = ['Value' => $this->client->formatValue($v)];
}
$result = $this->client->updateItem(['TableName' => $storageName, 'Key' => ['id' => ['S' => $key]], 'AttributeUpdates' => $data]);
}
/**
* {@inheritDoc}
*/
public function update($storageName, $key, array $data)
{
$this->prepareData($key, $data);
unset($data['id']);
foreach ($data as $k => $v) {
$data[$k] = array("Value" => $this->client->formatValue($v));
}
$result = $this->client->updateItem(array('TableName' => $storageName, 'Key' => array("id" => array('S' => $key)), "AttributeUpdates" => $data));
}
/**
* updateItem
*
* @param array $values associative array
*
* $values = array(
* 'name' => 'John',
* 'age' => 30,
* );
*
* @param array $options
*
* $options = array(
* 'ReturnValues' => 'string',
* 'ReturnConsumedCapacity' => 'string',
* 'ReturnItemCollectionMetrics' => 'string',
* 'Action' => array('age' => 'ADD'),
* 'Exists' => array('age' => true),
* );
*
* @param array $expected
*
* @return \Guzzle\Service\Resource\Model
*
* @link http://docs.aws.amazon.com/aws-sdk-php/latest/class-Aws.DynamoDb.DynamoDbClient.html#_updateItem
*/
public function updateItem(array $values, array $options = array(), array $expected = array())
{
$conditions = $this->_getKeyConditions();
$action = array();
// Update Action (ADD|PUT|DELETE)
if (isset($options['Action'])) {
$action = $options['Action'];
}
$attributes = $this->_formatAttributeUpdates($values, $action);
foreach ($conditions as $key => $value) {
if (isset($attributes[$key])) {
unset($attributes[$key]);
}
}
$args = array('TableName' => $this->_table_name, 'Key' => $conditions, 'AttributeUpdates' => $attributes, 'ReturnValues' => 'ALL_NEW', 'ReturnConsumedCapacity' => 'TOTAL', 'ReturnItemCollectionMetrics' => 'SIZE');
// Set Expected if exists
if ($expected || isset($options['Exists'])) {
$exists = isset($options['Exists']) ? $options['Exists'] : array();
$args['Expected'] = $this->_formatAttributeExpected($expected, $exists);
}
// Merge $options to $args
$option_names = array('ReturnValues', 'ReturnConsumedCapacity', 'ReturnItemCollectionMetrics');
foreach ($option_names as $option_name) {
if (isset($options[$option_name])) {
$args[$option_name] = $options[$option_name];
}
}
$item = self::$_client->updateItem($args);
self::_logQuery('updateItem', $args, $item);
return $item;
}
/**
* Update an item via the update_item call
* @param string $table The item table
* @param mixed $hash The primary hash key
* @param mixed|null $range The primary range key
* @param AttributeUpdate $update
* @param Context\Update|null $context The call context
* @return array|null
* @throws Exception\AttributesException
*/
public function update($table, $hash, $range = null, AttributeUpdate $update, Context\Update $context = null)
{
if (null !== $this->logger) {
$this->log('Update on table' . $table);
}
// Primary key
$hash = new Attribute($hash);
$key = array('HashKeyElement' => $hash->getForDynamoDB());
// Range key
if (null !== $range) {
$range = new Attribute($range);
$key['RangeKeyElement'] = $range->getForDynamoDB();
}
$attributes = array();
foreach ($update as $name => $attribute) {
/** @var $attribute Attribute */
$attributes[$name] = $attribute->getForDynamoDB();
}
$parameters = array('TableName' => $table, 'Key' => $key, 'AttributeUpdates' => $attributes);
if (null !== $context) {
$parameters += $context->getForDynamoDB();
}
if (null !== $this->logger) {
$this->log('Update request paramaters : ' . print_r($parameters, true), Logger::DEBUG);
}
$response = $this->connector->updateItem($parameters);
if (null !== $this->logger) {
$this->log('Update request response : ' . print_r($response, true), Logger::DEBUG);
}
// Update write counter
$this->addConsumedWriteUnits($table, floatval($response['ConsumedCapacityUnits']));
return $this->populateAttributes($response);
}
/**
* Marks specified document as processed, since it doesn't have job
* @param $runId
*/
private function markDocumentAsProcessed($runId)
{
$updateExpression = <<<EXPR
SET hasSyrupJob = :hasSyrupJob
EXPR;
$expressionAttributeValues = [':hasSyrupJob' => ['N' => strval(0)]];
$params = ['TableName' => $this->tableName, 'Key' => ['runId' => ['S' => strval($runId)]], 'UpdateExpression' => $updateExpression, 'ExpressionAttributeValues' => $expressionAttributeValues, 'ReturnValues' => 'UPDATED_NEW'];
$this->dynamoDbClient->updateItem($params);
}
/**
* Edits an existing item's attributes, or inserts a new item if it does not already exist.
* You can put, delete, or add attribute values.
* You can also perform a conditional update (insert a new attribute name-value pair if it doesn't exist, or replace an existing name-value pair if it has certain expected attribute values).
*
* @param string $tableName The name of the table containing the item to update.
* @param array $key Associative array of <AttributeName> keys mapping to (associative-array) values.
* @param array $attributeUpdates The names of attributes to be modified, the action to perform on each, and the new value for each.
* @param array $expected A map of attribute/condition pairs. This is the conditional block for the UpdateItem operation. All the conditions must be met for the operation to succeed.
* @param string $conditionnalOperator Operator between each condition of $expected argument.
* @param string $returnValues Use ReturnValues if you want to get the item attributes as they appeared either before or after they were updated.
* @param string $returnConsumedCapacity Sets consumed capacity return mode.
* @param string $returnItemCollectionMetrics If set to SIZE, statistics about item collections, if any, that were modified during the operation are returned in the response.
*
* @return Guzzle\Service\Resource\Model
*
* @see http://docs.aws.amazon.com/aws-sdk-php/latest/class-Aws.DynamoDb.DynamoDbClient.html#_updateItem
*/
public function updateItem($tableName, array $key, array $attributeUpdates = null, array $expected = null, $conditionnalOperator = self::COND_AND, $returnValues = self::RETURN_NONE, $returnConsumedCapacity = self::CAPACITY_NONE, $returnItemCollectionMetrics = self::METRICS_NONE)
{
$args = ['TableName' => $tableName, 'Key' => $key, 'ConditionnalOperator' => $conditionnalOperator, 'ReturnValues' => $returnValues, 'ReturnConsumedCapacity' => $returnConsumedCapacity, 'ReturnItemCollectionMetrics' => $returnItemCollectionMetrics];
if ($attributeUpdates !== null) {
$args['AttributeUpdates'] = $attributeUpdates;
}
if ($expected !== null) {
$args['Expected'] = $expected;
}
return $this->client->updateItem($args);
}
/**
* Sets field in the hash stored at key to value. If key does not exist, a new key holding a hash is created. <br />
* If field already exists in the hash, it is overwritten. <br />
* @param string $key The key at which to set a hash
* @param string $field The field to set a value to
* @param mixed $value The value to be set
* @return integer Returns 1 if field is a new field in the hash and value was set, 0 if field already exists in the hash and the value was updated
*/
public function hSet($key, $field, $value)
{
if (!($key !== '') || !($field !== '')) {
return false;
}
try {
$result = $this->client->updateItem(array('TableName' => $this->tableName, 'Key' => array(self::SIMPLE_KEY_NAME => array('S' => $key)), 'AttributeUpdates' => array(self::HPREFIX . $field => array('Action' => 'PUT', 'Value' => array('B' => $value)))));
return true;
} catch (\Exception $ex) {
return false;
}
}
/**
* Updates/increments storage stats
* @param $body
*/
private function updateStorageStats($body)
{
$storageMetrics = $this->prepareStorageStats($body);
$updateExpression = <<<EXPR
SET sapiStorage.inBytes = sapiStorage.inBytes + :inBytes,
sapiStorage.inBytesUncompressed = sapiStorage.inBytesUncompressed + :inBytesUncompressed,
sapiStorage.outBytes = sapiStorage.outBytes + :outBytes,
sapiStorage.outBytesUncompressed = sapiStorage.outBytesUncompressed + :outBytesUncompressed,
sapiStorage.inBytesComputed = sapiStorage.inBytesComputed + :inBytesComputed,
sapiStorage.outBytesComputed = sapiStorage.outBytesComputed + :outBytesComputed
EXPR;
$expressionAttributeValues = [':inBytes' => ['N' => $storageMetrics['inBytes']], ':inBytesUncompressed' => ['N' => $storageMetrics['inBytesUncompressed']], ':outBytes' => ['N' => $storageMetrics['outBytes']], ':outBytesUncompressed' => ['N' => $storageMetrics['outBytesUncompressed']], ':inBytesComputed' => ['N' => $storageMetrics['inBytesUncompressed'] != 0 ? $storageMetrics['inBytesUncompressed'] : $storageMetrics['inBytes']], ':outBytesComputed' => ['N' => $storageMetrics['outBytesUncompressed'] != 0 ? $storageMetrics['outBytesUncompressed'] : $storageMetrics['outBytes']]];
$params = ['TableName' => $this->tableName, 'Key' => ['runId' => ['S' => $body['runId']]], 'UpdateExpression' => $updateExpression, 'ExpressionAttributeValues' => $expressionAttributeValues, 'ReturnValues' => 'UPDATED_NEW'];
try {
$this->dynamoDbClient->updateItem($params);
} catch (DynamoDbException $e) {
if (strpos($e->getMessage(), 'ConditionalCheckFailedException') === false) {
throw $e;
}
}
}
