/**
* Processes message from SQS Queue
* @param QueueMessage $message
*/
private function processMessage(QueueMessage $message)
{
$body = \Keboola\Utils\objectToArray($message->getBody());
if (isset($body['container_id'], $body['container_name'], $body['container_stats'])) {
$containerName = $body['container_name'];
$jobIdAndRunId = $this->getJobIdAndRunIdFromContainerName($containerName);
if (!empty($jobIdAndRunId)) {
$containerStats = $body['container_stats'];
$networkRxBytes = strval(isset($containerStats['network']['rx_bytes']) ? $containerStats['network']['rx_bytes'] : 0);
$networkTxBytes = strval(isset($containerStats['network']['tx_bytes']) ? $containerStats['network']['tx_bytes'] : 0);
$updateExpression = 'SET dockerNetwork = :dockerNetwork';
$expressionAttributeValues = [':dockerNetwork' => ['M' => ['rxBytes' => ['N' => $networkRxBytes], 'txBytes' => ['N' => $networkTxBytes]]], ':newRxBytes' => ['N' => $networkRxBytes], ':newTxBytes' => ['N' => $networkTxBytes]];
$updateExpression .= ', syrupJobId = if_not_exists(syrupJobId, :syrupJobId)';
$expressionAttributeValues[':syrupJobId'] = ['S' => $jobIdAndRunId['jobId']];
$params = ['TableName' => $this->tableName, 'Key' => ['runId' => ['S' => $jobIdAndRunId['runId']]], 'UpdateExpression' => $updateExpression, 'ConditionExpression' => <<<EXPR
attribute_not_exists(dockerNetwork)
or (dockerNetwork.rxBytes < :newRxBytes or dockerNetwork.txBytes < :newTxBytes)
EXPR
, 'ExpressionAttributeValues' => $expressionAttributeValues, 'ReturnValues' => 'UPDATED_NEW'];
try {
$this->dynamoDbClient->updateItem($params);
} catch (DynamoDbException $e) {
if (strpos($e->getMessage(), 'ConditionalCheckFailedException') === false) {
throw $e;
}
}
}
}
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$this->dynamoDbClient = $this->getContainer()->get('queue.dynamodb_client');
$this->tableName = $this->getContainer()->getParameter('container_stats_dynamodb.table_name');
$tableName = $this->tableName;
$updateTableJson = <<<JSON
{
"TableName": "{$tableName}",
"GlobalSecondaryIndexUpdates": [
{
"Create": {
"IndexName": "sapiProjectId-sapiCreatedTime-index",
"KeySchema": [
{
"AttributeName": "sapiProjectId",
"KeyType": "HASH"
},
{
"AttributeName": "sapiCreatedTime",
"KeyType": "RANGE"
}
],
"Projection": {
"ProjectionType": "ALL"
},
"ProvisionedThroughput": {
"ReadCapacityUnits": 3,
"WriteCapacityUnits": 7
}
}
}
],
"AttributeDefinitions": [
{
"AttributeName": "sapiCreatedTime",
"AttributeType": "S"
},
{
"AttributeName": "sapiProjectId",
"AttributeType": "S"
}
]
}
JSON;
$updateTableArgs = \json_decode($updateTableJson, true);
try {
$this->dynamoDbClient->updateTable($updateTableArgs);
echo 'Index has been created' . "\n";
} catch (DynamoDbException $e) {
if (strpos($e->getMessage(), 'Attempting to create an index which already exists') !== false) {
echo 'Index already exists' . "\n";
} else {
throw $e;
}
}
}
/**
* Save marshaled data
*
* @param $id
* @param $data
* @return bool|\Guzzle\Service\Resource\Model
*/
public function saveData($id, $data)
{
if (empty($this->table)) {
return false;
}
$client_data = ['TableName' => $this->table, 'Item' => ['id' => ['S' => $id], 'item_status' => ['S' => 'entered'], 'payload' => ['M' => $this->marshallData($data)], 'date' => ['S' => utf8_encode((string) date('Y-m-d H:i:s'))]]];
$result = $this->client->putItem($client_data);
return $result;
}
public function saved($model)
{
$attrs = $model->attributesToArray();
// $this->attributeFilter->filter($attrs);
try {
$this->dynamoDbClient->putItem(['TableName' => $model->getDynamoDbTableName(), 'Item' => $this->marshaler->marshalItem($attrs)]);
} catch (Exception $e) {
Log::info($e);
}
}
public function storeReferralCode($code, $user_id, $timestamp)
{
$data = ['code' => $code, 'user_id' => $user_id, 'timestamp' => $timestamp];
// TODO error handling
$response = $this->client->GetItem(['TableName' => 'tr_referralcodes', 'Key' => ['code' => ['S' => $code]]]);
if (!empty($response['Item'])) {
throw new \Exception('Code already exists');
}
$this->client->PutItem(['TableName' => 'tr_referralcodes', 'Item' => $this->marshaler->marshalItem($data)]);
}
/**
* {@inheritdoc}
*/
protected function write(array $record)
{
$filtered = $this->filterEmptyFields($record['formatted']);
if ($this->version === 3) {
$formatted = $this->marshaler->marshalItem($filtered);
} else {
$formatted = $this->client->formatAttributes($filtered);
}
$this->client->putItem(['TableName' => $this->table, 'Item' => $formatted]);
}
protected function seed($attributes = [])
{
$item = ['id' => ['S' => str_random(36)], 'name' => ['S' => str_random(36)], 'description' => ['S' => str_random(256)], 'count' => ['N' => rand()]];
$item = array_merge($item, $attributes);
$this->dynamoDbClient->putItem(['TableName' => $this->testModel->getTable(), 'Item' => $item]);
return $item;
}
/**
* Scans table for specific items we want to update
* @return array
*/
private function getJobsToFetchStatsFor()
{
$params = ['TableName' => $this->tableName, 'FilterExpression' => <<<EXPR
attribute_exists(sapiProjectId)
and attribute_not_exists(componentName)
and not begins_with(runId, :notBeginsWith)
and (
attribute_not_exists(syrupJobId)
or
(attribute_exists(syrupJobId) and syrupJobId <> :syrupJobIdSyncAction)
)
and attribute_not_exists(hasSyrupJob)
EXPR
, 'Limit' => self::MAX_DOCUMENTS_TO_PROCESS, 'ExpressionAttributeValues' => [':notBeginsWith' => ['S' => 'p'], ':syrupJobIdSyncAction' => ['S' => '0']]];
$ids = [];
$marshaler = new Marshaler();
do {
if (isset($response) && isset($response['LastEvaluatedKey'])) {
$params['ExclusiveStartKey'] = $response['LastEvaluatedKey'];
}
$response = $this->dynamoDbClient->scan($params);
foreach ($response['Items'] as $item) {
$ids[] = $marshaler->unmarshalValue($item['runId']);
}
} while (isset($response['LastEvaluatedKey']) && count($ids) <= 5000);
return $ids;
}
protected function getAll($columns = [], $limit = -1)
{
$query = ['TableName' => $this->getTable()];
$op = 'Scan';
if ($limit > -1) {
$query['limit'] = $limit;
}
if (!empty($columns)) {
$query['AttributesToGet'] = $columns;
}
// If the $where is not empty, we run getIterator.
if (!empty($this->where)) {
// Primary key or index key condition exists, then use Query instead of Scan.
// However, Query only supports a few conditions.
if ($key = $this->conditionsContainIndexKey()) {
$condition = array_get($this->where, "{$key}.ComparisonOperator");
if (ComparisonOperator::isValidQueryDynamoDbOperator($condition)) {
$op = 'Query';
$query['IndexName'] = $this->dynamoDbIndexKeys[$key];
$query['KeyConditions'] = $this->where;
}
}
$query['ScanFilter'] = $this->where;
}
$iterator = $this->client->getIterator($op, $query);
$results = [];
foreach ($iterator as $item) {
$item = $this->unmarshalItem($item);
$model = new static($item, $this->dynamoDb);
$model->setUnfillableAttributes($item);
$results[] = $model;
}
return new Collection($results);
}
/**
* List tables via the list_tables call
* @param integer $limit
* @param string $exclusiveStartTableName
* @return Table\TableCollection
*/
public function listTables($limit = null, $exclusiveStartTableName = null)
{
if (null !== $this->logger) {
$this->log('List tables');
}
$parameters = array();
if (null !== $limit) {
$parameters['Limit'] = $limit;
}
if (null !== $exclusiveStartTableName) {
$parameters['ExclusiveStartTableName'] = $exclusiveStartTableName;
}
if (null !== $this->logger) {
$this->log('ListTable request paramaters : ' . print_r($parameters, true), Logger::DEBUG);
}
$response = $this->connector->listTables($parameters);
if (null !== $this->logger) {
$this->log('ListTable request response : ' . print_r($response, true), Logger::DEBUG);
}
$tables = new Table\TableCollection(isset($response['LastEvaluatedTableName']) ? $response['LastEvaluatedTableName'] : null);
if (!empty($response['TableNames'])) {
foreach ($response['TableNames'] as $table) {
$tables->add($table);
}
}
return $tables;
}
/**
* Initialize the dynamodb client.
*/
public function init()
{
parent::init();
//For v2 compatibility.
//TODO: remove deprecated.
$this->_client = DynamoDbClient::factory($this->config);
}
/**
* Fill the table with data
*/
protected static function fillTable()
{
$data = ['key' => 'dev', 'settings' => self::getTestData()];
$marshaler = new Marshaler();
$item = $marshaler->marshalItem($data);
self::$dynamodb->putItem(['TableName' => self::TABLE_NAME, 'Item' => $item]);
}
/**
* @param string $tableName
*/
protected function createTable($tableName)
{
try {
$this->client->describeTable(['TableName' => $tableName]);
} catch (ResourceNotFoundException $e) {
$this->client->createTable(['AttributeDefinitions' => [['AttributeName' => 'id', 'AttributeType' => 'S']], 'TableName' => $tableName, 'KeySchema' => [['AttributeName' => 'id', 'KeyType' => 'HASH']], 'ProvisionedThroughput' => ['ReadCapacityUnits' => 1, 'WriteCapacityUnits' => 1]]);
}
}
/**
* Initialize the client.
*/
public function init()
{
$this->_client = DynamoDbClient::factory($this->config);
if ($this->keyPrefix === null) {
$this->keyPrefix = substr(md5(Yii::$app->id), 0, 5);
}
parent::init();
}
public function __construct($region = false)
{
if (!$region && !($region = getenv("AWS_DEFAULT_REGION"))) {
throw new \Exception("Set 'AWS_DEFAULT_REGION' environment variable!");
}
$this->region = $region;
$this->sns = SnsClient::factory(['region' => $region]);
$this->ddb = DynamoDbClient::factory(['region' => $region]);
}
public function testRemoveAKey()
{
// Arrange.
$key = 'bar';
$this->mockDynamoDbClient->expects($this->once())->method('deleteItem')->with(['TableName' => $this->storeKeyTableName, 'Key' => [$this->storeKeyAttribute => ['S' => $key]]]);
// Act.
$this->storeKeyClient->delete($key);
// Assert in arrange.
}
/**
* @return array
*/
public function getTables()
{
$out = [];
do {
$result = $this->dbConn->listTables(['Limit' => 100, 'ExclusiveStartTableName' => isset($result) ? $result['LastEvaluatedTableName'] : null]);
$out = array_merge($out, $result['TableNames']);
} while ($result['LastEvaluatedTableName']);
return $out;
}
/**
* Factory method to create a new Amazon DynamoDB Streams client using an array of configuration options.
*
* See http://docs.aws.amazon.com/aws-sdk-php/v2/guide/configuration.html#client-configuration-options
*
* @param array|Collection $config Client configuration data
*
* @return self
* @link http://docs.aws.amazon.com/aws-sdk-php/v2/guide/configuration.html#client-configuration-options
*/
public static function factory($config = array())
{
// Configure the custom exponential backoff plugin for DynamoDB throttling
$exceptionParser = new JsonQueryExceptionParser();
if (!isset($config[Options::BACKOFF])) {
$config[Options::BACKOFF] = new BackoffPlugin(DynamoDbClient::createDynamoDbBackoffStrategy($exceptionParser));
}
return ClientBuilder::factory(__NAMESPACE__)->setConfig($config)->setConfigDefaults(array(Options::VERSION => self::LATEST_API_VERSION, Options::SERVICE_DESCRIPTION => __DIR__ . '/Resources/dynamodbstreams-%s.php'))->setExceptionParser($exceptionParser)->build();
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$this->dynamoDbClient = $this->getContainer()->get('queue.dynamodb_client');
$this->tableName = $this->getContainer()->getParameter('container_stats_dynamodb.table_name');
try {
$this->dynamoDbClient->describeTable(['TableName' => $this->tableName]);
echo 'Table ' . $this->tableName . ' already exists.' . "\n";
} catch (DynamoDbException $e) {
if (strpos($e->getMessage(), 'ResourceNotFoundException') !== false) {
echo 'Creating ' . $this->tableName . ' table ...';
$this->dynamoDbClient->createTable($this->getTableConfiguration());
$this->dynamoDbClient->waitUntil('TableExists', ['TableName' => $this->tableName, '@waiter' => ['delay' => 3, 'maxAttempts' => 20]]);
echo ' done!' . "\n";
} else {
throw $e;
}
}
}
public function deleteTable($tablename)
{
$args = ['TableName' => $tablename];
$result = $this->db->deleteTable($args);
if (isset($result['TableDescription']) && $result['TableDescription']) {
return true;
} else {
return false;
}
}
/**
* Executes an asynchronous command.
*
* @param CommandInterface $command Command to execute.
*
* @return AwsDynamoDbResponse
*/
protected function executeAsyncCommand(CommandInterface $command)
{
try {
$result = $this->client->executeAsync($command);
} catch (AwsException $e) {
/* Return an error response. */
return new AwsDynamoDbResponse(null, $e);
}
return new AwsDynamoDbResponse($result, null);
}
public function testGetItem()
{
try {
$result = self::$client->getItem(['TableName' => self::$tableName, 'ConsistentRead' => true, 'Key' => ['id' => ['S' => $this->message->get('id')->toString()]]]);
} catch (\Exception $e) {
$this->fail($e->getMessage());
return;
}
$this->assertSame($result['Item']['id']['S'], $this->message->get('id')->toString());
$message = $this->marshaler->unmarshal($result['Item']);
foreach ($this->message->schema()->getFields() as $field) {
$expected = $this->message->get($field->getName());
$actual = $message->get($field->getName());
if ($field->isASet()) {
sort($expected);
sort($actual);
}
$this->assertSame(json_encode($expected), json_encode($actual));
}
//echo json_encode($message, JSON_PRETTY_PRINT);
}
public function __construct()
{
$bucket = Config::get('storage.bucket', 'default');
$key = Config::get('storage.key');
$secret = Config::get('storage.secret');
$client = DynamoDbClient::factory(array('key' => $key, 'secret' => $secret, 'region' => '<region name>'));
$config = new SessionHandlerConfig(array('table_name' => 'sessions'));
// Make sure locking strategy has been provided or provide a default
$factory = new LockingStrategyFactory();
$strategy = $factory->factory($strategy, $config);
// Return an instance of the session handler
parent::__construct($client, $strategy, $config);
}
public function __construct($connection, $config = array())
{
if (!$connection instanceof DynamoDbClient) {
if (!is_array($connection)) {
throw new \InvalidArgumentException('First argument to OAuth2\\Storage\\Dynamodb must be an instance a configuration array containt key, secret, region');
}
if (!array_key_exists("key", $connection) || !array_key_exists("secret", $connection) || !array_key_exists("region", $connection)) {
throw new \InvalidArgumentException('First argument to OAuth2\\Storage\\Dynamodb must be an instance a configuration array containt key, secret, region');
}
$this->client = DynamoDbClient::factory(array('key' => $connection["key"], 'secret' => $connection["secret"], 'region' => $connection["region"]));
} else {
$this->client = $connection;
}
$this->config = array_merge(array('client_table' => 'oauth_clients', 'access_token_table' => 'oauth_access_tokens', 'refresh_token_table' => 'oauth_refresh_tokens', 'code_table' => 'oauth_authorization_codes', 'user_table' => 'oauth_users', 'jwt_table' => 'oauth_jwt', 'scope_table' => 'oauth_scopes', 'public_key_table' => 'oauth_public_keys'), $config);
}
public static function get()
{
if (LocalDBClientBuilder::$client == null) {
LocalDBClientBuilder::$client = DynamoDbClient::factory(array('region' => 'us-west-2', 'version' => 'latest', 'endpoint' => 'http://localhost:8000', 'key' => 'myKey', 'secret' => 'mySecret'));
// AMO: TODO remove debug plugin
//LocalDBClientBuilder::$client->addSubscriber(LogPlugin::getDebugPlugin());
}
return LocalDBClientBuilder::$client;
/* $sdk = new Aws\Sdk([
'region' => 'us-west-2',
'version' => 'latest',
'endpoint' => 'http://localhost:8080'
]);
$dynamodb = $sdk->createDynamoDb();*/
}
/**
* @depends testUsesInstanceProfileCredentialsByDefault
*/
public function testClientsUseInstanceProfileCredentials(array $creds)
{
$this->skipIfNotEc2();
list($credentials, $client) = $creds;
$dynamo = DynamoDbClient::factory(array('credentials' => $credentials));
// Ensure that the correct credentials object and client are being used
$this->assertSame($credentials, $dynamo->getCredentials());
if ($this->useMocks()) {
$this->setMockResponse($client, array('metadata/iam_security_credentials', 'metadata/iam_security_credentials_webapp'));
$this->setMockResponse($dynamo, 'dynamodb/list_tables_final');
}
// Expire the credentials
$credentials->setExpiration(0);
// List a table, causing a credential refresh and list table request
$this->assertInternalType('array', $dynamo->listTables());
}
/**
* @depends testIteratesOverScan
*/
public function testImplementsCustomExponentialBackoffStrategy()
{
self::log('Getting an item a bunch of times in parallel');
$batch = BatchBuilder::factory()->transferCommands(100)->build();
$s = microtime(true);
$total = 300;
for ($i = 0; $i < $total; $i++) {
$command = $this->client->getCommand('GetItem', array('TableName' => $this->table, 'Key' => $this->client->formatAttributes(array('HashKeyElement' => 'Test', 'RangeKeyElement' => 10))));
$batch->add($command);
}
$retries = 0;
foreach ($batch->flush() as $command) {
$retries += $command->getRequest()->getParams()->get(BackoffPlugin::RETRY_PARAM);
}
$elapsed = microtime(true) - $s;
self::log("Got the item {$total} times with {$retries} retries in {$elapsed} seconds");
}
/**
* Returns all keys that match the pattern given in $pattern. If an asterisk is used it returns all keys that start with the string that precede the asterisk.<br />
* If an asterisk is not used then it returns all keys containing the $pattern.
* @param string $pattern
* @return array An array containing all matched keys
*/
public function keys($pattern)
{
$astPos = mb_strpos($pattern, '*');
if ($astPos !== false && $astPos > 0) {
$comparisonOpearator = 'BEGINS_WITH';
$comparisonValue = mb_substr($pattern, 0, $astPos);
} else {
$comparisonOpearator = 'CONTAINS';
$comparisonValue = $pattern;
}
$iterator = $this->client->getIterator('Scan', array('TableName' => $this->tableName, 'AttributesToGet' => array(self::SIMPLE_KEY_NAME), 'ReturnConsumedCapacity' => 'TOTAL', 'ScanFilter' => array(self::SIMPLE_KEY_NAME => array('AttributeValueList' => array(array('S' => $comparisonValue)), 'ComparisonOperator' => $comparisonOpearator))));
$keysArray = array();
foreach ($iterator as $item) {
$keysArray[] = $item[self::SIMPLE_KEY_NAME]['S'];
}
return $keysArray;
}
public function __construct($options, $table)
{
// refer to the Session class to find the session timeout value (if it exists)
// in terms of DynamoDB, session_lifetime is the time to mark the inactive
// session to be garbage collected
// if {@link GarbageCollectSessionCronTask} is running periodically on your
// server (via the silverstripe-crontask module), then the inactive session
// will get removed from the DynamoDB session table.
if (!isset($options['session_lifetime'])) {
$timeout = Config::inst()->get('Session', 'timeout');
if ($timeout != null) {
$options['session_lifetime'] = $timeout;
}
}
$this->client = DynamoDbClient::factory($options);
$this->table = $table;
$this->handler = SessionHandler::factory(array('dynamodb_client' => $this->client, 'table_name' => $this->table));
}
/**
* Updates the current session id with a newly generated one.
* Please refer to {@link http://php.net/session_regenerate_id} for more details.
* @param boolean $deleteOldSession Whether to delete the old associated session file or not.
* @since 1.1.8
*/
public function regenerateID($deleteOldSession = false)
{
$oldId = session_id();
parent::regenerateID(false);
$newId = session_id();
$row = $this->getData($oldId);
if (!is_null($row)) {
if ($deleteOldSession) {
// Delete + Put = Update
$this->dynamoDb->deleteItem(array('TableName' => $this->tableName, 'Key' => array('id' => array('S' => (string) $oldId))));
$this->dynamoDb->putItem(array('TableName' => $this->tableName, 'Item' => array($this->idColumn => array('S' => (string) $newId), $this->dataColumn => $row[$this->dataColumn], $this->expireColumn => $row[$this->expireColumn])));
} else {
$row[$this->idColumn] = array('S' => (string) $newId);
$this->dynamoDb->putItem(array('TableName' => $this->tableName, 'Item' => array($row)));
}
} else {
$this->dynamoDb->putItem(array('TableName' => $this->tableName, 'Item' => array($this->idColumn => array('S' => $newId), $this->expireColumn => array('N' => $this->getExpireTime()))));
}
}