/** * Processes message from SQS Queue * @param QueueMessage $message */ private function processMessage(QueueMessage $message) { $body = \Keboola\Utils\objectToArray($message->getBody()); if (isset($body['container_id'], $body['container_name'], $body['container_stats'])) { $containerName = $body['container_name']; $jobIdAndRunId = $this->getJobIdAndRunIdFromContainerName($containerName); if (!empty($jobIdAndRunId)) { $containerStats = $body['container_stats']; $networkRxBytes = strval(isset($containerStats['network']['rx_bytes']) ? $containerStats['network']['rx_bytes'] : 0); $networkTxBytes = strval(isset($containerStats['network']['tx_bytes']) ? $containerStats['network']['tx_bytes'] : 0); $updateExpression = 'SET dockerNetwork = :dockerNetwork'; $expressionAttributeValues = [':dockerNetwork' => ['M' => ['rxBytes' => ['N' => $networkRxBytes], 'txBytes' => ['N' => $networkTxBytes]]], ':newRxBytes' => ['N' => $networkRxBytes], ':newTxBytes' => ['N' => $networkTxBytes]]; $updateExpression .= ', syrupJobId = if_not_exists(syrupJobId, :syrupJobId)'; $expressionAttributeValues[':syrupJobId'] = ['S' => $jobIdAndRunId['jobId']]; $params = ['TableName' => $this->tableName, 'Key' => ['runId' => ['S' => $jobIdAndRunId['runId']]], 'UpdateExpression' => $updateExpression, 'ConditionExpression' => <<<EXPR attribute_not_exists(dockerNetwork) or (dockerNetwork.rxBytes < :newRxBytes or dockerNetwork.txBytes < :newTxBytes) EXPR , 'ExpressionAttributeValues' => $expressionAttributeValues, 'ReturnValues' => 'UPDATED_NEW']; try { $this->dynamoDbClient->updateItem($params); } catch (DynamoDbException $e) { if (strpos($e->getMessage(), 'ConditionalCheckFailedException') === false) { throw $e; } } } } }
protected function execute(InputInterface $input, OutputInterface $output) { $this->dynamoDbClient = $this->getContainer()->get('queue.dynamodb_client'); $this->tableName = $this->getContainer()->getParameter('container_stats_dynamodb.table_name'); $tableName = $this->tableName; $updateTableJson = <<<JSON { "TableName": "{$tableName}", "GlobalSecondaryIndexUpdates": [ { "Create": { "IndexName": "sapiProjectId-sapiCreatedTime-index", "KeySchema": [ { "AttributeName": "sapiProjectId", "KeyType": "HASH" }, { "AttributeName": "sapiCreatedTime", "KeyType": "RANGE" } ], "Projection": { "ProjectionType": "ALL" }, "ProvisionedThroughput": { "ReadCapacityUnits": 3, "WriteCapacityUnits": 7 } } } ], "AttributeDefinitions": [ { "AttributeName": "sapiCreatedTime", "AttributeType": "S" }, { "AttributeName": "sapiProjectId", "AttributeType": "S" } ] } JSON; $updateTableArgs = \json_decode($updateTableJson, true); try { $this->dynamoDbClient->updateTable($updateTableArgs); echo 'Index has been created' . "\n"; } catch (DynamoDbException $e) { if (strpos($e->getMessage(), 'Attempting to create an index which already exists') !== false) { echo 'Index already exists' . "\n"; } else { throw $e; } } }
/** * Save marshaled data * * @param $id * @param $data * @return bool|\Guzzle\Service\Resource\Model */ public function saveData($id, $data) { if (empty($this->table)) { return false; } $client_data = ['TableName' => $this->table, 'Item' => ['id' => ['S' => $id], 'item_status' => ['S' => 'entered'], 'payload' => ['M' => $this->marshallData($data)], 'date' => ['S' => utf8_encode((string) date('Y-m-d H:i:s'))]]]; $result = $this->client->putItem($client_data); return $result; }
public function saved($model) { $attrs = $model->attributesToArray(); // $this->attributeFilter->filter($attrs); try { $this->dynamoDbClient->putItem(['TableName' => $model->getDynamoDbTableName(), 'Item' => $this->marshaler->marshalItem($attrs)]); } catch (Exception $e) { Log::info($e); } }
public function storeReferralCode($code, $user_id, $timestamp) { $data = ['code' => $code, 'user_id' => $user_id, 'timestamp' => $timestamp]; // TODO error handling $response = $this->client->GetItem(['TableName' => 'tr_referralcodes', 'Key' => ['code' => ['S' => $code]]]); if (!empty($response['Item'])) { throw new \Exception('Code already exists'); } $this->client->PutItem(['TableName' => 'tr_referralcodes', 'Item' => $this->marshaler->marshalItem($data)]); }
/** * {@inheritdoc} */ protected function write(array $record) { $filtered = $this->filterEmptyFields($record['formatted']); if ($this->version === 3) { $formatted = $this->marshaler->marshalItem($filtered); } else { $formatted = $this->client->formatAttributes($filtered); } $this->client->putItem(['TableName' => $this->table, 'Item' => $formatted]); }
protected function seed($attributes = []) { $item = ['id' => ['S' => str_random(36)], 'name' => ['S' => str_random(36)], 'description' => ['S' => str_random(256)], 'count' => ['N' => rand()]]; $item = array_merge($item, $attributes); $this->dynamoDbClient->putItem(['TableName' => $this->testModel->getTable(), 'Item' => $item]); return $item; }
/** * Scans table for specific items we want to update * @return array */ private function getJobsToFetchStatsFor() { $params = ['TableName' => $this->tableName, 'FilterExpression' => <<<EXPR attribute_exists(sapiProjectId) and attribute_not_exists(componentName) and not begins_with(runId, :notBeginsWith) and ( attribute_not_exists(syrupJobId) or (attribute_exists(syrupJobId) and syrupJobId <> :syrupJobIdSyncAction) ) and attribute_not_exists(hasSyrupJob) EXPR , 'Limit' => self::MAX_DOCUMENTS_TO_PROCESS, 'ExpressionAttributeValues' => [':notBeginsWith' => ['S' => 'p'], ':syrupJobIdSyncAction' => ['S' => '0']]]; $ids = []; $marshaler = new Marshaler(); do { if (isset($response) && isset($response['LastEvaluatedKey'])) { $params['ExclusiveStartKey'] = $response['LastEvaluatedKey']; } $response = $this->dynamoDbClient->scan($params); foreach ($response['Items'] as $item) { $ids[] = $marshaler->unmarshalValue($item['runId']); } } while (isset($response['LastEvaluatedKey']) && count($ids) <= 5000); return $ids; }
protected function getAll($columns = [], $limit = -1) { $query = ['TableName' => $this->getTable()]; $op = 'Scan'; if ($limit > -1) { $query['limit'] = $limit; } if (!empty($columns)) { $query['AttributesToGet'] = $columns; } // If the $where is not empty, we run getIterator. if (!empty($this->where)) { // Primary key or index key condition exists, then use Query instead of Scan. // However, Query only supports a few conditions. if ($key = $this->conditionsContainIndexKey()) { $condition = array_get($this->where, "{$key}.ComparisonOperator"); if (ComparisonOperator::isValidQueryDynamoDbOperator($condition)) { $op = 'Query'; $query['IndexName'] = $this->dynamoDbIndexKeys[$key]; $query['KeyConditions'] = $this->where; } } $query['ScanFilter'] = $this->where; } $iterator = $this->client->getIterator($op, $query); $results = []; foreach ($iterator as $item) { $item = $this->unmarshalItem($item); $model = new static($item, $this->dynamoDb); $model->setUnfillableAttributes($item); $results[] = $model; } return new Collection($results); }
/** * List tables via the list_tables call * @param integer $limit * @param string $exclusiveStartTableName * @return Table\TableCollection */ public function listTables($limit = null, $exclusiveStartTableName = null) { if (null !== $this->logger) { $this->log('List tables'); } $parameters = array(); if (null !== $limit) { $parameters['Limit'] = $limit; } if (null !== $exclusiveStartTableName) { $parameters['ExclusiveStartTableName'] = $exclusiveStartTableName; } if (null !== $this->logger) { $this->log('ListTable request paramaters : ' . print_r($parameters, true), Logger::DEBUG); } $response = $this->connector->listTables($parameters); if (null !== $this->logger) { $this->log('ListTable request response : ' . print_r($response, true), Logger::DEBUG); } $tables = new Table\TableCollection(isset($response['LastEvaluatedTableName']) ? $response['LastEvaluatedTableName'] : null); if (!empty($response['TableNames'])) { foreach ($response['TableNames'] as $table) { $tables->add($table); } } return $tables; }
/** * Initialize the dynamodb client. */ public function init() { parent::init(); //For v2 compatibility. //TODO: remove deprecated. $this->_client = DynamoDbClient::factory($this->config); }
/** * Fill the table with data */ protected static function fillTable() { $data = ['key' => 'dev', 'settings' => self::getTestData()]; $marshaler = new Marshaler(); $item = $marshaler->marshalItem($data); self::$dynamodb->putItem(['TableName' => self::TABLE_NAME, 'Item' => $item]); }
/** * @param string $tableName */ protected function createTable($tableName) { try { $this->client->describeTable(['TableName' => $tableName]); } catch (ResourceNotFoundException $e) { $this->client->createTable(['AttributeDefinitions' => [['AttributeName' => 'id', 'AttributeType' => 'S']], 'TableName' => $tableName, 'KeySchema' => [['AttributeName' => 'id', 'KeyType' => 'HASH']], 'ProvisionedThroughput' => ['ReadCapacityUnits' => 1, 'WriteCapacityUnits' => 1]]); } }
/** * Initialize the client. */ public function init() { $this->_client = DynamoDbClient::factory($this->config); if ($this->keyPrefix === null) { $this->keyPrefix = substr(md5(Yii::$app->id), 0, 5); } parent::init(); }
public function __construct($region = false) { if (!$region && !($region = getenv("AWS_DEFAULT_REGION"))) { throw new \Exception("Set 'AWS_DEFAULT_REGION' environment variable!"); } $this->region = $region; $this->sns = SnsClient::factory(['region' => $region]); $this->ddb = DynamoDbClient::factory(['region' => $region]); }
public function testRemoveAKey() { // Arrange. $key = 'bar'; $this->mockDynamoDbClient->expects($this->once())->method('deleteItem')->with(['TableName' => $this->storeKeyTableName, 'Key' => [$this->storeKeyAttribute => ['S' => $key]]]); // Act. $this->storeKeyClient->delete($key); // Assert in arrange. }
/** * @return array */ public function getTables() { $out = []; do { $result = $this->dbConn->listTables(['Limit' => 100, 'ExclusiveStartTableName' => isset($result) ? $result['LastEvaluatedTableName'] : null]); $out = array_merge($out, $result['TableNames']); } while ($result['LastEvaluatedTableName']); return $out; }
/** * Factory method to create a new Amazon DynamoDB Streams client using an array of configuration options. * * See http://docs.aws.amazon.com/aws-sdk-php/v2/guide/configuration.html#client-configuration-options * * @param array|Collection $config Client configuration data * * @return self * @link http://docs.aws.amazon.com/aws-sdk-php/v2/guide/configuration.html#client-configuration-options */ public static function factory($config = array()) { // Configure the custom exponential backoff plugin for DynamoDB throttling $exceptionParser = new JsonQueryExceptionParser(); if (!isset($config[Options::BACKOFF])) { $config[Options::BACKOFF] = new BackoffPlugin(DynamoDbClient::createDynamoDbBackoffStrategy($exceptionParser)); } return ClientBuilder::factory(__NAMESPACE__)->setConfig($config)->setConfigDefaults(array(Options::VERSION => self::LATEST_API_VERSION, Options::SERVICE_DESCRIPTION => __DIR__ . '/Resources/dynamodbstreams-%s.php'))->setExceptionParser($exceptionParser)->build(); }
protected function execute(InputInterface $input, OutputInterface $output) { $this->dynamoDbClient = $this->getContainer()->get('queue.dynamodb_client'); $this->tableName = $this->getContainer()->getParameter('container_stats_dynamodb.table_name'); try { $this->dynamoDbClient->describeTable(['TableName' => $this->tableName]); echo 'Table ' . $this->tableName . ' already exists.' . "\n"; } catch (DynamoDbException $e) { if (strpos($e->getMessage(), 'ResourceNotFoundException') !== false) { echo 'Creating ' . $this->tableName . ' table ...'; $this->dynamoDbClient->createTable($this->getTableConfiguration()); $this->dynamoDbClient->waitUntil('TableExists', ['TableName' => $this->tableName, '@waiter' => ['delay' => 3, 'maxAttempts' => 20]]); echo ' done!' . "\n"; } else { throw $e; } } }
public function deleteTable($tablename) { $args = ['TableName' => $tablename]; $result = $this->db->deleteTable($args); if (isset($result['TableDescription']) && $result['TableDescription']) { return true; } else { return false; } }
/** * Executes an asynchronous command. * * @param CommandInterface $command Command to execute. * * @return AwsDynamoDbResponse */ protected function executeAsyncCommand(CommandInterface $command) { try { $result = $this->client->executeAsync($command); } catch (AwsException $e) { /* Return an error response. */ return new AwsDynamoDbResponse(null, $e); } return new AwsDynamoDbResponse($result, null); }
public function testGetItem() { try { $result = self::$client->getItem(['TableName' => self::$tableName, 'ConsistentRead' => true, 'Key' => ['id' => ['S' => $this->message->get('id')->toString()]]]); } catch (\Exception $e) { $this->fail($e->getMessage()); return; } $this->assertSame($result['Item']['id']['S'], $this->message->get('id')->toString()); $message = $this->marshaler->unmarshal($result['Item']); foreach ($this->message->schema()->getFields() as $field) { $expected = $this->message->get($field->getName()); $actual = $message->get($field->getName()); if ($field->isASet()) { sort($expected); sort($actual); } $this->assertSame(json_encode($expected), json_encode($actual)); } //echo json_encode($message, JSON_PRETTY_PRINT); }
public function __construct() { $bucket = Config::get('storage.bucket', 'default'); $key = Config::get('storage.key'); $secret = Config::get('storage.secret'); $client = DynamoDbClient::factory(array('key' => $key, 'secret' => $secret, 'region' => '<region name>')); $config = new SessionHandlerConfig(array('table_name' => 'sessions')); // Make sure locking strategy has been provided or provide a default $factory = new LockingStrategyFactory(); $strategy = $factory->factory($strategy, $config); // Return an instance of the session handler parent::__construct($client, $strategy, $config); }
public function __construct($connection, $config = array()) { if (!$connection instanceof DynamoDbClient) { if (!is_array($connection)) { throw new \InvalidArgumentException('First argument to OAuth2\\Storage\\Dynamodb must be an instance a configuration array containt key, secret, region'); } if (!array_key_exists("key", $connection) || !array_key_exists("secret", $connection) || !array_key_exists("region", $connection)) { throw new \InvalidArgumentException('First argument to OAuth2\\Storage\\Dynamodb must be an instance a configuration array containt key, secret, region'); } $this->client = DynamoDbClient::factory(array('key' => $connection["key"], 'secret' => $connection["secret"], 'region' => $connection["region"])); } else { $this->client = $connection; } $this->config = array_merge(array('client_table' => 'oauth_clients', 'access_token_table' => 'oauth_access_tokens', 'refresh_token_table' => 'oauth_refresh_tokens', 'code_table' => 'oauth_authorization_codes', 'user_table' => 'oauth_users', 'jwt_table' => 'oauth_jwt', 'scope_table' => 'oauth_scopes', 'public_key_table' => 'oauth_public_keys'), $config); }
public static function get() { if (LocalDBClientBuilder::$client == null) { LocalDBClientBuilder::$client = DynamoDbClient::factory(array('region' => 'us-west-2', 'version' => 'latest', 'endpoint' => 'http://localhost:8000', 'key' => 'myKey', 'secret' => 'mySecret')); // AMO: TODO remove debug plugin //LocalDBClientBuilder::$client->addSubscriber(LogPlugin::getDebugPlugin()); } return LocalDBClientBuilder::$client; /* $sdk = new Aws\Sdk([ 'region' => 'us-west-2', 'version' => 'latest', 'endpoint' => 'http://localhost:8080' ]); $dynamodb = $sdk->createDynamoDb();*/ }
/** * @depends testUsesInstanceProfileCredentialsByDefault */ public function testClientsUseInstanceProfileCredentials(array $creds) { $this->skipIfNotEc2(); list($credentials, $client) = $creds; $dynamo = DynamoDbClient::factory(array('credentials' => $credentials)); // Ensure that the correct credentials object and client are being used $this->assertSame($credentials, $dynamo->getCredentials()); if ($this->useMocks()) { $this->setMockResponse($client, array('metadata/iam_security_credentials', 'metadata/iam_security_credentials_webapp')); $this->setMockResponse($dynamo, 'dynamodb/list_tables_final'); } // Expire the credentials $credentials->setExpiration(0); // List a table, causing a credential refresh and list table request $this->assertInternalType('array', $dynamo->listTables()); }
/** * @depends testIteratesOverScan */ public function testImplementsCustomExponentialBackoffStrategy() { self::log('Getting an item a bunch of times in parallel'); $batch = BatchBuilder::factory()->transferCommands(100)->build(); $s = microtime(true); $total = 300; for ($i = 0; $i < $total; $i++) { $command = $this->client->getCommand('GetItem', array('TableName' => $this->table, 'Key' => $this->client->formatAttributes(array('HashKeyElement' => 'Test', 'RangeKeyElement' => 10)))); $batch->add($command); } $retries = 0; foreach ($batch->flush() as $command) { $retries += $command->getRequest()->getParams()->get(BackoffPlugin::RETRY_PARAM); } $elapsed = microtime(true) - $s; self::log("Got the item {$total} times with {$retries} retries in {$elapsed} seconds"); }
/** * Returns all keys that match the pattern given in $pattern. If an asterisk is used it returns all keys that start with the string that precede the asterisk.<br /> * If an asterisk is not used then it returns all keys containing the $pattern. * @param string $pattern * @return array An array containing all matched keys */ public function keys($pattern) { $astPos = mb_strpos($pattern, '*'); if ($astPos !== false && $astPos > 0) { $comparisonOpearator = 'BEGINS_WITH'; $comparisonValue = mb_substr($pattern, 0, $astPos); } else { $comparisonOpearator = 'CONTAINS'; $comparisonValue = $pattern; } $iterator = $this->client->getIterator('Scan', array('TableName' => $this->tableName, 'AttributesToGet' => array(self::SIMPLE_KEY_NAME), 'ReturnConsumedCapacity' => 'TOTAL', 'ScanFilter' => array(self::SIMPLE_KEY_NAME => array('AttributeValueList' => array(array('S' => $comparisonValue)), 'ComparisonOperator' => $comparisonOpearator)))); $keysArray = array(); foreach ($iterator as $item) { $keysArray[] = $item[self::SIMPLE_KEY_NAME]['S']; } return $keysArray; }
public function __construct($options, $table) { // refer to the Session class to find the session timeout value (if it exists) // in terms of DynamoDB, session_lifetime is the time to mark the inactive // session to be garbage collected // if {@link GarbageCollectSessionCronTask} is running periodically on your // server (via the silverstripe-crontask module), then the inactive session // will get removed from the DynamoDB session table. if (!isset($options['session_lifetime'])) { $timeout = Config::inst()->get('Session', 'timeout'); if ($timeout != null) { $options['session_lifetime'] = $timeout; } } $this->client = DynamoDbClient::factory($options); $this->table = $table; $this->handler = SessionHandler::factory(array('dynamodb_client' => $this->client, 'table_name' => $this->table)); }
/** * Updates the current session id with a newly generated one. * Please refer to {@link http://php.net/session_regenerate_id} for more details. * @param boolean $deleteOldSession Whether to delete the old associated session file or not. * @since 1.1.8 */ public function regenerateID($deleteOldSession = false) { $oldId = session_id(); parent::regenerateID(false); $newId = session_id(); $row = $this->getData($oldId); if (!is_null($row)) { if ($deleteOldSession) { // Delete + Put = Update $this->dynamoDb->deleteItem(array('TableName' => $this->tableName, 'Key' => array('id' => array('S' => (string) $oldId)))); $this->dynamoDb->putItem(array('TableName' => $this->tableName, 'Item' => array($this->idColumn => array('S' => (string) $newId), $this->dataColumn => $row[$this->dataColumn], $this->expireColumn => $row[$this->expireColumn]))); } else { $row[$this->idColumn] = array('S' => (string) $newId); $this->dynamoDb->putItem(array('TableName' => $this->tableName, 'Item' => array($row))); } } else { $this->dynamoDb->putItem(array('TableName' => $this->tableName, 'Item' => array($this->idColumn => array('S' => $newId), $this->expireColumn => array('N' => $this->getExpireTime())))); } }