/**
     * Processes message from SQS Queue
     * @param QueueMessage $message
     */
    private function processMessage(QueueMessage $message)
    {
        $body = \Keboola\Utils\objectToArray($message->getBody());
        if (isset($body['container_id'], $body['container_name'], $body['container_stats'])) {
            $containerName = $body['container_name'];
            $jobIdAndRunId = $this->getJobIdAndRunIdFromContainerName($containerName);
            if (!empty($jobIdAndRunId)) {
                $containerStats = $body['container_stats'];
                $networkRxBytes = strval(isset($containerStats['network']['rx_bytes']) ? $containerStats['network']['rx_bytes'] : 0);
                $networkTxBytes = strval(isset($containerStats['network']['tx_bytes']) ? $containerStats['network']['tx_bytes'] : 0);
                $updateExpression = 'SET dockerNetwork = :dockerNetwork';
                $expressionAttributeValues = [':dockerNetwork' => ['M' => ['rxBytes' => ['N' => $networkRxBytes], 'txBytes' => ['N' => $networkTxBytes]]], ':newRxBytes' => ['N' => $networkRxBytes], ':newTxBytes' => ['N' => $networkTxBytes]];
                $updateExpression .= ', syrupJobId = if_not_exists(syrupJobId, :syrupJobId)';
                $expressionAttributeValues[':syrupJobId'] = ['S' => $jobIdAndRunId['jobId']];
                $params = ['TableName' => $this->tableName, 'Key' => ['runId' => ['S' => $jobIdAndRunId['runId']]], 'UpdateExpression' => $updateExpression, 'ConditionExpression' => <<<EXPR
attribute_not_exists(dockerNetwork)
or (dockerNetwork.rxBytes < :newRxBytes or dockerNetwork.txBytes < :newTxBytes)
EXPR
, 'ExpressionAttributeValues' => $expressionAttributeValues, 'ReturnValues' => 'UPDATED_NEW'];
                try {
                    $this->dynamoDbClient->updateItem($params);
                } catch (DynamoDbException $e) {
                    if (strpos($e->getMessage(), 'ConditionalCheckFailedException') === false) {
                        throw $e;
                    }
                }
            }
        }
    }
    protected function execute(InputInterface $input, OutputInterface $output)
    {
        $this->dynamoDbClient = $this->getContainer()->get('queue.dynamodb_client');
        $this->tableName = $this->getContainer()->getParameter('container_stats_dynamodb.table_name');
        $tableName = $this->tableName;
        $updateTableJson = <<<JSON
{
    "TableName": "{$tableName}",
    "GlobalSecondaryIndexUpdates": [
        {
            "Create": {
                "IndexName": "sapiProjectId-sapiCreatedTime-index",
                "KeySchema": [
                    {
                        "AttributeName": "sapiProjectId",
                        "KeyType": "HASH"
                    },
                    {
                        "AttributeName": "sapiCreatedTime",
                        "KeyType": "RANGE"
                    }
                ],
                "Projection": {
                    "ProjectionType": "ALL"
                },
                "ProvisionedThroughput": {
                    "ReadCapacityUnits": 3,
                    "WriteCapacityUnits": 7
                }

            }
        }
    ],
    "AttributeDefinitions": [
        {
            "AttributeName": "sapiCreatedTime",
            "AttributeType": "S"
        },
        {
            "AttributeName": "sapiProjectId",
            "AttributeType": "S"
        }
    ]
}
JSON;
        $updateTableArgs = \json_decode($updateTableJson, true);
        try {
            $this->dynamoDbClient->updateTable($updateTableArgs);
            echo 'Index has been created' . "\n";
        } catch (DynamoDbException $e) {
            if (strpos($e->getMessage(), 'Attempting to create an index which already exists') !== false) {
                echo 'Index already exists' . "\n";
            } else {
                throw $e;
            }
        }
    }
Exemplo n.º 3
0
 /**
  * Save marshaled data
  *
  * @param $id
  * @param $data
  * @return bool|\Guzzle\Service\Resource\Model
  */
 public function saveData($id, $data)
 {
     if (empty($this->table)) {
         return false;
     }
     $client_data = ['TableName' => $this->table, 'Item' => ['id' => ['S' => $id], 'item_status' => ['S' => 'entered'], 'payload' => ['M' => $this->marshallData($data)], 'date' => ['S' => utf8_encode((string) date('Y-m-d H:i:s'))]]];
     $result = $this->client->putItem($client_data);
     return $result;
 }
 public function saved($model)
 {
     $attrs = $model->attributesToArray();
     // $this->attributeFilter->filter($attrs);
     try {
         $this->dynamoDbClient->putItem(['TableName' => $model->getDynamoDbTableName(), 'Item' => $this->marshaler->marshalItem($attrs)]);
     } catch (Exception $e) {
         Log::info($e);
     }
 }
Exemplo n.º 5
0
 public function storeReferralCode($code, $user_id, $timestamp)
 {
     $data = ['code' => $code, 'user_id' => $user_id, 'timestamp' => $timestamp];
     // TODO error handling
     $response = $this->client->GetItem(['TableName' => 'tr_referralcodes', 'Key' => ['code' => ['S' => $code]]]);
     if (!empty($response['Item'])) {
         throw new \Exception('Code already exists');
     }
     $this->client->PutItem(['TableName' => 'tr_referralcodes', 'Item' => $this->marshaler->marshalItem($data)]);
 }
Exemplo n.º 6
0
 /**
  * {@inheritdoc}
  */
 protected function write(array $record)
 {
     $filtered = $this->filterEmptyFields($record['formatted']);
     if ($this->version === 3) {
         $formatted = $this->marshaler->marshalItem($filtered);
     } else {
         $formatted = $this->client->formatAttributes($filtered);
     }
     $this->client->putItem(['TableName' => $this->table, 'Item' => $formatted]);
 }
 protected function seed($attributes = [])
 {
     $item = ['id' => ['S' => str_random(36)], 'name' => ['S' => str_random(36)], 'description' => ['S' => str_random(256)], 'count' => ['N' => rand()]];
     $item = array_merge($item, $attributes);
     $this->dynamoDbClient->putItem(['TableName' => $this->testModel->getTable(), 'Item' => $item]);
     return $item;
 }
    /**
     * Scans table for specific items we want to update
     * @return array
     */
    private function getJobsToFetchStatsFor()
    {
        $params = ['TableName' => $this->tableName, 'FilterExpression' => <<<EXPR
attribute_exists(sapiProjectId)
and attribute_not_exists(componentName)
and not begins_with(runId, :notBeginsWith)
and (
  attribute_not_exists(syrupJobId)
  or
  (attribute_exists(syrupJobId) and syrupJobId <> :syrupJobIdSyncAction)
)
and attribute_not_exists(hasSyrupJob)
EXPR
, 'Limit' => self::MAX_DOCUMENTS_TO_PROCESS, 'ExpressionAttributeValues' => [':notBeginsWith' => ['S' => 'p'], ':syrupJobIdSyncAction' => ['S' => '0']]];
        $ids = [];
        $marshaler = new Marshaler();
        do {
            if (isset($response) && isset($response['LastEvaluatedKey'])) {
                $params['ExclusiveStartKey'] = $response['LastEvaluatedKey'];
            }
            $response = $this->dynamoDbClient->scan($params);
            foreach ($response['Items'] as $item) {
                $ids[] = $marshaler->unmarshalValue($item['runId']);
            }
        } while (isset($response['LastEvaluatedKey']) && count($ids) <= 5000);
        return $ids;
    }
 protected function getAll($columns = [], $limit = -1)
 {
     $query = ['TableName' => $this->getTable()];
     $op = 'Scan';
     if ($limit > -1) {
         $query['limit'] = $limit;
     }
     if (!empty($columns)) {
         $query['AttributesToGet'] = $columns;
     }
     // If the $where is not empty, we run getIterator.
     if (!empty($this->where)) {
         // Primary key or index key condition exists, then use Query instead of Scan.
         // However, Query only supports a few conditions.
         if ($key = $this->conditionsContainIndexKey()) {
             $condition = array_get($this->where, "{$key}.ComparisonOperator");
             if (ComparisonOperator::isValidQueryDynamoDbOperator($condition)) {
                 $op = 'Query';
                 $query['IndexName'] = $this->dynamoDbIndexKeys[$key];
                 $query['KeyConditions'] = $this->where;
             }
         }
         $query['ScanFilter'] = $this->where;
     }
     $iterator = $this->client->getIterator($op, $query);
     $results = [];
     foreach ($iterator as $item) {
         $item = $this->unmarshalItem($item);
         $model = new static($item, $this->dynamoDb);
         $model->setUnfillableAttributes($item);
         $results[] = $model;
     }
     return new Collection($results);
 }
Exemplo n.º 10
0
 /**
  * List tables via the list_tables call
  * @param integer $limit
  * @param string $exclusiveStartTableName
  * @return Table\TableCollection
  */
 public function listTables($limit = null, $exclusiveStartTableName = null)
 {
     if (null !== $this->logger) {
         $this->log('List tables');
     }
     $parameters = array();
     if (null !== $limit) {
         $parameters['Limit'] = $limit;
     }
     if (null !== $exclusiveStartTableName) {
         $parameters['ExclusiveStartTableName'] = $exclusiveStartTableName;
     }
     if (null !== $this->logger) {
         $this->log('ListTable request paramaters : ' . print_r($parameters, true), Logger::DEBUG);
     }
     $response = $this->connector->listTables($parameters);
     if (null !== $this->logger) {
         $this->log('ListTable request response : ' . print_r($response, true), Logger::DEBUG);
     }
     $tables = new Table\TableCollection(isset($response['LastEvaluatedTableName']) ? $response['LastEvaluatedTableName'] : null);
     if (!empty($response['TableNames'])) {
         foreach ($response['TableNames'] as $table) {
             $tables->add($table);
         }
     }
     return $tables;
 }
Exemplo n.º 11
0
 /**
  * Initialize the dynamodb client.
  */
 public function init()
 {
     parent::init();
     //For v2 compatibility.
     //TODO: remove deprecated.
     $this->_client = DynamoDbClient::factory($this->config);
 }
Exemplo n.º 12
0
 /**
  * Fill the table with data
  */
 protected static function fillTable()
 {
     $data = ['key' => 'dev', 'settings' => self::getTestData()];
     $marshaler = new Marshaler();
     $item = $marshaler->marshalItem($data);
     self::$dynamodb->putItem(['TableName' => self::TABLE_NAME, 'Item' => $item]);
 }
Exemplo n.º 13
0
 /**
  * @param string $tableName
  */
 protected function createTable($tableName)
 {
     try {
         $this->client->describeTable(['TableName' => $tableName]);
     } catch (ResourceNotFoundException $e) {
         $this->client->createTable(['AttributeDefinitions' => [['AttributeName' => 'id', 'AttributeType' => 'S']], 'TableName' => $tableName, 'KeySchema' => [['AttributeName' => 'id', 'KeyType' => 'HASH']], 'ProvisionedThroughput' => ['ReadCapacityUnits' => 1, 'WriteCapacityUnits' => 1]]);
     }
 }
Exemplo n.º 14
0
 /**
  * Initialize the client.
  */
 public function init()
 {
     $this->_client = DynamoDbClient::factory($this->config);
     if ($this->keyPrefix === null) {
         $this->keyPrefix = substr(md5(Yii::$app->id), 0, 5);
     }
     parent::init();
 }
Exemplo n.º 15
0
 public function __construct($region = false)
 {
     if (!$region && !($region = getenv("AWS_DEFAULT_REGION"))) {
         throw new \Exception("Set 'AWS_DEFAULT_REGION' environment variable!");
     }
     $this->region = $region;
     $this->sns = SnsClient::factory(['region' => $region]);
     $this->ddb = DynamoDbClient::factory(['region' => $region]);
 }
Exemplo n.º 16
0
 public function testRemoveAKey()
 {
     // Arrange.
     $key = 'bar';
     $this->mockDynamoDbClient->expects($this->once())->method('deleteItem')->with(['TableName' => $this->storeKeyTableName, 'Key' => [$this->storeKeyAttribute => ['S' => $key]]]);
     // Act.
     $this->storeKeyClient->delete($key);
     // Assert in arrange.
 }
Exemplo n.º 17
0
 /**
  * @return array
  */
 public function getTables()
 {
     $out = [];
     do {
         $result = $this->dbConn->listTables(['Limit' => 100, 'ExclusiveStartTableName' => isset($result) ? $result['LastEvaluatedTableName'] : null]);
         $out = array_merge($out, $result['TableNames']);
     } while ($result['LastEvaluatedTableName']);
     return $out;
 }
 /**
  * Factory method to create a new Amazon DynamoDB Streams client using an array of configuration options.
  *
  * See http://docs.aws.amazon.com/aws-sdk-php/v2/guide/configuration.html#client-configuration-options
  *
  * @param array|Collection $config Client configuration data
  *
  * @return self
  * @link http://docs.aws.amazon.com/aws-sdk-php/v2/guide/configuration.html#client-configuration-options
  */
 public static function factory($config = array())
 {
     // Configure the custom exponential backoff plugin for DynamoDB throttling
     $exceptionParser = new JsonQueryExceptionParser();
     if (!isset($config[Options::BACKOFF])) {
         $config[Options::BACKOFF] = new BackoffPlugin(DynamoDbClient::createDynamoDbBackoffStrategy($exceptionParser));
     }
     return ClientBuilder::factory(__NAMESPACE__)->setConfig($config)->setConfigDefaults(array(Options::VERSION => self::LATEST_API_VERSION, Options::SERVICE_DESCRIPTION => __DIR__ . '/Resources/dynamodbstreams-%s.php'))->setExceptionParser($exceptionParser)->build();
 }
Exemplo n.º 19
0
 protected function execute(InputInterface $input, OutputInterface $output)
 {
     $this->dynamoDbClient = $this->getContainer()->get('queue.dynamodb_client');
     $this->tableName = $this->getContainer()->getParameter('container_stats_dynamodb.table_name');
     try {
         $this->dynamoDbClient->describeTable(['TableName' => $this->tableName]);
         echo 'Table ' . $this->tableName . ' already exists.' . "\n";
     } catch (DynamoDbException $e) {
         if (strpos($e->getMessage(), 'ResourceNotFoundException') !== false) {
             echo 'Creating ' . $this->tableName . ' table ...';
             $this->dynamoDbClient->createTable($this->getTableConfiguration());
             $this->dynamoDbClient->waitUntil('TableExists', ['TableName' => $this->tableName, '@waiter' => ['delay' => 3, 'maxAttempts' => 20]]);
             echo ' done!' . "\n";
         } else {
             throw $e;
         }
     }
 }
Exemplo n.º 20
0
 public function deleteTable($tablename)
 {
     $args = ['TableName' => $tablename];
     $result = $this->db->deleteTable($args);
     if (isset($result['TableDescription']) && $result['TableDescription']) {
         return true;
     } else {
         return false;
     }
 }
 /**
  * Executes an asynchronous command.
  *
  * @param CommandInterface $command Command to execute.
  *
  * @return AwsDynamoDbResponse
  */
 protected function executeAsyncCommand(CommandInterface $command)
 {
     try {
         $result = $this->client->executeAsync($command);
     } catch (AwsException $e) {
         /* Return an error response. */
         return new AwsDynamoDbResponse(null, $e);
     }
     return new AwsDynamoDbResponse($result, null);
 }
Exemplo n.º 22
0
 public function testGetItem()
 {
     try {
         $result = self::$client->getItem(['TableName' => self::$tableName, 'ConsistentRead' => true, 'Key' => ['id' => ['S' => $this->message->get('id')->toString()]]]);
     } catch (\Exception $e) {
         $this->fail($e->getMessage());
         return;
     }
     $this->assertSame($result['Item']['id']['S'], $this->message->get('id')->toString());
     $message = $this->marshaler->unmarshal($result['Item']);
     foreach ($this->message->schema()->getFields() as $field) {
         $expected = $this->message->get($field->getName());
         $actual = $message->get($field->getName());
         if ($field->isASet()) {
             sort($expected);
             sort($actual);
         }
         $this->assertSame(json_encode($expected), json_encode($actual));
     }
     //echo json_encode($message, JSON_PRETTY_PRINT);
 }
Exemplo n.º 23
0
 public function __construct()
 {
     $bucket = Config::get('storage.bucket', 'default');
     $key = Config::get('storage.key');
     $secret = Config::get('storage.secret');
     $client = DynamoDbClient::factory(array('key' => $key, 'secret' => $secret, 'region' => '<region name>'));
     $config = new SessionHandlerConfig(array('table_name' => 'sessions'));
     // Make sure locking strategy has been provided or provide a default
     $factory = new LockingStrategyFactory();
     $strategy = $factory->factory($strategy, $config);
     // Return an instance of the session handler
     parent::__construct($client, $strategy, $config);
 }
Exemplo n.º 24
0
 public function __construct($connection, $config = array())
 {
     if (!$connection instanceof DynamoDbClient) {
         if (!is_array($connection)) {
             throw new \InvalidArgumentException('First argument to OAuth2\\Storage\\Dynamodb must be an instance a configuration array containt key, secret, region');
         }
         if (!array_key_exists("key", $connection) || !array_key_exists("secret", $connection) || !array_key_exists("region", $connection)) {
             throw new \InvalidArgumentException('First argument to OAuth2\\Storage\\Dynamodb must be an instance a configuration array containt key, secret, region');
         }
         $this->client = DynamoDbClient::factory(array('key' => $connection["key"], 'secret' => $connection["secret"], 'region' => $connection["region"]));
     } else {
         $this->client = $connection;
     }
     $this->config = array_merge(array('client_table' => 'oauth_clients', 'access_token_table' => 'oauth_access_tokens', 'refresh_token_table' => 'oauth_refresh_tokens', 'code_table' => 'oauth_authorization_codes', 'user_table' => 'oauth_users', 'jwt_table' => 'oauth_jwt', 'scope_table' => 'oauth_scopes', 'public_key_table' => 'oauth_public_keys'), $config);
 }
Exemplo n.º 25
0
 public static function get()
 {
     if (LocalDBClientBuilder::$client == null) {
         LocalDBClientBuilder::$client = DynamoDbClient::factory(array('region' => 'us-west-2', 'version' => 'latest', 'endpoint' => 'http://localhost:8000', 'key' => 'myKey', 'secret' => 'mySecret'));
         // AMO: TODO remove debug plugin
         //LocalDBClientBuilder::$client->addSubscriber(LogPlugin::getDebugPlugin());
     }
     return LocalDBClientBuilder::$client;
     /* $sdk = new Aws\Sdk([
                 'region'   => 'us-west-2',
                 'version'  => 'latest',
                 'endpoint' => 'http://localhost:8080'
             ]);
     
             $dynamodb = $sdk->createDynamoDb();*/
 }
 /**
  * @depends testUsesInstanceProfileCredentialsByDefault
  */
 public function testClientsUseInstanceProfileCredentials(array $creds)
 {
     $this->skipIfNotEc2();
     list($credentials, $client) = $creds;
     $dynamo = DynamoDbClient::factory(array('credentials' => $credentials));
     // Ensure that the correct credentials object and client are being used
     $this->assertSame($credentials, $dynamo->getCredentials());
     if ($this->useMocks()) {
         $this->setMockResponse($client, array('metadata/iam_security_credentials', 'metadata/iam_security_credentials_webapp'));
         $this->setMockResponse($dynamo, 'dynamodb/list_tables_final');
     }
     // Expire the credentials
     $credentials->setExpiration(0);
     // List a table, causing a credential refresh and list table request
     $this->assertInternalType('array', $dynamo->listTables());
 }
Exemplo n.º 27
0
 /**
  * @depends testIteratesOverScan
  */
 public function testImplementsCustomExponentialBackoffStrategy()
 {
     self::log('Getting an item a bunch of times in parallel');
     $batch = BatchBuilder::factory()->transferCommands(100)->build();
     $s = microtime(true);
     $total = 300;
     for ($i = 0; $i < $total; $i++) {
         $command = $this->client->getCommand('GetItem', array('TableName' => $this->table, 'Key' => $this->client->formatAttributes(array('HashKeyElement' => 'Test', 'RangeKeyElement' => 10))));
         $batch->add($command);
     }
     $retries = 0;
     foreach ($batch->flush() as $command) {
         $retries += $command->getRequest()->getParams()->get(BackoffPlugin::RETRY_PARAM);
     }
     $elapsed = microtime(true) - $s;
     self::log("Got the item {$total} times with {$retries} retries in {$elapsed} seconds");
 }
Exemplo n.º 28
0
 /**
  * Returns all keys that match the pattern given in $pattern. If an asterisk is used it returns all keys that start with the string that precede the asterisk.<br />
  * If an asterisk is not used then it returns all keys containing the $pattern.
  * @param string $pattern
  * @return array An array containing all matched keys
  */
 public function keys($pattern)
 {
     $astPos = mb_strpos($pattern, '*');
     if ($astPos !== false && $astPos > 0) {
         $comparisonOpearator = 'BEGINS_WITH';
         $comparisonValue = mb_substr($pattern, 0, $astPos);
     } else {
         $comparisonOpearator = 'CONTAINS';
         $comparisonValue = $pattern;
     }
     $iterator = $this->client->getIterator('Scan', array('TableName' => $this->tableName, 'AttributesToGet' => array(self::SIMPLE_KEY_NAME), 'ReturnConsumedCapacity' => 'TOTAL', 'ScanFilter' => array(self::SIMPLE_KEY_NAME => array('AttributeValueList' => array(array('S' => $comparisonValue)), 'ComparisonOperator' => $comparisonOpearator))));
     $keysArray = array();
     foreach ($iterator as $item) {
         $keysArray[] = $item[self::SIMPLE_KEY_NAME]['S'];
     }
     return $keysArray;
 }
 public function __construct($options, $table)
 {
     // refer to the Session class to find the session timeout value (if it exists)
     // in terms of DynamoDB, session_lifetime is the time to mark the inactive
     // session to be garbage collected
     // if {@link GarbageCollectSessionCronTask} is running periodically on your
     // server (via the silverstripe-crontask module), then the inactive session
     // will get removed from the DynamoDB session table.
     if (!isset($options['session_lifetime'])) {
         $timeout = Config::inst()->get('Session', 'timeout');
         if ($timeout != null) {
             $options['session_lifetime'] = $timeout;
         }
     }
     $this->client = DynamoDbClient::factory($options);
     $this->table = $table;
     $this->handler = SessionHandler::factory(array('dynamodb_client' => $this->client, 'table_name' => $this->table));
 }
 /**
  * Updates the current session id with a newly generated one.
  * Please refer to {@link http://php.net/session_regenerate_id} for more details.
  * @param boolean $deleteOldSession Whether to delete the old associated session file or not.
  * @since 1.1.8
  */
 public function regenerateID($deleteOldSession = false)
 {
     $oldId = session_id();
     parent::regenerateID(false);
     $newId = session_id();
     $row = $this->getData($oldId);
     if (!is_null($row)) {
         if ($deleteOldSession) {
             // Delete + Put = Update
             $this->dynamoDb->deleteItem(array('TableName' => $this->tableName, 'Key' => array('id' => array('S' => (string) $oldId))));
             $this->dynamoDb->putItem(array('TableName' => $this->tableName, 'Item' => array($this->idColumn => array('S' => (string) $newId), $this->dataColumn => $row[$this->dataColumn], $this->expireColumn => $row[$this->expireColumn])));
         } else {
             $row[$this->idColumn] = array('S' => (string) $newId);
             $this->dynamoDb->putItem(array('TableName' => $this->tableName, 'Item' => array($row)));
         }
     } else {
         $this->dynamoDb->putItem(array('TableName' => $this->tableName, 'Item' => array($this->idColumn => array('S' => $newId), $this->expireColumn => array('N' => $this->getExpireTime()))));
     }
 }