/**
* @expectedException \Aws\Sqs\Exception\SqsException
* @expectedExceptionMessage Body MD5 mismatch for
*/
public function testValidatesSuccessfulMd5OfBody()
{
$mock = new MockPlugin(array(Response::fromMessage("HTTP/1.1 200 OK\r\nContent-Type: application/xml\r\n\r\n" . "<ReceiveMessageResponse>\n <ReceiveMessageResult>\n <Message>\n <MD5OfBody>fooo</MD5OfBody>\n <Body>This is a test message</Body>\n </Message>\n </ReceiveMessageResult>\n </ReceiveMessageResponse>")));
$sqs = SqsClient::factory(array('key' => 'abc', 'secret' => '123', 'region' => 'us-east-1'));
$sqs->addSubscriber($mock);
$sqs->receiveMessage(array('QueueUrl' => 'http://foo.com'));
}
/**
* @param string $name
* @throws OperationException
* @return array
*/
protected function readQueueAttributes($name)
{
try {
// grab url before in order to read attributes
$queueUrl = $this->client->getQueueUrl(['QueueName' => $name]);
$attributes = $this->client->getQueueAttributes(['QueueUrl' => $queueUrl->get('QueueUrl'), 'AttributeNames' => ['All']]);
return $attributes->get('Attributes');
} catch (\Exception $e) {
throw new OperationException(sprintf('Cannot read attributes for queue "%s":%s', $name, $e->getMessage()), $e->getCode(), $e);
}
}
public function testReturnMessage()
{
$expectedArg = ["QueueUrl" => $this->queueUrl, "ReceiptHandle" => $this->receiptId, "VisibilityTimeout" => 0];
$this->sqsClient->shouldReceive("changeMessageVisibility")->with($expectedArg)->once();
$queue = new Queue($this->sqsClient, $this->messageFactory);
$queue->returnMessage($this->queueMessage);
}
public function testCreateQueue()
{
$queueName = 'test_queue';
$db = DriverManager::getConnection(['driver' => 'pdo_mysql', 'host' => SYRUP_DATABASE_HOST, 'dbname' => SYRUP_DATABASE_NAME, 'user' => SYRUP_DATABASE_USER, 'password' => SYRUP_DATABASE_PASSWORD, 'port' => SYRUP_DATABASE_PORT]);
$db->query("DELETE FROM queues WHERE id='{$queueName}'")->execute();
$command = $this->application->find('syrup:queue:create');
$commandTester = new CommandTester($command);
$commandTester->execute(['name' => $queueName, '--register' => null, '--no-watch' => null]);
$this->assertEquals(0, $commandTester->getStatusCode());
$factory = new QueueFactory($db, ['db_table' => 'queues'], SYRUP_APP_NAME);
$dbQueue = $factory->get($queueName);
$queueUrlArr = explode('/', $dbQueue->getUrl());
$this->assertEquals($queueName, array_pop($queueUrlArr));
$sqsClient = new SqsClient(['version' => '2012-11-05', 'region' => 'us-east-1']);
$sqsClient->deleteQueue(['QueueUrl' => $dbQueue->getUrl()]);
}
/**
* Connect to the queueing server. (AWS, Iron.io and Beanstalkd)
* @param array $config
* @return
*/
public function openConnection($config)
{
$this->queue = \Aws\Sqs\SqsClient::factory(['credentials' => new \Aws\Common\Credentials\Credentials($config['key'], $config['secret']), 'region' => $config['region']]);
if (!$this->queue) {
throw new TestRuntime('connection failed or timed-out.');
}
}
/**
*
* @return number
*/
public function is_message($queue = '')
{
$result = $this->sqs->getQueueAttributes(['QueueUrl' => $queue, 'AttributeNames' => ['ApproximateNumberOfMessages', 'ApproximateNumberOfMessagesNotVisible', 'ApproximateNumberOfMessagesDelayed']]);
$queuecount = 0;
foreach ($result->get("Attributes") as $key => $val) {
$queuecount += $val;
}
return $queuecount;
}
/**
* Process a single message and push it to the server URL
*
* @link http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/using-features-managing-env-tiers.html
* @param array $message
* @param string $uri
* @param string $queueName
* @param string $queueUrl
* @param OutputInterface $output
* @return void
*/
private function processMessage(array $message, string $uri, string $queueName, string $queueUrl, OutputInterface $output)
{
$response = $this->httpClient->post($uri, ['headers' => ['User-Agent' => 'aws-sqsd/2.0', 'Content-Type' => 'application/json', 'X-Aws-Sqsd-Msgid' => $message['MessageId'], 'X-Aws-Sqsd-Queue' => $queueName, 'X-Aws-Sqsd-First-Received-At' => $message['Attributes']['ApproximateFirstReceiveTimestamp'], 'X-Aws-Sqsd-Receive-Count' => $message['Attributes']['ApproximateReceiveCount'], 'X-Aws-Sqsd-Sender-Id' => $message['Attributes']['SenderId']], 'json' => json_decode($message['Body'], true)]);
if ($response->getStatusCode() === 200) {
$this->sqsClient->deleteMessage(['QueueUrl' => $queueUrl, 'ReceiptHandle' => $message['ReceiptHandle']]);
$output->writeln(sprintf('<info>Message "%s" has been processed and deleted from the queue "%s".</info>', $message['MessageId'], $queueName));
} else {
$output->writeln(sprintf('<error>Message "%s" could not be processed and back-end returned error %s. Reason: %s</error>', $message['MessageId'], $response->getStatusCode(), $response->getBody()));
}
}
/**
* {@inheritDoc}
*/
public function delete(MessageInterface $message)
{
if (null === $message->getAttribute('ReceiptHandle')) {
throw new QueueException('Message does not contain ReceiptHandle');
}
$result = $this->client->deleteMessage(array('ReceiptHandle' => $message->getAttribute('ReceiptHandle'), 'QueueUrl' => $this->queueUrl));
}
/**
* Send message to sqs and trigger lambda
*
* @param string $message to be send via sqs
*
*/
public function publish($message)
{
// http://docs.aws.amazon.com/aws-sdk-php/v2/api/class-Aws.Sqs.SqsClient.html#_sendMessage
$this->sqs->sendMessage(['QueueUrl' => $this->sqsQueueUrl, 'MessageBody' => $message]);
// http://docs.aws.amazon.com/aws-sdk-php/v2/api/class-Aws.Sns.SnsClient.html#_publish
$this->sns->publish(['TopicArn' => $this->snsTopicArn, 'Message' => 'SQS Message send', 'Subject' => 'SQS Message send']);
}
/**
* Pop the next job off of the queue.
*
* @param string $queue
* @return \Illuminate\Contracts\Queue\Job|null
*/
public function pop($queue = null)
{
$queue = $this->getQueue($queue);
$response = $this->sqs->receiveMessage(['QueueUrl' => $queue, 'AttributeNames' => ['ApproximateReceiveCount']]);
if (count($response['Messages']) > 0) {
return new SqsJob($this->container, $this->sqs, $queue, $response['Messages'][0]);
}
}
/**
* {@inheritdoc}
*/
public function dequeue(SerializerInterface $serializer)
{
$message = $this->client->receiveMessage(['QueueUrl' => $this->url])->get('Messages');
if (count($message)) {
return new SqsJob($this->client, $this->url, $message[0]['ReceiptHandle'], $serializer->unserialize($message[0]['Body']));
}
return null;
}
/**
* Pop the next job off of the queue.
*
* @param string $queue
* @return \Illuminate\Queue\Jobs\Job|null
*/
public function pop($queue = null)
{
$queue = $this->getQueue($queue);
$response = $this->sqs->receiveMessage(array('QueueUrl' => $queue));
if (count($response['Messages']) > 0) {
return new SqsJob($this->container, $this->sqs, $queue, $response['Messages'][0]);
}
}
public function create($name, $region = 'us-east-1', $key = null, $secret = null)
{
$data = ['region' => $region, 'version' => '2012-11-05', 'retries' => 40];
if ($key != null && $secret != null) {
$data['credentials'] = ['key' => $key, 'secret' => $secret];
}
$sqsClient = new SqsClient($data);
$sqsQueue = $sqsClient->createQueue(['QueueName' => $name]);
return $sqsQueue;
}
public function testCreate()
{
$db = DriverManager::getConnection(['driver' => 'pdo_mysql', 'host' => SYRUP_DATABASE_HOST, 'dbname' => SYRUP_DATABASE_NAME, 'user' => SYRUP_DATABASE_USER, 'password' => SYRUP_DATABASE_PASSWORD, 'port' => SYRUP_DATABASE_PORT]);
$factory = new QueueFactory($db, ['db_table' => 'queues'], SYRUP_APP_NAME);
$sqsQueue = $factory->create('test');
$queueUrlArr = explode('/', $sqsQueue->get('QueueUrl'));
$this->assertEquals('test', array_pop($queueUrlArr));
// delete the queue from AWS
$sqsClient = new SqsClient(['version' => '2012-11-05', 'region' => 'us-east-1']);
$sqsClient->deleteQueue(['QueueUrl' => $sqsQueue->get('QueueUrl')]);
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$noWatch = $input->getOption('no-watch');
$queueId = $this->getQueueId();
$snsConfig = $this->getContainer()->getParameter('sns');
$region = isset($snsConfig['region']) ? $snsConfig['region'] : 'us-east-1';
/** @var QueueFactory $queueFactory */
$queueFactory = $this->getContainer()->get('syrup.queue_factory');
$sqs = $queueFactory->create($queueId, $region, $snsConfig['key'], $snsConfig['secret']);
/** @var Connection $conn */
$conn = $this->getContainer()->get('doctrine.dbal.syrup_connection');
$stmt = $conn->query("SELECT * FROM queues WHERE id='{$queueId}'");
$res = $stmt->fetchAll();
if (empty($res)) {
$conn->insert('queues', ['id' => $queueId, 'access_key' => $snsConfig['key'], 'secret_key' => $snsConfig['secret'], 'region' => $region, 'url' => $sqs->get('QueueUrl')]);
}
$sqsClient = new SqsClient(['region' => $region, 'version' => '2012-11-05', 'credentials' => ['key' => $snsConfig['key'], 'secret' => $snsConfig['secret']]]);
$sqsArn = $sqsClient->getQueueArn($sqs->get('QueueUrl'));
// subscribe SQS to SNS
$snsClient = new SnsClient(['region' => $region, 'version' => '2010-03-31', 'credentials' => ['key' => $snsConfig['key'], 'secret' => $snsConfig['secret']]]);
$snsClient->subscribe(['TopicArn' => $snsConfig['topic_arn'], 'Protocol' => 'sqs', 'Endpoint' => $sqsArn]);
// add policy to SQS to allow SNS sending messages to it
$sqsPolicy = '{
"Version": "2008-10-17",
"Id": "' . $sqsArn . '/SQSDefaultPolicy",
"Statement": [
{
"Sid": "sqs-sns",
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": "SQS:SendMessage",
"Resource": "' . $sqsArn . '",
"Condition": {
"ArnEquals": {
"aws:SourceArn": "' . $snsConfig['topic_arn'] . '"
}
}
}
]
}';
$sqsClient->setQueueAttributes(['QueueUrl' => $sqs->get('QueueUrl'), 'Attributes' => ['Policy' => $sqsPolicy]]);
$output->writeln("SQS created and registered to SNS");
// Add Cloudwatch alarm
if (!$noWatch) {
$cwClient = new CloudWatchClient(['region' => $region, 'version' => '2010-08-01', 'credentials' => ['key' => $snsConfig['key'], 'secret' => $snsConfig['secret']]]);
$cwClient->putMetricAlarm(['AlarmName' => sprintf('Syrup %s queue is full', $queueId), 'ActionsEnabled' => true, 'AlarmActions' => [$snsConfig['alarm_topic_arn']], 'MetricName' => 'ApproximateNumberOfMessagesVisible', 'Namespace' => 'AWS/SQS', 'Statistic' => 'Average', 'Dimensions' => [['Name' => 'QueueName', 'Value' => $queueId]], 'Period' => 300, 'EvaluationPeriods' => 1, 'Threshold' => 5, 'ComparisonOperator' => 'GreaterThanOrEqualToThreshold']);
$output->writeln("Cloudwatch alarm created");
}
}
public function testGetQueueArn()
{
$url = 'https://sqs.us-east-1.amazonaws.com/057737625318/php-integ-sqs-queue-1359765974';
$arn = 'arn:aws:sqs:us-east-1:057737625318:php-integ-sqs-queue-1359765974';
$sqs = SqsClient::factory(array('region' => 'us-east-1'));
$this->assertEquals($arn, $sqs->getQueueArn($url));
}
function __construct()
{
$this->CI =& get_instance();
$this->client = SqsClient::factory(array('key' => $this->CI->config->item('sqs_access_key_id'), 'secret' => $this->CI->config->item('sqs_secret_key'), 'region' => $this->CI->config->item('aws_region')));
$this->uuid = $this->CI->config->item('uuid');
$this->queue = $this->CI->config->item('sqs_queue');
}
/**
* @inheritdoc
*
* @throws QueueAccessException
*/
public function listQueues($prefix = '')
{
$listQueues = [];
try {
if (empty($prefix)) {
$results = $this->sqsClient->listQueues();
} else {
$results = $this->sqsClient->listQueues(['QueueNamePrefix' => $prefix]);
}
} catch (SqsException $e) {
throw new QueueAccessException('Cannot list queues', 0, $e);
}
$results = $results->get('QueueUrls');
foreach ($results as $result) {
$result = explode('/', $result);
$result = array_pop($result);
$priorities = $this->priorityHandler->getAll();
foreach ($priorities as $priority) {
if (!empty($priority)) {
$result = str_replace(static::PRIORITY_SEPARATOR . $priority->getName(), '', $result);
}
}
$listQueues[] = $result;
}
$listQueues = array_unique($listQueues);
return $listQueues;
}
/**
* {@inheritdoc}
*/
public function wait(Closure $callback)
{
while (true) {
$result = $this->client->receiveMessage(array('QueueUrl' => $this->queueUrl, 'WaitTimeSeconds' => 20));
$messages = $result['Messages'];
if ($messages) {
foreach ($messages as $message) {
$hermesMessage = $this->serializer->unserialize($message['Body']);
$callback($hermesMessage);
$this->client->deleteMessage(array('QueueUrl' => $this->queueUrl, 'ReceiptHandle' => $message['ReceiptHandle']));
}
}
if ($this->sleepInterval) {
sleep($this->sleepInterval);
}
}
}
/**
* Get the Aws Client instance
*
* @return Aws\Sqs\SqsClient
*/
public function getConnection()
{
//we have a stored connection
if (!$this->_sqsClient) {
$_config = $this->_getConfiguration();
$this->_sqsClient = SqsClient::factory(array('key' => $_config['key'], 'secret' => $_config['secret'], 'region' => $_config['region']));
}
return $this->_sqsClient;
}
public function __construct($awsConfiguration)
{
if (!class_exists('\\Aws\\Sqs\\SqsClient')) {
throw new \LogicException("Can't find AWS SDK >= 2.0.0");
}
$this->sqs = SqsClient::factory($awsConfiguration);
}
/**
* @return $this
*/
public function connect()
{
$this->disconnect();
$key = $this->getConfigurationValue('key');
$secret = $this->getConfigurationValue('secret');
$region = $this->getConfigurationValue('region');
$this->instance = SqsClient::factory(['key' => $key, 'secret' => $secret, 'region' => $region]);
return $this;
}
/**
* Delete the job from the queue.
*
* @param Job $job
* @return boolean whether the operation succeed.
*/
public function delete(Job $job)
{
if (!empty($job->header['ReceiptHandle'])) {
$receiptHandle = $job->header['ReceiptHandle'];
$response = $this->_client->deleteMessage(['QueueUrl' => $this->url, 'ReceiptHandle' => $receiptHandle]);
return $response !== null;
} else {
return false;
}
}
/**
* @see http://docs.aws.amazon.com/aws-sdk-php/v3/api/api-sqs-2012-11-05.html#receivemessage
* Will throw an exception if $amount is > 10.000
*
* @param int $amount
* @param int $timeout seconds
* @return nothing
*/
public function consume($amount, $timeout = 0)
{
$limit = $amount > 0 ? $amount : $this->requestBatchSize;
if ($timeout > 0) {
$startTime = time();
$remaining = $timeout;
}
$receiveParams = array('QueueUrl' => $this->queueUrl, 'MaxNumberOfMessages' => $limit, 'AttributeNames' => array('All'), 'MessageAttributeNames' => array('All'));
while (true) {
$reqTime = microtime(true);
if ($timeout > 0) {
// according to the spec, this is maximum wait time. If messages are available sooner, they get delivered immediately
$receiveParams['WaitTimeSeconds'] = $remaining;
}
$result = $this->client->receiveMessage($receiveParams);
$messages = $result->get('Messages');
if (is_array($messages)) {
foreach ($messages as $message) {
// How we implement routing keys with SQS: since it is not supported natively, we check if the route
// matches after having downloaded the message. If it does not match, we just skip processing it.
// Since we will not call deleteMessage, SQS will requeue the message in a short time.
// This is far from optimal, but it might be better than nothing
if (!$this->matchRoutingKey($message)) {
continue;
}
// removing the message from the queue is manual with SQS
$this->client->deleteMessage(array('QueueUrl' => $this->queueUrl, 'ReceiptHandle' => $message['ReceiptHandle']));
$data = $message['Body'];
unset($message['Body']);
$contentType = isset($message['MessageAttributes'][$this->contentTypeAttribute]['StringValue']) ? $message['MessageAttributes'][$this->contentTypeAttribute]['StringValue'] : '';
if ($contentType != '') {
$this->callback->receive(new Message($data, $message, $contentType, $this->queueName));
} else {
if ($this->logger) {
$this->logger->warning('The SQS Consumer received a message with no content-type attribute. Assuming default');
}
$this->callback->receive(new Message($data, $message, null, $this->queueName));
}
}
}
$this->maybeStopConsumer();
if ($amount > 0) {
return;
}
if ($timeout > 0 && ($remaining = $startTime + $timeout - time()) <= 0) {
return;
}
/// @todo use a parameter to decide the polling interval
// observe MAX 5 requests per sec per queue: sleep for 0.2 secs in between requests
$passedMs = (microtime(true) - $reqTime) * 1000000;
if ($passedMs < 200000) {
usleep(200000 - $passedMs);
}
}
}
public function testDeleteFromS3()
{
$args = ['ReceiptHandle' => $modifiedReceiptHandle = json_encode(['s3_bucket_name' => 'MyBucket', 's3_key' => 'MyKey', 'original_receipt_handle' => 'MyReceiptHandle'])];
$this->sqsClient->method('__call')->with('deleteMessage')->willReturnCallback(function ($name, array $args) use($modifiedReceiptHandle) {
$params = array_key_exists(0, $args) ? $args[0] : [];
\PHPUnit_Framework_Assert::assertNotEquals($modifiedReceiptHandle, $params['ReceiptHandle']);
return new Result();
});
$actual = $this->sut->deleteMessage($args);
self::assertInstanceOf(Result::class, $actual);
}
/**
* @param array $config
* @return SqsClient
* @throws QueueReaderException
*/
public function createServiceFromConfig(array $config)
{
$hasConfiguration = isset($config['queue_client']['url']) && isset($config['queue_client']['params']) && isset($config['queue_client']['options']);
if (!$hasConfiguration) {
throw new QueueReaderException("Unable to find config for sqs client");
}
$queueUrl = $config['queue_client']['url'];
$awsSqsParams = $config['queue_client']['params'];
$options = $config['queue_client']['options'];
return new SqsClient(AwsSqsClient::factory($awsSqsParams), $queueUrl, $options);
}
/**
* Returns queue url
* @param string $queueName The name of the queue
* @return string The queue url
*/
public function getQueueUrl($queueName)
{
if (array_key_exists($queueName, $this->queueUrls)) {
return $this->queueUrls[$queueName];
}
$result = $this->client->getQueueUrl(array('QueueName' => $queueName));
if ($result && ($queueUrl = $result['QueueUrl'])) {
return $this->queueUrls[$queueName] = $queueUrl;
}
throw new \InvalidArgumentException("Queue url for queue {$queueName} not found");
}
public function testMetadataIsPopped()
{
$this->sqsClient->expects($this->once())->method('receiveMessage')->will($this->returnValue(array('Messages' => array(array('Body' => json_encode(array('content' => serialize('aa'), 'metadata' => array('__name__' => 'MyClass', 'foo' => 'bar'))), 'MessageId' => 'id_123', 'ReceiptHandle' => 'receipt_123', 'MD5OfBody' => 'funny')))));
$this->jobPluginManager->expects($this->once())->method('get')->with('MyClass')->will($this->returnValue(new Asset\SimpleJob()));
$job = $this->sqsQueue->pop();
$this->assertInstanceOf('SlmQueueSqsTest\\Asset\\SimpleJob', $job);
$this->assertEquals('aa', $job->getContent());
$this->assertEquals(array('__id__' => 'id_123', '__name__' => 'MyClass', 'receiptHandle' => 'receipt_123', 'md5' => 'funny', 'foo' => 'bar'), $job->getMetadata());
}
/**
* Release the job.
*
* @param Job $job The job to release.
*
* @return boolean whether the operation succeed.
*/
public function releaseJob(Job $job)
{
if (!empty($job->header['ReceiptHandle'])) {
$receiptHandle = $job->header['ReceiptHandle'];
$response = $this->_client->changeMessageVisibility(['QueueUrl' => $this->url, 'ReceiptHandle' => $receiptHandle, 'VisibilityTimeout' => 0]);
return $response !== null;
} else {
return false;
}
}
/**
* Releases a message back to the queue, making it visible again
*
* @param Message $message
* @return bool returns true if successful, false otherwise
*/
public function release(Message $message)
{
try {
// Set the visibility timeout to 0 to make the message visible in the queue again straight away
$this->sqs_client->changeMessageVisibility(array('QueueUrl' => $this->url, 'ReceiptHandle' => $message->receipt_handle, 'VisibilityTimeout' => 0));
return true;
} catch (Exception $e) {
echo 'Error releasing job back to queue ' . $e->getMessage();
return false;
}
}
