private function _storeShard($stream_name, $shard_id, $sequence_number) { $shard = new KinesisShard(); $shard->setStreamName($stream_name)->setShardId($shard_id)->setSequenceNumber($sequence_number); $data_store = new KinesisShardMemcacheDataStore($this->memcache); $data_store->modify($shard); return $shard; }
public function findDataRecords(KinesisShard $shard, $limit, $max_loop_count) { $result = array(); $option = null; if ($shard->getSequenceNumber() === '0') { $option = array('StreamName' => $shard->getStreamName(), 'ShardId' => $shard->getShardId(), 'ShardIteratorType' => 'TRIM_HORIZON'); } else { $option = array('StreamName' => $shard->getStreamName(), 'ShardId' => $shard->getShardId(), 'ShardIteratorType' => 'AFTER_SEQUENCE_NUMBER', 'StartingSequenceNumber' => $shard->getSequenceNumber()); } $kinesis = $this->getKinesis(); $shard_iterator_result = $kinesis->getShardIterator($option); $shard_iterator = $shard_iterator_result['ShardIterator']; for ($i = 0; $i < $max_loop_count; $i++) { $get_records_result = $kinesis->getRecords(array('ShardIterator' => $shard_iterator, 'Limit' => $limit)); $records = $get_records_result['Records']; foreach ($records as $record) { $data_record = new KinesisDataRecord(); $data_record->setStreamName($shard->getStreamName())->setShardId($shard->getShardId())->setSequenceNumber($record['SequenceNumber'])->setData($record['Data'])->setPartitionKey($record['PartitionKey']); $result[] = $data_record; } if (count($result) >= $limit) { break; } $shard_iterator = $get_records_result['NextShardIterator']; } return $result; }
/** * @Override */ public function restore($target_stream_name) { $result = array(); $keys = $this->_getKeys($target_stream_name); foreach ($keys as $key) { list($stream_name, $shard_id) = explode(':', $key); $sequence_number = $this->memcache->get($key); $shard = new KinesisShard(); $shard->setStreamName($stream_name)->setShardId($shard_id)->setSequenceNumber($sequence_number); $result[] = $shard; } return $result; }
/** * @Override */ public function restore($target_stream_name) { $result = array(); $store_dir = $this->data_store_dir . '/' . $target_stream_name; if (!file_exists($store_dir)) { return $result; } if ($dir_handle = opendir($store_dir)) { while (false !== ($file = readdir($dir_handle))) { $file_handle = fopen($store_dir . '/' . $file, "r"); while ($shard_info = fgetcsv($file_handle)) { list($stream_name, $shard_id, $sequence_number) = $shard_info; $shard = new KinesisShard(); $shard->setStreamName($stream_name)->setShardId($shard_id)->setSequenceNumber($sequence_number); $result[$shard_id] = $shard; } fclose($file_handle); } closedir($dir_handle); } return $result; }
public function testFindDataRecords_001() { // $data_store = $this->getMock("KinesisShardFileDataStore", array('restore')); // $data_store->expects($this->any()) // ->method('restore') // ->will($this->returnValue(array())); $kinesis = $this->getMock("KinesisClient", array('getShardIterator', 'getRecords')); $kinesis->expects($this->any())->method('getShardIterator')->will($this->returnValue(array('ShardIterator' => '123456789'))); $kinesis->expects($this->any())->method('getRecords')->will($this->returnValue(array('NextShardIterator' => '234567891', 'Records' => array(array('SequenceNumber' => '11111', 'Data' => 'hoge', 'PartitionKey' => 'pt1'), array('SequenceNumber' => '11112', 'Data' => 'foo', 'PartitionKey' => 'pt1'))))); $proxy = $this->getMockBuilder('Rf\\Aws\\Kinesis\\ClientLibrary\\KinesisProxy')->setMethods(array('getKinesis'))->disableOriginalConstructor()->getMock(); $proxy->expects($this->any())->method('getKinesis')->will($this->returnValue($kinesis)); // $proxy->expects($this->any()) // ->method('getDataStore') // ->will($this->returnValue($data_store)); $shard = new KinesisShard(); $shard->setStreamName('dummy-stream-name'); $shard->setShardId('dummy-shardId-000000000001'); $shard->setSequenceNumber('234567891'); $result = $proxy->findDataRecords($shard, 1000, 1); $this->assertCount(2, $result); $result_data_record1 = $result[0]; $this->assertEquals('dummy-stream-name', $result_data_record1->getStreamName()); $this->assertEquals('dummy-shardId-000000000001', $result_data_record1->getShardId()); $this->assertEquals('11111', $result_data_record1->getSequenceNumber()); $this->assertEquals('hoge', $result_data_record1->getData()); $this->assertEquals('pt1', $result_data_record1->getPartitionKey()); $result_data_record2 = $result[1]; $this->assertEquals('11112', $result_data_record2->getSequenceNumber()); $this->assertEquals('foo', $result_data_record2->getData()); $this->assertEquals('pt1', $result_data_record2->getPartitionKey()); }