public function findDataRecords(KinesisShard $shard, $limit, $max_loop_count) { $result = array(); $option = null; if ($shard->getSequenceNumber() === '0') { $option = array('StreamName' => $shard->getStreamName(), 'ShardId' => $shard->getShardId(), 'ShardIteratorType' => 'TRIM_HORIZON'); } else { $option = array('StreamName' => $shard->getStreamName(), 'ShardId' => $shard->getShardId(), 'ShardIteratorType' => 'AFTER_SEQUENCE_NUMBER', 'StartingSequenceNumber' => $shard->getSequenceNumber()); } $kinesis = $this->getKinesis(); $shard_iterator_result = $kinesis->getShardIterator($option); $shard_iterator = $shard_iterator_result['ShardIterator']; for ($i = 0; $i < $max_loop_count; $i++) { $get_records_result = $kinesis->getRecords(array('ShardIterator' => $shard_iterator, 'Limit' => $limit)); $records = $get_records_result['Records']; foreach ($records as $record) { $data_record = new KinesisDataRecord(); $data_record->setStreamName($shard->getStreamName())->setShardId($shard->getShardId())->setSequenceNumber($record['SequenceNumber'])->setData($record['Data'])->setPartitionKey($record['PartitionKey']); $result[] = $data_record; } if (count($result) >= $limit) { break; } $shard_iterator = $get_records_result['NextShardIterator']; } return $result; }
/** * @Override */ public function modify(KinesisShard $shard) { $store_dir = $this->data_store_dir . '/' . $shard->getStreamName(); if (!file_exists($store_dir)) { mkdir($store_dir, 0755, true); } $file_name = $store_dir . '/' . $shard->getShardId(); if ($file_handle = @fopen($file_name, 'x')) { fclose($file_handle); } $file_handle = fopen($file_name, "rb+"); $flag = flock($file_handle, LOCK_SH); fwrite($file_handle, implode(',', array($shard->getStreamName(), $shard->getShardId(), $shard->getSequenceNumber()))); fclose($file_handle); }
/** * @Override */ public function modify(KinesisShard $shard) { $key = sprintf("%s:%s", $shard->getStreamName(), $shard->getShardId()); $this->memcache->set($key, $shard->getSequenceNumber()); }