/** * get produce server offset * * @param string $topicName * @param integer $partitionId * @access public * @return int */ public function getProduceOffset($timeLine = self::LAST_OFFSET) { $topicName = $this->topicName; $partitionId = $this->partitionId; $requestData = array('data' => array(array('topic_name' => $this->topicName, 'partitions' => array(array('partition_id' => $this->partitionId, 'time' => $timeLine, 'max_offset' => 1))))); $this->encoder->offsetRequest($requestData); $result = $this->decoder->offsetResponse(); $this->client->freeStream($this->streamKey); if (!isset($result[$topicName][$partitionId]['offset'])) { if (isset($result[$topicName][$partitionId]['errCode'])) { throw new \Kafka\Exception(\Kafka\Protocol\Decoder::getError($result[$topicName][$partitionId]['errCode'])); } else { throw new \Kafka\Exception('get offset failed. topic name:' . $this->topicName . ' partitionId: ' . $this->partitionId); } } return array_shift($result[$topicName][$partitionId]['offset']); }
/** * testFetchOffsetResponseNotData * * @access public * @return void */ public function testFetchOffsetResponseNotData() { $this->setData(Decoder::Khex2bin('00000000')); $decoder = new \Kafka\Protocol\Decoder($this->stream); try { $actual = $decoder->fetchOffsetResponse(); } catch (\Kafka\Exception\Protocol $e) { $this->assertSame('fetch offset response invalid.', $e->getMessage()); } }
/** * testGetError * * @access public * @return void */ public function testGetError() { $this->assertEquals('Unknown error', Decoder::getError(19)); }
/** * load next partition * * @access public * @return bool */ public function loadNextPartition() { if ($this->validCount >= $this->partitionCount) { return false; } try { $partitionId = $this->stream->read(4, true); $partitionId = Decoder::unpack(Decoder::BIT_B32, $partitionId); $partitionId = array_shift($partitionId); \Kafka\Log::log("kafka client:fetch partition:" . $partitionId, LOG_INFO); $errCode = $this->stream->read(2, true); $errCode = Decoder::unpack(Decoder::BIT_B16, $errCode); $this->errCode = array_shift($errCode); if ($this->errCode != 0) { throw new \Kafka\Exception(\Kafka\Protocol\Decoder::getError($this->errCode)); } $offset = $this->stream->read(8, true); $this->offset = \Kafka\Protocol\Decoder::unpack(Decoder::BIT_B64, $offset); $this->key = $partitionId; $this->current = new MessageSet($this, $this->context); } catch (\Kafka\Exception $e) { \Kafka\Log::log($e->getMessage(), LOG_ERR); return false; } $this->validCount++; return true; }
/** * Send any records in the kafka client internal queue. */ protected function send() { try { $response = $this->produce->send(); } catch (\Kafka\Exception $e) { $ignore = $this->warning('Error sending records to kafka: {exception}', ['exception' => $e]); if (!$ignore) { throw $e; } else { return; } } if (is_bool($response)) { return; } $errors = []; foreach ($response as $topicName => $partitionResponse) { foreach ($partitionResponse as $partition => $info) { if ($info['errCode'] === 0) { // no error continue; } $errors[] = sprintf('Error producing to %s (errno %d): %s', $topicName, $info['errCode'], Decoder::getError($info['errCode'])); } } if ($errors) { $error = implode("\n", $errors); if (!$this->warning($error)) { throw new \RuntimeException($error); } } }
/** * load next message * * @access public * @return void */ public function loadNextMessage() { if ($this->validByteCount >= $this->messageSetSize) { return false; } try { $offset = $this->stream->read(8, true); $this->offset = \Kafka\Protocol\Decoder::unpack(Decoder::BIT_B64, $offset); $messageSize = $this->stream->read(4, true); $messageSize = Decoder::unpack(Decoder::BIT_B32, $messageSize); $messageSize = array_shift($messageSize); $msg = $this->stream->read($messageSize, true); $this->current = new Message($msg); } catch (\Kafka\Exception $e) { return false; } $this->validByteCount += 8 + 4 + $messageSize; return true; }
/** * load next topic * * @access public * @return void */ public function loadNextTopic() { if ($this->validCount >= $this->topicCount) { \Kafka\Protocol\Fetch\Helper\Helper::onStreamEof($this->currentStreamLockKey); return false; } if ($this->currentStreamCount >= $this->topicCounts[$this->currentStreamKey]) { \Kafka\Protocol\Fetch\Helper\Helper::onStreamEof($this->currentStreamLockKey); $this->currentStreamKey++; } $lockKeys = array_keys($this->streams); $streams = array_values($this->streams); if (!isset($streams[$this->currentStreamKey])) { return false; } $stream = $streams[$this->currentStreamKey]; $this->currentStreamLockKey = $lockKeys[$this->currentStreamKey]; try { $topicLen = $stream->read(2, true); $topicLen = Decoder::unpack(Decoder::BIT_B16, $topicLen); $topicLen = array_shift($topicLen); if ($topicLen <= 0) { return false; } // topic name $this->key = $stream->read($topicLen, true); $this->current = new Partition($this, $this->context); } catch (\Kafka\Exception $e) { return false; } $this->validCount++; $this->currentStreamCount++; return true; }
/** * __construct * * @param string(raw) $msg * @access public * @return void */ public function __construct($msg) { $offset = 0; $crc = Decoder::unpack(Decoder::BIT_B32, substr($msg, $offset, 4)); $offset += 4; $this->crc = array_shift($crc); $magic = Decoder::unpack(Decoder::BIT_B8, substr($msg, $offset, 1)); $this->magic = array_shift($magic); $offset += 1; $attr = Decoder::unpack(Decoder::BIT_B8, substr($msg, $offset, 1)); $this->attribute = array_shift($attr); $offset += 1; $keyLen = Decoder::unpack(Decoder::BIT_B32, substr($msg, $offset, 4)); $keyLen = array_shift($keyLen); $offset += 4; if ($keyLen > 0 && $keyLen != 0xffffffff) { $this->key = substr($msg, $offset, $keyLen); $offset += $keyLen; } $messageSize = Decoder::unpack(Decoder::BIT_B32, substr($msg, $offset, 4)); $messageSize = array_shift($messageSize); $offset += 4; if ($messageSize) { $this->value = substr($msg, $offset, $messageSize); } }
/** * load next message * * @access public * @return void */ public function loadNextMessage() { if ($this->validByteCount >= $this->messageSetSize) { return false; } try { if ($this->validByteCount + 12 > $this->messageSetSize) { // read socket buffer dirty data $this->stream->read($this->messageSetSize - $this->validByteCount); return false; } $offset = $this->stream->read(8, true); $this->offset = \Kafka\Protocol\Decoder::unpack(Decoder::BIT_B64, $offset); $messageSize = $this->stream->read(4, true); $messageSize = Decoder::unpack(Decoder::BIT_B32, $messageSize); $messageSize = array_shift($messageSize); $this->validByteCount += 12; if ($this->validByteCount + $messageSize > $this->messageSetSize) { // read socket buffer dirty data $this->stream->read($this->messageSetSize - $this->validByteCount); return false; } $msg = $this->stream->read($messageSize, true); $this->current = new Message($msg); } catch (\Kafka\Exception $e) { \Kafka\Log::log("already fetch: {$this->validByteCount}, {$e->getMessage()}", LOG_INFO); return false; } $this->validByteCount += $messageSize; return true; }
/** * load next partition * * @access public * @return void */ public function loadNextPartition() { if ($this->validCount >= $this->partitionCount) { return false; } try { $partitionId = $this->stream->read(4, true); $partitionId = Decoder::unpack(Decoder::BIT_B32, $partitionId); $partitionId = array_shift($partitionId); $errCode = $this->stream->read(2, true); $errCode = Decoder::unpack(Decoder::BIT_B16, $errCode); $this->errCode = array_shift($errCode); $offset = $this->stream->read(8, true); $this->offset = \Kafka\Protocol\Decoder::unpack(Decoder::BIT_B64, $offset); $this->key = $partitionId; $this->current = new MessageSet($this, $this->context); } catch (\Kafka\Exception $e) { return false; } $this->validCount++; return true; }