/**
* Write the Request Header
* <req_len> + <req_type> + <topic_len> + <topic> + <partition>
*
* @param Kafka_Socket $socket Socket
*
* @return void
*/
protected function writeRequestHeader(Kafka_Socket $socket)
{
// REQUEST_LENGTH (int) + REQUEST_TYPE (short)
$socket->write(pack('N', $this->sizeInBytes() + 2));
$socket->write(pack('n', $this->id));
// TOPIC_SIZE (short) + TOPIC (bytes)
$socket->write(pack('n', strlen($this->topic)) . $this->topic);
// PARTITION (int)
$socket->write(pack('N', $this->partition));
}
/**
* Read the next message
*
* @return string Message (raw)
* @throws Kafka_Exception when the message cannot be read from the stream buffer
*/
protected function getMessage()
{
try {
$size = $this->getMessageSize();
$msg = $this->socket->read($size, true);
} catch (Kafka_Exception_Socket_EOF $e) {
$size = isset($size) ? $size : 'enough';
$logMsg = 'Cannot read ' . $size . ' bytes, the message is likely bigger than the buffer';
throw new Kafka_Exception_OutOfRange($logMsg);
}
$this->validByteCount += 4 + $size;
return $msg;
}
/**
* Read the response error code
*
* @return integer Error code
*/
protected function getResponseCode()
{
$this->connect();
$data = $this->socket->read(2, true);
$unpack = unpack('n', $data);
return array_shift($unpack);
}
public function testWriteToOffset()
{
$this->offset = 14;
$this->req = new Kafka_FetchRequest($this->topic, $this->partition, $this->offset, $this->maxSize);
$stream = fopen('php://temp', 'w+b');
$socket = Kafka_Socket::createFromStream($stream);
$this->req->writeTo($socket);
rewind($stream);
//read it back
$headers = fread($stream, 6);
$topicLen = array_shift(unpack('n', fread($stream, 2)));
$this->assertEquals(strlen($this->topic), $topicLen);
$this->assertEquals($this->topic, fread($stream, $topicLen));
$this->assertEquals($this->partition, array_shift(unpack('N', fread($stream, 4))));
$int64bit = unpack('N2', fread($stream, 8));
$this->assertEquals($this->offset, $int64bit[2]);
$this->assertEquals($this->maxSize, array_shift(unpack('N', fread($stream, 4))));
}
public function connect()
{
if (null === $this->socket) {
$this->socket = Kafka_Socket::createFromStream(fopen('php://temp', 'w+b'));
}
}
/**
* Parse the response and return the array of offsets
*
* @param Kafka_Socket $socket Socket handle
*
* @return array
*/
public static function deserializeOffsetArray(Kafka_Socket $socket)
{
$nOffsets = array_shift(unpack('N', $socket->read(4)));
if ($nOffsets < 0) {
throw new Kafka_Exception_OutOfRange($nOffsets . ' is not a valid number of offsets');
}
$offsets = array();
for ($i = 0; $i < $nOffsets; ++$i) {
$offsets[] = self::unpackLong64bigendian($socket->read(8));
}
return $offsets;
}
/**
* Send messages to Kafka
*
* @param array $messages Messages to send
* @param string $topic Topic
* @param integer $partition Partition
*
* @return boolean
*/
public function send(array $messages, $topic, $partition = 0xffffffff)
{
$this->connect();
return $this->socket->write(Kafka_Encoder::encode_produce_request($topic, $partition, $messages, $this->compression));
}
/**
* @expectedException Kafka_Exception_Socket
*/
public function testWriteAfterClose()
{
$stream = fopen('php://temp', 'w+b');
$socket = Kafka_Socket::createFromStream($stream);
$socket->close();
$socket->write('test');
$this->fail('The above write() call should fail on a closed socket');
}
public function testMixedMessages()
{
$stream = fopen('php://temp', 'w+b');
$messages = array('message #1', 'message #2', 'message #3');
$this->writeDummyCompressedMessageSet($stream, $messages, Kafka_Encoder::COMPRESSION_GZIP);
$messages2 = array('message #4', 'message #5', 'message #6');
$this->writeDummyMessageSet($stream, $messages2, Kafka_Encoder::COMPRESSION_NONE);
$this->writeDummyCompressedMessageSet($stream, $messages, Kafka_Encoder::COMPRESSION_GZIP);
rewind($stream);
$allMessages = $messages;
foreach ($messages2 as $msg) {
$allMessages[] = $msg;
}
foreach ($messages as $msg) {
$allMessages[] = $msg;
}
$socket = Kafka_Socket::createFromStream($stream);
$set = new Kafka_MessageSet($socket, 0, 0);
$idx = 0;
foreach ($set as $offset => $msg) {
$this->assertEquals($allMessages[$idx++], $msg->payload());
}
$this->assertEquals(count($allMessages), $idx);
// test new offset
$readBytes = $set->validBytes();
$this->assertEquals(198, $readBytes);
// no more data
$set = new Kafka_MessageSet($socket, $readBytes, 0);
$cnt = 0;
foreach ($set as $offset => $msg) {
$cnt++;
}
$this->assertEquals(0, $cnt);
fclose($stream);
}
/**
* Decompress a message
*
* @param string $msg Message
* @param integer $compression 0=none, 1=gzip, 2=snappy
*
* @return string
* @throws Kafka_Exception
*/
public static function decompress($msg, $compression)
{
switch ($compression) {
case self::COMPRESSION_NONE:
return $msg;
case self::COMPRESSION_GZIP:
// NB: this is really a MessageSet, not just a single message
// although I'm not sure this is the best way to handle the inner offsets,
// as the symmetry with the outer collection iteration is broken.
// @see https://issues.apache.org/jira/browse/KAFKA-406
$stream = fopen('php://temp', 'w+b');
fwrite($stream, gzinflate(substr($msg, 10)));
rewind($stream);
$socket = Kafka_Socket::createFromStream($stream);
return new Kafka_MessageSetInternalIterator($socket, 0, 0);
case self::COMPRESSION_SNAPPY:
throw new Kafka_Exception_NotSupported('SNAPPY decompression not yet implemented');
default:
throw new Kafka_Exception_NotSupported('Unknown compression flag: ' . $compression);
}
}
/**
* Read the response error code
*
* @return integer Error code
*/
protected function getResponseCode()
{
$this->connect();
return array_shift(unpack('n', $this->socket->read(2, true)));
}
