/**
* @expectedException Kafka_Exception_Socket
*/
public function testConnectFailure()
{
$consumer = new Kafka_SimpleConsumer('invalid-host-name', 1234567890, 10, 1000000);
$consumer->connect();
$this->fail('The above call should throw an exception');
}
/**
* Get an approximate measure of the amount of data still to be consumed
*
* @return integer
*/
public function getRemainingSize()
{
try {
if (0 == $this->nIterators) {
$this->rewind();
// initialise simple consumers
}
} catch (Kafka_Exception_InvalidTopic $e) {
$logMsg = 'Invalid topic from ZookeeperConsumer::rewind(): Most likely cause is no topic yet as there is no data';
error_log($logMsg);
}
$totalSize = 0;
foreach ($this->iterators as $it) {
$readBytes = $it->offset + $it->uncommittedOffset;
if (null !== $it->messages) {
$readBytes += $it->messages->validBytes();
}
$consumer = new Kafka_SimpleConsumer($it->host, $it->port, $this->socketTimeout, $this->maxBatchSize);
$offsets = $consumer->getOffsetsBefore($this->topic, $it->partition, Kafka_SimpleConsumer::OFFSET_LAST, 1);
if (count($offsets) > 0) {
$remaining = $offsets[0] - $readBytes;
// remaining bytes for this broker/partition
if ($remaining > 0) {
$totalSize += $remaining;
}
}
$consumer->close();
}
return $totalSize;
}
* limitations under the License.
*/
set_include_path(implode(PATH_SEPARATOR, array(realpath(__DIR__ . '/../lib'), get_include_path())));
require 'autoloader.php';
$host = 'localhost';
$zkPort = 2181;
//zookeeper
$kPort = 9092;
//kafka server
$topic = 'test';
$maxSize = 10000000;
$socketTimeout = 2;
$offset = 0;
$partition = 0;
$nMessages = 0;
$consumer = new Kafka_SimpleConsumer($host, $kPort, $socketTimeout, $maxSize);
while (true) {
try {
//create a fetch request for topic "test", partition 0, current offset and fetch size of 1MB
$fetchRequest = new Kafka_FetchRequest($topic, $partition, $offset, $maxSize);
//get the message set from the consumer and print them out
$partialOffset = 0;
$messages = $consumer->fetch($fetchRequest);
foreach ($messages as $msg) {
++$nMessages;
echo "\nconsumed[{$offset}][{$partialOffset}][msg #{$nMessages}]: " . $msg->payload();
$partialOffset = $messages->validBytes();
}
//advance the offset after consuming each message
$offset += $messages->validBytes();
//echo "\n---[Advancing offset to $offset]------(".date('H:i:s').")";
/**
* Get an approximate measure of the amount of data still to be consumed
*
* @return integer
*/
public function getRemainingSize()
{
if (0 == $this->nIterators) {
$this->rewind();
// initialise simple consumers
}
$totalSize = 0;
foreach ($this->iterators as $it) {
$readBytes = $it->offset + $it->uncommittedOffset;
if (null !== $it->messages) {
$readBytes += $it->messages->validBytes();
}
$consumer = new Kafka_SimpleConsumer($it->host, $it->port, $this->socketTimeout, $this->maxBatchSize);
$offsets = $consumer->getOffsetsBefore($this->topic, $it->partition, Kafka_SimpleConsumer::OFFSET_LAST, 1);
if (count($offsets) > 0) {
$remaining = $offsets[0] - $readBytes;
// remaining bytes for this broker/partition
if ($remaining > 0) {
$totalSize += $remaining;
}
}
$consumer->close();
}
return $totalSize;
}
