public function addTopic($name, array $conf)
{
$producer = new \RdKafka\Producer();
$producer->addBrokers(implode(',', $conf['brokers']));
$producerTopicConf = new \RdKafka\TopicConf();
$producerTopic = $producer->newTopic($name, $producerTopicConf);
$consumer = new \RdKafka\Consumer();
$consumer->addBrokers(implode(',', $conf['brokers']));
$consumerTopicConf = new \RdKafka\TopicConf();
$consumerTopicConf->set("auto.commit.interval.ms", 1000.0);
$consumerTopicConf->set("offset.store.sync.interval.ms", 60000.0);
$consumerTopic = $consumer->newTopic($name, $consumerTopicConf);
$topic = new Topic($name, $producer, $producerTopic, $consumer, $consumerTopic);
$this->topics[$name] = $topic;
}
/**
* 消费初始化
* @param $groupName
* @param string $configKey
* @throws Exception
*/
function getCustomerInstance($groupName, $configKey = "default")
{
$configObj = new Config("kfk");
$config = $configObj->get($configKey);
try {
$conf = new \RdKafka\Conf();
$conf->setRebalanceCb(function (\RdKafka\KafkaConsumer $kafka, $err, array $partitions = null) {
switch ($err) {
case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS:
// echo "Assign: ";
// var_dump($partitions);
$kafka->assign($partitions);
break;
case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS:
// echo "Revoke: ";
// var_dump($partitions);
$kafka->assign(NULL);
break;
default:
throw new \Exception($err);
}
});
$conf->set('group.id', $groupName);
$conf->set('metadata.broker.list', $config);
$topicConf = new \RdKafka\TopicConf();
$topicConf->set('auto.offset.reset', 'smallest');
$conf->setDefaultTopicConf($topicConf);
$rk = new \RdKafka\KafkaConsumer($conf);
$this->type = 1;
$this->rk = $rk;
} catch (\Exception $e) {
throw new \Exception($e->getMessage());
}
}
public function toRdKafkaTopicConfig()
{
$topicConf = new \RdKafka\TopicConf();
$topicConf->set("auto.commit.interval.ms", $this->autoCommitIntervalMs);
$topicConf->set("offset.store.sync.interval.ms", $this->offsetStoreSyncIntervalMs);
$topicConf->set("offset.store.method", $this->offsetStoreMethod);
$topicConf->set("auto.commit.enable", $this->autoCommitEnable);
$topicConf->set("auto.offset.reset", $this->autoOffsetReset);
$topicConf->set("offset.store.path", $this->offsetStorePath);
$topicConf->set("consume.callback.max.messages", $this->consumeCallbackMaxMessages);
if ($this->groupId) {
$topicConf->set("group.id", $this->groupId);
}
}
public function toRdKafkaTopicConfig()
{
$topicConf = new \RdKafka\TopicConf();
$topicConf->set("auto.commit.interval.ms", $this->autoCommitIntervalMs);
$topicConf->set("offset.store.sync.interval.ms", $this->offsetStoreSyncIntervalMs);
$topicConf->set("offset.store.method", $this->offsetStoreMethod);
$topicConf->set("auto.commit.enable", $this->autoCommitEnable);
$topicConf->set("auto.offset.reset", $this->autoOffsetReset);
$topicConf->set("offset.store.path", $this->offsetStorePath);
if ($this->groupId) {
$topicConf->set("group.id", $this->groupId);
}
return $topicConf;
}
$val = $zk->get($path);
$val = json_decode($val, true);
$brokers[] = $val['host'] . ":" . $val['port'];
}
$brokersAddr = implode(',', $brokers);
//begin to consume
$conf = new RdKafka\Conf();
// Set the group id. This is required when storing offsets on the broker
$conf->set('group.id', 'my-alarm-group');
$conf->set('broker.version.fallback', '0.8.2.2');
// socket请求的超时时间。实际的超时时间为max.fetch.wait + socket.timeout.ms。
$conf->set('socket.timeout.ms', '400');
$consumer = new RdKafka\Consumer($conf);
$consumer->addBrokers($brokersAddr);
$consumer->setLogLevel(LOG_DEBUG);
$topicConf = new RdKafka\TopicConf();
$topicConf->set('auto.commit.interval.ms', 1000);
// Set the offset store method to 'file'
$topicConf->set('offset.store.method', 'file');
$topicConf->set('offset.store.path', sys_get_temp_dir());
//$topicConf->set('api.version.request', true);
//$topicConf->set('broker.version.fallback', '0.8.2.2');
// Alternatively, set the offset store method to 'broker'
// $topicConf->set('offset.store.method', 'broker');
// Set where to start consuming messages when there is no initial offset in
// offset store or the desired offset is out of range.
// 'smallest': start from the beginning
$topicConf->set('auto.offset.reset', 'largest');
$topic = $consumer->newTopic("Topic_Name", $topicConf);
// Start consuming partition 0
$metaData = $consumer->getMetadata(false, $topic, 1000);
public function toRdKafkaTopicConfig()
{
$topicConf = new \RdKafka\TopicConf();
$topicConf->set("request.required.acks", $this->requestRequiredAcks);
$topicConf->set("enforce.isr.cnt", $this->enforceIsrCnt);
$topicConf->set("request.timeout.ms", $this->requestTimeoutMs);
$topicConf->set("message.timeout.ms", $this->messageTimeoutMs);
return $topicConf;
}
private function setProducerTopic($mode = '')
{
if (!$this->producer_topic) {
if (!isset($this->producer)) {
echo "error : produce wrong identity";
exit;
}
$rd_topic_conf = new RdKafka\TopicConf();
$rd_topic_conf->set("request.required.acks", self::PRODUCER_REQUEST_ACK);
if ($mode == 'consistent') {
$rd_topic_conf->setPartitioner(RD_KAFKA_MSG_PARTITIONER_CONSISTENT);
}
//print_r($rd_topic_conf->dump());exit;
$this->producer_topic = $this->producer->newTopic($this->topic_name, $rd_topic_conf);
}
}