/**
* 消费初始化
* @param $groupName
* @param string $configKey
* @throws Exception
*/
function getCustomerInstance($groupName, $configKey = "default")
{
$configObj = new Config("kfk");
$config = $configObj->get($configKey);
try {
$conf = new \RdKafka\Conf();
$conf->setRebalanceCb(function (\RdKafka\KafkaConsumer $kafka, $err, array $partitions = null) {
switch ($err) {
case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS:
// echo "Assign: ";
// var_dump($partitions);
$kafka->assign($partitions);
break;
case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS:
// echo "Revoke: ";
// var_dump($partitions);
$kafka->assign(NULL);
break;
default:
throw new \Exception($err);
}
});
$conf->set('group.id', $groupName);
$conf->set('metadata.broker.list', $config);
$topicConf = new \RdKafka\TopicConf();
$topicConf->set('auto.offset.reset', 'smallest');
$conf->setDefaultTopicConf($topicConf);
$rk = new \RdKafka\KafkaConsumer($conf);
$this->type = 1;
$this->rk = $rk;
} catch (\Exception $e) {
throw new \Exception($e->getMessage());
}
}
/**
*
* @param ClusterMetadataInterface|null $clusterMetadata
* @param \RdKafka\Conf|null $config
*/
public function __construct(ClusterMetadataInterface $clusterMetadata = null, \RdKafka\Conf $config = null)
{
if (empty($config)) {
$config = new \RdKafka\Conf();
}
if (!empty($clusterMetadata)) {
$brokers = $clusterMetadata->getBrokers();
$brokers = array_map(function ($broker) {
return sprintf('%s:%d', $broker['host'], $broker['port']);
}, $brokers);
$config->set('metadata.broker.list', implode(',', $brokers));
}
parent::__construct($config);
}
function __construct($topic_name, $identity, $host_name = "kafka")
{
$conf = YII::app()->params[$host_name];
if ($conf == '' || $topic_name == '' || $identity == '') {
echo "error : no init param";
exit;
}
if (!in_array($identity, array('producer', 'consumer'))) {
echo "error : wrong identity";
exit;
}
$this->topic_name = $topic_name;
//初始化全局配置
$rd_conf = new RdKafka\Conf();
$rd_conf->set('metadata.broker.list', $conf['host']);
#$rd_conf -> set('socket.keepalive.enable',true);
#$rd_conf -> set('log_level',LOG_DEBUG);//Logging level (syslog(3) levels)
//print_r($rd_conf->dump());exit;
switch ($identity) {
case 'producer':
$rd_conf->set('compression.codec', 'none');
//Compression codec to use for compressing message sets: none, gzip or snappy;default none
$this->producer = new RdKafka\Producer($rd_conf);
break;
case 'consumer':
$rd_conf->set('fetch.message.max.bytes', self::CONSUMER_MESSAGE_MAX_BYTES);
$this->consumer = new RdKafka\Consumer($rd_conf);
$rd_topic_conf = new RdKafka\TopicConf();
$back = debug_backtrace();
$back = $back[1];
$group = $this->topic_name . "_" . $back['class'] . "_" . $back['function'];
$rd_topic_conf->set('group.id', $group);
$rd_topic_conf->set('auto.commit.interval.ms', 1000);
$rd_topic_conf->set("offset.store.path", $this->offset_path);
$rd_topic_conf->set('auto.offset.reset', 'smallest');
//$rd_topic_conf -> set('offset.store.method','broker');//flie(offset.store.path),broker
//$rd_topic_conf -> set('offset.store.sync.interval.ms',60000);//fsync() interval for the offset file, in milliseconds. Use -1 to disable syncing, and 0 for immediate sync after each write.
$this->consumer_topic = $this->consumer->newTopic($topic_name, $rd_topic_conf);
break;
}
}
/**
* https://github.com/andreiz/php-zookeeper/blob/master/examples/Zookeeper_Example.php
*/
$zk = new Zookeeper('127.0.0.1:port,...,ip:port/path');
$brokersPath = '/brokers/ids';
$ids = $zk->getChildren($brokersPath);
$brokers = array();
foreach ($ids as $id) {
$path = $brokersPath . "/{$id}";
$val = $zk->get($path);
$val = json_decode($val, true);
$brokers[] = $val['host'] . ":" . $val['port'];
}
$brokersAddr = implode(',', $brokers);
//begin to consume
$conf = new RdKafka\Conf();
// Set the group id. This is required when storing offsets on the broker
$conf->set('group.id', 'my-alarm-group');
$conf->set('broker.version.fallback', '0.8.2.2');
// socket请求的超时时间。实际的超时时间为max.fetch.wait + socket.timeout.ms。
$conf->set('socket.timeout.ms', '400');
$consumer = new RdKafka\Consumer($conf);
$consumer->addBrokers($brokersAddr);
$consumer->setLogLevel(LOG_DEBUG);
$topicConf = new RdKafka\TopicConf();
$topicConf->set('auto.commit.interval.ms', 1000);
// Set the offset store method to 'file'
$topicConf->set('offset.store.method', 'file');
$topicConf->set('offset.store.path', sys_get_temp_dir());
//$topicConf->set('api.version.request', true);
//$topicConf->set('broker.version.fallback', '0.8.2.2');