/** * 消费初始化 * @param $groupName * @param string $configKey * @throws Exception */ function getCustomerInstance($groupName, $configKey = "default") { $configObj = new Config("kfk"); $config = $configObj->get($configKey); try { $conf = new \RdKafka\Conf(); $conf->setRebalanceCb(function (\RdKafka\KafkaConsumer $kafka, $err, array $partitions = null) { switch ($err) { case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS: // echo "Assign: "; // var_dump($partitions); $kafka->assign($partitions); break; case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS: // echo "Revoke: "; // var_dump($partitions); $kafka->assign(NULL); break; default: throw new \Exception($err); } }); $conf->set('group.id', $groupName); $conf->set('metadata.broker.list', $config); $topicConf = new \RdKafka\TopicConf(); $topicConf->set('auto.offset.reset', 'smallest'); $conf->setDefaultTopicConf($topicConf); $rk = new \RdKafka\KafkaConsumer($conf); $this->type = 1; $this->rk = $rk; } catch (\Exception $e) { throw new \Exception($e->getMessage()); } }
function __construct($topic_name, $identity, $host_name = "kafka") { $conf = YII::app()->params[$host_name]; if ($conf == '' || $topic_name == '' || $identity == '') { echo "error : no init param"; exit; } if (!in_array($identity, array('producer', 'consumer'))) { echo "error : wrong identity"; exit; } $this->topic_name = $topic_name; //初始化全局配置 $rd_conf = new RdKafka\Conf(); $rd_conf->set('metadata.broker.list', $conf['host']); #$rd_conf -> set('socket.keepalive.enable',true); #$rd_conf -> set('log_level',LOG_DEBUG);//Logging level (syslog(3) levels) //print_r($rd_conf->dump());exit; switch ($identity) { case 'producer': $rd_conf->set('compression.codec', 'none'); //Compression codec to use for compressing message sets: none, gzip or snappy;default none $this->producer = new RdKafka\Producer($rd_conf); break; case 'consumer': $rd_conf->set('fetch.message.max.bytes', self::CONSUMER_MESSAGE_MAX_BYTES); $this->consumer = new RdKafka\Consumer($rd_conf); $rd_topic_conf = new RdKafka\TopicConf(); $back = debug_backtrace(); $back = $back[1]; $group = $this->topic_name . "_" . $back['class'] . "_" . $back['function']; $rd_topic_conf->set('group.id', $group); $rd_topic_conf->set('auto.commit.interval.ms', 1000); $rd_topic_conf->set("offset.store.path", $this->offset_path); $rd_topic_conf->set('auto.offset.reset', 'smallest'); //$rd_topic_conf -> set('offset.store.method','broker');//flie(offset.store.path),broker //$rd_topic_conf -> set('offset.store.sync.interval.ms',60000);//fsync() interval for the offset file, in milliseconds. Use -1 to disable syncing, and 0 for immediate sync after each write. $this->consumer_topic = $this->consumer->newTopic($topic_name, $rd_topic_conf); break; } }
/** * * @param ClusterMetadataInterface|null $clusterMetadata * @param \RdKafka\Conf|null $config */ public function __construct(ClusterMetadataInterface $clusterMetadata = null, \RdKafka\Conf $config = null) { if (empty($config)) { $config = new \RdKafka\Conf(); } if (!empty($clusterMetadata)) { $brokers = $clusterMetadata->getBrokers(); $brokers = array_map(function ($broker) { return sprintf('%s:%d', $broker['host'], $broker['port']); }, $brokers); $config->set('metadata.broker.list', implode(',', $brokers)); } parent::__construct($config); }
*/ $zk = new Zookeeper('127.0.0.1:port,...,ip:port/path'); $brokersPath = '/brokers/ids'; $ids = $zk->getChildren($brokersPath); $brokers = array(); foreach ($ids as $id) { $path = $brokersPath . "/{$id}"; $val = $zk->get($path); $val = json_decode($val, true); $brokers[] = $val['host'] . ":" . $val['port']; } $brokersAddr = implode(',', $brokers); //begin to consume $conf = new RdKafka\Conf(); // Set the group id. This is required when storing offsets on the broker $conf->set('group.id', 'my-alarm-group'); $conf->set('broker.version.fallback', '0.8.2.2'); // socket请求的超时时间。实际的超时时间为max.fetch.wait + socket.timeout.ms。 $conf->set('socket.timeout.ms', '400'); $consumer = new RdKafka\Consumer($conf); $consumer->addBrokers($brokersAddr); $consumer->setLogLevel(LOG_DEBUG); $topicConf = new RdKafka\TopicConf(); $topicConf->set('auto.commit.interval.ms', 1000); // Set the offset store method to 'file' $topicConf->set('offset.store.method', 'file'); $topicConf->set('offset.store.path', sys_get_temp_dir()); //$topicConf->set('api.version.request', true); //$topicConf->set('broker.version.fallback', '0.8.2.2'); // Alternatively, set the offset store method to 'broker' // $topicConf->set('offset.store.method', 'broker');