Esempio n. 1
0
 /**
  * 消费初始化
  * @param $groupName
  * @param string $configKey
  * @throws Exception
  */
 function getCustomerInstance($groupName, $configKey = "default")
 {
     $configObj = new Config("kfk");
     $config = $configObj->get($configKey);
     try {
         $conf = new \RdKafka\Conf();
         $conf->setRebalanceCb(function (\RdKafka\KafkaConsumer $kafka, $err, array $partitions = null) {
             switch ($err) {
                 case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS:
                     //                        echo "Assign: ";
                     //                        var_dump($partitions);
                     $kafka->assign($partitions);
                     break;
                 case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS:
                     //                        echo "Revoke: ";
                     //                        var_dump($partitions);
                     $kafka->assign(NULL);
                     break;
                 default:
                     throw new \Exception($err);
             }
         });
         $conf->set('group.id', $groupName);
         $conf->set('metadata.broker.list', $config);
         $topicConf = new \RdKafka\TopicConf();
         $topicConf->set('auto.offset.reset', 'smallest');
         $conf->setDefaultTopicConf($topicConf);
         $rk = new \RdKafka\KafkaConsumer($conf);
         $this->type = 1;
         $this->rk = $rk;
     } catch (\Exception $e) {
         throw new \Exception($e->getMessage());
     }
 }
Esempio n. 2
0
 /**
  *
  * @param ClusterMetadataInterface|null $clusterMetadata
  * @param \RdKafka\Conf|null            $config
  */
 public function __construct(ClusterMetadataInterface $clusterMetadata = null, \RdKafka\Conf $config = null)
 {
     if (empty($config)) {
         $config = new \RdKafka\Conf();
     }
     if (!empty($clusterMetadata)) {
         $brokers = $clusterMetadata->getBrokers();
         $brokers = array_map(function ($broker) {
             return sprintf('%s:%d', $broker['host'], $broker['port']);
         }, $brokers);
         $config->set('metadata.broker.list', implode(',', $brokers));
     }
     parent::__construct($config);
 }
Esempio n. 3
0
 function __construct($topic_name, $identity, $host_name = "kafka")
 {
     $conf = YII::app()->params[$host_name];
     if ($conf == '' || $topic_name == '' || $identity == '') {
         echo "error : no init param";
         exit;
     }
     if (!in_array($identity, array('producer', 'consumer'))) {
         echo "error : wrong identity";
         exit;
     }
     $this->topic_name = $topic_name;
     //初始化全局配置
     $rd_conf = new RdKafka\Conf();
     $rd_conf->set('metadata.broker.list', $conf['host']);
     #$rd_conf -> set('socket.keepalive.enable',true);
     #$rd_conf -> set('log_level',LOG_DEBUG);//Logging level (syslog(3) levels)
     //print_r($rd_conf->dump());exit;
     switch ($identity) {
         case 'producer':
             $rd_conf->set('compression.codec', 'none');
             //Compression codec to use for compressing message sets: none, gzip or snappy;default none
             $this->producer = new RdKafka\Producer($rd_conf);
             break;
         case 'consumer':
             $rd_conf->set('fetch.message.max.bytes', self::CONSUMER_MESSAGE_MAX_BYTES);
             $this->consumer = new RdKafka\Consumer($rd_conf);
             $rd_topic_conf = new RdKafka\TopicConf();
             $back = debug_backtrace();
             $back = $back[1];
             $group = $this->topic_name . "_" . $back['class'] . "_" . $back['function'];
             $rd_topic_conf->set('group.id', $group);
             $rd_topic_conf->set('auto.commit.interval.ms', 1000);
             $rd_topic_conf->set("offset.store.path", $this->offset_path);
             $rd_topic_conf->set('auto.offset.reset', 'smallest');
             //$rd_topic_conf -> set('offset.store.method','broker');//flie(offset.store.path),broker
             //$rd_topic_conf -> set('offset.store.sync.interval.ms',60000);//fsync() interval for the offset file, in milliseconds. Use -1 to disable syncing, and 0 for immediate sync after each write.
             $this->consumer_topic = $this->consumer->newTopic($topic_name, $rd_topic_conf);
             break;
     }
 }
Esempio n. 4
0
/**
 * https://github.com/andreiz/php-zookeeper/blob/master/examples/Zookeeper_Example.php
 */
$zk = new Zookeeper('127.0.0.1:port,...,ip:port/path');
$brokersPath = '/brokers/ids';
$ids = $zk->getChildren($brokersPath);
$brokers = array();
foreach ($ids as $id) {
    $path = $brokersPath . "/{$id}";
    $val = $zk->get($path);
    $val = json_decode($val, true);
    $brokers[] = $val['host'] . ":" . $val['port'];
}
$brokersAddr = implode(',', $brokers);
//begin to consume
$conf = new RdKafka\Conf();
// Set the group id. This is required when storing offsets on the broker
$conf->set('group.id', 'my-alarm-group');
$conf->set('broker.version.fallback', '0.8.2.2');
// socket请求的超时时间。实际的超时时间为max.fetch.wait + socket.timeout.ms。
$conf->set('socket.timeout.ms', '400');
$consumer = new RdKafka\Consumer($conf);
$consumer->addBrokers($brokersAddr);
$consumer->setLogLevel(LOG_DEBUG);
$topicConf = new RdKafka\TopicConf();
$topicConf->set('auto.commit.interval.ms', 1000);
// Set the offset store method to 'file'
$topicConf->set('offset.store.method', 'file');
$topicConf->set('offset.store.path', sys_get_temp_dir());
//$topicConf->set('api.version.request', true);
//$topicConf->set('broker.version.fallback', '0.8.2.2');