public function addTopic($name, array $conf)
 {
     $producer = new \RdKafka\Producer();
     $producer->addBrokers(implode(',', $conf['brokers']));
     $producerTopicConf = new \RdKafka\TopicConf();
     $producerTopic = $producer->newTopic($name, $producerTopicConf);
     $consumer = new \RdKafka\Consumer();
     $consumer->addBrokers(implode(',', $conf['brokers']));
     $consumerTopicConf = new \RdKafka\TopicConf();
     $consumerTopicConf->set("auto.commit.interval.ms", 1000.0);
     $consumerTopicConf->set("offset.store.sync.interval.ms", 60000.0);
     $consumerTopic = $consumer->newTopic($name, $consumerTopicConf);
     $topic = new Topic($name, $producer, $producerTopic, $consumer, $consumerTopic);
     $this->topics[$name] = $topic;
 }
Beispiel #2
0
 /**
  * 消费初始化
  * @param $groupName
  * @param string $configKey
  * @throws Exception
  */
 function getCustomerInstance($groupName, $configKey = "default")
 {
     $configObj = new Config("kfk");
     $config = $configObj->get($configKey);
     try {
         $conf = new \RdKafka\Conf();
         $conf->setRebalanceCb(function (\RdKafka\KafkaConsumer $kafka, $err, array $partitions = null) {
             switch ($err) {
                 case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS:
                     //                        echo "Assign: ";
                     //                        var_dump($partitions);
                     $kafka->assign($partitions);
                     break;
                 case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS:
                     //                        echo "Revoke: ";
                     //                        var_dump($partitions);
                     $kafka->assign(NULL);
                     break;
                 default:
                     throw new \Exception($err);
             }
         });
         $conf->set('group.id', $groupName);
         $conf->set('metadata.broker.list', $config);
         $topicConf = new \RdKafka\TopicConf();
         $topicConf->set('auto.offset.reset', 'smallest');
         $conf->setDefaultTopicConf($topicConf);
         $rk = new \RdKafka\KafkaConsumer($conf);
         $this->type = 1;
         $this->rk = $rk;
     } catch (\Exception $e) {
         throw new \Exception($e->getMessage());
     }
 }
 public function toRdKafkaTopicConfig()
 {
     $topicConf = new \RdKafka\TopicConf();
     $topicConf->set("auto.commit.interval.ms", $this->autoCommitIntervalMs);
     $topicConf->set("offset.store.sync.interval.ms", $this->offsetStoreSyncIntervalMs);
     $topicConf->set("offset.store.method", $this->offsetStoreMethod);
     $topicConf->set("auto.commit.enable", $this->autoCommitEnable);
     $topicConf->set("auto.offset.reset", $this->autoOffsetReset);
     $topicConf->set("offset.store.path", $this->offsetStorePath);
     $topicConf->set("consume.callback.max.messages", $this->consumeCallbackMaxMessages);
     if ($this->groupId) {
         $topicConf->set("group.id", $this->groupId);
     }
 }
 public function toRdKafkaTopicConfig()
 {
     $topicConf = new \RdKafka\TopicConf();
     $topicConf->set("auto.commit.interval.ms", $this->autoCommitIntervalMs);
     $topicConf->set("offset.store.sync.interval.ms", $this->offsetStoreSyncIntervalMs);
     $topicConf->set("offset.store.method", $this->offsetStoreMethod);
     $topicConf->set("auto.commit.enable", $this->autoCommitEnable);
     $topicConf->set("auto.offset.reset", $this->autoOffsetReset);
     $topicConf->set("offset.store.path", $this->offsetStorePath);
     if ($this->groupId) {
         $topicConf->set("group.id", $this->groupId);
     }
     return $topicConf;
 }
Beispiel #5
0
    $val = $zk->get($path);
    $val = json_decode($val, true);
    $brokers[] = $val['host'] . ":" . $val['port'];
}
$brokersAddr = implode(',', $brokers);
//begin to consume
$conf = new RdKafka\Conf();
// Set the group id. This is required when storing offsets on the broker
$conf->set('group.id', 'my-alarm-group');
$conf->set('broker.version.fallback', '0.8.2.2');
// socket请求的超时时间。实际的超时时间为max.fetch.wait + socket.timeout.ms。
$conf->set('socket.timeout.ms', '400');
$consumer = new RdKafka\Consumer($conf);
$consumer->addBrokers($brokersAddr);
$consumer->setLogLevel(LOG_DEBUG);
$topicConf = new RdKafka\TopicConf();
$topicConf->set('auto.commit.interval.ms', 1000);
// Set the offset store method to 'file'
$topicConf->set('offset.store.method', 'file');
$topicConf->set('offset.store.path', sys_get_temp_dir());
//$topicConf->set('api.version.request', true);
//$topicConf->set('broker.version.fallback', '0.8.2.2');
// Alternatively, set the offset store method to 'broker'
// $topicConf->set('offset.store.method', 'broker');
// Set where to start consuming messages when there is no initial offset in
// offset store or the desired offset is out of range.
// 'smallest': start from the beginning
$topicConf->set('auto.offset.reset', 'largest');
$topic = $consumer->newTopic("Topic_Name", $topicConf);
// Start consuming partition 0
$metaData = $consumer->getMetadata(false, $topic, 1000);
 public function toRdKafkaTopicConfig()
 {
     $topicConf = new \RdKafka\TopicConf();
     $topicConf->set("request.required.acks", $this->requestRequiredAcks);
     $topicConf->set("enforce.isr.cnt", $this->enforceIsrCnt);
     $topicConf->set("request.timeout.ms", $this->requestTimeoutMs);
     $topicConf->set("message.timeout.ms", $this->messageTimeoutMs);
     return $topicConf;
 }
Beispiel #7
0
 private function setProducerTopic($mode = '')
 {
     if (!$this->producer_topic) {
         if (!isset($this->producer)) {
             echo "error : produce wrong identity";
             exit;
         }
         $rd_topic_conf = new RdKafka\TopicConf();
         $rd_topic_conf->set("request.required.acks", self::PRODUCER_REQUEST_ACK);
         if ($mode == 'consistent') {
             $rd_topic_conf->setPartitioner(RD_KAFKA_MSG_PARTITIONER_CONSISTENT);
         }
         //print_r($rd_topic_conf->dump());exit;
         $this->producer_topic = $this->producer->newTopic($this->topic_name, $rd_topic_conf);
     }
 }