Exemplo n.º 1
0
 /**
  * Rewind the iterator
  *
  * @return void
  */
 public function rewind()
 {
     $this->iterators = array();
     foreach ($this->topicRegistry->partitions($this->topic) as $broker => $nPartitions) {
         for ($partition = 0; $partition < $nPartitions; ++$partition) {
             list($host, $port) = explode(':', $this->brokerRegistry->address($broker));
             $offset = $this->offsetRegistry->offset($this->topic, $broker, $partition);
             $this->iterators[] = (object) array('consumer' => null, 'host' => $host, 'port' => $port, 'broker' => $broker, 'partition' => $partition, 'offset' => $offset, 'uncommittedOffset' => 0, 'messages' => null);
             ++$this->nIterators;
         }
     }
     if (0 == count($this->iterators)) {
         throw new Kafka_Exception_InvalidTopic('Cannot find topic ' . $this->topic);
     }
     // get a random broker/partition every time
     $this->shuffle();
 }
 // pid dir
 $pid_dir = __DIR__ . '/log/import_workers';
 if (!file_exists($pid_dir)) {
     mkdir($pid_dir);
 }
 // save log pid
 file_put_contents($pid_dir . '/log.pid', posix_getpid());
 // zookeeper address (one or more, separated by commas)
 $zkaddress = '192.168.1.211:2121,192.168.1.212:2121,192.168.1.213:2121';
 // kafka topic to consume from
 $topic = 'topic_name';
 // start and monitor kafka by host and port
 while (true) {
     try {
         $zookeeper = new Zookeeper($zkaddress);
         $topicRegistry = new Kafka_Registry_Topic($zookeeper);
         $brokerRegistry = new Kafka_Registry_Broker($zookeeper);
         // all hosts and ports
         foreach ($topicRegistry->partitions($topic) as $broker => $nPartitions) {
             // get host and port
             list($host, $port) = explode(':', $brokerRegistry->address($broker));
             $lockfile = "{$pid_dir}/kafka_{$host}:{$port}.pid";
             $pid = @file_get_contents($lockfile);
             if ($pid === false || posix_getsid($pid) === false) {
                 file_put_contents(__DIR__ . '/log/kafka_error_' . date('Ymd') . '.log', '[' . date('H:i:s') . ']' . "Process({$host}:{$port}) has died! restarting...\n", FILE_APPEND);
                 system("/path/to/php/bin/php " . __DIR__ . "/import.php step kafka host {$host} port {$port} > /dev/null 2>&1 &");
             }
         }
     } catch (Exception $exception) {
         file_put_contents(__DIR__ . '/log/kafka_error_' . date('Ymd') . '.log', '[' . date('H:i:s') . ']' . 'EXCEPTION: ' . $exception->getMessage() . "\n", FILE_APPEND);
     }