コード例 #1
0
function run_tuning_sub($seq, $pid, $sp, $max_count)
{
    // 初始化log配置
    $log_conf = new BigpipeLogConf();
    $log_conf->file = sprintf('subscribe-%u-%u.php', $seq, $pid);
    if (BigpipeLog::init($log_conf)) {
        //echo "[Success] [open subscribe log]\n\n";
        //print_r($log_conf);
        //echo '\n';
    } else {
        echo '[Failure] [open subscribe log]\\n';
        print_r($log_conf);
        echo '\\n';
    }
    // subscriber的configure
    $conf = new BigpipeConf();
    $conf_dir = './conf';
    $conf_file = './for-tuning.conf';
    if (false === bigpipe_load_file($conf_dir, $conf_file, $conf)) {
        echo "[failure][when load configure]\n";
        return;
    }
    // 订阅参数
    $pipe_name = 'pipe2';
    $pipelet_id = $pid;
    //
    $start_point = $sp;
    $token = 'token';
    $peek_time_ms = 200;
    $suffix_str = sprintf('-%u-%u', $seq, $pid);
    $file_sub_succ = fopen('./sub_succ-' . $suffix_str . '.csv', 'w+');
    $file_sub_wait = fopen('./sub_wait-' . $suffix_str . '.csv', 'w+');
    $file_sub_fail = fopen('./sub_fail-' . $suffix_str . '.csv', 'w+');
    $sub = new BigpipeSubscriber();
    if ($sub->init($pipe_name, $token, $pipelet_id, $start_point, $conf)) {
        echo '[Success][init subscriber]\\n';
        $count = 0;
        $failure = 0;
        $success = 0;
        $wait_cnt = 0;
        $run_time_str = null;
        $total_start = BigpipeUtilities::get_time_us();
        while ($success < $max_count && $wait_cnt < $max_count && $failure < $max_count) {
            $pret = $sub->peek($peek_time_ms);
            if (BigpipeErrorCode::READABLE == $pret) {
                $msg = $sub->receive();
                if (false === $msg) {
                    $failure++;
                } else {
                    if (true === $msg->is_last_msg) {
                        $end_time = BigpipeUtilities::get_time_us();
                        if (0 == strncmp('timestamp:', $msg->content, 10)) {
                            $t_str = substr($msg->content, 10);
                            $run_time_str = sprintf("%u,%u,%s,%u\n", $pipelet_id, $msg->msg_id, $t_str, $end_time);
                            fwrite($file_sub_succ, $run_time_str);
                        }
                    }
                    $success++;
                }
            } else {
                if (BigpipeErrorCode::UNREADABLE == $pret) {
                    $wait_cnt++;
                } else {
                    $failure++;
                }
            }
            $count++;
        }
        $total_end = BigpipeUtilities::get_time_us();
        printf("[Leave][seq:%u][count:%u][succ:%u][fail:%u][wait:%u]\n\n", $seq, $count, $success, $failure, $wait_cnt);
        $avg = (double) ($total_end - $total_start) / (double) (1000 * $count);
        printf("[Subscriber][seq:%u][avg_time:%f(ms)]\n", $seq, $avg);
    } else {
        echo '[Failure][init subscribe]\\n';
    }
    $sub->uninit();
    BigpipeLog::close();
}
コード例 #2
0
$sub = new BigpipeSubscriber();
// 订阅流程从用户调用init接口成功后开始
if ($sub->init($pipe_name, $token, $pipelet_id, $start_point, $conf)) {
    echo '[Success][init subscriber]\\n';
    $loop_end = false;
    $count = 0;
    while (false === $loop_end) {
        // php的peek包括一下几步
        // 当还未订阅消息时,主动更新broker,向broker订阅消息
        // 订阅消息后,监听端口,等待消息到达
        $pret = $sub->peek($peek_time_ms);
        if (BigpipeErrorCode::READABLE == $pret) {
            // peek返回readable后,调用receive接收数据
            // 与send是发布message package不同,receive会主动解message package,
            // 每次receive成功,返回一条消息
            $msg = $sub->receive();
            if (false === $msg) {
                echo '[Receive][error]\\n';
                $loop_end = true;
                // 当receive发生错误时退出循环
            } else {
                $msg_str = sprintf("[begin]===\n[msg_id][%u]\n[seq_id][%u]\n[msg][%s]\n[end]===\n", $msg->msg_id, $msg->seq_id, $msg->content);
                fwrite($file, $msg_str);
                echo $msg->msg_id . "\t" . $msg->seq_id . "\n";
                $tmp = mc_pack_pack2array($msg->content);
                //				var_dump( $tmp );
                //				echo "=============================\n" ;
            }
            $count = 0;
        } else {
            if (BigpipeErrorCode::UNREADABLE == $pret) {
コード例 #3
0
ファイル: Data.php プロジェクト: focus-andy/Near-RealTime-ETL
 public function execute()
 {
     $minute_partition = parent::$arrAppConf['minute_partition'];
     $bigpipeLogConf = new BigpipeLogConf();
     $bigpipeLogConf->file = 'subscribe.php';
     if (BigpipeLog::init($bigpipeLogConf)) {
         echo "[Success] [open subscribe log]\n";
         //			print_r($bigpipeLogConf);
     } else {
         echo '[Failure] [open subscribe log]\\n';
         print_r($bigpipeLogConf);
         echo "\n";
     }
     $conf = new BigpipeConf();
     $conf_dir = ETL_CONF . parent::$app;
     $conf_file = './php-api.conf';
     if (false === bigpipe_load_file($conf_dir, $conf_file, $conf)) {
         echo "[failure][when load configure]\n";
         exit;
     }
     $pipeName = parent::$arrAppConf['pipe_name'];
     $pipelet = parent::$pipelet;
     $token = parent::$arrAppConf['token'];
     $peekTimeMs = parent::$arrAppConf['peek_time_ms'];
     $messageIdFile = ETL_DATA . parent::$app . "/message_pipelet_" . parent::$pipelet . ".flag";
     $startPoint = file_get_contents($messageIdFile);
     if ($startPoint == false) {
         $startPoint = -1;
     } else {
         if (trim($startPoint) != "-2") {
             $startPoint = intval(trim($startPoint)) + 1;
         }
     }
     $sub = new BigpipeSubscriber();
     if ($sub->init($pipeName, $token, $pipelet, $startPoint, $conf)) {
         $lastPartition = 0;
         while (true) {
             $pret = $sub->peek($peekTimeMs);
             if (BigpipeErrorCode::READABLE == $pret) {
                 $msg = $sub->receive();
                 if (false == $msg) {
                     echo "[Receive][error]\n";
                     continue;
                 } else {
                     //						echo $msg->msg_id . "\t" . $msg->seq_id. "\n" ;
                     $arrMsgContent = mc_pack_pack2array($msg->content);
                     //cut
                     $arrContent = cut($arrMsgContent['body']);
                     //map
                     $objEvent = new Event();
                     foreach (parent::$arrMap as $map) {
                         $arrParams = array();
                         foreach ($map['in'] as $in) {
                             $arrParams['in'][$in] = $arrContent[$in];
                         }
                         foreach ($map['out'] as $out) {
                             $arrParams['out'][$out] = null;
                         }
                         $hook_func_callback = $map['hook'];
                         $res = call_user_func($hook_func_callback, &$arrParams);
                         if ($res === true) {
                             foreach ($arrParams['out'] as $key => $value) {
                                 $objEvent->arrEvent[$key] = $value;
                             }
                             foreach ($arrParams['in'] as $key => $value) {
                                 unset($arrContent[$key]);
                             }
                         }
                     }
                     $objEvent->arrEvent['event_loginfo'] = common_loginfo($arrContent);
                     //write to data
                     $date = sprintf("%s%s%s", $objEvent->arrEvent['event_year'], $objEvent->arrEvent['event_month'], $objEvent->arrEvent['event_dayofmonth']);
                     list($hour, $minute, $second) = explode(":", $objEvent->arrEvent['event_time'], 3);
                     $partition = $date . $hour . sprintf("%02d", intval($minute / parent::$arrAppConf['minute_partition']) * parent::$arrAppConf['minute_partition']);
                     $dataFile = ETL_DATA . parent::$app . "/" . parent::$app . "_" . parent::$pipelet . "_{$partition}";
                     $strEvent = "";
                     foreach ($objEvent->arrEvent as $item) {
                         $item = str_replace("", "", $item);
                         $item = str_replace("\n", "", $item);
                         if ($strEvent == "") {
                             $strEvent .= $item;
                         } else {
                             $strEvent .= "" . $item;
                         }
                     }
                     $strEvent .= "\n";
                     file_put_contents($dataFile, $strEvent, FILE_APPEND | LOCK_EX);
                     $fpMessageId = fopen($messageIdFile, "w");
                     fwrite($fpMessageId, $msg->msg_id);
                     fclose($fpMessageId);
                 }
             } else {
                 if (BigpipeErrorCode::UNREADABLE == $pret) {
                     sleep(30);
                 } else {
                     echo "[Peek][Error][ret:{$pret}]\n";
                 }
             }
         }
     } else {
         echo '[Failure][init subscribe]\\n';
     }
     $sub->uninit();
     BigpipeLog::close();
 }