/** * 测试读取BigpieConf */ public function testBigpieConf() { $conf_dir = './conf'; $conf_file = 'php-api.conf'; $content = new BigpipeConf(); // 测试1 成功分支 $this->assertTrue(bigpipe_load_file($conf_dir, $conf_file, $content)); // 测试2 错误分支 $conf_arr = config_load($conf_dir, $conf_file); $this->assertTrue(false != $conf_arr); // 测试2.1 无stomp $stomp = $conf_arr['stomp']; $conf_arr['stomp'] = null; $this->assertFalse($content->load($conf_arr)); // 测试2.2 load stomp节点失败 // 无peek_timeo $conf_arr['stomp'] = $stomp; $peek_timeo = $conf_arr['stomp']['peek_timeo']; unset($conf_arr['stomp']['peek_timeo']); $this->assertFalse($content->load($conf_arr)); $conf_arr['stomp']['peek_timeo'] = $peek_timeo; // 无connection $connection = $conf_arr['stomp']['connection']; unset($conf_arr['stomp']['connection']); $this->assertFalse($content->load($conf_arr)); $conf_arr['stomp']['connection'] = $connection; // 测试2.3 无meta_agent $meta_agent = $conf_arr['meta_agent']; $conf_arr['meta_agent'] = null; $this->assertFalse($content->load($conf_arr)); $conf_arr['meta_agent'] = $meta_agent; // 测试2.4 load meta agent节点失败 // meta 没设 $meta = $conf_arr['meta_agent']['meta']; unset($conf_arr['meta_agent']['meta']); $this->assertFalse($content->load($conf_arr)); $conf_arr['meta_agent']['meta'] = $meta; // agent 没设 $agent = $conf_arr['meta_agent']['agent']; unset($conf_arr['meta_agent']['agent']); $this->assertFalse($content->load($conf_arr)); $conf_arr['meta_agent']['agent'] = $agent; // connection 没设 $connection = $conf_arr['meta_agent']['connection']; unset($conf_arr['meta_agent']['connection']); $this->assertFalse($content->load($conf_arr)); $conf_arr['meta_agent']['connection'] = $connection; }
$log_conf = new BigpipeLogConf(); $log_conf->file = 'publisher.php'; if (BigpipeLog::init($log_conf)) { echo "[success] ==> [open publisher log]\n"; } else { echo "[failure] ==> [open publisher log]\n"; print_r($log_conf); die; } // 以下configure文件加载方式是通过php-configure,读入ub-style的configure文件 // ub-style的configure文件与c-api的confige文件相似。 // BigpipeConf还可以通过load接口接收array形式的configure输入。 $conf = new BigpipeConf(); $conf_dir = './conf'; $conf_file = './php-api.conf'; if (false === bigpipe_load_file($conf_dir, $conf_file, $conf)) { echo "[failure][when load configure]\n"; die; } // 发布参数包括 // pipe name, token, paritioner和BigpipeConf $pipe_name = 'yzytest1'; $token = 'token'; $partitioner = new TestPubPartitioner(); // 定义了包中最多包涵的message条数 $max_package_message_count = 300; // 定义测试中发包条数 $max_package_count = 50000; echo "[Publish {$max_package_count} packages]\n"; $pub_file = fopen('./pub.txt', 'w+'); // 以下描述了一个bigpipe publisher的发布流程
function run_tuning_sub($seq, $pid, $sp, $max_count) { // 初始化log配置 $log_conf = new BigpipeLogConf(); $log_conf->file = sprintf('subscribe-%u-%u.php', $seq, $pid); if (BigpipeLog::init($log_conf)) { //echo "[Success] [open subscribe log]\n\n"; //print_r($log_conf); //echo '\n'; } else { echo '[Failure] [open subscribe log]\\n'; print_r($log_conf); echo '\\n'; } // subscriber的configure $conf = new BigpipeConf(); $conf_dir = './conf'; $conf_file = './for-tuning.conf'; if (false === bigpipe_load_file($conf_dir, $conf_file, $conf)) { echo "[failure][when load configure]\n"; return; } // 订阅参数 $pipe_name = 'pipe2'; $pipelet_id = $pid; // $start_point = $sp; $token = 'token'; $peek_time_ms = 200; $suffix_str = sprintf('-%u-%u', $seq, $pid); $file_sub_succ = fopen('./sub_succ-' . $suffix_str . '.csv', 'w+'); $file_sub_wait = fopen('./sub_wait-' . $suffix_str . '.csv', 'w+'); $file_sub_fail = fopen('./sub_fail-' . $suffix_str . '.csv', 'w+'); $sub = new BigpipeSubscriber(); if ($sub->init($pipe_name, $token, $pipelet_id, $start_point, $conf)) { echo '[Success][init subscriber]\\n'; $count = 0; $failure = 0; $success = 0; $wait_cnt = 0; $run_time_str = null; $total_start = BigpipeUtilities::get_time_us(); while ($success < $max_count && $wait_cnt < $max_count && $failure < $max_count) { $pret = $sub->peek($peek_time_ms); if (BigpipeErrorCode::READABLE == $pret) { $msg = $sub->receive(); if (false === $msg) { $failure++; } else { if (true === $msg->is_last_msg) { $end_time = BigpipeUtilities::get_time_us(); if (0 == strncmp('timestamp:', $msg->content, 10)) { $t_str = substr($msg->content, 10); $run_time_str = sprintf("%u,%u,%s,%u\n", $pipelet_id, $msg->msg_id, $t_str, $end_time); fwrite($file_sub_succ, $run_time_str); } } $success++; } } else { if (BigpipeErrorCode::UNREADABLE == $pret) { $wait_cnt++; } else { $failure++; } } $count++; } $total_end = BigpipeUtilities::get_time_us(); printf("[Leave][seq:%u][count:%u][succ:%u][fail:%u][wait:%u]\n\n", $seq, $count, $success, $failure, $wait_cnt); $avg = (double) ($total_end - $total_start) / (double) (1000 * $count); printf("[Subscriber][seq:%u][avg_time:%f(ms)]\n", $seq, $avg); } else { echo '[Failure][init subscribe]\\n'; } $sub->uninit(); BigpipeLog::close(); }
function run_tuning_pub($seq, $pid, $max_cnt, $one_msg) { // 初始化log配置 $log_conf = new BigpipeLogConf(); $log_conf->file = sprintf('publisher-%u-%u.log', $seq, $pid); if (BigpipeLog::init($log_conf)) { //echo '[Success] [open meta agent log]\n'; //print_r($log_conf); //echo '\n'; } else { echo "[Failure] [open meta agent log]\n"; print_r($log_conf); echo "\n"; return; } // subscriber的configure $conf = new BigpipeConf(); $conf_dir = './conf'; $conf_file = './for-tuning.conf'; if (false === bigpipe_load_file($conf_dir, $conf_file, $conf)) { echo "[failure][when load configure]\n"; return; } // 发布参数 $pipe_name = 'pipe2'; //$pipelet_id = 1; // 加1在哪里做好呢? $token = 'token'; $partitioner = new TestPubPartitioner($pid); // 生成一个1K左右的包 $max_msg_count = 12; // 500k $msg = null; $the_one = sprintf("[case:%u]\n", $seq); $uid = BigpipeUtilities::get_uid(); $msg_package = new BigpipeMessagePackage(); $msg_count = 0; while ($msg_count++ < $max_msg_count) { $msg = sprintf("[php-api-test][bigpipe_comlog_pvt_mm][uid:%s][package:%u][seq:%u]\n", $uid, 0, $msg_count); if (true === $one_msg) { $the_one .= $msg; } else { if (!$msg_package->push($msg)) { echo "[fail to pack message]{$msg}\n"; break; // 退出 } } } // end of add message to package if (true == $one_msg) { if (!$msg_package->push($the_one)) { echo "[fail to pack message]{$the_one}\n"; return; // 退出 } } // 定义测试中发包条数 $max_package_count = $max_cnt; echo "\n\n[Publish {$max_package_count} packages]\n\n"; $pub_file_name = sprintf('./pub-timer-%u-%u.csv', $seq, $pid); $pub_file = fopen($pub_file_name, 'w+'); $pub = new BigpipePublisher(); $stat = array(0 => 0, 1 => 0, 2 => 0); if ($pub->init($pipe_name, $token, $partitioner, $conf)) { echo "[Success][init publisher]\n\n"; $count = 0; $succeed = 0; $total_start = BigpipeUtilities::get_time_us(); $is_first = true; printf("[case:%u][start:%u]\n\n", $seq, $total_start); while ($count < $max_package_count) { $count++; //$max_msg_count = rand(1, $max_package_message_count); //echo "[Pack $max_msg_count messages to package <$count>]\n"; // if ($msg_count != $max_msg_count + 1) // { // echo "[expected:$max_msg_count][actual:$msg_count]\n"; // continue; // 打包失败,退出 // } $start_time = BigpipeUtilities::get_time_us(); $msg_package->push(sprintf('timestamp:%u', $start_time)); // 加入时间戳 $pub_result = $pub->send($msg_package); $end_time = BigpipeUtilities::get_time_us(); if (false === $pub_result) { //echo "[fail to publish message package][count:$count]\n"; //break; // 出错便停止 } else { // write result to file //$ret_str = sprintf("%d,%u,%u,%u\n", // $pub_result->error_no, // $pub_result->pipelet_id, // $pub_result->pipelet_msg_id, // $pub_result->session_msg_id); //fwrite($pub_file, $ret_str); if (true == $is_first) { printf('[%u][%u]\\n', $pub_result->pipelet_id, $pub_result->pipelet_msg_id); $is_first = false; } $succeed++; } $msg_package->pop(); // 弹出时间戳 $t = (double) ($end_time - $start_time) / 1000; $t_str = sprintf("%u\n", $t); fwrite($pub_file, $t_str); } $total_end = BigpipeUtilities::get_time_us(); echo "[Publisher][seq:{$seq}][count:{$count}][success:{$succeed}]====\n"; $avg = (double) ($total_end - $total_start) / (double) (1000 * $max_package_count); printf("\n[Publisher][case:%u][avg_time:%f(ms)]\n", $seq, $avg); } else { echo '[Failure][init publisher]\\n'; } $pub->uninit(); BigpipeLog::close(); }
public function execute() { $minute_partition = parent::$arrAppConf['minute_partition']; $bigpipeLogConf = new BigpipeLogConf(); $bigpipeLogConf->file = 'subscribe.php'; if (BigpipeLog::init($bigpipeLogConf)) { echo "[Success] [open subscribe log]\n"; // print_r($bigpipeLogConf); } else { echo '[Failure] [open subscribe log]\\n'; print_r($bigpipeLogConf); echo "\n"; } $conf = new BigpipeConf(); $conf_dir = ETL_CONF . parent::$app; $conf_file = './php-api.conf'; if (false === bigpipe_load_file($conf_dir, $conf_file, $conf)) { echo "[failure][when load configure]\n"; exit; } $pipeName = parent::$arrAppConf['pipe_name']; $pipelet = parent::$pipelet; $token = parent::$arrAppConf['token']; $peekTimeMs = parent::$arrAppConf['peek_time_ms']; $messageIdFile = ETL_DATA . parent::$app . "/message_pipelet_" . parent::$pipelet . ".flag"; $startPoint = file_get_contents($messageIdFile); if ($startPoint == false) { $startPoint = -1; } else { if (trim($startPoint) != "-2") { $startPoint = intval(trim($startPoint)) + 1; } } $sub = new BigpipeSubscriber(); if ($sub->init($pipeName, $token, $pipelet, $startPoint, $conf)) { $lastPartition = 0; while (true) { $pret = $sub->peek($peekTimeMs); if (BigpipeErrorCode::READABLE == $pret) { $msg = $sub->receive(); if (false == $msg) { echo "[Receive][error]\n"; continue; } else { // echo $msg->msg_id . "\t" . $msg->seq_id. "\n" ; $arrMsgContent = mc_pack_pack2array($msg->content); //cut $arrContent = cut($arrMsgContent['body']); //map $objEvent = new Event(); foreach (parent::$arrMap as $map) { $arrParams = array(); foreach ($map['in'] as $in) { $arrParams['in'][$in] = $arrContent[$in]; } foreach ($map['out'] as $out) { $arrParams['out'][$out] = null; } $hook_func_callback = $map['hook']; $res = call_user_func($hook_func_callback, &$arrParams); if ($res === true) { foreach ($arrParams['out'] as $key => $value) { $objEvent->arrEvent[$key] = $value; } foreach ($arrParams['in'] as $key => $value) { unset($arrContent[$key]); } } } $objEvent->arrEvent['event_loginfo'] = common_loginfo($arrContent); //write to data $date = sprintf("%s%s%s", $objEvent->arrEvent['event_year'], $objEvent->arrEvent['event_month'], $objEvent->arrEvent['event_dayofmonth']); list($hour, $minute, $second) = explode(":", $objEvent->arrEvent['event_time'], 3); $partition = $date . $hour . sprintf("%02d", intval($minute / parent::$arrAppConf['minute_partition']) * parent::$arrAppConf['minute_partition']); $dataFile = ETL_DATA . parent::$app . "/" . parent::$app . "_" . parent::$pipelet . "_{$partition}"; $strEvent = ""; foreach ($objEvent->arrEvent as $item) { $item = str_replace("", "", $item); $item = str_replace("\n", "", $item); if ($strEvent == "") { $strEvent .= $item; } else { $strEvent .= "" . $item; } } $strEvent .= "\n"; file_put_contents($dataFile, $strEvent, FILE_APPEND | LOCK_EX); $fpMessageId = fopen($messageIdFile, "w"); fwrite($fpMessageId, $msg->msg_id); fclose($fpMessageId); } } else { if (BigpipeErrorCode::UNREADABLE == $pret) { sleep(30); } else { echo "[Peek][Error][ret:{$pret}]\n"; } } } } else { echo '[Failure][init subscribe]\\n'; } $sub->uninit(); BigpipeLog::close(); }