/**
* 测试读取BigpieConf
*/
public function testBigpieConf()
{
$conf_dir = './conf';
$conf_file = 'php-api.conf';
$content = new BigpipeConf();
// 测试1 成功分支
$this->assertTrue(bigpipe_load_file($conf_dir, $conf_file, $content));
// 测试2 错误分支
$conf_arr = config_load($conf_dir, $conf_file);
$this->assertTrue(false != $conf_arr);
// 测试2.1 无stomp
$stomp = $conf_arr['stomp'];
$conf_arr['stomp'] = null;
$this->assertFalse($content->load($conf_arr));
// 测试2.2 load stomp节点失败
// 无peek_timeo
$conf_arr['stomp'] = $stomp;
$peek_timeo = $conf_arr['stomp']['peek_timeo'];
unset($conf_arr['stomp']['peek_timeo']);
$this->assertFalse($content->load($conf_arr));
$conf_arr['stomp']['peek_timeo'] = $peek_timeo;
// 无connection
$connection = $conf_arr['stomp']['connection'];
unset($conf_arr['stomp']['connection']);
$this->assertFalse($content->load($conf_arr));
$conf_arr['stomp']['connection'] = $connection;
// 测试2.3 无meta_agent
$meta_agent = $conf_arr['meta_agent'];
$conf_arr['meta_agent'] = null;
$this->assertFalse($content->load($conf_arr));
$conf_arr['meta_agent'] = $meta_agent;
// 测试2.4 load meta agent节点失败
// meta 没设
$meta = $conf_arr['meta_agent']['meta'];
unset($conf_arr['meta_agent']['meta']);
$this->assertFalse($content->load($conf_arr));
$conf_arr['meta_agent']['meta'] = $meta;
// agent 没设
$agent = $conf_arr['meta_agent']['agent'];
unset($conf_arr['meta_agent']['agent']);
$this->assertFalse($content->load($conf_arr));
$conf_arr['meta_agent']['agent'] = $agent;
// connection 没设
$connection = $conf_arr['meta_agent']['connection'];
unset($conf_arr['meta_agent']['connection']);
$this->assertFalse($content->load($conf_arr));
$conf_arr['meta_agent']['connection'] = $connection;
}
$log_conf = new BigpipeLogConf();
$log_conf->file = 'publisher.php';
if (BigpipeLog::init($log_conf)) {
echo "[success] ==> [open publisher log]\n";
} else {
echo "[failure] ==> [open publisher log]\n";
print_r($log_conf);
die;
}
// 以下configure文件加载方式是通过php-configure，读入ub-style的configure文件
// ub-style的configure文件与c-api的confige文件相似。
// BigpipeConf还可以通过load接口接收array形式的configure输入。
$conf = new BigpipeConf();
$conf_dir = './conf';
$conf_file = './php-api.conf';
if (false === bigpipe_load_file($conf_dir, $conf_file, $conf)) {
echo "[failure][when load configure]\n";
die;
}
// 发布参数包括
// pipe name, token, paritioner和BigpipeConf
$pipe_name = 'yzytest1';
$token = 'token';
$partitioner = new TestPubPartitioner();
// 定义了包中最多包涵的message条数
$max_package_message_count = 300;
// 定义测试中发包条数
$max_package_count = 50000;
echo "[Publish {$max_package_count} packages]\n";
$pub_file = fopen('./pub.txt', 'w+');
// 以下描述了一个bigpipe publisher的发布流程
function run_tuning_sub($seq, $pid, $sp, $max_count)
{
// 初始化log配置
$log_conf = new BigpipeLogConf();
$log_conf->file = sprintf('subscribe-%u-%u.php', $seq, $pid);
if (BigpipeLog::init($log_conf)) {
//echo "[Success] [open subscribe log]\n\n";
//print_r($log_conf);
//echo '\n';
} else {
echo '[Failure] [open subscribe log]\\n';
print_r($log_conf);
echo '\\n';
}
// subscriber的configure
$conf = new BigpipeConf();
$conf_dir = './conf';
$conf_file = './for-tuning.conf';
if (false === bigpipe_load_file($conf_dir, $conf_file, $conf)) {
echo "[failure][when load configure]\n";
return;
}
// 订阅参数
$pipe_name = 'pipe2';
$pipelet_id = $pid;
//
$start_point = $sp;
$token = 'token';
$peek_time_ms = 200;
$suffix_str = sprintf('-%u-%u', $seq, $pid);
$file_sub_succ = fopen('./sub_succ-' . $suffix_str . '.csv', 'w+');
$file_sub_wait = fopen('./sub_wait-' . $suffix_str . '.csv', 'w+');
$file_sub_fail = fopen('./sub_fail-' . $suffix_str . '.csv', 'w+');
$sub = new BigpipeSubscriber();
if ($sub->init($pipe_name, $token, $pipelet_id, $start_point, $conf)) {
echo '[Success][init subscriber]\\n';
$count = 0;
$failure = 0;
$success = 0;
$wait_cnt = 0;
$run_time_str = null;
$total_start = BigpipeUtilities::get_time_us();
while ($success < $max_count && $wait_cnt < $max_count && $failure < $max_count) {
$pret = $sub->peek($peek_time_ms);
if (BigpipeErrorCode::READABLE == $pret) {
$msg = $sub->receive();
if (false === $msg) {
$failure++;
} else {
if (true === $msg->is_last_msg) {
$end_time = BigpipeUtilities::get_time_us();
if (0 == strncmp('timestamp:', $msg->content, 10)) {
$t_str = substr($msg->content, 10);
$run_time_str = sprintf("%u,%u,%s,%u\n", $pipelet_id, $msg->msg_id, $t_str, $end_time);
fwrite($file_sub_succ, $run_time_str);
}
}
$success++;
}
} else {
if (BigpipeErrorCode::UNREADABLE == $pret) {
$wait_cnt++;
} else {
$failure++;
}
}
$count++;
}
$total_end = BigpipeUtilities::get_time_us();
printf("[Leave][seq:%u][count:%u][succ:%u][fail:%u][wait:%u]\n\n", $seq, $count, $success, $failure, $wait_cnt);
$avg = (double) ($total_end - $total_start) / (double) (1000 * $count);
printf("[Subscriber][seq:%u][avg_time:%f(ms)]\n", $seq, $avg);
} else {
echo '[Failure][init subscribe]\\n';
}
$sub->uninit();
BigpipeLog::close();
}
function run_tuning_pub($seq, $pid, $max_cnt, $one_msg)
{
// 初始化log配置
$log_conf = new BigpipeLogConf();
$log_conf->file = sprintf('publisher-%u-%u.log', $seq, $pid);
if (BigpipeLog::init($log_conf)) {
//echo '[Success] [open meta agent log]\n';
//print_r($log_conf);
//echo '\n';
} else {
echo "[Failure] [open meta agent log]\n";
print_r($log_conf);
echo "\n";
return;
}
// subscriber的configure
$conf = new BigpipeConf();
$conf_dir = './conf';
$conf_file = './for-tuning.conf';
if (false === bigpipe_load_file($conf_dir, $conf_file, $conf)) {
echo "[failure][when load configure]\n";
return;
}
// 发布参数
$pipe_name = 'pipe2';
//$pipelet_id = 1; // 加1在哪里做好呢？
$token = 'token';
$partitioner = new TestPubPartitioner($pid);
// 生成一个1K左右的包
$max_msg_count = 12;
// 500k
$msg = null;
$the_one = sprintf("[case:%u]\n", $seq);
$uid = BigpipeUtilities::get_uid();
$msg_package = new BigpipeMessagePackage();
$msg_count = 0;
while ($msg_count++ < $max_msg_count) {
$msg = sprintf("[php-api-test][bigpipe_comlog_pvt_mm][uid:%s][package:%u][seq:%u]\n", $uid, 0, $msg_count);
if (true === $one_msg) {
$the_one .= $msg;
} else {
if (!$msg_package->push($msg)) {
echo "[fail to pack message]{$msg}\n";
break;
// 退出
}
}
}
// end of add message to package
if (true == $one_msg) {
if (!$msg_package->push($the_one)) {
echo "[fail to pack message]{$the_one}\n";
return;
// 退出
}
}
// 定义测试中发包条数
$max_package_count = $max_cnt;
echo "\n\n[Publish {$max_package_count} packages]\n\n";
$pub_file_name = sprintf('./pub-timer-%u-%u.csv', $seq, $pid);
$pub_file = fopen($pub_file_name, 'w+');
$pub = new BigpipePublisher();
$stat = array(0 => 0, 1 => 0, 2 => 0);
if ($pub->init($pipe_name, $token, $partitioner, $conf)) {
echo "[Success][init publisher]\n\n";
$count = 0;
$succeed = 0;
$total_start = BigpipeUtilities::get_time_us();
$is_first = true;
printf("[case:%u][start:%u]\n\n", $seq, $total_start);
while ($count < $max_package_count) {
$count++;
//$max_msg_count = rand(1, $max_package_message_count);
//echo "[Pack $max_msg_count messages to package <$count>]\n";
// if ($msg_count != $max_msg_count + 1)
// {
// echo "[expected:$max_msg_count][actual:$msg_count]\n";
// continue; // 打包失败，退出
// }
$start_time = BigpipeUtilities::get_time_us();
$msg_package->push(sprintf('timestamp:%u', $start_time));
// 加入时间戳
$pub_result = $pub->send($msg_package);
$end_time = BigpipeUtilities::get_time_us();
if (false === $pub_result) {
//echo "[fail to publish message package][count:$count]\n";
//break; // 出错便停止
} else {
// write result to file
//$ret_str = sprintf("%d,%u,%u,%u\n",
// $pub_result->error_no,
// $pub_result->pipelet_id,
// $pub_result->pipelet_msg_id,
// $pub_result->session_msg_id);
//fwrite($pub_file, $ret_str);
if (true == $is_first) {
printf('[%u][%u]\\n', $pub_result->pipelet_id, $pub_result->pipelet_msg_id);
$is_first = false;
}
$succeed++;
}
$msg_package->pop();
// 弹出时间戳
$t = (double) ($end_time - $start_time) / 1000;
$t_str = sprintf("%u\n", $t);
fwrite($pub_file, $t_str);
}
$total_end = BigpipeUtilities::get_time_us();
echo "[Publisher][seq:{$seq}][count:{$count}][success:{$succeed}]====\n";
$avg = (double) ($total_end - $total_start) / (double) (1000 * $max_package_count);
printf("\n[Publisher][case:%u][avg_time:%f(ms)]\n", $seq, $avg);
} else {
echo '[Failure][init publisher]\\n';
}
$pub->uninit();
BigpipeLog::close();
}
public function execute()
{
$minute_partition = parent::$arrAppConf['minute_partition'];
$bigpipeLogConf = new BigpipeLogConf();
$bigpipeLogConf->file = 'subscribe.php';
if (BigpipeLog::init($bigpipeLogConf)) {
echo "[Success] [open subscribe log]\n";
// print_r($bigpipeLogConf);
} else {
echo '[Failure] [open subscribe log]\\n';
print_r($bigpipeLogConf);
echo "\n";
}
$conf = new BigpipeConf();
$conf_dir = ETL_CONF . parent::$app;
$conf_file = './php-api.conf';
if (false === bigpipe_load_file($conf_dir, $conf_file, $conf)) {
echo "[failure][when load configure]\n";
exit;
}
$pipeName = parent::$arrAppConf['pipe_name'];
$pipelet = parent::$pipelet;
$token = parent::$arrAppConf['token'];
$peekTimeMs = parent::$arrAppConf['peek_time_ms'];
$messageIdFile = ETL_DATA . parent::$app . "/message_pipelet_" . parent::$pipelet . ".flag";
$startPoint = file_get_contents($messageIdFile);
if ($startPoint == false) {
$startPoint = -1;
} else {
if (trim($startPoint) != "-2") {
$startPoint = intval(trim($startPoint)) + 1;
}
}
$sub = new BigpipeSubscriber();
if ($sub->init($pipeName, $token, $pipelet, $startPoint, $conf)) {
$lastPartition = 0;
while (true) {
$pret = $sub->peek($peekTimeMs);
if (BigpipeErrorCode::READABLE == $pret) {
$msg = $sub->receive();
if (false == $msg) {
echo "[Receive][error]\n";
continue;
} else {
// echo $msg->msg_id . "\t" . $msg->seq_id. "\n" ;
$arrMsgContent = mc_pack_pack2array($msg->content);
//cut
$arrContent = cut($arrMsgContent['body']);
//map
$objEvent = new Event();
foreach (parent::$arrMap as $map) {
$arrParams = array();
foreach ($map['in'] as $in) {
$arrParams['in'][$in] = $arrContent[$in];
}
foreach ($map['out'] as $out) {
$arrParams['out'][$out] = null;
}
$hook_func_callback = $map['hook'];
$res = call_user_func($hook_func_callback, &$arrParams);
if ($res === true) {
foreach ($arrParams['out'] as $key => $value) {
$objEvent->arrEvent[$key] = $value;
}
foreach ($arrParams['in'] as $key => $value) {
unset($arrContent[$key]);
}
}
}
$objEvent->arrEvent['event_loginfo'] = common_loginfo($arrContent);
//write to data
$date = sprintf("%s%s%s", $objEvent->arrEvent['event_year'], $objEvent->arrEvent['event_month'], $objEvent->arrEvent['event_dayofmonth']);
list($hour, $minute, $second) = explode(":", $objEvent->arrEvent['event_time'], 3);
$partition = $date . $hour . sprintf("%02d", intval($minute / parent::$arrAppConf['minute_partition']) * parent::$arrAppConf['minute_partition']);
$dataFile = ETL_DATA . parent::$app . "/" . parent::$app . "_" . parent::$pipelet . "_{$partition}";
$strEvent = "";
foreach ($objEvent->arrEvent as $item) {
$item = str_replace("", "", $item);
$item = str_replace("\n", "", $item);
if ($strEvent == "") {
$strEvent .= $item;
} else {
$strEvent .= "" . $item;
}
}
$strEvent .= "\n";
file_put_contents($dataFile, $strEvent, FILE_APPEND | LOCK_EX);
$fpMessageId = fopen($messageIdFile, "w");
fwrite($fpMessageId, $msg->msg_id);
fclose($fpMessageId);
}
} else {
if (BigpipeErrorCode::UNREADABLE == $pret) {
sleep(30);
} else {
echo "[Peek][Error][ret:{$pret}]\n";
}
}
}
} else {
echo '[Failure][init subscribe]\\n';
}
$sub->uninit();
BigpipeLog::close();
}
