// 发布/订阅参数 $pipe_name = 'pipe1'; $pipelet_id = 2; // 加1在哪里做好呢? $start_point = -1; // 取得可发布broker $broker = $adapter->get_pub_broker($pipe_name, $pipelet_id); if (false === $broker) { echo "[Failure][get pub info]\n"; } else { echo "[Success][get pub info]\n"; // 向broker发起连接请求 $pub_dest = array("socket_address" => $broker['ip'], "socket_port" => $broker['port'], "socket_timeout" => 300); $stomp->set_destination($pub_dest); $stomp->role = BStompRoleType::PUBLISHER; $stomp->topic_name = $broker['stripe']; $stomp->session_id = BigpipeUtilities::get_pipelet_name($pipe_name, $pipelet_id) . '_' . BigpipeUtilities::get_uid(); if ($stomp->connect()) { echo '[Success][connected on broker][ip:' . $broker['ip'] . '][port:' . $broker['port'] . ']\\n'; echo '[session message id][' . $stomp->session_message_id . ']\\n'; } echo "\n"; $ofs = fopen('pub.json', 'w+'); $oval = json_encode($broker); fwrite($ofs, $oval); fclose($ofs); } // 取得可订阅broker_group // todo 用户操作 $stomp->close(); $adapter->uninit();
/** * Éú³Éhead buffer */ private function _gen_head_buff() { $body = 'test body'; $body_len = strlen($body); $head = array(); $head['body_len'] = $body_len; $head['magic_num'] = BigpipeCommonDefine::NSHEAD_CHECKSUM_MAGICNUM; $head['reserved'] = BigpipeUtilities::adler32($body); $head_builder = new NsHead(); $head_data = $head_builder->build_nshead($head); $head['magic_num'] = 0; $no_magic_num = $head_builder->build_nshead($head); $head['body_len'] = 0; $bad_head = $head_builder->build_nshead($head); $ret_arr = array('good' => $head_data, 'bad' => $bad_head, 'no_magic' => $no_magic_num); return $ret_arr; }
die; } // 发布参数包括 // pipe name, token, paritioner和BigpipeConf $pipe_name = 'yzytest1'; $token = 'token'; $partitioner = new TestPubPartitioner(); // 定义了包中最多包涵的message条数 $max_package_message_count = 300; // 定义测试中发包条数 $max_package_count = 50000; echo "[Publish {$max_package_count} packages]\n"; $pub_file = fopen('./pub.txt', 'w+'); // 以下描述了一个bigpipe publisher的发布流程 // 该流程适合用于无状态发布(既每次都是init + send) $uid = BigpipeUtilities::get_uid(); $count = 0; $succeed = 0; while ($count < $max_package_count) { $pub = new BigpipePublisher(); if ($pub->init_ex($pipe_name, $token, $partitioner, $conf)) { // 发布流程从调用init接口成功后开始 echo '[Success][init publisher]\\n'; // 与c-api不同, // bigpipe php-api 发送的是一个message package // message package大小不能超过2MB // 用户声明BigpipeMessagePackage后, 调用push接口向package添加单条message $max_msg_count = rand(1, $max_package_message_count); echo "[Pack {$max_msg_count} messages to package <{$count}>]\n"; $msg = null; $msg_package = new BigpipeMessagePackage();
/** * 生成ack响应供测试程序使用 * @return ack响应集供test case选择 */ private function _gen_ack_response($subject) { $ack = new BStompAckFrame(); $ack->status = BStompIdAckType::TOPIC_ID; $ack->ack_type = BStompFrameType::ACK; $ack->session_message_id = TestUtilities::get_private_var($subject, '_session_msg_id') + 1; $this->assertFalse(false === $ack->session_message_id); $ack->topic_message_id = 369; $ack->global_message_id = 7659; $ack->delay_time = 0; $ack->destination = 'unknown'; $ack->receipt_id = 'fake-receipt-id'; $ack->store(); $good_ack = $ack->buffer(); $orig_smid = $ack->session_message_id; $ack->session_message_id = $orig_smid + 10; $ack->store(); $bad_session = $ack->buffer(); $ack->session_message_id = $orig_smid; $ack->receipt_id = BigpipeUtilities::get_uid(); $ack->store(); $bad_receipt = $ack->buffer(); $res_arr = array('good' => $good_ack, 'bad_session' => $bad_session, 'bad_receipt' => $bad_receipt, 'error_body' => 'error'); return $res_arr; }
function run_tuning_pub($seq, $pid, $max_cnt, $one_msg) { // 初始化log配置 $log_conf = new BigpipeLogConf(); $log_conf->file = sprintf('publisher-%u-%u.log', $seq, $pid); if (BigpipeLog::init($log_conf)) { //echo '[Success] [open meta agent log]\n'; //print_r($log_conf); //echo '\n'; } else { echo "[Failure] [open meta agent log]\n"; print_r($log_conf); echo "\n"; return; } // subscriber的configure $conf = new BigpipeConf(); $conf_dir = './conf'; $conf_file = './for-tuning.conf'; if (false === bigpipe_load_file($conf_dir, $conf_file, $conf)) { echo "[failure][when load configure]\n"; return; } // 发布参数 $pipe_name = 'pipe2'; //$pipelet_id = 1; // 加1在哪里做好呢? $token = 'token'; $partitioner = new TestPubPartitioner($pid); // 生成一个1K左右的包 $max_msg_count = 12; // 500k $msg = null; $the_one = sprintf("[case:%u]\n", $seq); $uid = BigpipeUtilities::get_uid(); $msg_package = new BigpipeMessagePackage(); $msg_count = 0; while ($msg_count++ < $max_msg_count) { $msg = sprintf("[php-api-test][bigpipe_comlog_pvt_mm][uid:%s][package:%u][seq:%u]\n", $uid, 0, $msg_count); if (true === $one_msg) { $the_one .= $msg; } else { if (!$msg_package->push($msg)) { echo "[fail to pack message]{$msg}\n"; break; // 退出 } } } // end of add message to package if (true == $one_msg) { if (!$msg_package->push($the_one)) { echo "[fail to pack message]{$the_one}\n"; return; // 退出 } } // 定义测试中发包条数 $max_package_count = $max_cnt; echo "\n\n[Publish {$max_package_count} packages]\n\n"; $pub_file_name = sprintf('./pub-timer-%u-%u.csv', $seq, $pid); $pub_file = fopen($pub_file_name, 'w+'); $pub = new BigpipePublisher(); $stat = array(0 => 0, 1 => 0, 2 => 0); if ($pub->init($pipe_name, $token, $partitioner, $conf)) { echo "[Success][init publisher]\n\n"; $count = 0; $succeed = 0; $total_start = BigpipeUtilities::get_time_us(); $is_first = true; printf("[case:%u][start:%u]\n\n", $seq, $total_start); while ($count < $max_package_count) { $count++; //$max_msg_count = rand(1, $max_package_message_count); //echo "[Pack $max_msg_count messages to package <$count>]\n"; // if ($msg_count != $max_msg_count + 1) // { // echo "[expected:$max_msg_count][actual:$msg_count]\n"; // continue; // 打包失败,退出 // } $start_time = BigpipeUtilities::get_time_us(); $msg_package->push(sprintf('timestamp:%u', $start_time)); // 加入时间戳 $pub_result = $pub->send($msg_package); $end_time = BigpipeUtilities::get_time_us(); if (false === $pub_result) { //echo "[fail to publish message package][count:$count]\n"; //break; // 出错便停止 } else { // write result to file //$ret_str = sprintf("%d,%u,%u,%u\n", // $pub_result->error_no, // $pub_result->pipelet_id, // $pub_result->pipelet_msg_id, // $pub_result->session_msg_id); //fwrite($pub_file, $ret_str); if (true == $is_first) { printf('[%u][%u]\\n', $pub_result->pipelet_id, $pub_result->pipelet_msg_id); $is_first = false; } $succeed++; } $msg_package->pop(); // 弹出时间戳 $t = (double) ($end_time - $start_time) / 1000; $t_str = sprintf("%u\n", $t); fwrite($pub_file, $t_str); } $total_end = BigpipeUtilities::get_time_us(); echo "[Publisher][seq:{$seq}][count:{$count}][success:{$succeed}]====\n"; $avg = (double) ($total_end - $total_start) / (double) (1000 * $max_package_count); printf("\n[Publisher][case:%u][avg_time:%f(ms)]\n", $seq, $avg); } else { echo '[Failure][init publisher]\\n'; } $pub->uninit(); BigpipeLog::close(); }
function run_tuning_sub($seq, $pid, $sp, $max_count) { // 初始化log配置 $log_conf = new BigpipeLogConf(); $log_conf->file = sprintf('subscribe-%u-%u.php', $seq, $pid); if (BigpipeLog::init($log_conf)) { //echo "[Success] [open subscribe log]\n\n"; //print_r($log_conf); //echo '\n'; } else { echo '[Failure] [open subscribe log]\\n'; print_r($log_conf); echo '\\n'; } // subscriber的configure $conf = new BigpipeConf(); $conf_dir = './conf'; $conf_file = './for-tuning.conf'; if (false === bigpipe_load_file($conf_dir, $conf_file, $conf)) { echo "[failure][when load configure]\n"; return; } // 订阅参数 $pipe_name = 'pipe2'; $pipelet_id = $pid; // $start_point = $sp; $token = 'token'; $peek_time_ms = 200; $suffix_str = sprintf('-%u-%u', $seq, $pid); $file_sub_succ = fopen('./sub_succ-' . $suffix_str . '.csv', 'w+'); $file_sub_wait = fopen('./sub_wait-' . $suffix_str . '.csv', 'w+'); $file_sub_fail = fopen('./sub_fail-' . $suffix_str . '.csv', 'w+'); $sub = new BigpipeSubscriber(); if ($sub->init($pipe_name, $token, $pipelet_id, $start_point, $conf)) { echo '[Success][init subscriber]\\n'; $count = 0; $failure = 0; $success = 0; $wait_cnt = 0; $run_time_str = null; $total_start = BigpipeUtilities::get_time_us(); while ($success < $max_count && $wait_cnt < $max_count && $failure < $max_count) { $pret = $sub->peek($peek_time_ms); if (BigpipeErrorCode::READABLE == $pret) { $msg = $sub->receive(); if (false === $msg) { $failure++; } else { if (true === $msg->is_last_msg) { $end_time = BigpipeUtilities::get_time_us(); if (0 == strncmp('timestamp:', $msg->content, 10)) { $t_str = substr($msg->content, 10); $run_time_str = sprintf("%u,%u,%s,%u\n", $pipelet_id, $msg->msg_id, $t_str, $end_time); fwrite($file_sub_succ, $run_time_str); } } $success++; } } else { if (BigpipeErrorCode::UNREADABLE == $pret) { $wait_cnt++; } else { $failure++; } } $count++; } $total_end = BigpipeUtilities::get_time_us(); printf("[Leave][seq:%u][count:%u][succ:%u][fail:%u][wait:%u]\n\n", $seq, $count, $success, $failure, $wait_cnt); $avg = (double) ($total_end - $total_start) / (double) (1000 * $count); printf("[Subscriber][seq:%u][avg_time:%f(ms)]\n", $seq, $avg); } else { echo '[Failure][init subscribe]\\n'; } $sub->uninit(); BigpipeLog::close(); }
private function _gen_recv_response() { $topic_id = 65535; $bad_topic_id = 0; $pkg = new BigpipeMessagePackage(); $msg = 'This is a test case'; $pkg->push($msg); $msg_body = null; $pkg->store($msg_body); $sign = creat_sign_mds64($msg_body); $frame = new BStompMessageFrame(); $frame->priority = 10; $frame->persistent = 1; $frame->no_dedupe = 1; $frame->timeout = BigpipeUtilities::get_time_us(); $frame->destination = 'cluster-for-unittest'; $frame->session_id = BigpipeUtilities::get_uid(); $frame->subscribe_id = BigpipeUtilities::get_uid(); $frame->receipt_id = BigpipeUtilities::gen_receipt_id(); $frame->session_message_id = BigpipeUtilities::get_uid(); $frame->topic_message_id = $topic_id; $frame->global_message_id = 76248; $frame->cur_checksum = $sign[2]; $frame->last_checksum = 0; $frame->message_body = $msg_body; $frame->store(); $good = $frame->buffer(); // topic message id ´íÎóµÄcase $frame->topic_message_id = $bad_topic_id; $frame->store(); $bad_topic = $frame->buffer(); // message body ´íÎóµÄcase $frame->topic_message_id = $topic_id; $frame->message_body = ''; $frame->store(); $bad_body = $frame->buffer(); // checksum´íÎóµÄcase $frame->message_body = $msg_body; $frame->cur_checksum = 201; $frame->store(); $bad_checksum = $frame->buffer(); // ´´ÔìÒ»¸öerrorµÄ°ü $err_pkg = '1'; $frame->message_body = $err_pkg; $err_sign = creat_sign_mds64($err_pkg); $frame->cur_checksum = $err_sign[2]; $frame->store(); $bad_pkg = $frame->buffer(); // ´´ÔìÒ»¸öpop errorµÄ°ü $frame->message_body = pack("L2", 1, 5); // ÕâÊÇÒ»¸ö³¤¶ÈΪ5£¬µ«ÊÇûÓÐÊý¾ÝµÄ»µ°ü $empty_sign = creat_sign_mds64($frame->message_body); $frame->cur_checksum = $empty_sign[2]; $frame->store(); $empty_pkg = $frame->buffer(); $res_arr = array('good' => $good, 'bad_topic' => $bad_topic, 'bad_body' => $bad_body, 'bad_checksum' => $bad_checksum, 'bad_pkg' => $bad_pkg, 'empty_pkg' => $empty_pkg); return $res_arr; }
/** * 解析一条nshead封装的消息 * @return 如果成功,返回收到的nshead body; 如果失败,返回null */ public function receive() { if (null == $this->_socket) { BigpipeLog::warning('[receive error][lose connection with %s:%u]', $this->_target['socket_address'], $this->_target['socket_port']); return null; // 无连接 } $res = array(); $buff = $this->_socket->read(36); // 读取一个nshead头 BigpipeLog::debug("[receive nshead][{$buff}]"); if (empty($buff) || strlen($buff) < 36) { $buff_size = strlen($buff); BigpipeLog::warning("[receive nshead error][head_size:{$buff_size}]"); return null; } $nshead = new NsHead(); $res['head'] = $nshead->split_nshead($buff, false); $is_ok = false; $err_msg = null; do { // 读取nshead // 下述判断理论上到不了 $res_head = $res['head']; if (false === $res_head) { $err_msg = '[fail to read head]'; break; } if (!isset($res_head['body_len']) || 0 == $res_head['body_len']) { // 不可能有size为0的包 $err_msg = '[no body_len]'; break; } //读取数据包内容 $res['body'] = $this->_socket->read($res_head['body_len']); if (false === $res['body']) { $err_msg = '[no message body]'; break; } if (true === $this->_check_frame) { // 整包校验 if (BigpipeCommonDefine::NSHEAD_CHECKSUM_MAGICNUM != $res_head['magic_num']) { // 包不带checksum也没问题,但是要打印日志追一下 BigpipeLog::debug('[receive][frame does not have checksum, skip checksum]'); } else { // checksum在nshead, reserved字段中 $checksum = BigpipeUtilities::adler32($res['body']); if ($res_head['reserved'] != $checksum) { $err_msg = sprintf('[checksum failed][send:%u][recv:%u]', $res_head['reserved'], $checksum); $res['body'] = null; break; // 校验失败,丢弃包 } } } // end of check frame integerity $is_ok = true; } while (false); // 用于分支检测 if (!$is_ok) { // 打印错误日志 BigpipeLog::warning("[receive][frame error]%s", $err_msg); return null; } return $res['body']; }
/** * 发布一条消息 * @param BigpipeMessagePackage $msg_pacakge * @return BigpipePubResult on success or false on failure */ public function send($msg_package) { if (!$this->_is_started) { BigpipeLog::warning("[connection is not established][pipelet:%d]", $this->_pipelet_id); return false; } // 发送CONNECT $cmd = new BStompSendFrame(); if (!$msg_package->store($cmd->message_body)) { // 包错误,不用考虑failover了 BigpipeLog::warning("[fail to store message body]"); return false; } if (true === $this->_last_send_ok) { // 当上次发送成功才增加session mesage id // 否则message id不变,防止重复发送 $this->_session_msg_id++; } $cmd->destination = $this->_get_stripe_name(); // echo "[stripe name][$cmd->destination]<br>"; $cmd->no_dedupe = $this->_no_dedupe; $cmd->session_id = $this->_session_id; $cmd->session_message_id = $this->_session_msg_id; $cmd->receipt_id = isset($this->unittest) ? 'fake-receipt-id' : BigpipeUtilities::gen_receipt_id(); if ($this->_enable_checksum) { // 发送message包校验 $cmd->last_checksum = $this->_last_sign; $sign = creat_sign_mds64($cmd->message_body); $cmd->cur_checksum = $sign[2]; // 注意函数返回值 $this->_last_sign = $cmd->cur_checksum; } // 进入发送流程, // 发送失败则进入failover流程,直到failover失败。 $send_result = false; do { if (false === $this->_stomp_adapter->send($cmd)) { BigpipeLog::warning("[send message error][session:%s][session_msg_id:%u]", $cmd->session_id, $cmd->session_message_id); continue; } $send_result = $this->_check_ack($cmd->receipt_id, $send_result); if (false === $send_result) { // ACK失败 // 不用考虑BMQ_E_COMMAND_DUPLICATEMSG, 去重在broker做, // 用户不会收到duplicate返回. BigpipeLog::warning("[send message ack error][session:%s][session_msg_id:%u]", $cmd->session_id, $cmd->session_message_id); continue; } else { BigpipeLog::notice("[send message success][pipelet:%u][session:%s][session_msg_id:%u]", $this->_pipelet_id, $cmd->session_id, $cmd->session_message_id); break; // 发布成功 } } while ($this->_failover()); if (false !== $send_result) { $this->_active(); // 清理failover状态 } else { $this->_last_send_ok = false; } return $send_result; }
/** * 更新meta信息, 记录重试次数 * @return boolean */ private function _failover() { // failover时, 订阅、发布状态无效,重置状态 if (true == $this->_is_subscribed) { $this->_unsubscribe(); // 先尝试取消订阅, 但是不用考虑错误 (因为failover中有错误是常态) $this->_is_subscribed = false; } if ($this->_fo_count > $this->_max_fo_cnt) { // 重置failover BigpipeLog::fatal("[%s:%u][%s][can not do more]", __FILE__, __LINE__, __FUNCTION__); $this->_fo_sleep_time = 0; $this->_fo_count = 0; return false; } if (0 == $this->_fo_sleep_time) { // 第一次flush subscribe时,我们往往不希望等待, // 因此这时跳过sleep过程 // php中只有微秒级的usleep和秒级的sleep $this->_fo_sleep_time = BigpipeCommonDefine::INIT_FO_SLEEP_TIME * 1000; } else { usleep($this->_fo_sleep_time); } $this->_fo_count++; $this->_fo_sleep_time *= 2; // increase failover sleep time if ($this->_fo_sleep_time > BigpipeCommonDefine::MAX_FO_SLEEP_TIME) { // failover sleep time不能无限制增长 $this->_fo_sleep_time = BigpipeCommonDefine::MAX_FO_SLEEP_TIME; } // 通过meta跟新stripe if (false === $this->_update_meta()) { BigpipeLog::fatal("[%s:%u][%s][can not update meta from meta agent]", __FILE__, __LINE__, __FUNCTION__); return false; } // 随机选择并连接一个broker $is_ok = false; do { $broker = $this->_random_select_broker(); if (false === $broker) { // 无新broker可选, failover失败 BigpipeLog::fatal("[%s:%u][%s][no broker to subcribe]", __FILE__, __LINE__, __FUNCTION__); break; } // try to connect to broker $sub_dest = array("socket_address" => $broker->ip, "socket_port" => $broker->port, "socket_timeout" => $this->_conn_timeo); $this->_stomp_adapter->set_destination($sub_dest); $this->_stomp_adapter->role = BStompRoleType::SUBSCRIBER; $this->_stomp_adapter->topic_name = $this->_stripe['stripe_name']; $this->_stomp_adapter->session_id = BigpipeUtilities::get_pipelet_name($this->_pipe_name, $this->_pipelet_id) . '_' . BigpipeUtilities::get_uid(); if ($this->_stomp_adapter->connect()) { BigpipeLog::debug("[%s:%u][%s][Success][connected on broker][ip:%s][port:%u]", __FILE__, __LINE__, __FUNCTION__, $broker->ip, $broker->port); BigpipeLog::debug('[%s:%u][%s][session message id][smid:%s]', __FILE__, __LINE__, __FUNCTION__, $this->_stomp_adapter->session_message_id); $is_ok = true; break; // 跳出连接 } } while (true); return $is_ok; }