public function get_headersession($arrHeader)
{
$arrDataSessionData = array();
if (empty($arrHeader['PS-DATASESSIONDATA'])) {
return $arrDataSessionData;
}
$arrDataSessionData = mc_pack_text2pack($arrHeader['PS-DATASESSIONDATA']);
$arrDataSessionData = empty($arrDataSessionData) ? $arrDataSessionData : mc_pack_pack2array($arrDataSessionData);
if (!empty($arrDataSessionData)) {
$strEncoding = empty($arrDataSessionData['encoding']) ? 'gbk' : trim($arrDataSessionData['encoding']);
$strUname = iconv($strEncoding, 'utf-8', $arrDataSessionData['uname']);
$strDisplayName = iconv($strEncoding, 'utf-8', $arrDataSessionData['displayname']);
$arrDataSessionData['uname'] = empty($strUname) ? '' : $strUname;
$arrDataSessionData['displayname'] = empty($strDisplayName) ? '' : $strDisplayName;
} else {
$status = $GLOBALS['STATUS_CODE']['UNPACK_FAIL'];
CLog::warning($GLOBALS['STATUS_MSG'][$status] . ",sessiondata raw pack:" . urlencode($arrHeader['PS-DATASESSIONDATA']), $status, $GLOBALS['logArr']);
}
return $arrDataSessionData;
}
// 订阅消息后，监听端口，等待消息到达
$pret = $sub->peek($peek_time_ms);
if (BigpipeErrorCode::READABLE == $pret) {
// peek返回readable后，调用receive接收数据
// 与send是发布message package不同，receive会主动解message package，
// 每次receive成功，返回一条消息
$msg = $sub->receive();
if (false === $msg) {
echo '[Receive][error]\\n';
$loop_end = true;
// 当receive发生错误时退出循环
} else {
$msg_str = sprintf("[begin]===\n[msg_id][%u]\n[seq_id][%u]\n[msg][%s]\n[end]===\n", $msg->msg_id, $msg->seq_id, $msg->content);
fwrite($file, $msg_str);
echo $msg->msg_id . "\t" . $msg->seq_id . "\n";
$tmp = mc_pack_pack2array($msg->content);
// var_dump( $tmp );
// echo "=============================\n" ;
}
$count = 0;
} else {
if (BigpipeErrorCode::UNREADABLE == $pret) {
$count++;
$msg_str = sprintf("[try][count:%d]\n", $count);
fwrite($file, $msg_str);
} else {
echo "[Peek][Error][ret:{$pret}]\n";
$loop_end = true;
}
}
}
protected function _deserialize($strOutput)
{
return mc_pack_pack2array($strOutput);
}
private function getExtendDispInfoLog($strLog)
{
$arrInfo = mc_pack_pack2array($strLog);
$strTmp = '';
if (false != $arrInfo) {
foreach ($arrInfo as $key => $value) {
if (!empty($strTmp)) {
$strTmp .= '&';
}
$strTmp .= $key . '=' . $value;
}
}
return $strTmp;
}
private function _talkWithServer($handler, $buffer)
{
if (!$handler->write($buffer)) {
$this->_last_errno = ZCACHE_CLIENT_ERR_WRITE;
$this->_last_errmsg = "net write err";
$this->_putHandler($handler, 1);
return false;
}
$retbuffer = $this->_readResponse($handler);
if (!$retbuffer) {
$this->_putHandler($handler, 1);
//print("retry connect\n");
$handler = $this->_getHandler();
// read error: retry connect , write , read once
if (!$handler) {
$this->_last_errno = ZCACHE_CLIENT_ERR_CONNECT;
$this->_last_errmsg = "retry net connect err";
return false;
}
//print("retry write\n");
if (!$handler->write($buffer)) {
$this->_last_errno = ZCACHE_CLIENT_ERR_WRITE;
$this->_last_errmsg = "retry net write err";
$this->_putHandler($handler, 1);
return false;
}
//print("retry read\n");
$retbuffer = $this->_readResponse($handler);
if (!$retbuffer) {
$this->_last_errno = ZCACHE_CLIENT_ERR_READ;
$this->_last_errmsg = "retry net read err";
$this->_putHandler($handler, 1);
return false;
}
}
//print("query ok\n");
$this->_putHandler($handler);
$ret_arr = mc_pack_pack2array($retbuffer);
if (isset($ret_arr['err_no']) && ZCACHE_OK == $ret_arr['err_no']) {
$this->_last_errno = ZCACHE_OK;
if (isset($ret_arr['error'])) {
$this->_last_errmsg = $ret_arr['error'];
}
return $ret_arr;
} else {
if (isset($ret_arr['err_no'])) {
$this->_last_errno = $ret_arr['err_no'];
if (isset($ret_arr['error'])) {
$this->_last_errmsg = $ret_arr['error'];
}
} else {
$this->_last_errno = ZCACHE_CLIENT_ERR_MCPACK;
$this->_last_errmsg = "mcpack err";
}
return false;
}
}
#!/usr/bin/env php
<?php
require_once dirname(dirname(__FILE__)) . '/init_env.php';
require_once Da\Sys_App::lib_path("CNsHead.class.php");
$opt = getopt('h::p::', ['2json', '2sp']);
$host = isset($opt['h']) ? $opt['h'] : '0.0.0.0';
$port = isset($opt['p']) ? $opt['p'] : '8765';
$socket = stream_socket_server("tcp://{$host}:{$port}", $errno, $errstr);
echo 'Listening at port: ' . $port . PHP_EOL;
if (!$socket) {
echo "{$errstr} ({$errno})\n";
} else {
while ($conn = stream_socket_accept($socket, 86400)) {
$ns = new NsHead();
$data = $ns->nshead_read($conn);
$result = mc_pack_pack2array($data['buf']);
isset($opt['2json']) ? print json_encode($result) : (isset($opt['2sp']) ? print serialize($result) : print_r($result));
echo PHP_EOL;
$data = ['errno' => 0, 'data' => 'ok'];
$body = mc_pack_array2pack($data);
$hdr = array('body_len' => strlen($body));
$ns->nshead_write($conn, $hdr, $body);
}
}
protected function call_acm($group_key, $key, $sub_key = null)
{
$acm_zk_path = $this->conf['zk_path'] . '/acm';
$zk_expire = $this->conf['zk_expire'];
$acm_conf = Ak_Zookeeper::getCached($acm_zk_path, 2, $zk_expire, 'Ak_McClient::checkAcmConf');
if (!is_array($acm_conf)) {
Ak_Log::warning("acm zk conf formate error!");
return false;
}
for ($i = 1; $i <= $acm_conf['acm_idc_num']; ++$i) {
$server_name = "server{$i}";
if (!is_array($acm_conf['children'][$server_name])) {
Ak_Log::warning("acm {$server_name} not exist!");
continue;
}
$server_conf = $acm_conf['children'][$server_name];
$this->span_idc->registerResource(self::ACM_RESOURCE, $server_conf['idc'], $server_name);
}
$resources = $this->span_idc->getResource($this->conf['span_idc_strategy_index'], self::ACM_RESOURCE, $this->conf['curr_idc']);
if (count($resources) == 0) {
Ak_Log::warning("get resources is empty!");
return false;
}
foreach ($resources as $server_name) {
$acm_path = $this->conf['zk_path'] . "/acm/{$server_name}";
$aclient_conf = array('Source' => 'Galileo', 'Protocol' => 'Nshead', 'Scheduler' => 'Closest', 'GalileoConf' => array('Path' => $acm_path), 'ClosestConf' => array('AlwaysRetry' => true), 'NsheadConf' => array('ConnectTimeOut' => $this->conf['acm_connect_timeout'], 'WriteTimeOut' => $this->conf['acm_write_timeout'], 'ReadTimeOut' => $this->conf['acm_read_timeout']));
$client = new Ak_AClient();
$client->SetConf($aclient_conf);
$idc_list = array();
for ($i = 0; $i < $this->zk_conf['idc_num']; $i++) {
$idc_list[] = $i;
}
$data = array('command_no' => $this->zk_conf['delete_cmd'], 'pid' => $this->conf['pid'], 'group_key' => (string) $group_key, 'key' => (string) $key, 'idc_list' => $idc_list);
if ($sub_key !== null) {
$data['has_sub_key'] = 1;
$data['sub_key'] = $sub_key;
}
$input = array('body' => mc_pack_array2pack($data, PHP_MC_PACK_V2));
$output = $client->Call($input);
if ($output == null) {
Ak_Log::warning("aclient call acm [{$server_name}] failed");
} else {
$res = mc_pack_pack2array($output['body']);
if ($res['error_no'] !== 0) {
Ak_Log::warning("call acm [{$server_name}] output err_no is {$res['errno']}");
} else {
return true;
}
}
}
return false;
}
public function execute()
{
$minute_partition = parent::$arrAppConf['minute_partition'];
$bigpipeLogConf = new BigpipeLogConf();
$bigpipeLogConf->file = 'subscribe.php';
if (BigpipeLog::init($bigpipeLogConf)) {
echo "[Success] [open subscribe log]\n";
// print_r($bigpipeLogConf);
} else {
echo '[Failure] [open subscribe log]\\n';
print_r($bigpipeLogConf);
echo "\n";
}
$conf = new BigpipeConf();
$conf_dir = ETL_CONF . parent::$app;
$conf_file = './php-api.conf';
if (false === bigpipe_load_file($conf_dir, $conf_file, $conf)) {
echo "[failure][when load configure]\n";
exit;
}
$pipeName = parent::$arrAppConf['pipe_name'];
$pipelet = parent::$pipelet;
$token = parent::$arrAppConf['token'];
$peekTimeMs = parent::$arrAppConf['peek_time_ms'];
$messageIdFile = ETL_DATA . parent::$app . "/message_pipelet_" . parent::$pipelet . ".flag";
$startPoint = file_get_contents($messageIdFile);
if ($startPoint == false) {
$startPoint = -1;
} else {
if (trim($startPoint) != "-2") {
$startPoint = intval(trim($startPoint)) + 1;
}
}
$sub = new BigpipeSubscriber();
if ($sub->init($pipeName, $token, $pipelet, $startPoint, $conf)) {
$lastPartition = 0;
while (true) {
$pret = $sub->peek($peekTimeMs);
if (BigpipeErrorCode::READABLE == $pret) {
$msg = $sub->receive();
if (false == $msg) {
echo "[Receive][error]\n";
continue;
} else {
// echo $msg->msg_id . "\t" . $msg->seq_id. "\n" ;
$arrMsgContent = mc_pack_pack2array($msg->content);
//cut
$arrContent = cut($arrMsgContent['body']);
//map
$objEvent = new Event();
foreach (parent::$arrMap as $map) {
$arrParams = array();
foreach ($map['in'] as $in) {
$arrParams['in'][$in] = $arrContent[$in];
}
foreach ($map['out'] as $out) {
$arrParams['out'][$out] = null;
}
$hook_func_callback = $map['hook'];
$res = call_user_func($hook_func_callback, &$arrParams);
if ($res === true) {
foreach ($arrParams['out'] as $key => $value) {
$objEvent->arrEvent[$key] = $value;
}
foreach ($arrParams['in'] as $key => $value) {
unset($arrContent[$key]);
}
}
}
$objEvent->arrEvent['event_loginfo'] = common_loginfo($arrContent);
//write to data
$date = sprintf("%s%s%s", $objEvent->arrEvent['event_year'], $objEvent->arrEvent['event_month'], $objEvent->arrEvent['event_dayofmonth']);
list($hour, $minute, $second) = explode(":", $objEvent->arrEvent['event_time'], 3);
$partition = $date . $hour . sprintf("%02d", intval($minute / parent::$arrAppConf['minute_partition']) * parent::$arrAppConf['minute_partition']);
$dataFile = ETL_DATA . parent::$app . "/" . parent::$app . "_" . parent::$pipelet . "_{$partition}";
$strEvent = "";
foreach ($objEvent->arrEvent as $item) {
$item = str_replace("", "", $item);
$item = str_replace("\n", "", $item);
if ($strEvent == "") {
$strEvent .= $item;
} else {
$strEvent .= "" . $item;
}
}
$strEvent .= "\n";
file_put_contents($dataFile, $strEvent, FILE_APPEND | LOCK_EX);
$fpMessageId = fopen($messageIdFile, "w");
fwrite($fpMessageId, $msg->msg_id);
fclose($fpMessageId);
}
} else {
if (BigpipeErrorCode::UNREADABLE == $pret) {
sleep(30);
} else {
echo "[Peek][Error][ret:{$pret}]\n";
}
}
}
} else {
echo '[Failure][init subscribe]\\n';
}
$sub->uninit();
BigpipeLog::close();
}
private function addQueryInfo($arrData)
{
if (empty($arrData)) {
return;
}
$arrQueryInfo = $arrData['uiData']['queryInfo'];
$arrResult = $arrData['uiData']['asResult'];
$rsseResult = $arrData['uiData']['rsseResult'];
$appResult = $arrData['uiData']['appResult'];
$favoResult = $arrData['uiData']['favoResult'];
$topResult = $arrData['uiData']['topResult'];
$rightResult = $arrData['uiData']['rightResult'];
$disableFlag = $arrData['uiData']['disableFlag'];
$ecResult = $arrData['uiData']['ecResult'];
$this->BaiduLog->setQueryId($arrQueryInfo['queryId']);
$this->BaiduLog->setPageNo($arrQueryInfo['pageNo']);
// 页面所有结果数，包括广告
// ec队列
$intAsNum = $arrResult['asResultNum'] + $arrResult['spResutlNum'] + $arrResult['adjResultNum'];
foreach ($ecResult['ecResultItem'] as $result) {
if ($result['source'] == SRC_PPIM || $result['source'] == SRC_NEWPP) {
$intAsNum += $result['adNum'];
}
}
$this->BaiduLog->setAsNum($intAsNum);
$this->BaiduLog->setUserUintIp($arrQueryInfo['ip']);
$this->BaiduLog->setBaiduid($arrQueryInfo['baiduId']);
$this->BaiduLog->setAccountName($arrQueryInfo['accountName']);
$this->BaiduLog->setNeedSp($arrQueryInfo['needSp']);
$this->BaiduLog->setAsDatadispNum($arrResult['asDataDispNum']);
$this->BaiduLog->setAsDatalistNum($arrResult['asDataListNum']);
foreach ($rsseResult['rsphrase'] as $arrPhrase) {
$rsPhrase = new RsPhrase();
$rsPhrase->setPhrase($arrPhrase['phrase']);
$rsPhrase->setRsComefrom((string) $arrPhrase['rsComeFrom']);
$this->BaiduLog->addRsPhrase($rsPhrase);
}
if (!empty($arrData['uiData']['queryInfo']['superSeFlag']) && $arrData['uiData']['queryInfo']['superSeFlag'] == 1) {
$this->BaiduLog->addSeWord($arrData['uiData']['queryInfo']['wordNoSyntax']);
} else {
foreach ($rsseResult['seword'] as $strWord) {
$this->BaiduLog->addSeWord($strWord);
}
}
$this->BaiduLog->setUrlParam(urlencode($arrQueryInfo['urlParam']));
$this->BaiduLog->setAdJpNum($arrQueryInfo['extResCount']);
$intSeInfo = $arrQueryInfo['seInfo'] > 0 ? 1 : 0;
$this->BaiduLog->setIsSeInfo($intSeInfo);
$intListNum = $appResult['listNum'] > 0 ? $appResult['listNum'] : 0;
$this->BaiduLog->setAppListNum($intListNum);
$intFavoNum = is_array($favoResult['items']) ? count($favoResult['items']) : 0;
$this->BaiduLog->setFavoItemNum($intFavoNum);
foreach ($arrQueryInfo['samplingId'] as $intSamplingId) {
$this->BaiduLog->addSamplingId($intSamplingId);
}
$arrInfo = mc_pack_pack2array($arrQueryInfo['gSampleLog']);
if (false != $arrInfo) {
foreach ($arrInfo as $key => $value) {
$dispData = new DispData();
$dispData->setKey($key);
$dispData->setVal($value);
$this->BaiduLog->addGSampleLog($dispData);
}
}
$intTopNum = is_array($topResult['item']) ? count($topResult['item']) : 0;
$this->BaiduLog->setTopResultNum($intTopNum);
$intRightNum = is_array($rightResult['item']) ? count($rightResult['item']) : 0;
$this->BaiduLog->setRightResultNum($intRightNum);
$this->BaiduLog->setAdRightOld($disableFlag['adRight']);
$this->BaiduLog->setAdRightNew($ecResult['advRightNum'] > 1 ? 0 : 1);
$this->BaiduLog->setAdvStrategyV1($ecResult['advStrategyV1']);
$this->BaiduLog->setAdvStrategyV2($ecResult['advStrategyV2']);
$this->BaiduLog->setAdSecurityPlanTip($ecResult['securityPlanTip']);
$rightPromptStra = empty($arrData['uiData']['queryInfo']['rightPromptStraFlag']) ? 0 : $arrData['uiData']['queryInfo']['rightPromptStraFlag'];
$this->BaiduLog->setRightPromptStra($rightPromptStra);
$this->BaiduLog->setInterClickLog($arrData['uiData']['interClickLog']);
$this->BaiduLog->setBqid($arrQueryInfo['bqid']);
foreach ($arrQueryInfo['promptNode'] as $arrPromptNode) {
$this->BaiduLog->addPromptNode($arrPromptNode['id']);
}
$this->BaiduLog->setCq($arrQueryInfo['otherResultQuery']);
$this->BaiduLog->setDispl($arrQueryInfo['language']);
$this->BaiduLog->setLa($arrQueryInfo['resultLang']);
$this->BaiduLog->setQryl($arrQueryInfo['query_lang']);
if (empty($arrData['uiData']['queryInfo']['superSeFlag'])) {
$this->BaiduLog->setSse(false);
} else {
$this->BaiduLog->setSse(true);
}
$this->BaiduLog->setDatetime(date('Y-m-d H:i:s'));
$this->BaiduLog->setBufTempLog($arrQueryInfo['bufTempLog']);
}
