/**
* @brief 获取db对象
*
* @param $clusterName 集群名称
* @param $key 负载均衡key
* @param $getNew 是否重新连接
*
* @return
*/
public static function getConn($clusterName, $key = NULL, $getNew = false)
{
$hookBeforeInit = Bd_Conf::getConf('/db/hook_before_init');
if ($hookBeforeInit === false) {
//cannot find hookBeforeInit in conf file
self::$_error['errno'] = LOAD_CONF_ERROR;
self::$_error['error'] = 'Can not read hookBeforeInit, please check db/global.conf';
Bd_Log::warning(self::$_error['error'], self::$_error['errno']);
return false;
}
if ($hookBeforeInit != NULL) {
//user sets hookBeforeInit
if (is_callable($hookBeforeInit)) {
$clusterName = call_user_func($hookBeforeInit, $clusterName);
} else {
//warnning
self::$_error['errno'] = SET_HOOK_ERROR;
self::$_error['error'] = 'Hook(beforinit):' . $before . 'is not callable';
Bd_Log::warning(self::$_error['error'], self::$_error['errno']);
}
}
$conf =& self::$_conf;
$hosts =& self::$_hosts;
$dbData =& self::$_dbData;
$lastDb =& self::$_lastDb;
//(1) alreay save a connection (2)user do not need to recreate
if (!empty($lastDb[$clusterName]) && !$getNew) {
Bd_Log::trace('Return an existing connection', 0, array('db_cluster' => $clusterName));
return $lastDb[$clusterName];
}
if (self::_init($clusterName) === false) {
return false;
}
//create a new db object
$db = new Bd_DB(Bd_Db_ConnMgr::$ENABLE_PROFILING);
//add hook
if ('' !== ($before = $conf['hook_before_query'])) {
if (!$db->addHook(Bd_Db::HK_BEFORE_QUERY, $clusterName . '-before', $before)) {
self::$_error['errno'] = SET_HOOK_ERROR;
self::$_error['error'] = 'Hook(befor query):' . $before . ' is not callable';
Bd_Log::warning(self::$_error['error'], self::$_error['errno']);
}
}
if ('' !== ($after = $conf['hook_after_query'])) {
if (!$db->addHook(Bd_Db::HK_AFTER_QUERY, $clusterName . '-after', $after)) {
self::$_error['errno'] = SET_HOOK_ERROR;
self::$_error['error'] = 'Hook(after query):' . $after . ' is not callable';
Bd_Log::warning(self::$_error['error'], self::$_error['errno']);
}
}
if ('' !== ($onFail = $conf['hook_on_fail'])) {
if (!$db->onFail($onFail)) {
self::$_error['errno'] = SET_HOOK_ERROR;
self::$_error['error'] = 'Hook(on fail):' . $onFail . ' is not callable';
Bd_Log::warning(self::$_error['error'], self::$_error['errno']);
}
}
//try to connect host until there is not host or connecting successfully
while (true) {
//balancer could not select a valid host to connect
if (count($hosts['valid_hosts']) === 0 || ($index = $dbData['host_selector']->select($hosts, $key)) === false) {
self::$_error['errno'] = ALL_CONNECT_ERROR;
self::$_error['error'] = 'No host could be connected in the cluster';
Bd_Log::warning(self::$_error['error'], self::$_error['errno'], array('db_cluster' => $clusterName));
$hookOnConnFail = $conf['hook_on_connect_fail'];
if ($hookOnConnFail != NULL) {
if (is_callable($hookOnConnFail)) {
call_user_func($hookOnConnFail);
} else {
//warnning
self::$_error['errno'] = SET_HOOK_ERROR;
self::$_error['error'] = 'Hook(on connect fail):' . $hookOnConnFail . 'is not callable';
Bd_Log::warning(self::$_error['error'], self::$_error['errno']);
}
}
return false;
}
//log parameters
$logPara = array('db_cluster' => $clusterName, 'db_host' => $hosts['valid_hosts'][$index]['ip'], 'db_port' => $hosts['valid_hosts'][$index]['port'], 'default_db' => $conf['default_db']);
for ($i = 1; $i <= $conf['retry_times']; $i++) {
$timeout = $conf['connect_timeout_s'];
if ($timeout > 0) {
$db->setConnectTimeOut($timeout);
}
$r_timeout = $conf['read_timeout_s'];
if ($r_timeout > 0) {
$db->setOption(MYSQL_OPT_READ_TIMEOUT, $r_timeout);
}
$w_timeout = $conf['write_timeout_s'];
if ($w_timeout > 0) {
$db->setOption(MYSQL_OPT_WRITE_TIMEOUT, $w_timeout);
}
Bd_Log::debug("retry times: {$i}");
$start = microtime(true) * 1000;
//connect
$ret = $db->connect($hosts['valid_hosts'][$index]['ip'], $conf['username'], $conf['password'], $conf['default_db'], $hosts['valid_hosts'][$index]['port'], $conf['connect_flag']);
$end = microtime(true) * 1000;
if ($ret) {
if (empty($conf['charset']) || $db->charset($conf['charset'])) {
$logPara['time_ms'] = $end - $start;
Bd_Log::trace('Connect to Mysql successfully', 0, $logPara);
$lastDb[$clusterName] = $db;
return $lastDb[$clusterName];
} else {
Bd_Log::debug('Set charset failed');
}
}
}
//connect failed
self::$_error['errno'] = CONNECT_ERROR;
self::$_error['error'] = 'Connect to Mysql failed';
Bd_Log::warning(self::$_error['error'], self::$_error['errno'], $logPara);
self::_recordFailedHost($index);
}
return false;
}
if ($ret == 0) {
//对上传成功文件打上done标记
$strCmdMd5 = "cd {$dataPath} && touch {$key}.done";
exec($strCmdMd5);
//更新该pipelet状态
$db = new mysqli($arrAppConf['mysql_host'], $arrAppConf['mysql_user'], $arrAppConf['mysql_pwd'], $arrAppConf['mysql_db'], $arrAppConf['mysql_port']);
$sql = sprintf("INSERT INTO %s_%s VALUES(%s%s%s, %s) ON DUPLICATE KEY UPDATE pipelet_id=pipelet_id | %d", $arrAppConf['mysql_table'], $app, $event_day, $event_hour, $event_minute, 1 << $pipelet, 1 << $pipelet);
$db->query($sql);
//检查所有pipelet上传完成状态
$sql = sprintf("SELECT pipelet_id FROM %s_%s WHERE partition=%s%s%s FOR UPDATE", $arrAppConf['mysql_table'], $app, $event_day, $event_hour, $event_minute);
$ret = $db->query($sql);
if ($ret === false) {
echo "query faild!\n";
Bd_Log::fatal("[UPDATE] pipelet_id:{$pipelet} partition:{$event_day}{$event_hour}{$event_minute} query_failed");
}
$tmp = $ret->fetch_row();
$pipeletFull = intval($arrAppConf['pipelet_full']);
Bd_Log::trace("[UPDATE] pipelet_id:{$pipelet} partition:{$event_day}{$event_hour}{$event_minute} [{$tmp[0]}/{$pipeletFull}]");
if ($tmp[0] == $pipeletFull) {
//创建新PARTITION
$hql = "ALTER TABLE iknowetl_{$app} ADD PARTITION (event_day='{$event_day}', event_hour='{$event_hour}', event_minute='{$event_minute}') LOCATION '{$hdfsPath}'";
$strCmdPartition = "source ~/.bashrc && sh hive_tool.sh hql \"{$hql}\"";
Bd_Log::trace("[UPDATE] pipelet_id:{$pipelet} partition:{$event_day}{$event_hour}{$event_minute} add_partition_start:{$strCmdPartition}");
exec($strCmdPartition, $out, $ret);
Bd_Log::trace("[UPDATE] pipelet_id:{$pipelet} partition:{$event_day}{$event_hour}{$event_minute} add_partition_res:{$ret}");
}
}
}
sleep($arrAppConf['update_sleep_s']);
}
/* vim: set ts=4 sw=4 sts=4 tw=100 noet: */
/**
* @brief 查询接口
*
* @param $sql 查询sql
* @param $fetchType 结果集抽取类型
* @param $bolUseResult 是否使用MYSQLI_USE_RESULT
*
* @return 结果数组：成功；false：失败
*/
public function query($sql, $fetchType = Bd_DB::FETCH_ASSOC, $bolUseResult = false)
{
/*
if(!$this->isConnected())
{
return false;
}
*/
$logPara = array('db_host' => $this->dbConf['host'], 'db_port' => $this->dbConf['port'], 'default_db' => $this->dbConf['dbname']);
if (!is_string($sql)) {
// get sql text
if (!$sql instanceof ISQL || !($sql = $sql->getSQL())) {
$this->_error['errno'] = INVALID_SQL;
$this->_error['error'] = 'Input SQL is not valid,please use string or ISQL instance';
Bd_Log::warning($this->_error['error'], $this->_error['errno'], $logPara);
return false;
}
}
// execute hooks before query
foreach ($this->hkBeforeQ as $arrCallback) {
$func = $arrCallback[0];
$extArgs = $arrCallback[1];
if (call_user_func_array($func, array($this, &$sql, $extArgs)) === false) {
return false;
}
}
$this->lastSQL = $sql;
Bd_Profiler::start();
$res = $this->mysql->query($sql, $bolUseResult ? MYSQLI_USE_RESULT : MYSQLI_STORE_RESULT);
// record cost
$this->lastCost = intval(microtime(true) * 1000000) - $beg;
$this->totalCost += $this->lastCost;
// do profiling
if ($this->enableProfiling) {
$this->arrCost[] = array($sql, $this->lastCost);
}
$ret = false;
$pos = strpos($sql, "\n");
if ($pos) {
$logPara['sql'] = strstr($sql, array("\n", ' '));
} else {
$logPara['sql'] = $sql;
}
// res is NULL if mysql is disconnected
if (is_bool($res) || $res === NULL) {
$arrInfo = array('ns' => $this->dbConf['dbname'], 'query' => $logPara['sql'], 'retry' => 1, 'local_ip' => $_SERVER['SERVER_ADDR'], 'remote_ip' => $this->dbConf['host'] . ':' . $this->dbConf['port'], 'res_len' => 0, 'errno' => QUERY_ERROR);
Bd_Profiler::end($arrInfo);
$ret = $res == true;
// call fail handler
if (!$ret) {
$this->_error['errno'] = QUERY_ERROR;
$this->_error['error'] = 'Query failed';
Bd_Log::warning($this->_error['error'], $this->_error['errno'], $logPara);
if ($this->onfail !== NULL) {
call_user_func_array($this->onfail, array($this, &$ret));
}
}
} else {
$logPara['time_ms'] = $this->lastCost / 1000;
$logPara['affected_rows'] = $this->mysql->affected_rows;
$arrInfo = array('ns' => $this->dbConf['dbname'], 'query' => $logPara['sql'], 'retry' => 1, 'local_ip' => $_SERVER['SERVER_ADDR'], 'remote_ip' => $this->dbConf['host'] . ':' . $this->dbConf['port'], 'res_len' => $logPara['affected_rows']);
Bd_Profiler::end($arrInfo);
Bd_Log::trace("Query successfully", 0, $logPara);
switch ($fetchType) {
case Bd_DB::FETCH_OBJ:
$ret = new Bd_Db_DBResult($res);
break;
case Bd_DB::FETCH_ASSOC:
$ret = array();
while ($row = $res->fetch_assoc()) {
$ret[] = $row;
}
$res->free();
break;
case Bd_DB::FETCH_ROW:
$ret = array();
while ($row = $res->fetch_row()) {
$ret[] = $row;
}
$res->free();
break;
default:
$ret = $res;
break;
}
}
// execute hooks after query
foreach ($this->hkAfterQ as $arrCallback) {
$func = $arrCallback[0];
$extArgs = $arrCallback[1];
call_user_func_array($func, array($this, &$ret, $extArgs));
}
return $ret;
}
/**
* @brief 根据$key删除cache中对应的数据
*
* @param $key cache中的key
* @return true or false
* @retval boolean
* @author chenyijie
* @date 2012/09/27 20:32:42
**/
public function deleteCacheKey($key)
{
if (!parent::_getCacheClient()) {
return false;
}
$result = $this->objCacheClient->delete($key);
if ($result === false) {
$arrErr = array('caller' => 'DsCacheProxy', 'class' => get_class($this->objClass), 'key' => $key);
Bd_Log::trace("Delete data from cache failed.", parent::DELETE_DATA_FAILED, $arrErr);
parent::$errCode = parent::DELETE_DATA_FAILED;
return false;
}
Bd_Log::trace("Delete data in cache success.", 0, $arrErr);
return true;
}
