/** * @brief 获取db对象 * * @param $clusterName 集群名称 * @param $key 负载均衡key * @param $getNew 是否重新连接 * * @return */ public static function getConn($clusterName, $key = NULL, $getNew = false) { $hookBeforeInit = Bd_Conf::getConf('/db/hook_before_init'); if ($hookBeforeInit === false) { //cannot find hookBeforeInit in conf file self::$_error['errno'] = LOAD_CONF_ERROR; self::$_error['error'] = 'Can not read hookBeforeInit, please check db/global.conf'; Bd_Log::warning(self::$_error['error'], self::$_error['errno']); return false; } if ($hookBeforeInit != NULL) { //user sets hookBeforeInit if (is_callable($hookBeforeInit)) { $clusterName = call_user_func($hookBeforeInit, $clusterName); } else { //warnning self::$_error['errno'] = SET_HOOK_ERROR; self::$_error['error'] = 'Hook(beforinit):' . $before . 'is not callable'; Bd_Log::warning(self::$_error['error'], self::$_error['errno']); } } $conf =& self::$_conf; $hosts =& self::$_hosts; $dbData =& self::$_dbData; $lastDb =& self::$_lastDb; //(1) alreay save a connection (2)user do not need to recreate if (!empty($lastDb[$clusterName]) && !$getNew) { Bd_Log::trace('Return an existing connection', 0, array('db_cluster' => $clusterName)); return $lastDb[$clusterName]; } if (self::_init($clusterName) === false) { return false; } //create a new db object $db = new Bd_DB(Bd_Db_ConnMgr::$ENABLE_PROFILING); //add hook if ('' !== ($before = $conf['hook_before_query'])) { if (!$db->addHook(Bd_Db::HK_BEFORE_QUERY, $clusterName . '-before', $before)) { self::$_error['errno'] = SET_HOOK_ERROR; self::$_error['error'] = 'Hook(befor query):' . $before . ' is not callable'; Bd_Log::warning(self::$_error['error'], self::$_error['errno']); } } if ('' !== ($after = $conf['hook_after_query'])) { if (!$db->addHook(Bd_Db::HK_AFTER_QUERY, $clusterName . '-after', $after)) { self::$_error['errno'] = SET_HOOK_ERROR; self::$_error['error'] = 'Hook(after query):' . $after . ' is not callable'; Bd_Log::warning(self::$_error['error'], self::$_error['errno']); } } if ('' !== ($onFail = $conf['hook_on_fail'])) { if (!$db->onFail($onFail)) { self::$_error['errno'] = SET_HOOK_ERROR; self::$_error['error'] = 'Hook(on fail):' . $onFail . ' is not callable'; Bd_Log::warning(self::$_error['error'], self::$_error['errno']); } } //try to connect host until there is not host or connecting successfully while (true) { //balancer could not select a valid host to connect if (count($hosts['valid_hosts']) === 0 || ($index = $dbData['host_selector']->select($hosts, $key)) === false) { self::$_error['errno'] = ALL_CONNECT_ERROR; self::$_error['error'] = 'No host could be connected in the cluster'; Bd_Log::warning(self::$_error['error'], self::$_error['errno'], array('db_cluster' => $clusterName)); $hookOnConnFail = $conf['hook_on_connect_fail']; if ($hookOnConnFail != NULL) { if (is_callable($hookOnConnFail)) { call_user_func($hookOnConnFail); } else { //warnning self::$_error['errno'] = SET_HOOK_ERROR; self::$_error['error'] = 'Hook(on connect fail):' . $hookOnConnFail . 'is not callable'; Bd_Log::warning(self::$_error['error'], self::$_error['errno']); } } return false; } //log parameters $logPara = array('db_cluster' => $clusterName, 'db_host' => $hosts['valid_hosts'][$index]['ip'], 'db_port' => $hosts['valid_hosts'][$index]['port'], 'default_db' => $conf['default_db']); for ($i = 1; $i <= $conf['retry_times']; $i++) { $timeout = $conf['connect_timeout_s']; if ($timeout > 0) { $db->setConnectTimeOut($timeout); } $r_timeout = $conf['read_timeout_s']; if ($r_timeout > 0) { $db->setOption(MYSQL_OPT_READ_TIMEOUT, $r_timeout); } $w_timeout = $conf['write_timeout_s']; if ($w_timeout > 0) { $db->setOption(MYSQL_OPT_WRITE_TIMEOUT, $w_timeout); } Bd_Log::debug("retry times: {$i}"); $start = microtime(true) * 1000; //connect $ret = $db->connect($hosts['valid_hosts'][$index]['ip'], $conf['username'], $conf['password'], $conf['default_db'], $hosts['valid_hosts'][$index]['port'], $conf['connect_flag']); $end = microtime(true) * 1000; if ($ret) { if (empty($conf['charset']) || $db->charset($conf['charset'])) { $logPara['time_ms'] = $end - $start; Bd_Log::trace('Connect to Mysql successfully', 0, $logPara); $lastDb[$clusterName] = $db; return $lastDb[$clusterName]; } else { Bd_Log::debug('Set charset failed'); } } } //connect failed self::$_error['errno'] = CONNECT_ERROR; self::$_error['error'] = 'Connect to Mysql failed'; Bd_Log::warning(self::$_error['error'], self::$_error['errno'], $logPara); self::_recordFailedHost($index); } return false; }
if ($ret == 0) { //对上传成功文件打上done标记 $strCmdMd5 = "cd {$dataPath} && touch {$key}.done"; exec($strCmdMd5); //更新该pipelet状态 $db = new mysqli($arrAppConf['mysql_host'], $arrAppConf['mysql_user'], $arrAppConf['mysql_pwd'], $arrAppConf['mysql_db'], $arrAppConf['mysql_port']); $sql = sprintf("INSERT INTO %s_%s VALUES(%s%s%s, %s) ON DUPLICATE KEY UPDATE pipelet_id=pipelet_id | %d", $arrAppConf['mysql_table'], $app, $event_day, $event_hour, $event_minute, 1 << $pipelet, 1 << $pipelet); $db->query($sql); //检查所有pipelet上传完成状态 $sql = sprintf("SELECT pipelet_id FROM %s_%s WHERE partition=%s%s%s FOR UPDATE", $arrAppConf['mysql_table'], $app, $event_day, $event_hour, $event_minute); $ret = $db->query($sql); if ($ret === false) { echo "query faild!\n"; Bd_Log::fatal("[UPDATE] pipelet_id:{$pipelet} partition:{$event_day}{$event_hour}{$event_minute} query_failed"); } $tmp = $ret->fetch_row(); $pipeletFull = intval($arrAppConf['pipelet_full']); Bd_Log::trace("[UPDATE] pipelet_id:{$pipelet} partition:{$event_day}{$event_hour}{$event_minute} [{$tmp[0]}/{$pipeletFull}]"); if ($tmp[0] == $pipeletFull) { //创建新PARTITION $hql = "ALTER TABLE iknowetl_{$app} ADD PARTITION (event_day='{$event_day}', event_hour='{$event_hour}', event_minute='{$event_minute}') LOCATION '{$hdfsPath}'"; $strCmdPartition = "source ~/.bashrc && sh hive_tool.sh hql \"{$hql}\""; Bd_Log::trace("[UPDATE] pipelet_id:{$pipelet} partition:{$event_day}{$event_hour}{$event_minute} add_partition_start:{$strCmdPartition}"); exec($strCmdPartition, $out, $ret); Bd_Log::trace("[UPDATE] pipelet_id:{$pipelet} partition:{$event_day}{$event_hour}{$event_minute} add_partition_res:{$ret}"); } } } sleep($arrAppConf['update_sleep_s']); } /* vim: set ts=4 sw=4 sts=4 tw=100 noet: */
/** * @brief 查询接口 * * @param $sql 查询sql * @param $fetchType 结果集抽取类型 * @param $bolUseResult 是否使用MYSQLI_USE_RESULT * * @return 结果数组:成功;false:失败 */ public function query($sql, $fetchType = Bd_DB::FETCH_ASSOC, $bolUseResult = false) { /* if(!$this->isConnected()) { return false; } */ $logPara = array('db_host' => $this->dbConf['host'], 'db_port' => $this->dbConf['port'], 'default_db' => $this->dbConf['dbname']); if (!is_string($sql)) { // get sql text if (!$sql instanceof ISQL || !($sql = $sql->getSQL())) { $this->_error['errno'] = INVALID_SQL; $this->_error['error'] = 'Input SQL is not valid,please use string or ISQL instance'; Bd_Log::warning($this->_error['error'], $this->_error['errno'], $logPara); return false; } } // execute hooks before query foreach ($this->hkBeforeQ as $arrCallback) { $func = $arrCallback[0]; $extArgs = $arrCallback[1]; if (call_user_func_array($func, array($this, &$sql, $extArgs)) === false) { return false; } } $this->lastSQL = $sql; Bd_Profiler::start(); $res = $this->mysql->query($sql, $bolUseResult ? MYSQLI_USE_RESULT : MYSQLI_STORE_RESULT); // record cost $this->lastCost = intval(microtime(true) * 1000000) - $beg; $this->totalCost += $this->lastCost; // do profiling if ($this->enableProfiling) { $this->arrCost[] = array($sql, $this->lastCost); } $ret = false; $pos = strpos($sql, "\n"); if ($pos) { $logPara['sql'] = strstr($sql, array("\n", ' ')); } else { $logPara['sql'] = $sql; } // res is NULL if mysql is disconnected if (is_bool($res) || $res === NULL) { $arrInfo = array('ns' => $this->dbConf['dbname'], 'query' => $logPara['sql'], 'retry' => 1, 'local_ip' => $_SERVER['SERVER_ADDR'], 'remote_ip' => $this->dbConf['host'] . ':' . $this->dbConf['port'], 'res_len' => 0, 'errno' => QUERY_ERROR); Bd_Profiler::end($arrInfo); $ret = $res == true; // call fail handler if (!$ret) { $this->_error['errno'] = QUERY_ERROR; $this->_error['error'] = 'Query failed'; Bd_Log::warning($this->_error['error'], $this->_error['errno'], $logPara); if ($this->onfail !== NULL) { call_user_func_array($this->onfail, array($this, &$ret)); } } } else { $logPara['time_ms'] = $this->lastCost / 1000; $logPara['affected_rows'] = $this->mysql->affected_rows; $arrInfo = array('ns' => $this->dbConf['dbname'], 'query' => $logPara['sql'], 'retry' => 1, 'local_ip' => $_SERVER['SERVER_ADDR'], 'remote_ip' => $this->dbConf['host'] . ':' . $this->dbConf['port'], 'res_len' => $logPara['affected_rows']); Bd_Profiler::end($arrInfo); Bd_Log::trace("Query successfully", 0, $logPara); switch ($fetchType) { case Bd_DB::FETCH_OBJ: $ret = new Bd_Db_DBResult($res); break; case Bd_DB::FETCH_ASSOC: $ret = array(); while ($row = $res->fetch_assoc()) { $ret[] = $row; } $res->free(); break; case Bd_DB::FETCH_ROW: $ret = array(); while ($row = $res->fetch_row()) { $ret[] = $row; } $res->free(); break; default: $ret = $res; break; } } // execute hooks after query foreach ($this->hkAfterQ as $arrCallback) { $func = $arrCallback[0]; $extArgs = $arrCallback[1]; call_user_func_array($func, array($this, &$ret, $extArgs)); } return $ret; }
/** * @brief 根据$key删除cache中对应的数据 * * @param $key cache中的key * @return true or false * @retval boolean * @author chenyijie * @date 2012/09/27 20:32:42 **/ public function deleteCacheKey($key) { if (!parent::_getCacheClient()) { return false; } $result = $this->objCacheClient->delete($key); if ($result === false) { $arrErr = array('caller' => 'DsCacheProxy', 'class' => get_class($this->objClass), 'key' => $key); Bd_Log::trace("Delete data from cache failed.", parent::DELETE_DATA_FAILED, $arrErr); parent::$errCode = parent::DELETE_DATA_FAILED; return false; } Bd_Log::trace("Delete data in cache success.", 0, $arrErr); return true; }