public function actionTestDemo() { // $request_type["POST"] = 10 ; // $request_type["GET"] = 2 ; // AccessLogService::countRequest($request_type,21,"app.tuandai.com") ; header("Content-type: text/html; charset=utf-8"); set_time_limit(0); ini_set('memory_limit', '4024M'); $save_rs = false; //如果有传入时间参数那么以时间参数为准 if (empty($fitdata)) { $fitdata = date("Ymd", time()); } $source = 17; $step = FALSE; $dir = Yii::getAlias("@backend") . "/resource17"; $handle = dir($dir); while ($entry = $handle->read()) { if (!in_array($entry, array('.', '..'))) { $file_url = $dir . "/" . $entry; //获取文件名22 $short_name = ToolService::parseFileName($entry); //判断是否用cdn格式 $isCdn = ToolService::isCdn($short_name); $cur_date = $fitdata; $deal_date = Yii::$app->cache->get($source . "deal_date" . $entry); //日期不一致时,删除上次读到的最后一行, //为隔天时,可以从第一行读取 $end_num_cache_name = "end_num" . $entry; if ($deal_date != $cur_date) { Yii::$app->cache->delete($end_num_cache_name); Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date); } Yii::$app->cache->delete($end_num_cache_name); Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date); echo $end_num_cache_name; echo "\n"; //读取上次读到 的最后一行 $last_end_num = empty(Yii::$app->cache->get($end_num_cache_name)) ? 0 : Yii::$app->cache->get($end_num_cache_name); echo 'filename:'; echo "\n"; echo $file_url; echo 'thisRemark'; echo $last_end_num; echo "\n"; $total_line = ToolService::count_line($file_url); $start_num = $last_end_num + 1; $end_num = $total_line; $save_rs = []; //一次处理500条数据 while ($start_num < $end_num) { //设定要处理的终结行 $fit_endNum = $start_num + 1000; //是否是最后一条数据的处理 $endDateNumFit = false; if ($fit_endNum >= $end_num) { $fit_endNum = $end_num; $endDateNumFit = true; } //开始处理行数 $content_arr = ToolService::getFileLines($file_url, $start_num, $fit_endNum); //$save_rs是前一次处理留下来的数据。这里做一下判断处理 $st_check_t = 0; //上次的检查时间 $preA = []; //上次处理留下的数据 if (!empty($save_rs)) { $st_check_t = $save_rs['str_check_time']; $preA = $save_rs['leaveDate']; } $save_rs = AccessLogService::analyForNginx($content_arr, $isCdn, $short_name, $source, $endDateNumFit, $st_check_t, $preA, $start_num, $end_num_cache_name, $step); $start_num = $fit_endNum; } unset($content_arr); //记录读到的最后一行 Yii::$app->cache->set($end_num_cache_name, $total_line); //记录日期 Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date); if (empty($save_rs)) { //处理完后删除文件,防范重复入库 // @unlink($file_url); } } } if (empty($file_url)) { $msg = $dir . " 目录下没有可读取的文件"; } else { $msg = "处理成功"; } if (empty($file_url)) { $msg = $dir . " 目录下没有可读取的文件"; } else { $msg = "处理成功"; } }
/** * 分析nginx配置文件 * $message string 指定是那个代理的文件 * $fitdata string 指定是那一天的数据 * @return type */ public function actionNginxaccessfile($message = '17', $fitdata = '') { // set_time_limit(0); ini_set('memory_limit', '4024M'); $save_rs = false; //步增判断,当天的数据,最后一部分的数据不会入库 $step = TRUE; //如果有传入时间参数那么以时间参数为准 if (empty($fitdata)) { $fitdata = date("Ymd", time()); } else { $step = FALSE; } if ($message == '17') { $dir = \Yii::$app->params['proxy17']; $source = "17"; } else { $dir = \Yii::$app->params['proxy21']; $source = "21"; } $dir .= '/' . $fitdata; //test data //$dir = \Yii::$app->params['proxytest']; $handle = dir($dir); $fitFileArray = \Yii::$app->params['nginxfitfile']; while ($entry = $handle->read()) { if (!in_array($entry, array('.', '..'))) { $file_url = $dir . "/" . $entry; //获取文件名22 $short_name = ToolService::parseFileName($entry); //一个用来处理来源的简化字符串 $li_short_name = ''; if (strpos($short_name, ".") === false) { $li_short_name = $short_name; } else { $li_short_name = substr($short_name, 0, strpos($short_name, ".")); } if (!in_array($short_name, $fitFileArray)) { continue; } //判断是否用cdn格式 $isCdn = ToolService::isCdn($short_name); $cur_date = $fitdata; $deal_date = Yii::$app->cache->get($source . "deal_date" . $entry); //日期不一致时,删除上次读到的最后一行, //为隔天时,可以从第一行读取 $end_num_cache_name = $source . "end_num" . $entry; if ($deal_date != $cur_date) { Yii::$app->cache->delete($end_num_cache_name); Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date); } echo $end_num_cache_name; echo "\n"; //读取上次读到 的最后一行 $last_end_num = empty(Yii::$app->cache->get($end_num_cache_name)) ? 0 : Yii::$app->cache->get($end_num_cache_name); echo 'filename:'; echo "\n"; echo $file_url; echo 'thisRemark'; echo $last_end_num; echo "\n"; $total_line = ToolService::count_line($file_url); $start_num = $last_end_num + 1; $end_num = $total_line; $save_rs = []; //一次处理500条数据 while ($start_num < $end_num) { //设定要处理的终结行 $fit_endNum = $start_num + 1000; //是否是最后一条数据的处理 $endDateNumFit = false; if ($fit_endNum >= $end_num) { $fit_endNum = $end_num; $endDateNumFit = true; } //开始处理行数 $content_arr = ToolService::getFileLines($file_url, $start_num, $fit_endNum); //$save_rs是前一次处理留下来的数据。这里做一下判断处理 $st_check_t = 0; //上次的检查时间 $preA = []; //上次处理留下的数据 if (!empty($save_rs)) { $st_check_t = $save_rs['str_check_time']; $preA = $save_rs['leaveDate']; } $save_rs = AccessLogService::analyForNginx($content_arr, $isCdn, $short_name, $source, $endDateNumFit, $st_check_t, $preA, $start_num, $end_num_cache_name, $step, $li_short_name); $start_num = $fit_endNum; } unset($content_arr); //记录读到的最后一行 Yii::$app->cache->set($end_num_cache_name, $total_line); //记录日期 Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date); if (empty($save_rs)) { //处理完后删除文件,防范重复入库 // @unlink($file_url); } } } if (empty($file_url)) { $msg = $dir . " 目录下没有可读取的文件"; } else { $msg = "处理成功"; } echo $msg; }