public function actionTestDemo() { // $request_type["POST"] = 10 ; // $request_type["GET"] = 2 ; // AccessLogService::countRequest($request_type,21,"app.tuandai.com") ; header("Content-type: text/html; charset=utf-8"); set_time_limit(0); ini_set('memory_limit', '4024M'); $save_rs = false; //如果有传入时间参数那么以时间参数为准 if (empty($fitdata)) { $fitdata = date("Ymd", time()); } $source = 17; $step = FALSE; $dir = Yii::getAlias("@backend") . "/resource17"; $handle = dir($dir); while ($entry = $handle->read()) { if (!in_array($entry, array('.', '..'))) { $file_url = $dir . "/" . $entry; //获取文件名22 $short_name = ToolService::parseFileName($entry); //判断是否用cdn格式 $isCdn = ToolService::isCdn($short_name); $cur_date = $fitdata; $deal_date = Yii::$app->cache->get($source . "deal_date" . $entry); //日期不一致时,删除上次读到的最后一行, //为隔天时,可以从第一行读取 $end_num_cache_name = "end_num" . $entry; if ($deal_date != $cur_date) { Yii::$app->cache->delete($end_num_cache_name); Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date); } Yii::$app->cache->delete($end_num_cache_name); Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date); echo $end_num_cache_name; echo "\n"; //读取上次读到 的最后一行 $last_end_num = empty(Yii::$app->cache->get($end_num_cache_name)) ? 0 : Yii::$app->cache->get($end_num_cache_name); echo 'filename:'; echo "\n"; echo $file_url; echo 'thisRemark'; echo $last_end_num; echo "\n"; $total_line = ToolService::count_line($file_url); $start_num = $last_end_num + 1; $end_num = $total_line; $save_rs = []; //一次处理500条数据 while ($start_num < $end_num) { //设定要处理的终结行 $fit_endNum = $start_num + 1000; //是否是最后一条数据的处理 $endDateNumFit = false; if ($fit_endNum >= $end_num) { $fit_endNum = $end_num; $endDateNumFit = true; } //开始处理行数 $content_arr = ToolService::getFileLines($file_url, $start_num, $fit_endNum); //$save_rs是前一次处理留下来的数据。这里做一下判断处理 $st_check_t = 0; //上次的检查时间 $preA = []; //上次处理留下的数据 if (!empty($save_rs)) { $st_check_t = $save_rs['str_check_time']; $preA = $save_rs['leaveDate']; } $save_rs = AccessLogService::analyForNginx($content_arr, $isCdn, $short_name, $source, $endDateNumFit, $st_check_t, $preA, $start_num, $end_num_cache_name, $step); $start_num = $fit_endNum; } unset($content_arr); //记录读到的最后一行 Yii::$app->cache->set($end_num_cache_name, $total_line); //记录日期 Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date); if (empty($save_rs)) { //处理完后删除文件,防范重复入库 // @unlink($file_url); } } } if (empty($file_url)) { $msg = $dir . " 目录下没有可读取的文件"; } else { $msg = "处理成功"; } if (empty($file_url)) { $msg = $dir . " 目录下没有可读取的文件"; } else { $msg = "处理成功"; } }
public function actionFrontDistribute($str_num, $file_url, $short_name, $source, $total_line) { ini_set('memory_limit', '1024M'); $isCdn = ToolService::isCdn($short_name); $total_page = ceil(ToolService::DISTRIBUTE_NUM / ToolService::READ_LINE); $end_num = $str_num + ToolService::DISTRIBUTE_NUM - 1; if ($end_num >= $total_line) { $end_num = $total_line; } $id = DistributeLogService::saveToDb($str_num, $end_num, $short_name, $file_url, $source); for ($i = 0; $i < $total_page; $i++) { $start_num = $i * ToolService::READ_LINE + $str_num; $end_num = $start_num + ToolService::READ_LINE - 1; $content_arr = ToolService::getFileLines($file_url, $start_num, $end_num); AccessLogService::saveToDbForNginx($content_arr, $isCdn, $short_name, $source); unset($content_arr); } DistributeLogService::updateToDb($id); }