Example #1
0
 public function actionTestDemo()
 {
     //        $request_type["POST"] = 10 ;
     //        $request_type["GET"] = 2 ;
     //        AccessLogService::countRequest($request_type,21,"app.tuandai.com") ;
     header("Content-type: text/html; charset=utf-8");
     set_time_limit(0);
     ini_set('memory_limit', '4024M');
     $save_rs = false;
     //如果有传入时间参数那么以时间参数为准
     if (empty($fitdata)) {
         $fitdata = date("Ymd", time());
     }
     $source = 17;
     $step = FALSE;
     $dir = Yii::getAlias("@backend") . "/resource17";
     $handle = dir($dir);
     while ($entry = $handle->read()) {
         if (!in_array($entry, array('.', '..'))) {
             $file_url = $dir . "/" . $entry;
             //获取文件名22
             $short_name = ToolService::parseFileName($entry);
             //判断是否用cdn格式
             $isCdn = ToolService::isCdn($short_name);
             $cur_date = $fitdata;
             $deal_date = Yii::$app->cache->get($source . "deal_date" . $entry);
             //日期不一致时,删除上次读到的最后一行,
             //为隔天时,可以从第一行读取
             $end_num_cache_name = "end_num" . $entry;
             if ($deal_date != $cur_date) {
                 Yii::$app->cache->delete($end_num_cache_name);
                 Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date);
             }
             Yii::$app->cache->delete($end_num_cache_name);
             Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date);
             echo $end_num_cache_name;
             echo "\n";
             //读取上次读到 的最后一行
             $last_end_num = empty(Yii::$app->cache->get($end_num_cache_name)) ? 0 : Yii::$app->cache->get($end_num_cache_name);
             echo 'filename:';
             echo "\n";
             echo $file_url;
             echo 'thisRemark';
             echo $last_end_num;
             echo "\n";
             $total_line = ToolService::count_line($file_url);
             $start_num = $last_end_num + 1;
             $end_num = $total_line;
             $save_rs = [];
             //一次处理500条数据
             while ($start_num < $end_num) {
                 //设定要处理的终结行
                 $fit_endNum = $start_num + 1000;
                 //是否是最后一条数据的处理
                 $endDateNumFit = false;
                 if ($fit_endNum >= $end_num) {
                     $fit_endNum = $end_num;
                     $endDateNumFit = true;
                 }
                 //开始处理行数
                 $content_arr = ToolService::getFileLines($file_url, $start_num, $fit_endNum);
                 //$save_rs是前一次处理留下来的数据。这里做一下判断处理
                 $st_check_t = 0;
                 //上次的检查时间
                 $preA = [];
                 //上次处理留下的数据
                 if (!empty($save_rs)) {
                     $st_check_t = $save_rs['str_check_time'];
                     $preA = $save_rs['leaveDate'];
                 }
                 $save_rs = AccessLogService::analyForNginx($content_arr, $isCdn, $short_name, $source, $endDateNumFit, $st_check_t, $preA, $start_num, $end_num_cache_name, $step);
                 $start_num = $fit_endNum;
             }
             unset($content_arr);
             //记录读到的最后一行
             Yii::$app->cache->set($end_num_cache_name, $total_line);
             //记录日期
             Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date);
             if (empty($save_rs)) {
                 //处理完后删除文件,防范重复入库
                 //                  @unlink($file_url);
             }
         }
     }
     if (empty($file_url)) {
         $msg = $dir . " 目录下没有可读取的文件";
     } else {
         $msg = "处理成功";
     }
     if (empty($file_url)) {
         $msg = $dir . " 目录下没有可读取的文件";
     } else {
         $msg = "处理成功";
     }
 }
 public function actionFrontDistribute($str_num, $file_url, $short_name, $source, $total_line)
 {
     ini_set('memory_limit', '1024M');
     $isCdn = ToolService::isCdn($short_name);
     $total_page = ceil(ToolService::DISTRIBUTE_NUM / ToolService::READ_LINE);
     $end_num = $str_num + ToolService::DISTRIBUTE_NUM - 1;
     if ($end_num >= $total_line) {
         $end_num = $total_line;
     }
     $id = DistributeLogService::saveToDb($str_num, $end_num, $short_name, $file_url, $source);
     for ($i = 0; $i < $total_page; $i++) {
         $start_num = $i * ToolService::READ_LINE + $str_num;
         $end_num = $start_num + ToolService::READ_LINE - 1;
         $content_arr = ToolService::getFileLines($file_url, $start_num, $end_num);
         AccessLogService::saveToDbForNginx($content_arr, $isCdn, $short_name, $source);
         unset($content_arr);
     }
     DistributeLogService::updateToDb($id);
 }