public function actionTestDemo()
{
// $request_type["POST"] = 10 ;
// $request_type["GET"] = 2 ;
// AccessLogService::countRequest($request_type,21,"app.tuandai.com") ;
header("Content-type: text/html; charset=utf-8");
set_time_limit(0);
ini_set('memory_limit', '4024M');
$save_rs = false;
//如果有传入时间参数那么以时间参数为准
if (empty($fitdata)) {
$fitdata = date("Ymd", time());
}
$source = 17;
$step = FALSE;
$dir = Yii::getAlias("@backend") . "/resource17";
$handle = dir($dir);
while ($entry = $handle->read()) {
if (!in_array($entry, array('.', '..'))) {
$file_url = $dir . "/" . $entry;
//获取文件名22
$short_name = ToolService::parseFileName($entry);
//判断是否用cdn格式
$isCdn = ToolService::isCdn($short_name);
$cur_date = $fitdata;
$deal_date = Yii::$app->cache->get($source . "deal_date" . $entry);
//日期不一致时,删除上次读到的最后一行,
//为隔天时,可以从第一行读取
$end_num_cache_name = "end_num" . $entry;
if ($deal_date != $cur_date) {
Yii::$app->cache->delete($end_num_cache_name);
Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date);
}
Yii::$app->cache->delete($end_num_cache_name);
Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date);
echo $end_num_cache_name;
echo "\n";
//读取上次读到 的最后一行
$last_end_num = empty(Yii::$app->cache->get($end_num_cache_name)) ? 0 : Yii::$app->cache->get($end_num_cache_name);
echo 'filename:';
echo "\n";
echo $file_url;
echo 'thisRemark';
echo $last_end_num;
echo "\n";
$total_line = ToolService::count_line($file_url);
$start_num = $last_end_num + 1;
$end_num = $total_line;
$save_rs = [];
//一次处理500条数据
while ($start_num < $end_num) {
//设定要处理的终结行
$fit_endNum = $start_num + 1000;
//是否是最后一条数据的处理
$endDateNumFit = false;
if ($fit_endNum >= $end_num) {
$fit_endNum = $end_num;
$endDateNumFit = true;
}
//开始处理行数
$content_arr = ToolService::getFileLines($file_url, $start_num, $fit_endNum);
//$save_rs是前一次处理留下来的数据。这里做一下判断处理
$st_check_t = 0;
//上次的检查时间
$preA = [];
//上次处理留下的数据
if (!empty($save_rs)) {
$st_check_t = $save_rs['str_check_time'];
$preA = $save_rs['leaveDate'];
}
$save_rs = AccessLogService::analyForNginx($content_arr, $isCdn, $short_name, $source, $endDateNumFit, $st_check_t, $preA, $start_num, $end_num_cache_name, $step);
$start_num = $fit_endNum;
}
unset($content_arr);
//记录读到的最后一行
Yii::$app->cache->set($end_num_cache_name, $total_line);
//记录日期
Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date);
if (empty($save_rs)) {
//处理完后删除文件,防范重复入库
// @unlink($file_url);
}
}
}
if (empty($file_url)) {
$msg = $dir . " 目录下没有可读取的文件";
} else {
$msg = "处理成功";
}
if (empty($file_url)) {
$msg = $dir . " 目录下没有可读取的文件";
} else {
$msg = "处理成功";
}
}
public function actionFrontDistribute($str_num, $file_url, $short_name, $source, $total_line)
{
ini_set('memory_limit', '1024M');
$isCdn = ToolService::isCdn($short_name);
$total_page = ceil(ToolService::DISTRIBUTE_NUM / ToolService::READ_LINE);
$end_num = $str_num + ToolService::DISTRIBUTE_NUM - 1;
if ($end_num >= $total_line) {
$end_num = $total_line;
}
$id = DistributeLogService::saveToDb($str_num, $end_num, $short_name, $file_url, $source);
for ($i = 0; $i < $total_page; $i++) {
$start_num = $i * ToolService::READ_LINE + $str_num;
$end_num = $start_num + ToolService::READ_LINE - 1;
$content_arr = ToolService::getFileLines($file_url, $start_num, $end_num);
AccessLogService::saveToDbForNginx($content_arr, $isCdn, $short_name, $source);
unset($content_arr);
}
DistributeLogService::updateToDb($id);
}