public function actionTestDemo()
{
// $request_type["POST"] = 10 ;
// $request_type["GET"] = 2 ;
// AccessLogService::countRequest($request_type,21,"app.tuandai.com") ;
header("Content-type: text/html; charset=utf-8");
set_time_limit(0);
ini_set('memory_limit', '4024M');
$save_rs = false;
//如果有传入时间参数那么以时间参数为准
if (empty($fitdata)) {
$fitdata = date("Ymd", time());
}
$source = 17;
$step = FALSE;
$dir = Yii::getAlias("@backend") . "/resource17";
$handle = dir($dir);
while ($entry = $handle->read()) {
if (!in_array($entry, array('.', '..'))) {
$file_url = $dir . "/" . $entry;
//获取文件名22
$short_name = ToolService::parseFileName($entry);
//判断是否用cdn格式
$isCdn = ToolService::isCdn($short_name);
$cur_date = $fitdata;
$deal_date = Yii::$app->cache->get($source . "deal_date" . $entry);
//日期不一致时,删除上次读到的最后一行,
//为隔天时,可以从第一行读取
$end_num_cache_name = "end_num" . $entry;
if ($deal_date != $cur_date) {
Yii::$app->cache->delete($end_num_cache_name);
Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date);
}
Yii::$app->cache->delete($end_num_cache_name);
Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date);
echo $end_num_cache_name;
echo "\n";
//读取上次读到 的最后一行
$last_end_num = empty(Yii::$app->cache->get($end_num_cache_name)) ? 0 : Yii::$app->cache->get($end_num_cache_name);
echo 'filename:';
echo "\n";
echo $file_url;
echo 'thisRemark';
echo $last_end_num;
echo "\n";
$total_line = ToolService::count_line($file_url);
$start_num = $last_end_num + 1;
$end_num = $total_line;
$save_rs = [];
//一次处理500条数据
while ($start_num < $end_num) {
//设定要处理的终结行
$fit_endNum = $start_num + 1000;
//是否是最后一条数据的处理
$endDateNumFit = false;
if ($fit_endNum >= $end_num) {
$fit_endNum = $end_num;
$endDateNumFit = true;
}
//开始处理行数
$content_arr = ToolService::getFileLines($file_url, $start_num, $fit_endNum);
//$save_rs是前一次处理留下来的数据。这里做一下判断处理
$st_check_t = 0;
//上次的检查时间
$preA = [];
//上次处理留下的数据
if (!empty($save_rs)) {
$st_check_t = $save_rs['str_check_time'];
$preA = $save_rs['leaveDate'];
}
$save_rs = AccessLogService::analyForNginx($content_arr, $isCdn, $short_name, $source, $endDateNumFit, $st_check_t, $preA, $start_num, $end_num_cache_name, $step);
$start_num = $fit_endNum;
}
unset($content_arr);
//记录读到的最后一行
Yii::$app->cache->set($end_num_cache_name, $total_line);
//记录日期
Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date);
if (empty($save_rs)) {
//处理完后删除文件,防范重复入库
// @unlink($file_url);
}
}
}
if (empty($file_url)) {
$msg = $dir . " 目录下没有可读取的文件";
} else {
$msg = "处理成功";
}
if (empty($file_url)) {
$msg = $dir . " 目录下没有可读取的文件";
} else {
$msg = "处理成功";
}
}
/**
* 分析nginx配置文件
* $message string 指定是那个代理的文件
* $fitdata string 指定是那一天的数据
* @return type
*/
public function actionNginxaccessfile($message = '17', $fitdata = '')
{
//
set_time_limit(0);
ini_set('memory_limit', '4024M');
$save_rs = false;
//步增判断，当天的数据，最后一部分的数据不会入库
$step = TRUE;
//如果有传入时间参数那么以时间参数为准
if (empty($fitdata)) {
$fitdata = date("Ymd", time());
} else {
$step = FALSE;
}
if ($message == '17') {
$dir = \Yii::$app->params['proxy17'];
$source = "17";
} else {
$dir = \Yii::$app->params['proxy21'];
$source = "21";
}
$dir .= '/' . $fitdata;
//test data
//$dir = \Yii::$app->params['proxytest'];
$handle = dir($dir);
$fitFileArray = \Yii::$app->params['nginxfitfile'];
while ($entry = $handle->read()) {
if (!in_array($entry, array('.', '..'))) {
$file_url = $dir . "/" . $entry;
//获取文件名22
$short_name = ToolService::parseFileName($entry);
//一个用来处理来源的简化字符串
$li_short_name = '';
if (strpos($short_name, ".") === false) {
$li_short_name = $short_name;
} else {
$li_short_name = substr($short_name, 0, strpos($short_name, "."));
}
if (!in_array($short_name, $fitFileArray)) {
continue;
}
//判断是否用cdn格式
$isCdn = ToolService::isCdn($short_name);
$cur_date = $fitdata;
$deal_date = Yii::$app->cache->get($source . "deal_date" . $entry);
//日期不一致时,删除上次读到的最后一行,
//为隔天时,可以从第一行读取
$end_num_cache_name = $source . "end_num" . $entry;
if ($deal_date != $cur_date) {
Yii::$app->cache->delete($end_num_cache_name);
Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date);
}
echo $end_num_cache_name;
echo "\n";
//读取上次读到 的最后一行
$last_end_num = empty(Yii::$app->cache->get($end_num_cache_name)) ? 0 : Yii::$app->cache->get($end_num_cache_name);
echo 'filename:';
echo "\n";
echo $file_url;
echo 'thisRemark';
echo $last_end_num;
echo "\n";
$total_line = ToolService::count_line($file_url);
$start_num = $last_end_num + 1;
$end_num = $total_line;
$save_rs = [];
//一次处理500条数据
while ($start_num < $end_num) {
//设定要处理的终结行
$fit_endNum = $start_num + 1000;
//是否是最后一条数据的处理
$endDateNumFit = false;
if ($fit_endNum >= $end_num) {
$fit_endNum = $end_num;
$endDateNumFit = true;
}
//开始处理行数
$content_arr = ToolService::getFileLines($file_url, $start_num, $fit_endNum);
//$save_rs是前一次处理留下来的数据。这里做一下判断处理
$st_check_t = 0;
//上次的检查时间
$preA = [];
//上次处理留下的数据
if (!empty($save_rs)) {
$st_check_t = $save_rs['str_check_time'];
$preA = $save_rs['leaveDate'];
}
$save_rs = AccessLogService::analyForNginx($content_arr, $isCdn, $short_name, $source, $endDateNumFit, $st_check_t, $preA, $start_num, $end_num_cache_name, $step, $li_short_name);
$start_num = $fit_endNum;
}
unset($content_arr);
//记录读到的最后一行
Yii::$app->cache->set($end_num_cache_name, $total_line);
//记录日期
Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date);
if (empty($save_rs)) {
//处理完后删除文件,防范重复入库
// @unlink($file_url);
}
}
}
if (empty($file_url)) {
$msg = $dir . " 目录下没有可读取的文件";
} else {
$msg = "处理成功";
}
echo $msg;
}
