/**
* 分析用户的访问流并进行相应的记录
* @param type $fitDataArray
*/
public static function userVisitFlow($fitDataArray, $li_short_name)
{
//获得解释过后的一个数组
$mat = $fitDataArray['mat'];
//获得请求页面
if (!isset($mat[3][0])) {
return false;
}
$accPage = $mat[3][0];
$newInfo = ToolService::parseRequestInfo($accPage);
//获得正常的表达式
if (!isset($newInfo[2][0])) {
return FALSE;
}
$realUrl = self::trimURl($newInfo[2][0], $li_short_name);
//如果请求连接有问题那么直接返回错误
if ($realUrl === FALSE) {
return false;
}
//获得来源页面
if (!isset($mat[6][0])) {
return FALSE;
}
$sourcePage = $mat[6][0];
$fromUrl = self::trimFromURL($sourcePage);
//保存数据到文件缓冲
$target = \Yii::$app->cache->get('target');
if ($target !== false) {
if (isset($target[$realUrl][$fromUrl])) {
$target[$realUrl][$fromUrl] = $target[$realUrl][$fromUrl] + 1;
} else {
$target[$realUrl][$fromUrl] = 1;
}
} else {
$target[$realUrl][$fromUrl] = 1;
}
//保存这个数据到缓冲文件,10分钟处理一次数据时，会把数据入库
\Yii::$app->cache->set('target', $target);
}
<?php
/* @var $this yii\web\View */
use yii\helpers\Html;
use yii\widgets\Breadcrumbs;
use yii\widgets\LinkPager;
use yii\widgets\ActiveForm;
use dosamigos\datetimepicker\DateTimePicker;
use backend\services\SqlAttackService;
use backend\services\ToolService;
$this->title = '日志列表';
$month_info = ToolService::getMonthFirstAndLastInfo();
$searchModel->start_date = empty($searchModel->start_date) ? date("Y-m-d H:i:s", $month_info['str_time']) : $searchModel->start_date;
$searchModel->end_date = empty($searchModel->end_date) ? date("Y-m-d H:i:s", $month_info['end_time']) : $searchModel->end_date;
$begin = $pager->page * $pager->pageSize + 1;
$end = $begin + $pager->pageSize - 1;
if ($begin > $end) {
$begin = $end;
}
#获得日志统计记录
?>
<div class="site-index">
<?php
echo Breadcrumbs::widget(['itemTemplate' => "<li><i>{link}</i></li>\n", 'links' => [['label' => '首页']]]);
$id = \Yii::$app->controller->action->id;
?>
<div class="body-content">
<div class="panel panel-default">
<?php
echo $this->render('common_top.php');
?>
public function actionTestDemo()
{
// $request_type["POST"] = 10 ;
// $request_type["GET"] = 2 ;
// AccessLogService::countRequest($request_type,21,"app.tuandai.com") ;
header("Content-type: text/html; charset=utf-8");
set_time_limit(0);
ini_set('memory_limit', '4024M');
$save_rs = false;
//如果有传入时间参数那么以时间参数为准
if (empty($fitdata)) {
$fitdata = date("Ymd", time());
}
$source = 17;
$step = FALSE;
$dir = Yii::getAlias("@backend") . "/resource17";
$handle = dir($dir);
while ($entry = $handle->read()) {
if (!in_array($entry, array('.', '..'))) {
$file_url = $dir . "/" . $entry;
//获取文件名22
$short_name = ToolService::parseFileName($entry);
//判断是否用cdn格式
$isCdn = ToolService::isCdn($short_name);
$cur_date = $fitdata;
$deal_date = Yii::$app->cache->get($source . "deal_date" . $entry);
//日期不一致时,删除上次读到的最后一行,
//为隔天时,可以从第一行读取
$end_num_cache_name = "end_num" . $entry;
if ($deal_date != $cur_date) {
Yii::$app->cache->delete($end_num_cache_name);
Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date);
}
Yii::$app->cache->delete($end_num_cache_name);
Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date);
echo $end_num_cache_name;
echo "\n";
//读取上次读到 的最后一行
$last_end_num = empty(Yii::$app->cache->get($end_num_cache_name)) ? 0 : Yii::$app->cache->get($end_num_cache_name);
echo 'filename:';
echo "\n";
echo $file_url;
echo 'thisRemark';
echo $last_end_num;
echo "\n";
$total_line = ToolService::count_line($file_url);
$start_num = $last_end_num + 1;
$end_num = $total_line;
$save_rs = [];
//一次处理500条数据
while ($start_num < $end_num) {
//设定要处理的终结行
$fit_endNum = $start_num + 1000;
//是否是最后一条数据的处理
$endDateNumFit = false;
if ($fit_endNum >= $end_num) {
$fit_endNum = $end_num;
$endDateNumFit = true;
}
//开始处理行数
$content_arr = ToolService::getFileLines($file_url, $start_num, $fit_endNum);
//$save_rs是前一次处理留下来的数据。这里做一下判断处理
$st_check_t = 0;
//上次的检查时间
$preA = [];
//上次处理留下的数据
if (!empty($save_rs)) {
$st_check_t = $save_rs['str_check_time'];
$preA = $save_rs['leaveDate'];
}
$save_rs = AccessLogService::analyForNginx($content_arr, $isCdn, $short_name, $source, $endDateNumFit, $st_check_t, $preA, $start_num, $end_num_cache_name, $step);
$start_num = $fit_endNum;
}
unset($content_arr);
//记录读到的最后一行
Yii::$app->cache->set($end_num_cache_name, $total_line);
//记录日期
Yii::$app->cache->set($source . "deal_date" . $entry, $cur_date);
if (empty($save_rs)) {
//处理完后删除文件,防范重复入库
// @unlink($file_url);
}
}
}
if (empty($file_url)) {
$msg = $dir . " 目录下没有可读取的文件";
} else {
$msg = "处理成功";
}
if (empty($file_url)) {
$msg = $dir . " 目录下没有可读取的文件";
} else {
$msg = "处理成功";
}
}
/**
* 首页信息
* @return type
*/
public function actionIndex()
{
$params = Yii::$app->request->get();
$searchModel = new SqlLogSearch();
$dataProvider = $searchModel->search($params);
$query = $dataProvider->query;
$sort = new Sort(['attributes' => ['executedate'], 'defaultOrder' => ['executedate' => SORT_DESC]]);
$locals = ToolService::getPagedRows($query, $tablename = 'SqlTrace', $params, ['orderBy' => $sort->orders, 'pageSize' => 30]);
$locals['searchModel'] = $searchModel;
return $this->render('sql', $locals);
}
public function actionIndex()
{
$params = Yii::$app->request->get();
$searchModel = new ErrorLogSearch();
$dataProvider = $searchModel->search($params);
$query = $dataProvider->query;
$sort = new Sort(['attributes' => ['AddDate'], 'defaultOrder' => ['AddDate' => SORT_DESC]]);
$locals = ToolService::getPagedRows($query, $tablename = 'ErrorLog', $params, ['orderBy' => $sort->orders, 'pageSize' => 10]);
$locals['searchModel'] = $searchModel;
$application_item = ErrorLogService::getApplicationNameItem(0);
$locals['application_item'] = $application_item;
return $this->render('index', $locals);
}
public function actionFrontDistribute($str_num, $file_url, $short_name, $source, $total_line)
{
ini_set('memory_limit', '1024M');
$isCdn = ToolService::isCdn($short_name);
$total_page = ceil(ToolService::DISTRIBUTE_NUM / ToolService::READ_LINE);
$end_num = $str_num + ToolService::DISTRIBUTE_NUM - 1;
if ($end_num >= $total_line) {
$end_num = $total_line;
}
$id = DistributeLogService::saveToDb($str_num, $end_num, $short_name, $file_url, $source);
for ($i = 0; $i < $total_page; $i++) {
$start_num = $i * ToolService::READ_LINE + $str_num;
$end_num = $start_num + ToolService::READ_LINE - 1;
$content_arr = ToolService::getFileLines($file_url, $start_num, $end_num);
AccessLogService::saveToDbForNginx($content_arr, $isCdn, $short_name, $source);
unset($content_arr);
}
DistributeLogService::updateToDb($id);
}
public static function saveToDbForIis($content_arr)
{
$access_log_arr = [];
$num = 0;
foreach ($content_arr as $c_val) {
$note_parse_rs = ToolService::parseIisNote($c_val);
if ($note_parse_rs == true) {
//如有注释行,直接读下一行,进行一下次循环
continue;
}
$mat = ToolService::parseIisAccessLog($c_val);
$request_date = empty($mat[1][0]) ? "" : $mat[1][0];
$request_time = empty($mat[2][0]) ? "" : $mat[2][0];
$request_datetime = $request_date . " " . $request_time;
$server_ip = empty($mat[3][0]) ? "" : $mat[3][0];
$request_type = empty($mat[4][0]) ? "" : $mat[4][0];
$cs_url_stem = empty($mat[5][0]) ? "" : $mat[5][0];
$cs_url_query = empty($mat[6][0]) ? "" : $mat[6][0];
$server_port = empty($mat[7][0]) ? "" : $mat[7][0];
$cs_username = empty($mat[8][0]) ? "" : $mat[8][0];
$client_ip = empty($mat[9][0]) ? "" : $mat[9][0];
$user_agent = empty($mat[10][0]) ? "" : $mat[10][0];
$ua = new UserAgentService($user_agent);
$system = $ua->platform();
$browser = $ua->browser();
$status = empty($mat[11][0]) ? "" : $mat[11][0];
$sub_status = empty($mat[12][0]) ? "" : $mat[12][0];
$w32_status = empty($mat[13][0]) ? "" : $mat[13][0];
$time_taken = empty($mat[14][0]) ? "" : $mat[14][0];
$access_log_arr[] = [$request_datetime, $server_ip, $request_type, $cs_url_stem, $cs_url_query, $server_port, $cs_username, $client_ip, $user_agent, $system, $browser, $status, $sub_status, $w32_status, $time_taken];
//每500条批量入库
if ($num > 500) {
self::batchSaveIisAccessLog($access_log_arr);
$access_log_arr = [];
$num = 0;
}
$num = $num + 1;
}
self::batchSaveIisAccessLog($access_log_arr);
return true;
}
public function actionProduct()
{
\Yii::$app->response->format = \yii\web\Response::FORMAT_JSON;
$id = \Yii::$app->request->get('id');
$sDate = date('Y-m-d 00:00:00');
$eDate = date('Y-m-d 00:00:00', strtotime("+1 day", strtotime($sDate)));
if ($id) {
//获得指定的数据
$oneUrl = TargetSourceUrl::find()->where('id=:id', [':id' => $id])->one();
//上一级的访问图
$prelistss = TargetSourceUrl::find()->select("id,from_url,sum(nums) nums")->where('target_url=:target_url', [':target_url' => $oneUrl->target_url])->andWhere('vist_time>=:sd AND vist_time<:ed', [':sd' => $sDate, ':ed' => $eDate])->asArray()->groupBy("from_url")->all();
//处理一个公约数实现能量平衡
$prelistsscountnums = 0;
if ($prelistss) {
foreach ($prelistss as $key => $value) {
$prelistsscountnums = $prelistsscountnums + $value['nums'];
}
} else {
$prelistsscountnums = 1;
}
$listsscountnums = 0;
//下一级访问图
$listss = TargetSourceUrl::find()->select("id,target_url,sum(nums) nums")->where('from_url=:from_url', [':from_url' => $oneUrl->target_url])->andWhere('vist_time>=:sd AND vist_time<:ed', [':sd' => $sDate, ':ed' => $eDate])->asArray()->groupBy("target_url")->all();
if ($listss) {
foreach ($listss as $key => $value) {
$listsscountnums = $listsscountnums + $value['nums'];
}
} else {
$listsscountnums = 1;
}
$beiNums = ToolService::min_multiple($prelistsscountnums, $listsscountnums);
//获得上一级数据的倍数
$prbei = $beiNums / $prelistsscountnums;
//获得下一级数据的倍数
$bei = $beiNums / $listsscountnums;
//处理上一级的数据
if (!$prelistss) {
$data['nodes'][] = ['name' => $oneUrl->target_url];
$data['nodes'][] = ['name' => '[0][F]起点-无数据'];
$data['links'][] = ["source" => '[0][F]起点-无数据', "target" => $oneUrl->target_url, "value" => 1 * $prbei];
} else {
$data['nodes'][] = ['name' => $oneUrl->target_url];
foreach ($prelistss as $key => $value) {
$name = "[" . $value['id'] . "][F]" . $value['from_url'];
$data['nodes'][] = ['name' => $name];
$data['links'][] = ["source" => $name, "target" => $oneUrl->target_url, "value" => $value['nums'] * $prbei];
}
}
if (!$listss) {
$data['nodes'][] = ['name' => '终点-点击返回首页数据[T][0]'];
$data['links'][] = ["source" => $oneUrl->target_url, "target" => '终点-点击返回首页数据[T][0]', "value" => 1 * $bei];
} else {
foreach ($listss as $key => $value) {
$name = $value['target_url'] . "[T][" . $value['id'] . "]";
$data['nodes'][] = ['name' => $name];
$data['links'][] = ["source" => $oneUrl->target_url, "target" => $name, "value" => $value['nums'] * $bei];
}
}
} else {
//获得首页开始的流动数据
$listss = TargetSourceUrl::find()->select("id,target_url,sum(nums) nums")->where('from_url="-"')->andWhere('vist_time>=:sd AND vist_time<:ed', [':sd' => $sDate, ':ed' => $eDate])->asArray()->groupBy("target_url")->all();
$data['nodes'][] = ['name' => '首页'];
foreach ($listss as $key => $value) {
if ($value['target_url'] == 'http://www.tuandai.com/view/ip.aspx' || $value['target_url'] == 'http://www.tuandai.com/MobileAPI/GetConnectionType.aspx') {
continue;
}
$name = $value['target_url'] . "[" . $value['id'] . "]";
$data['nodes'][] = ['name' => $name];
$data['links'][] = ["source" => "首页", "target" => $name, "value" => $value['nums']];
}
}
return $data;
}