function awstats()
{
$sock = new sockets();
$unix = new unix();
$pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid";
$pidTime = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".time";
if ($unix->file_time_min($pidTime) < 60) {
return;
}
$pid = $unix->get_pid_from_file($pidfile);
if ($unix->process_exists($pid, basename(__FILE__))) {
$time = $unix->PROCCESS_TIME_MIN($pid);
return;
}
@file_put_contents($pidfile, getmypid());
@unlink($pidTime);
@file_put_contents($pidTime, time());
$sock = new sockets();
$EnableNginxStats = $sock->GET_INFO("EnableNginxStats");
if (!is_numeric($EnableNginxStats)) {
$EnableNginxStats = 0;
}
if ($EnableNginxStats == 1) {
return;
}
include_once dirname(__FILE__) . "/ressources/class.awstats.inc";
include_once dirname(__FILE__) . "/ressources/class.mysql.syslogs.inc";
$awstats_bin = $unix->LOCATE_AWSTATS_BIN();
$nice = EXEC_NICE();
$perl = $unix->find_program("perl");
$awstats_buildstaticpages = $unix->LOCATE_AWSTATS_BUILDSTATICPAGES_BIN();
if ($GLOBALS["VERBOSE"]) {
echo "awstats......: {$awstats_bin}\n";
echo "statics Pages: {$awstats_buildstaticpages}\n";
echo "Nice.........: {$nice}\n";
echo "perl.........: {$perl}\n";
}
if (!is_file($awstats_buildstaticpages)) {
echo "buildstaticpages no such binary...\n";
return;
}
$sock = new sockets();
$kill = $unix->find_program("kill");
$NginxWorkLogsDir = $sock->GET_INFO("NginxWorkLogsDir");
if ($NginxWorkLogsDir == null) {
$NginxWorkLogsDir = "/home/nginx/logsWork";
}
$sys = new mysql_storelogs();
$files = $unix->DirFiles($NginxWorkLogsDir, "-([0-9\\-]+)\\.log");
while (list($filename, $line) = each($files)) {
if (!preg_match("#^(.+?)-[0-9]+-[0-9]+-[0-9]+-[0-9]+\\.log\$#", $filename, $re)) {
if ($GLOBALS["VERBOSE"]) {
echo "{$filename}, skip\n";
}
continue;
}
if ($GLOBALS["VERBOSE"]) {
echo "{$filename}, domain:{$re[1]}\n";
}
$servername = $re[1];
$GLOBALS["nice"] = $nice;
$aw = new awstats($servername);
$aw->set_LogFile("{$NginxWorkLogsDir}/{$filename}");
$aw->set_LogType("W");
$aw->set_LogFormat(1);
$config = $aw->buildconf();
$SOURCE_FILE_PATH = "{$NginxWorkLogsDir}/{$filename}";
$configlength = strlen($config);
if ($configlength < 10) {
if ($GLOBALS["VERBOSE"]) {
echo "configuration file lenght failed {$configlength} bytes, aborting {$servername}\n";
}
return;
}
@file_put_contents("/etc/awstats/awstats.{$servername}.conf", $config);
@chmod("/etc/awstats/awstats.{$servername}.conf", 644);
$Lang = $aw->GET("Lang");
if ($Lang == null) {
$Lang = "auto";
}
@mkdir("/var/tmp/awstats/{$servername}", 666, true);
$t1 = time();
$cmd = "{$nice}{$perl} {$awstats_buildstaticpages} -config={$servername} -update -lang={$Lang} -awstatsprog={$awstats_bin} -dir=/var/tmp/awstats/{$servername} -LogFile=\"{$SOURCE_FILE_PATH}\" 2>&1";
if ($GLOBALS["VERBOSE"]) {
echo $cmd . "\n";
}
shell_exec($cmd);
$filedate = date('Y-m-d H:i:s', filemtime($SOURCE_FILE_PATH));
if (!awstats_import_sql($servername)) {
continue;
}
$sys->ROTATE_TOMYSQL($SOURCE_FILE_PATH, $filedate);
}
}
function CleanRotatedFiles()
{
include_once dirname(__FILE__) . "/ressources/class.mysql.syslog.inc";
$unix = new unix();
$sock = new sockets();
$LogRotateCompress = 1;
$LogRotatePath = $sock->GET_INFO("LogRotatePath");
$ApacheLogRotate = $sock->GET_INFO("ApacheLogRotate");
$cpbin = $unix->find_program("cp");
$php5 = $unix->LOCATE_PHP5_BIN();
$tmpdir = $unix->TEMP_DIR();
if (!is_numeric($ApacheLogRotate)) {
$ApacheLogRotate = 1;
}
if (!is_numeric($LogRotatePath)) {
$LogRotatePath = "/home/logrotate";
}
$DirsToScan["/var/log"] = true;
$DirsToScan["/var/log/apache2"] = true;
$DirsToScan["/var/log/lighttpd"] = true;
$DirsToScan["/var/log/ejabberd"] = true;
$apache2 = $unix->dirdir("/var/log/apache2");
while (list($WorkingDir, $ligne) = each($apache2)) {
$DirsToScan[$WorkingDir] = true;
}
$q = new mysql_storelogs();
while (list($WorkingDir, $ligne) = each($DirsToScan)) {
$RotateSquid = false;
if ($WorkingDir == "/var/log/squid") {
continue;
}
$table = $unix->DirFiles($WorkingDir, "(\\.|-)[0-9]+.*?\$");
$compressed["gz"] = true;
$compressed["bz"] = true;
$compressed["bz2"] = true;
while (list($filename, $ligne) = each($table)) {
$path = "{$WorkingDir}/{$filename}";
if ($unix->file_time_min($path) < 1440) {
continue;
}
$filedate = date('Y-m-d H:i:s', filemtime($path));
$q->events("Injecting {$path} {$filedate}");
if (!$q->ROTATE_TOMYSQL($path, $filedate)) {
continue;
}
}
}
}
function rotate($filename)
{
$filename = basename($filename);
$filePath = "/var/log/squid/{$filename}";
$unix = new unix();
$pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".{$filename}.pid";
$pid = @file_get_contents("{$pidfile}");
if ($unix->process_exists($pid, basename(__FILE__))) {
events("Process {$pid} already exists...aborting");
die;
}
include_once dirname(__FILE__) . "/ressources/class.mysql.syslogs.inc";
$sarg_bin = $unix->find_program("sarg");
$q = new mysql_storelogs();
if (!is_file($filePath)) {
events("{$filePath} no such file");
}
if (!is_file($sarg_bin)) {
sarg_admin_events("Fatal, ({$filePath}) unable to locate sarg binary, aborting...", __FUNCTION__, __FILE__, __LINE__, "sarg");
$q->ROTATE_TOMYSQL($filePath);
return;
}
$t = time();
sargToFile($filePath);
$q->ROTATE_TOMYSQL($filePath);
progress("{$filename} done " . $unix->distanceOfTimeInWords($t, time()));
backup();
}
function ParseFile($servername, $fullpath)
{
events("[{$servername}]: Parsing {$fullpath}", __FUNCTION__, __LINE__);
$unix = new unix();
$size = @filesize($fullpath);
events("[{$servername}]: open {$fullpath} {$size} bytes", __FUNCTION__, __LINE__);
$handle = @fopen($fullpath, "r");
if (!$handle) {
events("[{$servername}]: open {$fullpath} fatal, unable to open ", __FUNCTION__, __LINE__);
return;
}
$c = 0;
$d = 0;
$t = time();
$WORKARRAY = array();
while (!feof($handle)) {
$d++;
$line = trim(fgets($handle, 4096));
if ($line == null) {
continue;
}
if (!preg_match('#(.*?)\\s+(.*?)\\s+(.*?)\\s+\\[(.*?)\\]\\s+([A-Z]+)\\s+(.*?)\\s+HTTP.*?\\/.*?"([0-9]+)"\\s+([0-9]+)\\s+"(.*?)"\\s+"(.*?)"\\s+"(.*?)"#', $line, $re)) {
events("[{$servername}]: {{$line}} unable to parse...", __FUNCTION__, __LINE__);
continue;
}
while (list($a, $b) = each($re)) {
$re[$a] = mysql_escape_string2($b);
}
$c++;
$md5 = md5($re[0]);
$ipaddr = $re[1];
$time = strtotime($re[4]);
$proto = $re[5];
$uri = $re[6];
$code = $re[7];
$size = $re[8];
$UserAgent = $re[10];
$Country = mysql_escape_string2(GeoLoc($ipaddr));
$currDate = date("Y-m-d H:i:s");
$linesql = "('{$md5}','{$currDate}','{$ipaddr}','{$proto}','{$uri}','{$code}','{$size}','{$UserAgent}','{$Country}')";
$table = "hour_" . date("YmdH", $time);
$WORKARRAY[$table][] = $linesql;
if ($c > 500) {
if (!ParseArray($servername, $WORKARRAY)) {
return;
}
$WORKARRAY = array();
$c = 0;
}
}
if (count($WORKARRAY) > 0) {
if (!ParseArray($servername, $WORKARRAY)) {
return;
}
}
$timeTOScan = $unix->distanceOfTimeInWords($t, time(), true);
events("[{$servername}]: {$fullpath} {$timeTOScan} {$d} lines", __FUNCTION__, __LINE__);
if ($d == 0) {
@unlink($fullpath);
}
$sys = new mysql_storelogs();
$filedate = date('Y-m-d H:i:s', filemtime($fullpath));
$sys->ROTATE_TOMYSQL($fullpath, $filedate);
}
function check_all_squid()
{
$sock = new sockets();
$unix = new unix();
$syslog = new mysql_storelogs();
$php5 = $unix->LOCATE_PHP5_BIN();
$pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid";
$timefile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".time";
$pid = file_get_contents("{$pidfile}");
if (system_is_overloaded(basename(__FILE__))) {
die;
}
if ($unix->process_exists($pid, basename(__FILE__))) {
$timeMin = $unix->PROCCESS_TIME_MIN($pid);
if ($timeMin > 240) {
system_admin_events("Too many TTL, {$pid} will be killed", __FUNCTION__, __FILE__, __LINE__, "logrotate");
$kill = $unix->find_program("kill");
unix_system_kill_force($pid);
} else {
die;
}
}
$time = $unix->file_time_min($timefile);
if ($time < 300) {
return;
}
@file_put_contents($pidfile, getmypid());
@file_put_contents($timefile, time());
$php = $unix->LOCATE_PHP5_BIN();
$bzip2 = $unix->find_program("bzip2");
$ALREADYCOMP["gz"] = true;
$ALREADYCOMP["bz2"] = true;
$LogRotateCompress = 1;
$LogRotatePath = $sock->GET_INFO("LogRotatePath");
$ApacheLogRotate = $sock->GET_INFO("ApacheLogRotate");
if (!is_numeric($ApacheLogRotate)) {
$ApacheLogRotate = 1;
}
if ($LogRotatePath == null) {
$LogRotatePath = "/home/logrotate";
}
$LogsRotateDefaultSizeRotation = $sock->GET_INFO("LogsRotateDefaultSizeRotation");
if (!is_numeric($LogsRotateDefaultSizeRotation)) {
$LogsRotateDefaultSizeRotation = 100;
}
foreach (glob("/var/log/squid/*") as $filename) {
if (is_dir($filename)) {
continue;
}
$size = $unix->file_size($filename);
$time = $unix->file_time_min($filename);
$size = round($size / 1024 / 1000, 2);
if ($size > $LogsRotateDefaultSizeRotation) {
if ($filename == "/var/log/squid/access.log") {
events("{$filename} -> is a production log for Squid, launch the rotation procedure.");
squid_admin_mysql(1, "{$filename} {$size}M exceed {$LogsRotateDefaultSizeRotation}M, launch rotation", null, __FILE__, __LINE__);
shell_exec("{$php} /usr/share/artica-postfix/exec.squid.php --rotate");
continue;
}
$TOROT[$filename] = true;
events("{$filename} -> Add to queue {$size}M exceed {$LogsRotateDefaultSizeRotation}M");
continue;
}
if ($time > 1440) {
if ($filename == "/var/log/squid/access.log") {
events("{$filename} -> is a production log for Squid, launch the rotation procedure.");
squid_admin_mysql(1, "{$filename} {$size}M exceed {$LogsRotateDefaultSizeRotation}M, launch rotation", null, __FILE__, __LINE__);
shell_exec("{$php} /usr/share/artica-postfix/exec.squid.php --rotate");
continue;
}
events("{$filename} -> Add to queue {$time}mn exceed 1440mn");
$TOROT[$filename] = true;
continue;
}
}
if (count($TOROT) == 0) {
return;
}
while (list($filename, $none) = each($TOROT)) {
$extension = pathinfo($filename, PATHINFO_EXTENSION);
$filedate = date('Y-m-d H:i:s', filemtime($filename));
$basename = basename($filename);
if (preg_match("#sarg\\.#", $filename)) {
shell_exec("{$php5} " . dirname(__FILE__) . "/exec.sarg.php --rotate {$basename} >/dev/null 2>&1 &");
continue;
}
if (preg_match("#access\\.log\\.[0-9]+\$#", $filename)) {
continue;
}
if ($extension != "gz") {
if (!$unix->compress($filename, "{$filename}.gz")) {
continue;
}
$filename = $filename . ".gz";
$extension = "gz";
}
echo "[{$filedate}]: {$filename} ({$extension})\n";
if ($syslog->ROTATE_TOMYSQL($filename, $filedate)) {
@unlink($filename);
}
}
foreach (glob("/home/squid/cache-logs/*") as $filename) {
$filedate = date('Y-m-d H:i:s', filemtime($filename));
$filename = $filename . ".gz";
if ($syslog->ROTATE_TOMYSQL($filename, $filedate)) {
@unlink($filename);
}
}
}
function start_import($aspid = false)
{
$sock = new sockets();
$syslog = new mysql_storelogs();
$unix = new unix();
$pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid";
$unix = new unix();
if (!$aspid) {
$pid = @file_get_contents($pidfile);
if ($unix->process_exists($pid, basename(__FILE__))) {
$time = $unix->PROCESS_TTL($pid);
stats_admin_events(2, "A already Importation Task is executed pid:{$pid} since {$time}Mn", null, __FILE__, __LINE__);
writelogs("Already executed pid:{$pid}", __FUNCTION__, __FILE__, __LINE__);
return;
}
$pid = @file_get_contents("/var/run/squid-stats-central.pid");
if ($unix->process_exists($pid, basename(__FILE__))) {
$time = $unix->PROCESS_TTL($pid);
stats_admin_events(2, "An anlready Importation Task is executed pid:{$pid} since {$time}Mn", null, __FILE__, __LINE__);
writelogs("Already executed pid:{$pid}", __FUNCTION__, __FILE__, __LINE__);
return;
}
}
$import_processes = import_processes();
if ($import_processes > 6) {
Import_logs("Too many processes ({$import_processes})...aborting task...");
return;
}
$mypid = getmypid();
@file_put_contents($pidfile, $mypid);
$ArticaProxyStatisticsBackupFolder = $sock->GET_INFO("ArticaProxyStatisticsBackupFolder");
if ($ArticaProxyStatisticsBackupFolder == null) {
$ArticaProxyStatisticsBackupFolder = "/home/artica/squid/backup-statistics";
}
$ArticaProxyStatisticsBackupFolder = $ArticaProxyStatisticsBackupFolder . "/import";
$files = $unix->DirFiles($ArticaProxyStatisticsBackupFolder);
if ($GLOBALS["VERBOSE"]) {
echo "PUSH Scanning {$ArticaProxyStatisticsBackupFolder}\n";
}
$mysql = $unix->find_program("mysql");
$bzip2 = $unix->find_program("bzip2");
$total = count($files);
$c = 0;
if ($total == 0) {
return;
}
stats_admin_events(2, "Importing {$total} files to MySQL", null, __FILE__, __LINE__);
while (list($filename, $none) = each($files)) {
$c++;
if (if_process_import_exists($filename)) {
Import_logs("SKIP {$ArticaProxyStatisticsBackupFolder}/{$filename} Already running...");
continue;
}
percentage("Extracting {$filename} {$c}/{$total}", 2);
$size = @filesize("{$ArticaProxyStatisticsBackupFolder}/{$filename}");
Import_logs("IMPORT {$ArticaProxyStatisticsBackupFolder}/{$filename}");
$f = array();
$results = array();
$f[] = "{$bzip2} -d -c {$ArticaProxyStatisticsBackupFolder}/{$filename} |";
$f[] = "{$mysql} --show-warnings";
$f[] = "--socket=/var/run/mysqld/squid-db.sock";
$f[] = "--protocol=socket --user=root --batch --force";
$f[] = "--debug-info --database=squidlogs 2>&1";
$cmd = @implode(" ", $f);
$results[] = $cmd;
percentage("Importing {$filename} to MySQL {$c}/{$total}", 2);
exec($cmd, $results);
Import_logs(@implode("\n", $results));
stats_admin_events(2, "Success importing {$filename} to MySQL", @implode("\n", $results), __FILE__, __LINE__);
Import_logs("Backup {$ArticaProxyStatisticsBackupFolder}/{$filename}");
$syslog->ROTATE_TOMYSQL("{$ArticaProxyStatisticsBackupFolder}/{$filename}");
}
}
