function awstats() { $sock = new sockets(); $unix = new unix(); $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid"; $pidTime = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".time"; if ($unix->file_time_min($pidTime) < 60) { return; } $pid = $unix->get_pid_from_file($pidfile); if ($unix->process_exists($pid, basename(__FILE__))) { $time = $unix->PROCCESS_TIME_MIN($pid); return; } @file_put_contents($pidfile, getmypid()); @unlink($pidTime); @file_put_contents($pidTime, time()); $sock = new sockets(); $EnableNginxStats = $sock->GET_INFO("EnableNginxStats"); if (!is_numeric($EnableNginxStats)) { $EnableNginxStats = 0; } if ($EnableNginxStats == 1) { return; } include_once dirname(__FILE__) . "/ressources/class.awstats.inc"; include_once dirname(__FILE__) . "/ressources/class.mysql.syslogs.inc"; $awstats_bin = $unix->LOCATE_AWSTATS_BIN(); $nice = EXEC_NICE(); $perl = $unix->find_program("perl"); $awstats_buildstaticpages = $unix->LOCATE_AWSTATS_BUILDSTATICPAGES_BIN(); if ($GLOBALS["VERBOSE"]) { echo "awstats......: {$awstats_bin}\n"; echo "statics Pages: {$awstats_buildstaticpages}\n"; echo "Nice.........: {$nice}\n"; echo "perl.........: {$perl}\n"; } if (!is_file($awstats_buildstaticpages)) { echo "buildstaticpages no such binary...\n"; return; } $sock = new sockets(); $kill = $unix->find_program("kill"); $NginxWorkLogsDir = $sock->GET_INFO("NginxWorkLogsDir"); if ($NginxWorkLogsDir == null) { $NginxWorkLogsDir = "/home/nginx/logsWork"; } $sys = new mysql_storelogs(); $files = $unix->DirFiles($NginxWorkLogsDir, "-([0-9\\-]+)\\.log"); while (list($filename, $line) = each($files)) { if (!preg_match("#^(.+?)-[0-9]+-[0-9]+-[0-9]+-[0-9]+\\.log\$#", $filename, $re)) { if ($GLOBALS["VERBOSE"]) { echo "{$filename}, skip\n"; } continue; } if ($GLOBALS["VERBOSE"]) { echo "{$filename}, domain:{$re[1]}\n"; } $servername = $re[1]; $GLOBALS["nice"] = $nice; $aw = new awstats($servername); $aw->set_LogFile("{$NginxWorkLogsDir}/{$filename}"); $aw->set_LogType("W"); $aw->set_LogFormat(1); $config = $aw->buildconf(); $SOURCE_FILE_PATH = "{$NginxWorkLogsDir}/{$filename}"; $configlength = strlen($config); if ($configlength < 10) { if ($GLOBALS["VERBOSE"]) { echo "configuration file lenght failed {$configlength} bytes, aborting {$servername}\n"; } return; } @file_put_contents("/etc/awstats/awstats.{$servername}.conf", $config); @chmod("/etc/awstats/awstats.{$servername}.conf", 644); $Lang = $aw->GET("Lang"); if ($Lang == null) { $Lang = "auto"; } @mkdir("/var/tmp/awstats/{$servername}", 666, true); $t1 = time(); $cmd = "{$nice}{$perl} {$awstats_buildstaticpages} -config={$servername} -update -lang={$Lang} -awstatsprog={$awstats_bin} -dir=/var/tmp/awstats/{$servername} -LogFile=\"{$SOURCE_FILE_PATH}\" 2>&1"; if ($GLOBALS["VERBOSE"]) { echo $cmd . "\n"; } shell_exec($cmd); $filedate = date('Y-m-d H:i:s', filemtime($SOURCE_FILE_PATH)); if (!awstats_import_sql($servername)) { continue; } $sys->ROTATE_TOMYSQL($SOURCE_FILE_PATH, $filedate); } }
function CleanRotatedFiles() { include_once dirname(__FILE__) . "/ressources/class.mysql.syslog.inc"; $unix = new unix(); $sock = new sockets(); $LogRotateCompress = 1; $LogRotatePath = $sock->GET_INFO("LogRotatePath"); $ApacheLogRotate = $sock->GET_INFO("ApacheLogRotate"); $cpbin = $unix->find_program("cp"); $php5 = $unix->LOCATE_PHP5_BIN(); $tmpdir = $unix->TEMP_DIR(); if (!is_numeric($ApacheLogRotate)) { $ApacheLogRotate = 1; } if (!is_numeric($LogRotatePath)) { $LogRotatePath = "/home/logrotate"; } $DirsToScan["/var/log"] = true; $DirsToScan["/var/log/apache2"] = true; $DirsToScan["/var/log/lighttpd"] = true; $DirsToScan["/var/log/ejabberd"] = true; $apache2 = $unix->dirdir("/var/log/apache2"); while (list($WorkingDir, $ligne) = each($apache2)) { $DirsToScan[$WorkingDir] = true; } $q = new mysql_storelogs(); while (list($WorkingDir, $ligne) = each($DirsToScan)) { $RotateSquid = false; if ($WorkingDir == "/var/log/squid") { continue; } $table = $unix->DirFiles($WorkingDir, "(\\.|-)[0-9]+.*?\$"); $compressed["gz"] = true; $compressed["bz"] = true; $compressed["bz2"] = true; while (list($filename, $ligne) = each($table)) { $path = "{$WorkingDir}/{$filename}"; if ($unix->file_time_min($path) < 1440) { continue; } $filedate = date('Y-m-d H:i:s', filemtime($path)); $q->events("Injecting {$path} {$filedate}"); if (!$q->ROTATE_TOMYSQL($path, $filedate)) { continue; } } } }
function rotate($filename) { $filename = basename($filename); $filePath = "/var/log/squid/{$filename}"; $unix = new unix(); $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".{$filename}.pid"; $pid = @file_get_contents("{$pidfile}"); if ($unix->process_exists($pid, basename(__FILE__))) { events("Process {$pid} already exists...aborting"); die; } include_once dirname(__FILE__) . "/ressources/class.mysql.syslogs.inc"; $sarg_bin = $unix->find_program("sarg"); $q = new mysql_storelogs(); if (!is_file($filePath)) { events("{$filePath} no such file"); } if (!is_file($sarg_bin)) { sarg_admin_events("Fatal, ({$filePath}) unable to locate sarg binary, aborting...", __FUNCTION__, __FILE__, __LINE__, "sarg"); $q->ROTATE_TOMYSQL($filePath); return; } $t = time(); sargToFile($filePath); $q->ROTATE_TOMYSQL($filePath); progress("{$filename} done " . $unix->distanceOfTimeInWords($t, time())); backup(); }
function ParseFile($servername, $fullpath) { events("[{$servername}]: Parsing {$fullpath}", __FUNCTION__, __LINE__); $unix = new unix(); $size = @filesize($fullpath); events("[{$servername}]: open {$fullpath} {$size} bytes", __FUNCTION__, __LINE__); $handle = @fopen($fullpath, "r"); if (!$handle) { events("[{$servername}]: open {$fullpath} fatal, unable to open ", __FUNCTION__, __LINE__); return; } $c = 0; $d = 0; $t = time(); $WORKARRAY = array(); while (!feof($handle)) { $d++; $line = trim(fgets($handle, 4096)); if ($line == null) { continue; } if (!preg_match('#(.*?)\\s+(.*?)\\s+(.*?)\\s+\\[(.*?)\\]\\s+([A-Z]+)\\s+(.*?)\\s+HTTP.*?\\/.*?"([0-9]+)"\\s+([0-9]+)\\s+"(.*?)"\\s+"(.*?)"\\s+"(.*?)"#', $line, $re)) { events("[{$servername}]: {{$line}} unable to parse...", __FUNCTION__, __LINE__); continue; } while (list($a, $b) = each($re)) { $re[$a] = mysql_escape_string2($b); } $c++; $md5 = md5($re[0]); $ipaddr = $re[1]; $time = strtotime($re[4]); $proto = $re[5]; $uri = $re[6]; $code = $re[7]; $size = $re[8]; $UserAgent = $re[10]; $Country = mysql_escape_string2(GeoLoc($ipaddr)); $currDate = date("Y-m-d H:i:s"); $linesql = "('{$md5}','{$currDate}','{$ipaddr}','{$proto}','{$uri}','{$code}','{$size}','{$UserAgent}','{$Country}')"; $table = "hour_" . date("YmdH", $time); $WORKARRAY[$table][] = $linesql; if ($c > 500) { if (!ParseArray($servername, $WORKARRAY)) { return; } $WORKARRAY = array(); $c = 0; } } if (count($WORKARRAY) > 0) { if (!ParseArray($servername, $WORKARRAY)) { return; } } $timeTOScan = $unix->distanceOfTimeInWords($t, time(), true); events("[{$servername}]: {$fullpath} {$timeTOScan} {$d} lines", __FUNCTION__, __LINE__); if ($d == 0) { @unlink($fullpath); } $sys = new mysql_storelogs(); $filedate = date('Y-m-d H:i:s', filemtime($fullpath)); $sys->ROTATE_TOMYSQL($fullpath, $filedate); }
function check_all_squid() { $sock = new sockets(); $unix = new unix(); $syslog = new mysql_storelogs(); $php5 = $unix->LOCATE_PHP5_BIN(); $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid"; $timefile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".time"; $pid = file_get_contents("{$pidfile}"); if (system_is_overloaded(basename(__FILE__))) { die; } if ($unix->process_exists($pid, basename(__FILE__))) { $timeMin = $unix->PROCCESS_TIME_MIN($pid); if ($timeMin > 240) { system_admin_events("Too many TTL, {$pid} will be killed", __FUNCTION__, __FILE__, __LINE__, "logrotate"); $kill = $unix->find_program("kill"); unix_system_kill_force($pid); } else { die; } } $time = $unix->file_time_min($timefile); if ($time < 300) { return; } @file_put_contents($pidfile, getmypid()); @file_put_contents($timefile, time()); $php = $unix->LOCATE_PHP5_BIN(); $bzip2 = $unix->find_program("bzip2"); $ALREADYCOMP["gz"] = true; $ALREADYCOMP["bz2"] = true; $LogRotateCompress = 1; $LogRotatePath = $sock->GET_INFO("LogRotatePath"); $ApacheLogRotate = $sock->GET_INFO("ApacheLogRotate"); if (!is_numeric($ApacheLogRotate)) { $ApacheLogRotate = 1; } if ($LogRotatePath == null) { $LogRotatePath = "/home/logrotate"; } $LogsRotateDefaultSizeRotation = $sock->GET_INFO("LogsRotateDefaultSizeRotation"); if (!is_numeric($LogsRotateDefaultSizeRotation)) { $LogsRotateDefaultSizeRotation = 100; } foreach (glob("/var/log/squid/*") as $filename) { if (is_dir($filename)) { continue; } $size = $unix->file_size($filename); $time = $unix->file_time_min($filename); $size = round($size / 1024 / 1000, 2); if ($size > $LogsRotateDefaultSizeRotation) { if ($filename == "/var/log/squid/access.log") { events("{$filename} -> is a production log for Squid, launch the rotation procedure."); squid_admin_mysql(1, "{$filename} {$size}M exceed {$LogsRotateDefaultSizeRotation}M, launch rotation", null, __FILE__, __LINE__); shell_exec("{$php} /usr/share/artica-postfix/exec.squid.php --rotate"); continue; } $TOROT[$filename] = true; events("{$filename} -> Add to queue {$size}M exceed {$LogsRotateDefaultSizeRotation}M"); continue; } if ($time > 1440) { if ($filename == "/var/log/squid/access.log") { events("{$filename} -> is a production log for Squid, launch the rotation procedure."); squid_admin_mysql(1, "{$filename} {$size}M exceed {$LogsRotateDefaultSizeRotation}M, launch rotation", null, __FILE__, __LINE__); shell_exec("{$php} /usr/share/artica-postfix/exec.squid.php --rotate"); continue; } events("{$filename} -> Add to queue {$time}mn exceed 1440mn"); $TOROT[$filename] = true; continue; } } if (count($TOROT) == 0) { return; } while (list($filename, $none) = each($TOROT)) { $extension = pathinfo($filename, PATHINFO_EXTENSION); $filedate = date('Y-m-d H:i:s', filemtime($filename)); $basename = basename($filename); if (preg_match("#sarg\\.#", $filename)) { shell_exec("{$php5} " . dirname(__FILE__) . "/exec.sarg.php --rotate {$basename} >/dev/null 2>&1 &"); continue; } if (preg_match("#access\\.log\\.[0-9]+\$#", $filename)) { continue; } if ($extension != "gz") { if (!$unix->compress($filename, "{$filename}.gz")) { continue; } $filename = $filename . ".gz"; $extension = "gz"; } echo "[{$filedate}]: {$filename} ({$extension})\n"; if ($syslog->ROTATE_TOMYSQL($filename, $filedate)) { @unlink($filename); } } foreach (glob("/home/squid/cache-logs/*") as $filename) { $filedate = date('Y-m-d H:i:s', filemtime($filename)); $filename = $filename . ".gz"; if ($syslog->ROTATE_TOMYSQL($filename, $filedate)) { @unlink($filename); } } }
function start_import($aspid = false) { $sock = new sockets(); $syslog = new mysql_storelogs(); $unix = new unix(); $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid"; $unix = new unix(); if (!$aspid) { $pid = @file_get_contents($pidfile); if ($unix->process_exists($pid, basename(__FILE__))) { $time = $unix->PROCESS_TTL($pid); stats_admin_events(2, "A already Importation Task is executed pid:{$pid} since {$time}Mn", null, __FILE__, __LINE__); writelogs("Already executed pid:{$pid}", __FUNCTION__, __FILE__, __LINE__); return; } $pid = @file_get_contents("/var/run/squid-stats-central.pid"); if ($unix->process_exists($pid, basename(__FILE__))) { $time = $unix->PROCESS_TTL($pid); stats_admin_events(2, "An anlready Importation Task is executed pid:{$pid} since {$time}Mn", null, __FILE__, __LINE__); writelogs("Already executed pid:{$pid}", __FUNCTION__, __FILE__, __LINE__); return; } } $import_processes = import_processes(); if ($import_processes > 6) { Import_logs("Too many processes ({$import_processes})...aborting task..."); return; } $mypid = getmypid(); @file_put_contents($pidfile, $mypid); $ArticaProxyStatisticsBackupFolder = $sock->GET_INFO("ArticaProxyStatisticsBackupFolder"); if ($ArticaProxyStatisticsBackupFolder == null) { $ArticaProxyStatisticsBackupFolder = "/home/artica/squid/backup-statistics"; } $ArticaProxyStatisticsBackupFolder = $ArticaProxyStatisticsBackupFolder . "/import"; $files = $unix->DirFiles($ArticaProxyStatisticsBackupFolder); if ($GLOBALS["VERBOSE"]) { echo "PUSH Scanning {$ArticaProxyStatisticsBackupFolder}\n"; } $mysql = $unix->find_program("mysql"); $bzip2 = $unix->find_program("bzip2"); $total = count($files); $c = 0; if ($total == 0) { return; } stats_admin_events(2, "Importing {$total} files to MySQL", null, __FILE__, __LINE__); while (list($filename, $none) = each($files)) { $c++; if (if_process_import_exists($filename)) { Import_logs("SKIP {$ArticaProxyStatisticsBackupFolder}/{$filename} Already running..."); continue; } percentage("Extracting {$filename} {$c}/{$total}", 2); $size = @filesize("{$ArticaProxyStatisticsBackupFolder}/{$filename}"); Import_logs("IMPORT {$ArticaProxyStatisticsBackupFolder}/{$filename}"); $f = array(); $results = array(); $f[] = "{$bzip2} -d -c {$ArticaProxyStatisticsBackupFolder}/{$filename} |"; $f[] = "{$mysql} --show-warnings"; $f[] = "--socket=/var/run/mysqld/squid-db.sock"; $f[] = "--protocol=socket --user=root --batch --force"; $f[] = "--debug-info --database=squidlogs 2>&1"; $cmd = @implode(" ", $f); $results[] = $cmd; percentage("Importing {$filename} to MySQL {$c}/{$total}", 2); exec($cmd, $results); Import_logs(@implode("\n", $results)); stats_admin_events(2, "Success importing {$filename} to MySQL", @implode("\n", $results), __FILE__, __LINE__); Import_logs("Backup {$ArticaProxyStatisticsBackupFolder}/{$filename}"); $syslog->ROTATE_TOMYSQL("{$ArticaProxyStatisticsBackupFolder}/{$filename}"); } }