Exemple #1
0
function awstats()
{
    $sock = new sockets();
    $unix = new unix();
    $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid";
    $pidTime = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".time";
    if ($unix->file_time_min($pidTime) < 60) {
        return;
    }
    $pid = $unix->get_pid_from_file($pidfile);
    if ($unix->process_exists($pid, basename(__FILE__))) {
        $time = $unix->PROCCESS_TIME_MIN($pid);
        return;
    }
    @file_put_contents($pidfile, getmypid());
    @unlink($pidTime);
    @file_put_contents($pidTime, time());
    $sock = new sockets();
    $EnableNginxStats = $sock->GET_INFO("EnableNginxStats");
    if (!is_numeric($EnableNginxStats)) {
        $EnableNginxStats = 0;
    }
    if ($EnableNginxStats == 1) {
        return;
    }
    include_once dirname(__FILE__) . "/ressources/class.awstats.inc";
    include_once dirname(__FILE__) . "/ressources/class.mysql.syslogs.inc";
    $awstats_bin = $unix->LOCATE_AWSTATS_BIN();
    $nice = EXEC_NICE();
    $perl = $unix->find_program("perl");
    $awstats_buildstaticpages = $unix->LOCATE_AWSTATS_BUILDSTATICPAGES_BIN();
    if ($GLOBALS["VERBOSE"]) {
        echo "awstats......: {$awstats_bin}\n";
        echo "statics Pages: {$awstats_buildstaticpages}\n";
        echo "Nice.........: {$nice}\n";
        echo "perl.........: {$perl}\n";
    }
    if (!is_file($awstats_buildstaticpages)) {
        echo "buildstaticpages no such binary...\n";
        return;
    }
    $sock = new sockets();
    $kill = $unix->find_program("kill");
    $NginxWorkLogsDir = $sock->GET_INFO("NginxWorkLogsDir");
    if ($NginxWorkLogsDir == null) {
        $NginxWorkLogsDir = "/home/nginx/logsWork";
    }
    $sys = new mysql_storelogs();
    $files = $unix->DirFiles($NginxWorkLogsDir, "-([0-9\\-]+)\\.log");
    while (list($filename, $line) = each($files)) {
        if (!preg_match("#^(.+?)-[0-9]+-[0-9]+-[0-9]+-[0-9]+\\.log\$#", $filename, $re)) {
            if ($GLOBALS["VERBOSE"]) {
                echo "{$filename}, skip\n";
            }
            continue;
        }
        if ($GLOBALS["VERBOSE"]) {
            echo "{$filename}, domain:{$re[1]}\n";
        }
        $servername = $re[1];
        $GLOBALS["nice"] = $nice;
        $aw = new awstats($servername);
        $aw->set_LogFile("{$NginxWorkLogsDir}/{$filename}");
        $aw->set_LogType("W");
        $aw->set_LogFormat(1);
        $config = $aw->buildconf();
        $SOURCE_FILE_PATH = "{$NginxWorkLogsDir}/{$filename}";
        $configlength = strlen($config);
        if ($configlength < 10) {
            if ($GLOBALS["VERBOSE"]) {
                echo "configuration file lenght failed {$configlength} bytes, aborting {$servername}\n";
            }
            return;
        }
        @file_put_contents("/etc/awstats/awstats.{$servername}.conf", $config);
        @chmod("/etc/awstats/awstats.{$servername}.conf", 644);
        $Lang = $aw->GET("Lang");
        if ($Lang == null) {
            $Lang = "auto";
        }
        @mkdir("/var/tmp/awstats/{$servername}", 666, true);
        $t1 = time();
        $cmd = "{$nice}{$perl} {$awstats_buildstaticpages} -config={$servername} -update -lang={$Lang} -awstatsprog={$awstats_bin} -dir=/var/tmp/awstats/{$servername} -LogFile=\"{$SOURCE_FILE_PATH}\" 2>&1";
        if ($GLOBALS["VERBOSE"]) {
            echo $cmd . "\n";
        }
        shell_exec($cmd);
        $filedate = date('Y-m-d H:i:s', filemtime($SOURCE_FILE_PATH));
        if (!awstats_import_sql($servername)) {
            continue;
        }
        $sys->ROTATE_TOMYSQL($SOURCE_FILE_PATH, $filedate);
    }
}
function CleanRotatedFiles()
{
    include_once dirname(__FILE__) . "/ressources/class.mysql.syslog.inc";
    $unix = new unix();
    $sock = new sockets();
    $LogRotateCompress = 1;
    $LogRotatePath = $sock->GET_INFO("LogRotatePath");
    $ApacheLogRotate = $sock->GET_INFO("ApacheLogRotate");
    $cpbin = $unix->find_program("cp");
    $php5 = $unix->LOCATE_PHP5_BIN();
    $tmpdir = $unix->TEMP_DIR();
    if (!is_numeric($ApacheLogRotate)) {
        $ApacheLogRotate = 1;
    }
    if (!is_numeric($LogRotatePath)) {
        $LogRotatePath = "/home/logrotate";
    }
    $DirsToScan["/var/log"] = true;
    $DirsToScan["/var/log/apache2"] = true;
    $DirsToScan["/var/log/lighttpd"] = true;
    $DirsToScan["/var/log/ejabberd"] = true;
    $apache2 = $unix->dirdir("/var/log/apache2");
    while (list($WorkingDir, $ligne) = each($apache2)) {
        $DirsToScan[$WorkingDir] = true;
    }
    $q = new mysql_storelogs();
    while (list($WorkingDir, $ligne) = each($DirsToScan)) {
        $RotateSquid = false;
        if ($WorkingDir == "/var/log/squid") {
            continue;
        }
        $table = $unix->DirFiles($WorkingDir, "(\\.|-)[0-9]+.*?\$");
        $compressed["gz"] = true;
        $compressed["bz"] = true;
        $compressed["bz2"] = true;
        while (list($filename, $ligne) = each($table)) {
            $path = "{$WorkingDir}/{$filename}";
            if ($unix->file_time_min($path) < 1440) {
                continue;
            }
            $filedate = date('Y-m-d H:i:s', filemtime($path));
            $q->events("Injecting {$path} {$filedate}");
            if (!$q->ROTATE_TOMYSQL($path, $filedate)) {
                continue;
            }
        }
    }
}
Exemple #3
0
function rotate($filename)
{
    $filename = basename($filename);
    $filePath = "/var/log/squid/{$filename}";
    $unix = new unix();
    $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".{$filename}.pid";
    $pid = @file_get_contents("{$pidfile}");
    if ($unix->process_exists($pid, basename(__FILE__))) {
        events("Process {$pid} already exists...aborting");
        die;
    }
    include_once dirname(__FILE__) . "/ressources/class.mysql.syslogs.inc";
    $sarg_bin = $unix->find_program("sarg");
    $q = new mysql_storelogs();
    if (!is_file($filePath)) {
        events("{$filePath} no such file");
    }
    if (!is_file($sarg_bin)) {
        sarg_admin_events("Fatal, ({$filePath}) unable to locate sarg binary, aborting...", __FUNCTION__, __FILE__, __LINE__, "sarg");
        $q->ROTATE_TOMYSQL($filePath);
        return;
    }
    $t = time();
    sargToFile($filePath);
    $q->ROTATE_TOMYSQL($filePath);
    progress("{$filename} done " . $unix->distanceOfTimeInWords($t, time()));
    backup();
}
function ParseFile($servername, $fullpath)
{
    events("[{$servername}]: Parsing {$fullpath}", __FUNCTION__, __LINE__);
    $unix = new unix();
    $size = @filesize($fullpath);
    events("[{$servername}]: open {$fullpath} {$size} bytes", __FUNCTION__, __LINE__);
    $handle = @fopen($fullpath, "r");
    if (!$handle) {
        events("[{$servername}]: open {$fullpath} fatal, unable to open ", __FUNCTION__, __LINE__);
        return;
    }
    $c = 0;
    $d = 0;
    $t = time();
    $WORKARRAY = array();
    while (!feof($handle)) {
        $d++;
        $line = trim(fgets($handle, 4096));
        if ($line == null) {
            continue;
        }
        if (!preg_match('#(.*?)\\s+(.*?)\\s+(.*?)\\s+\\[(.*?)\\]\\s+([A-Z]+)\\s+(.*?)\\s+HTTP.*?\\/.*?"([0-9]+)"\\s+([0-9]+)\\s+"(.*?)"\\s+"(.*?)"\\s+"(.*?)"#', $line, $re)) {
            events("[{$servername}]: {{$line}} unable to parse...", __FUNCTION__, __LINE__);
            continue;
        }
        while (list($a, $b) = each($re)) {
            $re[$a] = mysql_escape_string2($b);
        }
        $c++;
        $md5 = md5($re[0]);
        $ipaddr = $re[1];
        $time = strtotime($re[4]);
        $proto = $re[5];
        $uri = $re[6];
        $code = $re[7];
        $size = $re[8];
        $UserAgent = $re[10];
        $Country = mysql_escape_string2(GeoLoc($ipaddr));
        $currDate = date("Y-m-d H:i:s");
        $linesql = "('{$md5}','{$currDate}','{$ipaddr}','{$proto}','{$uri}','{$code}','{$size}','{$UserAgent}','{$Country}')";
        $table = "hour_" . date("YmdH", $time);
        $WORKARRAY[$table][] = $linesql;
        if ($c > 500) {
            if (!ParseArray($servername, $WORKARRAY)) {
                return;
            }
            $WORKARRAY = array();
            $c = 0;
        }
    }
    if (count($WORKARRAY) > 0) {
        if (!ParseArray($servername, $WORKARRAY)) {
            return;
        }
    }
    $timeTOScan = $unix->distanceOfTimeInWords($t, time(), true);
    events("[{$servername}]: {$fullpath} {$timeTOScan} {$d} lines", __FUNCTION__, __LINE__);
    if ($d == 0) {
        @unlink($fullpath);
    }
    $sys = new mysql_storelogs();
    $filedate = date('Y-m-d H:i:s', filemtime($fullpath));
    $sys->ROTATE_TOMYSQL($fullpath, $filedate);
}
function check_all_squid()
{
    $sock = new sockets();
    $unix = new unix();
    $syslog = new mysql_storelogs();
    $php5 = $unix->LOCATE_PHP5_BIN();
    $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid";
    $timefile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".time";
    $pid = file_get_contents("{$pidfile}");
    if (system_is_overloaded(basename(__FILE__))) {
        die;
    }
    if ($unix->process_exists($pid, basename(__FILE__))) {
        $timeMin = $unix->PROCCESS_TIME_MIN($pid);
        if ($timeMin > 240) {
            system_admin_events("Too many TTL, {$pid} will be killed", __FUNCTION__, __FILE__, __LINE__, "logrotate");
            $kill = $unix->find_program("kill");
            unix_system_kill_force($pid);
        } else {
            die;
        }
    }
    $time = $unix->file_time_min($timefile);
    if ($time < 300) {
        return;
    }
    @file_put_contents($pidfile, getmypid());
    @file_put_contents($timefile, time());
    $php = $unix->LOCATE_PHP5_BIN();
    $bzip2 = $unix->find_program("bzip2");
    $ALREADYCOMP["gz"] = true;
    $ALREADYCOMP["bz2"] = true;
    $LogRotateCompress = 1;
    $LogRotatePath = $sock->GET_INFO("LogRotatePath");
    $ApacheLogRotate = $sock->GET_INFO("ApacheLogRotate");
    if (!is_numeric($ApacheLogRotate)) {
        $ApacheLogRotate = 1;
    }
    if ($LogRotatePath == null) {
        $LogRotatePath = "/home/logrotate";
    }
    $LogsRotateDefaultSizeRotation = $sock->GET_INFO("LogsRotateDefaultSizeRotation");
    if (!is_numeric($LogsRotateDefaultSizeRotation)) {
        $LogsRotateDefaultSizeRotation = 100;
    }
    foreach (glob("/var/log/squid/*") as $filename) {
        if (is_dir($filename)) {
            continue;
        }
        $size = $unix->file_size($filename);
        $time = $unix->file_time_min($filename);
        $size = round($size / 1024 / 1000, 2);
        if ($size > $LogsRotateDefaultSizeRotation) {
            if ($filename == "/var/log/squid/access.log") {
                events("{$filename} -> is a production log for Squid, launch the rotation procedure.");
                squid_admin_mysql(1, "{$filename} {$size}M exceed {$LogsRotateDefaultSizeRotation}M, launch rotation", null, __FILE__, __LINE__);
                shell_exec("{$php} /usr/share/artica-postfix/exec.squid.php --rotate");
                continue;
            }
            $TOROT[$filename] = true;
            events("{$filename} -> Add to queue {$size}M exceed {$LogsRotateDefaultSizeRotation}M");
            continue;
        }
        if ($time > 1440) {
            if ($filename == "/var/log/squid/access.log") {
                events("{$filename} -> is a production log for Squid, launch the rotation procedure.");
                squid_admin_mysql(1, "{$filename} {$size}M exceed {$LogsRotateDefaultSizeRotation}M, launch rotation", null, __FILE__, __LINE__);
                shell_exec("{$php} /usr/share/artica-postfix/exec.squid.php --rotate");
                continue;
            }
            events("{$filename} -> Add to queue {$time}mn exceed 1440mn");
            $TOROT[$filename] = true;
            continue;
        }
    }
    if (count($TOROT) == 0) {
        return;
    }
    while (list($filename, $none) = each($TOROT)) {
        $extension = pathinfo($filename, PATHINFO_EXTENSION);
        $filedate = date('Y-m-d H:i:s', filemtime($filename));
        $basename = basename($filename);
        if (preg_match("#sarg\\.#", $filename)) {
            shell_exec("{$php5} " . dirname(__FILE__) . "/exec.sarg.php --rotate {$basename} >/dev/null 2>&1 &");
            continue;
        }
        if (preg_match("#access\\.log\\.[0-9]+\$#", $filename)) {
            continue;
        }
        if ($extension != "gz") {
            if (!$unix->compress($filename, "{$filename}.gz")) {
                continue;
            }
            $filename = $filename . ".gz";
            $extension = "gz";
        }
        echo "[{$filedate}]: {$filename} ({$extension})\n";
        if ($syslog->ROTATE_TOMYSQL($filename, $filedate)) {
            @unlink($filename);
        }
    }
    foreach (glob("/home/squid/cache-logs/*") as $filename) {
        $filedate = date('Y-m-d H:i:s', filemtime($filename));
        $filename = $filename . ".gz";
        if ($syslog->ROTATE_TOMYSQL($filename, $filedate)) {
            @unlink($filename);
        }
    }
}
function start_import($aspid = false)
{
    $sock = new sockets();
    $syslog = new mysql_storelogs();
    $unix = new unix();
    $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid";
    $unix = new unix();
    if (!$aspid) {
        $pid = @file_get_contents($pidfile);
        if ($unix->process_exists($pid, basename(__FILE__))) {
            $time = $unix->PROCESS_TTL($pid);
            stats_admin_events(2, "A already Importation Task is executed pid:{$pid} since {$time}Mn", null, __FILE__, __LINE__);
            writelogs("Already executed pid:{$pid}", __FUNCTION__, __FILE__, __LINE__);
            return;
        }
        $pid = @file_get_contents("/var/run/squid-stats-central.pid");
        if ($unix->process_exists($pid, basename(__FILE__))) {
            $time = $unix->PROCESS_TTL($pid);
            stats_admin_events(2, "An anlready Importation Task is executed pid:{$pid} since {$time}Mn", null, __FILE__, __LINE__);
            writelogs("Already executed pid:{$pid}", __FUNCTION__, __FILE__, __LINE__);
            return;
        }
    }
    $import_processes = import_processes();
    if ($import_processes > 6) {
        Import_logs("Too many processes ({$import_processes})...aborting task...");
        return;
    }
    $mypid = getmypid();
    @file_put_contents($pidfile, $mypid);
    $ArticaProxyStatisticsBackupFolder = $sock->GET_INFO("ArticaProxyStatisticsBackupFolder");
    if ($ArticaProxyStatisticsBackupFolder == null) {
        $ArticaProxyStatisticsBackupFolder = "/home/artica/squid/backup-statistics";
    }
    $ArticaProxyStatisticsBackupFolder = $ArticaProxyStatisticsBackupFolder . "/import";
    $files = $unix->DirFiles($ArticaProxyStatisticsBackupFolder);
    if ($GLOBALS["VERBOSE"]) {
        echo "PUSH Scanning {$ArticaProxyStatisticsBackupFolder}\n";
    }
    $mysql = $unix->find_program("mysql");
    $bzip2 = $unix->find_program("bzip2");
    $total = count($files);
    $c = 0;
    if ($total == 0) {
        return;
    }
    stats_admin_events(2, "Importing {$total} files to MySQL", null, __FILE__, __LINE__);
    while (list($filename, $none) = each($files)) {
        $c++;
        if (if_process_import_exists($filename)) {
            Import_logs("SKIP {$ArticaProxyStatisticsBackupFolder}/{$filename} Already running...");
            continue;
        }
        percentage("Extracting {$filename} {$c}/{$total}", 2);
        $size = @filesize("{$ArticaProxyStatisticsBackupFolder}/{$filename}");
        Import_logs("IMPORT {$ArticaProxyStatisticsBackupFolder}/{$filename}");
        $f = array();
        $results = array();
        $f[] = "{$bzip2} -d -c {$ArticaProxyStatisticsBackupFolder}/{$filename} |";
        $f[] = "{$mysql} --show-warnings";
        $f[] = "--socket=/var/run/mysqld/squid-db.sock";
        $f[] = "--protocol=socket --user=root --batch --force";
        $f[] = "--debug-info --database=squidlogs 2>&1";
        $cmd = @implode(" ", $f);
        $results[] = $cmd;
        percentage("Importing {$filename} to MySQL {$c}/{$total}", 2);
        exec($cmd, $results);
        Import_logs(@implode("\n", $results));
        stats_admin_events(2, "Success importing {$filename} to MySQL", @implode("\n", $results), __FILE__, __LINE__);
        Import_logs("Backup {$ArticaProxyStatisticsBackupFolder}/{$filename}");
        $syslog->ROTATE_TOMYSQL("{$ArticaProxyStatisticsBackupFolder}/{$filename}");
    }
}