Exemple #1
0
function sigtool()
{
    $unix = new unix();
    $sigtool = $unix->find_program("sigtool");
    if (strlen($sigtool) < 5) {
        die;
    }
    if (is_file("/usr/share/artica-postfix/ressources/interface-cache/ClamAVBases")) {
        $ttim = $unix->file_time_min("/usr/share/artica-postfix/ressources/interface-cache/ClamAVBases");
        if ($ttim < 30) {
            return;
        }
    }
    $baseDir = "/var/lib/clamav";
    $patnz = $unix->DirFiles($baseDir, "\\.(cvd|cld|hdb|ign2|ndb)\$");
    while (list($path, $none) = each($patnz)) {
        $patterns[basename($path)] = true;
    }
    while (list($pattern, $none) = each($patterns)) {
        if (!is_file("{$baseDir}/{$pattern}")) {
            continue;
        }
        $results = array();
        exec("{$sigtool} --info={$baseDir}/{$pattern} 2>&1", $results);
        while (list($index, $line) = each($results)) {
            if (preg_match("#Build time:\\s+(.+)#", $line, $re)) {
                $time = strtotime($re[1]);
                $MAIN[$pattern]["zDate"] = date("Y-m-d H:i:s");
                continue;
            }
            if (preg_match("#Version:\\s+([0-9]+)#", $line, $re)) {
                $MAIN[$pattern]["version"] = $re[1];
                continue;
            }
            if (preg_match("#Signatures:\\s+([0-9]+)#", $line, $re)) {
                $MAIN[$pattern]["signatures"] = $re[1];
                continue;
            }
        }
        if (!isset($MAIN[$pattern]["zDate"])) {
            $time = filemtime("{$baseDir}/{$pattern}");
            $MAIN[$pattern]["zDate"] = date("Y-m-d H:i:s", $time);
            if (!isset($MAIN[$pattern]["version"])) {
                $MAIN[$pattern]["version"] = date("YmdHi", $time);
            }
        }
        if (!isset($MAIN[$pattern]["signatures"])) {
            $MAIN[$pattern]["signatures"] = $unix->COUNT_LINES_OF_FILE("{$baseDir}/{$pattern}");
        }
    }
    if (count($MAIN) == 0) {
        return;
    }
    @file_put_contents("/usr/share/artica-postfix/ressources/interface-cache/ClamAVBases", serialize($MAIN));
}
function BuildDatabaseStatus()
{
    if (!ifMustBeExecuted()) {
        WriteMyLogs("No make sense to execute this script...", __FUNCTION__, __FILE__, __LINE__);
        if ($GLOBALS["VERBOSE"]) {
            echo "No make sense to execute this script...\n";
        }
        die;
    }
    $q = new mysql_squid_builder();
    $unix = new unix();
    $dirs = $unix->dirdir("/var/lib/ftpunivtlse1fr");
    $TLSE_CONVERTION = $q->TLSE_CONVERTION();
    while (list($directory, $line) = each($dirs)) {
        $catzname = $TLSE_CONVERTION[basename($directory)];
        if ($catzname == null) {
            continue;
        }
        if (!is_file("{$directory}/domains")) {
            if ($GLOBALS["VERBOSE"]) {
                echo "{$catzname}=0\n";
            }
            $f[$catzname]["F"] = 0;
            $f[$catzname]["D"] = "0000-00-00 00:00:00";
            $f[$catzname]["FF"] = "{$directory}/domains";
        } else {
            $f[$catzname]["F"] = $unix->COUNT_LINES_OF_FILE("{$directory}/domains");
            if ($GLOBALS["VERBOSE"]) {
                echo "{$catzname}={$f[$catzname]["F"]}\n";
            }
            $f[$catzname]["D"] = date("Y-m-d H:i:s", filemtime("{$directory}/domains"));
            $f[$catzname]["FF"] = "{$directory}/domains";
        }
    }
    $LastC = trim(@file_get_contents("/etc/artica-postfix/ftpunivtlse1frCount"));
    $c = 0;
    while (list($cat, $array) = each($f)) {
        $count = $array["F"];
        $date = $array["D"];
        $c = $c + $count;
        $sql[] = "('{$cat}','{$count}','{$date}')";
    }
    if (count($sql) > 0) {
        $q->QUERY_SQL("TRUNCATE TABLE univtlse1fr");
        $q->QUERY_SQL("INSERT IGNORE INTO univtlse1fr (category,websitesnum,zDate) VALUES " . @implode(",", $sql));
        if (!$q->ok) {
            artica_update_event(2, "Fatal {$q->mysql_error}", null, __FILE__, __LINE__);
            return;
        }
        if ($c != $LastC) {
            //artica_update_event("Toulouse University status: $c items in database",__FUNCTION__,__FILE__,__LINE__);
            @file_put_contents("/etc/artica-postfix/ftpunivtlse1frCount", $c);
        }
    }
    //univtlse1fr
}
function ExplodeFile($filepath, $md5file = null)
{
    $unix = new unix();
    $LastScannLine = 0;
    $GLOBALS["MYSQL_CATZ"] = new mysql_catz();
    $GLOBALS["SQUID_FAMILY_CLASS"] = new squid_familysite();
    if (!isset($GLOBALS["MYHOSTNAME"])) {
        $unix = new unix();
        $GLOBALS["MYHOSTNAME"] = $unix->hostname_g();
    }
    $GLOBALS["SEQUENCE"] = md5_file($filepath);
    if (!is_file("{$filepath}.last")) {
        if (is_file("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log")) {
            $influx = new influx();
            if ($influx->files_inject("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log")) {
                @unlink("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log");
                return true;
            }
        }
    }
    $handle = @fopen($filepath, "r");
    if (!$handle) {
        echo "Fopen failed on {$filepath}\n";
        if ($md5file != null) {
            mysql_progress($md5file, 100, 3, "Fopen {failed} on {$filepath}");
        }
        return false;
    }
    $countlines = 0;
    if ($md5file != null) {
        $countlines = $unix->COUNT_LINES_OF_FILE($filepath);
        if ($md5file != null) {
            mysql_progress($md5file, 10, 0, "Parsing {$countlines}");
        }
    }
    if (is_file("{$filepath}.last")) {
        $LastScannLine = intval(@file_get_contents("{$filepath}.last"));
    }
    $c = 0;
    $d = 0;
    $e = 0;
    $prc = 0;
    $prc_text = 0;
    $mysql_first_time = 0;
    if ($LastScannLine > 0) {
        fseek($handle, $LastScannLine, SEEK_SET);
    }
    while (!feof($handle)) {
        $c++;
        $d++;
        $e++;
        if ($countlines > 0) {
            $prc = $c / $countlines;
            $prc = round($prc * 100);
            if (!isset($GLOBALS["LAST_PRC"])) {
                if ($GLOBALS["PROGRESS"]) {
                    echo "{$prc}%\n";
                }
                $GLOBALS["LAST_PRC"] = $prc;
            } else {
                if ($GLOBALS["LAST_PRC"] != $prc) {
                    if ($GLOBALS["PROGRESS"]) {
                        echo "{$prc}%\n";
                    }
                    $GLOBALS["LAST_PRC"] = $prc;
                }
            }
            if ($prc > 10) {
                if ($prc < 99) {
                    if ($prc > $prc_text) {
                        $array_load = sys_getloadavg();
                        $internal_load = $array_load[0];
                        $mem = round(memory_get_usage() / 1024 / 1000, 2);
                        $prc_design = FormatNumber($c) . "/" . FormatNumber($countlines);
                        if ($md5file != null) {
                            mysql_progress($md5file, $prc, 1, "{parsing} {$prc_design} {load}:{$internal_load} {memory}:{$mem}MB");
                        }
                        $prc_text = $prc;
                    }
                }
            }
        }
        if ($d > 50) {
            $iSeek = ftell($handle);
            @file_put_contents("{$filepath}.last", $iSeek);
            if ($GLOBALS["VERBOSE"]) {
                $prc_design = FormatNumber($c) . "/" . FormatNumber($countlines);
                echo "{$prc}% {$prc_design}\n";
            }
            $d = 0;
        }
        if ($e > 500) {
            $mem = round(memory_get_usage() / 1024 / 1000, 2);
            $prc_design = FormatNumber($c) . "/" . FormatNumber($countlines);
            if ($md5file != null) {
                mysql_progress($md5file, $prc, 1, "{parsing} {$prc_design} {load}:{$internal_load} {memory}:{$mem}MB");
            }
            $e = 0;
        }
        $buffer = trim(fgets($handle));
        if ($buffer == null) {
            continue;
        }
        $array = parseAccessLine($buffer);
        if (count($array) == 0) {
            continue;
        }
        if ($mysql_first_time == 0) {
            if (date("Y", $array["TIME"]) > 2001) {
                $mysql_first_time = $array["TIME"];
                mysql_first_time($md5file, $mysql_first_time);
            }
        }
        CachedSizeMem($array["TIME"], $array["CACHED"], $array["SIZE"]);
        if (intval($array["SIZE"]) == 0) {
            if ($GLOBALS["VERBOSE"]) {
                echo "Size = 0 " . __LINE__ . "\n";
            }
        }
        CachedUserMem($array["TIME"], $array["SITENAME"], $array["SIZE"], null, $array["UID"], $array["IPADDR"], $array["CATEGORY"], $array["FAMILYSITE"]);
    }
    @unlink("{$filepath}.last");
    mysql_last_time($md5file, $array["TIME"]);
    CachedUserMem_dump();
    CachedSizeMem_dump();
    $influx = new influx();
    $size = filesize("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log");
    $size = $size / 1024;
    $size = $size / 1024;
    echo "Importing {$size}MB of data....\n";
    if (!$influx->files_inject("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log")) {
        @unlink("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log");
        return false;
    }
    @unlink("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log");
    return true;
}
function BuildPatterns()
{
    $unix = new unix();
    cpulimit();
    $dirs = $unix->dirdir("/etc/dansguardian/lists/blacklists");
    if ($GLOBALS["VERBOSE"]) {
        echo "open /etc/dansguardian/lists/blacklists array of " . count($dirs) . "\n";
    }
    if (!is_array($dirs)) {
        writelogs("Unable to dir /etc/dansguardian/lists/blacklists", __FUNCTION__, __FILE__, __LINE__);
        return;
    }
    reset($dirs);
    while (list($num, $val) = each($dirs)) {
        $category = basename($num);
        if ($GLOBALS["VERBOSE"]) {
            echo "{$category}:: {$num} -> {$val}\n";
        }
        writelogs("Checking {$category}", __FUNCTION__, __FILE__, __LINE__);
        if ($category == "blacklists") {
            if ($GLOBALS["VERBOSE"]) {
                echo "{$category} == blacklists, aborting\n";
            }
            continue;
        }
        $domains = 0;
        $urls = 0;
        $expressions = 0;
        if (is_file("{$num}/domains")) {
            $domains = $unix->COUNT_LINES_OF_FILE("{$num}/domains");
            if ($GLOBALS["VERBOSE"]) {
                echo "{$category}:: {$domains} number\n";
            }
            $filetime = date("Y-m-d H:i:s", filemtime("{$num}/domains"));
        } else {
            if ($GLOBALS["VERBOSE"]) {
                echo "{$category}:: unable to stat {$num}/domains\n";
            }
        }
        if (is_file("{$num}/urls")) {
            $urls = $unix->COUNT_LINES_OF_FILE("{$num}/urls");
        } else {
            if ($GLOBALS["VERBOSE"]) {
                echo "{$category}:: unable to stat {$num}/urls\n";
            }
        }
        if ($GLOBALS["VERBOSE"]) {
            echo "{$category}={$domains},{$urls},{$filetime}\n";
        }
        $array["{$category}"] = array($domains, $urls, $filetime);
    }
    $datas = base64_encode(serialize($array));
    writelogs("writing /usr/share/artica-postfix/ressources/logs/dansguardian.patterns", __FUNCTION__, __FILE__, __LINE__);
    @file_put_contents("/usr/share/artica-postfix/ressources/logs/dansguardian.patterns", $datas);
    @chmod("/usr/share/artica-postfix/ressources/logs/dansguardian.patterns", 0755);
    if (!is_file("/usr/share/artica-postfix/ressources/logs/dansguardian.patterns")) {
        writelogs("Error writing dansguardian.patterns", __FUNCTION__, __FILE__, __LINE__);
    }
    return;
}
function GetIndex()
{
    $WORKING_DIR = "/home/working_toulouse_databases";
    $WORKING_DOWNLOAD = "{$WORKING_DIR}/dowloads";
    $WORKING_UPLOAD = "{$WORKING_DIR}/uploads";
    @mkdir($WORKING_DOWNLOAD, 0755, true);
    @mkdir($WORKING_UPLOAD, 0755, true);
    $unix = new unix();
    $tar = $unix->find_program("tar");
    $catz = new mysql_catz();
    $tmpfile = $unix->FILE_TEMP();
    $tmpdir = $unix->TEMP_DIR();
    $rm = $unix->find_program("rm");
    $mainuri = "ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib";
    $EXEC_NICE = $unix->EXEC_NICE();
    $ufdbGenTable = $unix->find_program("ufdbGenTable");
    $curl = new ccurl("{$mainuri}/MD5SUM.LST");
    if (!$curl->GetFile($tmpfile)) {
        echo "Failed to download MD5SUM.LST\n";
        return;
    }
    $tr = explode("\n", @file_get_contents($tmpfile));
    while (list($index, $line) = each($tr)) {
        if (preg_match("#(.+?)\\s+(.+)#", $line, $re)) {
            $filename = trim($re[1]);
            $md5 = trim($re[2]);
            $array[$md5] = $filename;
        }
    }
    @unlink($tmpfile);
    $q = new mysql_squid_builder();
    $TLSE_CONVERTION = TLSE_CONVERTION();
    $FINAL_ARRAY = array();
    while (list($filename, $md5) = each($array)) {
        $category = str_replace(".tar.gz", "", $filename);
        if (isset($TLSE_CONVERTION[$category])) {
            $FINAL_ARRAY[$filename] = $md5;
        }
    }
    $UPDATED = 0;
    $MAIN_ARRAY = unserialize(base64_decode(@file_get_contents("{$WORKING_UPLOAD}/index.txt")));
    while (list($filename, $md5) = each($FINAL_ARRAY)) {
        $TargetFile = "{$WORKING_DOWNLOAD}/{$filename}";
        $categoryname = str_replace(".tar.gz", "", $filename);
        if ($categoryname == "adult") {
            continue;
        }
        if ($categoryname == "aggressive") {
            continue;
        }
        if ($categoryname == "agressif") {
            continue;
        }
        if ($categoryname == "redirector") {
            continue;
        }
        if ($categoryname == "ads") {
            continue;
        }
        if ($categoryname == "drogue") {
            continue;
        }
        $MyStoreMd5 = md5_file($TargetFile);
        if ($MyStoreMd5 == $md5) {
            echo "Skipping {$filename}\n";
            continue;
        }
        echo "Downloading {$filename}\n";
        $curl = new ccurl("{$mainuri}/{$filename}");
        $tmpfile = $unix->FILE_TEMP();
        if (!$curl->GetFile($tmpfile)) {
            echo "Failed {$curl->error}\n";
            @unlink($tmpfile);
            continue;
        }
        $md5_tmp = md5_file($tmpfile);
        if ($md5_tmp != $md5) {
            echo "Failed Corrupted file\n";
            @unlink($tmpfile);
            continue;
        }
        if (is_file($TargetFile)) {
            @unlink($TargetFile);
        }
        if (!@copy($tmpfile, $TargetFile)) {
            echo "Failed Copy file\n";
            @unlink($tmpfile);
            @unlink($TargetFile);
            continue;
        }
        @unlink($tmpfile);
        $MyStoreMd5 = md5_file($TargetFile);
        if ($MyStoreMd5 != $md5) {
            echo "Failed MD5 file\n";
            @unlink($TargetFile);
            continue;
        }
        @mkdir("{$WORKING_DIR}/{$categoryname}", 0755, true);
        echo "Extracting {$TargetFile}\n";
        $cmd = "{$tar} xvf {$TargetFile} -C {$WORKING_DIR}/{$categoryname}/";
        echo $cmd . "\n";
        system($cmd);
        $SOURCE_DIR = find_sources("{$WORKING_DIR}/{$categoryname}");
        if (!is_file("{$SOURCE_DIR}/domains")) {
            echo "Failed {$SOURCE_DIR}/domains no such file\n";
            @unlink($TargetFile);
            continue;
        }
        $COUNT_OF_DOMAINS = $unix->COUNT_LINES_OF_FILE("{$SOURCE_DIR}/domains");
        echo "{$categoryname} {$COUNT_OF_DOMAINS} domains\n";
        if ($COUNT_OF_DOMAINS == 0) {
            shell_exec("{$rm} -rf {$WORKING_DIR}/{$categoryname}");
            @unlink($TargetFile);
            continue;
        }
        if (is_file("{$SOURCE_DIR}/domains.ufdb")) {
            @unlink("{$SOURCE_DIR}/domains.ufdb");
        }
        if (!is_file("{$SOURCE_DIR}/urls")) {
            @touch("{$SOURCE_DIR}/urls");
        }
        $u = " -u {$SOURCE_DIR}/urls";
        $d = " -d {$SOURCE_DIR}/domains";
        $cmd = "{$EXEC_NICE}{$ufdbGenTable} -n -q -W -t {$categoryname}{$d}{$u}";
        echo $cmd . "\n";
        shell_exec($cmd);
        if (!is_file("{$SOURCE_DIR}/domains.ufdb")) {
            echo "Failed to compile {$categoryname}\n";
            @unlink($TargetFile);
            continue;
        }
        $MD5SRC = md5_file("{$SOURCE_DIR}/domains.ufdb");
        if (is_file("{$WORKING_UPLOAD}/{$categoryname}.gz")) {
            @unlink("{$WORKING_UPLOAD}/{$categoryname}.gz");
        }
        $unix->compress("{$SOURCE_DIR}/domains.ufdb", "{$WORKING_UPLOAD}/{$categoryname}.gz");
        $MD5GZ = md5_file("{$WORKING_UPLOAD}/{$categoryname}.gz");
        $UPDATED++;
        $NOTIFICATIONS[] = "{$categoryname} updated with  {$COUNT_OF_DOMAINS} domains";
        $MAIN_ARRAY[$categoryname]["ROWS"] = $COUNT_OF_DOMAINS;
        $MAIN_ARRAY[$categoryname]["MD5SRC"] = $MD5SRC;
        $MAIN_ARRAY[$categoryname]["MD5GZ"] = $MD5GZ;
        $MAIN_ARRAY[$categoryname]["TIME"] = time();
        $MAIN_ARRAY[$categoryname]["SIZE"] = @filesize("{$WORKING_UPLOAD}/{$categoryname}.gz");
        @file_put_contents("{$WORKING_UPLOAD}/index.txt", base64_encode(serialize($MAIN_ARRAY)));
    }
    if ($UPDATED > 0) {
        PushToRepo_alls();
        sendEmail("{$UPDATED} Toulouse Unversity databases uploaded.", @implode("\n", $NOTIFICATIONS));
    }
}
function xstart()
{
    $unix = new unix();
    $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . ".pid";
    $timefile = "/etc/artica-postfix/pids/exec.ufdb.parse-categories.php.time";
    $unix = new unix();
    $me = basename(__FILE__);
    $pid = $unix->get_pid_from_file($pidfile);
    if (system_is_overloaded()) {
        die;
    }
    if ($unix->process_exists($pid, $me)) {
        if ($GLOBALS["VERBOSE"]) {
            echo " {$pid} --> Already executed.. aborting the process\n";
        }
        $time = $unix->PROCCESS_TIME_MIN($pid);
        die;
    }
    @file_put_contents($pidfile, getmypid());
    if ($unix->file_time_min($timefile) < 60) {
        return;
    }
    @unlink($timefile);
    @file_put_contents($timefile, time());
    $q = new mysql_squid_builder();
    $DirsArtica = $unix->dirdir("/var/lib/ufdbartica");
    $sql = "CREATE TABLE IF NOT EXISTS `UPDATE_DBWF_INFOS` ( \n\t`category` varchar(90) NOT NULL, `size_artica` INT UNSIGNED NOT NULL, `date_artica` INT UNSIGNED NOT NULL, `count_artica` INT UNSIGNED NOT NULL, `size_tlse` INT UNSIGNED NOT NULL, `date_tlse` INT UNSIGNED NOT NULL, `count_tlse` INT UNSIGNED NOT NULL, `size_perso` INT UNSIGNED NOT NULL, `date_perso` INT UNSIGNED NOT NULL, `count_perso` INT UNSIGNED NOT NULL, PRIMARY KEY (`category`) \n\t\t\t) ENGINE=MYISAM;";
    $q->QUERY_SQL($sql);
    if (!$q->ok) {
        echo $q->mysql_error;
        return;
    }
    $MAX = 144;
    $c = 0;
    $UFDB = array();
    $UFDBCOUNT = unserialize(base64_decode(@file_get_contents("/etc/artica-postfix/ufdbcounts.txt")));
    while (list($dir, $line) = each($DirsArtica)) {
        if (is_link($dir)) {
            continue;
        }
        $database_path = "{$dir}/domains.ufdb";
        if (!is_file($database_path)) {
            continue;
        }
        $tablename = basename($dir);
        $size = @filesize("{$dir}/domains.ufdb");
        $time = filemtime("{$dir}/domains.ufdb");
        $cat = $q->tablename_tocat($tablename);
        $MAIN[$cat]["ART"]["SIZE"] = $size;
        $MAIN[$cat]["ART"]["TIME"] = $time;
        $MAIN[$cat]["ART"]["COUNT"] = $UFDBCOUNT[$tablename];
    }
    $DirsArtica = $unix->dirdir("/var/lib/ftpunivtlse1fr");
    while (list($dir, $line) = each($DirsArtica)) {
        $database_path = "{$dir}/domains.ufdb";
        $sourcefile = "{$dir}/domains";
        if (!is_file($database_path)) {
            continue;
        }
        $cat = basename($dir);
        $cat = $q->filaname_tocat($cat);
        $size = @filesize("{$dir}/domains.ufdb");
        $time = filemtime("{$dir}/domains.ufdb");
        $MAIN[$cat]["TLSE"]["SIZE"] = $size;
        $MAIN[$cat]["TLSE"]["TIME"] = $time;
        $MAIN[$cat]["TLSE"]["COUNT"] = $unix->COUNT_LINES_OF_FILE($sourcefile);
        if (system_is_overloaded()) {
            @unlink("{$timefile}");
            die;
        }
    }
    $DirsArtica = $unix->dirdir("/var/lib/squidguard");
    while (list($dir, $line) = each($DirsArtica)) {
        $database_path = "{$dir}/domains.ufdb";
        if (!is_file($database_path)) {
            continue;
        }
        $tablename = "category_" . basename($dir);
        $cat = $q->tablename_tocat($tablename);
        $size = @filesize("{$dir}/domains.ufdb");
        $time = filemtime("{$dir}/domains.ufdb");
        $sourcefile = "{$dir}/domains";
        $MAIN[$cat]["PERSO"]["SIZE"] = $size;
        $MAIN[$cat]["PERSO"]["PATH"] = $dir;
        $MAIN[$cat]["PERSO"]["CATEGORY"] = $cat;
        $MAIN[$cat]["PERSO"]["TIME"] = $time;
        $MAIN[$cat]["PERSO"]["COUNT"] = $unix->COUNT_LINES_OF_FILE($sourcefile);
        if (system_is_overloaded()) {
            @unlink("{$timefile}");
            die;
        }
    }
    $prefix = "INSERT IGNORE INTO `UPDATE_DBWF_INFOS` (`category`,\n\t`size_artica` ,\n\t`date_artica` ,\n\t`count_artica` ,\n\n\t`size_tlse` ,\n\t`date_tlse` ,\n\t`count_tlse` ,\t\t\t\n\t\t\t\n\t`size_perso` ,\n\t`date_perso` ,\n\t`count_perso`) VALUES ";
    while (list($category, $MAINZ) = each($MAIN)) {
        $f[] = "('{$category}','{$MAINZ["ART"]["SIZE"]}','{$MAINZ["ART"]["TIME"]}','{$MAINZ["ART"]["COUNT"]}','{$MAINZ["TLSE"]["SIZE"]}','{$MAINZ["TLSE"]["TIME"]}','{$MAINZ["TLSE"]["COUNT"]}','{$MAINZ["PERSO"]["SIZE"]}','{$MAINZ["PERSO"]["TIME"]}','{$MAINZ["PERSO"]["COUNT"]}')";
    }
    $q->QUERY_SQL("TRUNCATE TABLE `UPDATE_DBWF_INFOS`");
    $sql = $prefix . @implode(",", $f);
    $q->QUERY_SQL($sql);
    if (!$q->ok) {
        echo $q->mysql_error . "\n{$sql}\n";
        return;
    }
}
function restore($filename, $storeid)
{
    $filename = trim($filename);
    $unix = new unix();
    $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . ".{$filename}.pid";
    $pid = @file_get_contents("{$pidfile}");
    if ($unix->process_exists($pid, basename(__FILE__))) {
        die;
    }
    @file_put_contents($pidfile, getmypid());
    $EnableSyslogDB = @file_get_contents("/etc/artica-postfix/settings/Daemons/EnableSyslogDB");
    if (!is_numeric($EnableSyslogDB)) {
        $EnableSyslogDB = 0;
    }
    @mkdir("/var/log/artica-postfix/squid-brut", 0777, true);
    @mkdir("/var/log/artica-postfix/squid-reverse", 0777, true);
    $GLOBALS["filename"] = $filename;
    $sock = new sockets();
    $TempDir = "/home/artica-extract-temp";
    @mkdir($TempDir, 0777);
    @chown($TempDir, "mysql");
    @chdir($TempDir, "mysql");
    $BackupMaxDaysDir = $sock->GET_INFO("BackupMaxDaysDir");
    $bzip2 = $unix->find_program("bzip2");
    $gunzip = $unix->find_program("gunzip");
    progress("Extract {$filename} from MySQL database into {$TempDir}", 4);
    if ($EnableSyslogDB == 1) {
        $q = new mysql_storelogs();
        $sql = "SELECT filecontent INTO DUMPFILE '{$TempDir}/{$filename}' FROM files_store WHERE ID = '{$storeid}'";
        $q->QUERY_SQL($sql);
    } else {
        $q = new mysql_syslog();
        $sql = "SELECT filedata INTO DUMPFILE '{$TempDir}/{$filename}' FROM store WHERE filename = '{$filename}'";
        $q->QUERY_SQL($sql);
    }
    if (!$q->ok) {
        progress("Failed!!! {$q->mysql_error}", 100);
        return;
    }
    $file_extension = file_extension($filename);
    progress("Extract {$filename} extension: {$file_extension}", 5);
    $newtFile = $filename . ".log";
    if ($file_extension == "bz2") {
        $cmdline = "bzip2 -d \"{$TempDir}/{$filename}\" -c >\"{$TempDir}/{$newtFile}.log\" 2>&1";
        exec($cmdline, $results);
    }
    if ($file_extension == "gz") {
        $cmdline = "gunzip -d \"{$TempDir}/{$filename}\" -c >\"{$TempDir}/{$newtFile}.log\" 2>&1";
    }
    if ($cmdline != null) {
        exec($cmdline, $results);
        progress("Extract done " . @implode(" ", $results), 7);
    } else {
        if (!@copy("{$TempDir}/{$filename}", "{$TempDir}/{$newtFile}.log")) {
            progress("Failed!!! Copy error", 100);
            return;
        }
    }
    @unlink("{$TempDir}/{$filename}");
    if (!is_file("{$TempDir}/{$newtFile}.log")) {
        progress("Failed!!! {$TempDir}/{$newtFile}.log error no such file", 100);
        return;
    }
    $linesNumber = $unix->COUNT_LINES_OF_FILE("{$TempDir}/{$newtFile}.log");
    progress("Open {$TempDir}/{$newtFile}.log {$linesNumber}", 10);
    $handle = @fopen("{$TempDir}/{$newtFile}.log", "r");
    if (!$handle) {
        progress("Failed!!! {$TempDir}/{$newtFile}.log open failed", 100);
        return;
    }
    $c = 0;
    $d = 0;
    $TTEV = 0;
    while (!feof($handle)) {
        $c++;
        $buffer = trim(fgets($handle, 4096));
        if (!preg_match("#MAC:.*?\\[([0-9]+)\\/(.*?)\\/([0-9]+).*?:([0-9]+):([0-9]+):([0-9]+)\\s+(.*?)\\]\\s+\"#", $buffer, $re)) {
            continue;
        }
        $dteStr = "{$re[1]}/{$re[2]}/{$re[3]}:{$re[4]}:{$re[5]}:{$re[6]} {$re[7]}";
        $ttime = strtotime($dteStr);
        $newDate = date("Y-m-d H", $ttime) . "h";
        $datelog = date("Y-m-d-h", $ttime);
        $MD5Buffer = md5($buffer);
        $TTEV++;
        @mkdir("/var/log/artica-postfix/squid-brut/{$datelog}", 0777, true);
        @file_put_contents("/var/log/artica-postfix/squid-brut/{$datelog}/{$MD5Buffer}", $buffer);
        if ($c > 10) {
            $d = $d + $c;
            $pp = $d / $linesNumber;
            $pp = $pp * 100;
            $pp = round($pp, 1);
            if ($pp > 10) {
                if ($pp > 100) {
                    $pp = 99;
                }
                progress("Processing {$d}/{$linesNumber} - {$newDate} ", $pp);
                $c = 0;
            }
        }
    }
    progress("Success, {$TTEV} events sent to MySQL injector ", 100);
    @unlink("{$TempDir}/{$newtFile}.log");
}
function xstart()
{
    $T1 = time();
    $curl = new ccurl();
    $unix = new unix();
    $GLOBALS["MYPID"] = getmypid();
    $pidfile = "/etc/artica-postfix/pids/windowupdate.processor.pid";
    $pid = $unix->get_pid_from_file($pidfile);
    if ($unix->process_exists($pid, basename(__FILE__))) {
        die;
    }
    $pids = $unix->PIDOF_PATTERN_ALL(basename(__FILE__), true);
    if (count($pids) > 0) {
        while (list($i, $line) = each($pids)) {
            events("Already executed PID:{$i}... aborting ", __LINE__);
        }
        die;
    }
    $TEMPDIR = $unix->TEMP_DIR() . "/WindowsUpdates";
    $rm = $unix->find_program("rm");
    @file_put_contents($pidfile, $GLOBALS["MYPID"]);
    if (is_dir($TEMPDIR)) {
        shell_exec("{$rm} -rf {$TEMPDIR}");
    }
    @mkdir($TEMPDIR, 0755, true);
    $WindowsUpdateMaxPartition = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateMaxPartition"));
    if ($WindowsUpdateMaxPartition == 0) {
        $WindowsUpdateMaxPartition = 80;
    }
    $CheckPartitionPercentage = CheckPartitionPercentage();
    if ($CheckPartitionPercentage > $WindowsUpdateMaxPartition) {
        $time = $unix->file_time_min("/etc/squid3/WindowsUpdatePartitionExceed");
        if ($time > 10) {
            @unlink("/etc/squid3/WindowsUpdatePartitionExceed");
            events("Failed: Storage Partition exceed {$WindowsUpdateMaxPartition}%, Stopping retreivals", __LINE__);
            @touch("/etc/squid3/WindowsUpdatePartitionExceed");
            DirectorySize();
        }
        return;
    }
    $WindowsUpdateInProduction = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateInProduction"));
    if ($WindowsUpdateInProduction == 0) {
        if ($unix->IsProductionTime()) {
            $time = $unix->file_time_min("/etc/artica-postfix/pids/WindowsUpdateInProduction");
            if ($time > 15) {
                @unlink("/etc/artica-postfix/pids/WindowsUpdateInProduction");
                @touch("/etc/artica-postfix/pids/WindowsUpdateInProduction");
                events("INFO: Aborting, No download during production time", __LINE__);
                DirectorySize();
            }
            return;
        }
    }
    if (is_file("/etc/squid3/WindowsUpdatePartitionExceed")) {
        @unlink("/etc/squid3/WindowsUpdatePartitionExceed");
    }
    $q = new mysql_squid_builder();
    $sql = "CREATE TABLE IF NOT EXISTS `windowsupdate` (\n\t\t\t`filemd5` VARCHAR( 90 ) NOT NULL ,\n\t\t\t`zDate` DATETIME NOT NULL ,\n\t\t\t`zUri` VARCHAR( 255 ) NOT NULL ,\n\t\t\t`localpath` VARCHAR( 255 ) NOT NULL ,\n\t\t\t`filesize` BIGINT UNSIGNED DEFAULT '0',\n\t\t\t INDEX ( `filesize` ,`zDate`) ,\n\t\t\t KEY `localpath`(`localpath`),\n\t\t\t KEY `zUri`(`zUri`),\n\t\t\t PRIMARY KEY (`filemd5`)) ENGINE=MYISAM;";
    $q->QUERY_SQL($sql);
    if (!$q->ok) {
        events("MySQL Failed {$q->mysql_error}", __LINE__);
        die;
    }
    $GLOBALS["WindowsUpdateMaxToPartialQueue"] = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateMaxToPartialQueue"));
    if ($GLOBALS["WindowsUpdateMaxToPartialQueue"] == 0) {
        $GLOBALS["WindowsUpdateMaxToPartialQueue"] = 350;
    }
    $GLOBALS["WindowsUpdateCachingDir"] = @file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateCachingDir");
    if ($GLOBALS["WindowsUpdateCachingDir"] == null) {
        $GLOBALS["WindowsUpdateCachingDir"] = "/home/squid/WindowsUpdate";
    }
    $filepath = "{$GLOBALS["WindowsUpdateCachingDir"]}/Queue.log";
    $WindowsUpdateDownTimeout = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateDownTimeout"));
    if ($WindowsUpdateDownTimeout == 0) {
        $WindowsUpdateDownTimeout = 600;
    }
    $WindowsUpdateMaxToPartialQueue = $GLOBALS["WindowsUpdateMaxToPartialQueue"] * 1000;
    $WindowsUpdateMaxToPartialQueue = $WindowsUpdateMaxToPartialQueue * 1000;
    $LinesCount = $unix->COUNT_LINES_OF_FILE($filepath);
    if (!is_file($filepath)) {
        return;
    }
    $md5start = md5_file($filepath);
    $handle = @fopen($filepath, "r");
    if (!$handle) {
        events("Fopen failed on {$filepath}", __LINE__);
        return false;
    }
    $NEWBUFFER = array();
    $URLALREADY = array();
    $FinalSize = 0;
    $FF = 0;
    $c = 0;
    while (!feof($handle)) {
        $buffer = trim(fgets($handle));
        $c++;
        if ($buffer == null) {
            continue;
        }
        $TR = explode("|||", $buffer);
        $prc = $c / $LinesCount;
        $prc = round($prc * 100);
        $LocalFile = $TR[0];
        $URI = $TR[1];
        $ExpectedSize = 0;
        if (strpos($URI, $GLOBALS["WindowsUpdateCachingDir"]) > 0) {
            events("FOUND! directory in  URI", __LINE__);
            $TTR = explode($GLOBALS["WindowsUpdateCachingDir"], $URI);
            $URI = $TTR[0];
            $LocalFile = "{$GLOBALS["WindowsUpdateCachingDir"]}{$TTR[1]}";
            events("FOUND! URI:{$URI}", __LINE__);
            events("FOUND! NEXT:{$LocalFile}", __LINE__);
        }
        $BASENAMELL = basename($LocalFile);
        if (strlen($BASENAMELL) > 20) {
            $BASENAMELL = substr($BASENAMELL, 0, 17) . "...";
        }
        build_progressG("{$BASENAMELL} {$c}/{$LinesCount} {files}", $prc);
        if (isset($URLALREADY[$URI])) {
            continue;
        }
        $URLALREADY[$URI] = true;
        if (isBlacklisted($URI)) {
            events(basename($URI) . " blacklisted...");
            continue;
        }
        if (is_file($LocalFile)) {
            $size = @filesize($LocalFile);
            if ($size > 5) {
                events("SKIP " . basename($LocalFile) . " " . xFormatBytes($size / 1024), __LINE__);
                update_mysql($LocalFile, $URI);
                continue;
            } else {
                @unlink($LocalFile);
            }
        }
        $dirname = dirname($LocalFile);
        if (!is_dir($dirname)) {
            @mkdir($dirname, true, 0755);
        }
        $curl = new ccurl($URI);
        $Headers = $curl->getHeaders();
        $TIMEDOWN = time();
        $TMPFILE = "{$TEMPDIR}/" . basename($LocalFile);
        $GLOBALS["previousProgress"] = 0;
        $GLOBALS["DOWNLOADED_FILE"] = basename($LocalFile);
        $GLOBALS["TMPFILE"] = $TMPFILE;
        $ExpectedSize = GetTargetedSize($URI);
        if ($ExpectedSize == 0) {
            events("Failed to download {$URI} ( unable to get expected size)", __LINE__);
            continue;
        }
        if ($ExpectedSize > $WindowsUpdateMaxToPartialQueue) {
            $ExpectedSizeText = xFormatBytes($ExpectedSize / 1024, true);
            events(basename($URI) . " ({$ExpectedSizeText} {$ExpectedSize}/{$WindowsUpdateMaxToPartialQueue}) Limit {$WindowsUpdateMaxToPartialQueue} to BigFiles queue", __LINE__);
            AddToPartialQueue($URI, $ExpectedSize, $LocalFile);
            continue;
        }
        $curl = new ccurl($URI);
        $curl->WriteProgress = true;
        $curl->Timeout = $WindowsUpdateDownTimeout * 60;
        $curl->ProgressFunction = "xdownload_progress";
        events("Downloading " . basename($URI) . " to {$TMPFILE} (" . xFormatBytes($ExpectedSize / 1024, true) . " max:{$WindowsUpdateDownTimeout} Minutes)", __LINE__);
        if (!$curl->GetFile($TMPFILE)) {
            events("Failed: TMP: &laquo;{$TMPFILE}&raquo;", __LINE__);
            events("Failed: URL: &laquo;{$URI}&raquo;", __LINE__);
            events("Failed: After: " . $unix->distanceOfTimeInWords($TIMEDOWN, time(), true), __LINE__);
            events("Failed: With error: {$curl->error} http code: {$curl->CURLINFO_HTTP_CODE} (" . count($curl->CURL_ALL_INFOS) . ") infos", __LINE__);
            reset($curl->CURL_ALL_INFOS);
            while (list($index, $value) = each($curl->CURL_ALL_INFOS)) {
                events("Failed: &laquo;{$index}&raquo; [{$value}]", __LINE__);
            }
            if ($curl->CURLINFO_HTTP_CODE == 404) {
                continue;
            }
            @unlink($TMPFILE);
            $NEWBUFFER[] = "{$buffer}";
            continue;
        }
        if (!is_file($TMPFILE)) {
            events("Fatal {$TMPFILE}: no such file", __LINE__);
            continue;
        }
        $size = filesize($TMPFILE);
        $sizeT = xFormatBytes($size / 1024);
        if ($size < 5) {
            @unlink($TMPFILE);
            events("Failed: File less than 5 Bytes ({$size}), aborting", __LINE__);
            continue;
        }
        if ($ExpectedSize > 0) {
            if ($size != $ExpectedSize) {
                $ExpectedSizeT = xFormatBytes($ExpectedSize / 1024);
                events("Failed: corrupted download " . basename($URI) . " expected size {$ExpectedSizeT}/{$ExpectedSize} current:({$sizeT}/{$size})", __LINE__);
                @unlink($TMPFILE);
                continue;
            }
        }
        if (!@copy($TMPFILE, $LocalFile)) {
            @unlink($TMPFILE);
            events("Failed: Translating to {$LocalFile}", __LINE__);
            $NEWBUFFER[] = "{$buffer}";
            continue;
        }
        events("Success: " . basename($TMPFILE) . " ({$sizeT})", __LINE__);
        @unlink($TMPFILE);
        $FF++;
        $size = @filesize($LocalFile);
        $FinalSize = $FinalSize + $size;
        update_mysql($LocalFile, $URI);
    }
    $took = $unix->distanceOfTimeInWords($T1, time(), true);
    if ($FinalSize > 0) {
        $CURLINFO_SPEED_DOWNLOAD = $curl->CURL_ALL_INFOS["CURLINFO_SPEED_DOWNLOAD"];
        events("Downloaded {$FF} files for " . xFormatBytes($FinalSize / 1024, true) . " ({$CURLINFO_SPEED_DOWNLOAD}) took: {$took}", __LINE__);
    }
    $md5finish = md5_file($filepath);
    if (count($NEWBUFFER) > 0) {
        events("Retry " . count($NEWBUFFER) . " requests next time...", __LINE__);
        @file_put_contents($filepath, @implode("\n", $NEWBUFFER));
    } else {
        events("No new file downloaded....", __LINE__);
        events("Removing queue {$filepath}", __LINE__);
        @unlink($filepath);
    }
    DirectorySize();
    events("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; * * * END TOOK: {$took} * * *", __LINE__);
}
function ArticaUfdb()
{
    $DB = array();
    $unix = new unix();
    $FILEDBS = array();
    $prc = 0;
    $GLOBALS["MAIN_ARRAY"]["TLSE_PRC"] = 0;
    if ($GLOBALS["CLASS_SOCKETS"]->EnableUfdbGuard() == 0) {
        if ($GLOBALS["VERBOSE"]) {
            echo "EnableUfdbGuard report false\n";
        }
        $GLOBALS["MAIN_ARRAY"]["TLSE_ENABLED"] = 0;
    }
    if (is_file("/etc/artica-postfix/univtoulouse-global_usage")) {
        $contentF = explode("\n", @file_get_contents("/etc/artica-postfix/univtoulouse-global_usage"));
        while (list($index, $line) = each($contentF)) {
            if (preg_match("#NAME:\\s+(.+)#", $line, $re)) {
                $DB[trim($re[1])] = trim($re[1]);
            }
        }
    }
    $Dirs = $unix->dirdir("/var/lib/ftpunivtlse1fr");
    while (list($Dir, $line) = each($Dirs)) {
        if (!is_file("{$Dir}/domains.ufdb")) {
            continue;
        }
        $DB[basename($Dir)] = true;
    }
    if (count($DB) > 0) {
        $q = new mysql_squid_builder();
        $TLSE_CONVERTION = $q->TLSE_CONVERTION();
        while (list($TLSE, $line) = each($DB)) {
            $catzname = $TLSE_CONVERTION[$TLSE];
            if ($catzname == null) {
                if ($GLOBALS["VERBOSE"]) {
                    echo "Unable to understand {$TLSE}\n";
                }
                continue;
            }
            $catzname = str_replace("/", "_", $catzname);
            $FILEDBS[$catzname] = "/var/lib/ftpunivtlse1fr/{$catzname}/domains.ufdb";
            $FILEDBSC[] = "/var/lib/ftpunivtlse1fr/{$catzname}/domains";
        }
    }
    if (count($FILEDBS) > 0) {
        $c = 0;
        $MAX = count($FILEDBS);
        while (list($table, $path) = each($FILEDBS)) {
            if (!is_file($path)) {
                echo "{$path} such file\n";
                continue;
            }
            $size = @filesize($path);
            if ($size < 10) {
                continue;
            }
            $c++;
            $prc = intval($c) / intval($MAX);
            $prc = round($prc * 100);
            $GLOBALS["MAIN_ARRAY"]["TLSE_COUNT"] = $c;
            $TLSE_COUNTZ[$table]["SIZE"] = $size;
            if ($GLOBALS["VERBOSE"]) {
                echo "COUNT OF {$path}\n";
            }
            $path_db1 = dirname($path) . "/" . "domains";
            $TLSE_COUNTZ[$table]["ITEMS"] = $unix->COUNT_LINES_OF_FILE($path_db1);
            $TLSE_COUNTZ[$table]["TIME"] = filemtime($path);
        }
    }
    if ($GLOBALS["CLASS_UNIX"]->file_time_min("/etc/artica-postfix/UNIVTLSE_STAT_DB_SIZE") > 60) {
        $UNIVTLSE_STAT_DB_SIZE = $GLOBALS["CLASS_UNIX"]->DIRSIZE_KO("/var/lib/ftpunivtlse1fr");
        @unlink("/etc/artica-postfix/UNIVTLSE_STAT_DB_SIZE");
        @file_put_contents("/etc/artica-postfix/UNIVTLSE_STAT_DB_SIZE", $UNIVTLSE_STAT_DB_SIZE);
    }
    $C = 0;
    if ($GLOBALS["CLASS_UNIX"]->file_time_min("/etc/artica-postfix/UNIVTLSE_STAT_DB_ITEMS") > 60) {
        while (list($table, $path) = each($FILEDBSC)) {
            $unix = new unix();
            $C = $C + intval($unix->COUNT_LINES_OF_FILE($path));
        }
        @unlink("/etc/artica-postfix/UNIVTLSE_STAT_DB_ITEMS");
        @file_put_contents("/etc/artica-postfix/UNIVTLSE_STAT_DB_ITEMS", $C);
    }
    $STATUS = unserialize(@file_get_contents("/etc/artica-postfix/TLSE_LAST_DOWNLOAD"));
    if (!isset($STATUS["LAST_DOWNLOAD"])) {
        $STATUS["LAST_DOWNLOAD"] = array();
    }
    $GLOBALS["MAIN_ARRAY"]["TLSE_PRC"] = $prc;
    $GLOBALS["MAIN_ARRAY"]["TLSE_STAT_SIZE"] = trim(@file_get_contents("/etc/artica-postfix/UNIVTLSE_STAT_DB_SIZE"));
    $GLOBALS["MAIN_ARRAY"]["TLSE_STAT_ITEMS"] = trim(@file_get_contents("/etc/artica-postfix/UNIVTLSE_STAT_DB_ITEMS"));
    $GLOBALS["MAIN_ARRAY"]["TLSE_LAST_SINCE"] = $GLOBALS["CLASS_UNIX"]->distanceOfTimeInWords($STATUS["LAST_DOWNLOAD"]["TIME"], time());
    $GLOBALS["MAIN_ARRAY"]["TLSE_LAST_CAT"] = $STATUS["LAST_DOWNLOAD"]["CATEGORY"];
    $GLOBALS["MAIN_ARRAY"]["TLSE_LAST_SIZE"] = $STATUS["LAST_DOWNLOAD"]["SIZE"];
    $GLOBALS["MAIN_ARRAY"]["TLSE_LAST_CHECK"] = $STATUS["LAST_CHECK"];
    $GLOBALS["MAIN_ARRAY"]["TLSE_ARRAY"] = $TLSE_COUNTZ;
    if ($GLOBALS["VERBOSE"]) {
        print_r($GLOBALS["MAIN_ARRAY"]);
    }
}
function xstart()
{
    $T1 = time();
    $curl = new ccurl();
    $unix = new unix();
    $GLOBALS["MYPID"] = getmypid();
    $pidfile = "/etc/artica-postfix/pids/windowupdate.partial.processor.pid";
    $pid = $unix->get_pid_from_file($pidfile);
    if ($unix->process_exists($pid, basename(__FILE__))) {
        die;
    }
    $pids = $unix->PIDOF_PATTERN_ALL(basename(__FILE__), true);
    if (count($pids) > 0) {
        while (list($i, $line) = each($pids)) {
            events("Already executed PID:{$i}... aborting ", __LINE__);
        }
        die;
    }
    $TEMPDIR = $unix->TEMP_DIR() . "/WindowsUpdates";
    $rm = $unix->find_program("rm");
    @file_put_contents($pidfile, $GLOBALS["MYPID"]);
    if (is_dir($TEMPDIR)) {
        shell_exec("{$rm} -rf {$TEMPDIR}");
        @mkdir($TEMPDIR);
    }
    $GLOBALS["WindowsUpdateMaxToPartialQueue"] = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateMaxToPartialQueue"));
    $GLOBALS["WindowsUpdateCachingDir"] = @file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateCachingDir");
    if ($GLOBALS["WindowsUpdateCachingDir"] == null) {
        $GLOBALS["WindowsUpdateCachingDir"] = "/home/squid/WindowsUpdate";
    }
    $WindowsUpdateDownTimeout = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateDownTimeout"));
    if ($WindowsUpdateDownTimeout == 0) {
        $WindowsUpdateDownTimeout = 600;
    }
    $WindowsUpdateBandwidthMaxFailed = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateBandwidthMaxFailed"));
    if ($WindowsUpdateBandwidthMaxFailed == 0) {
        $WindowsUpdateBandwidthMaxFailed = 50;
    }
    $WindowsUpdateMaxPartition = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateMaxPartition"));
    if ($WindowsUpdateMaxPartition == 0) {
        $WindowsUpdateMaxPartition = 80;
    }
    $fileSource = "{$GLOBALS["WindowsUpdateCachingDir"]}/Partials/Queue.log";
    $LinesCount = $unix->COUNT_LINES_OF_FILE($fileSource);
    if (!is_file($fileSource)) {
        return;
    }
    $md5start = md5_file($fileSource);
    $CheckPartitionPercentage = CheckPartitionPercentage();
    if ($CheckPartitionPercentage > $WindowsUpdateMaxPartition) {
        events("Failed: Storage Partition exceed {$WindowsUpdateMaxPartition}% Stopping retreivals", __LINE__);
        @touch("/etc/squid3/WindowsUpdatePartitionExceed");
        DirectorySize();
        return;
    }
    $WindowsUpdateInProduction = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateInProduction"));
    if ($WindowsUpdateInProduction == 0) {
        if ($unix->IsProductionTime()) {
            $time = $unix->file_time_min("/etc/artica-postfix/pids/WindowsUpdateInProduction");
            if ($time > 15) {
                @unlink("/etc/artica-postfix/pids/WindowsUpdateInProduction");
                @touch("/etc/artica-postfix/pids/WindowsUpdateInProduction");
                events("INFO: Aborting, No download during production time", __LINE__);
                DirectorySize();
            }
            return;
        }
    }
    if (is_file("/etc/squid3/WindowsUpdatePartitionExceed")) {
        @unlink("/etc/squid3/WindowsUpdatePartitionExceed");
    }
    $handle = @fopen($fileSource, "r");
    if (!$handle) {
        events("Fopen failed on {$fileSource}", __LINE__);
        return false;
    }
    $NEWBUFFER = array();
    $URLALREADY = array();
    $NewBuffer = array();
    $FinalSize = 0;
    $FF = 0;
    $c = 0;
    while (!feof($handle)) {
        $buffer = trim(fgets($handle));
        $c++;
        if ($buffer == null) {
            continue;
        }
        $TR = explode("|||", $buffer);
        $LocalFile = $TR[0];
        $URI = $TR[1];
        $ExpectedSize = $TR[2];
        if (!isset($TR[3])) {
            $TR[3] = 1;
        }
        if (isset($URLALREADY[$URI])) {
            continue;
        }
        $URLALREADY[$URI] = true;
        $BaseNameOfFile = basename($URI);
        events("INFO: {$BaseNameOfFile} " . xFormatBytes($ExpectedSize / 1024) . " Retry:{$TR[3]} [{$c}/{$LinesCount}]", __LINE__);
        if (DownloadByRange($URI, $ExpectedSize, $LocalFile)) {
            update_mysql($LocalFile, $URI);
            continue;
        }
        $TR[3] = $TR[3] + 1;
        if ($TR[3] > $WindowsUpdateBandwidthMaxFailed) {
            events("Error: Max retry ({$WindowsUpdateBandwidthMaxFailed}) for filename: {$BaseNameOfFile}", __LINE__);
            SaveToBlacklists($BaseNameOfFile);
            RemoveTempOf($URI, $LocalFile);
            continue;
        }
        $NewBuffer[] = @implode("|||", $TR);
    }
    if (count($NewBuffer) == 0) {
        @unlink($fileSource);
        return;
    }
    events("INFO: add " . count($NewBuffer) . " orders to queue", __LINE__);
    @file_put_contents($fileSource, @implode("\n", $NewBuffer));
    DirectorySize();
}
function xrun()
{
    build_progress("{scanning} {files}", 20);
    $BaseWorkDir = "/home/postfix/logrotate";
    if (!($handle = opendir($BaseWorkDir))) {
        echo "Failed open {$BaseWorkDir}\n";
        return;
    }
    $MAIN = array();
    while (false !== ($filename = readdir($handle))) {
        if ($filename == ".") {
            continue;
        }
        if ($filename == "..") {
            continue;
        }
        $targetfile = "{$BaseWorkDir}/{$filename}";
        if (strpos($filename, ".gz") == 0) {
            continue;
        }
        $fileKey = str_replace(".gz", "", $filename);
        $MAIN[$fileKey] = $targetfile;
    }
    if (count($MAIN) == 0) {
        echo "No files to scan....\n";
        build_progress("{scanning} {files} {failed}", 110);
        return;
    }
    ksort($MAIN);
    $sock = new sockets();
    $PostfixHistorySearch = $sock->GET_INFO("PostfixHistorySearch");
    if ($PostfixHistorySearch == null) {
        echo "No search pattern, aborting...\n";
        build_progress("{scanning} {files} {failed}", 110);
        return;
    }
    if (!preg_match("#regex\\s+(.+)#", $PostfixHistorySearch, $re)) {
        $PostfixHistorySearch = str_replace(".", "\\.", $PostfixHistorySearch);
        $PostfixHistorySearch = str_replace("*", ".*?", $PostfixHistorySearch);
        $PostfixHistorySearch = str_replace("[", "\\[", $PostfixHistorySearch);
        $PostfixHistorySearch = str_replace("]", "\\]", $PostfixHistorySearch);
        $PostfixHistorySearch = str_replace("(", "\\(", $PostfixHistorySearch);
        $PostfixHistorySearch = str_replace(")", "\\)", $PostfixHistorySearch);
        $PostfixHistorySearch = str_replace("/", "\\/", $PostfixHistorySearch);
    } else {
        $PostfixHistorySearch = $re[1];
    }
    $unix = new unix();
    $zcat = $unix->find_program("zcat");
    $grep = $unix->find_program("grep");
    if (is_file("/usr/share/artica-postfix/ressources/logs/web/mail-history.log")) {
        @unlink("/usr/share/artica-postfix/ressources/logs/web/mail-history.log");
    }
    $perc = 20;
    while (list($zDate, $filepath) = each($MAIN)) {
        $perc = $perc + 1;
        if ($perc > 95) {
            $perc = 95;
        }
        $size = @filesize("/usr/share/artica-postfix/ressources/logs/web/mail-history.log");
        $size = $size / 1024;
        $size = round($size, 2);
        echo "Scanning {$filepath}\n";
        build_progress("{scanning} {$zDate} for {$PostfixHistorySearch} ( {$size}KB )", $perc);
        $cmd = "{$zcat} {$filepath} | {$grep} -E '{$PostfixHistorySearch}' >>/usr/share/artica-postfix/ressources/logs/web/mail-history.log 2>&1";
        echo "{$cmd}\n";
        shell_exec($cmd);
    }
    $countlines = $unix->COUNT_LINES_OF_FILE("/usr/share/artica-postfix/ressources/logs/web/mail-history.log");
    $size = @filesize("/usr/share/artica-postfix/ressources/logs/web/mail-history.log");
    $size = $size / 1024;
    $size = round($size, 2);
    build_progress("{scanning} {done} {$countlines} {lines} {$size}KB", 100);
    @chmod("/usr/share/artica-postfix/ressources/logs/web/mail-history.log", 0755);
}
function update_milter_greylist()
{
    $unix = new unix();
    $mirror = "http://mirror.articatech.net/webfilters-databases";
    if ($GLOBALS["VERBOSE"]) {
        echo "Downloading {$mirror}/milter-greylist-database.txt\n";
    }
    $curl = new ccurl("{$mirror}/milter-greylist-database.txt");
    $curl->NoHTTP_POST = true;
    $temppath = $unix->TEMP_DIR();
    if (!$curl->GetFile("{$temppath}/milter-greylist-database.txt")) {
        postfix_admin_mysql(0, "Unable to get Milter-greylist index file", $curl->error);
        return;
    }
    if (!is_file("{$temppath}/milter-greylist-database.txt")) {
        postfix_admin_mysql(0, "Unable to get Milter-greylist index file (no such file)", $curl->error);
        return;
    }
    $data = @file_get_contents("{$temppath}/milter-greylist-database.txt");
    $MAIN = unserialize($data);
    if ($GLOBALS["VERBOSE"]) {
        echo $data . "\n";
    }
    if ($GLOBALS["VERBOSE"]) {
        print_r($MAIN);
    }
    @unlink("{$temppath}/milter-greylist-database.txt");
    $TIME = $MAIN["PATTERN"]["TIME"];
    $MD5 = $MAIN["PATTERN"]["MD5"];
    $sock = new sockets();
    $MyTime = $sock->GET_INFO("MilterGreyListPatternTime");
    if (!is_file("/etc/mail/milter-greylist-database.conf")) {
        $MyTime = 0;
    }
    if ($TIME == $MyTime) {
        if ($GLOBALS["VERBOSE"]) {
            echo "{$TIME}=={$MyTime} No new update\n";
        }
        return;
    }
    $curl = new ccurl("{$mirror}/milter-greylist-database.gz");
    $curl->NoHTTP_POST = true;
    if (!$curl->GetFile("{$temppath}/milter-greylist-database.gz")) {
        postfix_admin_mysql(0, "Unable to get milter-greylist-database.gz", $curl->error, __FILE__, __LINE__);
        return;
    }
    $md5f = md5_file("{$temppath}/milter-greylist-database.gz");
    if ($md5f != $MD5) {
        @unlink("{$temppath}/milter-greylist-database.gz");
        postfix_admin_mysql(0, "Unable to get milter-greylist-database.gz (corrupted)", $curl->error, __FILE__, __LINE__);
        return;
    }
    if (!$unix->uncompress("{$temppath}/milter-greylist-database.gz", "{$temppath}/milter-greylist-database.conf")) {
        @unlink("{$temppath}/milter-greylist-database.gz");
        postfix_admin_mysql(0, "Unable to extract milter-greylist-database.gz (corrupted)", null, __FILE__, __LINE__);
        return;
    }
    @unlink("{$temppath}/milter-greylist-database.gz");
    @unlink("/etc/mail/milter-greylist-database.conf");
    @copy("{$temppath}/milter-greylist-database.conf", "/etc/mail/milter-greylist-database.conf");
    @unlink("{$temppath}/milter-greylist-database.conf");
    postfix_admin_mysql(0, "Success updating new Milter-greylist database version {$TIME}", null, __FILE__, __LINE__);
    $sock->SET_INFO("MilterGreyListPatternTime", $TIME);
    $sock->SET_INFO("MilterGreyListPatternCount", $unix->COUNT_LINES_OF_FILE("/etc/mail/milter-greylist-database.conf"));
    $main = new maincf_multi("master", "master");
    $check_client_access = $main->check_client_access();
    $postfix = $unix->find_program("postfix");
    $php5 = $unix->LOCATE_PHP5_BIN();
    $nohup = $unix->find_program("nohup");
    shell_exec("{$php5} /usr/share/artica-postfix/exec.postfix.maincf.php --body-checks >/dev/null 2>&1 &");
    shell_exec("{$postfix} stop");
    shell_exec("{$postfix} start");
    postfix_admin_mysql(1, "Restarting Milter-greylist service", null, __FILE__, __LINE__);
    shell_exec("/etc/init.d/milter-greylist restart");
}
Exemple #13
0
function xrun()
{
    $unix = new unix();
    $siege = $unix->find_program("siege");
    $sock = new sockets();
    $ARRAY = unserialize(base64_decode($sock->GET_INFO("SquidSiegeConfig")));
    if (!is_numeric($ARRAY["GRAB_URLS"])) {
        $ARRAY["GRAB_URLS"] = 0;
    }
    if (!is_numeric($ARRAY["USE_LOCAL_PROXY"])) {
        $ARRAY["USE_LOCAL_PROXY"] = 1;
    }
    if (!is_numeric($ARRAY["SESSIONS"])) {
        $ARRAY["SESSIONS"] = 150;
    }
    if (!is_numeric($ARRAY["MAX_TIME"])) {
        $ARRAY["MAX_TIME"] = 30;
    }
    build_progress_disconnect("{starting}", 5);
    if (!is_file($siege)) {
        build_progress_disconnect("{please_wait} {installing} SIEGE", 50);
        $unix->DEBIAN_INSTALL_PACKAGE("siege");
        $siege = $unix->find_program("siege");
        if (!is_file($siege)) {
            build_progress_disconnect("{installing} SIEGE {failed}", 110);
        }
    }
    $f[] = "internet = true";
    if ($ARRAY["USE_LOCAL_PROXY"] == 1) {
        $squid = new squidbee();
        if ($squid->hasProxyTransparent == 1) {
            $port = $squid->second_listen_port;
        } else {
            $port = $squid->listen_port;
        }
        $addr = "127.0.0.1";
    } else {
        $addr = $ARRAY["REMOTE_PROXY"];
        $port = intval($ARRAY["REMOTE_PROXY_PORT"]);
    }
    if ($addr == null) {
        build_progress_disconnect("{failed} No proxy address", 110);
        return;
    }
    if ($port == 0) {
        build_progress_disconnect("{failed} No proxy port", 110);
        return;
    }
    if ($ARRAY["SESSIONS"] == 0) {
        build_progress_disconnect("{failed} {simulate} 0 sessions", 110);
        return;
    }
    $f[] = "proxy-host ={$addr}";
    $f[] = "proxy-port = {$port}";
    $f[] = "user-agent = Mozilla/5.0 (compatible; IE 11.0; Win32; Trident/7.0)";
    $f[] = "file = /etc/siege/urls.txt";
    $f[] = "concurrent = {$ARRAY["SESSIONS"]}";
    $f[] = "time = {$ARRAY["MAX_TIME"]}S";
    $f[] = "timeout = 5";
    $f[] = "logfile = /var/log/siege.log";
    if (trim($ARRAY["USERNAME"]) != null) {
        $f[] = "username = {$ARRAY["USERNAME"]}";
        $f[] = "password = {$ARRAY["PASSWORD"]}";
    }
    @file_put_contents("/root/.siegerc", @implode("\n", $f));
    $filetemp = $unix->FILE_TEMP();
    $nohup = $unix->find_program("nohup");
    $URLS_NUMBER = $unix->COUNT_LINES_OF_FILE("/etc/siege/urls.txt");
    if ($URLS_NUMBER < 20) {
        @unlink("/etc/siege/urls.txt");
        if ($ARRAY["GRAB_URLS"] == 1) {
            import_urls();
        } else {
            @copy("/usr/share/artica-postfix/bin/install/squid/urls.txt", "/etc/siege/urls.txt");
        }
        $URLS_NUMBER = $unix->COUNT_LINES_OF_FILE("/etc/siege/urls.txt");
    }
    $FINAL["urls"] = $URLS_NUMBER;
    $FINAL["START_TIME"] = time();
    $ss[] = "{$nohup} {$siege} --concurrent={$ARRAY["SESSIONS"]}";
    $ss[] = "--internet --file=/etc/siege/urls.txt --time={$ARRAY["MAX_TIME"]}S";
    $ss[] = "--benchmark --rc=/root/.siegerc >{$filetemp} 2>&1 &";
    $cmd = @implode(" ", $ss);
    echo "{$cmd}\n";
    build_progress_disconnect("{executing}", 50);
    system($cmd);
    sleep(2);
    $pid = $unix->PIDOF($siege);
    while ($unix->process_exists($pid)) {
        $array_mem = getSystemMemInfo();
        $MemFree = $array_mem["MemFree"];
        $array_load = sys_getloadavg();
        $internal_load = $array_load[0];
        echo "Memory Free: " . round($MemFree / 1024) . " MB\n";
        echo "Load: {$internal_load}\n";
        build_progress_disconnect("{please_wait} Load:{$internal_load}", 50);
        sleep(2);
        $pid = $unix->PIDOF($siege);
    }
    build_progress_disconnect("{please_wait} {analyze}...", 90);
    $array = explode("\n", @file_get_contents($filetemp));
    @unlink($filetemp);
    while (list($num, $val) = each($array)) {
        echo "{$val}\n";
        if (preg_match("#alert#", $val)) {
            continue;
        }
        if (preg_match("#ERROR#", $val)) {
            continue;
        }
        if (preg_match("#(.+?):\\s+(.+)#", $val, $re)) {
            $FINAL[trim($re[1])] = trim($re[2]);
        }
    }
    $FINAL["STOP_TIME"] = time();
    build_progress_disconnect("{done}...", 99);
    sleep(5);
    build_progress_disconnect("{done}...", 100);
    @file_put_contents("/usr/share/artica-postfix/ressources/logs/web/siege.report.txt", serialize($FINAL));
    @chmod("/usr/share/artica-postfix/ressources/logs/web/siege.report.txt", 0755);
}
function _analyze_file($filepath, $zmd5)
{
    if (!is_file($filepath)) {
        events("{$filepath} no such file");
        return false;
    }
    $sock = new sockets();
    $unix = new unix();
    $EnableImportWithSarg = $sock->GET_INFO("EnableImportWithSarg");
    if (!is_numeric($EnableImportWithSarg)) {
        $EnableImportWithSarg = 1;
    }
    $SargOutputDir = $sock->GET_INFO("SargOutputDir");
    if ($SargOutputDir == null) {
        $SargOutputDir = "/var/www/html/squid-reports";
    }
    $basename = basename($filepath);
    $timeStart = time();
    $unix = new unix();
    $sock = new sockets();
    $q = new mysql_squid_builder();
    $TimeOfFile = strtotime(GetDateOfFile($filepath));
    $ContainerDir = "/var/log/artica-postfix/squid/queues/" . date("Y-m-d-h", $TimeOfFile);
    @mkdir($ContainerDir, 0755, true);
    $handle = @fopen($filepath, "r");
    if (!$handle) {
        events("Failed to open file {$filepath}");
        echo "Failed to open file\n";
        return;
    }
    $c = 0;
    $max = $unix->COUNT_LINES_OF_FILE($filepath);
    $GLOBALS["BUFFER_FILE_ANALYZED"] = $filepath;
    events("{$filepath} {$max} lines");
    $ligne = mysql_fetch_array($q->QUERY_SQL("SELECT * FROM accesslogs_import WHERE zmd5='{$zmd5}'"));
    $FileStatus = $ligne["status"];
    if (!$GLOBALS["SIMULATE"]) {
        if ($ligne["filename"] == null) {
            echo "{$filepath}: {$zmd5} did not match expected md5\n";
            @fclose($handle);
            return;
        }
    }
    if ($GLOBALS["VERBOSE"]) {
        echo "Container: {$ContainerDir}\n";
        echo "Status...: {$FileStatus}\n";
        echo "Lines....: {$max}\n";
    }
    if (!$GLOBALS["FORCE"]) {
        if ($FileStatus == 3) {
            events("{$filepath} already analyzed, skip it...");
            @fclose($handle);
            return true;
        }
    }
    if ($EnableImportWithSarg == 1) {
        $u = null;
        $nice = EXEC_NICE();
        $sarg = $unix->find_program("sarg");
        $php = $unix->LOCATE_PHP5_BIN();
        $squid = new squidbee();
        if ($squid->LDAP_AUTH == 1) {
            $usersauth = true;
        }
        if ($squid->LDAP_EXTERNAL_AUTH == 1) {
            $usersauth = true;
        }
        if ($usersauth) {
            echo "Starting......: " . date("H:i:s") . " Sarg, user authentification enabled\n";
            $u = " -i ";
        }
        if (is_file($sarg)) {
            shell_exec("{$php} /usr/share/artica-postfix/exec.sarg.php --conf >/dev/null 2>&1");
            exec("{$nice}{$sarg} {$u}-f /etc/squid3/sarg.conf -l {$filepath} -o \"{$SargOutputDir}\" 2>&1", $sargR);
            while (list($index, $line) = each($sargR)) {
                events("Sarg: {$line}\n");
            }
        }
    }
    $percent_ret = 0;
    while (!feof($handle)) {
        $c++;
        $buffer = trim(fgets($handle));
        if ($buffer == null) {
            continue;
        }
        $array = parseline($buffer);
        if (count($array) == 0) {
            continue;
        }
        $ip = null;
        $user = null;
        $xtime = $array["TIME"];
        $ip = $array["IPADDR"];
        $user = $array["UID"];
        $code_error = $array["ERRCODE"];
        $size = $array["SIZE"];
        $uri = $array["URI"];
        $cached = $array["CACHED"];
        // $q->QUERY_SQL("INSERT INTO accesslogs_import (zmd5,filename,zDate,size,status,percent) VALUES ('$md5','$filename','$date','$size',0,0)");
        $HOSTNAME = $array["HOSTNAME"];
        if (is_numeric($user)) {
            echo "\n\n\n****************\n\nNumeric user:{$user}\n{$buffer}\n\n";
            die;
        }
        if ($ip == null) {
            if ($HOSTNAME != null) {
                $ip = $HOSTNAME;
            }
        }
        $GLOBALS["BUFFER_ANALYZED"] = $buffer;
        $GLOBALS["squidtail"]->Builsql($ip, $user, $uri, $code_error, $size, $xtime, $cached, null, $xtime);
        if ($GLOBALS["SIMULATE"]) {
            continue;
        }
        $percent = $c / $max * 100;
        $percent = round($percent);
        if ($percent != $percent_ret) {
            if ($GLOBALS["VERBOSE"]) {
                echo "****************** {$percent}% ********************\n";
            }
            $percent_ret = $percent;
            events("{$percent_ret}% " . count($GLOBALS["squidtail"]->GLOBAL_QUEUE) . " in memory - {$filepath}");
            $q->QUERY_SQL("UPDATE accesslogs_import SET percent='{$percent}',status=1 WHERE zmd5='{$zmd5}'");
        }
        if (count($GLOBALS["squidtail"]->GLOBAL_QUEUE) > 2000) {
            events("analyze_file()::{$basename}::{$percent}% GLOBAL_RTTSIZE......: " . count($GLOBALS["squidtail"]->GLOBAL_RTTSIZE) . " items...", __LINE__);
            events("analyze_file()::{$basename}::{$percent}% GLOBAL_PAGEKEEPER...: " . count($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER) . " items...", __LINE__);
            events("analyze_file()::{$basename}::{$percent}% GLOBAL_YOUTUBE......: " . count($GLOBALS["squidtail"]->GLOBAL_YOUTUBE) . " items...", __LINE__);
            events("analyze_file()::{$basename}::{$percent}% GLOBAL_SQUIDUSERS...: " . count($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS) . " items...", __LINE__);
            events("analyze_file()::{$basename}::{$percent}% GLOBAL_SEARCHWORDS..: " . count($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS) . " items...", __LINE__);
            PURGE_GLOBAL_QUEUE($GLOBALS["squidtail"]->GLOBAL_QUEUE);
            $GLOBALS["squidtail"]->GLOBAL_QUEUE = array();
            if (count($GLOBALS["squidtail"]->GLOBAL_RTTSIZE) > 500) {
                @mkdir("{$ContainerDir}/RTTSize", 0755, true);
                @file_put_contents("{$ContainerDir}/RTTSize/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_RTTSIZE)), serialize($GLOBALS["squidtail"]->GLOBAL_RTTSIZE));
                $GLOBALS["squidtail"]->GLOBAL_RTTSIZE = array();
                $RTTSIZE = true;
            }
            if (count($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER) > 500) {
                @mkdir("{$ContainerDir}/PageKeeper", 0755, true);
                @file_put_contents("{$ContainerDir}/PageKeeper/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER)), serialize($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER));
                $GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER = array();
                $PAGEKEEP = true;
            }
            if (count($GLOBALS["squidtail"]->GLOBAL_YOUTUBE) > 500) {
                @mkdir("{$ContainerDir}/Youtube", 0755, true);
                $md5 = md5(serialize($GLOBALS["squidtail"]->GLOBAL_YOUTUBE));
                youtube_events("Saving queue:(2000) {$ContainerDir}/Youtube/" . $md5, __LINE__);
                @file_put_contents("{$ContainerDir}/Youtube/" . $md5, serialize($GLOBALS["squidtail"]->GLOBAL_YOUTUBE));
                $GLOBALS["squidtail"]->GLOBAL_YOUTUBE = array();
                $YOUTUBE = true;
            }
            if (count($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS) > 500) {
                @mkdir("{$ContainerDir}/Members", 0755, true);
                @file_put_contents("{$ContainerDir}/Members/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS)), serialize($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS));
                $GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS = array();
            }
            if (count($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS) > 500) {
                @mkdir("{$ContainerDir}/SearchWords", 0755, true);
                @file_put_contents("{$ContainerDir}/SearchWords/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS)), serialize($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS));
                $GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS = array();
            }
        }
        // PURGE OVER 2000
    }
    // END GLOBAL LOOP
    @fclose($handle);
    PURGE_GLOBAL_QUEUE($GLOBALS["squidtail"]->GLOBAL_QUEUE);
    events("analyze_file()::{$basename}::  Container.........: `{$ContainerDir}` ", __LINE__);
    events("analyze_file()::{$basename}::  GLOBAL_RTTSIZE....: " . count($GLOBALS["squidtail"]->GLOBAL_RTTSIZE) . " items...", __LINE__);
    events("analyze_file()::{$basename}::  GLOBAL_PAGEKEEPER.: " . count($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER) . " items...", __LINE__);
    events("analyze_file()::{$basename}::  GLOBAL_YOUTUBE....: " . count($GLOBALS["squidtail"]->GLOBAL_YOUTUBE) . " items...", __LINE__);
    events("analyze_file()::{$basename}::  GLOBAL_SQUIDUSERS.: " . count($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS) . " items...", __LINE__);
    events("analyze_file()::{$basename}::  GLOBAL_SEARCHWORDS: " . count($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS) . " items...", __LINE__);
    $q->QUERY_SQL("UPDATE accesslogs_import SET percent='100',status='3' WHERE zmd5='{$zmd5}'");
    if (count($GLOBALS["squidtail"]->GLOBAL_RTTSIZE) > 0) {
        @mkdir("{$ContainerDir}/RTTSize", 0755, true);
        @file_put_contents("{$ContainerDir}/RTTSize/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_RTTSIZE)), serialize($GLOBALS["squidtail"]->GLOBAL_RTTSIZE));
        $RTTSIZE = true;
    }
    if (count($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER) > 0) {
        @mkdir("{$ContainerDir}/PageKeeper", 0755, true);
        @file_put_contents("{$ContainerDir}/PageKeeper/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER)), serialize($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER));
        $PAGEKEEP = true;
    }
    if (count($GLOBALS["squidtail"]->GLOBAL_YOUTUBE) > 0) {
        @mkdir("{$ContainerDir}/Youtube", 0755, true);
        $md5 = md5(serialize($GLOBALS["squidtail"]->GLOBAL_YOUTUBE));
        youtube_events("Saving queue: {$ContainerDir}/Youtube/" . $md5, __LINE__);
        @file_put_contents("{$ContainerDir}/Youtube/" . $md5, serialize($GLOBALS["squidtail"]->GLOBAL_YOUTUBE));
        $YOUTUBE = true;
    }
    if (count($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS) > 0) {
        @mkdir("{$ContainerDir}/Members", 0755, true);
        @file_put_contents("{$ContainerDir}/Members/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS)), serialize($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS));
    }
    if (count($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS) > 0) {
        @mkdir("{$ContainerDir}/SearchWords", 0755, true);
        @file_put_contents("{$ContainerDir}/SearchWords/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS)), serialize($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS));
    }
    $size = round($size / 1024, 2);
    events("analyze_file()::{$basename}:: {$max}: lines parsed in " . $unix->distanceOfTimeInWords($timeStart, time()) . __LINE__);
    if (system_is_overloaded(basename(__FILE__))) {
        return;
    }
    $php5 = $unix->LOCATE_PHP5_BIN();
    $nice = EXEC_NICE();
    $nohup = $unix->find_program("nohup");
    $cmd = "{$nohup} {$php5} " . __FILE__ . " --squid >/dev/null 2>&1 &";
    events("analyze_file()::{$cmd}");
    shell_exec($cmd);
    if ($PAGEKEEP) {
        $cmd = "{$nohup} {$php5} /usr/share/artica-postfix/exec.squid.stats.php --thumbs-parse >/dev/null 2>&1 &";
        events(__FUNCTION__ . ":: {$cmd}", __LINE__);
        shell_exec($cmd);
    }
    if ($YOUTUBE) {
        $cmd = "{$nohup} {$php5} " . __FILE__ . " --youtube >/dev/null 2>&1 &";
        events(__FUNCTION__ . ":: {$cmd}", __LINE__);
        shell_exec($cmd);
    }
    if ($RTTSIZE) {
        $cmd = "{$nohup} {$php5} /usr/share/artica-postfix/exec.squid-users-rttsize.php --now >/dev/null 2>&1 &";
        events(__FUNCTION__ . ":: {$cmd}", __LINE__);
        shell_exec($cmd);
    }
}
function parse($filename)
{
    $unix = new unix();
    $LastScannLine = 0;
    $GLOBALS["MYSQL_CATZ"] = new mysql_catz();
    $GLOBALS["SQUID_FAMILY_CLASS"] = new squid_familysite();
    if (!isset($GLOBALS["MYHOSTNAME"])) {
        $unix = new unix();
        $GLOBALS["MYHOSTNAME"] = $unix->hostname_g();
    }
    $filesource = dirname(__FILE__) . "/ressources/conf/upload/{$filename}";
    if (!is_file($filesource)) {
        build_progress(110, "{$filename} no such file");
        return;
    }
    $tmpfile = $unix->FILE_TEMP();
    if (!@copy($filesource, $tmpfile)) {
        @unlink($filesource);
        build_progress(110, "{$filename} -> {$tmpfile} {failed}");
        return;
    }
    @unlink($filesource);
    $SumOflines = $unix->COUNT_LINES_OF_FILE($tmpfile);
    echo "{$tmpfile} {$SumOflines} lines\n";
    $handle = @fopen($tmpfile, "r");
    if (!$handle) {
        echo "Fopen failed on {$tmpfile}\n";
        build_progress(110, "{$tmpfile} {failed}");
        @unlink($tmpfile);
        return false;
    }
    $c = 0;
    $d = 0;
    $e = 0;
    $prc = 0;
    $prc_text = 0;
    $mysql_first_time = 0;
    $SITES = array();
    $NOT_CATEGORIZED = array();
    $CATEGORIZED = array();
    $RQS = array();
    $IPClass = new IP();
    $FIRSTTIME = 0;
    $LASTTIME = 0;
    $TIME1 = time();
    while (!feof($handle)) {
        $c++;
        $d++;
        $e++;
        $prc = $c / $SumOflines;
        $prc = round($prc * 100);
        $buffer = trim(fgets($handle));
        if ($buffer == null) {
            continue;
        }
        $stats_sites = count($SITES);
        $stats_categorized = count($CATEGORIZED);
        $stats_not_categorized = count($NOT_CATEGORIZED);
        if (!isset($GLOBALS["LAST_PRC"])) {
            build_progress($prc, "{$c}/{$SumOflines} {please_wait}");
            $GLOBALS["LAST_PRC"] = $prc;
        }
        if ($prc > 5) {
            if ($prc < 95) {
                if ($GLOBALS["LAST_PRC"] != $prc) {
                    $array_load = sys_getloadavg();
                    $internal_load = $array_load[0];
                    $mem = round(memory_get_usage() / 1024 / 1000, 2);
                    echo "Load: {$internal_load}, Memory {$mem}MB\n";
                    echo "Categorized: " . FormatNumber($stats_categorized) . "\n";
                    echo "Unknown....: " . FormatNumber($stats_not_categorized) . "\n";
                    echo "Web sites..: " . FormatNumber($stats_sites) . "\n";
                    build_progress($prc, FormatNumber($c) . "/" . FormatNumber($SumOflines) . " {please_wait} - {$mem}MB {memory}");
                    $GLOBALS["LAST_PRC"] = $prc;
                }
            }
        }
        $array = parseAccessLine($buffer);
        if (count($array) == 0) {
            continue;
        }
        $TIME = $array["TIME"];
        $LASTTIME = $TIME;
        if ($FIRSTTIME == 0) {
            $FIRSTTIME = $TIME;
        }
        $CATEGORY = $array["CATEGORY"];
        $FAMILYSITE = $array["FAMILYSITE"];
        $SIZE = intval($array["SIZE"]);
        if ($IPClass->isIPAddress($FAMILYSITE)) {
            if (!isset($IPADDRESSES[$FAMILYSITE]["RQS"])) {
                $IPADDRESSES[$FAMILYSITE]["RQS"] = 1;
                $IPADDRESSES[$FAMILYSITE]["SIZE"] = 0;
                $IPADDRESSES[$FAMILYSITE]["CATEGORY"] = $CATEGORY;
            } else {
                $IPADDRESSES[$FAMILYSITE]["RQS"] = $IPADDRESSES[$FAMILYSITE]["RQS"] + 1;
                $IPADDRESSES[$FAMILYSITE]["SIZE"] = $IPADDRESSES[$FAMILYSITE]["SIZE"] + $SIZE;
            }
            continue;
        }
        if (!isset($SITES[$FAMILYSITE])) {
            $SITES[$FAMILYSITE] = 0;
        }
        if (!isset($RQS[$FAMILYSITE])) {
            $RQS[$FAMILYSITE] = 0;
        }
        $SITES[$FAMILYSITE] = $SITES[$FAMILYSITE] + $SIZE;
        $RQS[$FAMILYSITE] = $RQS[$FAMILYSITE] + 1;
        if ($CATEGORY != null) {
            $CATEGORIZED[$FAMILYSITE] = $CATEGORY;
            continue;
        }
        $NOT_CATEGORIZED[$FAMILYSITE] = true;
    }
    fclose($handle);
    @unlink($tmpfile);
    build_progress(91, "{building_report}");
    $TIME2 = time();
    $stats_sites = count($SITES);
    $stats_categorized = count($CATEGORIZED);
    $stats_not_categorized = count($NOT_CATEGORIZED);
    $ARRAY["DURATION"] = $unix->distanceOfTimeInWords($TIME1, $TIME2);
    $ARRAY["SumOflines"] = $SumOflines;
    $ARRAY["stats_sites"] = $stats_sites;
    $ARRAY["stats_ip"] = count($IPADDRESSES);
    $ARRAY["firsttime"] = $FIRSTTIME;
    $ARRAY["lasttime"] = $LASTTIME;
    $ARRAY["stats_categorized"] = $stats_categorized;
    $ARRAY["stats_not_categorized"] = $stats_not_categorized;
    build_progress(92, "{building_report}");
    $CSV1[] = array("website", "size", "requests");
    while (list($familysite, $ligne) = each($NOT_CATEGORIZED)) {
        $CSV1[] = array($familysite, $SITES[$familysite], $RQS[$familysite]);
    }
    build_progress(95, "{building_report}");
    $CSV2[] = array("website", "category", "size", "requests");
    while (list($familysite, $category) = each($CATEGORIZED)) {
        $CSV2[] = array($familysite, $category, $SITES[$familysite], $RQS[$familysite]);
    }
    build_progress(97, "{building_report}");
    $CSV3[] = array("Public IP addresses", "category", "size", "requests");
    while (list($ip, $ARRAYIPS) = each($IPADDRESSES)) {
        $category = $ARRAYIPS["CATEGORY"];
        $size = $ARRAYIPS["SIZE"];
        $RQS = $ARRAYIPS["RQS"];
        $CSV3[] = array($ip, $category, $size, $RQS);
    }
    build_progress(99, "{saving_reports}");
    outputCSV($CSV1, "/usr/share/artica-postfix/ressources/logs/notcategorized.csv");
    outputCSV($CSV2, "/usr/share/artica-postfix/ressources/logs/categorized.csv");
    outputCSV($CSV3, "/usr/share/artica-postfix/ressources/logs/ipcategorized.csv");
    @file_put_contents("/usr/share/artica-postfix/ressources/logs/categorized.array", serialize($ARRAY));
    @chmod("/usr/share/artica-postfix/ressources/logs/notcategorized.csv", 0755);
    @chmod("/usr/share/artica-postfix/ressources/logs/ipcategorized.csv", 0755);
    @chmod("/usr/share/artica-postfix/ressources/logs/categorized.csv", 0755);
    @chmod("/usr/share/artica-postfix/ressources/logs/categorized.array", 0755);
    build_progress(100, "{done}");
}
Exemple #16
0
function sargToFile($filePath)
{
    if (!is_file($filePath)) {
        progress("Fatal {$filePath} no such file", 10);
        return;
    }
    $unix = new unix();
    $sarg_bin = $unix->find_program("sarg");
    $linesNumber = $unix->COUNT_LINES_OF_FILE($filePath);
    $basename = basename($filePath);
    progress("Open {$filePath} {$linesNumber} lines", 10);
    $sock = new sockets();
    $SargOutputDir = $sock->GET_INFO("SargOutputDir");
    if ($SargOutputDir == null) {
        $SargOutputDir = "/var/www/html/squid-reports";
    }
    $nice = EXEC_NICE();
    $usersauth = false;
    $t = time();
    $squid = new squidbee();
    if ($squid->is_auth()) {
        $usersauth = true;
    }
    if ($usersauth) {
        events("User authentification enabled");
        $u = " -i ";
    } else {
        events("User authentification disabled");
    }
    $t = time();
    $cmd = "{$nice}{$sarg_bin} {$u}-f /etc/squid3/sarg.conf -l \"{$filePath}\" -o \"{$SargOutputDir}\" -x -z 2>&1";
    progress("Open {$cmd}", 10);
    exec($cmd, $results);
    while (list($index, $line) = each($results)) {
        if (preg_match("#SARG: OPTION:#", $line)) {
            continue;
        }
        events($line);
    }
    if ($basename == "sarg.log") {
        continue;
    }
    $took = $unix->distanceOfTimeInWords($t, time(), true);
    sarg_admin_events("{$basename} generated took: {$took}\n" . @implode("\n", $results), __FUNCTION__, __FILE__, __LINE__, "sarg");
    build_index_page();
}
function parse_db($id)
{
    $unix = new unix();
    $pidfile = "/etc/artica-postfix/" . basename(__FILE__) . ".{$id}.pid";
    $pid = trim(@file_get_contents($pidfile));
    if ($unix->process_exists($pid)) {
        $pid = getmypid();
        echo "[{$pid}]:: Process {$pid} already running...\n";
        die;
    }
    $pid = getmypid();
    file_put_contents($pidfile, $pid);
    $sql = "SELECT * FROM emailing_db_paths WHERE ID={$id}";
    $q = new mysql();
    $ligne = mysql_fetch_array($q->QUERY_SQL($sql, "artica_backup"));
    $unzipbin = $unix->find_program("unzip");
    if ($unzipbin == null) {
        echo "[{$pid}]:: unzip tool no such file or directory...\n";
        update_status($id, "110", 1, "unzip tool no such file or directory");
        die;
    }
    $zip_path = $ligne["filepath"];
    if (!is_file($zip_path)) {
        echo "[{$pid}]:: {$zip_path} no such file or directory\n";
        update_status($id, "110", 1, "zip db no such file or directory");
        die;
    }
    $tmp_path = "/tmp/emailing-import/{$id}";
    writeevent("using {$tmp_path}", $id);
    @mkdir($tmp_path, 666, true);
    writeevent("Uncompress {$zip_path}", $id);
    echo "{$unzipbin} -o {$zip_path} -d {$tmp_path}\n";
    shell_exec("{$unzipbin} -o {$zip_path} -d {$tmp_path}");
    $files = $unix->DirFiles($tmp_path);
    if (!is_array($files)) {
        update_status($id, "110", 1, "{$zip_path} corrupted or no files stored");
        die;
    }
    while (list($filename, $file_name) = each($files)) {
        writeevent("parsing {$file_name}", $pid);
        $max = $unix->COUNT_LINES_OF_FILE("{$tmp_path}/{$file_name}");
        writeevent("parsing {$file_name} {$max} entries", $pid);
        $handle = @fopen("{$tmp_path}/{$file_name}", "r");
        $ligne["databasename"] = format_mysql_table($ligne["databasename"]);
        $q = new mysql();
        $q->CheckTableEmailingContacts("emailing_{$ligne["databasename"]}");
        $sql = "INSERT INTO emailing_{$ligne["databasename"]} (`gender`,`firstname`, `lastname`,`email`,`phone`,`city`,`cp`,`postaladdress`,`domain`) VALUES";
        if ($handle) {
            while (!feof($handle)) {
                $tw = $tw + 1;
                $count = $count + 1;
                unset($re);
                $buffer = fgets($handle, 4096);
                if (trim($buffer) == null) {
                    continue;
                }
                $buffer = str_replace('"', "", $buffer);
                $buffer = str_replace(';', ",", $buffer);
                $lines = explode(",", addslashes($buffer));
                if (!is_array($lines)) {
                    if (preg_match("#.+?@.+#", $buffer)) {
                        for ($i = 0; $i < 8; $i++) {
                            $lines[$i] = "";
                        }
                        $lines[3] = trim($buffer);
                    }
                }
                if (count($lines) < 2) {
                    if (preg_match("#.+?@.+#", $buffer)) {
                        for ($i = 0; $i < 8; $i++) {
                            $lines[$i] = "";
                        }
                        $lines[3] = trim($buffer);
                    }
                }
                $lines[3] = str_replace(";", ".", $lines[3]);
                $lines[3] = str_replace("?", ".", $lines[3]);
                $lines[3] = str_replace("@.", "@", $lines[3]);
                $lines[3] = str_replace('^', '@', $lines[3]);
                $lines[3] = str_replace(',', '.', $lines[3]);
                if (trim($lines[3]) == null) {
                    writeevent("failed 3:[{$buffer}] [" . __LINE__ . "]", $pid);
                    $GLOBALS["FAILED_CONTACTS"] = $GLOBALS["FAILED_CONTACTS"] + 1;
                    unset($lines);
                    continue;
                }
                if (!preg_match("#(.+?)@(.+?)\\.(.+)#", $lines[3], $re)) {
                    writeevent("failed 3:{$lines[3]} bad email address [" . __LINE__ . "]", $pid);
                    $GLOBALS["FAILED_CONTACTS"] = $GLOBALS["FAILED_CONTACTS"] + 1;
                    unset($lines);
                    continue;
                }
                $domain = $re[2];
                if (substr($domain, strlen($domain) - 1, 1) == '.') {
                    $domain = substr($domain, 0, strlen($domain) - 1);
                }
                if (preg_match("#^\\..+#", $domain)) {
                    writeevent("failed 3:{$lines[3]} bad domain {$domain} [" . __LINE__ . "]", $pid);
                    $GLOBALS["FAILED_CONTACTS"] = $GLOBALS["FAILED_CONTACTS"] + 1;
                    unset($lines);
                    continue;
                }
                if ($GLOBALS["EMAILS"][$lines[3]]) {
                    writeevent("failed 3:{$lines[3]} already exists [" . __LINE__ . "]", $pid);
                    $GLOBALS["FAILED_CONTACTS"] = $GLOBALS["FAILED_CONTACTS"] + 1;
                    continue;
                }
                if (isBlockedMail($lines[3])) {
                    writeevent("failed 3:{$lines[3]} is blacklisted [" . __LINE__ . "]", $pid);
                    $GLOBALS["FAILED_CONTACTS"] = $GLOBALS["FAILED_CONTACTS"] + 1;
                    continue;
                }
                $sqla[] = "('{$lines[0]}',\n\t\t\t\t'{$lines[1]}',\n\t\t\t\t'{$lines[2]}',\n\t\t\t\t'{$lines[3]}',\n\t\t\t\t'{$lines[4]}',\n\t\t\t\t'{$lines[5]}',\n\t\t\t\t'{$lines[6]}',\n\t\t\t\t'{$lines[7]}','{$domain}')";
                $GLOBALS["SUCCESS_CONTACTS"] = $GLOBALS["SUCCESS_CONTACTS"] + 1;
                $GLOBALS["EMAILS"][$lines[3]] = true;
                if (count($GLOBALS["EMAILS"]) > 10000) {
                    $GLOBALS["EMAILS"] = array();
                }
                unset($lines);
                if ($tw > 100) {
                    $fullsql = $sql . "\n" . @implode(",", $sqla);
                    $q = new mysql();
                    $q->QUERY_SQL($fullsql, "artica_backup");
                    if (!$q->ok) {
                        writeevent($id, "{$q->mysql_error}\n{$fullsql}");
                    }
                    unset($sqla);
                    $purc = $count / $max;
                    $purc = $purc * 100;
                    $purc = round($purc, 0);
                    update_status($id, $purc, 0);
                    $tw = 0;
                }
            }
            fclose($handle);
            if (is_array($sqla)) {
                $fullsql = $sql . "\n" . @implode(",", $sqla);
                $q->QUERY_SQL($fullsql, "artica_backup");
                if (!$q->ok) {
                    writeevent($id, "{$q->mysql_error}\n{$fullsql}");
                }
            }
        }
    }
    if ($GLOBALS["SUCCESS_CONTACTS"] > 0) {
        writeevent("Failed.:{$GLOBALS["FAILED_CONTACTS"]}", $id);
        writeevent("Success:{$GLOBALS["SUCCESS_CONTACTS"]}", $id);
        update_status($id, 100, 1, null);
    } else {
        update_status($id, 110, 0, null);
    }
}