Ejemplo n.º 1
0
function ufdbtables($nopid = false)
{
    $unix = new unix();
    $sock = new sockets();
    if (!$nopid) {
        $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid";
        $nohup = $unix->find_program("nohup");
        $pid = @file_get_contents($pidfile);
        if ($unix->process_exists($pid, __FILE__)) {
            $timepid = $unix->PROCCESS_TIME_MIN($pid);
            updatev2_progress(110, "Error, {$timepid} already running [" . __LINE__ . "]");
            if (!$GLOBALS["NOLOGS"]) {
                // ufdbguard_admin_events("UFDB::Warning: Task already executed PID: $pid since {$timepid}Mn",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica");
                return;
            }
        }
        @file_put_contents($pidfile, getmypid());
    }
    $GLOBALS["EVENTS"] = array();
    $CACHE_FILE = "/etc/artica-postfix/ufdb.tables.db";
    updatev2_progress(10, "CACHE_FILE = {$CACHE_FILE} [" . __LINE__ . "]");
    $sock = new sockets();
    $EnableArticaMetaClient = intval($sock->GET_INFO("EnableArticaMetaClient"));
    if ($EnableArticaMetaClient == 1) {
        updatev2_progress(10, "Using Artica Meta console [" . __LINE__ . "]");
        return ufdbtables_artica_meta();
    }
    $UseRemoteUfdbguardService = $sock->GET_INFO('UseRemoteUfdbguardService');
    if (!is_numeric($UseRemoteUfdbguardService)) {
        $UseRemoteUfdbguardService = 0;
    }
    if ($UseRemoteUfdbguardService == 1) {
        updatev2_progress(10, "UseRemoteUfdbguardService = TRUE - aborting [" . __LINE__ . "]");
        return;
    }
    if ($GLOBALS["MIRROR"] == null) {
        updatev2_progress(10, "MIRROR is null #2 [" . __LINE__ . "]");
        unset($GLOBALS["updatev2_checkversion"]);
        updatev2_checkversion();
    }
    if ($GLOBALS["MIRROR"] == null) {
        updatev2_progress(110, "MIRROR is null #1 [" . __LINE__ . "]");
        return;
    }
    $tmpdir = $unix->TEMP_DIR();
    $URIBASE = $GLOBALS["MIRROR"];
    $WORKDIR = $GLOBALS["WORKDIR_LOCAL"];
    if (is_link($WORKDIR)) {
        $WORKDIR = readlink($WORKDIR);
    }
    $CategoriesDatabasesByCron = $sock->GET_INFO("CategoriesDatabaseByCron");
    if (!is_numeric($CategoriesDatabasesByCron)) {
        $CategoriesDatabasesByCron = 1;
    }
    if (!$GLOBALS["FORCE"]) {
        if ($CategoriesDatabasesByCron == 1) {
            if (!$GLOBALS["BYCRON"]) {
                updatev2_progress(110, "{done} CategoriesDatabasesByCron [" . __LINE__ . "]");
                return;
            }
        }
    }
    updatev2_progress(15, "{$URIBASE}/index.txt [" . __LINE__ . "]");
    $curl = new ccurl("{$URIBASE}/index.txt");
    $STATUS = unserialize(@file_get_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD"));
    $STATUS["LAST_DOWNLOAD"]["LAST_CHECK"] = time();
    @file_put_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD", serialize($STATUS));
    if (!$curl->GetHeads()) {
        if ($GLOBALS["VERBOSE"]) {
            echo "Fatal ! {$URIBASE}/index.txt ERROR NUMBER {$curl->CURLINFO_HTTP_CODE}\n";
        }
        if ($curl->CURLINFO_HTTP_CODE == 404 or $curl->CURLINFO_HTTP_CODE == 300) {
            if (!preg_match("#\\/ufdb#", $URIBASE)) {
                $URIBASE = "{$URIBASE}/ufdb";
            }
            $curl = new ccurl("{$URIBASE}/index.txt");
            if (!$curl->GetHeads()) {
                $GLOBALS["EVENTS"][] = "{$URIBASE}/index.txt";
                $GLOBALS["EVENTS"][] = "Failed with error {$curl->error}";
                while (list($a, $b) = each($GLOBALS["CURLDEBUG"])) {
                    $GLOBALS["EVENTS"][] = $b;
                }
                squid_admin_mysql(0, "Unable to download blacklist index file with error: `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
                artica_update_event(0, "Unable to download Artica blacklist index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
                // ufdbguard_admin_events("UFDB::Fatal: Unable to download blacklist index file $curl->error",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica");
                echo "UFDB: Failed to retreive {$URIBASE}/index.txt ({$curl->error})\n";
                updatev2_progress(110, "Failed to retreive {$URIBASE}/index.txt [" . __LINE__ . "]");
                updatev2_adblock();
                return;
            }
        }
    }
    updatev2_progress(20, "Downloading MD5Strings.txt [" . __LINE__ . "]");
    $curl = new ccurl("{$URIBASE}/MD5Strings.txt");
    if (!$curl->GetFile("{$tmpdir}/MD5Strings.txt")) {
        artica_update_event(2, "{$URIBASE}: Unable to download Artica blacklist MD5 table `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
        @unlink("{$tmpdir}/MD5Strings.txt");
    }
    $MD5_strings = unserialize(@file_get_contents("{$tmpdir}/MD5Strings.txt"));
    $source_filetime = $curl->CURL_ALL_INFOS["filetime"];
    if ($GLOBALS["VERBOSE"]) {
        echo "{$URIBASE}/index.txt filetime: {$source_filetime} " . date("Y-m-d H:i:s", $source_filetime) . "\n";
    }
    $GLOBALS["EVENTS"][] = "{$URIBASE}/index.txt";
    $GLOBALS["EVENTS"][] = "filetime: {$source_filetime} " . date("Y-m-d H:i:s", $source_filetime);
    $UFDBGUARD_LAST_INDEX_TIME = "/etc/artica-postfix/UFDBGUARD_LAST_INDEX_TIME";
    $old_time = intval(@file_get_contents("{$UFDBGUARD_LAST_INDEX_TIME}"));
    $GLOBALS["EVENTS"][] = "Old filetime: {$old_time} " . date("Y-m-d H:i:s", $old_time);
    if (is_file("/etc/artica-postfix/ufdbcounts.txt")) {
        if ($source_filetime == $old_time) {
            ufdbtables_md5($MD5_strings, $URIBASE);
            $GLOBALS["EVENTS"][] = "No new updates";
            updatev2_progress(100, "{up-to-date} [" . __LINE__ . "]");
            return true;
        }
        if ($source_filetime < $old_time) {
            ufdbtables_md5($MD5_strings, $URIBASE);
            $GLOBALS["EVENTS"][] = "No new updates";
            updatev2_progress(100, "{up-to-date} [" . __LINE__ . "]");
            return true;
        }
    }
    updatev2_progress(99, "{$URIBASE}/ufdbcounts.txt [" . __LINE__ . "]");
    $curl = new ccurl("{$URIBASE}/ufdbcounts.txt");
    if (!$curl->GetFile("/etc/artica-postfix/ufdbcounts.txt")) {
        $GLOBALS["EVENTS"][] = "{$URIBASE}/ufdbcounts.txt";
        $GLOBALS["EVENTS"][] = "Failed with error {$curl->error}";
        while (list($a, $b) = each($GLOBALS["CURLDEBUG"])) {
            $GLOBALS["EVENTS"][] = $b;
        }
        squid_admin_mysql(0, "Unable to download {$URIBASE}/ufdbcounts.txt index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
        artica_update_event(0, "Unable to download Artica {$URIBASE}/ufdbcounts.txt index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
        // ufdbguard_admin_events("UFDB::Fatal: Unable to download $URIBASE/ufdbcounts.txt index file $curl->error",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica");
    }
    updatev2_progress(99, "{$URIBASE}/index.txt [" . __LINE__ . "]");
    $curl = new ccurl("{$URIBASE}/index.txt");
    if (!$curl->GetFile("/etc/artica-postfix/artica-webfilter-db-index.txt")) {
        $GLOBALS["EVENTS"][] = "{$URIBASE}/index.txt";
        $GLOBALS["EVENTS"][] = "Failed with error {$curl->error}";
        while (list($a, $b) = each($GLOBALS["CURLDEBUG"])) {
            $GLOBALS["EVENTS"][] = $b;
        }
        squid_admin_mysql(0, "Unable to download blacklist index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
        artica_update_event(0, "Unable to download Artica blacklist index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
        // ufdbguard_admin_events("UFDB::Fatal: Unable to download blacklist index file $curl->error",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica");
        echo "UFDB: Failed to retreive {$URIBASE}/index.txt ({$curl->error})\n";
        updatev2_progress2(110, "Unable to download blacklist index file");
        updatev2_adblock();
        return;
    }
    if (ufdbtables_md5($MD5_strings, $URIBASE)) {
        updatev2_progress(100, "{done} [" . __LINE__ . "]");
        return;
    }
    $LOCAL_CACHE = unserialize(base64_decode(@file_get_contents($CACHE_FILE)));
    $REMOTE_CACHE = unserialize(base64_decode(@file_get_contents("/etc/artica-postfix/artica-webfilter-db-index.txt")));
    $CALCULATED_SIZE = 0;
    $MAx = count($REMOTE_CACHE);
    $BigSize = 0;
    $c = 0;
    $ERRORDB = 0;
    while (list($tablename, $size) = each($REMOTE_CACHE)) {
        $OriginalSize = $size;
        if (blacklisted_tables($tablename)) {
            continue;
        }
        $STATUS["LAST_DOWNLOAD"]["CATEGORY"] = $tablename;
        if ($size != $LOCAL_CACHE[$tablename]) {
            ufdbevents("{$tablename}  {$size} <> {$LOCAL_CACHE[$tablename]}");
            $c++;
            if (!ufdbtables_DownloadInstall($URIBASE, $tablename, $size, null)) {
                $ERRORDB++;
                $STATUS["LAST_DOWNLOAD"]["TIME"] = time();
                $STATUS["LAST_DOWNLOAD"]["SIZE"] = $GLOBALS["CURL_LAST_SIZE_DOWNLOAD"] / 1024;
                $STATUS["LAST_DOWNLOAD"]["FAILED"] = $ERRORDB;
                @file_put_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD", serialize($STATUS));
                continue;
            }
            $prc = $c / $MAx * 100;
            updatev2_progress2($prc, "{$tablename} ok");
            $GLOBALS["UFDB_SIZE"] = $GLOBALS["CALCULATED_SIZE"];
            $size = $unix->file_size("{$WORKDIR}/{$tablename}/domains.ufdb");
            $size = round($size / 1024, 2);
            $BigSize = $BigSize + $size;
            @chown("{$WORKDIR}/{$tablename}", "squid");
            @chgrp("{$WORKDIR}/{$tablename}", "squid");
            $LOCAL_CACHE[$tablename] = $OriginalSize;
            $GLOBALS["EVENTS"][] = "Success updating category `{$tablename}` with {$size} Ko";
            $STATUS["LAST_DOWNLOAD"]["TIME"] = time();
            $STATUS["LAST_DOWNLOAD"]["SIZE"] = $GLOBALS["CURL_LAST_SIZE_DOWNLOAD"] / 1024;
            $STATUS["LAST_DOWNLOAD"]["FAILED"] = $ERRORDB;
            @file_put_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD", serialize($STATUS));
        }
    }
    @file_put_contents($CACHE_FILE, base64_encode(serialize($LOCAL_CACHE)));
    updatev2_progress2(100, "DONE ok");
    $ufdbguard_admin_memory = @implode("\n", $GLOBALS["ufdbguard_admin_memory"]);
    $php5 = $unix->LOCATE_PHP5_BIN();
    if (!is_file("/opt/ufdbcat/bin/ufdbcatdd")) {
        system("{$php5} /usr/share/artica-postfix/exec.ufdbcat.php --install --noupdate");
    }
    if ($c > 0) {
        $BigSizeMB = round($BigSize / 1024, 2);
        shell_exec("/etc/init.d/ufdbcat reload");
        squid_admin_mysql(2, "Artica Web filtering Databases Success updated {$c} categories {$BigSizeMB}MB extracted on disk", "{$ufdbguard_admin_memory}" . @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
        artica_update_event(2, "Artica Web filtering Databases Success updated {$c} categories {$BigSizeMB}MB extracted on disk", "{$ufdbguard_admin_memory}" . @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
        // ufdbguard_admin_events("UFDB::Success update $c categories $BigSize extracted\n$ufdbguard_admin_memory",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica");
        @file_put_contents($UFDBGUARD_LAST_INDEX_TIME, $source_filetime);
        shell_exec("/etc/init.d/ufdbcat reload");
        META_MASTER_UFDBTABLES(true);
        calculate_categorized_websites(true);
    } else {
        if ($GLOBALS["FORCE"]) {
            echo "No update available\n{$ufdbguard_admin_memory}\n";
        }
    }
    @chown("{$WORKDIR}", "squid");
    @chgrp("{$WORKDIR}", "squid");
    updatev2_adblock();
    scan_artica_databases();
    updatev2_progress(100, "{done} [" . __LINE__ . "]");
    return true;
}
Ejemplo n.º 2
0
}
if ($argv[1] == "--ufdb") {
    ufdbtables();
    die;
}
if ($argv[1] == "--ufdbsum") {
    calculate_categorized_websites();
    die;
}
if ($argv[1] == "--ufdbmaster") {
    $GLOBALS["FORCE"] = true;
    updatev2();
    if ($GLOBALS["VERBOSE"]) {
        echo " **************** META_MASTER_UFDBTABLES ***************\n";
    }
    META_MASTER_UFDBTABLES();
    die;
}
WriteMyLogs("Executed: {$GLOBALS["CMDLINE"]} task:{$GLOBALS["SCHEDULE_ID"]}", __FUNCTION__, __FILE__, __LINE__);
$sock = new sockets();
$EnableRemoteStatisticsAppliance = $sock->GET_INFO("EnableRemoteStatisticsAppliance");
if (!is_numeric($EnableRemoteStatisticsAppliance)) {
    $EnableRemoteStatisticsAppliance = 0;
}
$DisableCategoriesDatabasesUpdates = intval($sock->GET_INFO("DisableCategoriesDatabasesUpdates"));
if ($DisableCategoriesDatabasesUpdates == 1) {
    ufdbevents("DisableCategoriesDatabasesUpdates IS ACTIVE ,ABORTING TASK", __FUNCTION__, __FILE__, __LINE__);
    die;
}
if ($EnableRemoteStatisticsAppliance == 1) {
    ufdbevents("EnableRemoteStatisticsAppliance IS ACTIVE ,ABORTING TASK", __FUNCTION__, __FILE__, __LINE__);