function ufdbtables($nopid = false) { $unix = new unix(); $sock = new sockets(); if (!$nopid) { $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid"; $nohup = $unix->find_program("nohup"); $pid = @file_get_contents($pidfile); if ($unix->process_exists($pid, __FILE__)) { $timepid = $unix->PROCCESS_TIME_MIN($pid); updatev2_progress(110, "Error, {$timepid} already running [" . __LINE__ . "]"); if (!$GLOBALS["NOLOGS"]) { // ufdbguard_admin_events("UFDB::Warning: Task already executed PID: $pid since {$timepid}Mn",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica"); return; } } @file_put_contents($pidfile, getmypid()); } $GLOBALS["EVENTS"] = array(); $CACHE_FILE = "/etc/artica-postfix/ufdb.tables.db"; updatev2_progress(10, "CACHE_FILE = {$CACHE_FILE} [" . __LINE__ . "]"); $sock = new sockets(); $EnableArticaMetaClient = intval($sock->GET_INFO("EnableArticaMetaClient")); if ($EnableArticaMetaClient == 1) { updatev2_progress(10, "Using Artica Meta console [" . __LINE__ . "]"); return ufdbtables_artica_meta(); } $UseRemoteUfdbguardService = $sock->GET_INFO('UseRemoteUfdbguardService'); if (!is_numeric($UseRemoteUfdbguardService)) { $UseRemoteUfdbguardService = 0; } if ($UseRemoteUfdbguardService == 1) { updatev2_progress(10, "UseRemoteUfdbguardService = TRUE - aborting [" . __LINE__ . "]"); return; } if ($GLOBALS["MIRROR"] == null) { updatev2_progress(10, "MIRROR is null #2 [" . __LINE__ . "]"); unset($GLOBALS["updatev2_checkversion"]); updatev2_checkversion(); } if ($GLOBALS["MIRROR"] == null) { updatev2_progress(110, "MIRROR is null #1 [" . __LINE__ . "]"); return; } $tmpdir = $unix->TEMP_DIR(); $URIBASE = $GLOBALS["MIRROR"]; $WORKDIR = $GLOBALS["WORKDIR_LOCAL"]; if (is_link($WORKDIR)) { $WORKDIR = readlink($WORKDIR); } $CategoriesDatabasesByCron = $sock->GET_INFO("CategoriesDatabaseByCron"); if (!is_numeric($CategoriesDatabasesByCron)) { $CategoriesDatabasesByCron = 1; } if (!$GLOBALS["FORCE"]) { if ($CategoriesDatabasesByCron == 1) { if (!$GLOBALS["BYCRON"]) { updatev2_progress(110, "{done} CategoriesDatabasesByCron [" . __LINE__ . "]"); return; } } } updatev2_progress(15, "{$URIBASE}/index.txt [" . __LINE__ . "]"); $curl = new ccurl("{$URIBASE}/index.txt"); $STATUS = unserialize(@file_get_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD")); $STATUS["LAST_DOWNLOAD"]["LAST_CHECK"] = time(); @file_put_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD", serialize($STATUS)); if (!$curl->GetHeads()) { if ($GLOBALS["VERBOSE"]) { echo "Fatal ! {$URIBASE}/index.txt ERROR NUMBER {$curl->CURLINFO_HTTP_CODE}\n"; } if ($curl->CURLINFO_HTTP_CODE == 404 or $curl->CURLINFO_HTTP_CODE == 300) { if (!preg_match("#\\/ufdb#", $URIBASE)) { $URIBASE = "{$URIBASE}/ufdb"; } $curl = new ccurl("{$URIBASE}/index.txt"); if (!$curl->GetHeads()) { $GLOBALS["EVENTS"][] = "{$URIBASE}/index.txt"; $GLOBALS["EVENTS"][] = "Failed with error {$curl->error}"; while (list($a, $b) = each($GLOBALS["CURLDEBUG"])) { $GLOBALS["EVENTS"][] = $b; } squid_admin_mysql(0, "Unable to download blacklist index file with error: `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); artica_update_event(0, "Unable to download Artica blacklist index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); // ufdbguard_admin_events("UFDB::Fatal: Unable to download blacklist index file $curl->error",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica"); echo "UFDB: Failed to retreive {$URIBASE}/index.txt ({$curl->error})\n"; updatev2_progress(110, "Failed to retreive {$URIBASE}/index.txt [" . __LINE__ . "]"); updatev2_adblock(); return; } } } updatev2_progress(20, "Downloading MD5Strings.txt [" . __LINE__ . "]"); $curl = new ccurl("{$URIBASE}/MD5Strings.txt"); if (!$curl->GetFile("{$tmpdir}/MD5Strings.txt")) { artica_update_event(2, "{$URIBASE}: Unable to download Artica blacklist MD5 table `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); @unlink("{$tmpdir}/MD5Strings.txt"); } $MD5_strings = unserialize(@file_get_contents("{$tmpdir}/MD5Strings.txt")); $source_filetime = $curl->CURL_ALL_INFOS["filetime"]; if ($GLOBALS["VERBOSE"]) { echo "{$URIBASE}/index.txt filetime: {$source_filetime} " . date("Y-m-d H:i:s", $source_filetime) . "\n"; } $GLOBALS["EVENTS"][] = "{$URIBASE}/index.txt"; $GLOBALS["EVENTS"][] = "filetime: {$source_filetime} " . date("Y-m-d H:i:s", $source_filetime); $UFDBGUARD_LAST_INDEX_TIME = "/etc/artica-postfix/UFDBGUARD_LAST_INDEX_TIME"; $old_time = intval(@file_get_contents("{$UFDBGUARD_LAST_INDEX_TIME}")); $GLOBALS["EVENTS"][] = "Old filetime: {$old_time} " . date("Y-m-d H:i:s", $old_time); if (is_file("/etc/artica-postfix/ufdbcounts.txt")) { if ($source_filetime == $old_time) { ufdbtables_md5($MD5_strings, $URIBASE); $GLOBALS["EVENTS"][] = "No new updates"; updatev2_progress(100, "{up-to-date} [" . __LINE__ . "]"); return true; } if ($source_filetime < $old_time) { ufdbtables_md5($MD5_strings, $URIBASE); $GLOBALS["EVENTS"][] = "No new updates"; updatev2_progress(100, "{up-to-date} [" . __LINE__ . "]"); return true; } } updatev2_progress(99, "{$URIBASE}/ufdbcounts.txt [" . __LINE__ . "]"); $curl = new ccurl("{$URIBASE}/ufdbcounts.txt"); if (!$curl->GetFile("/etc/artica-postfix/ufdbcounts.txt")) { $GLOBALS["EVENTS"][] = "{$URIBASE}/ufdbcounts.txt"; $GLOBALS["EVENTS"][] = "Failed with error {$curl->error}"; while (list($a, $b) = each($GLOBALS["CURLDEBUG"])) { $GLOBALS["EVENTS"][] = $b; } squid_admin_mysql(0, "Unable to download {$URIBASE}/ufdbcounts.txt index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); artica_update_event(0, "Unable to download Artica {$URIBASE}/ufdbcounts.txt index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); // ufdbguard_admin_events("UFDB::Fatal: Unable to download $URIBASE/ufdbcounts.txt index file $curl->error",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica"); } updatev2_progress(99, "{$URIBASE}/index.txt [" . __LINE__ . "]"); $curl = new ccurl("{$URIBASE}/index.txt"); if (!$curl->GetFile("/etc/artica-postfix/artica-webfilter-db-index.txt")) { $GLOBALS["EVENTS"][] = "{$URIBASE}/index.txt"; $GLOBALS["EVENTS"][] = "Failed with error {$curl->error}"; while (list($a, $b) = each($GLOBALS["CURLDEBUG"])) { $GLOBALS["EVENTS"][] = $b; } squid_admin_mysql(0, "Unable to download blacklist index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); artica_update_event(0, "Unable to download Artica blacklist index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); // ufdbguard_admin_events("UFDB::Fatal: Unable to download blacklist index file $curl->error",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica"); echo "UFDB: Failed to retreive {$URIBASE}/index.txt ({$curl->error})\n"; updatev2_progress2(110, "Unable to download blacklist index file"); updatev2_adblock(); return; } if (ufdbtables_md5($MD5_strings, $URIBASE)) { updatev2_progress(100, "{done} [" . __LINE__ . "]"); return; } $LOCAL_CACHE = unserialize(base64_decode(@file_get_contents($CACHE_FILE))); $REMOTE_CACHE = unserialize(base64_decode(@file_get_contents("/etc/artica-postfix/artica-webfilter-db-index.txt"))); $CALCULATED_SIZE = 0; $MAx = count($REMOTE_CACHE); $BigSize = 0; $c = 0; $ERRORDB = 0; while (list($tablename, $size) = each($REMOTE_CACHE)) { $OriginalSize = $size; if (blacklisted_tables($tablename)) { continue; } $STATUS["LAST_DOWNLOAD"]["CATEGORY"] = $tablename; if ($size != $LOCAL_CACHE[$tablename]) { ufdbevents("{$tablename} {$size} <> {$LOCAL_CACHE[$tablename]}"); $c++; if (!ufdbtables_DownloadInstall($URIBASE, $tablename, $size, null)) { $ERRORDB++; $STATUS["LAST_DOWNLOAD"]["TIME"] = time(); $STATUS["LAST_DOWNLOAD"]["SIZE"] = $GLOBALS["CURL_LAST_SIZE_DOWNLOAD"] / 1024; $STATUS["LAST_DOWNLOAD"]["FAILED"] = $ERRORDB; @file_put_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD", serialize($STATUS)); continue; } $prc = $c / $MAx * 100; updatev2_progress2($prc, "{$tablename} ok"); $GLOBALS["UFDB_SIZE"] = $GLOBALS["CALCULATED_SIZE"]; $size = $unix->file_size("{$WORKDIR}/{$tablename}/domains.ufdb"); $size = round($size / 1024, 2); $BigSize = $BigSize + $size; @chown("{$WORKDIR}/{$tablename}", "squid"); @chgrp("{$WORKDIR}/{$tablename}", "squid"); $LOCAL_CACHE[$tablename] = $OriginalSize; $GLOBALS["EVENTS"][] = "Success updating category `{$tablename}` with {$size} Ko"; $STATUS["LAST_DOWNLOAD"]["TIME"] = time(); $STATUS["LAST_DOWNLOAD"]["SIZE"] = $GLOBALS["CURL_LAST_SIZE_DOWNLOAD"] / 1024; $STATUS["LAST_DOWNLOAD"]["FAILED"] = $ERRORDB; @file_put_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD", serialize($STATUS)); } } @file_put_contents($CACHE_FILE, base64_encode(serialize($LOCAL_CACHE))); updatev2_progress2(100, "DONE ok"); $ufdbguard_admin_memory = @implode("\n", $GLOBALS["ufdbguard_admin_memory"]); $php5 = $unix->LOCATE_PHP5_BIN(); if (!is_file("/opt/ufdbcat/bin/ufdbcatdd")) { system("{$php5} /usr/share/artica-postfix/exec.ufdbcat.php --install --noupdate"); } if ($c > 0) { $BigSizeMB = round($BigSize / 1024, 2); shell_exec("/etc/init.d/ufdbcat reload"); squid_admin_mysql(2, "Artica Web filtering Databases Success updated {$c} categories {$BigSizeMB}MB extracted on disk", "{$ufdbguard_admin_memory}" . @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); artica_update_event(2, "Artica Web filtering Databases Success updated {$c} categories {$BigSizeMB}MB extracted on disk", "{$ufdbguard_admin_memory}" . @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); // ufdbguard_admin_events("UFDB::Success update $c categories $BigSize extracted\n$ufdbguard_admin_memory",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica"); @file_put_contents($UFDBGUARD_LAST_INDEX_TIME, $source_filetime); shell_exec("/etc/init.d/ufdbcat reload"); META_MASTER_UFDBTABLES(true); calculate_categorized_websites(true); } else { if ($GLOBALS["FORCE"]) { echo "No update available\n{$ufdbguard_admin_memory}\n"; } } @chown("{$WORKDIR}", "squid"); @chgrp("{$WORKDIR}", "squid"); updatev2_adblock(); scan_artica_databases(); updatev2_progress(100, "{done} [" . __LINE__ . "]"); return true; }
function updatev2() { $sock = new sockets(); $unix = new unix(); $GLOBALS["TEMP_PATH"] = $unix->TEMP_DIR(); updatev2_progress(10, "{checking} [" . __LINE__ . "]"); $timeFile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".time"; $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid"; $ArticaDbReplicate = $sock->GET_INFO("ArticaDbReplicate"); $CategoriesDatabasesByCron = $sock->GET_INFO("CategoriesDatabaseByCron"); if (!is_numeric($CategoriesDatabasesByCron)) { $CategoriesDatabasesByCron = 1; } $DisableArticaProxyStatistics = $sock->GET_INFO("DisableArticaProxyStatistics"); $CategoriesDatabasesUpdatesAllTimes = intval($sock->GET_INFO("CategoriesDatabasesUpdatesAllTimes")); $ArticaDBPath = $sock->GET_INFO("ArticaDBPath"); if ($ArticaDBPath == null) { $ArticaDBPath = "/opt/articatech"; } $ManualArticaDBPath = $sock->GET_INFO("ManualArticaDBPath"); if ($ManualArticaDBPath == null) { $ManualArticaDBPath = "/home/manualupdate/articadb.tar.gz"; } $ManualArticaDBPathNAS = $sock->GET_INFO("ManualArticaDBPathNAS"); $EnableArticaMetaServer = intval($sock->GET_INFO("EnableArticaMetaServer")); $datas = unserialize(base64_decode($sock->GET_INFO("ufdbguardConfig"))); updatev2_progress(10, "{checking} [" . __LINE__ . "]"); if (!is_numeric($ManualArticaDBPathNAS)) { $ManualArticaDBPathNAS = 0; } if (!is_numeric($DisableArticaProxyStatistics)) { $DisableArticaProxyStatistics = 0; } if (!is_numeric($ArticaDbReplicate)) { $ArticaDbReplicate = 0; } $WizardStatsAppliance = unserialize(base64_decode($sock->GET_INFO("WizardStatsAppliance"))); if (!isset($WizardStatsAppliance["SERVER"])) { $WizardStatsAppliance["SERVER"] = null; } if ($EnableArticaMetaServer == 0) { if ($DisableArticaProxyStatistics == 1) { updatev2_progress(110, "Error: Artica statistics are disabled"); } } if ($EnableArticaMetaServer == 0) { if ($datas["UseRemoteUfdbguardService"] == 1) { updatev2_progress(110, "Error: - UseRemoteUfdbguardService - Only used by {$WizardStatsAppliance["SERVER"]}"); return; } } $CHECKTIME = $unix->file_time_min($timeFile); ufdbevents(" **"); ufdbevents(" **"); ufdbevents("{$timeFile} = {$CHECKTIME}Mn"); ufdbevents(" **"); ufdbevents(" **"); if (!$GLOBALS["FORCE"]) { if ($CategoriesDatabasesByCron == 1) { if ($EnableArticaMetaServer == 0) { if (!$GLOBALS["BYCRON"]) { updatev2_progress(110, "Error: Only executed by schedule [" . __LINE__ . "]"); if ($CHECKTIME > 60) { updatev2_checkversions(); } return; } } } if ($CategoriesDatabasesUpdatesAllTimes == 0) { if ($EnableArticaMetaServer == 0) { if ($unix->IsProductionTime()) { webupdate_admin_mysql(2, "Update aborted, only allowed outside the production time", null, __FILE__, __LINE__); updatev2_progress(110, "Error: Only outside production time"); if ($CHECKTIME > 60) { updatev2_checkversions(); } return; } } } } if ($GLOBALS["FORCE"]) { ufdbevents("***** Force enabled ***** "); ufdbevents("*****"); ufdbevents("*****"); ufdbevents("Executed as {$GLOBALS["CMDLINE"]}"); ufdbevents("*****"); ufdbevents("*****"); } $MaxCheckTime = 240; if ($EnableArticaMetaServer == 1) { $MaxCheckTime = 60; } if (!$GLOBALS["FORCE"]) { if ($CHECKTIME < 240) { updatev2_progress(110, "STOP: current {$CHECKTIME}Mn, require {$MaxCheckTime}mn"); if ($CHECKTIME > 60) { updatev2_checkversions(); } return; } } updatev2_progress(10, "{checking} [" . __LINE__ . "]"); $pid = @file_get_contents($pidfile); if ($unix->process_exists($pid, __FILE__)) { $time = $unix->PROCCESS_TIME_MIN($pid); if ($time < 10200) { updatev2_progress(110, "Error: already running pid {$pid} since {$time}Mn"); return; } else { $kill = $unix->find_program("kill"); unix_system_kill_force($pid); if ($GLOBALS["SCHEDULE_ID"] > 0) { artica_update_event(1, "Warning: Old task pid {$pid} since {$time}Mn has been killed, (reach 7200mn)", null, __FILE__, __LINE__); } } } updatev2_progress(10, "{checking} [" . __LINE__ . "]"); ufdbevents("Stamp {$timeFile}"); @unlink($timeFile); $tlse_force_token = null; @file_put_contents($timeFile, time()); @file_put_contents($pidfile, getmypid()); $tlse_token = null; if ($GLOBALS["BYCRON"]) { $tlse_token == " --bycron --force"; } if ($GLOBALS["FORCE"]) { $tlse_force_token = " --force"; } $php = $unix->LOCATE_PHP5_BIN(); $nohup = $unix->find_program("nohup"); $PrivoxyEnabled = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/PrivoxyEnabled")); if ($PrivoxyEnabled == 1) { @chmod("/usr/share/artica-postfix/bin/privoxy-blocklist.sh", 0755); system("{$nohup} /usr/share/artica-postfix/bin/privoxy-blocklist.sh >/dev/null 2>&1 &"); } system("{$nohup} {$php} /usr/share/artica-postfix/exec.infected.urls.php >/dev/null 2>&1 &"); $EnableArticaMetaClient = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/EnableArticaMetaClient")); $EnableArticaMetaServer = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/EnableArticaMetaServer")); if ($EnableArticaMetaServer == 1) { $EnableArticaMetaClient = 0; } if ($EnableArticaMetaClient == 1) { updatev2_progress(10, "Using Meta console [" . __LINE__ . "]"); return ufdbtables_artica_meta(); } updatev2_progress(10, "{checking} [" . __LINE__ . "]"); updatev2_checkversions(); updatev2_progress(12, "{running} [" . __LINE__ . "]"); $GLOBALS["DOWNLOADED_INSTALLED"] = 0; ufdbtables(true); tlsetables(true); ufdb_phistank(); TranslateToMetaServer(); if ($GLOBALS["DOWNLOADED_INSTALLED"] > 0) { updatev2_progress(96, "{restarting_webfiltering_service} [" . __LINE__ . "]"); squid_admin_mysql(2, "{$GLOBALS["DOWNLOADED_INSTALLED"]} updated blacklists databases [action=restart]", __FILE__, __LINE__); system("/etc/init.d/ufdb restart --updater"); system("/etc/init.d/ufdbcat restart --updater"); $squidbin = $unix->LOCATE_SQUID_BIN(); squid_admin_mysql(1, "Reloading proxy service after updating Web filtering databases", __FILE__, __LINE__); system("{$squidbin} -f /etc/squid3/squid.conf -k reconfigure"); } if ($GLOBALS["VERBOSE"]) { echo " **************** C_ICAP_TABLES ***************\n"; } C_ICAP_TABLES(true); if ($GLOBALS["VERBOSE"]) { echo " **************** schedulemaintenance ***************\n"; } schedulemaintenance(); if ($GLOBALS["VERBOSE"]) { echo " **************** EXECUTE_BLACK_INSTANCE ***************\n"; } EXECUTE_BLACK_INSTANCE(); if ($GLOBALS["VERBOSE"]) { echo " **************** FINISH ***************\n"; } updatev2_progress(100, "{done}"); }