function HyperCacheGetMimeType($uri, $ID) { $curl = new ccurl($uri); if (!$curl->GetHeads()) { events("headers {failed} {$uri} Error number: {$curl->CURLINFO_HTTP_CODE}", $ID, 1, __LINE__); return "NONE_{$curl->CURLINFO_HTTP_CODE}"; } if ($curl->CURLINFO_HTTP_CODE == 501) { return "NONE_501"; } $content_type = $curl->CURL_ALL_INFOS["content_type"]; if (strpos($content_type, ";") > 0) { $tbl = explode(";", $content_type); $content_type = trim($tbl[0]); } return $content_type; }
function ufdbtables($nopid = false) { $unix = new unix(); $sock = new sockets(); if (!$nopid) { $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid"; $nohup = $unix->find_program("nohup"); $pid = @file_get_contents($pidfile); if ($unix->process_exists($pid, __FILE__)) { $timepid = $unix->PROCCESS_TIME_MIN($pid); updatev2_progress(110, "Error, {$timepid} already running [" . __LINE__ . "]"); if (!$GLOBALS["NOLOGS"]) { // ufdbguard_admin_events("UFDB::Warning: Task already executed PID: $pid since {$timepid}Mn",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica"); return; } } @file_put_contents($pidfile, getmypid()); } $GLOBALS["EVENTS"] = array(); $CACHE_FILE = "/etc/artica-postfix/ufdb.tables.db"; updatev2_progress(10, "CACHE_FILE = {$CACHE_FILE} [" . __LINE__ . "]"); $sock = new sockets(); $EnableArticaMetaClient = intval($sock->GET_INFO("EnableArticaMetaClient")); if ($EnableArticaMetaClient == 1) { updatev2_progress(10, "Using Artica Meta console [" . __LINE__ . "]"); return ufdbtables_artica_meta(); } $UseRemoteUfdbguardService = $sock->GET_INFO('UseRemoteUfdbguardService'); if (!is_numeric($UseRemoteUfdbguardService)) { $UseRemoteUfdbguardService = 0; } if ($UseRemoteUfdbguardService == 1) { updatev2_progress(10, "UseRemoteUfdbguardService = TRUE - aborting [" . __LINE__ . "]"); return; } if ($GLOBALS["MIRROR"] == null) { updatev2_progress(10, "MIRROR is null #2 [" . __LINE__ . "]"); unset($GLOBALS["updatev2_checkversion"]); updatev2_checkversion(); } if ($GLOBALS["MIRROR"] == null) { updatev2_progress(110, "MIRROR is null #1 [" . __LINE__ . "]"); return; } $tmpdir = $unix->TEMP_DIR(); $URIBASE = $GLOBALS["MIRROR"]; $WORKDIR = $GLOBALS["WORKDIR_LOCAL"]; if (is_link($WORKDIR)) { $WORKDIR = readlink($WORKDIR); } $CategoriesDatabasesByCron = $sock->GET_INFO("CategoriesDatabaseByCron"); if (!is_numeric($CategoriesDatabasesByCron)) { $CategoriesDatabasesByCron = 1; } if (!$GLOBALS["FORCE"]) { if ($CategoriesDatabasesByCron == 1) { if (!$GLOBALS["BYCRON"]) { updatev2_progress(110, "{done} CategoriesDatabasesByCron [" . __LINE__ . "]"); return; } } } updatev2_progress(15, "{$URIBASE}/index.txt [" . __LINE__ . "]"); $curl = new ccurl("{$URIBASE}/index.txt"); $STATUS = unserialize(@file_get_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD")); $STATUS["LAST_DOWNLOAD"]["LAST_CHECK"] = time(); @file_put_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD", serialize($STATUS)); if (!$curl->GetHeads()) { if ($GLOBALS["VERBOSE"]) { echo "Fatal ! {$URIBASE}/index.txt ERROR NUMBER {$curl->CURLINFO_HTTP_CODE}\n"; } if ($curl->CURLINFO_HTTP_CODE == 404 or $curl->CURLINFO_HTTP_CODE == 300) { if (!preg_match("#\\/ufdb#", $URIBASE)) { $URIBASE = "{$URIBASE}/ufdb"; } $curl = new ccurl("{$URIBASE}/index.txt"); if (!$curl->GetHeads()) { $GLOBALS["EVENTS"][] = "{$URIBASE}/index.txt"; $GLOBALS["EVENTS"][] = "Failed with error {$curl->error}"; while (list($a, $b) = each($GLOBALS["CURLDEBUG"])) { $GLOBALS["EVENTS"][] = $b; } squid_admin_mysql(0, "Unable to download blacklist index file with error: `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); artica_update_event(0, "Unable to download Artica blacklist index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); // ufdbguard_admin_events("UFDB::Fatal: Unable to download blacklist index file $curl->error",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica"); echo "UFDB: Failed to retreive {$URIBASE}/index.txt ({$curl->error})\n"; updatev2_progress(110, "Failed to retreive {$URIBASE}/index.txt [" . __LINE__ . "]"); updatev2_adblock(); return; } } } updatev2_progress(20, "Downloading MD5Strings.txt [" . __LINE__ . "]"); $curl = new ccurl("{$URIBASE}/MD5Strings.txt"); if (!$curl->GetFile("{$tmpdir}/MD5Strings.txt")) { artica_update_event(2, "{$URIBASE}: Unable to download Artica blacklist MD5 table `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); @unlink("{$tmpdir}/MD5Strings.txt"); } $MD5_strings = unserialize(@file_get_contents("{$tmpdir}/MD5Strings.txt")); $source_filetime = $curl->CURL_ALL_INFOS["filetime"]; if ($GLOBALS["VERBOSE"]) { echo "{$URIBASE}/index.txt filetime: {$source_filetime} " . date("Y-m-d H:i:s", $source_filetime) . "\n"; } $GLOBALS["EVENTS"][] = "{$URIBASE}/index.txt"; $GLOBALS["EVENTS"][] = "filetime: {$source_filetime} " . date("Y-m-d H:i:s", $source_filetime); $UFDBGUARD_LAST_INDEX_TIME = "/etc/artica-postfix/UFDBGUARD_LAST_INDEX_TIME"; $old_time = intval(@file_get_contents("{$UFDBGUARD_LAST_INDEX_TIME}")); $GLOBALS["EVENTS"][] = "Old filetime: {$old_time} " . date("Y-m-d H:i:s", $old_time); if (is_file("/etc/artica-postfix/ufdbcounts.txt")) { if ($source_filetime == $old_time) { ufdbtables_md5($MD5_strings, $URIBASE); $GLOBALS["EVENTS"][] = "No new updates"; updatev2_progress(100, "{up-to-date} [" . __LINE__ . "]"); return true; } if ($source_filetime < $old_time) { ufdbtables_md5($MD5_strings, $URIBASE); $GLOBALS["EVENTS"][] = "No new updates"; updatev2_progress(100, "{up-to-date} [" . __LINE__ . "]"); return true; } } updatev2_progress(99, "{$URIBASE}/ufdbcounts.txt [" . __LINE__ . "]"); $curl = new ccurl("{$URIBASE}/ufdbcounts.txt"); if (!$curl->GetFile("/etc/artica-postfix/ufdbcounts.txt")) { $GLOBALS["EVENTS"][] = "{$URIBASE}/ufdbcounts.txt"; $GLOBALS["EVENTS"][] = "Failed with error {$curl->error}"; while (list($a, $b) = each($GLOBALS["CURLDEBUG"])) { $GLOBALS["EVENTS"][] = $b; } squid_admin_mysql(0, "Unable to download {$URIBASE}/ufdbcounts.txt index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); artica_update_event(0, "Unable to download Artica {$URIBASE}/ufdbcounts.txt index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); // ufdbguard_admin_events("UFDB::Fatal: Unable to download $URIBASE/ufdbcounts.txt index file $curl->error",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica"); } updatev2_progress(99, "{$URIBASE}/index.txt [" . __LINE__ . "]"); $curl = new ccurl("{$URIBASE}/index.txt"); if (!$curl->GetFile("/etc/artica-postfix/artica-webfilter-db-index.txt")) { $GLOBALS["EVENTS"][] = "{$URIBASE}/index.txt"; $GLOBALS["EVENTS"][] = "Failed with error {$curl->error}"; while (list($a, $b) = each($GLOBALS["CURLDEBUG"])) { $GLOBALS["EVENTS"][] = $b; } squid_admin_mysql(0, "Unable to download blacklist index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); artica_update_event(0, "Unable to download Artica blacklist index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); // ufdbguard_admin_events("UFDB::Fatal: Unable to download blacklist index file $curl->error",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica"); echo "UFDB: Failed to retreive {$URIBASE}/index.txt ({$curl->error})\n"; updatev2_progress2(110, "Unable to download blacklist index file"); updatev2_adblock(); return; } if (ufdbtables_md5($MD5_strings, $URIBASE)) { updatev2_progress(100, "{done} [" . __LINE__ . "]"); return; } $LOCAL_CACHE = unserialize(base64_decode(@file_get_contents($CACHE_FILE))); $REMOTE_CACHE = unserialize(base64_decode(@file_get_contents("/etc/artica-postfix/artica-webfilter-db-index.txt"))); $CALCULATED_SIZE = 0; $MAx = count($REMOTE_CACHE); $BigSize = 0; $c = 0; $ERRORDB = 0; while (list($tablename, $size) = each($REMOTE_CACHE)) { $OriginalSize = $size; if (blacklisted_tables($tablename)) { continue; } $STATUS["LAST_DOWNLOAD"]["CATEGORY"] = $tablename; if ($size != $LOCAL_CACHE[$tablename]) { ufdbevents("{$tablename} {$size} <> {$LOCAL_CACHE[$tablename]}"); $c++; if (!ufdbtables_DownloadInstall($URIBASE, $tablename, $size, null)) { $ERRORDB++; $STATUS["LAST_DOWNLOAD"]["TIME"] = time(); $STATUS["LAST_DOWNLOAD"]["SIZE"] = $GLOBALS["CURL_LAST_SIZE_DOWNLOAD"] / 1024; $STATUS["LAST_DOWNLOAD"]["FAILED"] = $ERRORDB; @file_put_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD", serialize($STATUS)); continue; } $prc = $c / $MAx * 100; updatev2_progress2($prc, "{$tablename} ok"); $GLOBALS["UFDB_SIZE"] = $GLOBALS["CALCULATED_SIZE"]; $size = $unix->file_size("{$WORKDIR}/{$tablename}/domains.ufdb"); $size = round($size / 1024, 2); $BigSize = $BigSize + $size; @chown("{$WORKDIR}/{$tablename}", "squid"); @chgrp("{$WORKDIR}/{$tablename}", "squid"); $LOCAL_CACHE[$tablename] = $OriginalSize; $GLOBALS["EVENTS"][] = "Success updating category `{$tablename}` with {$size} Ko"; $STATUS["LAST_DOWNLOAD"]["TIME"] = time(); $STATUS["LAST_DOWNLOAD"]["SIZE"] = $GLOBALS["CURL_LAST_SIZE_DOWNLOAD"] / 1024; $STATUS["LAST_DOWNLOAD"]["FAILED"] = $ERRORDB; @file_put_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD", serialize($STATUS)); } } @file_put_contents($CACHE_FILE, base64_encode(serialize($LOCAL_CACHE))); updatev2_progress2(100, "DONE ok"); $ufdbguard_admin_memory = @implode("\n", $GLOBALS["ufdbguard_admin_memory"]); $php5 = $unix->LOCATE_PHP5_BIN(); if (!is_file("/opt/ufdbcat/bin/ufdbcatdd")) { system("{$php5} /usr/share/artica-postfix/exec.ufdbcat.php --install --noupdate"); } if ($c > 0) { $BigSizeMB = round($BigSize / 1024, 2); shell_exec("/etc/init.d/ufdbcat reload"); squid_admin_mysql(2, "Artica Web filtering Databases Success updated {$c} categories {$BigSizeMB}MB extracted on disk", "{$ufdbguard_admin_memory}" . @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); artica_update_event(2, "Artica Web filtering Databases Success updated {$c} categories {$BigSizeMB}MB extracted on disk", "{$ufdbguard_admin_memory}" . @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__); // ufdbguard_admin_events("UFDB::Success update $c categories $BigSize extracted\n$ufdbguard_admin_memory",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica"); @file_put_contents($UFDBGUARD_LAST_INDEX_TIME, $source_filetime); shell_exec("/etc/init.d/ufdbcat reload"); META_MASTER_UFDBTABLES(true); calculate_categorized_websites(true); } else { if ($GLOBALS["FORCE"]) { echo "No update available\n{$ufdbguard_admin_memory}\n"; } } @chown("{$WORKDIR}", "squid"); @chgrp("{$WORKDIR}", "squid"); updatev2_adblock(); scan_artica_databases(); updatev2_progress(100, "{done} [" . __LINE__ . "]"); return true; }