function ufdbtables($nopid = false)
{
$unix = new unix();
$sock = new sockets();
if (!$nopid) {
$pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid";
$nohup = $unix->find_program("nohup");
$pid = @file_get_contents($pidfile);
if ($unix->process_exists($pid, __FILE__)) {
$timepid = $unix->PROCCESS_TIME_MIN($pid);
updatev2_progress(110, "Error, {$timepid} already running [" . __LINE__ . "]");
if (!$GLOBALS["NOLOGS"]) {
// ufdbguard_admin_events("UFDB::Warning: Task already executed PID: $pid since {$timepid}Mn",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica");
return;
}
}
@file_put_contents($pidfile, getmypid());
}
$GLOBALS["EVENTS"] = array();
$CACHE_FILE = "/etc/artica-postfix/ufdb.tables.db";
updatev2_progress(10, "CACHE_FILE = {$CACHE_FILE} [" . __LINE__ . "]");
$sock = new sockets();
$EnableArticaMetaClient = intval($sock->GET_INFO("EnableArticaMetaClient"));
if ($EnableArticaMetaClient == 1) {
updatev2_progress(10, "Using Artica Meta console [" . __LINE__ . "]");
return ufdbtables_artica_meta();
}
$UseRemoteUfdbguardService = $sock->GET_INFO('UseRemoteUfdbguardService');
if (!is_numeric($UseRemoteUfdbguardService)) {
$UseRemoteUfdbguardService = 0;
}
if ($UseRemoteUfdbguardService == 1) {
updatev2_progress(10, "UseRemoteUfdbguardService = TRUE - aborting [" . __LINE__ . "]");
return;
}
if ($GLOBALS["MIRROR"] == null) {
updatev2_progress(10, "MIRROR is null #2 [" . __LINE__ . "]");
unset($GLOBALS["updatev2_checkversion"]);
updatev2_checkversion();
}
if ($GLOBALS["MIRROR"] == null) {
updatev2_progress(110, "MIRROR is null #1 [" . __LINE__ . "]");
return;
}
$tmpdir = $unix->TEMP_DIR();
$URIBASE = $GLOBALS["MIRROR"];
$WORKDIR = $GLOBALS["WORKDIR_LOCAL"];
if (is_link($WORKDIR)) {
$WORKDIR = readlink($WORKDIR);
}
$CategoriesDatabasesByCron = $sock->GET_INFO("CategoriesDatabaseByCron");
if (!is_numeric($CategoriesDatabasesByCron)) {
$CategoriesDatabasesByCron = 1;
}
if (!$GLOBALS["FORCE"]) {
if ($CategoriesDatabasesByCron == 1) {
if (!$GLOBALS["BYCRON"]) {
updatev2_progress(110, "{done} CategoriesDatabasesByCron [" . __LINE__ . "]");
return;
}
}
}
updatev2_progress(15, "{$URIBASE}/index.txt [" . __LINE__ . "]");
$curl = new ccurl("{$URIBASE}/index.txt");
$STATUS = unserialize(@file_get_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD"));
$STATUS["LAST_DOWNLOAD"]["LAST_CHECK"] = time();
@file_put_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD", serialize($STATUS));
if (!$curl->GetHeads()) {
if ($GLOBALS["VERBOSE"]) {
echo "Fatal ! {$URIBASE}/index.txt ERROR NUMBER {$curl->CURLINFO_HTTP_CODE}\n";
}
if ($curl->CURLINFO_HTTP_CODE == 404 or $curl->CURLINFO_HTTP_CODE == 300) {
if (!preg_match("#\\/ufdb#", $URIBASE)) {
$URIBASE = "{$URIBASE}/ufdb";
}
$curl = new ccurl("{$URIBASE}/index.txt");
if (!$curl->GetHeads()) {
$GLOBALS["EVENTS"][] = "{$URIBASE}/index.txt";
$GLOBALS["EVENTS"][] = "Failed with error {$curl->error}";
while (list($a, $b) = each($GLOBALS["CURLDEBUG"])) {
$GLOBALS["EVENTS"][] = $b;
}
squid_admin_mysql(0, "Unable to download blacklist index file with error: `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
artica_update_event(0, "Unable to download Artica blacklist index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
// ufdbguard_admin_events("UFDB::Fatal: Unable to download blacklist index file $curl->error",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica");
echo "UFDB: Failed to retreive {$URIBASE}/index.txt ({$curl->error})\n";
updatev2_progress(110, "Failed to retreive {$URIBASE}/index.txt [" . __LINE__ . "]");
updatev2_adblock();
return;
}
}
}
updatev2_progress(20, "Downloading MD5Strings.txt [" . __LINE__ . "]");
$curl = new ccurl("{$URIBASE}/MD5Strings.txt");
if (!$curl->GetFile("{$tmpdir}/MD5Strings.txt")) {
artica_update_event(2, "{$URIBASE}: Unable to download Artica blacklist MD5 table `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
@unlink("{$tmpdir}/MD5Strings.txt");
}
$MD5_strings = unserialize(@file_get_contents("{$tmpdir}/MD5Strings.txt"));
$source_filetime = $curl->CURL_ALL_INFOS["filetime"];
if ($GLOBALS["VERBOSE"]) {
echo "{$URIBASE}/index.txt filetime: {$source_filetime} " . date("Y-m-d H:i:s", $source_filetime) . "\n";
}
$GLOBALS["EVENTS"][] = "{$URIBASE}/index.txt";
$GLOBALS["EVENTS"][] = "filetime: {$source_filetime} " . date("Y-m-d H:i:s", $source_filetime);
$UFDBGUARD_LAST_INDEX_TIME = "/etc/artica-postfix/UFDBGUARD_LAST_INDEX_TIME";
$old_time = intval(@file_get_contents("{$UFDBGUARD_LAST_INDEX_TIME}"));
$GLOBALS["EVENTS"][] = "Old filetime: {$old_time} " . date("Y-m-d H:i:s", $old_time);
if (is_file("/etc/artica-postfix/ufdbcounts.txt")) {
if ($source_filetime == $old_time) {
ufdbtables_md5($MD5_strings, $URIBASE);
$GLOBALS["EVENTS"][] = "No new updates";
updatev2_progress(100, "{up-to-date} [" . __LINE__ . "]");
return true;
}
if ($source_filetime < $old_time) {
ufdbtables_md5($MD5_strings, $URIBASE);
$GLOBALS["EVENTS"][] = "No new updates";
updatev2_progress(100, "{up-to-date} [" . __LINE__ . "]");
return true;
}
}
updatev2_progress(99, "{$URIBASE}/ufdbcounts.txt [" . __LINE__ . "]");
$curl = new ccurl("{$URIBASE}/ufdbcounts.txt");
if (!$curl->GetFile("/etc/artica-postfix/ufdbcounts.txt")) {
$GLOBALS["EVENTS"][] = "{$URIBASE}/ufdbcounts.txt";
$GLOBALS["EVENTS"][] = "Failed with error {$curl->error}";
while (list($a, $b) = each($GLOBALS["CURLDEBUG"])) {
$GLOBALS["EVENTS"][] = $b;
}
squid_admin_mysql(0, "Unable to download {$URIBASE}/ufdbcounts.txt index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
artica_update_event(0, "Unable to download Artica {$URIBASE}/ufdbcounts.txt index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
// ufdbguard_admin_events("UFDB::Fatal: Unable to download $URIBASE/ufdbcounts.txt index file $curl->error",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica");
}
updatev2_progress(99, "{$URIBASE}/index.txt [" . __LINE__ . "]");
$curl = new ccurl("{$URIBASE}/index.txt");
if (!$curl->GetFile("/etc/artica-postfix/artica-webfilter-db-index.txt")) {
$GLOBALS["EVENTS"][] = "{$URIBASE}/index.txt";
$GLOBALS["EVENTS"][] = "Failed with error {$curl->error}";
while (list($a, $b) = each($GLOBALS["CURLDEBUG"])) {
$GLOBALS["EVENTS"][] = $b;
}
squid_admin_mysql(0, "Unable to download blacklist index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
artica_update_event(0, "Unable to download Artica blacklist index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
// ufdbguard_admin_events("UFDB::Fatal: Unable to download blacklist index file $curl->error",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica");
echo "UFDB: Failed to retreive {$URIBASE}/index.txt ({$curl->error})\n";
updatev2_progress2(110, "Unable to download blacklist index file");
updatev2_adblock();
return;
}
if (ufdbtables_md5($MD5_strings, $URIBASE)) {
updatev2_progress(100, "{done} [" . __LINE__ . "]");
return;
}
$LOCAL_CACHE = unserialize(base64_decode(@file_get_contents($CACHE_FILE)));
$REMOTE_CACHE = unserialize(base64_decode(@file_get_contents("/etc/artica-postfix/artica-webfilter-db-index.txt")));
$CALCULATED_SIZE = 0;
$MAx = count($REMOTE_CACHE);
$BigSize = 0;
$c = 0;
$ERRORDB = 0;
while (list($tablename, $size) = each($REMOTE_CACHE)) {
$OriginalSize = $size;
if (blacklisted_tables($tablename)) {
continue;
}
$STATUS["LAST_DOWNLOAD"]["CATEGORY"] = $tablename;
if ($size != $LOCAL_CACHE[$tablename]) {
ufdbevents("{$tablename} {$size} <> {$LOCAL_CACHE[$tablename]}");
$c++;
if (!ufdbtables_DownloadInstall($URIBASE, $tablename, $size, null)) {
$ERRORDB++;
$STATUS["LAST_DOWNLOAD"]["TIME"] = time();
$STATUS["LAST_DOWNLOAD"]["SIZE"] = $GLOBALS["CURL_LAST_SIZE_DOWNLOAD"] / 1024;
$STATUS["LAST_DOWNLOAD"]["FAILED"] = $ERRORDB;
@file_put_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD", serialize($STATUS));
continue;
}
$prc = $c / $MAx * 100;
updatev2_progress2($prc, "{$tablename} ok");
$GLOBALS["UFDB_SIZE"] = $GLOBALS["CALCULATED_SIZE"];
$size = $unix->file_size("{$WORKDIR}/{$tablename}/domains.ufdb");
$size = round($size / 1024, 2);
$BigSize = $BigSize + $size;
@chown("{$WORKDIR}/{$tablename}", "squid");
@chgrp("{$WORKDIR}/{$tablename}", "squid");
$LOCAL_CACHE[$tablename] = $OriginalSize;
$GLOBALS["EVENTS"][] = "Success updating category `{$tablename}` with {$size} Ko";
$STATUS["LAST_DOWNLOAD"]["TIME"] = time();
$STATUS["LAST_DOWNLOAD"]["SIZE"] = $GLOBALS["CURL_LAST_SIZE_DOWNLOAD"] / 1024;
$STATUS["LAST_DOWNLOAD"]["FAILED"] = $ERRORDB;
@file_put_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD", serialize($STATUS));
}
}
@file_put_contents($CACHE_FILE, base64_encode(serialize($LOCAL_CACHE)));
updatev2_progress2(100, "DONE ok");
$ufdbguard_admin_memory = @implode("\n", $GLOBALS["ufdbguard_admin_memory"]);
$php5 = $unix->LOCATE_PHP5_BIN();
if (!is_file("/opt/ufdbcat/bin/ufdbcatdd")) {
system("{$php5} /usr/share/artica-postfix/exec.ufdbcat.php --install --noupdate");
}
if ($c > 0) {
$BigSizeMB = round($BigSize / 1024, 2);
shell_exec("/etc/init.d/ufdbcat reload");
squid_admin_mysql(2, "Artica Web filtering Databases Success updated {$c} categories {$BigSizeMB}MB extracted on disk", "{$ufdbguard_admin_memory}" . @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
artica_update_event(2, "Artica Web filtering Databases Success updated {$c} categories {$BigSizeMB}MB extracted on disk", "{$ufdbguard_admin_memory}" . @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
// ufdbguard_admin_events("UFDB::Success update $c categories $BigSize extracted\n$ufdbguard_admin_memory",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica");
@file_put_contents($UFDBGUARD_LAST_INDEX_TIME, $source_filetime);
shell_exec("/etc/init.d/ufdbcat reload");
META_MASTER_UFDBTABLES(true);
calculate_categorized_websites(true);
} else {
if ($GLOBALS["FORCE"]) {
echo "No update available\n{$ufdbguard_admin_memory}\n";
}
}
@chown("{$WORKDIR}", "squid");
@chgrp("{$WORKDIR}", "squid");
updatev2_adblock();
scan_artica_databases();
updatev2_progress(100, "{done} [" . __LINE__ . "]");
return true;
}
}
if ($argv[1] == "--ufdb") {
ufdbtables();
die;
}
if ($argv[1] == "--ufdbsum") {
calculate_categorized_websites();
die;
}
if ($argv[1] == "--ufdbmaster") {
$GLOBALS["FORCE"] = true;
updatev2();
if ($GLOBALS["VERBOSE"]) {
echo " **************** META_MASTER_UFDBTABLES ***************\n";
}
META_MASTER_UFDBTABLES();
die;
}
WriteMyLogs("Executed: {$GLOBALS["CMDLINE"]} task:{$GLOBALS["SCHEDULE_ID"]}", __FUNCTION__, __FILE__, __LINE__);
$sock = new sockets();
$EnableRemoteStatisticsAppliance = $sock->GET_INFO("EnableRemoteStatisticsAppliance");
if (!is_numeric($EnableRemoteStatisticsAppliance)) {
$EnableRemoteStatisticsAppliance = 0;
}
$DisableCategoriesDatabasesUpdates = intval($sock->GET_INFO("DisableCategoriesDatabasesUpdates"));
if ($DisableCategoriesDatabasesUpdates == 1) {
ufdbevents("DisableCategoriesDatabasesUpdates IS ACTIVE ,ABORTING TASK", __FUNCTION__, __FILE__, __LINE__);
die;
}
if ($EnableRemoteStatisticsAppliance == 1) {
ufdbevents("EnableRemoteStatisticsAppliance IS ACTIVE ,ABORTING TASK", __FUNCTION__, __FILE__, __LINE__);