function ufdbtables($nopid = false)
{
$unix = new unix();
$sock = new sockets();
if (!$nopid) {
$pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid";
$nohup = $unix->find_program("nohup");
$pid = @file_get_contents($pidfile);
if ($unix->process_exists($pid, __FILE__)) {
$timepid = $unix->PROCCESS_TIME_MIN($pid);
updatev2_progress(110, "Error, {$timepid} already running [" . __LINE__ . "]");
if (!$GLOBALS["NOLOGS"]) {
// ufdbguard_admin_events("UFDB::Warning: Task already executed PID: $pid since {$timepid}Mn",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica");
return;
}
}
@file_put_contents($pidfile, getmypid());
}
$GLOBALS["EVENTS"] = array();
$CACHE_FILE = "/etc/artica-postfix/ufdb.tables.db";
updatev2_progress(10, "CACHE_FILE = {$CACHE_FILE} [" . __LINE__ . "]");
$sock = new sockets();
$EnableArticaMetaClient = intval($sock->GET_INFO("EnableArticaMetaClient"));
if ($EnableArticaMetaClient == 1) {
updatev2_progress(10, "Using Artica Meta console [" . __LINE__ . "]");
return ufdbtables_artica_meta();
}
$UseRemoteUfdbguardService = $sock->GET_INFO('UseRemoteUfdbguardService');
if (!is_numeric($UseRemoteUfdbguardService)) {
$UseRemoteUfdbguardService = 0;
}
if ($UseRemoteUfdbguardService == 1) {
updatev2_progress(10, "UseRemoteUfdbguardService = TRUE - aborting [" . __LINE__ . "]");
return;
}
if ($GLOBALS["MIRROR"] == null) {
updatev2_progress(10, "MIRROR is null #2 [" . __LINE__ . "]");
unset($GLOBALS["updatev2_checkversion"]);
updatev2_checkversion();
}
if ($GLOBALS["MIRROR"] == null) {
updatev2_progress(110, "MIRROR is null #1 [" . __LINE__ . "]");
return;
}
$tmpdir = $unix->TEMP_DIR();
$URIBASE = $GLOBALS["MIRROR"];
$WORKDIR = $GLOBALS["WORKDIR_LOCAL"];
if (is_link($WORKDIR)) {
$WORKDIR = readlink($WORKDIR);
}
$CategoriesDatabasesByCron = $sock->GET_INFO("CategoriesDatabaseByCron");
if (!is_numeric($CategoriesDatabasesByCron)) {
$CategoriesDatabasesByCron = 1;
}
if (!$GLOBALS["FORCE"]) {
if ($CategoriesDatabasesByCron == 1) {
if (!$GLOBALS["BYCRON"]) {
updatev2_progress(110, "{done} CategoriesDatabasesByCron [" . __LINE__ . "]");
return;
}
}
}
updatev2_progress(15, "{$URIBASE}/index.txt [" . __LINE__ . "]");
$curl = new ccurl("{$URIBASE}/index.txt");
$STATUS = unserialize(@file_get_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD"));
$STATUS["LAST_DOWNLOAD"]["LAST_CHECK"] = time();
@file_put_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD", serialize($STATUS));
if (!$curl->GetHeads()) {
if ($GLOBALS["VERBOSE"]) {
echo "Fatal ! {$URIBASE}/index.txt ERROR NUMBER {$curl->CURLINFO_HTTP_CODE}\n";
}
if ($curl->CURLINFO_HTTP_CODE == 404 or $curl->CURLINFO_HTTP_CODE == 300) {
if (!preg_match("#\\/ufdb#", $URIBASE)) {
$URIBASE = "{$URIBASE}/ufdb";
}
$curl = new ccurl("{$URIBASE}/index.txt");
if (!$curl->GetHeads()) {
$GLOBALS["EVENTS"][] = "{$URIBASE}/index.txt";
$GLOBALS["EVENTS"][] = "Failed with error {$curl->error}";
while (list($a, $b) = each($GLOBALS["CURLDEBUG"])) {
$GLOBALS["EVENTS"][] = $b;
}
squid_admin_mysql(0, "Unable to download blacklist index file with error: `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
artica_update_event(0, "Unable to download Artica blacklist index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
// ufdbguard_admin_events("UFDB::Fatal: Unable to download blacklist index file $curl->error",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica");
echo "UFDB: Failed to retreive {$URIBASE}/index.txt ({$curl->error})\n";
updatev2_progress(110, "Failed to retreive {$URIBASE}/index.txt [" . __LINE__ . "]");
updatev2_adblock();
return;
}
}
}
updatev2_progress(20, "Downloading MD5Strings.txt [" . __LINE__ . "]");
$curl = new ccurl("{$URIBASE}/MD5Strings.txt");
if (!$curl->GetFile("{$tmpdir}/MD5Strings.txt")) {
artica_update_event(2, "{$URIBASE}: Unable to download Artica blacklist MD5 table `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
@unlink("{$tmpdir}/MD5Strings.txt");
}
$MD5_strings = unserialize(@file_get_contents("{$tmpdir}/MD5Strings.txt"));
$source_filetime = $curl->CURL_ALL_INFOS["filetime"];
if ($GLOBALS["VERBOSE"]) {
echo "{$URIBASE}/index.txt filetime: {$source_filetime} " . date("Y-m-d H:i:s", $source_filetime) . "\n";
}
$GLOBALS["EVENTS"][] = "{$URIBASE}/index.txt";
$GLOBALS["EVENTS"][] = "filetime: {$source_filetime} " . date("Y-m-d H:i:s", $source_filetime);
$UFDBGUARD_LAST_INDEX_TIME = "/etc/artica-postfix/UFDBGUARD_LAST_INDEX_TIME";
$old_time = intval(@file_get_contents("{$UFDBGUARD_LAST_INDEX_TIME}"));
$GLOBALS["EVENTS"][] = "Old filetime: {$old_time} " . date("Y-m-d H:i:s", $old_time);
if (is_file("/etc/artica-postfix/ufdbcounts.txt")) {
if ($source_filetime == $old_time) {
ufdbtables_md5($MD5_strings, $URIBASE);
$GLOBALS["EVENTS"][] = "No new updates";
updatev2_progress(100, "{up-to-date} [" . __LINE__ . "]");
return true;
}
if ($source_filetime < $old_time) {
ufdbtables_md5($MD5_strings, $URIBASE);
$GLOBALS["EVENTS"][] = "No new updates";
updatev2_progress(100, "{up-to-date} [" . __LINE__ . "]");
return true;
}
}
updatev2_progress(99, "{$URIBASE}/ufdbcounts.txt [" . __LINE__ . "]");
$curl = new ccurl("{$URIBASE}/ufdbcounts.txt");
if (!$curl->GetFile("/etc/artica-postfix/ufdbcounts.txt")) {
$GLOBALS["EVENTS"][] = "{$URIBASE}/ufdbcounts.txt";
$GLOBALS["EVENTS"][] = "Failed with error {$curl->error}";
while (list($a, $b) = each($GLOBALS["CURLDEBUG"])) {
$GLOBALS["EVENTS"][] = $b;
}
squid_admin_mysql(0, "Unable to download {$URIBASE}/ufdbcounts.txt index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
artica_update_event(0, "Unable to download Artica {$URIBASE}/ufdbcounts.txt index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
// ufdbguard_admin_events("UFDB::Fatal: Unable to download $URIBASE/ufdbcounts.txt index file $curl->error",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica");
}
updatev2_progress(99, "{$URIBASE}/index.txt [" . __LINE__ . "]");
$curl = new ccurl("{$URIBASE}/index.txt");
if (!$curl->GetFile("/etc/artica-postfix/artica-webfilter-db-index.txt")) {
$GLOBALS["EVENTS"][] = "{$URIBASE}/index.txt";
$GLOBALS["EVENTS"][] = "Failed with error {$curl->error}";
while (list($a, $b) = each($GLOBALS["CURLDEBUG"])) {
$GLOBALS["EVENTS"][] = $b;
}
squid_admin_mysql(0, "Unable to download blacklist index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
artica_update_event(0, "Unable to download Artica blacklist index file `{$curl->error}`", @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
// ufdbguard_admin_events("UFDB::Fatal: Unable to download blacklist index file $curl->error",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica");
echo "UFDB: Failed to retreive {$URIBASE}/index.txt ({$curl->error})\n";
updatev2_progress2(110, "Unable to download blacklist index file");
updatev2_adblock();
return;
}
if (ufdbtables_md5($MD5_strings, $URIBASE)) {
updatev2_progress(100, "{done} [" . __LINE__ . "]");
return;
}
$LOCAL_CACHE = unserialize(base64_decode(@file_get_contents($CACHE_FILE)));
$REMOTE_CACHE = unserialize(base64_decode(@file_get_contents("/etc/artica-postfix/artica-webfilter-db-index.txt")));
$CALCULATED_SIZE = 0;
$MAx = count($REMOTE_CACHE);
$BigSize = 0;
$c = 0;
$ERRORDB = 0;
while (list($tablename, $size) = each($REMOTE_CACHE)) {
$OriginalSize = $size;
if (blacklisted_tables($tablename)) {
continue;
}
$STATUS["LAST_DOWNLOAD"]["CATEGORY"] = $tablename;
if ($size != $LOCAL_CACHE[$tablename]) {
ufdbevents("{$tablename} {$size} <> {$LOCAL_CACHE[$tablename]}");
$c++;
if (!ufdbtables_DownloadInstall($URIBASE, $tablename, $size, null)) {
$ERRORDB++;
$STATUS["LAST_DOWNLOAD"]["TIME"] = time();
$STATUS["LAST_DOWNLOAD"]["SIZE"] = $GLOBALS["CURL_LAST_SIZE_DOWNLOAD"] / 1024;
$STATUS["LAST_DOWNLOAD"]["FAILED"] = $ERRORDB;
@file_put_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD", serialize($STATUS));
continue;
}
$prc = $c / $MAx * 100;
updatev2_progress2($prc, "{$tablename} ok");
$GLOBALS["UFDB_SIZE"] = $GLOBALS["CALCULATED_SIZE"];
$size = $unix->file_size("{$WORKDIR}/{$tablename}/domains.ufdb");
$size = round($size / 1024, 2);
$BigSize = $BigSize + $size;
@chown("{$WORKDIR}/{$tablename}", "squid");
@chgrp("{$WORKDIR}/{$tablename}", "squid");
$LOCAL_CACHE[$tablename] = $OriginalSize;
$GLOBALS["EVENTS"][] = "Success updating category `{$tablename}` with {$size} Ko";
$STATUS["LAST_DOWNLOAD"]["TIME"] = time();
$STATUS["LAST_DOWNLOAD"]["SIZE"] = $GLOBALS["CURL_LAST_SIZE_DOWNLOAD"] / 1024;
$STATUS["LAST_DOWNLOAD"]["FAILED"] = $ERRORDB;
@file_put_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD", serialize($STATUS));
}
}
@file_put_contents($CACHE_FILE, base64_encode(serialize($LOCAL_CACHE)));
updatev2_progress2(100, "DONE ok");
$ufdbguard_admin_memory = @implode("\n", $GLOBALS["ufdbguard_admin_memory"]);
$php5 = $unix->LOCATE_PHP5_BIN();
if (!is_file("/opt/ufdbcat/bin/ufdbcatdd")) {
system("{$php5} /usr/share/artica-postfix/exec.ufdbcat.php --install --noupdate");
}
if ($c > 0) {
$BigSizeMB = round($BigSize / 1024, 2);
shell_exec("/etc/init.d/ufdbcat reload");
squid_admin_mysql(2, "Artica Web filtering Databases Success updated {$c} categories {$BigSizeMB}MB extracted on disk", "{$ufdbguard_admin_memory}" . @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
artica_update_event(2, "Artica Web filtering Databases Success updated {$c} categories {$BigSizeMB}MB extracted on disk", "{$ufdbguard_admin_memory}" . @implode("\n", $GLOBALS["EVENTS"]), __FUNCTION__, __LINE__);
// ufdbguard_admin_events("UFDB::Success update $c categories $BigSize extracted\n$ufdbguard_admin_memory",__FUNCTION__,__FILE__,__LINE__,"ufbd-artica");
@file_put_contents($UFDBGUARD_LAST_INDEX_TIME, $source_filetime);
shell_exec("/etc/init.d/ufdbcat reload");
META_MASTER_UFDBTABLES(true);
calculate_categorized_websites(true);
} else {
if ($GLOBALS["FORCE"]) {
echo "No update available\n{$ufdbguard_admin_memory}\n";
}
}
@chown("{$WORKDIR}", "squid");
@chgrp("{$WORKDIR}", "squid");
updatev2_adblock();
scan_artica_databases();
updatev2_progress(100, "{done} [" . __LINE__ . "]");
return true;
}
function META_MASTER_UFDBTABLES($force = false)
{
$sock = new sockets();
$unix = new unix();
$sourcefile = "/usr/share/artica-postfix/ressources/logs/web/cache/CATZ_ARRAY";
$EnableArticaMetaServer = intval($sock->GET_INFO("EnableArticaMetaServer"));
if ($EnableArticaMetaServer == 0) {
return;
}
$ArticaMetaStorage = $sock->GET_INFO("ArticaMetaStorage");
if ($ArticaMetaStorage == null) {
$ArticaMetaStorage = "/home/artica-meta";
}
@mkdir("{$ArticaMetaStorage}/nightlys", 0755, true);
@mkdir("{$ArticaMetaStorage}/releases", 0755, true);
@mkdir("{$ArticaMetaStorage}/webfiltering", 0755, true);
$srcdir = $GLOBALS["WORKDIR_LOCAL"];
$destfile = "{$ArticaMetaStorage}/webfiltering/ufdbartica.tgz";
$destdir = "{$ArticaMetaStorage}/webfiltering/ufdbartica";
META_MASTER_UFDBTABLES_ufdbartica_txt();
$rm = $unix->find_program("rm");
$CATZ_ARRAY = unserialize(base64_decode(@file_get_contents($sourcefile)));
$DATABASES_VERSION = $CATZ_ARRAY["TIME"];
$STATUS = unserialize(@file_get_contents("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD"));
$LAST_DOWNLOAD = $STATUS["LAST_DOWNLOAD"]["TIME"];
if ($GLOBALS["VERBOSE"]) {
echo "LAST_DOWNLOAD = {$LAST_DOWNLOAD}\n";
}
$STATUS_STORAGE = unserialize(@file_get_contents("{$ArticaMetaStorage}/webfiltering/ARTICAUFDB_LAST_DOWNLOAD"));
$LAST_DOWNLOAD_STORAGE = $STATUS_STORAGE["LAST_DOWNLOAD"]["TIME"];
if ($GLOBALS["VERBOSE"]) {
echo "LAST_DOWNLOAD_STORAGE = {$LAST_DOWNLOAD_STORAGE}\n";
}
if (is_file("{$ArticaMetaStorage}/webfiltering/ufdbartica.txt")) {
if ($LAST_DOWNLOAD_STORAGE == $LAST_DOWNLOAD) {
if (is_file($destfile)) {
$timeFile = "/etc/artica-postfix/pids/exec.artica-meta-server.php.checkufdb.time";
$time = $unix->file_time_min($timeFile);
if ($GLOBALS["VERBOSE"]) {
echo "META_MASTER_UFDBTABLES: {$timeFile} = {$time}\n";
}
if ($time < 1440) {
return;
}
}
}
}
$tar = $unix->find_program("tar");
$split = $unix->find_program("split");
if ($GLOBALS["VERBOSE"]) {
echo "REMOVE {$destfile}\n";
}
@unlink($destfile);
if ($GLOBALS["VERBOSE"]) {
echo "CD {$srcdir}\n";
}
chdir($srcdir);
if ($GLOBALS["VERBOSE"]) {
echo "{$tar} czf {$destfile} *\n";
}
shell_exec("{$tar} czf {$destfile} *");
if (is_dir($destdir)) {
shell_exec("{$rm} -rf {$destdir}");
}
@mkdir("{$destdir}", 0755, true);
chdir("{$destdir}");
system("cd {$destdir}");
if ($GLOBALS["VERBOSE"]) {
echo "META_MASTER_UFDBTABLES: {$destfile} -> {$destdir}/ufdbartica.tgz\n";
}
@copy($destfile, "{$destdir}/ufdbartica.tgz");
echo "Split...ufdbartica.tgz\n";
shell_exec("{$split} -a 3 -b 1m -d ufdbartica.tgz ufdbartica.tgz.");
@unlink("{$destdir}/ufdbartica.tgz");
$files = $unix->DirFiles("{$destdir}");
while (list($num, $ligne) = each($files)) {
$Splited_md5 = md5_file("{$destdir}/{$num}");
$ARRAY["{$num}"] = $Splited_md5;
}
if ($GLOBALS["VERBOSE"]) {
echo "META_MASTER_UFDBTABLES: {$destdir}/ufdbartica.txt OK\n";
}
@file_put_contents("{$destdir}/ufdbartica.txt", serialize($ARRAY));
@unlink("{$ArticaMetaStorage}/webfiltering/ufdbartica.txt");
@unlink("{$ArticaMetaStorage}/webfiltering/ARTICAUFDB_LAST_DOWNLOAD");
@copy("/etc/artica-postfix/ARTICAUFDB_LAST_DOWNLOAD", "{$ArticaMetaStorage}/webfiltering/ARTICAUFDB_LAST_DOWNLOAD");
@file_put_contents("{$ArticaMetaStorage}/webfiltering/ufdbartica.txt", $DATABASES_VERSION);
if (is_file("/etc/artica-postfix/artica-webfilter-db-index.txt")) {
@unlink("{$ArticaMetaStorage}/webfiltering/index.txt");
@copy("/etc/artica-postfix/artica-webfilter-db-index.txt", "{$ArticaMetaStorage}/webfiltering/index.txt");
}
if (is_file("/etc/artica-postfix/ufdbcounts.txt")) {
@unlink("{$ArticaMetaStorage}/webfiltering/ufdbcounts.txt");
@copy("/etc/artica-postfix/ufdbcounts.txt", "{$ArticaMetaStorage}/webfiltering/ufdbcounts.txt");
}
calculate_categorized_websites(true);
artica_update_event(2, "Artica Webfiltering databases: Success update Meta Server webfiltering repository", @implode("\n", $GLOBALS["EVENTS"]), __FILE__, __LINE__);
meta_admin_mysql(2, "Success update webfiltering repository with Webfiltering databases", null, __FILE__, __LINE__);
}
