function update_remote_file($BASE_URI, $filename, $md5, $prc)
{
if (!isset($GLOBALS["UFDB_SIZE"])) {
$GLOBALS["UFDB_SIZE"] = 0;
}
WriteMyLogs("update_remote_file({$BASE_URI},{$filename},{$md5})", __FUNCTION__, __FILE__, __LINE__);
$STATUS = unserialize(@file_get_contents("/etc/artica-postfix/TLSE_LAST_DOWNLOAD"));
$indexuri = "{$BASE_URI}/{$filename}";
$unix = new unix();
$q = new mysql_squid_builder();
$tar = $unix->find_program("tar");
$rm = $unix->find_program("rm");
$ln = $unix->find_program("ln");
$ufdbGenTable = $unix->find_program("ufdbGenTable");
$Conversion = $q->TLSE_CONVERTION();
$ufdb = new compile_ufdbguard();
$curl = new ccurl($indexuri);
$curl->Timeout = 360;
echo "Downloading {$indexuri}\n";
$cache_temp = "/tmp/{$filename}";
if (!$curl->GetFile($cache_temp)) {
build_progress("Fatal error downloading {$indexuri} {$curl->error}", $prc);
echo "Fatal error downloading {$indexuri} {$curl->error}\n";
$errorDetails = @implode("\n", $GLOBALS["CURLDEBUG"]);
artica_update_event(0, "Web filtering databases, unable to download {$indexuri}", "Fatal error downloading {$indexuri} {$curl->error}\n{$errorDetails}", __FILE__, __LINE__);
return;
}
$filesize = $unix->file_size($cache_temp);
$GLOBALS["UFDB_SIZE"] = $GLOBALS["UFDB_SIZE"] + $filesize;
@mkdir("/var/lib/ftpunivtlse1fr", 755, true);
$categoryname = str_replace(".tar.gz", "", $filename);
$categoryDISK = $categoryname;
if (isset($Conversion[$categoryname])) {
$categoryDISK = $Conversion[$categoryname];
}
$STATUS["LAST_DOWNLOAD"]["TIME"] = time();
$STATUS["LAST_DOWNLOAD"]["CATEGORY"] = $categoryname;
$STATUS["LAST_DOWNLOAD"]["SIZE"] = $GLOBALS["CURL_LAST_SIZE_DOWNLOAD"] / 1024;
@file_put_contents("/etc/artica-postfix/TLSE_LAST_DOWNLOAD", serialize($STATUS));
$categoryDISK = str_replace("/", "_", $categoryDISK);
if (is_link("/var/lib/ftpunivtlse1fr/{$categoryname}")) {
if ($GLOBALS["VERBOSE"]) {
echo "/var/lib/ftpunivtlse1fr/{$categoryname} is a link of " . @readlink("/var/lib/ftpunivtlse1fr/{$categoryname}") . "\n";
}
if ($GLOBALS["VERBOSE"]) {
echo "Removing /var/lib/ftpunivtlse1fr/{$categoryname}/\n";
}
shell_exec("{$rm} -rf /var/lib/ftpunivtlse1fr/{$categoryname}");
}
if (is_dir("/var/lib/ftpunivtlse1fr/{$categoryname}")) {
if ($GLOBALS["VERBOSE"]) {
echo "Removing /var/lib/ftpunivtlse1fr/{$categoryname}/\n";
}
shell_exec("{$rm} -rf /var/lib/ftpunivtlse1fr/{$categoryname}");
}
if ($GLOBALS["VERBOSE"]) {
echo "Creating /var/lib/ftpunivtlse1fr/{$categoryname}/\n";
}
@mkdir("/var/lib/ftpunivtlse1fr/{$categoryname}", 0755, true);
if ($GLOBALS["VERBOSE"]) {
echo "Extracting {$cache_temp} to /var/lib/ftpunivtlse1fr/\n";
}
shell_exec("{$tar} -xf {$cache_temp} -C /var/lib/ftpunivtlse1fr/");
if (!is_file("/var/lib/ftpunivtlse1fr/{$categoryname}/domains")) {
build_progress("Fatal!!: {$categoryname}/domains no such file", $prc);
ufdbevents("Fatal!!: /var/lib/ftpunivtlse1fr/{$categoryname}/domains no such file", __FUNCTION__, __FILE__, __LINE__);
return;
}
$CountDeSitesFile = CountDeSitesFile("/var/lib/ftpunivtlse1fr/{$categoryname}/domains");
if ($GLOBALS["VERBOSE"]) {
echo "/var/lib/ftpunivtlse1fr/{$categoryname}/domains -> {$CountDeSitesFile} websites\n";
}
if ($CountDeSitesFile == 0) {
build_progress("Fatal!!: {$categoryname}/domains corrupted, no website", $prc);
ufdbevents("Fatal!!: /var/lib/ftpunivtlse1fr/{$categoryname}/domains corrupted, no website", __FUNCTION__, __FILE__, __LINE__, "Toulouse DB");
shell_exec("{$rm} -rf /var/lib/ftpunivtlse1fr/{$categoryname}");
return;
}
if (trim(strtolower($categoryDISK)) != trim(strtolower($categoryname))) {
if (is_dir("/var/lib/ftpunivtlse1fr/{$categoryDISK}")) {
shell_exec("{$rm} -rf /var/lib/ftpunivtlse1fr/{$categoryDISK}");
}
shell_exec("ln -sf /var/lib/ftpunivtlse1fr/{$categoryDISK} /var/lib/ftpunivtlse1fr/{$categoryname}");
}
$q->QUERY_SQL("DELETE FROM ftpunivtlse1fr WHERE filename='{$filename}'");
if (!$q->ok) {
ufdbevents("Fatal!!: {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "Toulouse DB");
return;
}
$q->QUERY_SQL("INSERT INTO ftpunivtlse1fr (`filename`,`zmd5`,`websitesnum`) VALUES ('{$filename}','{$md5}','{$CountDeSitesFile}')");
if (!$q->ok) {
ufdbevents("Fatal!!: {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "Toulouse DB");
return;
}
$GLOBALS["UFDB_COUNT_OF_DOWNLOADED"] = $GLOBALS["UFDB_COUNT_OF_DOWNLOADED"] + 1;
build_progress("{$categoryname} {$CountDeSitesFile} websites", $prc);
$GLOBALS["squid_admin_mysql"][] = "Success updating category `{$categoryname}` with {$CountDeSitesFile} websites";
if ($GLOBALS["VERBOSE"]) {
echo "ufdbGenTable={$ufdbGenTable}\n";
}
if (is_file($ufdbGenTable)) {
$t = time();
ufdbevents("Compiling /var/lib/ftpunivtlse1fr/{$categoryname}");
build_progress("{$categoryname} Compiling....", $prc);
$ufdb->UfdbGenTable("/var/lib/ftpunivtlse1fr/{$categoryname}", $categoryname);
}
}
function updatev2()
{
$sock = new sockets();
$unix = new unix();
$GLOBALS["TEMP_PATH"] = $unix->TEMP_DIR();
updatev2_progress(10, "{checking} [" . __LINE__ . "]");
$timeFile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".time";
$pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid";
$ArticaDbReplicate = $sock->GET_INFO("ArticaDbReplicate");
$CategoriesDatabasesByCron = $sock->GET_INFO("CategoriesDatabaseByCron");
if (!is_numeric($CategoriesDatabasesByCron)) {
$CategoriesDatabasesByCron = 1;
}
$DisableArticaProxyStatistics = $sock->GET_INFO("DisableArticaProxyStatistics");
$ArticaDBPath = $sock->GET_INFO("ArticaDBPath");
if ($ArticaDBPath == null) {
$ArticaDBPath = "/opt/articatech";
}
$ManualArticaDBPath = $sock->GET_INFO("ManualArticaDBPath");
if ($ManualArticaDBPath == null) {
$ManualArticaDBPath = "/home/manualupdate/articadb.tar.gz";
}
$ManualArticaDBPathNAS = $sock->GET_INFO("ManualArticaDBPathNAS");
$datas = unserialize(base64_decode($sock->GET_INFO("ufdbguardConfig")));
updatev2_progress(10, "{checking} [" . __LINE__ . "]");
if (!is_numeric($ManualArticaDBPathNAS)) {
$ManualArticaDBPathNAS = 0;
}
if (!is_numeric($DisableArticaProxyStatistics)) {
$DisableArticaProxyStatistics = 0;
}
if (!is_numeric($ArticaDbReplicate)) {
$ArticaDbReplicate = 0;
}
$WizardStatsAppliance = unserialize(base64_decode($sock->GET_INFO("WizardStatsAppliance")));
if (!isset($WizardStatsAppliance["SERVER"])) {
$WizardStatsAppliance["SERVER"] = null;
}
if ($DisableArticaProxyStatistics == 1) {
updatev2_progress(110, "Error: Artica statistics are disabled");
}
if ($datas["UseRemoteUfdbguardService"] == 1) {
updatev2_progress(110, "Error: - UseRemoteUfdbguardService - Only used by {$WizardStatsAppliance["SERVER"]}");
return;
}
if (!$GLOBALS["FORCE"]) {
if ($CategoriesDatabasesByCron == 1) {
if (!$GLOBALS["BYCRON"]) {
updatev2_progress(110, "Error: Only executed by schedule...");
return;
}
}
}
if ($GLOBALS["FORCE"]) {
ufdbevents("***** Force enabled ***** ");
ufdbevents("*****");
ufdbevents("*****");
ufdbevents("Executed as {$GLOBALS["CMDLINE"]}");
ufdbevents("*****");
ufdbevents("*****");
}
if (!$GLOBALS["CHECKTIME"]) {
ufdbevents("***** CHECKTIME disabled ***** ");
}
$CHECKTIME = $unix->file_time_min($timeFile);
ufdbevents(" **");
ufdbevents(" **");
ufdbevents("{$timeFile} = {$CHECKTIME}Mn");
ufdbevents(" **");
ufdbevents(" **");
if (!$GLOBALS["FORCE"]) {
if ($CHECKTIME < 240) {
updatev2_progress(110, "STOP: current {$CHECKTIME}Mn, require 240mn");
return;
}
}
updatev2_progress(10, "{checking} [" . __LINE__ . "]");
$pid = @file_get_contents($pidfile);
if ($unix->process_exists($pid, __FILE__)) {
$time = $unix->PROCCESS_TIME_MIN($pid);
if ($time < 10200) {
updatev2_progress(110, "Error: already running pid {$pid} since {$time}Mn");
return;
} else {
$kill = $unix->find_program("kill");
unix_system_kill_force($pid);
if ($GLOBALS["SCHEDULE_ID"] > 0) {
artica_update_event(1, "Warning: Old task pid {$pid} since {$time}Mn has been killed, (reach 7200mn)", null, __FILE__, __LINE__);
}
}
}
updatev2_progress(10, "{checking} [" . __LINE__ . "]");
ufdbevents("Stamp {$timeFile}");
@unlink($timeFile);
$tlse_force_token = null;
@file_put_contents($timeFile, time());
@file_put_contents($pidfile, getmypid());
$tlse_token = null;
if ($GLOBALS["BYCRON"]) {
$tlse_token == " --bycron --force";
}
if ($GLOBALS["FORCE"]) {
$tlse_force_token = " --force";
}
$php = $unix->LOCATE_PHP5_BIN();
$nohup = $unix->find_program("nohup");
ufdbevents("Running exec.update.squid.tlse.php");
shell_exec("{$nohup} {$php} /usr/share/artica-postfix/exec.update.squid.tlse.php --schedule-id={$GLOBALS["SCHEDULE_ID"]}{$tlse_force_token}{$tlse_token} >/dev/null 2>&1 &");
updatev2_progress(10, "{checking} [" . __LINE__ . "]");
updatev2_checkversion();
updatev2_progress(12, "{runing} [" . __LINE__ . "]");
ufdbtables(true);
C_ICAP_TABLES(true);
schedulemaintenance();
EXECUTE_BLACK_INSTANCE();
}
function updatev2()
{
$sock = new sockets();
$unix = new unix();
$GLOBALS["TEMP_PATH"] = $unix->TEMP_DIR();
updatev2_progress(10, "{checking} [" . __LINE__ . "]");
$timeFile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".time";
$pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid";
$ArticaDbReplicate = $sock->GET_INFO("ArticaDbReplicate");
$CategoriesDatabasesByCron = $sock->GET_INFO("CategoriesDatabaseByCron");
if (!is_numeric($CategoriesDatabasesByCron)) {
$CategoriesDatabasesByCron = 1;
}
$DisableArticaProxyStatistics = $sock->GET_INFO("DisableArticaProxyStatistics");
$CategoriesDatabasesUpdatesAllTimes = intval($sock->GET_INFO("CategoriesDatabasesUpdatesAllTimes"));
$ArticaDBPath = $sock->GET_INFO("ArticaDBPath");
if ($ArticaDBPath == null) {
$ArticaDBPath = "/opt/articatech";
}
$ManualArticaDBPath = $sock->GET_INFO("ManualArticaDBPath");
if ($ManualArticaDBPath == null) {
$ManualArticaDBPath = "/home/manualupdate/articadb.tar.gz";
}
$ManualArticaDBPathNAS = $sock->GET_INFO("ManualArticaDBPathNAS");
$EnableArticaMetaServer = intval($sock->GET_INFO("EnableArticaMetaServer"));
$datas = unserialize(base64_decode($sock->GET_INFO("ufdbguardConfig")));
updatev2_progress(10, "{checking} [" . __LINE__ . "]");
if (!is_numeric($ManualArticaDBPathNAS)) {
$ManualArticaDBPathNAS = 0;
}
if (!is_numeric($DisableArticaProxyStatistics)) {
$DisableArticaProxyStatistics = 0;
}
if (!is_numeric($ArticaDbReplicate)) {
$ArticaDbReplicate = 0;
}
$WizardStatsAppliance = unserialize(base64_decode($sock->GET_INFO("WizardStatsAppliance")));
if (!isset($WizardStatsAppliance["SERVER"])) {
$WizardStatsAppliance["SERVER"] = null;
}
if ($EnableArticaMetaServer == 0) {
if ($DisableArticaProxyStatistics == 1) {
updatev2_progress(110, "Error: Artica statistics are disabled");
}
}
if ($EnableArticaMetaServer == 0) {
if ($datas["UseRemoteUfdbguardService"] == 1) {
updatev2_progress(110, "Error: - UseRemoteUfdbguardService - Only used by {$WizardStatsAppliance["SERVER"]}");
return;
}
}
$CHECKTIME = $unix->file_time_min($timeFile);
ufdbevents(" **");
ufdbevents(" **");
ufdbevents("{$timeFile} = {$CHECKTIME}Mn");
ufdbevents(" **");
ufdbevents(" **");
if (!$GLOBALS["FORCE"]) {
if ($CategoriesDatabasesByCron == 1) {
if ($EnableArticaMetaServer == 0) {
if (!$GLOBALS["BYCRON"]) {
updatev2_progress(110, "Error: Only executed by schedule [" . __LINE__ . "]");
if ($CHECKTIME > 60) {
updatev2_checkversions();
}
return;
}
}
}
if ($CategoriesDatabasesUpdatesAllTimes == 0) {
if ($EnableArticaMetaServer == 0) {
if ($unix->IsProductionTime()) {
webupdate_admin_mysql(2, "Update aborted, only allowed outside the production time", null, __FILE__, __LINE__);
updatev2_progress(110, "Error: Only outside production time");
if ($CHECKTIME > 60) {
updatev2_checkversions();
}
return;
}
}
}
}
if ($GLOBALS["FORCE"]) {
ufdbevents("***** Force enabled ***** ");
ufdbevents("*****");
ufdbevents("*****");
ufdbevents("Executed as {$GLOBALS["CMDLINE"]}");
ufdbevents("*****");
ufdbevents("*****");
}
$MaxCheckTime = 240;
if ($EnableArticaMetaServer == 1) {
$MaxCheckTime = 60;
}
if (!$GLOBALS["FORCE"]) {
if ($CHECKTIME < 240) {
updatev2_progress(110, "STOP: current {$CHECKTIME}Mn, require {$MaxCheckTime}mn");
if ($CHECKTIME > 60) {
updatev2_checkversions();
}
return;
}
}
updatev2_progress(10, "{checking} [" . __LINE__ . "]");
$pid = @file_get_contents($pidfile);
if ($unix->process_exists($pid, __FILE__)) {
$time = $unix->PROCCESS_TIME_MIN($pid);
if ($time < 10200) {
updatev2_progress(110, "Error: already running pid {$pid} since {$time}Mn");
return;
} else {
$kill = $unix->find_program("kill");
unix_system_kill_force($pid);
if ($GLOBALS["SCHEDULE_ID"] > 0) {
artica_update_event(1, "Warning: Old task pid {$pid} since {$time}Mn has been killed, (reach 7200mn)", null, __FILE__, __LINE__);
}
}
}
updatev2_progress(10, "{checking} [" . __LINE__ . "]");
ufdbevents("Stamp {$timeFile}");
@unlink($timeFile);
$tlse_force_token = null;
@file_put_contents($timeFile, time());
@file_put_contents($pidfile, getmypid());
$tlse_token = null;
if ($GLOBALS["BYCRON"]) {
$tlse_token == " --bycron --force";
}
if ($GLOBALS["FORCE"]) {
$tlse_force_token = " --force";
}
$php = $unix->LOCATE_PHP5_BIN();
$nohup = $unix->find_program("nohup");
$PrivoxyEnabled = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/PrivoxyEnabled"));
if ($PrivoxyEnabled == 1) {
@chmod("/usr/share/artica-postfix/bin/privoxy-blocklist.sh", 0755);
system("{$nohup} /usr/share/artica-postfix/bin/privoxy-blocklist.sh >/dev/null 2>&1 &");
}
system("{$nohup} {$php} /usr/share/artica-postfix/exec.infected.urls.php >/dev/null 2>&1 &");
$EnableArticaMetaClient = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/EnableArticaMetaClient"));
$EnableArticaMetaServer = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/EnableArticaMetaServer"));
if ($EnableArticaMetaServer == 1) {
$EnableArticaMetaClient = 0;
}
if ($EnableArticaMetaClient == 1) {
updatev2_progress(10, "Using Meta console [" . __LINE__ . "]");
return ufdbtables_artica_meta();
}
updatev2_progress(10, "{checking} [" . __LINE__ . "]");
updatev2_checkversions();
updatev2_progress(12, "{running} [" . __LINE__ . "]");
$GLOBALS["DOWNLOADED_INSTALLED"] = 0;
ufdbtables(true);
tlsetables(true);
ufdb_phistank();
TranslateToMetaServer();
if ($GLOBALS["DOWNLOADED_INSTALLED"] > 0) {
updatev2_progress(96, "{restarting_webfiltering_service} [" . __LINE__ . "]");
squid_admin_mysql(2, "{$GLOBALS["DOWNLOADED_INSTALLED"]} updated blacklists databases [action=restart]", __FILE__, __LINE__);
system("/etc/init.d/ufdb restart --updater");
system("/etc/init.d/ufdbcat restart --updater");
$squidbin = $unix->LOCATE_SQUID_BIN();
squid_admin_mysql(1, "Reloading proxy service after updating Web filtering databases", __FILE__, __LINE__);
system("{$squidbin} -f /etc/squid3/squid.conf -k reconfigure");
}
if ($GLOBALS["VERBOSE"]) {
echo " **************** C_ICAP_TABLES ***************\n";
}
C_ICAP_TABLES(true);
if ($GLOBALS["VERBOSE"]) {
echo " **************** schedulemaintenance ***************\n";
}
schedulemaintenance();
if ($GLOBALS["VERBOSE"]) {
echo " **************** EXECUTE_BLACK_INSTANCE ***************\n";
}
EXECUTE_BLACK_INSTANCE();
if ($GLOBALS["VERBOSE"]) {
echo " **************** FINISH ***************\n";
}
updatev2_progress(100, "{done}");
}
