function re_categorize() { $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid"; $oldpid = @file_get_contents($pidfile); if ($oldpid < 100) { $oldpid = null; } $unix = new unix(); if ($unix->process_exists($oldpid, basename(__FILE__))) { if ($GLOBALS["VERBOSE"]) { echo "Already executed pid {$oldpid}\n"; } die; } if (systemMaxOverloaded()) { writelogs_squid("Fatal: VERY Overloaded system, die();", __FUNCTION__, __FILE__, __LINE__, "stats"); return; } $mypid = getmypid(); @file_put_contents($pidfile, $mypid); $sock = new sockets(); $RecategorizeSecondsToWaitOverload = $sock->GET_INFO("RecategorizeSecondsToWaitOverload"); $RecategorizeMaxExecutionTime = $sock->GET_INFO("RecategorizeSecondsToWaitOverload"); $RecategorizeProxyStats = $sock->GET_INFO("RecategorizeProxyStats"); if (!is_numeric($RecategorizeProxyStats)) { $RecategorizeProxyStats = 1; } if (!is_numeric($RecategorizeSecondsToWaitOverload)) { $RecategorizeSecondsToWaitOverload = 30; } if (!is_numeric($RecategorizeMaxExecutionTime)) { $RecategorizeMaxExecutionTime = 210; } if ($RecategorizeProxyStats == 0) { return; } $t = time(); $sql = "SELECT * FROM visited_sites"; $results = $GLOBALS["Q"]->QUERY_SQL($sql); $num_rows = mysql_num_rows($results); $c = 0; while ($ligne = @mysql_fetch_array($results, MYSQL_ASSOC)) { $website = trim($ligne["sitename"]); if ($website == null) { continue; } $category = trim($GLOBALS["Q"]->GET_CATEGORIES($website, true)); $GLOBALS["Q"]->QUERY_SQL("UPDATE visited_sites SET category='{$category}' WHERE sitename='{$website}'"); if (!$GLOBALS["Q"]->ok) { writelogs_squid("Fatal: mysql error {$GLOBALS["Q"]->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "stats"); return; } $c++; if ($c > 5000) { $distanceInSeconds = round(abs(time() - $t)); $distanceInMinutes = round($distanceInSeconds / 60); if ($distanceInMinutes > $RecategorizeMaxExecutionTime) { $took = $unix->distanceOfTimeInWords($t, time()); writelogs_squid("Re-categorized websites task aborted (Max execution time {$RecategorizeMaxExecutionTime}Mn) ({$took})", __FUNCTION__, __FILE__, __LINE__, "stats"); return; } $c = 0; } } $took = $unix->distanceOfTimeInWords($t, time()); writelogs_squid("{$num_rows} re-categorized websites in main table ({$took})", __FUNCTION__, __FILE__, __LINE__, "stats"); __re_categorize_subtables($t); }
function re_categorize($nopid = false) { $q = $GLOBALS["Q"]; if (!$nopid) { $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid"; $pid = @file_get_contents($pidfile); if ($pid < 100) { $pid = null; } $unix = new unix(); if ($unix->process_exists($pid, basename(__FILE__))) { ufdbguard_admin_events("Already executed pid {$pid}", __FUNCTION__, __FILE__, __LINE__, "stats"); if ($GLOBALS["VERBOSE"]) { echo "Already executed pid {$pid}\n"; } return; } $mypid = getmypid(); @file_put_contents($pidfile, $mypid); } $sock = new sockets(); $RecategorizeSecondsToWaitOverload = $sock->GET_INFO("RecategorizeSecondsToWaitOverload"); $RecategorizeMaxExecutionTime = $sock->GET_INFO("RecategorizeSecondsToWaitOverload"); $RecategorizeProxyStats = $sock->GET_INFO("RecategorizeProxyStats"); if (!is_numeric($RecategorizeProxyStats)) { $RecategorizeProxyStats = 1; } if (!is_numeric($RecategorizeSecondsToWaitOverload)) { $RecategorizeSecondsToWaitOverload = 30; } if (!is_numeric($RecategorizeMaxExecutionTime)) { $RecategorizeMaxExecutionTime = 210; } if ($RecategorizeProxyStats == 0) { ufdbguard_admin_events("RecategorizeProxyStats=0, aborting...", __FUNCTION__, __FILE__, __LINE__, "stats"); return; } $t = time(); if (!$GLOBALS["Q"]->FIELD_EXISTS("visited_sites", "recatgorized")) { $GLOBALS["Q"]->QUERY_SQL("ALTER TABLE `visited_sites` ADD `recatgorized` smallint(1) NOT NULL ,\n\t\t\t\tADD KEY `recatgorized` (`recatgorized`)"); } $sql = "SELECT * FROM visited_sites WHERE recatgorized='0' LIMIT 0,300"; $results = $GLOBALS["Q"]->QUERY_SQL($sql); $num_rows = mysql_num_rows($results); stats_admin_events(2, "Trying to recategorize {$num_rows} visited websites", "", __FILE__, __LINE__); $t = time(); $c = 0; $L = 0; while ($ligne = @mysql_fetch_array($results, MYSQL_ASSOC)) { $website = trim($ligne["sitename"]); $categorySrc = $ligne["category"]; if ($website == null) { continue; } $category = trim($GLOBALS["Q"]->GET_CATEGORIES($website, true)); if ($category == $categorySrc) { $GLOBALS["Q"]->QUERY_SQL("UPDATE visited_sites SET recatgorized=1 WHERE sitename='{$website}'"); continue; } if ($category == null) { $GLOBALS["Q"]->QUERY_SQL("UPDATE visited_sites SET recatgorized=1 WHERE sitename='{$website}'"); continue; } $GLOBALS["Q"]->QUERY_SQL("UPDATE visited_sites SET category='{$category}',recatgorized=1 WHERE sitename='{$website}'"); if (!$GLOBALS["Q"]->ok) { stats_admin_events(0, "Fatal: mysql error", "{$GLOBALS["Q"]->mysql_error}", __FILE__, __LINE__); return; } $c++; $L++; $websites[$website] = $category; if ($L > 400) { $took = $unix->distanceOfTimeInWords($t, time()); if (SquidStatisticsTasksOverTime()) { stats_admin_events(1, "Statistics overtime... Aborting", null, __FILE__, __LINE__); return; } $L = 0; } } $took = $unix->distanceOfTimeInWords($t, time()); stats_admin_events(2, "{$c} re-categorized websites in main table ({$took})", __FUNCTION__, __FILE__, __LINE__, "stats"); if (count($websites) > 0) { __re_categorize_subtables($t, $websites); } }