function WordScanners()
{
if (isset($GLOBALS["WordScanners_executed"])) {
return;
}
$GLOBALS["WordScanners_executed"] = true;
$workdir = "/var/log/artica-postfix/searchwords";
@mkdir($workdir, 0755, true);
if ($GLOBALS["VERBOSE"]) {
echo "Open {$workdir}\n";
}
$handle = opendir($workdir);
if (!$handle) {
if ($GLOBALS["VERBOSE"]) {
echo "Fatal unable to opendir {$workdir}\n";
}
events("Fatal unable to opendir {$workdir}", __FUNCTION__, __FILE__, __LINE__);
return;
}
if ($GLOBALS["VERBOSE"]) {
events("WordScanners(" . __LINE__ . "):Open {$workdir} done.. instanciate mysql_squid_builder()");
}
$q = new mysql_squid_builder();
if ($GLOBALS["VERBOSE"]) {
events("WordScanners(" . __LINE__ . "):Start looping()");
}
$FF = array();
while (false !== ($filename = readdir($handle))) {
if ($filename == ".") {
continue;
}
if ($filename == "..") {
continue;
}
$targetFile = "{$workdir}/{$filename}";
if ($GLOBALS["VERBOSE"]) {
echo "Scanning {$targetFile}\n";
}
$searchWords = unserialize(@file_get_contents($targetFile));
while (list($key, $val) = each($searchWords)) {
$searchWords[$key] = addslashes(str_replace("'", "`", $val));
}
$zmd5 = md5(serialize($searchWords));
$ipaddr = $searchWords["ipaddr"];
$date = $searchWords["date"];
$Time = strtotime($date);
$uid = $searchWords["uid"];
if (strlen($uid) < 3) {
$uid = null;
}
$mac = trim($searchWords["mac"]);
if ($mac == null) {
$mac = GetMacFromIP($ipaddr);
}
$hostname = $searchWords["hostname"];
$words = $searchWords["WORDS"];
if (trim($words) == null) {
@unlink($targetFile);
continue;
}
$sitename = $searchWords["SITENAME"];
$sitename = addslashes($sitename);
if (!isset($GLOBALS["familysite"][$sitename])) {
$GLOBALS["familysite"][$sitename] = $q->GetFamilySites($sitename);
}
$familysite = $GLOBALS["familysite"][$sitename];
if ($mac != null) {
if ($uid == null) {
$uid = $q->UID_FROM_MAC($mac);
}
}
if ($ipaddr != null) {
if ($uid == null) {
$uid = $q->UID_FROM_IP($ipaddr);
}
}
$FF[date("Ymdh", $Time)][] = "('{$zmd5}','{$sitename}','{$date}','{$ipaddr}','{$hostname}','{$uid}','{$mac}','0','{$familysite}','{$words}')";
@unlink($targetFile);
}
events("WordScanners(" . __LINE__ . "): End looping...\n");
$q = new mysql_squid_builder();
while (list($tablePrefix, $f) = each($FF)) {
if (count($f) > 0) {
if ($q->check_SearchWords_hour($tablePrefix)) {
$tablename = "searchwords_{$tablePrefix}";
$prefix = "INSERT IGNORE INTO {$tablename} (`zmd5`,`sitename`,`zDate`,`ipaddr`,`hostname`,`uid`,`MAC`,`account`,`familysite`,`words`) VALUES " . @implode(",", $f);
$q->QUERY_SQL($prefix);
if (!$q->ok) {
writelogs_squid("{$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "stats");
@mkdir("/var/log/artica-postfix/searchwords-sql-errors", 0755, true);
@file_put_contents("/var/log/artica-postfix/searchwords-sql-errors/" . md5($prefix), $prefix);
}
}
}
}
}
function parse_realtime_events($nopid = false)
{
events("parse_realtime_events():: nopid => {$nopid}");
$unix = new unix();
$TimePID = "/etc/artica-postfix/pids/" . basename(__FILE__) . ".pid";
$TimeExec = "/etc/artica-postfix/pids/" . basename(__FILE__) . ".time";
if (!$nopid) {
$pid = @file_get_contents($TimePID);
if ($unix->process_exists($pid)) {
$timePid = $unix->PROCCESS_TIME_MIN($pid);
events("parse_realtime_events():: Already process exists {$pid} since {$timePid}Mn");
if ($timePid > 10) {
$kill = $unix->find_program("kill");
events("parse_realtime_events():: Killing {$pid} running since {$timePid}Mn");
unix_system_kill_force($pid);
} else {
if ($GLOBALS["VERBOSE"]) {
echo "Already running PID {$pid} since {$timePid}mn";
}
die;
}
}
@file_put_contents($TimePID, getmypid());
}
events("parse_realtime_events():: Time File: {$TimeExec}");
@unlink($TimeExec);
@file_put_contents($TimeExec, time());
events("Wakup...");
Wakeup();
events("parse_realtime_events():: -> parse_realtime_hash()");
$GLOBALS["PARSE_SECOND_TIME"] = false;
parse_realtime_hash();
if (!$GLOBALS["PARSE_SECOND_TIME"]) {
return;
}
@mkdir("/var/log/squid/mysql-queue", 0755, true);
if (!($handle = opendir("/var/log/squid/mysql-queue"))) {
return;
}
$q = new mysql_squid_builder();
$q->check_youtube_hour(date("YmdH"));
$countDeFiles = 0;
while (false !== ($filename = readdir($handle))) {
if ($filename == ".") {
continue;
}
if ($filename == "..") {
continue;
}
$filepath = "/var/log/squid/mysql-queue/{$filename}";
if (preg_match("#^UserAutDB#", $filename)) {
$content = unserialize(@file_get_contents($filepath));
$contentSize = filesize($filepath) / 1024;
$ArraySize = count($content);
$sql = "INSERT IGNORE INTO UserAutDB (zmd5,MAC,ipaddr,uid,hostname,UserAgent) VALUES " . @implode(",", $content);
$q->QUERY_SQL($sql);
if (!$q->ok) {
echo $q->mysql_error . "\n";
} else {
if ($GLOBALS["VERBOSE"]) {
echo $filepath . " ({$contentSize} KB) done with {$ArraySize} elements...\n";
}
@unlink($filepath);
}
continue;
}
if (preg_match("#^macscan#", $filename)) {
$content = unserialize(@file_get_contents($filepath));
$contentSize = filesize($filepath) / 1024;
$ArraySize = count($content);
$sql = "INSERT IGNORE INTO `macscan` (`MAC`,`ipaddr`) VALUES " . @implode(",", $content);
$q->QUERY_SQL($sql);
if (!$q->ok) {
echo $q->mysql_error . "\n";
} else {
if ($GLOBALS["VERBOSE"]) {
echo $filepath . " ({$contentSize} KB) done with {$ArraySize} elements...\n";
}
@unlink($filepath);
}
continue;
}
if (preg_match("#^YoutubeRTT#", $filename)) {
$sql = trim(@file_get_contents($filepath));
$contentSize = strlen($sql) / 1024;
if (preg_match("#INSERT IGNORE INTO `(.+?)`#", $sql, $re)) {
$tablename = $re[1];
if (!preg_match("#youtubehours_([0-9]+)#", $tablename)) {
echo "***** replace {$tablename} to youtubehours_date(YmdH) ****\n";
$sql = str_replace($tablename, "youtubehours_" . date("YmdH"), $sql);
}
}
$q->QUERY_SQL($sql);
if (!$q->ok) {
ToSyslog("{$q->mysql_error} in line [" . __LINE__ . "]");
if (preg_match("#Table\\s+'.+?\\.youtubehours_(.+?)'\\s+doesn't exist#", $q->mysql_error, $re)) {
ToSyslog("Building youtubehours_{$re[1]} table");
$q->check_youtube_hour($re[1]);
$q->QUERY_SQL($sql);
}
}
if (!$q->ok) {
ToSyslog("Failed ->{$filename}");
continue;
}
if ($GLOBALS["VERBOSE"]) {
echo $filepath . " ({$contentSize} KB) done with 1 element...\n";
}
@unlink($filepath);
}
if (preg_match("#^sizehour_([0-9]+)\\.#", $filename, $re)) {
$TableSizeHours = "sizehour_{$re[1]}";
$content = unserialize(@file_get_contents($filepath));
$contentSize = filesize($filepath) / 1024;
$q->check_sizehour($TableSizeHours);
$sql = "INSERT IGNORE INTO `{$TableSizeHours}` (`zDate`,`size`,`cached`) VALUES " . @implode(",", $content);
$q->QUERY_SQL($sql);
if (!$q->ok) {
echo $q->mysql_error . "\n";
} else {
if ($GLOBALS["VERBOSE"]) {
echo $filepath . " ({$contentSize} KB) done with " . count($content) . " elements...\n";
}
@unlink($filepath);
}
continue;
}
if (preg_match("#^searchwords_([0-9]+)\\.#", $filename, $re)) {
$TableSource = "searchwords_{$re[1]}";
$content = unserialize(@file_get_contents($filepath));
$contentSize = filesize($filepath) / 1024;
$q->check_SearchWords_hour(null, $TableSource);
$sql = "INSERT IGNORE INTO `{$TableSource}`\n\t\t(`zmd5`,`sitename`,`zDate`,`ipaddr`,`hostname`,`uid`,`MAC`,`account`,`familysite`,`words`)\n\t\tVALUES " . @implode(",", $content);
$q->QUERY_SQL($sql);
if (!$q->ok) {
echo $q->mysql_error . "\n";
} else {
if ($GLOBALS["VERBOSE"]) {
echo $filepath . " ({$contentSize} KB) done with " . count($content) . " elements...\n";
}
@unlink($filepath);
}
continue;
}
if (preg_match("#^quotatemp_([0-9]+)\\.#", $filename, $re)) {
$TableSource = "quotatemp_{$re[1]}";
$q->check_quota_hour_tmp($re[1]);
$q->check_quota_hour($re[1]);
$content = unserialize(@file_get_contents($filepath));
$contentSize = filesize($filepath) / 1024;
$sql = "INSERT IGNORE INTO `{$TableSource}` (`xtime`,`keyr`,`ipaddr`,`familysite`,`servername`,`uid`,`MAC`,`size`) VALUES " . @implode(",", $content);
$q->QUERY_SQL($sql);
if (!$q->ok) {
echo $q->mysql_error . "\n";
} else {
if ($GLOBALS["VERBOSE"]) {
echo $filepath . " ({$contentSize} KB) done with " . count($content) . " elements...\n";
}
@unlink($filepath);
}
continue;
}
}
events("{$countDeFiles} Scanned files...");
$php = $unix->LOCATE_PHP5_BIN();
$nohup = $unix->find_program("nohup");
$cmd = "{$nohup} {$php} " . __FILE__ . " --tables-primaires >/dev/null 2>&1 &";
if ($GLOBALS["VERBOSE"]) {
echo $cmd . "\n";
}
shell_exec($cmd);
}
function inject_array($array)
{
$q = new mysql_squid_builder();
while (list($tablePrefix, $f) = each($array)) {
if (count($f) > 0) {
$tablename = "searchwords_{$tablePrefix}";
if ($GLOBALS["VERBOSE"]) {
echo "-> {$tablename} -> " . count($f) . "\n";
}
$q->check_SearchWords_hour($tablePrefix);
$prefix = "INSERT IGNORE INTO {$tablename} (`zmd5`,`sitename`,`zDate`,`ipaddr`,`hostname`,`uid`,`MAC`,`account`,`familysite`,`words`) VALUES " . @implode(",", $f);
$q->QUERY_SQL($prefix);
if (!$q->ok) {
writelogs_squid("{$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "stats");
@mkdir("/var/log/artica-postfix/searchwords-sql-errors", 0755, true);
@file_put_contents("/var/log/artica-postfix/searchwords-sql-errors/" . md5($prefix), $prefix);
}
}
}
}
}
$pid = $unix->get_pid_from_file($pidfile);
if ($unix->process_exists($pid)) {
die;
}
}
@unlink($pidTime);
@file_put_contents($pidTime, time());
@file_put_contents($pidfile, getmypid());
access_db();
$php5 = $unix->LOCATE_PHP5_BIN();
$xtime = time();
$q = new mysql_squid_builder();
$q->TablePrimaireHour(date("YmdH", $xtime));
$q->check_youtube_hour(date("YmdH", $xtime));
$q->check_SearchWords_hour(date("YmdH", $xtime));
$q->check_quota_hour(date("YmdH", $xtime));
$f[] = "#!/bin/sh";
$f[] = "export LC_ALL=C";
$f[] = "{$php5} " . __FILE__ . " >/dev/null 2>&1";
$f[] = "";
@file_put_contents("/etc/cron.hourly/SquidHourlyTables.sh", @implode("\n", $f));
@chmod("/etc/cron.hourly/SquidHourlyTables.sh", 0755);
unset($f);
function tests($fullpath)
{
$berekley = new parse_berekley_dbs();
$filename = basename($fullpath);
preg_match("#([0-9]+)_#", $filename, $re);
$xre = $re[1];
$xtime = $berekley->TIME_FROM_HOUR_INT($re[1]);
