function build_whitelist_dstdomain($gpid, $negation) { $q = new mysql_squid_builder(); $fam = new familysite(); $sql = "SELECT pattern FROM webfilters_sqitems WHERE gpid={$gpid} AND enabled=1"; $results = $q->QUERY_SQL($sql); $exclam = null; if (!$q->ok) { writelogs("{$gpid} {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__); return false; } if (mysql_num_rows($results) == 0) { return false; } if ($negation == 1) { $exclam = "!"; } $f = array(); while ($ligne = mysql_fetch_assoc($results)) { $pattern = trim(strtolower($ligne["pattern"])); $Family = $fam->GetFamilySites($pattern); pack_debug("Group::[{$gpid}] Item: \"{$pattern}\" -> {$Family}", __FUNCTION__, __LINE__); if (strpos(" {$pattern}", "*") > 0) { if (preg_match("#^\\^(.+)#", $ligne["pattern"], $re)) { $pattern = $re[1]; } $f[] = "\tif( shExpMatch(host ,\"{$pattern}\") ){ return \"DIRECT\";}"; continue; } if (preg_match("#^\\^(.+)#", $ligne["pattern"], $re)) { $f[] = "\tif( {$exclam}dnsDomainIs(host, \"{$re[1]}\") ){ return \"DIRECT\"; }"; continue; } if ($Family == $ligne["pattern"]) { if (!preg_match("#^\\.#", $ligne["pattern"])) { $f[] = "\tif( {$exclam}dnsDomainIs(host, \".{$ligne["pattern"]}\") ){ return \"DIRECT\"; }"; continue; } } $f[] = "\tif( {$exclam}dnsDomainIs(host, \"{$ligne["pattern"]}\") ){ return \"DIRECT\"; }"; } return @implode("\n", $f); }
function ifAlreadyDownloaded($uri) { $fam = new familysite(); $parse_url = parse_url($uri); $hostname = $parse_url["host"]; $familysite = $fam->GetFamilySites($hostname); $dbfile = "{$GLOBALS["HyperCacheStoragePath"]}/cache.db"; if (!is_file($dbfile)) { events("ifAlreadyDownloaded:: {$dbfile} no such file..."); return false; } $db_con = @dba_open($dbfile, "c", "db4"); if (!$db_con) { events("analyze:: FATAL!!!::{$dbfile}, unable to open"); return false; } if (@dba_exists($uri, $db_con)) { $array = unserialize(dba_fetch($uri, $db_con)); $filepath = $array["filepath"]; if (is_file("{$GLOBALS["HyperCacheStoragePath"]}/{$filepath}")) { $filesize = $array["filesize"]; if ($filesize == @filesize("{$GLOBALS["HyperCacheStoragePath"]}/{$filepath}")) { events("ifAlreadyDownloaded:: {$GLOBALS["HyperCacheStoragePath"]}/{$filepath} already exists"); @dba_close($db_con); return true; } } } else { events("ifAlreadyDownloaded:: {$uri} doesn't exists..."); } @dba_close($db_con); return false; }
function Postfix_Addconnection($hostname = null, $ip = null) { $time = time(); events("Addconnection: {$hostname} - > {$ip}"); $fam = new familysite(); if ($hostname == null) { $hostname = $fam->GetComputerName($ip); } $curdate = date("YmdH"); $tablename = "{$curdate}_hcnx"; $zDate = date("Y-m-d H:i:s"); $GLOBALS["CLASS_POSTFIX_SQL"]->postfix_buildhour_connections(); $domain = $fam->GetFamilySites($hostname); $zmd5 = md5("{$time}{$hostname}{$ip}"); $sql = "INSERT IGNORE INTO {$tablename} (`zmd5`,`zDate`,`hostname`,`domain`,`ipaddr`) VALUES ('{$zmd5}','{$zDate}','{$hostname}','{$domain}','{$ip}')"; events("Addconnection: QUERY_SQL"); $GLOBALS["CLASS_POSTFIX_SQL"]->QUERY_SQL($sql); }
function WhitelistedBase($url) { $db_path = "/var/log/squid/ufdbgclient.white.db"; $H = parse_url($url); $domain = $H["host"]; $fam = new familysite(); $familysite = $fam->GetFamilySites($domain); $WhitelistedBase_domain = WhitelistedBase_domain($domain); if ($WhitelistedBase_domain == 1) { return true; } $WhitelistedBase_domain = WhitelistedBase_domain($familysite); if ($WhitelistedBase_domain == 1) { return true; } if ($WhitelistedBase_domain == 2) { return false; } if (!is_file($db_path)) { if ($GLOBALS["DEBUG_WHITELIST"]) { events("WHITELIST:: {$db_path} -> no such file"); } return false; } $db_con = dba_open($db_path, "r", "db4"); if (!$db_con) { return false; } $mainkey = dba_firstkey($db_con); $domain_regex = str_replace(".", "\\.", $domain); $family_regex = str_replace(".", "\\.", $familysite); while ($mainkey != false) { $val = 0; if (trim($mainkey) == null) { $mainkey = dba_nextkey($db_con); continue; } if ($GLOBALS["DEBUG_WHITELIST"]) { events("WHITELIST:: WhitelistedBase: Checking {$mainkey} -> {$domain}"); } if (preg_match("#{$mainkey}#", $domain)) { if ($GLOBALS["DEBUG_WHITELIST"]) { events("WHITELIST:: WhitelistedBase {$mainkey} MATCH {$domain}"); } $GLOBALS["WhitelistedBase"][$domain] = true; dba_close($db_con); return true; } if ($GLOBALS["DEBUG_WHITELIST"]) { events("WHITELIST:: #{$mainkey}# NO MATCH {$domain}"); } if (preg_match("#{$mainkey}#", $familysite)) { if ($GLOBALS["DEBUG_WHITELIST"]) { events("WHITELIST:: WhitelistedBase {$mainkey} MATCH {$familysite}"); } $GLOBALS["WhitelistedBase"][$familysite] = true; dba_close($db_con); return true; } if ($GLOBALS["DEBUG_WHITELIST"]) { events("WHITELIST:: #{$mainkey}# NO MATCH {$domain}"); } $mainkey = dba_nextkey($db_con); } dba_close($db_con); if ($GLOBALS["DEBUG_WHITELIST"]) { events("WHITELIST:: Assume {$domain} FALSE"); } if ($GLOBALS["DEBUG_WHITELIST"]) { events("WHITELIST:: Assume {$familysite} FALSE"); } if (!isset($GLOBALS["WhitelistedBase"])) { $GLOBALS["WhitelistedBase"] = array(); } $CountOf = count($GLOBALS["WhitelistedBase"]); if ($GLOBALS["DEBUG_WHITELIST"]) { events("WHITELIST:: {$CountOf} domains in memory"); } if ($CountOf > 5000) { $GLOBALS["WhitelistedBase"] = array(); } $GLOBALS["WhitelistedBase"][$domain] = false; $GLOBALS["WhitelistedBase"][$familysite] = false; return false; }
function cyrus_imap_conx($service, $hostname, $ip, $user) { $time = time(); events("{$service}-connection: {$hostname} - > {$ip}"); $fam = new familysite(); if ($hostname == null) { $hostname = $fam->GetComputerName($ip); } $curdate = date("YmdH"); $tablename = "{$curdate}_hcnx"; $zDate = date("Y-m-d H:i:s"); $GLOBALS["CLASS_POSTFIX_SQL"]->postfix_buildhour_connections(); $domain = $fam->GetFamilySites($hostname); $zmd5 = md5("{$time}{$hostname}{$ip}"); $tablename = "{$curdate}_hmbx"; $sql = "INSERT IGNORE INTO `{$tablename}` (`zmd5`,`zDate`,`mbx_service`,`hostname`,`ipaddr`,`uid`,`imap_server`,`domain`)\n\tVALUES('{$zmd5}','{$zDate}','{$service}','{$hostname}','{$ip}','{$user}','{$GLOBALS["MYHOSTNAME"]}','{$domain}')"; $GLOBALS["CLASS_POSTFIX_SQL"]->QUERY_SQL($sql); }
function STATS_CNX_ACCEPT_parse($path) { if ($GLOBALS["VERBOSE"]) { echo "Parsing {$path}\n"; } $db_con = dba_open($path, "r", "db4"); if (!$db_con) { echo "DB open {$path} failed\n"; return false; } $fam = new familysite(); $SQL1 = array(); $mainkey = dba_firstkey($db_con); while ($mainkey != false) { $data = dba_fetch($mainkey, $db_con); $ARRAY = unserialize($data); if ($ARRAY["IPADDR"] == "127.0.0.1") { $mainkey = dba_nextkey($db_con); continue; } $zmd5 = md5($data); if ($ARRAY["HOSTNAME"] == null) { $ARRAY["HOSTNAME"] = $fam->GetComputerName($ARRAY["IPADDR"]); } $familysite = $fam->GetFamilySites($ARRAY["HOSTNAME"]); $SQL1[date("YmdH", $ARRAY["TIME"])][] = "('{$zmd5}','{$ARRAY["DATE"]}','{$ARRAY["HOSTNAME"]}','{$familysite}','{$ARRAY["IPADDR"]}')"; if ($GLOBALS["VERBOSE"]) { echo "('{$zmd5}','{$ARRAY["DATE"]}','{$ARRAY["HOSTNAME"]}','{$familysite}','{$ARRAY["IPADDR"]}')\n"; } $mainkey = dba_nextkey($db_con); } dba_close($db_con); if (count($SQL1) > 0) { $q = new mysql_postfix_builder(); while (list($TIMESTAMP, $rows) = each($SQL1)) { $q->postfix_buildhour_connections($TIMESTAMP); $sql = "INSERT IGNORE INTO {$TIMESTAMP}_hcnx (`zmd5`,`zDate`,`hostname`,`domain`,`ipaddr`) VALUES " . @implode(",", $rows); $q->QUERY_SQL($sql); if (!$q->ok) { echo $q->mysql_error; return false; } } } if ($GLOBALS["VERBOSE"]) { echo "Parsing {$path} END\n"; } return true; }
function connect_from($logpath) { $unix = new unix(); $q = new mysql(); $q->QUERY_SQL("CREATE TABLE IF NOT EXISTS `smtpstats_day` (\n\t`zmd5` VARCHAR(90) NOT NULL PRIMARY KEY,\n\t`zDate` DATETIME,\n\t`domain` VARCHAR(128),\n\t`GREY` BIGINT UNSIGNED,\n\t`BLACK` BIGINT UNSIGNED,\n\t`CNX` BIGINT UNSIGNED,\n\t`HOSTS` BIGINT UNSIGNED,\n\t`IPS` BIGINT UNSIGNED,\n\t`INFOS` TINYTEXT,\n\tKEY `zDate` (`zDate`),\n\tKEY `domain` (`domain`),\n\tKEY `GREY` (`GREY`),\n\tKEY `BLACK` (`BLACK`),\n\tKEY `CNX` (`CNX`),\n\tKEY `IPS` (`IPS`),\n\tKEY `HOSTS` (`HOSTS`)\n\t) ENGINE=MYISAM;", "artica_events"); if (!$q->ok) { echo $q->mysql_error . "\n"; return; } $q->QUERY_SQL("CREATE TABLE IF NOT EXISTS `smtpcdir_day` (\n\t`zmd5` VARCHAR(90) NOT NULL PRIMARY KEY,\n\t`zDate` DATETIME,\n\t`CDIR` VARCHAR(90),\n\t`GREY` BIGINT UNSIGNED,\n\t`BLACK` BIGINT UNSIGNED,\n\t`CNX` BIGINT UNSIGNED,\n\t`HOSTS` BIGINT UNSIGNED,\n\t`DOMAINS` BIGINT UNSIGNED,\n\t`INFOS` TINYTEXT,\n\tKEY `zDate` (`zDate`),\n\tKEY `DOMAINS` (`DOMAINS`),\n\tKEY `GREY` (`GREY`),\n\tKEY `BLACK` (`BLACK`),\n\tKEY `CNX` (`CNX`),\n\tKEY `CDIR` (`CDIR`),\n\tKEY `HOSTS` (`HOSTS`)\n\t) ENGINE=MYISAM;", "artica_events"); if (!$q->ok) { echo $q->mysql_error . "\n"; return; } $grep = $unix->find_program("grep"); $tmpfile = $unix->FILE_TEMP(); shell_exec("{$grep} -e \"smtpd.*: connect from\" {$logpath} >{$tmpfile}"); $fp = @fopen($tmpfile, "r"); if (!$fp) { return false; } $t = array(); $fam = new familysite(); while (!feof($fp)) { $line = trim(fgets($fp, 4096)); $line = str_replace("\r\n", "", $line); $line = str_replace("\n", "", $line); $line = str_replace("\r", "", $line); $line = trim($line); if (!preg_match("#^(.+?)\\s+([0-9]+)\\s+([0-9:]+)\\s+.*?\\[[0-9]+\\]:\\s+connect from\\s+(.+?)\\[([0-9\\.]+)\\]#", $line, $re)) { continue; } $date = strtotime("{$re[1]} {$re[2]} {$re[3]}"); $ipaddr = $re[5]; $day = date("Y-m-d", $date); $NETZ = explode(".", $ipaddr); $network = "{$NETZ[0]}.{$NETZ[1]}.{$NETZ[2]}.0/24"; $hostname = $re[4]; $familysite = $fam->GetFamilySites($hostname); if (!isset($MAINNETS[$day][$network]["CNX"])) { $MAINNETS[$day][$network]["CNX"] = 1; } else { $MAINNETS[$day][$network]["CNX"] = $MAINNETS[$day][$network]["CNX"] + 1; } if (!isset($MAINNETS[$day][$network]["FAM"][$familysite])) { $MAINNETS[$day][$network]["FAM"][$familysite] = 1; } else { $MAINNETS[$day][$network]["FAM"][$familysite] = $MAINNETS[$day][$network]["FAM"][$familysite] + 1; } if (!isset($MAIN[$day][$familysite]["IPS"][$ipaddr])) { $MAIN[$day][$familysite]["IPS"][$ipaddr] = 1; } else { $MAIN[$day][$familysite]["IPS"][$ipaddr] = $MAIN[$day][$familysite]["IPS"][$ipaddr] + 1; } if (!isset($MAIN[$day][$familysite]["COUNT"])) { $MAIN[$day][$familysite]["COUNT"] = 1; } else { $MAIN[$day][$familysite]["COUNT"] = $MAIN[$day][$familysite]["COUNT"] + 1; } if (!isset($MAIN[$day][$familysite]["HOSTS"][$hostname])) { $MAIN[$day][$familysite]["HOSTS"][$hostname] = 1; } else { $MAIN[$day][$familysite]["HOSTS"][$hostname] = $MAIN[$day][$familysite]["HOSTS"][$hostname] + 1; } //echo date("Y-m-d")." $hostname $ipaddr\n"; } @fclose($fp); @unlink($tmpfile); shell_exec("{$grep} -e \"NOQUEUE: milter-reject: RCPT from\" {$logpath} >{$tmpfile}"); $fp = @fopen($tmpfile, "r"); if (!$fp) { return false; } while (!feof($fp)) { $line = trim(fgets($fp, 4096)); $line = str_replace("\r\n", "", $line); $line = str_replace("\n", "", $line); $line = str_replace("\r", "", $line); $line = trim($line); if (!preg_match("#^(.+?)\\s+([0-9]+)\\s+([0-9:]+)\\s+.*?\\[[0-9]+\\]:\\s+NOQUEUE: milter-reject: RCPT from\\s+(.*?)\\[([0-9\\.]+)\\]:\\s+([0-9]+)\\s+#", $line, $re)) { echo "NO MATCH {$line}\n"; continue; } $date = strtotime("{$re[1]} {$re[2]} {$re[3]}"); $hostname = $re[4]; $ipaddr = $re[5]; $CODE = $re[6]; $day = date("Y-m-d", $date); $familysite = $fam->GetFamilySites($hostname); $NETZ = explode(".", $ipaddr); $network = "{$NETZ[0]}.{$NETZ[1]}.{$NETZ[2]}.0/24"; if (!isset($MAINNETS[$day][$network]["FAM"][$familysite])) { $MAINNETS[$day][$network]["FAM"][$familysite] = 1; } else { $MAINNETS[$day][$network]["FAM"][$familysite] = $MAINNETS[$day][$network]["FAM"][$familysite] + 1; } if ($CODE == 451) { if (!isset($MAINNETS[$day][$network]["GREY"])) { $MAINNETS[$day][$network]["GREY"] = 1; } else { $MAINNETS[$day][$network]["GREY"] = $MAINNETS[$day][$network]["GREY"] + 1; } if (!isset($MAIN[$day][$familysite]["GREY"])) { $MAIN[$day][$familysite]["GREY"] = 1; } else { $MAIN[$day][$familysite]["GREY"] = $MAIN[$day][$familysite]["GREY"] + 1; } } if ($CODE == 551) { if (!isset($MAIN[$day][$familysite]["BLACK"])) { $MAIN[$day][$familysite]["BLACK"] = 1; } else { $MAIN[$day][$familysite]["BLACK"] = $MAIN[$day][$familysite]["BLACK"] + 1; } if (!isset($MAINNETS[$day][$network]["BLACK"])) { $MAINNETS[$day][$network]["BLACK"] = 1; } else { $MAINNETS[$day][$network]["BLACK"] = $MAINNETS[$day][$network]["BLACK"] + 1; } } } @fclose($fp); @unlink($tmpfile); shell_exec("{$grep} -e \"NOQUEUE: reject: RCPT from\" {$logpath} >{$tmpfile}"); $fp = @fopen($tmpfile, "r"); if (!$fp) { return false; } while (!feof($fp)) { $line = trim(fgets($fp, 4096)); $line = str_replace("\r\n", "", $line); $line = str_replace("\n", "", $line); $line = str_replace("\r", "", $line); $line = trim($line); if (!preg_match("#^(.+?)\\s+([0-9]+)\\s+([0-9:]+)\\s+.*?\\[[0-9]+\\]:\\s+NOQUEUE: reject: RCPT from\\s+(.*?)\\[([0-9\\.]+)\\]:\\s+([0-9]+)\\s+#", $line, $re)) { echo "NO MATCH {$line}\n"; continue; } $date = strtotime("{$re[1]} {$re[2]} {$re[3]}"); $hostname = $re[4]; $ipaddr = $re[5]; $CODE = $re[6]; $day = date("Y-m-d", $date); $familysite = $fam->GetFamilySites($hostname); $NETZ = explode(".", $ipaddr); $network = "{$NETZ[0]}.{$NETZ[1]}.{$NETZ[2]}.0/24"; if ($CODE == 551 or $CODE == 554) { if (!isset($MAIN[$day][$familysite]["BLACK"])) { $MAIN[$day][$familysite]["BLACK"] = 1; } else { $MAIN[$day][$familysite]["BLACK"] = $MAIN[$day][$familysite]["BLACK"] + 1; } if (!isset($MAINNETS[$day][$network]["BLACK"])) { $MAINNETS[$day][$network]["BLACK"] = 1; } else { $MAINNETS[$day][$network]["BLACK"] = $MAINNETS[$day][$network]["BLACK"] + 1; } } } @fclose($fp); @unlink($tmpfile); $prefix = "INSERT IGNORE INTO smtpstats_day (`zmd5`,`zDate`,`domain`,`GREY`,`BLACK`,`CNX`,`HOSTS`,`IPS`,`INFOS`) VALUES "; while (list($zDate, $ARRAY) = each($MAIN)) { while (list($domain, $INFOS) = each($ARRAY)) { $GREY = 0; if (!isset($INFOS["BLACK"])) { $INFOS["BLACK"] = 0; } if (!isset($INFOS["GREY"])) { $INFOS["GREY"] = 0; } $HOSTS = count($INFOS["HOSTS"]); $IPS = count($INFOS["IPS"]); $BLACK = intval($INFOS["BLACK"]); $CNX = intval($INFOS["COUNT"]); $INFO["IPS"] = $INFOS["IPS"]; $INFO["HOSTS"] = $INFOS["HOSTS"]; $infotext = mysql_escape_string2(serialize($INFO)); if ($GLOBALS["VERBOSE"]) { echo "{$zDate}: {$domain} hosts:{$HOSTS} ips:{$IPS} blacklisted:{$BLACK} greylisted:{$GREY} cnx:{$CNX} {$infotext}\n"; } $md5 = md5("{$zDate}{$domain}{$HOSTS}{$IPS}{$BLACK}{$GREY}{$CNX}{$infotext}"); $f[] = "('{$md5}','{$zDate}','{$domain}','{$GREY}','{$BLACK}','{$CNX}','{$HOSTS}','{$IPS}','{$infotext}')"; if (count($f) > 500) { $q->QUERY_SQL($prefix . @implode(",", $f), "artica_events"); if (!$q->ok) { echo $q->mysql_error . "\n"; return; } $f = array(); } } } if (count($f) > 0) { $q->QUERY_SQL($prefix . @implode(",", $f), "artica_events"); if (!$q->ok) { echo $q->mysql_error . "\n"; return; } $f = array(); } $q->QUERY_SQL("CREATE TABLE IF NOT EXISTS `smtpcdir_day` (\n\t`zmd5` VARCHAR(90) NOT NULL PRIMARY KEY,\n\t`zDate` DATETIME,\n\t`CDIR` VARCHAR(90),\n\t`GREY` BIGINT UNSIGNED,\n\t`BLACK` BIGINT UNSIGNED,\n\t`CNX` BIGINT UNSIGNED,\n\t`HOSTS` BIGINT UNSIGNED,\n\t`DOMAINS` BIGINT UNSIGNED,\n\t`INFOS` TINYTEXT,\n\tKEY `zDate` (`zDate`),\n\tKEY `DOMAINS` (`DOMAINS`),\n\tKEY `GREY` (`GREY`),\n\tKEY `BLACK` (`BLACK`),\n\tKEY `CNX` (`CNX`),\n\tKEY `CDIR` (`CDIR`),\n\tKEY `HOSTS` (`HOSTS`)\n\t) ENGINE=MYISAM;", "artica_events"); if (!$q->ok) { echo $q->mysql_error . "\n"; return; } $prefix = "INSERT IGNORE INTO `smtpcdir_day` (`zmd5`,`zDate`,`CDIR`,`GREY`,`BLACK`,`CNX`,`DOMAINS`,`INFOS`) VALUES "; while (list($zDate, $ARRAY) = each($MAINNETS)) { while (list($CDIR, $INFOS) = each($ARRAY)) { if (!isset($INFOS["BLACK"])) { $INFOS["BLACK"] = 0; } if (!isset($INFOS["GREY"])) { $INFOS["GREY"] = 0; } $CNX = intval($INFOS["CNX"]); $GREY = intval($INFOS["GREY"]); $BLACK = intval($INFOS["BLACK"]); $DOMAINS = intval($INFOS["FAM"]); $infotext = mysql_escape_string2(serialize($INFOS["FAM"])); echo "{$zDate} {$CDIR} cnx:{$CNX} greylisted:{$GREY} blacklisted:{$BLACK} domains:{$DOMAINS}\n"; $md5 = md5("{$zDate}{$CDIR}{$DOMAINS}{$BLACK}{$GREY}{$CNX}{$infotext}"); $f[] = "('{$md5}','{$zDate}','{$CDIR}','{$GREY}','{$BLACK}','{$CNX}','{$DOMAINS}','{$infotext}')"; if (count($f) > 500) { $q->QUERY_SQL($prefix . @implode(",", $f), "artica_events"); if (!$q->ok) { echo $q->mysql_error . "\n"; return; } $f = array(); } } } if (count($f) > 0) { $q->QUERY_SQL($prefix . @implode(",", $f), "artica_events"); if (!$q->ok) { echo $q->mysql_error . "\n"; return; } $f = array(); } return true; //print_r($MAINNETS); }
function tool_get_familysite($uri){ $parse_url=parse_url($uri); $sitename=$parse_url["host"]; if(isset($GLOBALS["FAMILYSITES"][$sitename])){return $GLOBALS["FAMILYSITES"][$sitename];} $f=new familysite(); $GLOBALS["FAMILYSITES"][$sitename]=$f->GetFamilySites($sitename); return $GLOBALS["FAMILYSITES"][$sitename]; }
function xstart() { $unix = new unix(); $filetime = "/etc/artica-postfix/pids/exec.mvps.php.time"; if ($unix->file_time_min($filetime) < 240) { echo "Need 240mn, current is {$filetime}Mn\n"; return; } @unlink($filetime); @file_put_contents($filetime, time()); $q = new mysql_squid_builder(); $curl = new ccurl("http://winhelp2002.mvps.org/hosts.txt"); $targetpath = $unix->FILE_TEMP(); if (!$curl->GetFile($targetpath)) { squid_admin_mysql(1, "Unable to download hosts.txt from winhelp2002.mvps.org", null, __FILE__, __LINE__); return; } $f = explode("\n", @file_get_contents($targetpath)); @unlink($targetpath); $fam = new familysite(); $sql = "CREATE TABLE IF NOT EXISTS `squidlogs`.`ads_domains` (\n\t\t\t`servername` VARCHAR(255) PRIMARY KEY,\n\t\t\t`enabled` smallint(1) NOT NULL DEFAULT 1,\n\t\t\t KEY `enabled`(`enabled`)\n\t\t) ENGINE=MYISAM;"; $q->QUERY_SQL($sql); if (!$q->ok) { echo $q->mysql_error; return; } $COUNT1 = $q->COUNT_ROWS("ads_domains"); $QQR = array(); while (list($a, $line) = each($f)) { $line = trim($line); if (strpos($line, "localhost") > 0) { continue; } if (substr($line, 0, 1) == "#") { continue; } $line = str_replace("0.0.0.0 ", "", $line); if (strpos($line, "#") > 0) { $FI = explode("#", $line); $line = $FI[0]; } if (strpos($line, ".") == 0) { continue; } $line = trim($line); $familysite = $fam->GetFamilySites($line); if ($line == $familysite) { $line = ".{$line}"; } $QQR[] = "('{$line}','1')"; if (count($QQR) > 500) { $sql = "INSERT IGNORE INTO `ads_domains` (`servername`,`enabled`) VALUES " . @implode(",", $QQR); $q->QUERY_SQL($sql); $QQR = array(); } } if (count($QQR) > 0) { $sql = "INSERT IGNORE INTO `ads_domains` (`servername`,`enabled`) VALUES " . @implode(",", $QQR); $q->QUERY_SQL($sql); $QQR = array(); } $curl = new ccurl("http://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext"); $targetpath = $unix->FILE_TEMP(); if (!$curl->GetFile($targetpath)) { squid_admin_mysql(1, "Unable to download serverlist from yoyo.org", null, __FILE__, __LINE__); return; } $f = explode("\n", @file_get_contents($targetpath)); @unlink($targetpath); while (list($a, $line) = each($f)) { $line = trim($line); if (strpos($line, "localhost") > 0) { continue; } if (substr($line, 0, 1) == "#") { continue; } $line = str_replace("127.0.0.1 ", "", $line); if (strpos($line, "#") > 0) { $FI = explode("#", $line); $line = $FI[0]; } if (strpos($line, ".") == 0) { continue; } $line = trim($line); $QQR[] = "('{$line}','1')"; $familysite = $fam->GetFamilySites($line); if ($line == $familysite) { $line = ".{$line}"; } if (count($QQR) > 500) { $sql = "INSERT IGNORE INTO `ads_domains` (`servername`,`enabled`) VALUES " . @implode(",", $QQR); $q->QUERY_SQL($sql); $QQR = array(); } } if (count($QQR) > 0) { $sql = "INSERT IGNORE INTO `ads_domains` (`servername`,`enabled`) VALUES " . @implode(",", $QQR); $q->QUERY_SQL($sql); $QQR = array(); } $COUNT2 = $q->COUNT_ROWS("ads_domains"); if ($COUNT2 > $COUNT1) { $TOTAL = $COUNT2 - $COUNT1; squid_admin_mysql(1, "{$TOTAL} ads and tracker added in ACLs", null, __FILE__, __LINE__); } }
function connect_from($logpath) { $unix = new unix(); smtpstats_day_migrate_to_postgres(); smtpcdir_day_migrate_to_postgres(); smtpsum_day_migrate_to_postgres(); smtpgraph_day_migrate_to_postgres(); smtpdeliver_day_migrate_to_postgres(); smtpsenders_day_migrate_to_postgres(); smtprecipients_day_migrate_to_postgres(); $q = new postgres_sql(); $q->SMTP_TABLES(); $grep = $unix->find_program("grep"); $tmpfile = $unix->FILE_TEMP(); shell_exec("{$grep} -e \"smtpd.*: connect from\" {$logpath} >{$tmpfile}"); $fp = @fopen($tmpfile, "r"); if (!$fp) { return false; } $t = array(); $fam = new familysite(); while (!feof($fp)) { $line = trim(fgets($fp, 4096)); $line = str_replace("\r\n", "", $line); $line = str_replace("\n", "", $line); $line = str_replace("\r", "", $line); $line = trim($line); if (!preg_match("#^(.+?)\\s+([0-9]+)\\s+([0-9:]+)\\s+.*?\\[[0-9]+\\]:\\s+connect from\\s+(.+?)\\[([0-9\\.]+)\\]#", $line, $re)) { continue; } $date = strtotime("{$re[1]} {$re[2]} {$re[3]}"); $ipaddr = $re[5]; $day = date("Y-m-d", $date); $NETZ = explode(".", $ipaddr); $network = "{$NETZ[0]}.{$NETZ[1]}.{$NETZ[2]}.0/24"; $hostname = $re[4]; $familysite = $fam->GetFamilySites($hostname); if (!isset($MAINNETS[$day][$network]["CNX"])) { $MAINNETS[$day][$network]["CNX"] = 1; } else { $MAINNETS[$day][$network]["CNX"] = $MAINNETS[$day][$network]["CNX"] + 1; } if (!isset($MAINNETS[$day][$network]["FAM"][$familysite])) { $MAINNETS[$day][$network]["FAM"][$familysite] = 1; } else { $MAINNETS[$day][$network]["FAM"][$familysite] = $MAINNETS[$day][$network]["FAM"][$familysite] + 1; } if (!isset($MAIN[$day][$familysite]["IPS"][$ipaddr])) { $MAIN[$day][$familysite]["IPS"][$ipaddr] = 1; } else { $MAIN[$day][$familysite]["IPS"][$ipaddr] = $MAIN[$day][$familysite]["IPS"][$ipaddr] + 1; } if (!isset($MAIN[$day][$familysite]["COUNT"])) { $MAIN[$day][$familysite]["COUNT"] = 1; } else { $MAIN[$day][$familysite]["COUNT"] = $MAIN[$day][$familysite]["COUNT"] + 1; } if (!isset($MAIN[$day][$familysite]["HOSTS"][$hostname])) { $MAIN[$day][$familysite]["HOSTS"][$hostname] = 1; } else { $MAIN[$day][$familysite]["HOSTS"][$hostname] = $MAIN[$day][$familysite]["HOSTS"][$hostname] + 1; } //echo date("Y-m-d")." $hostname $ipaddr\n"; } @fclose($fp); @unlink($tmpfile); shell_exec("{$grep} -e \"NOQUEUE: milter-reject: RCPT from\" {$logpath} >{$tmpfile}"); $fp = @fopen($tmpfile, "r"); if (!$fp) { return false; } while (!feof($fp)) { $line = trim(fgets($fp, 4096)); $line = str_replace("\r\n", "", $line); $line = str_replace("\n", "", $line); $line = str_replace("\r", "", $line); $line = trim($line); if (!preg_match("#^(.+?)\\s+([0-9]+)\\s+([0-9:]+)\\s+.*?\\[[0-9]+\\]:\\s+NOQUEUE: milter-reject: RCPT from\\s+(.*?)\\[([0-9\\.]+)\\]:\\s+([0-9]+)\\s+#", $line, $re)) { echo "NO MATCH {$line}\n"; continue; } $date = strtotime("{$re[1]} {$re[2]} {$re[3]}"); $hostname = $re[4]; $ipaddr = $re[5]; $CODE = $re[6]; $day = date("Y-m-d", $date); $familysite = $fam->GetFamilySites($hostname); $NETZ = explode(".", $ipaddr); $network = "{$NETZ[0]}.{$NETZ[1]}.{$NETZ[2]}.0/24"; if (!isset($MAINNETS[$day][$network]["FAM"][$familysite])) { $MAINNETS[$day][$network]["FAM"][$familysite] = 1; } else { $MAINNETS[$day][$network]["FAM"][$familysite] = $MAINNETS[$day][$network]["FAM"][$familysite] + 1; } if ($CODE == 451) { if (!isset($MAINNETS[$day][$network]["GREY"])) { $MAINNETS[$day][$network]["GREY"] = 1; } else { $MAINNETS[$day][$network]["GREY"] = $MAINNETS[$day][$network]["GREY"] + 1; } if (!isset($MAIN[$day][$familysite]["GREY"])) { $MAIN[$day][$familysite]["GREY"] = 1; } else { $MAIN[$day][$familysite]["GREY"] = $MAIN[$day][$familysite]["GREY"] + 1; } } if ($CODE == 551) { if (!isset($MAIN[$day][$familysite]["BLACK"])) { $MAIN[$day][$familysite]["BLACK"] = 1; } else { $MAIN[$day][$familysite]["BLACK"] = $MAIN[$day][$familysite]["BLACK"] + 1; } if (!isset($MAINNETS[$day][$network]["BLACK"])) { $MAINNETS[$day][$network]["BLACK"] = 1; } else { $MAINNETS[$day][$network]["BLACK"] = $MAINNETS[$day][$network]["BLACK"] + 1; } } } @fclose($fp); @unlink($tmpfile); shell_exec("{$grep} -e \"NOQUEUE: reject: RCPT from\" {$logpath} >{$tmpfile}"); $fp = @fopen($tmpfile, "r"); if (!$fp) { return false; } while (!feof($fp)) { $line = trim(fgets($fp, 4096)); $line = str_replace("\r\n", "", $line); $line = str_replace("\n", "", $line); $line = str_replace("\r", "", $line); $line = trim($line); if (!preg_match("#^(.+?)\\s+([0-9]+)\\s+([0-9:]+)\\s+.*?\\[[0-9]+\\]:\\s+NOQUEUE: reject: RCPT from\\s+(.*?)\\[([0-9\\.]+)\\]:\\s+([0-9]+)\\s+#", $line, $re)) { echo "NO MATCH {$line}\n"; continue; } $date = strtotime("{$re[1]} {$re[2]} {$re[3]}"); $hostname = $re[4]; $ipaddr = $re[5]; $CODE = $re[6]; $day = date("Y-m-d", $date); $familysite = $fam->GetFamilySites($hostname); $NETZ = explode(".", $ipaddr); $network = "{$NETZ[0]}.{$NETZ[1]}.{$NETZ[2]}.0/24"; if ($CODE == 551 or $CODE == 554) { if (!isset($MAIN[$day][$familysite]["BLACK"])) { $MAIN[$day][$familysite]["BLACK"] = 1; } else { $MAIN[$day][$familysite]["BLACK"] = $MAIN[$day][$familysite]["BLACK"] + 1; } if (!isset($MAINNETS[$day][$network]["BLACK"])) { $MAINNETS[$day][$network]["BLACK"] = 1; } else { $MAINNETS[$day][$network]["BLACK"] = $MAINNETS[$day][$network]["BLACK"] + 1; } } } @fclose($fp); @unlink($tmpfile); $prefix = "INSERT INTO smtpstats_day (zmd5,zdate,domain,grey,black,cnx,hosts,ips,infos) VALUES "; $q = new postgres_sql(); while (list($zDate, $ARRAY) = each($MAIN)) { while (list($domain, $INFOS) = each($ARRAY)) { $GREY = 0; if (!isset($INFOS["BLACK"])) { $INFOS["BLACK"] = 0; } if (!isset($INFOS["GREY"])) { $INFOS["GREY"] = 0; } $HOSTS = count($INFOS["HOSTS"]); $IPS = count($INFOS["IPS"]); $BLACK = intval($INFOS["BLACK"]); $CNX = intval($INFOS["COUNT"]); $INFO["IPS"] = $INFOS["IPS"]; $INFO["HOSTS"] = $INFOS["HOSTS"]; $infotext = mysql_escape_string2(serialize($INFO)); if ($GLOBALS["VERBOSE"]) { echo "{$zDate}: {$domain} hosts:{$HOSTS} ips:{$IPS} blacklisted:{$BLACK} greylisted:{$GREY} cnx:{$CNX} {$infotext}\n"; } $md5 = md5("{$zDate}{$domain}{$HOSTS}{$IPS}{$BLACK}{$GREY}{$CNX}{$infotext}"); $f[] = "('{$md5}','{$zDate}','{$domain}','{$GREY}','{$BLACK}','{$CNX}','{$HOSTS}','{$IPS}','{$infotext}')"; if (count($f) > 500) { $q->QUERY_SQL($prefix . @implode(",", $f) . " ON CONFLICT DO NOTHING"); if (!$q->ok) { echo $q->mysql_error . "\n"; return; } $f = array(); } } } if (count($f) > 0) { $q->QUERY_SQL($prefix . @implode(",", $f), "artica_events"); if (!$q->ok) { echo $q->mysql_error . "\n"; return; } $f = array(); } $prefix = "INSERT INTO smtpcdir_day (zmd5,zdate,cdir,grey,black,cnx,domains,infos) VALUES "; $q = new postgres_sql(); $q->SMTP_TABLES(); while (list($zDate, $ARRAY) = each($MAINNETS)) { while (list($CDIR, $INFOS) = each($ARRAY)) { if (!isset($INFOS["BLACK"])) { $INFOS["BLACK"] = 0; } if (!isset($INFOS["GREY"])) { $INFOS["GREY"] = 0; } $CNX = intval($INFOS["CNX"]); $GREY = intval($INFOS["GREY"]); $BLACK = intval($INFOS["BLACK"]); $DOMAINS = intval($INFOS["FAM"]); $infotext = mysql_escape_string2(serialize($INFOS["FAM"])); echo "{$zDate} {$CDIR} cnx:{$CNX} greylisted:{$GREY} blacklisted:{$BLACK} domains:{$DOMAINS}\n"; $md5 = md5("{$zDate}{$CDIR}{$DOMAINS}{$BLACK}{$GREY}{$CNX}{$infotext}"); $f[] = "('{$md5}','{$zDate}','{$CDIR}','{$GREY}','{$BLACK}','{$CNX}','{$DOMAINS}','{$infotext}')"; if (count($f) > 500) { $q->QUERY_SQL($prefix . @implode(",", $f) . " ON CONFLICT DO NOTHING"); if (!$q->ok) { echo $q->mysql_error . "\n"; return; } $f = array(); } } } if (count($f) > 0) { $q->QUERY_SQL($prefix . @implode(",", $f), "artica_events"); if (!$q->ok) { echo $q->mysql_error . "\n"; return; } $f = array(); } return true; //print_r($MAINNETS); }