function build_whitelist_dstdomain($gpid, $negation)
{
$q = new mysql_squid_builder();
$fam = new familysite();
$sql = "SELECT pattern FROM webfilters_sqitems WHERE gpid={$gpid} AND enabled=1";
$results = $q->QUERY_SQL($sql);
$exclam = null;
if (!$q->ok) {
writelogs("{$gpid} {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__);
return false;
}
if (mysql_num_rows($results) == 0) {
return false;
}
if ($negation == 1) {
$exclam = "!";
}
$f = array();
while ($ligne = mysql_fetch_assoc($results)) {
$pattern = trim(strtolower($ligne["pattern"]));
$Family = $fam->GetFamilySites($pattern);
pack_debug("Group::[{$gpid}] Item: \"{$pattern}\" -> {$Family}", __FUNCTION__, __LINE__);
if (strpos(" {$pattern}", "*") > 0) {
if (preg_match("#^\\^(.+)#", $ligne["pattern"], $re)) {
$pattern = $re[1];
}
$f[] = "\tif( shExpMatch(host ,\"{$pattern}\") ){ return \"DIRECT\";}";
continue;
}
if (preg_match("#^\\^(.+)#", $ligne["pattern"], $re)) {
$f[] = "\tif( {$exclam}dnsDomainIs(host, \"{$re[1]}\") ){ return \"DIRECT\"; }";
continue;
}
if ($Family == $ligne["pattern"]) {
if (!preg_match("#^\\.#", $ligne["pattern"])) {
$f[] = "\tif( {$exclam}dnsDomainIs(host, \".{$ligne["pattern"]}\") ){ return \"DIRECT\"; }";
continue;
}
}
$f[] = "\tif( {$exclam}dnsDomainIs(host, \"{$ligne["pattern"]}\") ){ return \"DIRECT\"; }";
}
return @implode("\n", $f);
}
function ifAlreadyDownloaded($uri)
{
$fam = new familysite();
$parse_url = parse_url($uri);
$hostname = $parse_url["host"];
$familysite = $fam->GetFamilySites($hostname);
$dbfile = "{$GLOBALS["HyperCacheStoragePath"]}/cache.db";
if (!is_file($dbfile)) {
events("ifAlreadyDownloaded:: {$dbfile} no such file...");
return false;
}
$db_con = @dba_open($dbfile, "c", "db4");
if (!$db_con) {
events("analyze:: FATAL!!!::{$dbfile}, unable to open");
return false;
}
if (@dba_exists($uri, $db_con)) {
$array = unserialize(dba_fetch($uri, $db_con));
$filepath = $array["filepath"];
if (is_file("{$GLOBALS["HyperCacheStoragePath"]}/{$filepath}")) {
$filesize = $array["filesize"];
if ($filesize == @filesize("{$GLOBALS["HyperCacheStoragePath"]}/{$filepath}")) {
events("ifAlreadyDownloaded:: {$GLOBALS["HyperCacheStoragePath"]}/{$filepath} already exists");
@dba_close($db_con);
return true;
}
}
} else {
events("ifAlreadyDownloaded:: {$uri} doesn't exists...");
}
@dba_close($db_con);
return false;
}
function Postfix_Addconnection($hostname = null, $ip = null)
{
$time = time();
events("Addconnection: {$hostname} - > {$ip}");
$fam = new familysite();
if ($hostname == null) {
$hostname = $fam->GetComputerName($ip);
}
$curdate = date("YmdH");
$tablename = "{$curdate}_hcnx";
$zDate = date("Y-m-d H:i:s");
$GLOBALS["CLASS_POSTFIX_SQL"]->postfix_buildhour_connections();
$domain = $fam->GetFamilySites($hostname);
$zmd5 = md5("{$time}{$hostname}{$ip}");
$sql = "INSERT IGNORE INTO {$tablename} (`zmd5`,`zDate`,`hostname`,`domain`,`ipaddr`) VALUES ('{$zmd5}','{$zDate}','{$hostname}','{$domain}','{$ip}')";
events("Addconnection: QUERY_SQL");
$GLOBALS["CLASS_POSTFIX_SQL"]->QUERY_SQL($sql);
}
function WhitelistedBase($url)
{
$db_path = "/var/log/squid/ufdbgclient.white.db";
$H = parse_url($url);
$domain = $H["host"];
$fam = new familysite();
$familysite = $fam->GetFamilySites($domain);
$WhitelistedBase_domain = WhitelistedBase_domain($domain);
if ($WhitelistedBase_domain == 1) {
return true;
}
$WhitelistedBase_domain = WhitelistedBase_domain($familysite);
if ($WhitelistedBase_domain == 1) {
return true;
}
if ($WhitelistedBase_domain == 2) {
return false;
}
if (!is_file($db_path)) {
if ($GLOBALS["DEBUG_WHITELIST"]) {
events("WHITELIST:: {$db_path} -> no such file");
}
return false;
}
$db_con = dba_open($db_path, "r", "db4");
if (!$db_con) {
return false;
}
$mainkey = dba_firstkey($db_con);
$domain_regex = str_replace(".", "\\.", $domain);
$family_regex = str_replace(".", "\\.", $familysite);
while ($mainkey != false) {
$val = 0;
if (trim($mainkey) == null) {
$mainkey = dba_nextkey($db_con);
continue;
}
if ($GLOBALS["DEBUG_WHITELIST"]) {
events("WHITELIST:: WhitelistedBase: Checking {$mainkey} -> {$domain}");
}
if (preg_match("#{$mainkey}#", $domain)) {
if ($GLOBALS["DEBUG_WHITELIST"]) {
events("WHITELIST:: WhitelistedBase {$mainkey} MATCH {$domain}");
}
$GLOBALS["WhitelistedBase"][$domain] = true;
dba_close($db_con);
return true;
}
if ($GLOBALS["DEBUG_WHITELIST"]) {
events("WHITELIST:: #{$mainkey}# NO MATCH {$domain}");
}
if (preg_match("#{$mainkey}#", $familysite)) {
if ($GLOBALS["DEBUG_WHITELIST"]) {
events("WHITELIST:: WhitelistedBase {$mainkey} MATCH {$familysite}");
}
$GLOBALS["WhitelistedBase"][$familysite] = true;
dba_close($db_con);
return true;
}
if ($GLOBALS["DEBUG_WHITELIST"]) {
events("WHITELIST:: #{$mainkey}# NO MATCH {$domain}");
}
$mainkey = dba_nextkey($db_con);
}
dba_close($db_con);
if ($GLOBALS["DEBUG_WHITELIST"]) {
events("WHITELIST:: Assume {$domain} FALSE");
}
if ($GLOBALS["DEBUG_WHITELIST"]) {
events("WHITELIST:: Assume {$familysite} FALSE");
}
if (!isset($GLOBALS["WhitelistedBase"])) {
$GLOBALS["WhitelistedBase"] = array();
}
$CountOf = count($GLOBALS["WhitelistedBase"]);
if ($GLOBALS["DEBUG_WHITELIST"]) {
events("WHITELIST:: {$CountOf} domains in memory");
}
if ($CountOf > 5000) {
$GLOBALS["WhitelistedBase"] = array();
}
$GLOBALS["WhitelistedBase"][$domain] = false;
$GLOBALS["WhitelistedBase"][$familysite] = false;
return false;
}
function cyrus_imap_conx($service, $hostname, $ip, $user)
{
$time = time();
events("{$service}-connection: {$hostname} - > {$ip}");
$fam = new familysite();
if ($hostname == null) {
$hostname = $fam->GetComputerName($ip);
}
$curdate = date("YmdH");
$tablename = "{$curdate}_hcnx";
$zDate = date("Y-m-d H:i:s");
$GLOBALS["CLASS_POSTFIX_SQL"]->postfix_buildhour_connections();
$domain = $fam->GetFamilySites($hostname);
$zmd5 = md5("{$time}{$hostname}{$ip}");
$tablename = "{$curdate}_hmbx";
$sql = "INSERT IGNORE INTO `{$tablename}` (`zmd5`,`zDate`,`mbx_service`,`hostname`,`ipaddr`,`uid`,`imap_server`,`domain`)\n\tVALUES('{$zmd5}','{$zDate}','{$service}','{$hostname}','{$ip}','{$user}','{$GLOBALS["MYHOSTNAME"]}','{$domain}')";
$GLOBALS["CLASS_POSTFIX_SQL"]->QUERY_SQL($sql);
}
function STATS_CNX_ACCEPT_parse($path)
{
if ($GLOBALS["VERBOSE"]) {
echo "Parsing {$path}\n";
}
$db_con = dba_open($path, "r", "db4");
if (!$db_con) {
echo "DB open {$path} failed\n";
return false;
}
$fam = new familysite();
$SQL1 = array();
$mainkey = dba_firstkey($db_con);
while ($mainkey != false) {
$data = dba_fetch($mainkey, $db_con);
$ARRAY = unserialize($data);
if ($ARRAY["IPADDR"] == "127.0.0.1") {
$mainkey = dba_nextkey($db_con);
continue;
}
$zmd5 = md5($data);
if ($ARRAY["HOSTNAME"] == null) {
$ARRAY["HOSTNAME"] = $fam->GetComputerName($ARRAY["IPADDR"]);
}
$familysite = $fam->GetFamilySites($ARRAY["HOSTNAME"]);
$SQL1[date("YmdH", $ARRAY["TIME"])][] = "('{$zmd5}','{$ARRAY["DATE"]}','{$ARRAY["HOSTNAME"]}','{$familysite}','{$ARRAY["IPADDR"]}')";
if ($GLOBALS["VERBOSE"]) {
echo "('{$zmd5}','{$ARRAY["DATE"]}','{$ARRAY["HOSTNAME"]}','{$familysite}','{$ARRAY["IPADDR"]}')\n";
}
$mainkey = dba_nextkey($db_con);
}
dba_close($db_con);
if (count($SQL1) > 0) {
$q = new mysql_postfix_builder();
while (list($TIMESTAMP, $rows) = each($SQL1)) {
$q->postfix_buildhour_connections($TIMESTAMP);
$sql = "INSERT IGNORE INTO {$TIMESTAMP}_hcnx (`zmd5`,`zDate`,`hostname`,`domain`,`ipaddr`) VALUES " . @implode(",", $rows);
$q->QUERY_SQL($sql);
if (!$q->ok) {
echo $q->mysql_error;
return false;
}
}
}
if ($GLOBALS["VERBOSE"]) {
echo "Parsing {$path} END\n";
}
return true;
}
function connect_from($logpath)
{
$unix = new unix();
$q = new mysql();
$q->QUERY_SQL("CREATE TABLE IF NOT EXISTS `smtpstats_day` (\n\t`zmd5` VARCHAR(90) NOT NULL PRIMARY KEY,\n\t`zDate` DATETIME,\n\t`domain` VARCHAR(128),\n\t`GREY` BIGINT UNSIGNED,\n\t`BLACK` BIGINT UNSIGNED,\n\t`CNX` BIGINT UNSIGNED,\n\t`HOSTS` BIGINT UNSIGNED,\n\t`IPS` BIGINT UNSIGNED,\n\t`INFOS` TINYTEXT,\n\tKEY `zDate` (`zDate`),\n\tKEY `domain` (`domain`),\n\tKEY `GREY` (`GREY`),\n\tKEY `BLACK` (`BLACK`),\n\tKEY `CNX` (`CNX`),\n\tKEY `IPS` (`IPS`),\n\tKEY `HOSTS` (`HOSTS`)\n\t) ENGINE=MYISAM;", "artica_events");
if (!$q->ok) {
echo $q->mysql_error . "\n";
return;
}
$q->QUERY_SQL("CREATE TABLE IF NOT EXISTS `smtpcdir_day` (\n\t`zmd5` VARCHAR(90) NOT NULL PRIMARY KEY,\n\t`zDate` DATETIME,\n\t`CDIR` VARCHAR(90),\n\t`GREY` BIGINT UNSIGNED,\n\t`BLACK` BIGINT UNSIGNED,\n\t`CNX` BIGINT UNSIGNED,\n\t`HOSTS` BIGINT UNSIGNED,\n\t`DOMAINS` BIGINT UNSIGNED,\n\t`INFOS` TINYTEXT,\n\tKEY `zDate` (`zDate`),\n\tKEY `DOMAINS` (`DOMAINS`),\n\tKEY `GREY` (`GREY`),\n\tKEY `BLACK` (`BLACK`),\n\tKEY `CNX` (`CNX`),\n\tKEY `CDIR` (`CDIR`),\n\tKEY `HOSTS` (`HOSTS`)\n\t) ENGINE=MYISAM;", "artica_events");
if (!$q->ok) {
echo $q->mysql_error . "\n";
return;
}
$grep = $unix->find_program("grep");
$tmpfile = $unix->FILE_TEMP();
shell_exec("{$grep} -e \"smtpd.*: connect from\" {$logpath} >{$tmpfile}");
$fp = @fopen($tmpfile, "r");
if (!$fp) {
return false;
}
$t = array();
$fam = new familysite();
while (!feof($fp)) {
$line = trim(fgets($fp, 4096));
$line = str_replace("\r\n", "", $line);
$line = str_replace("\n", "", $line);
$line = str_replace("\r", "", $line);
$line = trim($line);
if (!preg_match("#^(.+?)\\s+([0-9]+)\\s+([0-9:]+)\\s+.*?\\[[0-9]+\\]:\\s+connect from\\s+(.+?)\\[([0-9\\.]+)\\]#", $line, $re)) {
continue;
}
$date = strtotime("{$re[1]} {$re[2]} {$re[3]}");
$ipaddr = $re[5];
$day = date("Y-m-d", $date);
$NETZ = explode(".", $ipaddr);
$network = "{$NETZ[0]}.{$NETZ[1]}.{$NETZ[2]}.0/24";
$hostname = $re[4];
$familysite = $fam->GetFamilySites($hostname);
if (!isset($MAINNETS[$day][$network]["CNX"])) {
$MAINNETS[$day][$network]["CNX"] = 1;
} else {
$MAINNETS[$day][$network]["CNX"] = $MAINNETS[$day][$network]["CNX"] + 1;
}
if (!isset($MAINNETS[$day][$network]["FAM"][$familysite])) {
$MAINNETS[$day][$network]["FAM"][$familysite] = 1;
} else {
$MAINNETS[$day][$network]["FAM"][$familysite] = $MAINNETS[$day][$network]["FAM"][$familysite] + 1;
}
if (!isset($MAIN[$day][$familysite]["IPS"][$ipaddr])) {
$MAIN[$day][$familysite]["IPS"][$ipaddr] = 1;
} else {
$MAIN[$day][$familysite]["IPS"][$ipaddr] = $MAIN[$day][$familysite]["IPS"][$ipaddr] + 1;
}
if (!isset($MAIN[$day][$familysite]["COUNT"])) {
$MAIN[$day][$familysite]["COUNT"] = 1;
} else {
$MAIN[$day][$familysite]["COUNT"] = $MAIN[$day][$familysite]["COUNT"] + 1;
}
if (!isset($MAIN[$day][$familysite]["HOSTS"][$hostname])) {
$MAIN[$day][$familysite]["HOSTS"][$hostname] = 1;
} else {
$MAIN[$day][$familysite]["HOSTS"][$hostname] = $MAIN[$day][$familysite]["HOSTS"][$hostname] + 1;
}
//echo date("Y-m-d")." $hostname $ipaddr\n";
}
@fclose($fp);
@unlink($tmpfile);
shell_exec("{$grep} -e \"NOQUEUE: milter-reject: RCPT from\" {$logpath} >{$tmpfile}");
$fp = @fopen($tmpfile, "r");
if (!$fp) {
return false;
}
while (!feof($fp)) {
$line = trim(fgets($fp, 4096));
$line = str_replace("\r\n", "", $line);
$line = str_replace("\n", "", $line);
$line = str_replace("\r", "", $line);
$line = trim($line);
if (!preg_match("#^(.+?)\\s+([0-9]+)\\s+([0-9:]+)\\s+.*?\\[[0-9]+\\]:\\s+NOQUEUE: milter-reject: RCPT from\\s+(.*?)\\[([0-9\\.]+)\\]:\\s+([0-9]+)\\s+#", $line, $re)) {
echo "NO MATCH {$line}\n";
continue;
}
$date = strtotime("{$re[1]} {$re[2]} {$re[3]}");
$hostname = $re[4];
$ipaddr = $re[5];
$CODE = $re[6];
$day = date("Y-m-d", $date);
$familysite = $fam->GetFamilySites($hostname);
$NETZ = explode(".", $ipaddr);
$network = "{$NETZ[0]}.{$NETZ[1]}.{$NETZ[2]}.0/24";
if (!isset($MAINNETS[$day][$network]["FAM"][$familysite])) {
$MAINNETS[$day][$network]["FAM"][$familysite] = 1;
} else {
$MAINNETS[$day][$network]["FAM"][$familysite] = $MAINNETS[$day][$network]["FAM"][$familysite] + 1;
}
if ($CODE == 451) {
if (!isset($MAINNETS[$day][$network]["GREY"])) {
$MAINNETS[$day][$network]["GREY"] = 1;
} else {
$MAINNETS[$day][$network]["GREY"] = $MAINNETS[$day][$network]["GREY"] + 1;
}
if (!isset($MAIN[$day][$familysite]["GREY"])) {
$MAIN[$day][$familysite]["GREY"] = 1;
} else {
$MAIN[$day][$familysite]["GREY"] = $MAIN[$day][$familysite]["GREY"] + 1;
}
}
if ($CODE == 551) {
if (!isset($MAIN[$day][$familysite]["BLACK"])) {
$MAIN[$day][$familysite]["BLACK"] = 1;
} else {
$MAIN[$day][$familysite]["BLACK"] = $MAIN[$day][$familysite]["BLACK"] + 1;
}
if (!isset($MAINNETS[$day][$network]["BLACK"])) {
$MAINNETS[$day][$network]["BLACK"] = 1;
} else {
$MAINNETS[$day][$network]["BLACK"] = $MAINNETS[$day][$network]["BLACK"] + 1;
}
}
}
@fclose($fp);
@unlink($tmpfile);
shell_exec("{$grep} -e \"NOQUEUE: reject: RCPT from\" {$logpath} >{$tmpfile}");
$fp = @fopen($tmpfile, "r");
if (!$fp) {
return false;
}
while (!feof($fp)) {
$line = trim(fgets($fp, 4096));
$line = str_replace("\r\n", "", $line);
$line = str_replace("\n", "", $line);
$line = str_replace("\r", "", $line);
$line = trim($line);
if (!preg_match("#^(.+?)\\s+([0-9]+)\\s+([0-9:]+)\\s+.*?\\[[0-9]+\\]:\\s+NOQUEUE: reject: RCPT from\\s+(.*?)\\[([0-9\\.]+)\\]:\\s+([0-9]+)\\s+#", $line, $re)) {
echo "NO MATCH {$line}\n";
continue;
}
$date = strtotime("{$re[1]} {$re[2]} {$re[3]}");
$hostname = $re[4];
$ipaddr = $re[5];
$CODE = $re[6];
$day = date("Y-m-d", $date);
$familysite = $fam->GetFamilySites($hostname);
$NETZ = explode(".", $ipaddr);
$network = "{$NETZ[0]}.{$NETZ[1]}.{$NETZ[2]}.0/24";
if ($CODE == 551 or $CODE == 554) {
if (!isset($MAIN[$day][$familysite]["BLACK"])) {
$MAIN[$day][$familysite]["BLACK"] = 1;
} else {
$MAIN[$day][$familysite]["BLACK"] = $MAIN[$day][$familysite]["BLACK"] + 1;
}
if (!isset($MAINNETS[$day][$network]["BLACK"])) {
$MAINNETS[$day][$network]["BLACK"] = 1;
} else {
$MAINNETS[$day][$network]["BLACK"] = $MAINNETS[$day][$network]["BLACK"] + 1;
}
}
}
@fclose($fp);
@unlink($tmpfile);
$prefix = "INSERT IGNORE INTO smtpstats_day (`zmd5`,`zDate`,`domain`,`GREY`,`BLACK`,`CNX`,`HOSTS`,`IPS`,`INFOS`) VALUES ";
while (list($zDate, $ARRAY) = each($MAIN)) {
while (list($domain, $INFOS) = each($ARRAY)) {
$GREY = 0;
if (!isset($INFOS["BLACK"])) {
$INFOS["BLACK"] = 0;
}
if (!isset($INFOS["GREY"])) {
$INFOS["GREY"] = 0;
}
$HOSTS = count($INFOS["HOSTS"]);
$IPS = count($INFOS["IPS"]);
$BLACK = intval($INFOS["BLACK"]);
$CNX = intval($INFOS["COUNT"]);
$INFO["IPS"] = $INFOS["IPS"];
$INFO["HOSTS"] = $INFOS["HOSTS"];
$infotext = mysql_escape_string2(serialize($INFO));
if ($GLOBALS["VERBOSE"]) {
echo "{$zDate}: {$domain} hosts:{$HOSTS} ips:{$IPS} blacklisted:{$BLACK} greylisted:{$GREY} cnx:{$CNX} {$infotext}\n";
}
$md5 = md5("{$zDate}{$domain}{$HOSTS}{$IPS}{$BLACK}{$GREY}{$CNX}{$infotext}");
$f[] = "('{$md5}','{$zDate}','{$domain}','{$GREY}','{$BLACK}','{$CNX}','{$HOSTS}','{$IPS}','{$infotext}')";
if (count($f) > 500) {
$q->QUERY_SQL($prefix . @implode(",", $f), "artica_events");
if (!$q->ok) {
echo $q->mysql_error . "\n";
return;
}
$f = array();
}
}
}
if (count($f) > 0) {
$q->QUERY_SQL($prefix . @implode(",", $f), "artica_events");
if (!$q->ok) {
echo $q->mysql_error . "\n";
return;
}
$f = array();
}
$q->QUERY_SQL("CREATE TABLE IF NOT EXISTS `smtpcdir_day` (\n\t`zmd5` VARCHAR(90) NOT NULL PRIMARY KEY,\n\t`zDate` DATETIME,\n\t`CDIR` VARCHAR(90),\n\t`GREY` BIGINT UNSIGNED,\n\t`BLACK` BIGINT UNSIGNED,\n\t`CNX` BIGINT UNSIGNED,\n\t`HOSTS` BIGINT UNSIGNED,\n\t`DOMAINS` BIGINT UNSIGNED,\n\t`INFOS` TINYTEXT,\n\tKEY `zDate` (`zDate`),\n\tKEY `DOMAINS` (`DOMAINS`),\n\tKEY `GREY` (`GREY`),\n\tKEY `BLACK` (`BLACK`),\n\tKEY `CNX` (`CNX`),\n\tKEY `CDIR` (`CDIR`),\n\tKEY `HOSTS` (`HOSTS`)\n\t) ENGINE=MYISAM;", "artica_events");
if (!$q->ok) {
echo $q->mysql_error . "\n";
return;
}
$prefix = "INSERT IGNORE INTO `smtpcdir_day` (`zmd5`,`zDate`,`CDIR`,`GREY`,`BLACK`,`CNX`,`DOMAINS`,`INFOS`) VALUES ";
while (list($zDate, $ARRAY) = each($MAINNETS)) {
while (list($CDIR, $INFOS) = each($ARRAY)) {
if (!isset($INFOS["BLACK"])) {
$INFOS["BLACK"] = 0;
}
if (!isset($INFOS["GREY"])) {
$INFOS["GREY"] = 0;
}
$CNX = intval($INFOS["CNX"]);
$GREY = intval($INFOS["GREY"]);
$BLACK = intval($INFOS["BLACK"]);
$DOMAINS = intval($INFOS["FAM"]);
$infotext = mysql_escape_string2(serialize($INFOS["FAM"]));
echo "{$zDate} {$CDIR} cnx:{$CNX} greylisted:{$GREY} blacklisted:{$BLACK} domains:{$DOMAINS}\n";
$md5 = md5("{$zDate}{$CDIR}{$DOMAINS}{$BLACK}{$GREY}{$CNX}{$infotext}");
$f[] = "('{$md5}','{$zDate}','{$CDIR}','{$GREY}','{$BLACK}','{$CNX}','{$DOMAINS}','{$infotext}')";
if (count($f) > 500) {
$q->QUERY_SQL($prefix . @implode(",", $f), "artica_events");
if (!$q->ok) {
echo $q->mysql_error . "\n";
return;
}
$f = array();
}
}
}
if (count($f) > 0) {
$q->QUERY_SQL($prefix . @implode(",", $f), "artica_events");
if (!$q->ok) {
echo $q->mysql_error . "\n";
return;
}
$f = array();
}
return true;
//print_r($MAINNETS);
}
function tool_get_familysite($uri){
$parse_url=parse_url($uri);
$sitename=$parse_url["host"];
if(isset($GLOBALS["FAMILYSITES"][$sitename])){return $GLOBALS["FAMILYSITES"][$sitename];}
$f=new familysite();
$GLOBALS["FAMILYSITES"][$sitename]=$f->GetFamilySites($sitename);
return $GLOBALS["FAMILYSITES"][$sitename];
}
function xstart()
{
$unix = new unix();
$filetime = "/etc/artica-postfix/pids/exec.mvps.php.time";
if ($unix->file_time_min($filetime) < 240) {
echo "Need 240mn, current is {$filetime}Mn\n";
return;
}
@unlink($filetime);
@file_put_contents($filetime, time());
$q = new mysql_squid_builder();
$curl = new ccurl("http://winhelp2002.mvps.org/hosts.txt");
$targetpath = $unix->FILE_TEMP();
if (!$curl->GetFile($targetpath)) {
squid_admin_mysql(1, "Unable to download hosts.txt from winhelp2002.mvps.org", null, __FILE__, __LINE__);
return;
}
$f = explode("\n", @file_get_contents($targetpath));
@unlink($targetpath);
$fam = new familysite();
$sql = "CREATE TABLE IF NOT EXISTS `squidlogs`.`ads_domains` (\n\t\t\t`servername` VARCHAR(255) PRIMARY KEY,\n\t\t\t`enabled` smallint(1) NOT NULL DEFAULT 1,\n\t\t\t KEY `enabled`(`enabled`)\n\t\t) ENGINE=MYISAM;";
$q->QUERY_SQL($sql);
if (!$q->ok) {
echo $q->mysql_error;
return;
}
$COUNT1 = $q->COUNT_ROWS("ads_domains");
$QQR = array();
while (list($a, $line) = each($f)) {
$line = trim($line);
if (strpos($line, "localhost") > 0) {
continue;
}
if (substr($line, 0, 1) == "#") {
continue;
}
$line = str_replace("0.0.0.0 ", "", $line);
if (strpos($line, "#") > 0) {
$FI = explode("#", $line);
$line = $FI[0];
}
if (strpos($line, ".") == 0) {
continue;
}
$line = trim($line);
$familysite = $fam->GetFamilySites($line);
if ($line == $familysite) {
$line = ".{$line}";
}
$QQR[] = "('{$line}','1')";
if (count($QQR) > 500) {
$sql = "INSERT IGNORE INTO `ads_domains` (`servername`,`enabled`) VALUES " . @implode(",", $QQR);
$q->QUERY_SQL($sql);
$QQR = array();
}
}
if (count($QQR) > 0) {
$sql = "INSERT IGNORE INTO `ads_domains` (`servername`,`enabled`) VALUES " . @implode(",", $QQR);
$q->QUERY_SQL($sql);
$QQR = array();
}
$curl = new ccurl("http://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext");
$targetpath = $unix->FILE_TEMP();
if (!$curl->GetFile($targetpath)) {
squid_admin_mysql(1, "Unable to download serverlist from yoyo.org", null, __FILE__, __LINE__);
return;
}
$f = explode("\n", @file_get_contents($targetpath));
@unlink($targetpath);
while (list($a, $line) = each($f)) {
$line = trim($line);
if (strpos($line, "localhost") > 0) {
continue;
}
if (substr($line, 0, 1) == "#") {
continue;
}
$line = str_replace("127.0.0.1 ", "", $line);
if (strpos($line, "#") > 0) {
$FI = explode("#", $line);
$line = $FI[0];
}
if (strpos($line, ".") == 0) {
continue;
}
$line = trim($line);
$QQR[] = "('{$line}','1')";
$familysite = $fam->GetFamilySites($line);
if ($line == $familysite) {
$line = ".{$line}";
}
if (count($QQR) > 500) {
$sql = "INSERT IGNORE INTO `ads_domains` (`servername`,`enabled`) VALUES " . @implode(",", $QQR);
$q->QUERY_SQL($sql);
$QQR = array();
}
}
if (count($QQR) > 0) {
$sql = "INSERT IGNORE INTO `ads_domains` (`servername`,`enabled`) VALUES " . @implode(",", $QQR);
$q->QUERY_SQL($sql);
$QQR = array();
}
$COUNT2 = $q->COUNT_ROWS("ads_domains");
if ($COUNT2 > $COUNT1) {
$TOTAL = $COUNT2 - $COUNT1;
squid_admin_mysql(1, "{$TOTAL} ads and tracker added in ACLs", null, __FILE__, __LINE__);
}
}
function connect_from($logpath)
{
$unix = new unix();
smtpstats_day_migrate_to_postgres();
smtpcdir_day_migrate_to_postgres();
smtpsum_day_migrate_to_postgres();
smtpgraph_day_migrate_to_postgres();
smtpdeliver_day_migrate_to_postgres();
smtpsenders_day_migrate_to_postgres();
smtprecipients_day_migrate_to_postgres();
$q = new postgres_sql();
$q->SMTP_TABLES();
$grep = $unix->find_program("grep");
$tmpfile = $unix->FILE_TEMP();
shell_exec("{$grep} -e \"smtpd.*: connect from\" {$logpath} >{$tmpfile}");
$fp = @fopen($tmpfile, "r");
if (!$fp) {
return false;
}
$t = array();
$fam = new familysite();
while (!feof($fp)) {
$line = trim(fgets($fp, 4096));
$line = str_replace("\r\n", "", $line);
$line = str_replace("\n", "", $line);
$line = str_replace("\r", "", $line);
$line = trim($line);
if (!preg_match("#^(.+?)\\s+([0-9]+)\\s+([0-9:]+)\\s+.*?\\[[0-9]+\\]:\\s+connect from\\s+(.+?)\\[([0-9\\.]+)\\]#", $line, $re)) {
continue;
}
$date = strtotime("{$re[1]} {$re[2]} {$re[3]}");
$ipaddr = $re[5];
$day = date("Y-m-d", $date);
$NETZ = explode(".", $ipaddr);
$network = "{$NETZ[0]}.{$NETZ[1]}.{$NETZ[2]}.0/24";
$hostname = $re[4];
$familysite = $fam->GetFamilySites($hostname);
if (!isset($MAINNETS[$day][$network]["CNX"])) {
$MAINNETS[$day][$network]["CNX"] = 1;
} else {
$MAINNETS[$day][$network]["CNX"] = $MAINNETS[$day][$network]["CNX"] + 1;
}
if (!isset($MAINNETS[$day][$network]["FAM"][$familysite])) {
$MAINNETS[$day][$network]["FAM"][$familysite] = 1;
} else {
$MAINNETS[$day][$network]["FAM"][$familysite] = $MAINNETS[$day][$network]["FAM"][$familysite] + 1;
}
if (!isset($MAIN[$day][$familysite]["IPS"][$ipaddr])) {
$MAIN[$day][$familysite]["IPS"][$ipaddr] = 1;
} else {
$MAIN[$day][$familysite]["IPS"][$ipaddr] = $MAIN[$day][$familysite]["IPS"][$ipaddr] + 1;
}
if (!isset($MAIN[$day][$familysite]["COUNT"])) {
$MAIN[$day][$familysite]["COUNT"] = 1;
} else {
$MAIN[$day][$familysite]["COUNT"] = $MAIN[$day][$familysite]["COUNT"] + 1;
}
if (!isset($MAIN[$day][$familysite]["HOSTS"][$hostname])) {
$MAIN[$day][$familysite]["HOSTS"][$hostname] = 1;
} else {
$MAIN[$day][$familysite]["HOSTS"][$hostname] = $MAIN[$day][$familysite]["HOSTS"][$hostname] + 1;
}
//echo date("Y-m-d")." $hostname $ipaddr\n";
}
@fclose($fp);
@unlink($tmpfile);
shell_exec("{$grep} -e \"NOQUEUE: milter-reject: RCPT from\" {$logpath} >{$tmpfile}");
$fp = @fopen($tmpfile, "r");
if (!$fp) {
return false;
}
while (!feof($fp)) {
$line = trim(fgets($fp, 4096));
$line = str_replace("\r\n", "", $line);
$line = str_replace("\n", "", $line);
$line = str_replace("\r", "", $line);
$line = trim($line);
if (!preg_match("#^(.+?)\\s+([0-9]+)\\s+([0-9:]+)\\s+.*?\\[[0-9]+\\]:\\s+NOQUEUE: milter-reject: RCPT from\\s+(.*?)\\[([0-9\\.]+)\\]:\\s+([0-9]+)\\s+#", $line, $re)) {
echo "NO MATCH {$line}\n";
continue;
}
$date = strtotime("{$re[1]} {$re[2]} {$re[3]}");
$hostname = $re[4];
$ipaddr = $re[5];
$CODE = $re[6];
$day = date("Y-m-d", $date);
$familysite = $fam->GetFamilySites($hostname);
$NETZ = explode(".", $ipaddr);
$network = "{$NETZ[0]}.{$NETZ[1]}.{$NETZ[2]}.0/24";
if (!isset($MAINNETS[$day][$network]["FAM"][$familysite])) {
$MAINNETS[$day][$network]["FAM"][$familysite] = 1;
} else {
$MAINNETS[$day][$network]["FAM"][$familysite] = $MAINNETS[$day][$network]["FAM"][$familysite] + 1;
}
if ($CODE == 451) {
if (!isset($MAINNETS[$day][$network]["GREY"])) {
$MAINNETS[$day][$network]["GREY"] = 1;
} else {
$MAINNETS[$day][$network]["GREY"] = $MAINNETS[$day][$network]["GREY"] + 1;
}
if (!isset($MAIN[$day][$familysite]["GREY"])) {
$MAIN[$day][$familysite]["GREY"] = 1;
} else {
$MAIN[$day][$familysite]["GREY"] = $MAIN[$day][$familysite]["GREY"] + 1;
}
}
if ($CODE == 551) {
if (!isset($MAIN[$day][$familysite]["BLACK"])) {
$MAIN[$day][$familysite]["BLACK"] = 1;
} else {
$MAIN[$day][$familysite]["BLACK"] = $MAIN[$day][$familysite]["BLACK"] + 1;
}
if (!isset($MAINNETS[$day][$network]["BLACK"])) {
$MAINNETS[$day][$network]["BLACK"] = 1;
} else {
$MAINNETS[$day][$network]["BLACK"] = $MAINNETS[$day][$network]["BLACK"] + 1;
}
}
}
@fclose($fp);
@unlink($tmpfile);
shell_exec("{$grep} -e \"NOQUEUE: reject: RCPT from\" {$logpath} >{$tmpfile}");
$fp = @fopen($tmpfile, "r");
if (!$fp) {
return false;
}
while (!feof($fp)) {
$line = trim(fgets($fp, 4096));
$line = str_replace("\r\n", "", $line);
$line = str_replace("\n", "", $line);
$line = str_replace("\r", "", $line);
$line = trim($line);
if (!preg_match("#^(.+?)\\s+([0-9]+)\\s+([0-9:]+)\\s+.*?\\[[0-9]+\\]:\\s+NOQUEUE: reject: RCPT from\\s+(.*?)\\[([0-9\\.]+)\\]:\\s+([0-9]+)\\s+#", $line, $re)) {
echo "NO MATCH {$line}\n";
continue;
}
$date = strtotime("{$re[1]} {$re[2]} {$re[3]}");
$hostname = $re[4];
$ipaddr = $re[5];
$CODE = $re[6];
$day = date("Y-m-d", $date);
$familysite = $fam->GetFamilySites($hostname);
$NETZ = explode(".", $ipaddr);
$network = "{$NETZ[0]}.{$NETZ[1]}.{$NETZ[2]}.0/24";
if ($CODE == 551 or $CODE == 554) {
if (!isset($MAIN[$day][$familysite]["BLACK"])) {
$MAIN[$day][$familysite]["BLACK"] = 1;
} else {
$MAIN[$day][$familysite]["BLACK"] = $MAIN[$day][$familysite]["BLACK"] + 1;
}
if (!isset($MAINNETS[$day][$network]["BLACK"])) {
$MAINNETS[$day][$network]["BLACK"] = 1;
} else {
$MAINNETS[$day][$network]["BLACK"] = $MAINNETS[$day][$network]["BLACK"] + 1;
}
}
}
@fclose($fp);
@unlink($tmpfile);
$prefix = "INSERT INTO smtpstats_day (zmd5,zdate,domain,grey,black,cnx,hosts,ips,infos) VALUES ";
$q = new postgres_sql();
while (list($zDate, $ARRAY) = each($MAIN)) {
while (list($domain, $INFOS) = each($ARRAY)) {
$GREY = 0;
if (!isset($INFOS["BLACK"])) {
$INFOS["BLACK"] = 0;
}
if (!isset($INFOS["GREY"])) {
$INFOS["GREY"] = 0;
}
$HOSTS = count($INFOS["HOSTS"]);
$IPS = count($INFOS["IPS"]);
$BLACK = intval($INFOS["BLACK"]);
$CNX = intval($INFOS["COUNT"]);
$INFO["IPS"] = $INFOS["IPS"];
$INFO["HOSTS"] = $INFOS["HOSTS"];
$infotext = mysql_escape_string2(serialize($INFO));
if ($GLOBALS["VERBOSE"]) {
echo "{$zDate}: {$domain} hosts:{$HOSTS} ips:{$IPS} blacklisted:{$BLACK} greylisted:{$GREY} cnx:{$CNX} {$infotext}\n";
}
$md5 = md5("{$zDate}{$domain}{$HOSTS}{$IPS}{$BLACK}{$GREY}{$CNX}{$infotext}");
$f[] = "('{$md5}','{$zDate}','{$domain}','{$GREY}','{$BLACK}','{$CNX}','{$HOSTS}','{$IPS}','{$infotext}')";
if (count($f) > 500) {
$q->QUERY_SQL($prefix . @implode(",", $f) . " ON CONFLICT DO NOTHING");
if (!$q->ok) {
echo $q->mysql_error . "\n";
return;
}
$f = array();
}
}
}
if (count($f) > 0) {
$q->QUERY_SQL($prefix . @implode(",", $f), "artica_events");
if (!$q->ok) {
echo $q->mysql_error . "\n";
return;
}
$f = array();
}
$prefix = "INSERT INTO smtpcdir_day (zmd5,zdate,cdir,grey,black,cnx,domains,infos) VALUES ";
$q = new postgres_sql();
$q->SMTP_TABLES();
while (list($zDate, $ARRAY) = each($MAINNETS)) {
while (list($CDIR, $INFOS) = each($ARRAY)) {
if (!isset($INFOS["BLACK"])) {
$INFOS["BLACK"] = 0;
}
if (!isset($INFOS["GREY"])) {
$INFOS["GREY"] = 0;
}
$CNX = intval($INFOS["CNX"]);
$GREY = intval($INFOS["GREY"]);
$BLACK = intval($INFOS["BLACK"]);
$DOMAINS = intval($INFOS["FAM"]);
$infotext = mysql_escape_string2(serialize($INFOS["FAM"]));
echo "{$zDate} {$CDIR} cnx:{$CNX} greylisted:{$GREY} blacklisted:{$BLACK} domains:{$DOMAINS}\n";
$md5 = md5("{$zDate}{$CDIR}{$DOMAINS}{$BLACK}{$GREY}{$CNX}{$infotext}");
$f[] = "('{$md5}','{$zDate}','{$CDIR}','{$GREY}','{$BLACK}','{$CNX}','{$DOMAINS}','{$infotext}')";
if (count($f) > 500) {
$q->QUERY_SQL($prefix . @implode(",", $f) . " ON CONFLICT DO NOTHING");
if (!$q->ok) {
echo $q->mysql_error . "\n";
return;
}
$f = array();
}
}
}
if (count($f) > 0) {
$q->QUERY_SQL($prefix . @implode(",", $f), "artica_events");
if (!$q->ok) {
echo $q->mysql_error . "\n";
return;
}
$f = array();
}
return true;
//print_r($MAINNETS);
}
