function ParseSizeBuffer($buffer) { if (!class_exists("class.logfile_daemon.inc")) { include_once "/usr/share/artica-postfix/ressources/class.logfile_daemon.inc"; } $re = explode(":::", $buffer); $mac = trim(strtolower($re[0])); if ($mac == "-") { $mac == null; } $mac = str_replace("-", ":", $mac); if ($mac == "00:00:00:00:00:00") { $mac = null; } $ipaddr = trim($re[1]); // uid $uid = $re[2]; $uid2 = $re[3]; if ($uid == "-") { $uid = null; } if ($uid2 == "-") { $uid2 = null; } if (preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $uid2)) { $uid2 = null; } if ($uid == null) { if ($uid2 != null) { $uid = $uid2; } } $zdate = $re[4]; $xtime = time(); $SUFFIX_DATE = date("YmdH", $xtime); $logzdate = date("Y-m-d H:i:s", $xtime); $proto = $re[5]; $uri = $re[6]; $code_error = $re[8]; $SIZE = $re[9]; $SquidCode = $re[10]; $UserAgent = urldecode($re[11]); $Forwarded = $re[12]; $sitename = trim($re[13]); $hostname = $re[14]; $response_time = $re[15]; $MimeType = $re[16]; $uid = str_replace("%20", " ", $uid); $uid = str_replace("%25", "-", $uid); if ($uid == "-") { $uid = null; } $Forwarded = str_replace("%25", "", $Forwarded); //events("MimeType: ......: $MimeType"); if (strpos($uid, '$') > 0) { if (substr($uid, strlen($uid) - 1, 1) == "\$") { $uid = null; } } if (preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $uid)) { $uid = null; } if (!preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $ipaddr)) { eventsfailed("***** WRONG LINE ipaddr:{$ipaddr} column 13 " . @implode(" | ", $re) . "*****"); return; } if ($sitename == "-") { $h = parse_url($uri); if (isset($h["host"])) { $sitename = $h["host"]; } if ($sitename == "-") { eventsfailed("***** WRONG SITENAME {$sitename} column 13 " . @implode(" | ", $re) . "*****"); eventsfailed("{$buffer}"); eventsfailed("*"); $GLOBALS["REFUSED_REQUESTS"] = $GLOBALS["REFUSED_REQUESTS"] + 1; return; } if ($sitename == null) { eventsfailed("***** WRONG SITENAME {$sitename} column 13 " . @implode(" | ", $re) . "*****"); eventsfailed("{$buffer}"); eventsfailed("*"); $GLOBALS["REFUSED_REQUESTS"] = $GLOBALS["REFUSED_REQUESTS"] + 1; return; } } if (strpos($sitename, ":") > 0) { $XA = explode(":", $sitename); $sitename = $XA[0]; } if ($sitename == "127.0.0.1") { $GLOBALS["REFUSED_REQUESTS"] = $GLOBALS["REFUSED_REQUESTS"] + 1; if ($GLOBALS["VERBOSE"]) { events("127.0.0.1 -> uid = null -> SKIP"); } return; } if ($Forwarded == "unknown") { $Forwarded = null; } if ($Forwarded == "-") { $Forwarded = null; } if ($Forwarded == "0.0.0.0") { $Forwarded = null; } if ($Forwarded == "255.255.255.255") { $Forwarded = null; } if (strlen($Forwarded) > 4) { $ipaddr = $Forwarded; $mac = null; } $ipaddr = str_replace("%25", "-", $ipaddr); $mac = str_replace("%25", "-", $mac); if ($mac == "-") { $mac = null; } if ($ipaddr == "127.0.0.1" or $ipaddr == "::") { if ($uid == null) { $GLOBALS["REFUSED_REQUESTS"] = $GLOBALS["REFUSED_REQUESTS"] + 1; if ($GLOBALS["VERBOSE"]) { events("127.0.0.1 -> uid = null -> SKIP"); } return; } } if (preg_match("#([0-9:a-z]+)\$#", $mac, $z)) { $mac = $z[1]; } if ($GLOBALS["VERBOSE"]) { events("ITEM: DATE......: {$logzdate}"); events("ITEM: MAC.......: {$mac}"); events("ITEM: IP........: {$ipaddr}"); events("ITEM: Size......: {$SIZE}"); events("ITEM: SQUID CODE: {$SquidCode}"); events("ITEM: HTTP CODE.: {$code_error}"); events("ITEM: uid.......: {$uid}"); events("ITEM: uri.......: {$uri}"); events("ITEM: UserAgent.: {$UserAgent}"); events("ITEM: Forwarded.: {$Forwarded}"); events("ITEM: SiteName..: {$sitename}"); } if ($UserAgent != null) { UserAuthDB($mac, $ipaddr, $uid, $hostname, $UserAgent); } else { events("No UserAgents in {$buffer}"); } $GLOBALS["COUNT_HASH_TABLE"] = $GLOBALS["COUNT_HASH_TABLE"] + 1; $arrayURI = parse_url($uri); $sitename = $arrayURI["host"]; if (strpos($sitename, ":")) { $xtr = explode(":", $sitename); $sitename = $xtr[0]; if (preg_match("#^www\\.(.+)#", $sitename, $rz)) { $sitename = $rz[1]; } } $TimeCache = date("YmdH"); if (!isset($GLOBALS["FAMLILYSITE"][$sitename])) { $fam = new squid_familysite(); $GLOBALS["FAMLILYSITE"][$sitename] = $fam->GetFamilySites($sitename); } $FamilySite = $GLOBALS["FAMLILYSITE"][$sitename]; $TablePrimaireHour = "squidhour_" . $TimeCache; $TableSizeHours = "sizehour_" . $TimeCache; $TableCacheHours = "cachehour_" . $TimeCache; $tableYoutube = "youtubehours_" . $TimeCache; $tableSearchWords = "searchwords_" . $TimeCache; $tableQuotaTemp = "quotatemp_" . $TimeCache; $category = null; if ($GLOBALS["DisableLogFileDaemonCategories"] == 0) { if ($GLOBALS["VERBOSE"]) { $time_start = microtime(true); } $category = ufdbcat($sitename); if ($GLOBALS["VERBOSE"]) { $time_end = microtime(true); $time_calc = $time_end - $time_start; } if ($GLOBALS["VERBOSE"]) { events("{$sitename} = {$category} {$time_calc}ms"); } } $logfile_daemon = new logfile_daemon(); $cached = $logfile_daemon->CACHEDORNOT($SquidCode); $SearchWords = $logfile_daemon->SearchWords($uri); $GLOBALS["ACCEPTED_REQUESTS"] = $GLOBALS["ACCEPTED_REQUESTS"] + 1; $MAIN["TIMESTAMP"] = time(); $MAIN["URI"] = $uri; $MAIN["sitename"] = $sitename; $MAIN["SIZE"] = $SIZE; $MAIN["CACHED"] = $cached; $MAIN["IPADDR"] = $ipaddr; $MAIN["CATEGORY"] = $category; $MAIN["MIMETYPE"] = $MimeType; $MAIN["FAMILYSITE"] = $GLOBALS["FAMLILYSITE"][$sitename]; $MAIN["MAC"] = $mac; $MAIN["UID"] = $uid; $MAIN["USERAGENT"] = $UserAgent; $MAIN["SQUID_CODE"] = $SquidCode; $MAIN["RESPONSE_TIME"] = $response_time; $MAIN["PROTO"] = $proto; $MAIN["HTTP_CODE"] = $code_error; if ($hostname != null) { $MAIN["HOSTNAME"] = $hostname; } if (is_array($SearchWords)) { $MAIN["WORDS"] = $SearchWords["WORDS"]; } $md5 = md5(serialize($MAIN)); berekley_add($md5, base64_encode(serialize($MAIN))); return; if (!isset($GLOBALS["RTTCREATED"][$TimeCache])) { events("Creating RTTH_{$TimeCache} table..."); if (create_tables($TimeCache)) { $GLOBALS["RTTCREATED"][$TimeCache] = true; } } $sql = "INSERT IGNORE INTO `squidlogs`.`RTTH_{$TimeCache}` (`xtime`,`sitename`,`ipaddr`,`uid`,`MAC`,`size`) VALUES('{$xtime}','{$FamilySite}','{$ipaddr}','{$uid}','{$mac}','{$SIZE}')"; if ($GLOBALS["VERBOSE"]) { $time_start = microtime(true); } if (!SEND_MYSQL($sql)) { @file_put_contents("/var/log/squid/mysql-rtterrors/" . md5($sql), serialize(array("TABLE" => "RTTH_{$TimeCache}", "CMD" => $sql))); } if ($GLOBALS["VERBOSE"]) { $time_end = microtime(true); $time_calc = $time_end - $time_start; } if ($GLOBALS["VERBOSE"]) { events("RTTH_{$TimeCache} {$time_calc}ms DisableLogFileDaemonMySQL={$GLOBALS["DisableLogFileDaemonMySQL"]}"); } $uri = xmysql_escape_string2($uri); if (!isset($GLOBALS["CODE_TO_STRING"][$code_error])) { $GLOBALS["CODE_TO_STRING"][$code_error] = $logfile_daemon->codeToString($code_error); } $zMD5 = md5("{$uri}{$xtime}{$mac}{$ipaddr}"); $TYPE = $GLOBALS["CODE_TO_STRING"][$code_error]; $cached = $GLOBALS["CACHEDX"][$SquidCode]; $UserAgent = xmysql_escape_string2($UserAgent); if ($GLOBALS["VERBOSE"]) { $time_start = microtime(true); } $sql = "INSERT IGNORE INTO `{$TableSizeHours}` (`zDate`,`size`,`cached`) VALUES ('{$logzdate}','{$SIZE}','{$cached}')"; if (!SEND_MYSQL($sql)) { @file_put_contents("/var/log/squid/mysql-rtterrors/" . md5($sql), serialize(array("TimeCache" => $TimeCache, "TABLE" => $TableSizeHours, "CMD" => $sql))); } if ($GLOBALS["VERBOSE"]) { $time_end = microtime(true); $time_calc = $time_end - $time_start; } if ($GLOBALS["VERBOSE"]) { events("{$TableSizeHours} = {$time_calc}ms"); } $sql = "INSERT IGNORE INTO `{$tableQuotaTemp}` (`xtime`,`keyr`,`ipaddr`,`familysite`,`servername`,`uid`,`MAC`,`size`) VALUES \n\t('{$logzdate}','{$zMD5}','{$ipaddr}','{$FamilySite}','{$FamilySite}','{$uid}','{$mac}','{$SIZE}')"; if (!SEND_MYSQL($sql)) { @file_put_contents("/var/log/squid/mysql-rtterrors/" . md5($sql), serialize(array("TimeCache" => $TimeCache, "TABLE" => $tableQuotaTemp, "CMD" => $sql))); } $sql = "INSERT IGNORE INTO `{$TablePrimaireHour}` (`sitename`,`uri`,`TYPE`,`REASON`,`CLIENT`,`hostname`,`zDate`,`zMD5`,`uid`,`QuerySize`,`cached`,`MAC`,`category`) VALUES ('{$sitename}','{$uri}','{$TYPE}','{$TYPE}','{$ipaddr}','{$hostname}','{$logzdate}','{$zMD5}','{$uid}','{$SIZE}','{$cached}','{$mac}','{$category}')"; if (!SEND_MYSQL($sql)) { @file_put_contents("/var/log/squid/mysql-rtterrors/" . md5($sql), serialize(array("TABLE" => $TablePrimaireHour, "CMD" => $sql))); } $sql = "INSERT IGNORE INTO `{$TableCacheHours}` (`zDate`,`size`,`cached`,`familysite`) VALUES ('{$logzdate}','{$SIZE}','{$cached}','{$FamilySite}')"; if (!SEND_MYSQL($sql)) { @file_put_contents("/var/log/squid/mysql-rtterrors/" . md5($sql), serialize(array("TimeCache" => $TimeCache, "TABLE" => $TableCacheHours, "CMD" => $sql))); } if (strpos(" {$uri}", "youtube") > 0) { $VIDEOID = $logfile_daemon->GetYoutubeID($uri); if ($VIDEOID != null) { $sql = "INSERT IGNORE INTO `{$tableYoutube}` (`zDate`,`ipaddr`,`hostname`,`uid`,`MAC` ,`account`,`youtubeid`) VALUES ('{$logzdate}','{$ipaddr}','','{$uid}','{$mac}','0','{$VIDEOID}')"; events_youtube($sql); if (!SEND_MYSQL($sql)) { @file_put_contents("/var/log/squid/mysql-rtterrors/" . md5($sql), serialize(array("TimeCache" => $TimeCache, "TABLE" => $tableYoutube, "CMD" => $sql))); } } } if (is_array($SearchWords)) { $words = xmysql_escape_string2($SearchWords["WORDS"]); $sql = "INSERT IGNORE INTO `{$tableSearchWords}` (`zmd5`,`sitename`,`zDate`,`ipaddr`,`hostname`,`uid`,`MAC`,`account`,`familysite`,`words`) VALUES ('{$zMD5}','{$sitename}','{$logzdate}','{$ipaddr}','{$hostname}','{$uid}','{$mac}','0','{$FamilySite}','{$words}')"; if (!SEND_MYSQL($sql)) { @file_put_contents("/var/log/squid/mysql-rtterrors/" . md5($sql), serialize(array("TimeCache" => $TimeCache, "TABLE" => $tableYoutube, "CMD" => $sql))); } } if (count($GLOBALS["CACHE_SQL"]) > 2) { events("CACHE_SQL = " . count($GLOBALS["CACHE_SQL"] . " seems 2 minutes")); empty_TableHash(); } $dd = date("Hi"); if (count($GLOBALS["CACHE_SQL"][$dd]) > 1000) { events("CACHE_SQL[{$dd}] = " . count($GLOBALS["CACHE_SQL"][$dd])); empty_TableHash(); } return; $GLOBALS["RTTHASH"][$SUFFIX_DATE][] = array("TIME" => $xtime, "MAC" => $mac, "IPADDR" => $ipaddr, "SIZE" => $SIZE, "SQUID_CODE" => $SquidCode, "HTTP_CODE" => $code_error, "UID" => $uid, "URI" => $uri, "USERAGENT" => $UserAgent, "SITENAME" => $sitename, "HOSTNAME" => $hostname, "RESPONSE_TIME" => $response_time); $GLOBALS["ACCEPTED_REQUESTS"] = $GLOBALS["ACCEPTED_REQUESTS"] + 1; if (count($GLOBALS["RTTHASH"][$SUFFIX_DATE]) > 50) { if ($GLOBALS["VERBOSE"]) { events("-> empty_TableHash()"); } empty_TableHash(); } if ($GLOBALS["VERBOSE"]) { events("---------------------- DONE ----------------------"); } }
function ParseSizeBuffer($buffer) { $proxyname = null; if (!class_exists("class.logfile_daemon.inc")) { include_once "/usr/share/artica-postfix/ressources/class.logfile_daemon.inc"; } $re = explode(":::", $buffer); if (preg_match("#^.*?\\):\\s+(.+)#", trim($re[0]), $rz)) { $re[0] = $rz[1]; } if ($GLOBALS["VERBOSE"]) { events($buffer); } if ($GLOBALS["VERBOSE"]) { events("ITEM: MAC......: {$re[0]} [" . __LINE__ . "]"); } $mac = trim(strtolower($re[0])); if ($mac == "-") { $mac == null; } $mac = str_replace("-", ":", $mac); if ($mac == "00:00:00:00:00:00") { $mac = null; } $ipaddr = trim($re[1]); if (!isset($GLOBALS["USER_MEM"])) { $GLOBALS["USER_MEM"] = 0; } // uid $uid = $re[2]; $uid2 = $re[3]; if ($uid == "-") { $uid = null; } if ($uid2 == "-") { $uid2 = null; } if (preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $uid2)) { $uid2 = null; } if ($uid == null) { if ($uid2 != null) { $uid = $uid2; } } $zdate = $re[4]; $xtime = time(); $SUFFIX_DATE = date("YmdH", $xtime); $logzdate = date("Y-m-d H:i:s", $xtime); $proto = $re[5]; $uri = $re[6]; $code_error = $re[8]; $SIZE = $re[9]; $SquidCode = $re[10]; $UserAgent = urldecode($re[11]); $Forwarded = $re[12]; $sitename = trim($re[13]); $hostname = trim($re[14]); $response_time = $re[15]; $MimeType = trim($re[16]); $sni = trim($re[17]); $proxyname = trim($re[18]); $OUGROUP = trim($re[19]); $uid = trim(strtolower(str_replace("%20", " ", $uid))); $uid = str_replace("%25", "-", $uid); if ($uid == "-") { $uid = null; } $Forwarded = str_replace("%25", "", $Forwarded); //events("MimeType: ......: $MimeType"); if ($sni == "-") { $sni = null; } if (strpos($uid, '$') > 0) { if (substr($uid, strlen($uid) - 1, 1) == "\$") { $uid = null; } } if ($sni != null) { if (preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $sitename)) { $sitename = $sni; } } if ($proxyname != null) { if (preg_match("#proxyname=(.+)#", $proxyname, $re)) { $GLOBALS["REMOTE_PROXY_NAME"] = $re[1]; } } else { $GLOBALS["REMOTE_PROXY_NAME"] = $GLOBALS["MYHOSTNAME"]; } $GLOBALS["REMOTE_PROXY_NAME"] = str_replace("proxyname=", "", $GLOBALS["REMOTE_PROXY_NAME"]); if (preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $uid)) { $uid = null; } if (!preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $ipaddr)) { eventsfailed("***** WRONG LINE ipaddr:{$ipaddr} column 13 " . @implode(" | ", $re) . "*****"); return; } if ($sitename == "-") { $h = parse_url($uri); if (isset($h["host"])) { $sitename = $h["host"]; } if ($sitename == "-") { eventsfailed("***** WRONG SITENAME \"{$sitename}\" column 13 " . @implode(" | ", $re) . "*****"); eventsfailed("{$buffer}"); eventsfailed("*"); $GLOBALS["REFUSED_REQUESTS"] = $GLOBALS["REFUSED_REQUESTS"] + 1; return; } if ($sitename == null) { eventsfailed("***** WRONG SITENAME \"{$sitename}\" column 13 " . @implode(" | ", $re) . "*****"); eventsfailed("{$buffer}"); eventsfailed("*"); $GLOBALS["REFUSED_REQUESTS"] = $GLOBALS["REFUSED_REQUESTS"] + 1; return; } } if (strpos($sitename, ":") > 0) { $XA = explode(":", $sitename); $sitename = $XA[0]; } if ($sitename == "127.0.0.1") { $GLOBALS["REFUSED_REQUESTS"] = $GLOBALS["REFUSED_REQUESTS"] + 1; if ($GLOBALS["VERBOSE"]) { events("127.0.0.1 -> uid = null -> SKIP"); } return; } if ($GLOBALS["ResolvIPStatistics"] == 1) { if (preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $sitename)) { $sitename = xRESOLV($sitename); } } if ($Forwarded == "unknown") { $Forwarded = null; } if ($Forwarded == "-") { $Forwarded = null; } if ($Forwarded == "0.0.0.0") { $Forwarded = null; } if ($Forwarded == "255.255.255.255") { $Forwarded = null; } if (strlen($Forwarded) > 4) { $ipaddr = $Forwarded; $mac = null; } $ipaddr = str_replace("%25", "-", $ipaddr); $mac = str_replace("%25", "-", $mac); if ($mac == "-") { $mac = null; } if ($ipaddr == "127.0.0.1" or $ipaddr == "::") { if ($uid == null) { $GLOBALS["REFUSED_REQUESTS"] = $GLOBALS["REFUSED_REQUESTS"] + 1; if ($GLOBALS["VERBOSE"]) { events("127.0.0.1 -> uid = null -> SKIP"); } return; } } if (preg_match("#([0-9:a-z]+)\$#", $mac, $z)) { $mac = $z[1]; } if ($SIZE == 0) { return; } if ($GLOBALS["VERBOSE"]) { events("ITEM: DATE......: {$logzdate}"); events("ITEM: MAC.......: {$mac}"); events("ITEM: IP........: {$ipaddr}"); events("ITEM: Size......: {$SIZE}"); events("ITEM: SQUID CODE: {$SquidCode}"); events("ITEM: HTTP CODE.: {$code_error}"); events("ITEM: uid.......: {$uid}"); events("ITEM: uri.......: {$uri}"); events("ITEM: UserAgent.: {$UserAgent}"); events("ITEM: Forwarded.: {$Forwarded}"); events("ITEM: SiteName..: {$sitename}"); events("ITEM: ProxyName.: {$GLOBALS["REMOTE_PROXY_NAME"]}"); } $GLOBALS["COUNT_HASH_TABLE"] = $GLOBALS["COUNT_HASH_TABLE"] + 1; $arrayURI = parse_url($uri); $sitename = $arrayURI["host"]; if (strpos($sitename, ":")) { $xtr = explode(":", $sitename); $sitename = $xtr[0]; if (preg_match("#^www\\.(.+)#", $sitename, $rz)) { $sitename = $rz[1]; } } // $uid=UID_MEM_CACHE($uid,$mac,$ipaddr); $TimeCache = date("YmdH"); $logfile_daemon = new logfile_daemon(); $cached = $logfile_daemon->CACHEDORNOT($SquidCode); if ($GLOBALS["DEBUG_MEM"]) { events("RTT: {$sitename} - {$SquidCode} = {$cached}"); } $GLOBALS["ACCEPTED_REQUESTS"] = $GLOBALS["ACCEPTED_REQUESTS"] + 1; if (!isset($GLOBALS["CATEGORIES"][$sitename])) { $GLOBALS["CATEGORIES"][$sitename] = $GLOBALS["MYSQL_CATZ"]->GET_CATEGORIES($sitename); } $MAIN["TIMESTAMP"] = time(); $MAIN["URI"] = $uri; $MAIN["sitename"] = $sitename; $familysite = $GLOBALS["SQUID_FAMILY_CLASS"]->GetFamilySites($sitename); $category = $GLOBALS["CATEGORIES"][$sitename]; $MAIN["SIZE"] = intval($SIZE); $MAIN["CACHED"] = $cached; if ($GLOBALS["UserAgentsStatistics"] == 1) { UserAgentsStatistics($UserAgent, $mac, $uid, $SIZE); } else { if ($GLOBALS["DEBUG_USERAGENT"]) { events("UserAgentsStatistics is disabled..."); } } CachedSizeMem($MAIN["CACHED"], $MAIN["SIZE"]); CachedUserMem($sitename, $SIZE, $mac, $uid, $ipaddr, $category, $familysite, $OUGROUP); }