function ExplodeFile($filepath, $md5file = null) { $unix = new unix(); $LastScannLine = 0; $GLOBALS["MYSQL_CATZ"] = new mysql_catz(); $GLOBALS["SQUID_FAMILY_CLASS"] = new squid_familysite(); if (!isset($GLOBALS["MYHOSTNAME"])) { $unix = new unix(); $GLOBALS["MYHOSTNAME"] = $unix->hostname_g(); } $GLOBALS["SEQUENCE"] = md5_file($filepath); if (!is_file("{$filepath}.last")) { if (is_file("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log")) { $influx = new influx(); if ($influx->files_inject("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log")) { @unlink("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log"); return true; } } } $handle = @fopen($filepath, "r"); if (!$handle) { echo "Fopen failed on {$filepath}\n"; if ($md5file != null) { mysql_progress($md5file, 100, 3, "Fopen {failed} on {$filepath}"); } return false; } $countlines = 0; if ($md5file != null) { $countlines = $unix->COUNT_LINES_OF_FILE($filepath); if ($md5file != null) { mysql_progress($md5file, 10, 0, "Parsing {$countlines}"); } } if (is_file("{$filepath}.last")) { $LastScannLine = intval(@file_get_contents("{$filepath}.last")); } $c = 0; $d = 0; $e = 0; $prc = 0; $prc_text = 0; $mysql_first_time = 0; if ($LastScannLine > 0) { fseek($handle, $LastScannLine, SEEK_SET); } while (!feof($handle)) { $c++; $d++; $e++; if ($countlines > 0) { $prc = $c / $countlines; $prc = round($prc * 100); if (!isset($GLOBALS["LAST_PRC"])) { if ($GLOBALS["PROGRESS"]) { echo "{$prc}%\n"; } $GLOBALS["LAST_PRC"] = $prc; } else { if ($GLOBALS["LAST_PRC"] != $prc) { if ($GLOBALS["PROGRESS"]) { echo "{$prc}%\n"; } $GLOBALS["LAST_PRC"] = $prc; } } if ($prc > 10) { if ($prc < 99) { if ($prc > $prc_text) { $array_load = sys_getloadavg(); $internal_load = $array_load[0]; $mem = round(memory_get_usage() / 1024 / 1000, 2); $prc_design = FormatNumber($c) . "/" . FormatNumber($countlines); if ($md5file != null) { mysql_progress($md5file, $prc, 1, "{parsing} {$prc_design} {load}:{$internal_load} {memory}:{$mem}MB"); } $prc_text = $prc; } } } } if ($d > 50) { $iSeek = ftell($handle); @file_put_contents("{$filepath}.last", $iSeek); if ($GLOBALS["VERBOSE"]) { $prc_design = FormatNumber($c) . "/" . FormatNumber($countlines); echo "{$prc}% {$prc_design}\n"; } $d = 0; } if ($e > 500) { $mem = round(memory_get_usage() / 1024 / 1000, 2); $prc_design = FormatNumber($c) . "/" . FormatNumber($countlines); if ($md5file != null) { mysql_progress($md5file, $prc, 1, "{parsing} {$prc_design} {load}:{$internal_load} {memory}:{$mem}MB"); } $e = 0; } $buffer = trim(fgets($handle)); if ($buffer == null) { continue; } $array = parseAccessLine($buffer); if (count($array) == 0) { continue; } if ($mysql_first_time == 0) { if (date("Y", $array["TIME"]) > 2001) { $mysql_first_time = $array["TIME"]; mysql_first_time($md5file, $mysql_first_time); } } CachedSizeMem($array["TIME"], $array["CACHED"], $array["SIZE"]); if (intval($array["SIZE"]) == 0) { if ($GLOBALS["VERBOSE"]) { echo "Size = 0 " . __LINE__ . "\n"; } } CachedUserMem($array["TIME"], $array["SITENAME"], $array["SIZE"], null, $array["UID"], $array["IPADDR"], $array["CATEGORY"], $array["FAMILYSITE"]); } @unlink("{$filepath}.last"); mysql_last_time($md5file, $array["TIME"]); CachedUserMem_dump(); CachedSizeMem_dump(); $influx = new influx(); $size = filesize("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log"); $size = $size / 1024; $size = $size / 1024; echo "Importing {$size}MB of data....\n"; if (!$influx->files_inject("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log")) { @unlink("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log"); return false; } @unlink("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log"); return true; }
function ParseSizeBuffer($buffer) { $proxyname = null; if (!class_exists("class.logfile_daemon.inc")) { include_once "/usr/share/artica-postfix/ressources/class.logfile_daemon.inc"; } $re = explode(":::", $buffer); if (preg_match("#^.*?\\):\\s+(.+)#", trim($re[0]), $rz)) { $re[0] = $rz[1]; } if ($GLOBALS["VERBOSE"]) { events($buffer); } if ($GLOBALS["VERBOSE"]) { events("ITEM: MAC......: {$re[0]} [" . __LINE__ . "]"); } $mac = trim(strtolower($re[0])); if ($mac == "-") { $mac == null; } $mac = str_replace("-", ":", $mac); if ($mac == "00:00:00:00:00:00") { $mac = null; } $ipaddr = trim($re[1]); if (!isset($GLOBALS["USER_MEM"])) { $GLOBALS["USER_MEM"] = 0; } // uid $uid = $re[2]; $uid2 = $re[3]; if ($uid == "-") { $uid = null; } if ($uid2 == "-") { $uid2 = null; } if (preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $uid2)) { $uid2 = null; } if ($uid == null) { if ($uid2 != null) { $uid = $uid2; } } $zdate = $re[4]; $xtime = time(); $SUFFIX_DATE = date("YmdH", $xtime); $logzdate = date("Y-m-d H:i:s", $xtime); $proto = $re[5]; $uri = $re[6]; $code_error = $re[8]; $SIZE = $re[9]; $SquidCode = $re[10]; $UserAgent = urldecode($re[11]); $Forwarded = $re[12]; $sitename = trim($re[13]); $hostname = trim($re[14]); $response_time = $re[15]; $MimeType = trim($re[16]); $sni = trim($re[17]); $proxyname = trim($re[18]); $OUGROUP = trim($re[19]); $uid = trim(strtolower(str_replace("%20", " ", $uid))); $uid = str_replace("%25", "-", $uid); if ($uid == "-") { $uid = null; } $Forwarded = str_replace("%25", "", $Forwarded); //events("MimeType: ......: $MimeType"); if ($sni == "-") { $sni = null; } if (strpos($uid, '$') > 0) { if (substr($uid, strlen($uid) - 1, 1) == "\$") { $uid = null; } } if ($sni != null) { if (preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $sitename)) { $sitename = $sni; } } if ($proxyname != null) { if (preg_match("#proxyname=(.+)#", $proxyname, $re)) { $GLOBALS["REMOTE_PROXY_NAME"] = $re[1]; } } else { $GLOBALS["REMOTE_PROXY_NAME"] = $GLOBALS["MYHOSTNAME"]; } $GLOBALS["REMOTE_PROXY_NAME"] = str_replace("proxyname=", "", $GLOBALS["REMOTE_PROXY_NAME"]); if (preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $uid)) { $uid = null; } if (!preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $ipaddr)) { eventsfailed("***** WRONG LINE ipaddr:{$ipaddr} column 13 " . @implode(" | ", $re) . "*****"); return; } if ($sitename == "-") { $h = parse_url($uri); if (isset($h["host"])) { $sitename = $h["host"]; } if ($sitename == "-") { eventsfailed("***** WRONG SITENAME \"{$sitename}\" column 13 " . @implode(" | ", $re) . "*****"); eventsfailed("{$buffer}"); eventsfailed("*"); $GLOBALS["REFUSED_REQUESTS"] = $GLOBALS["REFUSED_REQUESTS"] + 1; return; } if ($sitename == null) { eventsfailed("***** WRONG SITENAME \"{$sitename}\" column 13 " . @implode(" | ", $re) . "*****"); eventsfailed("{$buffer}"); eventsfailed("*"); $GLOBALS["REFUSED_REQUESTS"] = $GLOBALS["REFUSED_REQUESTS"] + 1; return; } } if (strpos($sitename, ":") > 0) { $XA = explode(":", $sitename); $sitename = $XA[0]; } if ($sitename == "127.0.0.1") { $GLOBALS["REFUSED_REQUESTS"] = $GLOBALS["REFUSED_REQUESTS"] + 1; if ($GLOBALS["VERBOSE"]) { events("127.0.0.1 -> uid = null -> SKIP"); } return; } if ($GLOBALS["ResolvIPStatistics"] == 1) { if (preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $sitename)) { $sitename = xRESOLV($sitename); } } if ($Forwarded == "unknown") { $Forwarded = null; } if ($Forwarded == "-") { $Forwarded = null; } if ($Forwarded == "0.0.0.0") { $Forwarded = null; } if ($Forwarded == "255.255.255.255") { $Forwarded = null; } if (strlen($Forwarded) > 4) { $ipaddr = $Forwarded; $mac = null; } $ipaddr = str_replace("%25", "-", $ipaddr); $mac = str_replace("%25", "-", $mac); if ($mac == "-") { $mac = null; } if ($ipaddr == "127.0.0.1" or $ipaddr == "::") { if ($uid == null) { $GLOBALS["REFUSED_REQUESTS"] = $GLOBALS["REFUSED_REQUESTS"] + 1; if ($GLOBALS["VERBOSE"]) { events("127.0.0.1 -> uid = null -> SKIP"); } return; } } if (preg_match("#([0-9:a-z]+)\$#", $mac, $z)) { $mac = $z[1]; } if ($SIZE == 0) { return; } if ($GLOBALS["VERBOSE"]) { events("ITEM: DATE......: {$logzdate}"); events("ITEM: MAC.......: {$mac}"); events("ITEM: IP........: {$ipaddr}"); events("ITEM: Size......: {$SIZE}"); events("ITEM: SQUID CODE: {$SquidCode}"); events("ITEM: HTTP CODE.: {$code_error}"); events("ITEM: uid.......: {$uid}"); events("ITEM: uri.......: {$uri}"); events("ITEM: UserAgent.: {$UserAgent}"); events("ITEM: Forwarded.: {$Forwarded}"); events("ITEM: SiteName..: {$sitename}"); events("ITEM: ProxyName.: {$GLOBALS["REMOTE_PROXY_NAME"]}"); } $GLOBALS["COUNT_HASH_TABLE"] = $GLOBALS["COUNT_HASH_TABLE"] + 1; $arrayURI = parse_url($uri); $sitename = $arrayURI["host"]; if (strpos($sitename, ":")) { $xtr = explode(":", $sitename); $sitename = $xtr[0]; if (preg_match("#^www\\.(.+)#", $sitename, $rz)) { $sitename = $rz[1]; } } // $uid=UID_MEM_CACHE($uid,$mac,$ipaddr); $TimeCache = date("YmdH"); $logfile_daemon = new logfile_daemon(); $cached = $logfile_daemon->CACHEDORNOT($SquidCode); if ($GLOBALS["DEBUG_MEM"]) { events("RTT: {$sitename} - {$SquidCode} = {$cached}"); } $GLOBALS["ACCEPTED_REQUESTS"] = $GLOBALS["ACCEPTED_REQUESTS"] + 1; if (!isset($GLOBALS["CATEGORIES"][$sitename])) { $GLOBALS["CATEGORIES"][$sitename] = $GLOBALS["MYSQL_CATZ"]->GET_CATEGORIES($sitename); } $MAIN["TIMESTAMP"] = time(); $MAIN["URI"] = $uri; $MAIN["sitename"] = $sitename; $familysite = $GLOBALS["SQUID_FAMILY_CLASS"]->GetFamilySites($sitename); $category = $GLOBALS["CATEGORIES"][$sitename]; $MAIN["SIZE"] = intval($SIZE); $MAIN["CACHED"] = $cached; if ($GLOBALS["UserAgentsStatistics"] == 1) { UserAgentsStatistics($UserAgent, $mac, $uid, $SIZE); } else { if ($GLOBALS["DEBUG_USERAGENT"]) { events("UserAgentsStatistics is disabled..."); } } CachedSizeMem($MAIN["CACHED"], $MAIN["SIZE"]); CachedUserMem($sitename, $SIZE, $mac, $uid, $ipaddr, $category, $familysite, $OUGROUP); }