function ExplodeFile($filepath, $md5file = null)
{
$unix = new unix();
$LastScannLine = 0;
$GLOBALS["MYSQL_CATZ"] = new mysql_catz();
$GLOBALS["SQUID_FAMILY_CLASS"] = new squid_familysite();
if (!isset($GLOBALS["MYHOSTNAME"])) {
$unix = new unix();
$GLOBALS["MYHOSTNAME"] = $unix->hostname_g();
}
$GLOBALS["SEQUENCE"] = md5_file($filepath);
if (!is_file("{$filepath}.last")) {
if (is_file("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log")) {
$influx = new influx();
if ($influx->files_inject("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log")) {
@unlink("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log");
return true;
}
}
}
$handle = @fopen($filepath, "r");
if (!$handle) {
echo "Fopen failed on {$filepath}\n";
if ($md5file != null) {
mysql_progress($md5file, 100, 3, "Fopen {failed} on {$filepath}");
}
return false;
}
$countlines = 0;
if ($md5file != null) {
$countlines = $unix->COUNT_LINES_OF_FILE($filepath);
if ($md5file != null) {
mysql_progress($md5file, 10, 0, "Parsing {$countlines}");
}
}
if (is_file("{$filepath}.last")) {
$LastScannLine = intval(@file_get_contents("{$filepath}.last"));
}
$c = 0;
$d = 0;
$e = 0;
$prc = 0;
$prc_text = 0;
$mysql_first_time = 0;
if ($LastScannLine > 0) {
fseek($handle, $LastScannLine, SEEK_SET);
}
while (!feof($handle)) {
$c++;
$d++;
$e++;
if ($countlines > 0) {
$prc = $c / $countlines;
$prc = round($prc * 100);
if (!isset($GLOBALS["LAST_PRC"])) {
if ($GLOBALS["PROGRESS"]) {
echo "{$prc}%\n";
}
$GLOBALS["LAST_PRC"] = $prc;
} else {
if ($GLOBALS["LAST_PRC"] != $prc) {
if ($GLOBALS["PROGRESS"]) {
echo "{$prc}%\n";
}
$GLOBALS["LAST_PRC"] = $prc;
}
}
if ($prc > 10) {
if ($prc < 99) {
if ($prc > $prc_text) {
$array_load = sys_getloadavg();
$internal_load = $array_load[0];
$mem = round(memory_get_usage() / 1024 / 1000, 2);
$prc_design = FormatNumber($c) . "/" . FormatNumber($countlines);
if ($md5file != null) {
mysql_progress($md5file, $prc, 1, "{parsing} {$prc_design} {load}:{$internal_load} {memory}:{$mem}MB");
}
$prc_text = $prc;
}
}
}
}
if ($d > 50) {
$iSeek = ftell($handle);
@file_put_contents("{$filepath}.last", $iSeek);
if ($GLOBALS["VERBOSE"]) {
$prc_design = FormatNumber($c) . "/" . FormatNumber($countlines);
echo "{$prc}% {$prc_design}\n";
}
$d = 0;
}
if ($e > 500) {
$mem = round(memory_get_usage() / 1024 / 1000, 2);
$prc_design = FormatNumber($c) . "/" . FormatNumber($countlines);
if ($md5file != null) {
mysql_progress($md5file, $prc, 1, "{parsing} {$prc_design} {load}:{$internal_load} {memory}:{$mem}MB");
}
$e = 0;
}
$buffer = trim(fgets($handle));
if ($buffer == null) {
continue;
}
$array = parseAccessLine($buffer);
if (count($array) == 0) {
continue;
}
if ($mysql_first_time == 0) {
if (date("Y", $array["TIME"]) > 2001) {
$mysql_first_time = $array["TIME"];
mysql_first_time($md5file, $mysql_first_time);
}
}
CachedSizeMem($array["TIME"], $array["CACHED"], $array["SIZE"]);
if (intval($array["SIZE"]) == 0) {
if ($GLOBALS["VERBOSE"]) {
echo "Size = 0 " . __LINE__ . "\n";
}
}
CachedUserMem($array["TIME"], $array["SITENAME"], $array["SIZE"], null, $array["UID"], $array["IPADDR"], $array["CATEGORY"], $array["FAMILYSITE"]);
}
@unlink("{$filepath}.last");
mysql_last_time($md5file, $array["TIME"]);
CachedUserMem_dump();
CachedSizeMem_dump();
$influx = new influx();
$size = filesize("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log");
$size = $size / 1024;
$size = $size / 1024;
echo "Importing {$size}MB of data....\n";
if (!$influx->files_inject("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log")) {
@unlink("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log");
return false;
}
@unlink("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log");
return true;
}
function ParseSizeBuffer($buffer)
{
$proxyname = null;
if (!class_exists("class.logfile_daemon.inc")) {
include_once "/usr/share/artica-postfix/ressources/class.logfile_daemon.inc";
}
$re = explode(":::", $buffer);
if (preg_match("#^.*?\\):\\s+(.+)#", trim($re[0]), $rz)) {
$re[0] = $rz[1];
}
if ($GLOBALS["VERBOSE"]) {
events($buffer);
}
if ($GLOBALS["VERBOSE"]) {
events("ITEM: MAC......: {$re[0]} [" . __LINE__ . "]");
}
$mac = trim(strtolower($re[0]));
if ($mac == "-") {
$mac == null;
}
$mac = str_replace("-", ":", $mac);
if ($mac == "00:00:00:00:00:00") {
$mac = null;
}
$ipaddr = trim($re[1]);
if (!isset($GLOBALS["USER_MEM"])) {
$GLOBALS["USER_MEM"] = 0;
}
// uid
$uid = $re[2];
$uid2 = $re[3];
if ($uid == "-") {
$uid = null;
}
if ($uid2 == "-") {
$uid2 = null;
}
if (preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $uid2)) {
$uid2 = null;
}
if ($uid == null) {
if ($uid2 != null) {
$uid = $uid2;
}
}
$zdate = $re[4];
$xtime = time();
$SUFFIX_DATE = date("YmdH", $xtime);
$logzdate = date("Y-m-d H:i:s", $xtime);
$proto = $re[5];
$uri = $re[6];
$code_error = $re[8];
$SIZE = $re[9];
$SquidCode = $re[10];
$UserAgent = urldecode($re[11]);
$Forwarded = $re[12];
$sitename = trim($re[13]);
$hostname = trim($re[14]);
$response_time = $re[15];
$MimeType = trim($re[16]);
$sni = trim($re[17]);
$proxyname = trim($re[18]);
$OUGROUP = trim($re[19]);
$uid = trim(strtolower(str_replace("%20", " ", $uid)));
$uid = str_replace("%25", "-", $uid);
if ($uid == "-") {
$uid = null;
}
$Forwarded = str_replace("%25", "", $Forwarded);
//events("MimeType: ......: $MimeType");
if ($sni == "-") {
$sni = null;
}
if (strpos($uid, '$') > 0) {
if (substr($uid, strlen($uid) - 1, 1) == "\$") {
$uid = null;
}
}
if ($sni != null) {
if (preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $sitename)) {
$sitename = $sni;
}
}
if ($proxyname != null) {
if (preg_match("#proxyname=(.+)#", $proxyname, $re)) {
$GLOBALS["REMOTE_PROXY_NAME"] = $re[1];
}
} else {
$GLOBALS["REMOTE_PROXY_NAME"] = $GLOBALS["MYHOSTNAME"];
}
$GLOBALS["REMOTE_PROXY_NAME"] = str_replace("proxyname=", "", $GLOBALS["REMOTE_PROXY_NAME"]);
if (preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $uid)) {
$uid = null;
}
if (!preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $ipaddr)) {
eventsfailed("***** WRONG LINE ipaddr:{$ipaddr} column 13 " . @implode(" | ", $re) . "*****");
return;
}
if ($sitename == "-") {
$h = parse_url($uri);
if (isset($h["host"])) {
$sitename = $h["host"];
}
if ($sitename == "-") {
eventsfailed("***** WRONG SITENAME \"{$sitename}\" column 13 " . @implode(" | ", $re) . "*****");
eventsfailed("{$buffer}");
eventsfailed("*");
$GLOBALS["REFUSED_REQUESTS"] = $GLOBALS["REFUSED_REQUESTS"] + 1;
return;
}
if ($sitename == null) {
eventsfailed("***** WRONG SITENAME \"{$sitename}\" column 13 " . @implode(" | ", $re) . "*****");
eventsfailed("{$buffer}");
eventsfailed("*");
$GLOBALS["REFUSED_REQUESTS"] = $GLOBALS["REFUSED_REQUESTS"] + 1;
return;
}
}
if (strpos($sitename, ":") > 0) {
$XA = explode(":", $sitename);
$sitename = $XA[0];
}
if ($sitename == "127.0.0.1") {
$GLOBALS["REFUSED_REQUESTS"] = $GLOBALS["REFUSED_REQUESTS"] + 1;
if ($GLOBALS["VERBOSE"]) {
events("127.0.0.1 -> uid = null -> SKIP");
}
return;
}
if ($GLOBALS["ResolvIPStatistics"] == 1) {
if (preg_match("#^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $sitename)) {
$sitename = xRESOLV($sitename);
}
}
if ($Forwarded == "unknown") {
$Forwarded = null;
}
if ($Forwarded == "-") {
$Forwarded = null;
}
if ($Forwarded == "0.0.0.0") {
$Forwarded = null;
}
if ($Forwarded == "255.255.255.255") {
$Forwarded = null;
}
if (strlen($Forwarded) > 4) {
$ipaddr = $Forwarded;
$mac = null;
}
$ipaddr = str_replace("%25", "-", $ipaddr);
$mac = str_replace("%25", "-", $mac);
if ($mac == "-") {
$mac = null;
}
if ($ipaddr == "127.0.0.1" or $ipaddr == "::") {
if ($uid == null) {
$GLOBALS["REFUSED_REQUESTS"] = $GLOBALS["REFUSED_REQUESTS"] + 1;
if ($GLOBALS["VERBOSE"]) {
events("127.0.0.1 -> uid = null -> SKIP");
}
return;
}
}
if (preg_match("#([0-9:a-z]+)\$#", $mac, $z)) {
$mac = $z[1];
}
if ($SIZE == 0) {
return;
}
if ($GLOBALS["VERBOSE"]) {
events("ITEM: DATE......: {$logzdate}");
events("ITEM: MAC.......: {$mac}");
events("ITEM: IP........: {$ipaddr}");
events("ITEM: Size......: {$SIZE}");
events("ITEM: SQUID CODE: {$SquidCode}");
events("ITEM: HTTP CODE.: {$code_error}");
events("ITEM: uid.......: {$uid}");
events("ITEM: uri.......: {$uri}");
events("ITEM: UserAgent.: {$UserAgent}");
events("ITEM: Forwarded.: {$Forwarded}");
events("ITEM: SiteName..: {$sitename}");
events("ITEM: ProxyName.: {$GLOBALS["REMOTE_PROXY_NAME"]}");
}
$GLOBALS["COUNT_HASH_TABLE"] = $GLOBALS["COUNT_HASH_TABLE"] + 1;
$arrayURI = parse_url($uri);
$sitename = $arrayURI["host"];
if (strpos($sitename, ":")) {
$xtr = explode(":", $sitename);
$sitename = $xtr[0];
if (preg_match("#^www\\.(.+)#", $sitename, $rz)) {
$sitename = $rz[1];
}
}
// $uid=UID_MEM_CACHE($uid,$mac,$ipaddr);
$TimeCache = date("YmdH");
$logfile_daemon = new logfile_daemon();
$cached = $logfile_daemon->CACHEDORNOT($SquidCode);
if ($GLOBALS["DEBUG_MEM"]) {
events("RTT: {$sitename} - {$SquidCode} = {$cached}");
}
$GLOBALS["ACCEPTED_REQUESTS"] = $GLOBALS["ACCEPTED_REQUESTS"] + 1;
if (!isset($GLOBALS["CATEGORIES"][$sitename])) {
$GLOBALS["CATEGORIES"][$sitename] = $GLOBALS["MYSQL_CATZ"]->GET_CATEGORIES($sitename);
}
$MAIN["TIMESTAMP"] = time();
$MAIN["URI"] = $uri;
$MAIN["sitename"] = $sitename;
$familysite = $GLOBALS["SQUID_FAMILY_CLASS"]->GetFamilySites($sitename);
$category = $GLOBALS["CATEGORIES"][$sitename];
$MAIN["SIZE"] = intval($SIZE);
$MAIN["CACHED"] = $cached;
if ($GLOBALS["UserAgentsStatistics"] == 1) {
UserAgentsStatistics($UserAgent, $mac, $uid, $SIZE);
} else {
if ($GLOBALS["DEBUG_USERAGENT"]) {
events("UserAgentsStatistics is disabled...");
}
}
CachedSizeMem($MAIN["CACHED"], $MAIN["SIZE"]);
CachedUserMem($sitename, $SIZE, $mac, $uid, $ipaddr, $category, $familysite, $OUGROUP);
}
