function DumpMemory() { if (!isset($GLOBALS["MEMORY"]["ACCESSES"])) { return; } if (tool_time_sec($GLOBALS["MEMORY"]["TIME"]) < 30) { return; } $GLOBALS["MEMORY"]["TIME"] = time(); $filename = "/home/apache/artica-stats/requests.log"; $c = 0; while (list($KEYMD5, $ARRAY) = each($GLOBALS["MEMORY"]["ACCESSES"])) { $RQS = $ARRAY["RQS"]; $CODE = $ARRAY["CODE"]; $IPADDR = $ARRAY["IPADDR"]; $SIZE = $ARRAY["SIZE"]; $TIME = $ARRAY["TIME"]; $HOSTNAME = $ARRAY["HOSTNAME"]; $LINE = "{$TIME};{$HOSTNAME};{$IPADDR};{$CODE};{$RQS};{$SIZE}"; $c++; writeCompresslogs($filename, $LINE); } $GLOBALS["MEMORY"]["ACCESSES"] = array(); $GLOBALS["MEMORY"]["TIME"] = time(); events("Writing {$c} events..."); @unlink("/etc/artica-postfix/apache-tail.time"); @file_put_contents("/etc/artica-postfix/apache-tail.time", time()); }
function GoogleSafeBrowsingGet($PROTO, $servername) { if (isset($GLOBALS["SafeBrowsingSTOP"])) { if ($GLOBALS["SafeBrowsingSTOP"] > 0) { if (tool_time_sec($GLOBALS["SafeBrowsingSTOP"]) < 300) { return null; } } } $start_time = microtime(true); if (!isset($GLOBALS["PROXY"]["ArticaProxyServerEnabled"])) { $GLOBALS["PROXY"]["ArticaProxyServerEnabled"] = "no"; $GLOBALS["PROXY"]["ArticaProxyServerName"] = null; $GLOBALS["PROXY"]["ArticaProxyServerPort"] = null; $GLOBALS["PROXY"]["ArticaProxyServerUsername"] = null; $GLOBALS["PROXY"]["ArticaProxyServerUserPassword"] = null; } $ArticaProxyServerEnabled = $GLOBALS["PROXY"]["ArticaProxyServerEnabled"]; $ArticaProxyServerName = $GLOBALS["PROXY"]["ArticaProxyServerName"]; $ArticaProxyServerPort = $GLOBALS["PROXY"]["ArticaProxyServerPort"]; $ArticaProxyServerUsername = trim($GLOBALS["PROXY"]["ArticaProxyServerUsername"]); $ArticaProxyServerUserPassword = $GLOBALS["PROXY"]["ArticaProxyServerUserPassword"]; $servername = urlencode("http://{$servername}/"); $url = "https://sb-ssl.google.com/safebrowsing/api/lookup?client=api&apikey={$GLOBALS["GoogleSafeBrowsingApiKey"]}&appver=1.5.2&pver=3.1&url={$servername}"; if ($GLOBALS["GOOGLE_SAFE"]) { events("GoogleSafeBrowsingGet: {$url}"); } $curl = curl_init($url); curl_setopt($curl, CURLOPT_RETURNTRANSFER, TRUE); curl_setopt($curl, CURLOPT_FAILONERROR, FALSE); curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, FALSE); curl_setopt($curl, CURLOPT_FRESH_CONNECT, TRUE); curl_setopt($curl, CURLOPT_FORBID_REUSE, TRUE); curl_setopt($curl, CURLOPT_DNS_CACHE_TIMEOUT, 3600); curl_setopt($curl, CURLOPT_CONNECTTIMEOUT, 3); curl_setopt($curl, CURLOPT_TIMEOUT, 10); if (trim($GLOBALS["GoogleSafeBrowsingDNS"]) != null) { @curl_setopt($curl, CURLOPT_DNS_SERVERS, $GLOBALS["GoogleSafeBrowsingDNS"]); } if ($GLOBALS["GoogleSafeBrowsingInterface"] != null) { curl_setopt($curl, CURLOPT_INTERFACE, $GLOBALS["GoogleSafeBrowsingInterface"]); } if ($ArticaProxyServerEnabled == "yes") { curl_setopt($curl, CURLOPT_HTTPPROXYTUNNEL, FALSE); curl_setopt($curl, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); curl_setopt($curl, CURLOPT_PROXY, $ArticaProxyServerName); curl_setopt($curl, CURLOPT_PROXYPORT, $ArticaProxyServerPort); if ($ArticaProxyServerUsername != null) { curl_setopt($curl, CURLOPT_PROXYAUTH, CURLAUTH_BASIC); curl_setopt($curl, CURLOPT_PROXYUSERPWD, $ArticaProxyServerUsername . ':' . $ArticaProxyServerUserPassword); } } $response = curl_exec($curl); $http_status = curl_getinfo($curl, CURLINFO_HTTP_CODE); $end_time = microtime(true); $Infos = curl_getinfo($curl); $TimedSec = $end_time - $start_time; if ($GLOBALS["GOOGLE_SAFE"]) { events("GoogleSafeBrowsingGet: Connection {$TimedSec}ms"); } if (!$response) { if ($http_status == 204) { @curl_close($curl); return "clean"; } $errno = curl_errno($curl); $error_message = curl_strerror($errno); if ($errno == 28) { events("GoogleSafeBrowsingGet: DNS...: {$GLOBALS["GoogleSafeBrowsingDNS"]}, Interface \"{$GLOBALS["GoogleSafeBrowsingInterface"]}\""); ufdbg_admin_mysql(1, "PID {$GLOBALS["MYPID"]}: Google Safe Browsing Timed Out, skipping protection for 5mn", "Requested URL: {$url}\nSleeping during 5 minutes", __FILE__, __LINE__); $GLOBALS["SafeBrowsingSTOP"] = time(); } curl_close($curl); if (isset($GLOBALS["SafeBrowsingERROR"])) { if ($GLOBALS["SafeBrowsingERROR"] > 0) { if (tool_time_sec($GLOBALS["SafeBrowsingERROR"]) < 180) { return null; } } } ufdbg_admin_mysql(1, "PID {$GLOBALS["MYPID"]}: Google Safe Browsing HTTP Error code {$errno} ({$error_message})", "Requested URL: {$url}\n", __FILE__, __LINE__); $GLOBALS["SafeBrowsingERROR"] = time(); return null; } if (isset($GLOBALS["SafeBrowsingSTOP"])) { if ($GLOBALS["SafeBrowsingSTOP"] > 0) { ufdbg_admin_mysql(1, "PID {$GLOBALS["MYPID"]}: Google Safe Browsing relinked", "", __FILE__, __LINE__); $GLOBALS["SafeBrowsingSTOP"] = 0; } } if (isset($GLOBALS["SafeBrowsingERROR"])) { if ($GLOBALS["SafeBrowsingERROR"] > 0) { ufdbg_admin_mysql(1, "PID {$GLOBALS["MYPID"]}: Google Safe Browsing relinked", "", __FILE__, __LINE__); $GLOBALS["SafeBrowsingERROR"] = 0; } } curl_close($curl); return $response; }
function Parseline($buffer) { if (!isset($GLOBALS["TIMEEXEC"])) { $GLOBALS["TIMEEXEC"] = time(); } $main = json_decode($buffer); $timestamp = strtotime($main->timestamp); $zdate = date("Y-m-d H:i:s", $timestamp); $zdate_min = date("Y-m-d H:i:00", $timestamp); $event_type = $main->event_type; $src_ip = $main->src_ip; $src_port = $main->src_port; $dest_port = $main->dest_port; $dest_ip = $main->dest_ip; $proto = $main->proto; $signature_id = $main->alert->signature_id; $signature_rev = $main->alert->rev; $signature_string = $main->alert->signature; $category = $main->alert->category; $severity = $main->alert->severity; $uduniq = md5($category); $class_id = getClassification($uduniq, $category); if ($GLOBALS["VERBOSE"]) { events("BUFFER: {$uduniq}/{$category} = {$class_id}"); } $md5 = md5("{$zdate_min}{$src_ip}{$proto}{$dest_ip}{$dest_port}{$signature_id}"); if (isset($GLOBALS["FIREWALL"][$signature_id])) { XDENY($signature_id, $src_ip, $dest_port, $proto); } if (!isset($RULES[$md5])) { $RULES[$md5]["DATE"] = $zdate_min; $RULES[$md5]["SRC"] = $src_ip; $RULES[$md5]["DEST"] = $dest_ip; $RULES[$md5]["PROTO"] = $proto; $RULES[$md5]["DEST_PORT"] = $dest_port; $RULES[$md5]["SIG"] = $signature_id; $RULES[$md5]["severity"] = $severity; $RULES[$md5]["COUNT"] = 1; } else { $RULES[$md5]["COUNT"] = $RULES[$md5]["COUNT"] + 1; } if (!isset($SIG[$signature_id])) { if ($GLOBALS["VERBOSE"]) { events("BUFFER: {$signature_id} = {$signature_string}"); } $SIG[$signature_id] = $signature_string; } $cacheTailTime = tool_time_sec($GLOBALS["TIMEEXEC"]); if ($GLOBALS["VERBOSE"]) { events("TIME: {$GLOBALS["TIMEEXEC"]} = {$cacheTailTime}s / 10"); } if ($cacheTailTime > 10) { XDUMP($RULES); XDUMP_RULES($SIG); $GLOBALS["TIMEEXEC"] = time(); $RULES = array(); $SIG = array(); } events("{$zdate} {$event_type} {$proto} {$src_ip}:{$src_port} -> {$dest_ip}:{$dest_port} {$signature_id}/{$class_id}"); if ($GLOBALS["COUNT_RQS"] == 0) { $GLOBALS["COUNT_RQS"] = 1; } $ctrqs = intval($GLOBALS["COUNT_RQS"]); $ctrqs++; $GLOBALS["COUNT_RQS"] = $ctrqs; if ($GLOBALS["COUNT_RQS_TIME"] == 0) { $GLOBALS["COUNT_RQS_TIME"] = time(); } if ($GLOBALS["VERBOSE"]) { events(__LINE__ . " {$GLOBALS["COUNT_RQS"]} connexions"); } $buffer = null; }
function SEND_LOGS($ARRAY) { if (!isset($GLOBALS["XTIMECACHE"])) { $GLOBALS["XTIMECACHE"] = time(); } $UID = $ARRAY["UID"]; $IPADDR = $ARRAY["IPADDR"]; $MAC = $ARRAY["MAC"]; $CONTENT_TYPE = $ARRAY["CONTENT_TYPE"]; $HOST = $ARRAY["HOST"]; $SIZE = $ARRAY["SIZE"]; $keyMD5 = md5("{$UID}{$IPADDR}{$MAC}{$CONTENT_TYPE}{$HOST}"); if (tool_time_sec($GLOBALS["XTIMECACHE"]) > 10) { CHUNK(); } if (!isset($GLOBALS["LOGS"][$keyMD5])) { $GLOBALS["LOGS"][$keyMD5]["TIME"] = time(); $GLOBALS["LOGS"][$keyMD5]["CONTENT_TYPE"] = $CONTENT_TYPE; $GLOBALS["LOGS"][$keyMD5]["HOST"] = $HOST; $GLOBALS["LOGS"][$keyMD5]["MAC"] = $MAC; $GLOBALS["LOGS"][$keyMD5]["IPADDR"] = $IPADDR; $GLOBALS["LOGS"][$keyMD5]["UID"] = $UID; $GLOBALS["LOGS"][$keyMD5]["HIT"] = 1; $GLOBALS["LOGS"][$keyMD5]["SIZE"] = intval($SIZE); return; } $GLOBALS["LOGS"][$keyMD5]["SIZE"] = $GLOBALS["LOGS"][$keyMD5]["SIZE"] + $SIZE; $GLOBALS["LOGS"][$keyMD5]["HIT"] = $GLOBALS["LOGS"][$keyMD5]["HIT"] + 1; $GLOBALS["LOGS"][$keyMD5]["TIME"] = time(); }
function CachedUserMemDump() { $f = @fopen("{$GLOBALS["LogFileDeamonLogDir"]}/USERAUTDB.LOG", 'a'); $xtime = tool_time_sec($GLOBALS["LOGACCESS_TIME"]); if ($xtime < 10) { return; } while (list($KEYMD5, $line) = each($GLOBALS["UserAutDB"])) { @fwrite($f, "{$line}\n"); } $GLOBALS["UserAutDB"] = array(); @fclose($f); $c = 0; $MAIN = $GLOBALS["CACHEDUSersMem"]; $q = new influx(); $xRQS = 0; while (list($KEYMD5, $ARRAY) = each($MAIN)) { $zArray = array(); $zArray2 = array(); if (!isset($GLOBALS["CACHEDUSersMem"][$KEYMD5]["SITE"])) { unset($GLOBALS["CACHEDUSersMem"][$KEYMD5]); continue; } $CATEGORY = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["CATEGORY"]; $USERID = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["USERID"]; $IPADDR = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["IPADDR"]; $MAC = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["MAC"]; $SIZE = intval($GLOBALS["CACHEDUSersMem"][$KEYMD5]["SIZE"]); $SITE = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["SITE"]; $FAM = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["FAM"]; $RQS = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["RQS"]; $PROXYNAME = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["PROXYNAME"]; $GROUP = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["GROUP"]; $ORGA = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["ORGA"]; if ($SIZE > 0) { FILL_DISK_SIZES($USERID, $IPADDR, $MAC, $SIZE, $CATEGORY, $FAM); } if ($MAC == null) { $MAC = "00:00:00:00:00:00"; } if ($USERID == null) { $USERID = "none"; } $xRQS = $xRQS + $RQS; $line = time() . ":::{$CATEGORY}:::{$USERID}:::{$IPADDR}:::{$MAC}:::{$SIZE}:::{$SITE}:::{$FAM}:::{$RQS}:::{$PROXYNAME}:::{$GROUP}:::{$ORGA}"; $c++; writeCompresslogs("{$GLOBALS["LogFileDeamonLogDir"]}/ACCESS_LOG", $line); unset($GLOBALS["CACHEDUSersMem"][$KEYMD5]); } if (count($GLOBALS["USERRTT"]) > 0) { while (list($KEYMD5, $ARRAY) = each($GLOBALS["USERRTT"])) { if (!isset($GLOBALS["USERRTT"][$KEYMD5]["ORGA"])) { $GLOBALS["USERRTT"][$KEYMD5]["ORGA"] = null; } $USERID = $GLOBALS["USERRTT"][$KEYMD5]["USERID"]; $IPADDR = $GLOBALS["USERRTT"][$KEYMD5]["IPADDR"]; $MAC = $GLOBALS["USERRTT"][$KEYMD5]["MAC"]; $SIZE = intval($GLOBALS["USERRTT"][$KEYMD5]["SIZE"]); $RQS = $GLOBALS["USERRTT"][$KEYMD5]["RQS"]; $PROXYNAME = $GLOBALS["USERRTT"][$KEYMD5]["PROXYNAME"]; $GROUP = $GLOBALS["USERRTT"][$KEYMD5]["GROUP"]; $ORGA = $GLOBALS["USERRTT"][$KEYMD5]["ORGA"]; $line = time() . ":::{$USERID}:::{$IPADDR}:::{$MAC}:::{$SIZE}:::{$RQS}:::{$PROXYNAME}::{$GROUP}:::{$ORGA}"; writeCompresslogs("{$GLOBALS["LogFileDeamonLogDir"]}/USERS_LOG", $line); unset($GLOBALS["USERRTT"][$KEYMD5]); } } events("CachedUserMemDump:: Saving {$c}/{$xRQS} requests time={$xtime}s"); $GLOBALS["CACHEDUSersMemTime"] = array(); $GLOBALS["USERRTT"] = array(); $GLOBALS["LOGACCESS_TIME"] = time(); }
function CachedUserMemDump() { $xtime = tool_time_sec($GLOBALS["LOGACCESS_TIME"]); if ($xtime < 10) { return; } $c = 0; $MAIN = $GLOBALS["CACHEDUSersMem"]; $q = new influx(); $xRQS = 0; while (list($KEYMD5, $ARRAY) = each($MAIN)) { $zArray = array(); $zArray2 = array(); if (!isset($GLOBALS["CACHEDUSersMem"][$KEYMD5]["SITE"])) { unset($GLOBALS["CACHEDUSersMem"][$KEYMD5]); continue; } $CATEGORY = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["CATEGORY"]; $USERID = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["USERID"]; $IPADDR = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["IPADDR"]; $MAC = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["MAC"]; $SIZE = intval($GLOBALS["CACHEDUSersMem"][$KEYMD5]["SIZE"]); $SITE = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["SITE"]; $FAM = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["FAM"]; $RQS = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["RQS"]; $PROXYNAME = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["PROXYNAME"]; $GROUP = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["GROUP"]; $ORGA = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["ORGA"]; if ($MAC == null) { $MAC = "00:00:00:00:00:00"; } if ($USERID == null) { $USERID = "none"; } $xRQS = $xRQS + $RQS; $line = time() . ":::{$CATEGORY}:::{$USERID}:::{$IPADDR}:::{$MAC}:::{$SIZE}:::{$SITE}:::{$FAM}:::{$RQS}:::{$PROXYNAME}:::{$GROUP}:::{$ORGA}"; $c++; if ($GLOBALS["NoCompressStatisticsByHour"] == 0) { writeCompresslogs("{$GLOBALS["LogFileDeamonLogDir"]}/ACCESS_LOG", $line); unset($GLOBALS["CACHEDUSersMem"][$KEYMD5]); continue; } $zArray["tags"]["GROUP"] = $GROUP; $zArray["tags"]["ORGA"] = $ORGA; $zArray["tags"]["CATEGORY"] = $CATEGORY; $zArray["tags"]["USERID"] = $USERID; $zArray["tags"]["IPADDR"] = $IPADDR; $zArray["tags"]["MAC"] = $MAC; $zArray["fields"]["SIZE"] = $SIZE; $zArray["tags"]["SITE"] = $SITE; $zArray["tags"]["FAMILYSITE"] = $FAM; $zArray["fields"]["ZDATE"] = time(); $zArray["fields"]["RQS"] = $RQS; $zArray["tags"]["proxyname"] = $PROXYNAME; if ($GLOBALS["DEBUG_MEM"]) { events("INSERT - [{$KEYMD5}] {$zArray["tags"]["IPADDR"]} - {$zArray["tags"]["FAMILYSITE"]} - {$zArray["fields"]["SIZE"]}bytes {$zArray["fields"]["RQS"]}rqs [" . __LINE__ . "]"); } $q->insert("access_log", $zArray); unset($GLOBALS["CACHEDUSersMem"][$KEYMD5]); } if (count($GLOBALS["USERRTT"]) > 0) { while (list($KEYMD5, $ARRAY) = each($GLOBALS["USERRTT"])) { $USERID = $GLOBALS["USERRTT"][$KEYMD5]["USERID"]; $IPADDR = $GLOBALS["USERRTT"][$KEYMD5]["IPADDR"]; $MAC = $GLOBALS["USERRTT"][$KEYMD5]["MAC"]; $SIZE = intval($GLOBALS["USERRTT"][$KEYMD5]["SIZE"]); $RQS = $GLOBALS["USERRTT"][$KEYMD5]["RQS"]; $PROXYNAME = $GLOBALS["USERRTT"][$KEYMD5]["PROXYNAME"]; $GROUP = $GLOBALS["USERRTT"][$KEYMD5]["GROUP"]; $ORGA = $GLOBALS["USERRTT"][$KEYMD5]["ORGA"]; $line = time() . ":::{$USERID}:::{$IPADDR}:::{$MAC}:::{$SIZE}:::{$RQS}:::{$PROXYNAME}::{$GROUP}:::{$ORGA}"; writeCompresslogs("{$GLOBALS["LogFileDeamonLogDir"]}/USERS_LOG", $line); unset($GLOBALS["USERRTT"][$KEYMD5]); } } events("CachedUserMemDump:: Saving {$c}/{$xRQS} requests time={$xtime}s"); $GLOBALS["CACHEDUSersMemTime"] = array(); $GLOBALS["USERRTT"] = array(); $GLOBALS["LOGACCESS_TIME"] = time(); }