function DumpMemory()
{
if (!isset($GLOBALS["MEMORY"]["ACCESSES"])) {
return;
}
if (tool_time_sec($GLOBALS["MEMORY"]["TIME"]) < 30) {
return;
}
$GLOBALS["MEMORY"]["TIME"] = time();
$filename = "/home/apache/artica-stats/requests.log";
$c = 0;
while (list($KEYMD5, $ARRAY) = each($GLOBALS["MEMORY"]["ACCESSES"])) {
$RQS = $ARRAY["RQS"];
$CODE = $ARRAY["CODE"];
$IPADDR = $ARRAY["IPADDR"];
$SIZE = $ARRAY["SIZE"];
$TIME = $ARRAY["TIME"];
$HOSTNAME = $ARRAY["HOSTNAME"];
$LINE = "{$TIME};{$HOSTNAME};{$IPADDR};{$CODE};{$RQS};{$SIZE}";
$c++;
writeCompresslogs($filename, $LINE);
}
$GLOBALS["MEMORY"]["ACCESSES"] = array();
$GLOBALS["MEMORY"]["TIME"] = time();
events("Writing {$c} events...");
@unlink("/etc/artica-postfix/apache-tail.time");
@file_put_contents("/etc/artica-postfix/apache-tail.time", time());
}
function GoogleSafeBrowsingGet($PROTO, $servername)
{
if (isset($GLOBALS["SafeBrowsingSTOP"])) {
if ($GLOBALS["SafeBrowsingSTOP"] > 0) {
if (tool_time_sec($GLOBALS["SafeBrowsingSTOP"]) < 300) {
return null;
}
}
}
$start_time = microtime(true);
if (!isset($GLOBALS["PROXY"]["ArticaProxyServerEnabled"])) {
$GLOBALS["PROXY"]["ArticaProxyServerEnabled"] = "no";
$GLOBALS["PROXY"]["ArticaProxyServerName"] = null;
$GLOBALS["PROXY"]["ArticaProxyServerPort"] = null;
$GLOBALS["PROXY"]["ArticaProxyServerUsername"] = null;
$GLOBALS["PROXY"]["ArticaProxyServerUserPassword"] = null;
}
$ArticaProxyServerEnabled = $GLOBALS["PROXY"]["ArticaProxyServerEnabled"];
$ArticaProxyServerName = $GLOBALS["PROXY"]["ArticaProxyServerName"];
$ArticaProxyServerPort = $GLOBALS["PROXY"]["ArticaProxyServerPort"];
$ArticaProxyServerUsername = trim($GLOBALS["PROXY"]["ArticaProxyServerUsername"]);
$ArticaProxyServerUserPassword = $GLOBALS["PROXY"]["ArticaProxyServerUserPassword"];
$servername = urlencode("http://{$servername}/");
$url = "https://sb-ssl.google.com/safebrowsing/api/lookup?client=api&apikey={$GLOBALS["GoogleSafeBrowsingApiKey"]}&appver=1.5.2&pver=3.1&url={$servername}";
if ($GLOBALS["GOOGLE_SAFE"]) {
events("GoogleSafeBrowsingGet: {$url}");
}
$curl = curl_init($url);
curl_setopt($curl, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt($curl, CURLOPT_FAILONERROR, FALSE);
curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, FALSE);
curl_setopt($curl, CURLOPT_FRESH_CONNECT, TRUE);
curl_setopt($curl, CURLOPT_FORBID_REUSE, TRUE);
curl_setopt($curl, CURLOPT_DNS_CACHE_TIMEOUT, 3600);
curl_setopt($curl, CURLOPT_CONNECTTIMEOUT, 3);
curl_setopt($curl, CURLOPT_TIMEOUT, 10);
if (trim($GLOBALS["GoogleSafeBrowsingDNS"]) != null) {
@curl_setopt($curl, CURLOPT_DNS_SERVERS, $GLOBALS["GoogleSafeBrowsingDNS"]);
}
if ($GLOBALS["GoogleSafeBrowsingInterface"] != null) {
curl_setopt($curl, CURLOPT_INTERFACE, $GLOBALS["GoogleSafeBrowsingInterface"]);
}
if ($ArticaProxyServerEnabled == "yes") {
curl_setopt($curl, CURLOPT_HTTPPROXYTUNNEL, FALSE);
curl_setopt($curl, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
curl_setopt($curl, CURLOPT_PROXY, $ArticaProxyServerName);
curl_setopt($curl, CURLOPT_PROXYPORT, $ArticaProxyServerPort);
if ($ArticaProxyServerUsername != null) {
curl_setopt($curl, CURLOPT_PROXYAUTH, CURLAUTH_BASIC);
curl_setopt($curl, CURLOPT_PROXYUSERPWD, $ArticaProxyServerUsername . ':' . $ArticaProxyServerUserPassword);
}
}
$response = curl_exec($curl);
$http_status = curl_getinfo($curl, CURLINFO_HTTP_CODE);
$end_time = microtime(true);
$Infos = curl_getinfo($curl);
$TimedSec = $end_time - $start_time;
if ($GLOBALS["GOOGLE_SAFE"]) {
events("GoogleSafeBrowsingGet: Connection {$TimedSec}ms");
}
if (!$response) {
if ($http_status == 204) {
@curl_close($curl);
return "clean";
}
$errno = curl_errno($curl);
$error_message = curl_strerror($errno);
if ($errno == 28) {
events("GoogleSafeBrowsingGet: DNS...: {$GLOBALS["GoogleSafeBrowsingDNS"]}, Interface \"{$GLOBALS["GoogleSafeBrowsingInterface"]}\"");
ufdbg_admin_mysql(1, "PID {$GLOBALS["MYPID"]}: Google Safe Browsing Timed Out, skipping protection for 5mn", "Requested URL: {$url}\nSleeping during 5 minutes", __FILE__, __LINE__);
$GLOBALS["SafeBrowsingSTOP"] = time();
}
curl_close($curl);
if (isset($GLOBALS["SafeBrowsingERROR"])) {
if ($GLOBALS["SafeBrowsingERROR"] > 0) {
if (tool_time_sec($GLOBALS["SafeBrowsingERROR"]) < 180) {
return null;
}
}
}
ufdbg_admin_mysql(1, "PID {$GLOBALS["MYPID"]}: Google Safe Browsing HTTP Error code {$errno} ({$error_message})", "Requested URL: {$url}\n", __FILE__, __LINE__);
$GLOBALS["SafeBrowsingERROR"] = time();
return null;
}
if (isset($GLOBALS["SafeBrowsingSTOP"])) {
if ($GLOBALS["SafeBrowsingSTOP"] > 0) {
ufdbg_admin_mysql(1, "PID {$GLOBALS["MYPID"]}: Google Safe Browsing relinked", "", __FILE__, __LINE__);
$GLOBALS["SafeBrowsingSTOP"] = 0;
}
}
if (isset($GLOBALS["SafeBrowsingERROR"])) {
if ($GLOBALS["SafeBrowsingERROR"] > 0) {
ufdbg_admin_mysql(1, "PID {$GLOBALS["MYPID"]}: Google Safe Browsing relinked", "", __FILE__, __LINE__);
$GLOBALS["SafeBrowsingERROR"] = 0;
}
}
curl_close($curl);
return $response;
}
function Parseline($buffer)
{
if (!isset($GLOBALS["TIMEEXEC"])) {
$GLOBALS["TIMEEXEC"] = time();
}
$main = json_decode($buffer);
$timestamp = strtotime($main->timestamp);
$zdate = date("Y-m-d H:i:s", $timestamp);
$zdate_min = date("Y-m-d H:i:00", $timestamp);
$event_type = $main->event_type;
$src_ip = $main->src_ip;
$src_port = $main->src_port;
$dest_port = $main->dest_port;
$dest_ip = $main->dest_ip;
$proto = $main->proto;
$signature_id = $main->alert->signature_id;
$signature_rev = $main->alert->rev;
$signature_string = $main->alert->signature;
$category = $main->alert->category;
$severity = $main->alert->severity;
$uduniq = md5($category);
$class_id = getClassification($uduniq, $category);
if ($GLOBALS["VERBOSE"]) {
events("BUFFER: {$uduniq}/{$category} = {$class_id}");
}
$md5 = md5("{$zdate_min}{$src_ip}{$proto}{$dest_ip}{$dest_port}{$signature_id}");
if (isset($GLOBALS["FIREWALL"][$signature_id])) {
XDENY($signature_id, $src_ip, $dest_port, $proto);
}
if (!isset($RULES[$md5])) {
$RULES[$md5]["DATE"] = $zdate_min;
$RULES[$md5]["SRC"] = $src_ip;
$RULES[$md5]["DEST"] = $dest_ip;
$RULES[$md5]["PROTO"] = $proto;
$RULES[$md5]["DEST_PORT"] = $dest_port;
$RULES[$md5]["SIG"] = $signature_id;
$RULES[$md5]["severity"] = $severity;
$RULES[$md5]["COUNT"] = 1;
} else {
$RULES[$md5]["COUNT"] = $RULES[$md5]["COUNT"] + 1;
}
if (!isset($SIG[$signature_id])) {
if ($GLOBALS["VERBOSE"]) {
events("BUFFER: {$signature_id} = {$signature_string}");
}
$SIG[$signature_id] = $signature_string;
}
$cacheTailTime = tool_time_sec($GLOBALS["TIMEEXEC"]);
if ($GLOBALS["VERBOSE"]) {
events("TIME: {$GLOBALS["TIMEEXEC"]} = {$cacheTailTime}s / 10");
}
if ($cacheTailTime > 10) {
XDUMP($RULES);
XDUMP_RULES($SIG);
$GLOBALS["TIMEEXEC"] = time();
$RULES = array();
$SIG = array();
}
events("{$zdate} {$event_type} {$proto} {$src_ip}:{$src_port} -> {$dest_ip}:{$dest_port} {$signature_id}/{$class_id}");
if ($GLOBALS["COUNT_RQS"] == 0) {
$GLOBALS["COUNT_RQS"] = 1;
}
$ctrqs = intval($GLOBALS["COUNT_RQS"]);
$ctrqs++;
$GLOBALS["COUNT_RQS"] = $ctrqs;
if ($GLOBALS["COUNT_RQS_TIME"] == 0) {
$GLOBALS["COUNT_RQS_TIME"] = time();
}
if ($GLOBALS["VERBOSE"]) {
events(__LINE__ . " {$GLOBALS["COUNT_RQS"]} connexions");
}
$buffer = null;
}
function SEND_LOGS($ARRAY)
{
if (!isset($GLOBALS["XTIMECACHE"])) {
$GLOBALS["XTIMECACHE"] = time();
}
$UID = $ARRAY["UID"];
$IPADDR = $ARRAY["IPADDR"];
$MAC = $ARRAY["MAC"];
$CONTENT_TYPE = $ARRAY["CONTENT_TYPE"];
$HOST = $ARRAY["HOST"];
$SIZE = $ARRAY["SIZE"];
$keyMD5 = md5("{$UID}{$IPADDR}{$MAC}{$CONTENT_TYPE}{$HOST}");
if (tool_time_sec($GLOBALS["XTIMECACHE"]) > 10) {
CHUNK();
}
if (!isset($GLOBALS["LOGS"][$keyMD5])) {
$GLOBALS["LOGS"][$keyMD5]["TIME"] = time();
$GLOBALS["LOGS"][$keyMD5]["CONTENT_TYPE"] = $CONTENT_TYPE;
$GLOBALS["LOGS"][$keyMD5]["HOST"] = $HOST;
$GLOBALS["LOGS"][$keyMD5]["MAC"] = $MAC;
$GLOBALS["LOGS"][$keyMD5]["IPADDR"] = $IPADDR;
$GLOBALS["LOGS"][$keyMD5]["UID"] = $UID;
$GLOBALS["LOGS"][$keyMD5]["HIT"] = 1;
$GLOBALS["LOGS"][$keyMD5]["SIZE"] = intval($SIZE);
return;
}
$GLOBALS["LOGS"][$keyMD5]["SIZE"] = $GLOBALS["LOGS"][$keyMD5]["SIZE"] + $SIZE;
$GLOBALS["LOGS"][$keyMD5]["HIT"] = $GLOBALS["LOGS"][$keyMD5]["HIT"] + 1;
$GLOBALS["LOGS"][$keyMD5]["TIME"] = time();
}
function CachedUserMemDump()
{
$f = @fopen("{$GLOBALS["LogFileDeamonLogDir"]}/USERAUTDB.LOG", 'a');
$xtime = tool_time_sec($GLOBALS["LOGACCESS_TIME"]);
if ($xtime < 10) {
return;
}
while (list($KEYMD5, $line) = each($GLOBALS["UserAutDB"])) {
@fwrite($f, "{$line}\n");
}
$GLOBALS["UserAutDB"] = array();
@fclose($f);
$c = 0;
$MAIN = $GLOBALS["CACHEDUSersMem"];
$q = new influx();
$xRQS = 0;
while (list($KEYMD5, $ARRAY) = each($MAIN)) {
$zArray = array();
$zArray2 = array();
if (!isset($GLOBALS["CACHEDUSersMem"][$KEYMD5]["SITE"])) {
unset($GLOBALS["CACHEDUSersMem"][$KEYMD5]);
continue;
}
$CATEGORY = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["CATEGORY"];
$USERID = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["USERID"];
$IPADDR = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["IPADDR"];
$MAC = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["MAC"];
$SIZE = intval($GLOBALS["CACHEDUSersMem"][$KEYMD5]["SIZE"]);
$SITE = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["SITE"];
$FAM = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["FAM"];
$RQS = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["RQS"];
$PROXYNAME = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["PROXYNAME"];
$GROUP = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["GROUP"];
$ORGA = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["ORGA"];
if ($SIZE > 0) {
FILL_DISK_SIZES($USERID, $IPADDR, $MAC, $SIZE, $CATEGORY, $FAM);
}
if ($MAC == null) {
$MAC = "00:00:00:00:00:00";
}
if ($USERID == null) {
$USERID = "none";
}
$xRQS = $xRQS + $RQS;
$line = time() . ":::{$CATEGORY}:::{$USERID}:::{$IPADDR}:::{$MAC}:::{$SIZE}:::{$SITE}:::{$FAM}:::{$RQS}:::{$PROXYNAME}:::{$GROUP}:::{$ORGA}";
$c++;
writeCompresslogs("{$GLOBALS["LogFileDeamonLogDir"]}/ACCESS_LOG", $line);
unset($GLOBALS["CACHEDUSersMem"][$KEYMD5]);
}
if (count($GLOBALS["USERRTT"]) > 0) {
while (list($KEYMD5, $ARRAY) = each($GLOBALS["USERRTT"])) {
if (!isset($GLOBALS["USERRTT"][$KEYMD5]["ORGA"])) {
$GLOBALS["USERRTT"][$KEYMD5]["ORGA"] = null;
}
$USERID = $GLOBALS["USERRTT"][$KEYMD5]["USERID"];
$IPADDR = $GLOBALS["USERRTT"][$KEYMD5]["IPADDR"];
$MAC = $GLOBALS["USERRTT"][$KEYMD5]["MAC"];
$SIZE = intval($GLOBALS["USERRTT"][$KEYMD5]["SIZE"]);
$RQS = $GLOBALS["USERRTT"][$KEYMD5]["RQS"];
$PROXYNAME = $GLOBALS["USERRTT"][$KEYMD5]["PROXYNAME"];
$GROUP = $GLOBALS["USERRTT"][$KEYMD5]["GROUP"];
$ORGA = $GLOBALS["USERRTT"][$KEYMD5]["ORGA"];
$line = time() . ":::{$USERID}:::{$IPADDR}:::{$MAC}:::{$SIZE}:::{$RQS}:::{$PROXYNAME}::{$GROUP}:::{$ORGA}";
writeCompresslogs("{$GLOBALS["LogFileDeamonLogDir"]}/USERS_LOG", $line);
unset($GLOBALS["USERRTT"][$KEYMD5]);
}
}
events("CachedUserMemDump:: Saving {$c}/{$xRQS} requests time={$xtime}s");
$GLOBALS["CACHEDUSersMemTime"] = array();
$GLOBALS["USERRTT"] = array();
$GLOBALS["LOGACCESS_TIME"] = time();
}
function CachedUserMemDump()
{
$xtime = tool_time_sec($GLOBALS["LOGACCESS_TIME"]);
if ($xtime < 10) {
return;
}
$c = 0;
$MAIN = $GLOBALS["CACHEDUSersMem"];
$q = new influx();
$xRQS = 0;
while (list($KEYMD5, $ARRAY) = each($MAIN)) {
$zArray = array();
$zArray2 = array();
if (!isset($GLOBALS["CACHEDUSersMem"][$KEYMD5]["SITE"])) {
unset($GLOBALS["CACHEDUSersMem"][$KEYMD5]);
continue;
}
$CATEGORY = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["CATEGORY"];
$USERID = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["USERID"];
$IPADDR = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["IPADDR"];
$MAC = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["MAC"];
$SIZE = intval($GLOBALS["CACHEDUSersMem"][$KEYMD5]["SIZE"]);
$SITE = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["SITE"];
$FAM = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["FAM"];
$RQS = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["RQS"];
$PROXYNAME = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["PROXYNAME"];
$GROUP = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["GROUP"];
$ORGA = $GLOBALS["CACHEDUSersMem"][$KEYMD5]["ORGA"];
if ($MAC == null) {
$MAC = "00:00:00:00:00:00";
}
if ($USERID == null) {
$USERID = "none";
}
$xRQS = $xRQS + $RQS;
$line = time() . ":::{$CATEGORY}:::{$USERID}:::{$IPADDR}:::{$MAC}:::{$SIZE}:::{$SITE}:::{$FAM}:::{$RQS}:::{$PROXYNAME}:::{$GROUP}:::{$ORGA}";
$c++;
if ($GLOBALS["NoCompressStatisticsByHour"] == 0) {
writeCompresslogs("{$GLOBALS["LogFileDeamonLogDir"]}/ACCESS_LOG", $line);
unset($GLOBALS["CACHEDUSersMem"][$KEYMD5]);
continue;
}
$zArray["tags"]["GROUP"] = $GROUP;
$zArray["tags"]["ORGA"] = $ORGA;
$zArray["tags"]["CATEGORY"] = $CATEGORY;
$zArray["tags"]["USERID"] = $USERID;
$zArray["tags"]["IPADDR"] = $IPADDR;
$zArray["tags"]["MAC"] = $MAC;
$zArray["fields"]["SIZE"] = $SIZE;
$zArray["tags"]["SITE"] = $SITE;
$zArray["tags"]["FAMILYSITE"] = $FAM;
$zArray["fields"]["ZDATE"] = time();
$zArray["fields"]["RQS"] = $RQS;
$zArray["tags"]["proxyname"] = $PROXYNAME;
if ($GLOBALS["DEBUG_MEM"]) {
events("INSERT - [{$KEYMD5}] {$zArray["tags"]["IPADDR"]} - {$zArray["tags"]["FAMILYSITE"]} - {$zArray["fields"]["SIZE"]}bytes {$zArray["fields"]["RQS"]}rqs [" . __LINE__ . "]");
}
$q->insert("access_log", $zArray);
unset($GLOBALS["CACHEDUSersMem"][$KEYMD5]);
}
if (count($GLOBALS["USERRTT"]) > 0) {
while (list($KEYMD5, $ARRAY) = each($GLOBALS["USERRTT"])) {
$USERID = $GLOBALS["USERRTT"][$KEYMD5]["USERID"];
$IPADDR = $GLOBALS["USERRTT"][$KEYMD5]["IPADDR"];
$MAC = $GLOBALS["USERRTT"][$KEYMD5]["MAC"];
$SIZE = intval($GLOBALS["USERRTT"][$KEYMD5]["SIZE"]);
$RQS = $GLOBALS["USERRTT"][$KEYMD5]["RQS"];
$PROXYNAME = $GLOBALS["USERRTT"][$KEYMD5]["PROXYNAME"];
$GROUP = $GLOBALS["USERRTT"][$KEYMD5]["GROUP"];
$ORGA = $GLOBALS["USERRTT"][$KEYMD5]["ORGA"];
$line = time() . ":::{$USERID}:::{$IPADDR}:::{$MAC}:::{$SIZE}:::{$RQS}:::{$PROXYNAME}::{$GROUP}:::{$ORGA}";
writeCompresslogs("{$GLOBALS["LogFileDeamonLogDir"]}/USERS_LOG", $line);
unset($GLOBALS["USERRTT"][$KEYMD5]);
}
}
events("CachedUserMemDump:: Saving {$c}/{$xRQS} requests time={$xtime}s");
$GLOBALS["CACHEDUSersMemTime"] = array();
$GLOBALS["USERRTT"] = array();
$GLOBALS["LOGACCESS_TIME"] = time();
}
