function DUMP_HOUR() { $TimeFile = "/etc/artica-postfix/pids/exec.squid.interface-size.php.DUMP_HOUR.time"; $unix = new unix(); $xtime = $unix->file_time_min($TimeFile); if ($xtime < 59) { events("Aborting current {$xtime}mn, require 1h minimal"); return; } @unlink($TimeFile); @file_put_contents($TimeFile, time()); $sock = new sockets(); $q = new mysql_squid_builder(); $q->QUERY_SQL("CREATE TABLE IF NOT EXISTS `dashboard_volume_day` (\n\t\t\t`TIME` DATETIME,\n\t\t\t`FAMILYSITE` VARCHAR(128),\n\t\t\t`USERID` VARCHAR(64),\n\t\t\t`IPADDR` VARCHAR(64),\n\t\t\t`MAC` VARCHAR(64),\n\t\t\t`CATEGORY` VARCHAR(64),\n\t\t\t`CONTENT_TYPE` VARCHAR(64),\n\t\t\t`SIZE` BIGINT UNSIGNED,\n\t\t\t`RQS` BIGINT UNSIGNED,\n\t\t\tKEY `TIME` (`TIME`),\n\t\t\tKEY `FAMILYSITE` (`FAMILYSITE`),\n\t\t\tKEY `USERID` (`USERID`),\n\t\t\tKEY `IPADDR` (`IPADDR`),\n\t\t\tKEY `MAC` (`MAC`),\n\t\t\tKEY `CONTENT_TYPE` (`CONTENT_TYPE`)\n\t\t\n\t\t\t) ENGINE=MYISAM;"); if (!$q->ok) { events("FATAL: {$q->mysql_error}"); } $MySQLStatisticsRetentionDays = intval($sock->GET_INFO("MySQLStatisticsRetentionDays")); if ($MySQLStatisticsRetentionDays == 0) { $MySQLStatisticsRetentionDays = 5; } $influx = new influx(); events("MySQL Statistics Retention Days:{$MySQLStatisticsRetentionDays}"); $c = 0; $TRUNCATE = false; $prefix = "INSERT IGNORE INTO `dashboard_volume_day` (`TIME`,`FAMILYSITE`,`USERID`,`IPADDR`,`MAC`,`CATEGORY`,`SIZE`,`RQS`) VALUES "; for ($i = 0; $i < $MySQLStatisticsRetentionDays + 1; $i++) { $timeQuery = time(); if ($i > 0) { $timeQuery = strtotime("-{$i} day"); } $sql = "SELECT * FROM access_log WHERE time > '" . date("Y-m-d 00:00:00", $timeQuery) . "' AND time < '" . date("Y-m-d 23:59:59", $timeQuery) . "'"; $main = $influx->QUERY_SQL($sql); if (!is_iterable($main)) { events("dashboard_volume_day:" . date("Y-m-d 00:00:00", $timeQuery) . " no data returned"); continue; } $d = 0; foreach ($main as $row) { $CATEGORY = null; $time = date("Y-m-d H:00:00", $row->ZDATE); $FAMILYSITE = mysql_escape_string2($row->FAMILYSITE); $IPADDR = mysql_escape_string2($row->IPADDR); $USERID = mysql_escape_string2($row->USERID); $MAC = mysql_escape_string2($row->MAC); $RQS = mysql_escape_string2($row->RQS); $SIZE = mysql_escape_string2($row->SIZE); if (property_exists($row, "CATEGORY")) { $CATEGORY = mysql_escape_string2($row->CATEGORY); } $RSQL[] = "('{$time}','{$FAMILYSITE}','{$USERID}','{$IPADDR}','{$MAC}','{$CATEGORY}','{$SIZE}','{$RQS}')"; $c++; $d++; if (count($RSQL) > 500) { if (!$TRUNCATE) { events("dashboard_volume_day:TRUNCATE TABLE"); $q->QUERY_SQL("TRUNCATE TABLE `dashboard_volume_day`"); $TRUNCATE = TRUE; } $q->QUERY_SQL($prefix . @implode(",", $RSQL)); if (!$q->ok) { events("FATAL! {$q->mysql_error}"); return; } $RSQL = array(); } } if (count($RSQL) > 0) { if (!$TRUNCATE) { events("dashboard_volume_day:TRUNCATE TABLE"); $q->QUERY_SQL("TRUNCATE TABLE `dashboard_volume_day`"); $TRUNCATE = TRUE; } $q->QUERY_SQL($prefix . @implode(",", $RSQL)); if (!$q->ok) { events("FATAL! {$q->mysql_error}"); return; } $RSQL = array(); } events("dashboard_volume_day:" . date("Y-m-d 00:00:00", $timeQuery) . " {$d} inserted rows"); } if (count($RSQL) > 0) { if (!$TRUNCATE) { events("dashboard_volume_day:TRUNCATE TABLE"); $q->QUERY_SQL("TRUNCATE TABLE `dashboard_volume_day`"); $TRUNCATE = TRUE; } $q->QUERY_SQL($prefix . @implode(",", $RSQL)); if (!$q->ok) { events("FATAL! {$q->mysql_error}"); return; } $RSQL = array(); } events("dashboard_volume_day: Total {$c} inserted rows"); FAMILY_SITES_DAY(); FULL_USERS_DAY(); }
function DUMP_HOUR() { $TimeFile = "/etc/artica-postfix/pids/exec.squid.interface-size.php.DUMP_HOUR.time"; $unix = new unix(); $xtime = $unix->file_time_min($TimeFile); if (!$GLOBALS["FORCE"]) { if (!$GLOBALS["VERBOSE"]) { if ($xtime < 119) { events("Aborting current {$xtime}mn, require 2h minimal"); return; } } } @unlink($TimeFile); @file_put_contents($TimeFile, time()); $sock = new sockets(); $q = new mysql_squid_builder(); $q->QUERY_SQL("CREATE TABLE IF NOT EXISTS `dashboard_volume_day` (\n\t\t\t`TIME` DATETIME,\n\t\t\t`FAMILYSITE` VARCHAR(128),\n\t\t\t`USERID` VARCHAR(64),\n\t\t\t`IPADDR` VARCHAR(64),\n\t\t\t`MAC` VARCHAR(64),\n\t\t\t`CATEGORY` VARCHAR(64),\n\t\t\t`CONTENT_TYPE` VARCHAR(64),\n\t\t\t`SIZE` BIGINT UNSIGNED,\n\t\t\t`RQS` BIGINT UNSIGNED,\n\t\t\tKEY `TIME` (`TIME`),\n\t\t\tKEY `FAMILYSITE` (`FAMILYSITE`),\n\t\t\tKEY `USERID` (`USERID`),\n\t\t\tKEY `IPADDR` (`IPADDR`),\n\t\t\tKEY `MAC` (`MAC`),\n\t\t\tKEY `CONTENT_TYPE` (`CONTENT_TYPE`)\n\t\t\n\t\t\t) ENGINE=MYISAM;"); if (!$q->ok) { DUMP_HOUR_PROGRESS("{mysql_error} CREATE TABLE", 110); echo $q->mysql_error . "\n"; events("FATAL: {$q->mysql_error}"); return; } $MySQLStatisticsRetentionDays = intval($sock->GET_INFO("MySQLStatisticsRetentionDays")); if ($MySQLStatisticsRetentionDays == 0) { $MySQLStatisticsRetentionDays = 5; } $postgres = new postgres_sql(); events("MySQL Statistics Retention Days:{$MySQLStatisticsRetentionDays}"); $c = 0; $TRUNCATE = false; $prefix = "INSERT IGNORE INTO `dashboard_volume_day` (`TIME`,`FAMILYSITE`,`USERID`,`IPADDR`,`MAC`,`CATEGORY`,`SIZE`,`RQS`) VALUES "; $timeQuery = time(); $TimeGroup = "date_trunc('hour', zdate) as zdate"; $timeQuery = date("Y-m-d H:i:s", strtotime("-{$MySQLStatisticsRetentionDays} day")); $sql = "SELECT SUM(size) as size, SUM(RQS) as RQS,\n\tfamilysite,ipaddr,mac,userid,category,\n\t{$TimeGroup}\tFROM access_log \n\tWHERE zdate > '{$timeQuery}'\n\tGROUP BY zdate,familysite,ipaddr,mac,userid,category ORDER by zdate"; echo "******************************\n{$sql}\n******************************\n"; DUMP_HOUR_PROGRESS("{query}...", 50); $results = $postgres->QUERY_SQL($sql); if (!$postgres->ok) { DUMP_HOUR_PROGRESS("{mysql_error} QUERY", 110); echo $postgres->mysql_error . "\n"; events("FATAL: {$q->mysql_error}"); return; } $d = 0; while ($ligne = @pg_fetch_assoc($results)) { $CATEGORY = null; $zDate = $ligne["zdate"]; $FAMILYSITE = mysql_escape_string2($ligne["familysite"]); $IPADDR = mysql_escape_string2($ligne["ipaddr"]); $USERID = mysql_escape_string2($ligne["userid"]); $MAC = mysql_escape_string2($ligne["mac"]); $RQS = mysql_escape_string2($ligne["rqs"]); $SIZE = mysql_escape_string2($ligne["size"]); $CATEGORY = mysql_escape_string2($ligne["category"]); $RSQL[] = "('{$zDate}','{$FAMILYSITE}','{$USERID}','{$IPADDR}','{$MAC}','{$CATEGORY}','{$SIZE}','{$RQS}')"; $c++; $d++; if (count($RSQL) > 500) { echo "{$c}...\n"; if (!$TRUNCATE) { events("dashboard_volume_day:TRUNCATE TABLE"); $q->QUERY_SQL("TRUNCATE TABLE `dashboard_volume_day`"); $TRUNCATE = TRUE; } $q->QUERY_SQL($prefix . @implode(",", $RSQL)); if (!$q->ok) { events("FATAL! {$q->mysql_error}"); DUMP_HOUR_PROGRESS("{mysql_error} at {$c}", 110); return; } $RSQL = array(); } } if (count($RSQL) > 0) { if (!$TRUNCATE) { events("dashboard_volume_day:TRUNCATE TABLE"); $q->QUERY_SQL("TRUNCATE TABLE `dashboard_volume_day`"); $TRUNCATE = TRUE; } $q->QUERY_SQL($prefix . @implode(",", $RSQL)); if (!$q->ok) { events("FATAL! {$q->mysql_error}"); DUMP_HOUR_PROGRESS("{mysql_error} at {$c}", 110); return; } $RSQL = array(); } DUMP_HOUR_PROGRESS("Total {$c} inserted rows", 80); events("dashboard_volume_day: Total {$c} inserted rows"); echo "******************************\nFAMILY_SITES_DAY();\n******************************\n"; echo "******************************\nFULL_USERS_DAY();\n******************************\n"; DUMP_HOUR_PROGRESS("FULL_USERS_DAY", 90); FULL_USERS_DAY(); DUMP_HOUR_PROGRESS("{done}", 100); }
exit; } if ($argv[1] == "--members-graph") { $GLOBALS["OUTPUT"] = true; exit; } if ($argv[1] == "--clean") { squidhour_clean(); exit; } if ($argv[1] == "--dump-hour") { DUMP_HOUR(); exit; } if ($argv[1] == "--dump-users") { FULL_USERS_DAY(); exit; } if ($argv[1] == "--cache-avg") { CACHES_AVG(); exit; } parse(); function build_progress($text, $pourc) { $cachefile = "/usr/share/artica-postfix/ressources/logs/admin.refresh.progress"; echo "{$pourc}% {$text}\n"; $array["POURC"] = $pourc; $array["TEXT"] = $text; events("{$pourc}% {$text}"); @file_put_contents($cachefile, serialize($array));