function scan()
{
$pidtime = "/etc/artica-postfix/pids/exec.haproxy.events-parser.php.scan.time";
$pidFile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid";
$unix = new unix();
$pid = $unix->get_pid_from_file($pidFile);
if ($unix->process_exists($pid)) {
events("A process, {$pid} Already exists...");
return;
}
$GLOBALS["MYHOSTNAME_PROXY"] = $unix->hostname_g();
@file_put_contents($pidFile, getmypid());
if (!$GLOBALS["VERBOSE"]) {
$time = $unix->file_time_min($pidtime);
if (!$GLOBALS["VERBOSE"]) {
if ($time < 5) {
events("{$time}mn, require minimal 5mn");
return;
}
}
}
$GLOBALS["LogFileDeamonLogDir"] = "/home/artica/haproxy-postgres/realtime-events";
$Workpath = "{$GLOBALS["LogFileDeamonLogDir"]}/access-work";
@file_put_contents($pidtime, time());
@mkdir($GLOBALS["LogFileDeamonLogDir"], 0755, true);
$postgres = new postgres_sql();
$postgres->CREATE_TABLES();
if (!$postgres->TABLE_EXISTS("haproxy_log")) {
events("haproxy_log, not such table");
die;
}
if (is_file("{$GLOBALS["LogFileDeamonLogDir"]}/HAPROXY.LOG")) {
HAPROXY_LOG("{$GLOBALS["LogFileDeamonLogDir"]}/HAPROXY.LOG");
}
HAPROXY_LOG_SCAN($Workpath);
}
function compressToYear($aspid = false)
{
$unix = new unix();
$pidfile = "/etc/artica-postfix/pids/SquidStatsYearQueue.pid";
$pidTime = "/etc/artica-postfix/pids/SquidStatsYearQueue.time";
if (!$aspid) {
$pid = $unix->get_pid_from_file($pidfile);
if ($unix->process_exists($pid, basename(__FILE__))) {
die("Already executed");
}
@file_put_contents($pidfile, getmypid());
$timeExec = $unix->file_time_min($pidTime);
if ($timeExec < 15) {
die("Only Each 15mn");
}
@unlink($pidfile);
@file_put_contents($pidfile, time());
}
$q = new postgres_sql();
if ($q->isRemote) {
return;
}
@file_put_contents("/etc/artica-postfix/settings/Daemons/SquidStatsYearQueue", 0);
if (system_is_overloaded()) {
@file_put_contents("/etc/artica-postfix/settings/Daemons/SquidStatsYearQueue", 1);
squid_admin_mysql(0, "Overloaded system {$GLOBALS["SYSTEM_INTERNAL_LOAD"]}, aborting task", null, __FILE__, __LINE__);
return;
}
$Curday = date("Y-m-d 00:00:00");
$sql = "SELECT date_trunc('month',zdate) as zdate FROM access_month GROUP BY date_trunc('month',zdate) ORDER BY zdate";
$q = new postgres_sql();
$q->CREATE_TABLES();
$results = $q->QUERY_SQL($sql);
if (!$q->ok) {
echo $q->mysql_error . "\n";
}
$SOURCES_MONTH = array();
while ($ligne = @pg_fetch_assoc($results)) {
$day = $ligne["zdate"];
if ($Curday == $day) {
continue;
}
$SOURCES_MONTH[$day] = true;
}
$sql = "SELECT date_trunc('month',zdate) as zdate FROM access_year GROUP BY date_trunc('month',zdate) ORDER BY zdate";
$q = new postgres_sql();
$results = $q->QUERY_SQL($sql);
if (!$q->ok) {
echo $q->mysql_error . "\n";
}
$DEST_MONTH = array();
while ($ligne = @pg_fetch_assoc($results)) {
$day = $ligne["zdate"];
if ($Curday == $day) {
continue;
}
$DEST_MONTH[$day] = true;
}
$CountOfSourcesDay = count($SOURCES_MONTH);
$c = 0;
$OverLoadedCount = 0;
$OverLoadedMax = round($OverLoadedCount / 2);
while (list($Day, $ligne) = each($SOURCES_MONTH)) {
if (isset($DEST_MONTH[$Day])) {
echo "{$Day} already done\n";
continue;
}
echo "{$Day} not imported done\n";
$c++;
if (!compress_month_perform($Day)) {
@file_put_contents("/etc/artica-postfix/settings/Daemons/SquidStatsYearQueue", 1);
return;
}
if (system_is_overloaded()) {
$OverLoadedCount++;
if ($OverLoadedCount > $OverLoadedMax) {
squid_admin_mysql(0, "Overloaded system {$GLOBALS["SYSTEM_INTERNAL_LOAD"]}, aborting tasks after compressing {$c} days", null, __FILE__, __LINE__);
@file_put_contents("/etc/artica-postfix/settings/Daemons/SquidStatsYearQueue", 1);
return;
}
}
}
}
function start($aspid = false)
{
$unix = new unix();
$sock = new sockets();
$users = new usersMenus();
$Masterbin = "/usr/local/ArticaStats/bin/postgres";
if (!is_file($Masterbin)) {
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]}, arpd not installed\n";
}
return;
}
if (!$aspid) {
$pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid";
$pid = $unix->get_pid_from_file($pidfile);
if ($unix->process_exists($pid, basename(__FILE__))) {
$time = $unix->PROCCESS_TIME_MIN($pid);
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} Already Artica task running PID {$pid} since {$time}mn\n";
}
return;
}
@file_put_contents($pidfile, getmypid());
}
$pid = PID_NUM();
if ($unix->MEM_TOTAL_INSTALLEE() < 624288) {
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} not enough memory\n";
}
if ($unix->process_exists($pid)) {
stop();
}
build_progress_restart("{starting} {failed} no memory", 110);
return;
}
if ($unix->process_exists($pid)) {
$timepid = $unix->PROCCESS_TIME_MIN($pid);
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} Service already started {$pid} since {$timepid}Mn...\n";
}
build_progress_restart("{starting} {success}", 30);
return true;
}
$php5 = $unix->LOCATE_PHP5_BIN();
$sysctl = $unix->find_program("sysctl");
$echo = $unix->find_program("echo");
$nohup = $unix->find_program("nohup");
$su = $unix->find_program("su");
$rm = $unix->find_program("rm");
$SquidPerformance = intval($sock->GET_INFO("SquidPerformance"));
$EnableInfluxDB = intval($sock->GET_INFO("EnableInfluxDB"));
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} EnableInfluxDB: {$EnableInfluxDB}\n";
}
$InfluxUseRemote = intval($sock->GET_INFO("InfluxUseRemote"));
if ($users->POSTFIX_INSTALLED) {
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} Postfix installed: True\n";
}
}
if ($InfluxUseRemote == 1) {
$EnableInfluxDB = 0;
}
$FreeZePostGres = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/FreeZePostGres"));
if ($FreeZePostGres == 1) {
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} Freeze !!! Aborting...\n";
}
return;
}
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} Use Remote statistics.: {$InfluxUseRemote}\n";
}
if (!$users->POSTFIX_INSTALLED) {
$EnableIntelCeleron = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/EnableIntelCeleron"));
if ($EnableIntelCeleron == 1) {
$EnableInfluxDB = 0;
}
}
if (is_file("/etc/artica-postfix/STATS_APPLIANCE")) {
$EnableInfluxDB = 1;
$SquidPerformance = 0;
$EnableIntelCeleron = 0;
}
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} Use Statistics DB.....: {$EnableInfluxDB}\n";
}
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} Use Intel Celeron mode: {$EnableIntelCeleron}\n";
}
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} Use Performance.......: {$SquidPerformance}\n";
}
if ($EnableInfluxDB == 0) {
build_progress_restart("{starting} {failed} {disabled}", 110);
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} service disabled (see EnableInflux)\n";
}
return;
}
if (!function_exists("pg_connect")) {
build_progress_restart("{starting} installing php5-pgsql", 35);
$unix->DEBIAN_INSTALL_PACKAGE("php5-pgsql");
system("/usr/share/artica-postfix/exec.php.ini.php");
if (!function_exists("pg_connect")) {
build_progress_restart("{starting} installing php5-pgsql {failed}", 110);
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} pg_connect no such function\n";
}
return;
}
system("/etc/init.d/artica-webconsole restart");
}
build_progress_restart("{starting}", 40);
if (!$unix->UnixUserExists("ArticaStats")) {
$unix->CreateUnixUser("ArticaStats", "ArticaStats");
}
@mkdir("/var/run/ArticaStats", 0755, true);
@mkdir("/home/ArticaStatsDB", 0700, true);
@mkdir("/var/log/ArticaStatsDB", 0755, true);
@chown("/home/ArticaStatsDB", "ArticaStats");
@chgrp("/home/ArticaStatsDB", "ArticaStats");
@chown("/var/run/ArticaStats", "ArticaStats");
@chgrp("/var/run/ArticaStats", "ArticaStats");
@chown("/var/log/ArticaStatsDB", "ArticaStats");
@chgrp("/var/log/ArticaStatsDB", "ArticaStats");
if (is_file("/var/log/ArticaStatsDB/ArticaStatsDB.log")) {
@unlink("/var/log/ArticaStatsDB/ArticaStatsDB.log");
@touch("/var/log/ArticaStatsDB/ArticaStatsDB.log");
}
@chown("/var/log/ArticaStatsDB/ArticaStatsDB.log", "ArticaStats");
@chgrp("/var/log/ArticaStatsDB/ArticaStatsDB.log", "ArticaStats");
if (is_file("/var/run/ArticaStats/.s.PGSQL.8086")) {
@unlink("/var/run/ArticaStats/.s.PGSQL.8086");
}
$php = $unix->LOCATE_PHP5_BIN();
if (!is_file("/etc/artica-postfix/locales.gen")) {
squid_admin_mysql(1, "Generating languages for the PostGreSQL compatibility", null, __FILE__, __LINE__);
build_progress_restart("{generating_langs}", 42);
system("{$php} /usr/share/artica-postfix/exec.locale.gen.php");
}
if (!is_dir("/home/ArticaStatsDB/base/1")) {
squid_admin_mysql(0, "Creating a new PostgreSQL database in ArticaStatsDB", null, __FILE__, __LINE__);
build_progress_restart("{starting}", 45);
$rm = $unix->find_program("rm");
shell_exec("{$rm} -rf /home/ArticaStatsDB/*");
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} initialize database...\n";
}
system("{$su} -c \"/usr/local/ArticaStats/bin/initdb --username=ArticaStats /home/ArticaStatsDB --no-locale -E UTF8\" ArticaStats");
}
if (!is_dir("/home/ArticaStatsDB/base/1")) {
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} initialize database failed...\n";
}
$rm = $unix->find_program("rm");
shell_exec("{$rm} -rf /home/ArticaStatsDB/*");
return;
}
build_progress_restart("{starting}", 50);
xbuild();
fuser_port();
build_progress_restart("{starting} {permissions}", 55);
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} Apply permissions on /home/ArticaStatsDB\n";
}
$chown = $unix->find_program("chown");
$chmod = $unix->find_program("chmod");
shell_exec("{$chown} -R ArticaStats:ArticaStats /home/ArticaStatsDB");
shell_exec("{$chmod} 0700 /home/ArticaStatsDB");
if (is_file("/home/ArticaStatsDB/postmaster.pid")) {
@unlink("/home/ArticaStatsDB/postmaster.pid");
}
$f[] = "su -l ArticaStats -c '";
$f[] = "/usr/local/ArticaStats/bin/pg_ctl -o \"-k /tmp,/var/run/ArticaStats\" -D /home/ArticaStatsDB -l /var/log/ArticaStatsDB/ArticaStatsDB.log start'";
$cmd = @implode(" ", $f) . " >/dev/null 2>&1 &";
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} service\n";
}
shell_exec($cmd);
for ($i = 1; $i < 5; $i++) {
build_progress_restart("{starting} {wait} {$i}/5", 70);
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} waiting {$i}/5\n";
}
sleep(1);
$pid = PID_NUM();
if ($unix->process_exists($pid)) {
break;
}
}
$pid = PID_NUM();
if ($unix->process_exists($pid)) {
build_progress_restart("{starting} {success}", 75);
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} Success PID {$pid}\n";
}
$pg = new postgres_sql();
$pg->CREATE_TABLES();
return true;
} else {
build_progress_restart("{starting} {failed}", 110);
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} Failed\n";
}
if ($GLOBALS["OUTPUT"]) {
echo "Starting......: " . date("H:i:s") . " [INIT]: {$GLOBALS["TITLENAME"]} {$cmd}\n";
}
}
}
function xtsart()
{
$unix = new unix();
$pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . ".pid";
$pidTime = "/etc/artica-postfix/pids/" . basename(__FILE__) . ".time";
if ($GLOBALS["VERBOSE"]) {
echo "TimeFile:{$pidTime}\n";
}
$unix = new unix();
if (!$GLOBALS["VERBOSE"]) {
if ($unix->file_time_min($pidTime) < 10) {
die;
}
if ($unix->process_exists(@file_get_contents($pidfile, basename(__FILE__)))) {
if ($GLOBALS["VERBOSE"]) {
echo " --> Already executed.. " . @file_get_contents($pidfile) . " aborting the process\n";
}
writelogs(basename(__FILE__) . ":Already executed.. aborting the process", basename(__FILE__), __FILE__, __LINE__);
die;
}
@file_put_contents($pidfile, getmypid());
@unlink($pidTime);
@file_put_contents($pidTime, time());
}
$array_load = sys_getloadavg();
$internal_load = $array_load[0];
$time = time();
$BASEDIR = "/usr/share/artica-postfix";
$hash_mem = array();
@chmod("/usr/share/artica-postfix/ressources/mem.pl", 0755);
$datas = shell_exec(dirname(__FILE__) . "/ressources/mem.pl");
if (preg_match('#T=([0-9]+) U=([0-9]+)#', $datas, $re)) {
$ram_used = $re[2];
}
$cpuUsage = null;
$ps = $unix->find_program("ps");
exec("{$ps} -aux 2>&1", $processes);
foreach ($processes as $process) {
$cols = explode(' ', preg_replace('# +#', ' ', $process));
if (strpos($cols[2], '.') > -1) {
$cpuUsage += floatval($cols[2]);
}
}
if ($GLOBALS["VERBOSE"]) {
echo "CPU: {$cpuUsage}, LOAD: {$internal_load}, MEM: {$ram_used}\n";
}
$q = new postgres_sql();
$q->CREATE_TABLES();
$proxyname = $unix->hostname_g();
$time = date("Y-m-d H:i:s");
$sql = "INSERT INTO system (zdate,proxyname,load_avg,mem_stats,cpu_stats)\n\tVALUES('{$time}','{$proxyname}','{$internal_load}','" . intval($ram_used) . "','{$cpuUsage}')";
if ($GLOBALS["VERBOSE"]) {
echo $sql . "\n";
}
$q->QUERY_SQL($sql);
if (!$q->ok) {
echo $q->mysql_error . "\n";
}
RXTX();
if (system_is_overloaded(basename(__FILE__))) {
$date = time();
@mkdir("/var/log/artica-postfix/sys_alerts", 0755, true);
if (!is_file("/var/log/artica-postfix/sys_alerts/{$date}")) {
$ps = $unix->find_program("ps");
$nohup = $unix->find_program($nohup);
$nice = $unix->EXEC_NICE();
$load = $GLOBALS["SYSTEM_INTERNAL_LOAD"];
if (!$unix->process_exists($unix->PIDOF_PATTERN("{$ps}"))) {
$cmd = trim("{$nohup} {$nice} {$ps} auxww >/var/log/artica-postfix/sys_alerts/{$date}-{$load} 2>&1");
shell_exec($cmd);
}
}
}
}
function run()
{
$TimeFile = "/etc/artica-postfix/pids/exec.squid.stats.categorize.php.time";
$MaxTime = "/etc/artica-postfix/pids/exec.squid.stats.categorize.php.maxtime";
$pidfile = "/etc/artica-postfix/pids/exec.squid.stats.categorize.php.pid";
$unix = new unix();
$skiptime = false;
$pid = @file_get_contents($pidfile);
if ($pid < 100) {
$pid = null;
}
if ($unix->process_exists($pid, basename(__FILE__))) {
$timepid = $unix->PROCCESS_TIME_MIN($pid);
if ($GLOBALS["VERBOSE"]) {
echo "{$pid} already executed since {$timepid}Mn\n";
}
if (!$GLOBALS["FORCE"]) {
if ($timepid < 14) {
build_progress("Already executed pid {$pid}", 110);
return;
}
$kill = $unix->find_program("kill");
unix_system_kill_force($pid);
}
}
if ($GLOBALS["PROGRESS"]) {
$skiptime = true;
}
if ($GLOBALS["FORCE"]) {
$skiptime = true;
}
@file_put_contents($pidfile, getmypid());
if (!$skiptime) {
$time = $unix->file_time_min($TimeFile);
if ($time < 240) {
echo "Current {$time}Mn, require at least 240mn\n";
return;
}
@unlink($TimeFile);
@file_put_contents($TimeFile, time());
}
@unlink($MaxTime);
@file_put_contents($MaxTime, time());
$now = date("Y-m-d H:i:s", strtotime("-24 hour"));
$catz = new mysql_catz();
if ($catz->UfdbCatEnabled == 0) {
build_progress("Categories Engine is disabled", 110);
squid_admin_mysql(1, "Categories Engine is disabled, skip parsing non-categorized websites.", null, __FILE__, __LINE__);
return;
}
build_progress("Updating databases", 10);
$php = $unix->LOCATE_PHP5_BIN();
system("{$php} /usr/share/artica-postfix/exec.squid.blacklists.php --bycron");
build_progress("Construct not categorized webistes...", 15);
$q = new postgres_sql();
$q->CREATE_TABLES();
$sql = "SELECT sum(size) as size,sum(rqs) as rqs, familysite from access_log WHERE category='' AND zdate>'{$now}' GROUP BY familysite ORDER BY size DESC LIMIT 5000";
$q = new postgres_sql();
$results = $q->QUERY_SQL($sql);
if (!$q->ok) {
echo $q->mysql_error . "\n";
}
$DEST_DAY = array();
$q->QUERY_SQL("truncate table not_categorized");
while ($ligne = @pg_fetch_assoc($results)) {
$familysite = $ligne["familysite"];
$size = $ligne["size"];
$rqs = $ligne["rqs"];
echo "{$familysite} {$size} ({$rqs})\n";
$q->QUERY_SQL("INSERT INTO not_categorized (zdate,familysite,size,rqs) VALUES (NOW(),'{$familysite}','{$size}','{$rqs}')");
}
@file_put_contents("{$GLOBALS["BASEDIR"]}/NOT_CATEGORIZED_TIME", time());
if (system_is_overloaded()) {
build_progress("Overloaded, aborting", 110);
return;
}
$sql = "SELECT familysite from not_categorized ORDER BY size DESC LIMIT 5000";
$c = 0;
$q = new postgres_sql();
build_progress("Query the system...", 20);
$results = $q->QUERY_SQL($sql);
$sum = pg_num_rows($results);
$c = 0;
while ($ligne = @pg_fetch_assoc($results)) {
$c++;
$perc = $c / $sum;
$perc = round($perc * 100);
$perc = $perc + 20;
if ($perc > 95) {
$perc = 95;
}
$familysite = $ligne["familysite"];
build_progress("Analyze {$familysite} {$c}/{$sum}", $perc);
$category = $catz->GET_CATEGORIES($familysite);
echo "{$familysite} = {$category}\n";
if ($category != null) {
build_progress("Analyze {$familysite} = {$category} {$c}/{$sum}", $perc);
echo "UPDATE access_log = {$category}\n";
$q->QUERY_SQL("UPDATE access_log SET category='{$category}' WHERE familysite='{$familysite}' AND category=''");
echo "UPDATE access_month = {$category}\n";
$q->QUERY_SQL("UPDATE access_month SET category='{$category}' WHERE familysite='{$familysite}' AND category=''");
echo "UPDATE access_year = {$category}\n";
$q->QUERY_SQL("UPDATE access_year SET category='{$category}' WHERE familysite='{$familysite}' AND category=''");
echo "DELETE not_categorized FOR {$familysite}\n";
$q->QUERY_SQL("DELETE FROM not_categorized WHERE familysite='{$familysite}'");
}
$timexec = $unix->file_time_min($MaxTime);
if ($timexec > 230) {
build_progress("Expired time, aborting {$c}/{$sum}", 110);
return;
}
if (system_is_overloaded()) {
build_progress("Overloaded, aborting {$c}/{$sum}", 110);
return;
}
}
build_progress("{success}", 100);
}
