function build() { $timefile = "/etc/artica-postfix/pids/exec.squid.rotate.php.build.time"; $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid"; // /etc/artica-postfix/pids/exec.squid.rotate.php.build.time $sock = new sockets(); $unix = new unix(); $ls = $unix->find_program("ls"); $pid = $unix->PIDOF_PATTERN(basename(__FILE__)); $MyPid = getmypid(); if ($MyPid != $pid) { if ($unix->process_exists($pid)) { $timeFile = $unix->PROCESS_TIME_INT($pid); $pidCmdline = @file_get_contents("/proc/{$pid}/cmdline"); if ($timeFile < 30) { echo "Already PID {$pid} is running since {$timeFile}Mn\n"; squid_admin_mysql(1, "[LOG ROTATION]: Skip task, already running {$pid} since {$timeFile}Mn", "Running: {$pidCmdline}", __FILE__, __LINE__); die; } else { squid_admin_mysql(1, "[LOG ROTATION]: Killing old task {$pid} running more than 30mn ({$timeFile}Mn)", "Running: {$pidCmdline}", __FILE__, __LINE__); $unix->KILL_PROCESS($pid); } } } @file_put_contents($pidfile, getmypid()); if (!$GLOBALS["FORCE"]) { $time = $unix->file_time_min($timefile); if ($time < 60) { echo "Only each 60mn\n"; die; } @unlink($timefile); @file_put_contents($timefile, time()); } $syslog = new mysql_storelogs(); $SquidLogRotateFreq = intval($sock->GET_INFO("SquidLogRotateFreq")); if ($SquidLogRotateFreq < 10) { $SquidLogRotateFreq = 1440; } $LastRotate = $unix->file_time_min("/etc/artica-postfix/pids/squid-rotate-cache.time"); $LogsRotateDefaultSizeRotation = $sock->GET_INFO("LogsRotateDefaultSizeRotation"); $SquidRotateOnlySchedule = intval($sock->GET_INFO("SquidRotateOnlySchedule")); if (!is_numeric($LogsRotateDefaultSizeRotation)) { $LogsRotateDefaultSizeRotation = 100; } $BackupMaxDaysDir = $sock->GET_INFO("BackupMaxDaysDir"); if ($BackupMaxDaysDir == null) { $BackupMaxDaysDir = "/home/logrotate_backup"; } $BackupSquidLogsUseNas = intval($sock->GET_INFO("BackupSquidLogsUseNas")); $SquidRotateAutomount = intval($sock->GET_INFO("SquidRotateAutomount")); $SquidRotateClean = intval($sock->GET_INFO("SquidRotateClean")); $SquidRotateAutomountRes = $sock->GET_INFO("SquidRotateAutomountRes"); $SquidRotateAutomountFolder = $sock->GET_INFO("SquidRotateAutomountFolder"); if ($SquidRotateAutomount == 1) { shell_exec("{$ls} /automounts/{$SquidRotateAutomountRes} >/dev/null 2>&1"); if (ifdirMounted("/automounts/{$SquidRotateAutomountRes}")) { $BackupSquidLogsUseNas = 0; $BackupMaxDaysDir = "/automounts/{$SquidRotateAutomountRes}/{$SquidRotateAutomountFolder}"; } else { $syslog->events("/automounts/{$SquidRotateAutomountRes} not mounted", __FUNCTION__, __LINE__); squid_admin_mysql(1, "[ROTATE],Auto-mount {$SquidRotateAutomountRes} not mounted", null, __FILE__, __LINE__); } } $BackupMaxDaysDir = str_replace("//", "/", $BackupMaxDaysDir); $BackupMaxDaysDir = str_replace("\\", "/", $BackupMaxDaysDir); if (!is_dir($BackupMaxDaysDir)) { @mkdir($BackupMaxDaysDir, 0755, true); } if (!is_dir($BackupMaxDaysDir)) { $syslog->events("{$BackupMaxDaysDir} not such directory or permission denied", __FUNCTION__, __LINE__); squid_admin_mysql(1, "[ROTATE],{$BackupMaxDaysDir} not such directory or permission denied", null, __FILE__, __LINE__); if ($SquidRotateAutomount == 1) { $BackupMaxDaysDir = $sock->GET_INFO("BackupMaxDaysDir"); if ($BackupMaxDaysDir == null) { $BackupMaxDaysDir = "/home/logrotate_backup"; } if (!is_dir($BackupMaxDaysDir)) { @mkdir($BackupMaxDaysDir, 0755, true); } $syslog->events("Return back to {$BackupMaxDaysDir}", __FUNCTION__, __LINE__); } else { return; } } $php = $unix->LOCATE_PHP5_BIN(); $hostname = $unix->hostname_g(); $InFluxBackupDatabaseDir = $sock->GET_INFO("InFluxBackupDatabaseDir"); if ($InFluxBackupDatabaseDir == null) { $InFluxBackupDatabaseDir = "/home/artica/influx/backup"; } $LogRotatePath = $sock->GET_INFO("LogRotatePath"); if ($LogRotatePath == null) { $LogRotatePath = "/home/logrotate"; } $LogRotateAccess = "{$LogRotatePath}/access"; $LogRotateTail = "{$LogRotatePath}/tail"; $LogRotateCache = "{$LogRotatePath}/cache"; $syslog->events("Launch rotation only by schedule.: {$SquidRotateOnlySchedule}", __FUNCTION__, __LINE__); $syslog->events("SquidLogRotateFreq...............: {$SquidLogRotateFreq}Mn", __FUNCTION__, __LINE__); $syslog->events("LastRotate.......................: {$LastRotate}Mn", __FUNCTION__, __LINE__); $syslog->events("Working directory................: {$LogRotatePath}", __FUNCTION__, __LINE__); $syslog->events("Launch rotation when exceed......: {$LogsRotateDefaultSizeRotation}M", __FUNCTION__, __LINE__); $syslog->events("Final storage directory..........: {$BackupMaxDaysDir}", __FUNCTION__, __LINE__); $syslog->events("Backup files to a NAS............: {$BackupSquidLogsUseNas}", __FUNCTION__, __LINE__); ss5_log($BackupMaxDaysDir); if ($handle = opendir("/var/run/squid")) { while (false !== ($file = readdir($handle))) { if ($file == ".") { continue; } if ($file == "..") { continue; } $path = "/var/run/squid/{$file}"; if (preg_match("#\\.[0-9]+\\.status\$#", $file)) { $time = $unix->file_time_min($path); if ($time > 1440) { $syslog->events("Removing {$path}", __FUNCTION__, __LINE__); @unlink($path); } continue; } if (preg_match("#\\.[0-9]+\\.state\$#", $file)) { $time = $unix->file_time_min($path); if ($time > 1440) { $syslog->events("Removing {$path}", __FUNCTION__, __LINE__); @unlink($path); } continue; } } } $size = @filesize("/var/log/squid/access.log"); $size = $size / 1024; $size = $size / 1024; $syslog->events("/var/log/squid/access.log........: {$size}M", __FUNCTION__, __LINE__); $syslog->events("Analyze /var/log/squid directory for cache.log", __FUNCTION__, __LINE__); if (!($handle = opendir("/var/log/squid"))) { build_progress_rotation("Unable to open /var/log/squid", 110); $syslog->events("Unable to open /var/log/squid directory.", __FUNCTION__, __LINE__); return; } build_progress_rotation("Scanning /var/log/squid", 40); while (false !== ($file = readdir($handle))) { if ($file != "." && $file != "..") { $path = "/var/log/squid/{$file}"; if (is_dir($path)) { continue; } if (!preg_match("#^cache\\.log\\.[0-9]+\$#", $file)) { continue; } @mkdir("{$LogRotateCache}", 0755, true); $size = @filesize($path); $size = $size / 1024; $size = $size / 1024; $destfile = "{$LogRotateCache}/{$file}." . time() . ".log"; if (!@copy($path, $destfile)) { $syslog->events("Unable to copy {$path} to {$destfile}", __FUNCTION__, __LINE__); @unlink($destfile); continue; } $syslog->events("Removed {$path}", __FUNCTION__, __LINE__); @unlink($path); } } $syslog->events("Analyze /var/log/squid directory for access.log", __FUNCTION__, __LINE__); if (!($handle = opendir("/var/log/squid"))) { $syslog->events("Unable to open /var/log/squid directory.", __FUNCTION__, __LINE__); return; } @mkdir($LogRotateAccess, 0755, true); @mkdir($LogRotateTail, 0755, true); while (false !== ($file = readdir($handle))) { if ($file != "." && $file != "..") { $path = "/var/log/squid/{$file}"; if (is_dir($path)) { continue; } if (preg_match("#^childs-access\\.log\\.[0-9]+\$#", $file)) { $destfile = "{$LogRotateAccess}/{$file}." . time() . ".log"; if (!@copy($path, $destfile)) { $syslog->events("Unable to copy {$path} to {$destfile}", __FUNCTION__, __LINE__); @unlink($destfile); continue; } $syslog->events("Removed {$path}", __FUNCTION__, __LINE__); @unlink($path); continue; } $syslog->events("Analyze {$file} ^squidtail\\.log\\.[0-9]+\$", __FUNCTION__, __LINE__); if (preg_match("#^squidtail\\.log\\.[0-9]+\$#", $file)) { $destfile = "{$LogRotateTail}/{$file}"; if (!@copy($path, $destfile)) { $syslog->events("Unable to copy {$path} to {$destfile}", __FUNCTION__, __LINE__); @unlink($destfile); continue; } $syslog->events("Removed {$path}", __FUNCTION__, __LINE__); @unlink($path); continue; } $syslog->events("Analyze {$file} ^access\\.log\\.[0-9]+\$", __FUNCTION__, __LINE__); if (!preg_match("#^access\\.log\\.[0-9]+\$#", $file)) { continue; } @mkdir("{$LogRotatePath}", 0755, true); $destfile = "{$LogRotateAccess}/{$file}." . time() . ".log"; if (!@copy($path, $destfile)) { $syslog->events("Unable to copy {$path} to {$destfile}", __FUNCTION__, __LINE__); @unlink($destfile); continue; } $syslog->events("Removed {$path}", __FUNCTION__, __LINE__); @unlink($path); } } $syslog->events("Analyze {$LogRotateAccess} for access.log", __FUNCTION__, __LINE__); if (!($handle = opendir($LogRotateAccess))) { $syslog->events("Unable to open {$LogRotateAccess} directory.", __FUNCTION__, __LINE__); return; } $ROTATED = false; while (false !== ($file = readdir($handle))) { if ($file != "." && $file != "..") { $path = "{$LogRotateAccess}/{$file}"; echo "OPEN {$path}\n"; if (is_dir($path)) { continue; } if (!preg_match("#^access\\.log#", $file)) { continue; } range_fichier_source($path, $BackupMaxDaysDir); $ROTATED = true; } } if ($handle = opendir($LogRotateTail)) { while (false !== ($file = readdir($handle))) { if ($file != "." && $file != "..") { $path = "{$LogRotateTail}/{$file}"; echo "OPEN {$path}\n"; if (is_dir($path)) { continue; } if (!preg_match("#^squidtail\\.log#", $file)) { continue; } range_fichier_tail($path, $BackupMaxDaysDir); $ROTATED = true; } } } if (!$ROTATED) { return; } $q = new mysql(); $sql = "CREATE TABLE IF NOT EXISTS `backuped_logs` (\n\t\t\t `path` CHAR(255) NOT NULL,\n\t\t\t `zDate` DATETIME,\n\t\t\t `size` INT UNSIGNED NOT NULL,\n\t\t\t PRIMARY KEY (`path`),\n\t\t\t KEY `zDate` (`zDate`)\n\t\t\t) ENGINE = MYISAM;"; $q->QUERY_SQL($sql, "artica_events"); $syslog->events("Analyze /home/logrotate/work for access.log", __FUNCTION__, __LINE__); build_progress_rotation("Scanning /home/logrotate/work", 45); analyze_directory("/home/logrotate/work", $BackupMaxDaysDir); $BackupMaxDaysDir2 = $sock->GET_INFO("BackupMaxDaysDir"); if ($BackupMaxDaysDir2 == null) { $BackupMaxDaysDir2 = "/home/logrotate_backup"; } if ($BackupMaxDaysDir2 != $BackupMaxDaysDir) { build_progress_rotation("Scanning {$BackupMaxDaysDir2}", 46); $syslog->events("{$BackupMaxDaysDir2} is different of {$BackupMaxDaysDir}", __FUNCTION__, __LINE__); analyze_directory($BackupMaxDaysDir2, $BackupMaxDaysDir); } build_progress_rotation("Scanning /home/logrotate/merged", 47); analyze_garbage_directory("/home/logrotate/merged", $BackupMaxDaysDir, 1440); build_progress_rotation("Scanning {$LogRotateCache}", 48); analyze_cache_directory($LogRotateCache, $BackupMaxDaysDir); build_progress_rotation("Scanning /home/logrotate/work", 49); analyze_cache_directory("/home/logrotate/work", $BackupMaxDaysDir); build_progress_rotation("Scanning /home/squid/cache-logs", 49); analyze_cache_directory("/home/squid/cache-logs", $BackupMaxDaysDir); if ($GLOBALS["VERBOSE"]) { echo "TRUNCATE TABLE backuped_logs !!!\n"; } $q->QUERY_SQL("TRUNCATE TABLE backuped_logs", "artica_events"); if ($BackupSquidLogsUseNas == 1) { build_progress_rotation("Backup to N.A.S", 50); BackupToNas($BackupMaxDaysDir); build_progress_rotation("Backup to N.A.S BigData backups", 50); BackupToNas($InFluxBackupDatabaseDir, false); } else { build_progress_rotation("Scanning {$BackupMaxDaysDir}", 50); analyze_destination_directory($BackupMaxDaysDir); } @file_put_contents("/etc/artica-postfix/settings/Daemons/BackupMaxDaysDirCurrentSize", $unix->DIRSIZE_KO($BackupMaxDaysDir)); @chmod("/etc/artica-postfix/settings/Daemons/BackupMaxDaysDirCurrentSize", 0777); return; }
function rotate_logs() { $pidfile = "/etc/artica-postfix/pids/exec.squid.php.rotate_logs.pid"; $unix = new unix(); $sock = new sockets(); $GLOBALS["SQUIDBIN"] = $unix->LOCATE_SQUID_BIN(); $php = $unix->LOCATE_PHP5_BIN(); $nohup = $unix->find_program("nohup"); $pid = $unix->get_pid_from_file($pidfile); if ($unix->process_exists($pid, basename(__FILE__))) { if ($GLOBALS["VERBOSE"]) { echo "Already executed pid {$pid}\n"; } return; } $getmypid = getmypid(); @file_put_contents($pidfile, getmypid()); build_progress_rotation("{starting}", 5); $LogsRotateDefaultSizeRotation = $sock->GET_INFO("LogsRotateDefaultSizeRotation"); if (!is_numeric($LogsRotateDefaultSizeRotation)) { $LogsRotateDefaultSizeRotation = 100; } $EnableProxyCompressor = intval($sock->GET_INFO("EnableProxyCompressor")); $ScheduleCMD = null; build_progress_rotation("{default} {$LogsRotateDefaultSizeRotation}MB", 6); $size = @filesize("/var/log/squid/access.log"); $squidtail_size = @filesize("/var/log/squid/squidtail.log"); $ROTATED = false; $size = $size / 1024; $size = round($size / 1024); $squidtail_size = $squidtail_size / 1024; $squidtail_size = round($squidtail_size / 1024); $echo = $unix->find_program("echo"); if ($GLOBALS["FORCE"]) { $forced_text = " - forced"; } if ($GLOBALS["SMOOTH"]) { $forced_text = " - smooth"; } $MustRotateAt = $LogsRotateDefaultSizeRotation; $LastRotate = $unix->file_time_min("/etc/artica-postfix/pids/squid-rotate-cache.time"); $SquidLogRotateFreq = intval($sock->GET_INFO("SquidLogRotateFreq")); $SquidRotateOnlySchedule = intval($sock->GET_INFO("SquidRotateOnlySchedule")); if ($SquidLogRotateFreq < 10) { $SquidLogRotateFreq = 1440; } @unlink("/etc/artica-postfix/pids/squid-rotate-cache.time"); @file_put_contents("/etc/artica-postfix/pids/squid-rotate-cache.time", time()); if ($SquidRotateOnlySchedule == 1) { if ($GLOBALS["BY_SCHEDULE"]) { $MustRotateAt = 0; } } if ($GLOBALS["FORCE"]) { $MustRotateAt = 0; } if ($GLOBALS["PROGRESS"]) { $MustRotateAt = 0; } $MUSTROTATE = false; if ($SquidRotateOnlySchedule == 0) { if ($LastRotate < $SquidLogRotateFreq) { if ($size >= $MustRotateAt) { $MUSTROTATE = true; build_progress_rotation("/var/log/squid/access.log Must be rotated ", 10); } if ($squidtail_size >= $squidtail_size) { $MUSTROTATE = true; build_progress_rotation("/var/log/squid/squidtail.log Must be rotated ", 10); } } if ($MUSTROTATE == false) { build_progress_rotation("Nothing to do...", 100); return; } } build_progress_rotation("Rotate access events current size:{$size}M rotate to {$MustRotateAt} M", 10); build_progress_rotation("Rotate tail events current size:{$squidtail_size}M rotate to {$MustRotateAt} M", 10); if ($size > $MustRotateAt) { build_progress_rotation("Backup source file", 15); if (!@copy("/var/log/squid/access.log", "/var/log/squid/access.log." . time())) { squid_admin_mysql(0, "[LOG ROTATION]: Unable to duplicate source log!", "", __FILE__, __LINE__); build_progress_rotation("Rotate events duplicate file failed", 110); return; } if (is_file("/var/log/squid/squidtail.log")) { if (!@copy("/var/log/squid/squidtail.log", "/var/log/squid/squidtail.log." . time())) { squid_admin_mysql(0, "[LOG ROTATION]: Unable to duplicate squidtail source log!", "", __FILE__, __LINE__); build_progress_rotation("Rotate events duplicate file failed", 110); return; } } if (is_file("/var/log/squid/childs-access.log")) { @copy("/var/log/squid/childs-access.log", "/var/log/squid/childs-access.log." . time()); shell_exec("{$echo} \"\" >/var/log/squid/childs-access.log"); } build_progress_rotation("Backup source file {success}", 20); build_progress_rotation("Empty source file", 21); shell_exec("{$echo} \"\" >/var/log/squid/access.log"); shell_exec("{$echo} \"\" >/var/log/squid/hypercache-plugin.log"); if (is_file("/var/log/squid/squidtail.log")) { shell_exec("{$echo} \" \"> /var/log/squid/squidtail.log 2>&1"); } build_progress_rotation("Backup source file {success}", 21); squid_admin_mysql(1, "[LOG ROTATION]: Rotate proxy events {$size}MB", "", __FILE__, __LINE__); build_progress_rotation("Ask proxy to rotate", 25); shell_exec("{$GLOBALS["SQUIDBIN"]} -k rotate >/dev/null 2>&1"); build_progress_rotation("Restarting watchdog services", 30); shell_exec("/etc/init.d/auth-tail restart >/dev/null 2>&1"); shell_exec("/etc/init.d/cache-tail restart >/dev/null 2>&1"); shell_exec("/etc/init.d/hypercache-tail restart >/dev/null 2>&1"); $php = $unix->LOCATE_PHP5_BIN(); $nohup = $unix->find_program("nohup"); if ($GLOBALS["PROGRESS"]) { build_progress_rotation("backup and store files", 35); system("{$php} /usr/share/artica-postfix/exec.squid.rotate.php --force --progress"); } else { shell_exec("{$php} /usr/share/artica-postfix/exec.squid.rotate.php --force >/dev/null 2>&1 &"); } build_progress_rotation("{done}...", 100); } else { build_progress_rotation("Rotate events Nothing to do", 100); } if ($EnableProxyCompressor == 1) { $ScheduleCMD = "--schedule-id={$GLOBALS["SCHEDULE_ID"]}"; if ($GLOBALS["BY_SCHEDULE"]) { $ScheduleCMD = " --byschedule"; } shell_exec("{$nohup} {$php} /usr/share/artica-postfix/exec.zipproxy.php --rotate {$ScheduleCMD} >/dev/null 2>&1 &"); } if (is_file("/var/log/squid/childs-access.log")) { $size = @filesize("/var/log/squid/childs-access.log"); $ROTATED = false; $size = $size / 1024; $size = round($size / 1024); if ($size > $LogsRotateDefaultSizeRotation + 50) { squid_admin_mysql(1, "Rotate Parent log proxy events {$size}MB exceed rule {$LogsRotateDefaultSizeRotation}M and is more than 50MB", "", __FILE__, __LINE__); @copy("/var/log/squid/childs-access.log", "/var/log/squid/childs-access.log." . time()); shell_exec("{$GLOBALS["SQUIDBIN"]} -k rotate >/dev/null 2>&1"); shell_exec("/etc/init.d/auth-tail restart >/dev/null 2>&1"); shell_exec("/etc/init.d/cache-tail restart >/dev/null 2>&1"); $php = $unix->LOCATE_PHP5_BIN(); $nohup = $unix->find_program("nohup"); shell_exec("/usr/share/artica-postfix/exec.squid.rotate.php --force >/dev/null 2>&1 &"); return; } } }