function build()
{
$timefile = "/etc/artica-postfix/pids/exec.squid.rotate.php.build.time";
$pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid";
// /etc/artica-postfix/pids/exec.squid.rotate.php.build.time
$sock = new sockets();
$unix = new unix();
$pid = $unix->get_pid_from_file($pidfile);
if ($unix->process_exists($pid)) {
echo "Already PID {$pid} is running\n";
die;
}
@file_put_contents($pidfile, getmypid());
if (!$GLOBALS["FORCE"]) {
$time = $unix->file_time_min($timefile);
if ($time < 60) {
echo "Only each 60mn\n";
die;
}
@unlink($timefile);
@file_put_contents($timefile, time());
}
$syslog = new mysql_storelogs();
$SquidLogRotateFreq = intval($sock->GET_INFO("SquidLogRotateFreq"));
if ($SquidLogRotateFreq < 10) {
$SquidLogRotateFreq = 1440;
}
$LastRotate = $unix->file_time_min("/etc/artica-postfix/pids/squid-rotate-cache.time");
$LogsRotateDefaultSizeRotation = $sock->GET_INFO("LogsRotateDefaultSizeRotation");
$SquidRotateOnlySchedule = intval($sock->GET_INFO("SquidRotateOnlySchedule"));
if (!is_numeric($LogsRotateDefaultSizeRotation)) {
$LogsRotateDefaultSizeRotation = 100;
}
$LogsRotateDeleteSize = intval($sock->GET_INFO("LogsRotateDeleteSize"));
if ($LogsRotateDeleteSize == 0) {
$LogsRotateDeleteSize = 5000;
}
$BackupMaxDaysDir = $sock->GET_INFO("BackupMaxDaysDir");
if ($BackupMaxDaysDir == null) {
$BackupMaxDaysDir = "/home/logrotate_backup";
}
$BackupSquidLogsUseNas = intval($sock->GET_INFO("BackupSquidLogsUseNas"));
$SquidRotateMergeFiles = $sock->GET_INFO("SquidRotateMergeFiles");
if (!is_numeric($SquidRotateMergeFiles)) {
$SquidRotateMergeFiles = 1;
}
$php = $unix->LOCATE_PHP5_BIN();
$hostname = $unix->hostname_g();
$LogRotatePath = $sock->GET_INFO("LogRotatePath");
if ($LogRotatePath == null) {
$LogRotatePath = "/home/logrotate";
}
$LogRotateAccess = "{$LogRotatePath}/access";
$LogRotateCache = "{$LogRotatePath}/cache";
$syslog->events("Launch rotation only by schedule.: {$SquidRotateOnlySchedule}", __FUNCTION__, __LINE__);
$syslog->events("SquidLogRotateFreq...............: {$SquidLogRotateFreq}Mn", __FUNCTION__, __LINE__);
$syslog->events("LastRotate.......................: {$LastRotate}Mn", __FUNCTION__, __LINE__);
$syslog->events("Working directory................: {$LogRotatePath}", __FUNCTION__, __LINE__);
$syslog->events("Launch rotation when exceed......: {$LogsRotateDefaultSizeRotation}M", __FUNCTION__, __LINE__);
$syslog->events("Delete the file when exceed......: {$LogsRotateDeleteSize}M", __FUNCTION__, __LINE__);
$syslog->events("Final storage directory..........: {$BackupMaxDaysDir}", __FUNCTION__, __LINE__);
$syslog->events("Merge rotated files to a big file: {$SquidRotateMergeFiles}", __FUNCTION__, __LINE__);
$syslog->events("Backup files to a NAS............: {$BackupSquidLogsUseNas}", __FUNCTION__, __LINE__);
if ($handle = opendir("/var/run/squid")) {
while (false !== ($file = readdir($handle))) {
if ($file == ".") {
continue;
}
if ($file == "..") {
continue;
}
$path = "/var/run/squid/{$file}";
if (preg_match("#\\.[0-9]+\\.status\$#", $file)) {
$time = $unix->file_time_min($path);
if ($time > 1440) {
$syslog->events("Removing {$path}", __FUNCTION__, __LINE__);
@unlink($path);
}
continue;
}
if (preg_match("#\\.[0-9]+\\.state\$#", $file)) {
$time = $unix->file_time_min($path);
if ($time > 1440) {
$syslog->events("Removing {$path}", __FUNCTION__, __LINE__);
@unlink($path);
}
continue;
}
}
}
$size = @filesize("/var/log/squid/access.log");
$size = $size / 1024;
$size = $size / 1024;
$syslog->events("/var/log/squid/access.log........: {$size}M", __FUNCTION__, __LINE__);
$syslog->events("Analyze /var/log/squid directory for cache.log", __FUNCTION__, __LINE__);
if (!($handle = opendir("/var/log/squid"))) {
$syslog->events("Unable to open /var/log/squid directory.", __FUNCTION__, __LINE__);
return;
}
while (false !== ($file = readdir($handle))) {
if ($file != "." && $file != "..") {
$path = "/var/log/squid/{$file}";
if (is_dir($path)) {
continue;
}
if (!preg_match("#^cache\\.log\\.[0-9]+\$#", $file)) {
continue;
}
@mkdir("{$LogRotateCache}", 0755, true);
$size = @filesize($path);
$size = $size / 1024;
$size = $size / 1024;
$destfile = "{$LogRotateCache}/{$file}." . time() . ".log";
if (!@copy($path, $destfile)) {
$syslog->events("Unable to copy {$path} to {$destfile}", __FUNCTION__, __LINE__);
@unlink($destfile);
continue;
}
$syslog->events("Removed {$path}", __FUNCTION__, __LINE__);
@unlink($path);
}
}
$syslog->events("Analyze /var/log/squid directory for access.log", __FUNCTION__, __LINE__);
if (!($handle = opendir("/var/log/squid"))) {
$syslog->events("Unable to open /var/log/squid directory.", __FUNCTION__, __LINE__);
return;
}
while (false !== ($file = readdir($handle))) {
if ($file != "." && $file != "..") {
$path = "/var/log/squid/{$file}";
if (is_dir($path)) {
continue;
}
if (!preg_match("#^access\\.log\\.[0-9]+\$#", $file)) {
continue;
}
@mkdir("{$LogRotatePath}", 0755, true);
$size = @filesize($path);
$size = $size / 1024;
$size = $size / 1024;
$destfile = "{$LogRotateAccess}/{$file}." . time() . ".log";
if (!@copy($path, $destfile)) {
$syslog->events("Unable to copy {$path} to {$destfile}", __FUNCTION__, __LINE__);
@unlink($destfile);
continue;
}
$syslog->events("Removed {$path}", __FUNCTION__, __LINE__);
@unlink($path);
}
}
@mkdir($LogRotateAccess, 0755, true);
$syslog->events("Analyze {$LogRotateAccess} for access.log", __FUNCTION__, __LINE__);
if (!($handle = opendir($LogRotateAccess))) {
$syslog->events("Unable to open {$LogRotateAccess} directory.", __FUNCTION__, __LINE__);
return;
}
while (false !== ($file = readdir($handle))) {
if ($file != "." && $file != "..") {
$path = "{$LogRotateAccess}/{$file}";
echo "OPEN {$path}\n";
if (is_dir($path)) {
continue;
}
if (!preg_match("#^access\\.log#", $file)) {
continue;
}
range_fichier_source($path, $BackupMaxDaysDir);
}
}
$q = new mysql();
$sql = "CREATE TABLE IF NOT EXISTS `backuped_logs` (\n\t\t\t `path` CHAR(255) NOT NULL,\n\t\t\t `zDate` DATETIME,\n\t\t\t `size` INT UNSIGNED NOT NULL,\n\t\t\t PRIMARY KEY (`path`),\n\t\t\t KEY `zDate` (`zDate`)\n\t\t\t) ENGINE = MYISAM;";
$q->QUERY_SQL($sql, "artica_events");
$syslog->events("Analyze /home/logrotate/work for access.log", __FUNCTION__, __LINE__);
analyze_directory("/home/logrotate/work", $BackupMaxDaysDir);
analyze_cache_directory($LogRotateCache, $BackupMaxDaysDir);
analyze_cache_directory("/home/logrotate/work", $BackupMaxDaysDir);
analyze_cache_directory("/home/squid/cache-logs", $BackupMaxDaysDir);
if ($SquidRotateMergeFiles == 1) {
Merge_files();
}
if ($BackupSquidLogsUseNas == 1) {
BackupToNas($BackupMaxDaysDir);
$q->QUERY_SQL("TRUNCATE TABLE backuped_logs", "artica_events");
} else {
analyze_destination_directory($BackupMaxDaysDir);
}
@file_put_contents("/etc/artica-postfix/settings/Daemons/BackupMaxDaysDirCurrentSize", $unix->DIRSIZE_KO($BackupMaxDaysDir));
@chmod("/etc/artica-postfix/settings/Daemons/BackupMaxDaysDirCurrentSize", 0777);
return;
}
function build()
{
$timefile = "/etc/artica-postfix/pids/exec.squid.rotate.php.build.time";
$pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".pid";
// /etc/artica-postfix/pids/exec.squid.rotate.php.build.time
$sock = new sockets();
$unix = new unix();
$ls = $unix->find_program("ls");
$pid = $unix->PIDOF_PATTERN(basename(__FILE__));
$MyPid = getmypid();
if ($MyPid != $pid) {
if ($unix->process_exists($pid)) {
$timeFile = $unix->PROCESS_TIME_INT($pid);
$pidCmdline = @file_get_contents("/proc/{$pid}/cmdline");
if ($timeFile < 30) {
echo "Already PID {$pid} is running since {$timeFile}Mn\n";
squid_admin_mysql(1, "[LOG ROTATION]: Skip task, already running {$pid} since {$timeFile}Mn", "Running: {$pidCmdline}", __FILE__, __LINE__);
die;
} else {
squid_admin_mysql(1, "[LOG ROTATION]: Killing old task {$pid} running more than 30mn ({$timeFile}Mn)", "Running: {$pidCmdline}", __FILE__, __LINE__);
$unix->KILL_PROCESS($pid);
}
}
}
@file_put_contents($pidfile, getmypid());
if (!$GLOBALS["FORCE"]) {
$time = $unix->file_time_min($timefile);
if ($time < 60) {
echo "Only each 60mn\n";
die;
}
@unlink($timefile);
@file_put_contents($timefile, time());
}
$syslog = new mysql_storelogs();
$SquidLogRotateFreq = intval($sock->GET_INFO("SquidLogRotateFreq"));
if ($SquidLogRotateFreq < 10) {
$SquidLogRotateFreq = 1440;
}
$LastRotate = $unix->file_time_min("/etc/artica-postfix/pids/squid-rotate-cache.time");
$LogsRotateDefaultSizeRotation = $sock->GET_INFO("LogsRotateDefaultSizeRotation");
$SquidRotateOnlySchedule = intval($sock->GET_INFO("SquidRotateOnlySchedule"));
if (!is_numeric($LogsRotateDefaultSizeRotation)) {
$LogsRotateDefaultSizeRotation = 100;
}
$BackupMaxDaysDir = $sock->GET_INFO("BackupMaxDaysDir");
if ($BackupMaxDaysDir == null) {
$BackupMaxDaysDir = "/home/logrotate_backup";
}
$BackupSquidLogsUseNas = intval($sock->GET_INFO("BackupSquidLogsUseNas"));
$SquidRotateAutomount = intval($sock->GET_INFO("SquidRotateAutomount"));
$SquidRotateClean = intval($sock->GET_INFO("SquidRotateClean"));
$SquidRotateAutomountRes = $sock->GET_INFO("SquidRotateAutomountRes");
$SquidRotateAutomountFolder = $sock->GET_INFO("SquidRotateAutomountFolder");
if ($SquidRotateAutomount == 1) {
shell_exec("{$ls} /automounts/{$SquidRotateAutomountRes} >/dev/null 2>&1");
if (ifdirMounted("/automounts/{$SquidRotateAutomountRes}")) {
$BackupSquidLogsUseNas = 0;
$BackupMaxDaysDir = "/automounts/{$SquidRotateAutomountRes}/{$SquidRotateAutomountFolder}";
} else {
$syslog->events("/automounts/{$SquidRotateAutomountRes} not mounted", __FUNCTION__, __LINE__);
squid_admin_mysql(1, "[ROTATE],Auto-mount {$SquidRotateAutomountRes} not mounted", null, __FILE__, __LINE__);
}
}
$BackupMaxDaysDir = str_replace("//", "/", $BackupMaxDaysDir);
$BackupMaxDaysDir = str_replace("\\", "/", $BackupMaxDaysDir);
if (!is_dir($BackupMaxDaysDir)) {
@mkdir($BackupMaxDaysDir, 0755, true);
}
if (!is_dir($BackupMaxDaysDir)) {
$syslog->events("{$BackupMaxDaysDir} not such directory or permission denied", __FUNCTION__, __LINE__);
squid_admin_mysql(1, "[ROTATE],{$BackupMaxDaysDir} not such directory or permission denied", null, __FILE__, __LINE__);
if ($SquidRotateAutomount == 1) {
$BackupMaxDaysDir = $sock->GET_INFO("BackupMaxDaysDir");
if ($BackupMaxDaysDir == null) {
$BackupMaxDaysDir = "/home/logrotate_backup";
}
if (!is_dir($BackupMaxDaysDir)) {
@mkdir($BackupMaxDaysDir, 0755, true);
}
$syslog->events("Return back to {$BackupMaxDaysDir}", __FUNCTION__, __LINE__);
} else {
return;
}
}
$php = $unix->LOCATE_PHP5_BIN();
$hostname = $unix->hostname_g();
$InFluxBackupDatabaseDir = $sock->GET_INFO("InFluxBackupDatabaseDir");
if ($InFluxBackupDatabaseDir == null) {
$InFluxBackupDatabaseDir = "/home/artica/influx/backup";
}
$LogRotatePath = $sock->GET_INFO("LogRotatePath");
if ($LogRotatePath == null) {
$LogRotatePath = "/home/logrotate";
}
$LogRotateAccess = "{$LogRotatePath}/access";
$LogRotateTail = "{$LogRotatePath}/tail";
$LogRotateCache = "{$LogRotatePath}/cache";
$syslog->events("Launch rotation only by schedule.: {$SquidRotateOnlySchedule}", __FUNCTION__, __LINE__);
$syslog->events("SquidLogRotateFreq...............: {$SquidLogRotateFreq}Mn", __FUNCTION__, __LINE__);
$syslog->events("LastRotate.......................: {$LastRotate}Mn", __FUNCTION__, __LINE__);
$syslog->events("Working directory................: {$LogRotatePath}", __FUNCTION__, __LINE__);
$syslog->events("Launch rotation when exceed......: {$LogsRotateDefaultSizeRotation}M", __FUNCTION__, __LINE__);
$syslog->events("Final storage directory..........: {$BackupMaxDaysDir}", __FUNCTION__, __LINE__);
$syslog->events("Backup files to a NAS............: {$BackupSquidLogsUseNas}", __FUNCTION__, __LINE__);
ss5_log($BackupMaxDaysDir);
if ($handle = opendir("/var/run/squid")) {
while (false !== ($file = readdir($handle))) {
if ($file == ".") {
continue;
}
if ($file == "..") {
continue;
}
$path = "/var/run/squid/{$file}";
if (preg_match("#\\.[0-9]+\\.status\$#", $file)) {
$time = $unix->file_time_min($path);
if ($time > 1440) {
$syslog->events("Removing {$path}", __FUNCTION__, __LINE__);
@unlink($path);
}
continue;
}
if (preg_match("#\\.[0-9]+\\.state\$#", $file)) {
$time = $unix->file_time_min($path);
if ($time > 1440) {
$syslog->events("Removing {$path}", __FUNCTION__, __LINE__);
@unlink($path);
}
continue;
}
}
}
$size = @filesize("/var/log/squid/access.log");
$size = $size / 1024;
$size = $size / 1024;
$syslog->events("/var/log/squid/access.log........: {$size}M", __FUNCTION__, __LINE__);
$syslog->events("Analyze /var/log/squid directory for cache.log", __FUNCTION__, __LINE__);
if (!($handle = opendir("/var/log/squid"))) {
build_progress_rotation("Unable to open /var/log/squid", 110);
$syslog->events("Unable to open /var/log/squid directory.", __FUNCTION__, __LINE__);
return;
}
build_progress_rotation("Scanning /var/log/squid", 40);
while (false !== ($file = readdir($handle))) {
if ($file != "." && $file != "..") {
$path = "/var/log/squid/{$file}";
if (is_dir($path)) {
continue;
}
if (!preg_match("#^cache\\.log\\.[0-9]+\$#", $file)) {
continue;
}
@mkdir("{$LogRotateCache}", 0755, true);
$size = @filesize($path);
$size = $size / 1024;
$size = $size / 1024;
$destfile = "{$LogRotateCache}/{$file}." . time() . ".log";
if (!@copy($path, $destfile)) {
$syslog->events("Unable to copy {$path} to {$destfile}", __FUNCTION__, __LINE__);
@unlink($destfile);
continue;
}
$syslog->events("Removed {$path}", __FUNCTION__, __LINE__);
@unlink($path);
}
}
$syslog->events("Analyze /var/log/squid directory for access.log", __FUNCTION__, __LINE__);
if (!($handle = opendir("/var/log/squid"))) {
$syslog->events("Unable to open /var/log/squid directory.", __FUNCTION__, __LINE__);
return;
}
@mkdir($LogRotateAccess, 0755, true);
@mkdir($LogRotateTail, 0755, true);
while (false !== ($file = readdir($handle))) {
if ($file != "." && $file != "..") {
$path = "/var/log/squid/{$file}";
if (is_dir($path)) {
continue;
}
if (preg_match("#^childs-access\\.log\\.[0-9]+\$#", $file)) {
$destfile = "{$LogRotateAccess}/{$file}." . time() . ".log";
if (!@copy($path, $destfile)) {
$syslog->events("Unable to copy {$path} to {$destfile}", __FUNCTION__, __LINE__);
@unlink($destfile);
continue;
}
$syslog->events("Removed {$path}", __FUNCTION__, __LINE__);
@unlink($path);
continue;
}
$syslog->events("Analyze {$file} ^squidtail\\.log\\.[0-9]+\$", __FUNCTION__, __LINE__);
if (preg_match("#^squidtail\\.log\\.[0-9]+\$#", $file)) {
$destfile = "{$LogRotateTail}/{$file}";
if (!@copy($path, $destfile)) {
$syslog->events("Unable to copy {$path} to {$destfile}", __FUNCTION__, __LINE__);
@unlink($destfile);
continue;
}
$syslog->events("Removed {$path}", __FUNCTION__, __LINE__);
@unlink($path);
continue;
}
$syslog->events("Analyze {$file} ^access\\.log\\.[0-9]+\$", __FUNCTION__, __LINE__);
if (!preg_match("#^access\\.log\\.[0-9]+\$#", $file)) {
continue;
}
@mkdir("{$LogRotatePath}", 0755, true);
$destfile = "{$LogRotateAccess}/{$file}." . time() . ".log";
if (!@copy($path, $destfile)) {
$syslog->events("Unable to copy {$path} to {$destfile}", __FUNCTION__, __LINE__);
@unlink($destfile);
continue;
}
$syslog->events("Removed {$path}", __FUNCTION__, __LINE__);
@unlink($path);
}
}
$syslog->events("Analyze {$LogRotateAccess} for access.log", __FUNCTION__, __LINE__);
if (!($handle = opendir($LogRotateAccess))) {
$syslog->events("Unable to open {$LogRotateAccess} directory.", __FUNCTION__, __LINE__);
return;
}
$ROTATED = false;
while (false !== ($file = readdir($handle))) {
if ($file != "." && $file != "..") {
$path = "{$LogRotateAccess}/{$file}";
echo "OPEN {$path}\n";
if (is_dir($path)) {
continue;
}
if (!preg_match("#^access\\.log#", $file)) {
continue;
}
range_fichier_source($path, $BackupMaxDaysDir);
$ROTATED = true;
}
}
if ($handle = opendir($LogRotateTail)) {
while (false !== ($file = readdir($handle))) {
if ($file != "." && $file != "..") {
$path = "{$LogRotateTail}/{$file}";
echo "OPEN {$path}\n";
if (is_dir($path)) {
continue;
}
if (!preg_match("#^squidtail\\.log#", $file)) {
continue;
}
range_fichier_tail($path, $BackupMaxDaysDir);
$ROTATED = true;
}
}
}
if (!$ROTATED) {
return;
}
$q = new mysql();
$sql = "CREATE TABLE IF NOT EXISTS `backuped_logs` (\n\t\t\t `path` CHAR(255) NOT NULL,\n\t\t\t `zDate` DATETIME,\n\t\t\t `size` INT UNSIGNED NOT NULL,\n\t\t\t PRIMARY KEY (`path`),\n\t\t\t KEY `zDate` (`zDate`)\n\t\t\t) ENGINE = MYISAM;";
$q->QUERY_SQL($sql, "artica_events");
$syslog->events("Analyze /home/logrotate/work for access.log", __FUNCTION__, __LINE__);
build_progress_rotation("Scanning /home/logrotate/work", 45);
analyze_directory("/home/logrotate/work", $BackupMaxDaysDir);
$BackupMaxDaysDir2 = $sock->GET_INFO("BackupMaxDaysDir");
if ($BackupMaxDaysDir2 == null) {
$BackupMaxDaysDir2 = "/home/logrotate_backup";
}
if ($BackupMaxDaysDir2 != $BackupMaxDaysDir) {
build_progress_rotation("Scanning {$BackupMaxDaysDir2}", 46);
$syslog->events("{$BackupMaxDaysDir2} is different of {$BackupMaxDaysDir}", __FUNCTION__, __LINE__);
analyze_directory($BackupMaxDaysDir2, $BackupMaxDaysDir);
}
build_progress_rotation("Scanning /home/logrotate/merged", 47);
analyze_garbage_directory("/home/logrotate/merged", $BackupMaxDaysDir, 1440);
build_progress_rotation("Scanning {$LogRotateCache}", 48);
analyze_cache_directory($LogRotateCache, $BackupMaxDaysDir);
build_progress_rotation("Scanning /home/logrotate/work", 49);
analyze_cache_directory("/home/logrotate/work", $BackupMaxDaysDir);
build_progress_rotation("Scanning /home/squid/cache-logs", 49);
analyze_cache_directory("/home/squid/cache-logs", $BackupMaxDaysDir);
if ($GLOBALS["VERBOSE"]) {
echo "TRUNCATE TABLE backuped_logs !!!\n";
}
$q->QUERY_SQL("TRUNCATE TABLE backuped_logs", "artica_events");
if ($BackupSquidLogsUseNas == 1) {
build_progress_rotation("Backup to N.A.S", 50);
BackupToNas($BackupMaxDaysDir);
build_progress_rotation("Backup to N.A.S BigData backups", 50);
BackupToNas($InFluxBackupDatabaseDir, false);
} else {
build_progress_rotation("Scanning {$BackupMaxDaysDir}", 50);
analyze_destination_directory($BackupMaxDaysDir);
}
@file_put_contents("/etc/artica-postfix/settings/Daemons/BackupMaxDaysDirCurrentSize", $unix->DIRSIZE_KO($BackupMaxDaysDir));
@chmod("/etc/artica-postfix/settings/Daemons/BackupMaxDaysDirCurrentSize", 0777);
return;
}
