function sigtool()
{
$unix = new unix();
$sigtool = $unix->find_program("sigtool");
if (strlen($sigtool) < 5) {
die;
}
if (is_file("/usr/share/artica-postfix/ressources/interface-cache/ClamAVBases")) {
$ttim = $unix->file_time_min("/usr/share/artica-postfix/ressources/interface-cache/ClamAVBases");
if ($ttim < 30) {
return;
}
}
$baseDir = "/var/lib/clamav";
$patnz = $unix->DirFiles($baseDir, "\\.(cvd|cld|hdb|ign2|ndb)\$");
while (list($path, $none) = each($patnz)) {
$patterns[basename($path)] = true;
}
while (list($pattern, $none) = each($patterns)) {
if (!is_file("{$baseDir}/{$pattern}")) {
continue;
}
$results = array();
exec("{$sigtool} --info={$baseDir}/{$pattern} 2>&1", $results);
while (list($index, $line) = each($results)) {
if (preg_match("#Build time:\\s+(.+)#", $line, $re)) {
$time = strtotime($re[1]);
$MAIN[$pattern]["zDate"] = date("Y-m-d H:i:s");
continue;
}
if (preg_match("#Version:\\s+([0-9]+)#", $line, $re)) {
$MAIN[$pattern]["version"] = $re[1];
continue;
}
if (preg_match("#Signatures:\\s+([0-9]+)#", $line, $re)) {
$MAIN[$pattern]["signatures"] = $re[1];
continue;
}
}
if (!isset($MAIN[$pattern]["zDate"])) {
$time = filemtime("{$baseDir}/{$pattern}");
$MAIN[$pattern]["zDate"] = date("Y-m-d H:i:s", $time);
if (!isset($MAIN[$pattern]["version"])) {
$MAIN[$pattern]["version"] = date("YmdHi", $time);
}
}
if (!isset($MAIN[$pattern]["signatures"])) {
$MAIN[$pattern]["signatures"] = $unix->COUNT_LINES_OF_FILE("{$baseDir}/{$pattern}");
}
}
if (count($MAIN) == 0) {
return;
}
@file_put_contents("/usr/share/artica-postfix/ressources/interface-cache/ClamAVBases", serialize($MAIN));
}
function BuildDatabaseStatus()
{
if (!ifMustBeExecuted()) {
WriteMyLogs("No make sense to execute this script...", __FUNCTION__, __FILE__, __LINE__);
if ($GLOBALS["VERBOSE"]) {
echo "No make sense to execute this script...\n";
}
die;
}
$q = new mysql_squid_builder();
$unix = new unix();
$dirs = $unix->dirdir("/var/lib/ftpunivtlse1fr");
$TLSE_CONVERTION = $q->TLSE_CONVERTION();
while (list($directory, $line) = each($dirs)) {
$catzname = $TLSE_CONVERTION[basename($directory)];
if ($catzname == null) {
continue;
}
if (!is_file("{$directory}/domains")) {
if ($GLOBALS["VERBOSE"]) {
echo "{$catzname}=0\n";
}
$f[$catzname]["F"] = 0;
$f[$catzname]["D"] = "0000-00-00 00:00:00";
$f[$catzname]["FF"] = "{$directory}/domains";
} else {
$f[$catzname]["F"] = $unix->COUNT_LINES_OF_FILE("{$directory}/domains");
if ($GLOBALS["VERBOSE"]) {
echo "{$catzname}={$f[$catzname]["F"]}\n";
}
$f[$catzname]["D"] = date("Y-m-d H:i:s", filemtime("{$directory}/domains"));
$f[$catzname]["FF"] = "{$directory}/domains";
}
}
$LastC = trim(@file_get_contents("/etc/artica-postfix/ftpunivtlse1frCount"));
$c = 0;
while (list($cat, $array) = each($f)) {
$count = $array["F"];
$date = $array["D"];
$c = $c + $count;
$sql[] = "('{$cat}','{$count}','{$date}')";
}
if (count($sql) > 0) {
$q->QUERY_SQL("TRUNCATE TABLE univtlse1fr");
$q->QUERY_SQL("INSERT IGNORE INTO univtlse1fr (category,websitesnum,zDate) VALUES " . @implode(",", $sql));
if (!$q->ok) {
artica_update_event(2, "Fatal {$q->mysql_error}", null, __FILE__, __LINE__);
return;
}
if ($c != $LastC) {
//artica_update_event("Toulouse University status: $c items in database",__FUNCTION__,__FILE__,__LINE__);
@file_put_contents("/etc/artica-postfix/ftpunivtlse1frCount", $c);
}
}
//univtlse1fr
}
function ExplodeFile($filepath, $md5file = null)
{
$unix = new unix();
$LastScannLine = 0;
$GLOBALS["MYSQL_CATZ"] = new mysql_catz();
$GLOBALS["SQUID_FAMILY_CLASS"] = new squid_familysite();
if (!isset($GLOBALS["MYHOSTNAME"])) {
$unix = new unix();
$GLOBALS["MYHOSTNAME"] = $unix->hostname_g();
}
$GLOBALS["SEQUENCE"] = md5_file($filepath);
if (!is_file("{$filepath}.last")) {
if (is_file("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log")) {
$influx = new influx();
if ($influx->files_inject("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log")) {
@unlink("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log");
return true;
}
}
}
$handle = @fopen($filepath, "r");
if (!$handle) {
echo "Fopen failed on {$filepath}\n";
if ($md5file != null) {
mysql_progress($md5file, 100, 3, "Fopen {failed} on {$filepath}");
}
return false;
}
$countlines = 0;
if ($md5file != null) {
$countlines = $unix->COUNT_LINES_OF_FILE($filepath);
if ($md5file != null) {
mysql_progress($md5file, 10, 0, "Parsing {$countlines}");
}
}
if (is_file("{$filepath}.last")) {
$LastScannLine = intval(@file_get_contents("{$filepath}.last"));
}
$c = 0;
$d = 0;
$e = 0;
$prc = 0;
$prc_text = 0;
$mysql_first_time = 0;
if ($LastScannLine > 0) {
fseek($handle, $LastScannLine, SEEK_SET);
}
while (!feof($handle)) {
$c++;
$d++;
$e++;
if ($countlines > 0) {
$prc = $c / $countlines;
$prc = round($prc * 100);
if (!isset($GLOBALS["LAST_PRC"])) {
if ($GLOBALS["PROGRESS"]) {
echo "{$prc}%\n";
}
$GLOBALS["LAST_PRC"] = $prc;
} else {
if ($GLOBALS["LAST_PRC"] != $prc) {
if ($GLOBALS["PROGRESS"]) {
echo "{$prc}%\n";
}
$GLOBALS["LAST_PRC"] = $prc;
}
}
if ($prc > 10) {
if ($prc < 99) {
if ($prc > $prc_text) {
$array_load = sys_getloadavg();
$internal_load = $array_load[0];
$mem = round(memory_get_usage() / 1024 / 1000, 2);
$prc_design = FormatNumber($c) . "/" . FormatNumber($countlines);
if ($md5file != null) {
mysql_progress($md5file, $prc, 1, "{parsing} {$prc_design} {load}:{$internal_load} {memory}:{$mem}MB");
}
$prc_text = $prc;
}
}
}
}
if ($d > 50) {
$iSeek = ftell($handle);
@file_put_contents("{$filepath}.last", $iSeek);
if ($GLOBALS["VERBOSE"]) {
$prc_design = FormatNumber($c) . "/" . FormatNumber($countlines);
echo "{$prc}% {$prc_design}\n";
}
$d = 0;
}
if ($e > 500) {
$mem = round(memory_get_usage() / 1024 / 1000, 2);
$prc_design = FormatNumber($c) . "/" . FormatNumber($countlines);
if ($md5file != null) {
mysql_progress($md5file, $prc, 1, "{parsing} {$prc_design} {load}:{$internal_load} {memory}:{$mem}MB");
}
$e = 0;
}
$buffer = trim(fgets($handle));
if ($buffer == null) {
continue;
}
$array = parseAccessLine($buffer);
if (count($array) == 0) {
continue;
}
if ($mysql_first_time == 0) {
if (date("Y", $array["TIME"]) > 2001) {
$mysql_first_time = $array["TIME"];
mysql_first_time($md5file, $mysql_first_time);
}
}
CachedSizeMem($array["TIME"], $array["CACHED"], $array["SIZE"]);
if (intval($array["SIZE"]) == 0) {
if ($GLOBALS["VERBOSE"]) {
echo "Size = 0 " . __LINE__ . "\n";
}
}
CachedUserMem($array["TIME"], $array["SITENAME"], $array["SIZE"], null, $array["UID"], $array["IPADDR"], $array["CATEGORY"], $array["FAMILYSITE"]);
}
@unlink("{$filepath}.last");
mysql_last_time($md5file, $array["TIME"]);
CachedUserMem_dump();
CachedSizeMem_dump();
$influx = new influx();
$size = filesize("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log");
$size = $size / 1024;
$size = $size / 1024;
echo "Importing {$size}MB of data....\n";
if (!$influx->files_inject("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log")) {
@unlink("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log");
return false;
}
@unlink("/home/artica/import-temp/{$GLOBALS["SEQUENCE"]}.working.log");
return true;
}
function BuildPatterns()
{
$unix = new unix();
cpulimit();
$dirs = $unix->dirdir("/etc/dansguardian/lists/blacklists");
if ($GLOBALS["VERBOSE"]) {
echo "open /etc/dansguardian/lists/blacklists array of " . count($dirs) . "\n";
}
if (!is_array($dirs)) {
writelogs("Unable to dir /etc/dansguardian/lists/blacklists", __FUNCTION__, __FILE__, __LINE__);
return;
}
reset($dirs);
while (list($num, $val) = each($dirs)) {
$category = basename($num);
if ($GLOBALS["VERBOSE"]) {
echo "{$category}:: {$num} -> {$val}\n";
}
writelogs("Checking {$category}", __FUNCTION__, __FILE__, __LINE__);
if ($category == "blacklists") {
if ($GLOBALS["VERBOSE"]) {
echo "{$category} == blacklists, aborting\n";
}
continue;
}
$domains = 0;
$urls = 0;
$expressions = 0;
if (is_file("{$num}/domains")) {
$domains = $unix->COUNT_LINES_OF_FILE("{$num}/domains");
if ($GLOBALS["VERBOSE"]) {
echo "{$category}:: {$domains} number\n";
}
$filetime = date("Y-m-d H:i:s", filemtime("{$num}/domains"));
} else {
if ($GLOBALS["VERBOSE"]) {
echo "{$category}:: unable to stat {$num}/domains\n";
}
}
if (is_file("{$num}/urls")) {
$urls = $unix->COUNT_LINES_OF_FILE("{$num}/urls");
} else {
if ($GLOBALS["VERBOSE"]) {
echo "{$category}:: unable to stat {$num}/urls\n";
}
}
if ($GLOBALS["VERBOSE"]) {
echo "{$category}={$domains},{$urls},{$filetime}\n";
}
$array["{$category}"] = array($domains, $urls, $filetime);
}
$datas = base64_encode(serialize($array));
writelogs("writing /usr/share/artica-postfix/ressources/logs/dansguardian.patterns", __FUNCTION__, __FILE__, __LINE__);
@file_put_contents("/usr/share/artica-postfix/ressources/logs/dansguardian.patterns", $datas);
@chmod("/usr/share/artica-postfix/ressources/logs/dansguardian.patterns", 0755);
if (!is_file("/usr/share/artica-postfix/ressources/logs/dansguardian.patterns")) {
writelogs("Error writing dansguardian.patterns", __FUNCTION__, __FILE__, __LINE__);
}
return;
}
function GetIndex()
{
$WORKING_DIR = "/home/working_toulouse_databases";
$WORKING_DOWNLOAD = "{$WORKING_DIR}/dowloads";
$WORKING_UPLOAD = "{$WORKING_DIR}/uploads";
@mkdir($WORKING_DOWNLOAD, 0755, true);
@mkdir($WORKING_UPLOAD, 0755, true);
$unix = new unix();
$tar = $unix->find_program("tar");
$catz = new mysql_catz();
$tmpfile = $unix->FILE_TEMP();
$tmpdir = $unix->TEMP_DIR();
$rm = $unix->find_program("rm");
$mainuri = "ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib";
$EXEC_NICE = $unix->EXEC_NICE();
$ufdbGenTable = $unix->find_program("ufdbGenTable");
$curl = new ccurl("{$mainuri}/MD5SUM.LST");
if (!$curl->GetFile($tmpfile)) {
echo "Failed to download MD5SUM.LST\n";
return;
}
$tr = explode("\n", @file_get_contents($tmpfile));
while (list($index, $line) = each($tr)) {
if (preg_match("#(.+?)\\s+(.+)#", $line, $re)) {
$filename = trim($re[1]);
$md5 = trim($re[2]);
$array[$md5] = $filename;
}
}
@unlink($tmpfile);
$q = new mysql_squid_builder();
$TLSE_CONVERTION = TLSE_CONVERTION();
$FINAL_ARRAY = array();
while (list($filename, $md5) = each($array)) {
$category = str_replace(".tar.gz", "", $filename);
if (isset($TLSE_CONVERTION[$category])) {
$FINAL_ARRAY[$filename] = $md5;
}
}
$UPDATED = 0;
$MAIN_ARRAY = unserialize(base64_decode(@file_get_contents("{$WORKING_UPLOAD}/index.txt")));
while (list($filename, $md5) = each($FINAL_ARRAY)) {
$TargetFile = "{$WORKING_DOWNLOAD}/{$filename}";
$categoryname = str_replace(".tar.gz", "", $filename);
if ($categoryname == "adult") {
continue;
}
if ($categoryname == "aggressive") {
continue;
}
if ($categoryname == "agressif") {
continue;
}
if ($categoryname == "redirector") {
continue;
}
if ($categoryname == "ads") {
continue;
}
if ($categoryname == "drogue") {
continue;
}
$MyStoreMd5 = md5_file($TargetFile);
if ($MyStoreMd5 == $md5) {
echo "Skipping {$filename}\n";
continue;
}
echo "Downloading {$filename}\n";
$curl = new ccurl("{$mainuri}/{$filename}");
$tmpfile = $unix->FILE_TEMP();
if (!$curl->GetFile($tmpfile)) {
echo "Failed {$curl->error}\n";
@unlink($tmpfile);
continue;
}
$md5_tmp = md5_file($tmpfile);
if ($md5_tmp != $md5) {
echo "Failed Corrupted file\n";
@unlink($tmpfile);
continue;
}
if (is_file($TargetFile)) {
@unlink($TargetFile);
}
if (!@copy($tmpfile, $TargetFile)) {
echo "Failed Copy file\n";
@unlink($tmpfile);
@unlink($TargetFile);
continue;
}
@unlink($tmpfile);
$MyStoreMd5 = md5_file($TargetFile);
if ($MyStoreMd5 != $md5) {
echo "Failed MD5 file\n";
@unlink($TargetFile);
continue;
}
@mkdir("{$WORKING_DIR}/{$categoryname}", 0755, true);
echo "Extracting {$TargetFile}\n";
$cmd = "{$tar} xvf {$TargetFile} -C {$WORKING_DIR}/{$categoryname}/";
echo $cmd . "\n";
system($cmd);
$SOURCE_DIR = find_sources("{$WORKING_DIR}/{$categoryname}");
if (!is_file("{$SOURCE_DIR}/domains")) {
echo "Failed {$SOURCE_DIR}/domains no such file\n";
@unlink($TargetFile);
continue;
}
$COUNT_OF_DOMAINS = $unix->COUNT_LINES_OF_FILE("{$SOURCE_DIR}/domains");
echo "{$categoryname} {$COUNT_OF_DOMAINS} domains\n";
if ($COUNT_OF_DOMAINS == 0) {
shell_exec("{$rm} -rf {$WORKING_DIR}/{$categoryname}");
@unlink($TargetFile);
continue;
}
if (is_file("{$SOURCE_DIR}/domains.ufdb")) {
@unlink("{$SOURCE_DIR}/domains.ufdb");
}
if (!is_file("{$SOURCE_DIR}/urls")) {
@touch("{$SOURCE_DIR}/urls");
}
$u = " -u {$SOURCE_DIR}/urls";
$d = " -d {$SOURCE_DIR}/domains";
$cmd = "{$EXEC_NICE}{$ufdbGenTable} -n -q -W -t {$categoryname}{$d}{$u}";
echo $cmd . "\n";
shell_exec($cmd);
if (!is_file("{$SOURCE_DIR}/domains.ufdb")) {
echo "Failed to compile {$categoryname}\n";
@unlink($TargetFile);
continue;
}
$MD5SRC = md5_file("{$SOURCE_DIR}/domains.ufdb");
if (is_file("{$WORKING_UPLOAD}/{$categoryname}.gz")) {
@unlink("{$WORKING_UPLOAD}/{$categoryname}.gz");
}
$unix->compress("{$SOURCE_DIR}/domains.ufdb", "{$WORKING_UPLOAD}/{$categoryname}.gz");
$MD5GZ = md5_file("{$WORKING_UPLOAD}/{$categoryname}.gz");
$UPDATED++;
$NOTIFICATIONS[] = "{$categoryname} updated with {$COUNT_OF_DOMAINS} domains";
$MAIN_ARRAY[$categoryname]["ROWS"] = $COUNT_OF_DOMAINS;
$MAIN_ARRAY[$categoryname]["MD5SRC"] = $MD5SRC;
$MAIN_ARRAY[$categoryname]["MD5GZ"] = $MD5GZ;
$MAIN_ARRAY[$categoryname]["TIME"] = time();
$MAIN_ARRAY[$categoryname]["SIZE"] = @filesize("{$WORKING_UPLOAD}/{$categoryname}.gz");
@file_put_contents("{$WORKING_UPLOAD}/index.txt", base64_encode(serialize($MAIN_ARRAY)));
}
if ($UPDATED > 0) {
PushToRepo_alls();
sendEmail("{$UPDATED} Toulouse Unversity databases uploaded.", @implode("\n", $NOTIFICATIONS));
}
}
function xstart()
{
$unix = new unix();
$pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . ".pid";
$timefile = "/etc/artica-postfix/pids/exec.ufdb.parse-categories.php.time";
$unix = new unix();
$me = basename(__FILE__);
$pid = $unix->get_pid_from_file($pidfile);
if (system_is_overloaded()) {
die;
}
if ($unix->process_exists($pid, $me)) {
if ($GLOBALS["VERBOSE"]) {
echo " {$pid} --> Already executed.. aborting the process\n";
}
$time = $unix->PROCCESS_TIME_MIN($pid);
die;
}
@file_put_contents($pidfile, getmypid());
if ($unix->file_time_min($timefile) < 60) {
return;
}
@unlink($timefile);
@file_put_contents($timefile, time());
$q = new mysql_squid_builder();
$DirsArtica = $unix->dirdir("/var/lib/ufdbartica");
$sql = "CREATE TABLE IF NOT EXISTS `UPDATE_DBWF_INFOS` ( \n\t`category` varchar(90) NOT NULL, `size_artica` INT UNSIGNED NOT NULL, `date_artica` INT UNSIGNED NOT NULL, `count_artica` INT UNSIGNED NOT NULL, `size_tlse` INT UNSIGNED NOT NULL, `date_tlse` INT UNSIGNED NOT NULL, `count_tlse` INT UNSIGNED NOT NULL, `size_perso` INT UNSIGNED NOT NULL, `date_perso` INT UNSIGNED NOT NULL, `count_perso` INT UNSIGNED NOT NULL, PRIMARY KEY (`category`) \n\t\t\t) ENGINE=MYISAM;";
$q->QUERY_SQL($sql);
if (!$q->ok) {
echo $q->mysql_error;
return;
}
$MAX = 144;
$c = 0;
$UFDB = array();
$UFDBCOUNT = unserialize(base64_decode(@file_get_contents("/etc/artica-postfix/ufdbcounts.txt")));
while (list($dir, $line) = each($DirsArtica)) {
if (is_link($dir)) {
continue;
}
$database_path = "{$dir}/domains.ufdb";
if (!is_file($database_path)) {
continue;
}
$tablename = basename($dir);
$size = @filesize("{$dir}/domains.ufdb");
$time = filemtime("{$dir}/domains.ufdb");
$cat = $q->tablename_tocat($tablename);
$MAIN[$cat]["ART"]["SIZE"] = $size;
$MAIN[$cat]["ART"]["TIME"] = $time;
$MAIN[$cat]["ART"]["COUNT"] = $UFDBCOUNT[$tablename];
}
$DirsArtica = $unix->dirdir("/var/lib/ftpunivtlse1fr");
while (list($dir, $line) = each($DirsArtica)) {
$database_path = "{$dir}/domains.ufdb";
$sourcefile = "{$dir}/domains";
if (!is_file($database_path)) {
continue;
}
$cat = basename($dir);
$cat = $q->filaname_tocat($cat);
$size = @filesize("{$dir}/domains.ufdb");
$time = filemtime("{$dir}/domains.ufdb");
$MAIN[$cat]["TLSE"]["SIZE"] = $size;
$MAIN[$cat]["TLSE"]["TIME"] = $time;
$MAIN[$cat]["TLSE"]["COUNT"] = $unix->COUNT_LINES_OF_FILE($sourcefile);
if (system_is_overloaded()) {
@unlink("{$timefile}");
die;
}
}
$DirsArtica = $unix->dirdir("/var/lib/squidguard");
while (list($dir, $line) = each($DirsArtica)) {
$database_path = "{$dir}/domains.ufdb";
if (!is_file($database_path)) {
continue;
}
$tablename = "category_" . basename($dir);
$cat = $q->tablename_tocat($tablename);
$size = @filesize("{$dir}/domains.ufdb");
$time = filemtime("{$dir}/domains.ufdb");
$sourcefile = "{$dir}/domains";
$MAIN[$cat]["PERSO"]["SIZE"] = $size;
$MAIN[$cat]["PERSO"]["PATH"] = $dir;
$MAIN[$cat]["PERSO"]["CATEGORY"] = $cat;
$MAIN[$cat]["PERSO"]["TIME"] = $time;
$MAIN[$cat]["PERSO"]["COUNT"] = $unix->COUNT_LINES_OF_FILE($sourcefile);
if (system_is_overloaded()) {
@unlink("{$timefile}");
die;
}
}
$prefix = "INSERT IGNORE INTO `UPDATE_DBWF_INFOS` (`category`,\n\t`size_artica` ,\n\t`date_artica` ,\n\t`count_artica` ,\n\n\t`size_tlse` ,\n\t`date_tlse` ,\n\t`count_tlse` ,\t\t\t\n\t\t\t\n\t`size_perso` ,\n\t`date_perso` ,\n\t`count_perso`) VALUES ";
while (list($category, $MAINZ) = each($MAIN)) {
$f[] = "('{$category}','{$MAINZ["ART"]["SIZE"]}','{$MAINZ["ART"]["TIME"]}','{$MAINZ["ART"]["COUNT"]}','{$MAINZ["TLSE"]["SIZE"]}','{$MAINZ["TLSE"]["TIME"]}','{$MAINZ["TLSE"]["COUNT"]}','{$MAINZ["PERSO"]["SIZE"]}','{$MAINZ["PERSO"]["TIME"]}','{$MAINZ["PERSO"]["COUNT"]}')";
}
$q->QUERY_SQL("TRUNCATE TABLE `UPDATE_DBWF_INFOS`");
$sql = $prefix . @implode(",", $f);
$q->QUERY_SQL($sql);
if (!$q->ok) {
echo $q->mysql_error . "\n{$sql}\n";
return;
}
}
function restore($filename, $storeid)
{
$filename = trim($filename);
$unix = new unix();
$pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . ".{$filename}.pid";
$pid = @file_get_contents("{$pidfile}");
if ($unix->process_exists($pid, basename(__FILE__))) {
die;
}
@file_put_contents($pidfile, getmypid());
$EnableSyslogDB = @file_get_contents("/etc/artica-postfix/settings/Daemons/EnableSyslogDB");
if (!is_numeric($EnableSyslogDB)) {
$EnableSyslogDB = 0;
}
@mkdir("/var/log/artica-postfix/squid-brut", 0777, true);
@mkdir("/var/log/artica-postfix/squid-reverse", 0777, true);
$GLOBALS["filename"] = $filename;
$sock = new sockets();
$TempDir = "/home/artica-extract-temp";
@mkdir($TempDir, 0777);
@chown($TempDir, "mysql");
@chdir($TempDir, "mysql");
$BackupMaxDaysDir = $sock->GET_INFO("BackupMaxDaysDir");
$bzip2 = $unix->find_program("bzip2");
$gunzip = $unix->find_program("gunzip");
progress("Extract {$filename} from MySQL database into {$TempDir}", 4);
if ($EnableSyslogDB == 1) {
$q = new mysql_storelogs();
$sql = "SELECT filecontent INTO DUMPFILE '{$TempDir}/{$filename}' FROM files_store WHERE ID = '{$storeid}'";
$q->QUERY_SQL($sql);
} else {
$q = new mysql_syslog();
$sql = "SELECT filedata INTO DUMPFILE '{$TempDir}/{$filename}' FROM store WHERE filename = '{$filename}'";
$q->QUERY_SQL($sql);
}
if (!$q->ok) {
progress("Failed!!! {$q->mysql_error}", 100);
return;
}
$file_extension = file_extension($filename);
progress("Extract {$filename} extension: {$file_extension}", 5);
$newtFile = $filename . ".log";
if ($file_extension == "bz2") {
$cmdline = "bzip2 -d \"{$TempDir}/{$filename}\" -c >\"{$TempDir}/{$newtFile}.log\" 2>&1";
exec($cmdline, $results);
}
if ($file_extension == "gz") {
$cmdline = "gunzip -d \"{$TempDir}/{$filename}\" -c >\"{$TempDir}/{$newtFile}.log\" 2>&1";
}
if ($cmdline != null) {
exec($cmdline, $results);
progress("Extract done " . @implode(" ", $results), 7);
} else {
if (!@copy("{$TempDir}/{$filename}", "{$TempDir}/{$newtFile}.log")) {
progress("Failed!!! Copy error", 100);
return;
}
}
@unlink("{$TempDir}/{$filename}");
if (!is_file("{$TempDir}/{$newtFile}.log")) {
progress("Failed!!! {$TempDir}/{$newtFile}.log error no such file", 100);
return;
}
$linesNumber = $unix->COUNT_LINES_OF_FILE("{$TempDir}/{$newtFile}.log");
progress("Open {$TempDir}/{$newtFile}.log {$linesNumber}", 10);
$handle = @fopen("{$TempDir}/{$newtFile}.log", "r");
if (!$handle) {
progress("Failed!!! {$TempDir}/{$newtFile}.log open failed", 100);
return;
}
$c = 0;
$d = 0;
$TTEV = 0;
while (!feof($handle)) {
$c++;
$buffer = trim(fgets($handle, 4096));
if (!preg_match("#MAC:.*?\\[([0-9]+)\\/(.*?)\\/([0-9]+).*?:([0-9]+):([0-9]+):([0-9]+)\\s+(.*?)\\]\\s+\"#", $buffer, $re)) {
continue;
}
$dteStr = "{$re[1]}/{$re[2]}/{$re[3]}:{$re[4]}:{$re[5]}:{$re[6]} {$re[7]}";
$ttime = strtotime($dteStr);
$newDate = date("Y-m-d H", $ttime) . "h";
$datelog = date("Y-m-d-h", $ttime);
$MD5Buffer = md5($buffer);
$TTEV++;
@mkdir("/var/log/artica-postfix/squid-brut/{$datelog}", 0777, true);
@file_put_contents("/var/log/artica-postfix/squid-brut/{$datelog}/{$MD5Buffer}", $buffer);
if ($c > 10) {
$d = $d + $c;
$pp = $d / $linesNumber;
$pp = $pp * 100;
$pp = round($pp, 1);
if ($pp > 10) {
if ($pp > 100) {
$pp = 99;
}
progress("Processing {$d}/{$linesNumber} - {$newDate} ", $pp);
$c = 0;
}
}
}
progress("Success, {$TTEV} events sent to MySQL injector ", 100);
@unlink("{$TempDir}/{$newtFile}.log");
}
function xstart()
{
$T1 = time();
$curl = new ccurl();
$unix = new unix();
$GLOBALS["MYPID"] = getmypid();
$pidfile = "/etc/artica-postfix/pids/windowupdate.processor.pid";
$pid = $unix->get_pid_from_file($pidfile);
if ($unix->process_exists($pid, basename(__FILE__))) {
die;
}
$pids = $unix->PIDOF_PATTERN_ALL(basename(__FILE__), true);
if (count($pids) > 0) {
while (list($i, $line) = each($pids)) {
events("Already executed PID:{$i}... aborting ", __LINE__);
}
die;
}
$TEMPDIR = $unix->TEMP_DIR() . "/WindowsUpdates";
$rm = $unix->find_program("rm");
@file_put_contents($pidfile, $GLOBALS["MYPID"]);
if (is_dir($TEMPDIR)) {
shell_exec("{$rm} -rf {$TEMPDIR}");
}
@mkdir($TEMPDIR, 0755, true);
$WindowsUpdateMaxPartition = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateMaxPartition"));
if ($WindowsUpdateMaxPartition == 0) {
$WindowsUpdateMaxPartition = 80;
}
$CheckPartitionPercentage = CheckPartitionPercentage();
if ($CheckPartitionPercentage > $WindowsUpdateMaxPartition) {
$time = $unix->file_time_min("/etc/squid3/WindowsUpdatePartitionExceed");
if ($time > 10) {
@unlink("/etc/squid3/WindowsUpdatePartitionExceed");
events("Failed: Storage Partition exceed {$WindowsUpdateMaxPartition}%, Stopping retreivals", __LINE__);
@touch("/etc/squid3/WindowsUpdatePartitionExceed");
DirectorySize();
}
return;
}
$WindowsUpdateInProduction = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateInProduction"));
if ($WindowsUpdateInProduction == 0) {
if ($unix->IsProductionTime()) {
$time = $unix->file_time_min("/etc/artica-postfix/pids/WindowsUpdateInProduction");
if ($time > 15) {
@unlink("/etc/artica-postfix/pids/WindowsUpdateInProduction");
@touch("/etc/artica-postfix/pids/WindowsUpdateInProduction");
events("INFO: Aborting, No download during production time", __LINE__);
DirectorySize();
}
return;
}
}
if (is_file("/etc/squid3/WindowsUpdatePartitionExceed")) {
@unlink("/etc/squid3/WindowsUpdatePartitionExceed");
}
$q = new mysql_squid_builder();
$sql = "CREATE TABLE IF NOT EXISTS `windowsupdate` (\n\t\t\t`filemd5` VARCHAR( 90 ) NOT NULL ,\n\t\t\t`zDate` DATETIME NOT NULL ,\n\t\t\t`zUri` VARCHAR( 255 ) NOT NULL ,\n\t\t\t`localpath` VARCHAR( 255 ) NOT NULL ,\n\t\t\t`filesize` BIGINT UNSIGNED DEFAULT '0',\n\t\t\t INDEX ( `filesize` ,`zDate`) ,\n\t\t\t KEY `localpath`(`localpath`),\n\t\t\t KEY `zUri`(`zUri`),\n\t\t\t PRIMARY KEY (`filemd5`)) ENGINE=MYISAM;";
$q->QUERY_SQL($sql);
if (!$q->ok) {
events("MySQL Failed {$q->mysql_error}", __LINE__);
die;
}
$GLOBALS["WindowsUpdateMaxToPartialQueue"] = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateMaxToPartialQueue"));
if ($GLOBALS["WindowsUpdateMaxToPartialQueue"] == 0) {
$GLOBALS["WindowsUpdateMaxToPartialQueue"] = 350;
}
$GLOBALS["WindowsUpdateCachingDir"] = @file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateCachingDir");
if ($GLOBALS["WindowsUpdateCachingDir"] == null) {
$GLOBALS["WindowsUpdateCachingDir"] = "/home/squid/WindowsUpdate";
}
$filepath = "{$GLOBALS["WindowsUpdateCachingDir"]}/Queue.log";
$WindowsUpdateDownTimeout = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateDownTimeout"));
if ($WindowsUpdateDownTimeout == 0) {
$WindowsUpdateDownTimeout = 600;
}
$WindowsUpdateMaxToPartialQueue = $GLOBALS["WindowsUpdateMaxToPartialQueue"] * 1000;
$WindowsUpdateMaxToPartialQueue = $WindowsUpdateMaxToPartialQueue * 1000;
$LinesCount = $unix->COUNT_LINES_OF_FILE($filepath);
if (!is_file($filepath)) {
return;
}
$md5start = md5_file($filepath);
$handle = @fopen($filepath, "r");
if (!$handle) {
events("Fopen failed on {$filepath}", __LINE__);
return false;
}
$NEWBUFFER = array();
$URLALREADY = array();
$FinalSize = 0;
$FF = 0;
$c = 0;
while (!feof($handle)) {
$buffer = trim(fgets($handle));
$c++;
if ($buffer == null) {
continue;
}
$TR = explode("|||", $buffer);
$prc = $c / $LinesCount;
$prc = round($prc * 100);
$LocalFile = $TR[0];
$URI = $TR[1];
$ExpectedSize = 0;
if (strpos($URI, $GLOBALS["WindowsUpdateCachingDir"]) > 0) {
events("FOUND! directory in URI", __LINE__);
$TTR = explode($GLOBALS["WindowsUpdateCachingDir"], $URI);
$URI = $TTR[0];
$LocalFile = "{$GLOBALS["WindowsUpdateCachingDir"]}{$TTR[1]}";
events("FOUND! URI:{$URI}", __LINE__);
events("FOUND! NEXT:{$LocalFile}", __LINE__);
}
$BASENAMELL = basename($LocalFile);
if (strlen($BASENAMELL) > 20) {
$BASENAMELL = substr($BASENAMELL, 0, 17) . "...";
}
build_progressG("{$BASENAMELL} {$c}/{$LinesCount} {files}", $prc);
if (isset($URLALREADY[$URI])) {
continue;
}
$URLALREADY[$URI] = true;
if (isBlacklisted($URI)) {
events(basename($URI) . " blacklisted...");
continue;
}
if (is_file($LocalFile)) {
$size = @filesize($LocalFile);
if ($size > 5) {
events("SKIP " . basename($LocalFile) . " " . xFormatBytes($size / 1024), __LINE__);
update_mysql($LocalFile, $URI);
continue;
} else {
@unlink($LocalFile);
}
}
$dirname = dirname($LocalFile);
if (!is_dir($dirname)) {
@mkdir($dirname, true, 0755);
}
$curl = new ccurl($URI);
$Headers = $curl->getHeaders();
$TIMEDOWN = time();
$TMPFILE = "{$TEMPDIR}/" . basename($LocalFile);
$GLOBALS["previousProgress"] = 0;
$GLOBALS["DOWNLOADED_FILE"] = basename($LocalFile);
$GLOBALS["TMPFILE"] = $TMPFILE;
$ExpectedSize = GetTargetedSize($URI);
if ($ExpectedSize == 0) {
events("Failed to download {$URI} ( unable to get expected size)", __LINE__);
continue;
}
if ($ExpectedSize > $WindowsUpdateMaxToPartialQueue) {
$ExpectedSizeText = xFormatBytes($ExpectedSize / 1024, true);
events(basename($URI) . " ({$ExpectedSizeText} {$ExpectedSize}/{$WindowsUpdateMaxToPartialQueue}) Limit {$WindowsUpdateMaxToPartialQueue} to BigFiles queue", __LINE__);
AddToPartialQueue($URI, $ExpectedSize, $LocalFile);
continue;
}
$curl = new ccurl($URI);
$curl->WriteProgress = true;
$curl->Timeout = $WindowsUpdateDownTimeout * 60;
$curl->ProgressFunction = "xdownload_progress";
events("Downloading " . basename($URI) . " to {$TMPFILE} (" . xFormatBytes($ExpectedSize / 1024, true) . " max:{$WindowsUpdateDownTimeout} Minutes)", __LINE__);
if (!$curl->GetFile($TMPFILE)) {
events("Failed: TMP: «{$TMPFILE}»", __LINE__);
events("Failed: URL: «{$URI}»", __LINE__);
events("Failed: After: " . $unix->distanceOfTimeInWords($TIMEDOWN, time(), true), __LINE__);
events("Failed: With error: {$curl->error} http code: {$curl->CURLINFO_HTTP_CODE} (" . count($curl->CURL_ALL_INFOS) . ") infos", __LINE__);
reset($curl->CURL_ALL_INFOS);
while (list($index, $value) = each($curl->CURL_ALL_INFOS)) {
events("Failed: «{$index}» [{$value}]", __LINE__);
}
if ($curl->CURLINFO_HTTP_CODE == 404) {
continue;
}
@unlink($TMPFILE);
$NEWBUFFER[] = "{$buffer}";
continue;
}
if (!is_file($TMPFILE)) {
events("Fatal {$TMPFILE}: no such file", __LINE__);
continue;
}
$size = filesize($TMPFILE);
$sizeT = xFormatBytes($size / 1024);
if ($size < 5) {
@unlink($TMPFILE);
events("Failed: File less than 5 Bytes ({$size}), aborting", __LINE__);
continue;
}
if ($ExpectedSize > 0) {
if ($size != $ExpectedSize) {
$ExpectedSizeT = xFormatBytes($ExpectedSize / 1024);
events("Failed: corrupted download " . basename($URI) . " expected size {$ExpectedSizeT}/{$ExpectedSize} current:({$sizeT}/{$size})", __LINE__);
@unlink($TMPFILE);
continue;
}
}
if (!@copy($TMPFILE, $LocalFile)) {
@unlink($TMPFILE);
events("Failed: Translating to {$LocalFile}", __LINE__);
$NEWBUFFER[] = "{$buffer}";
continue;
}
events("Success: " . basename($TMPFILE) . " ({$sizeT})", __LINE__);
@unlink($TMPFILE);
$FF++;
$size = @filesize($LocalFile);
$FinalSize = $FinalSize + $size;
update_mysql($LocalFile, $URI);
}
$took = $unix->distanceOfTimeInWords($T1, time(), true);
if ($FinalSize > 0) {
$CURLINFO_SPEED_DOWNLOAD = $curl->CURL_ALL_INFOS["CURLINFO_SPEED_DOWNLOAD"];
events("Downloaded {$FF} files for " . xFormatBytes($FinalSize / 1024, true) . " ({$CURLINFO_SPEED_DOWNLOAD}) took: {$took}", __LINE__);
}
$md5finish = md5_file($filepath);
if (count($NEWBUFFER) > 0) {
events("Retry " . count($NEWBUFFER) . " requests next time...", __LINE__);
@file_put_contents($filepath, @implode("\n", $NEWBUFFER));
} else {
events("No new file downloaded....", __LINE__);
events("Removing queue {$filepath}", __LINE__);
@unlink($filepath);
}
DirectorySize();
events(" * * * END TOOK: {$took} * * *", __LINE__);
}
function ArticaUfdb()
{
$DB = array();
$unix = new unix();
$FILEDBS = array();
$prc = 0;
$GLOBALS["MAIN_ARRAY"]["TLSE_PRC"] = 0;
if ($GLOBALS["CLASS_SOCKETS"]->EnableUfdbGuard() == 0) {
if ($GLOBALS["VERBOSE"]) {
echo "EnableUfdbGuard report false\n";
}
$GLOBALS["MAIN_ARRAY"]["TLSE_ENABLED"] = 0;
}
if (is_file("/etc/artica-postfix/univtoulouse-global_usage")) {
$contentF = explode("\n", @file_get_contents("/etc/artica-postfix/univtoulouse-global_usage"));
while (list($index, $line) = each($contentF)) {
if (preg_match("#NAME:\\s+(.+)#", $line, $re)) {
$DB[trim($re[1])] = trim($re[1]);
}
}
}
$Dirs = $unix->dirdir("/var/lib/ftpunivtlse1fr");
while (list($Dir, $line) = each($Dirs)) {
if (!is_file("{$Dir}/domains.ufdb")) {
continue;
}
$DB[basename($Dir)] = true;
}
if (count($DB) > 0) {
$q = new mysql_squid_builder();
$TLSE_CONVERTION = $q->TLSE_CONVERTION();
while (list($TLSE, $line) = each($DB)) {
$catzname = $TLSE_CONVERTION[$TLSE];
if ($catzname == null) {
if ($GLOBALS["VERBOSE"]) {
echo "Unable to understand {$TLSE}\n";
}
continue;
}
$catzname = str_replace("/", "_", $catzname);
$FILEDBS[$catzname] = "/var/lib/ftpunivtlse1fr/{$catzname}/domains.ufdb";
$FILEDBSC[] = "/var/lib/ftpunivtlse1fr/{$catzname}/domains";
}
}
if (count($FILEDBS) > 0) {
$c = 0;
$MAX = count($FILEDBS);
while (list($table, $path) = each($FILEDBS)) {
if (!is_file($path)) {
echo "{$path} such file\n";
continue;
}
$size = @filesize($path);
if ($size < 10) {
continue;
}
$c++;
$prc = intval($c) / intval($MAX);
$prc = round($prc * 100);
$GLOBALS["MAIN_ARRAY"]["TLSE_COUNT"] = $c;
$TLSE_COUNTZ[$table]["SIZE"] = $size;
if ($GLOBALS["VERBOSE"]) {
echo "COUNT OF {$path}\n";
}
$path_db1 = dirname($path) . "/" . "domains";
$TLSE_COUNTZ[$table]["ITEMS"] = $unix->COUNT_LINES_OF_FILE($path_db1);
$TLSE_COUNTZ[$table]["TIME"] = filemtime($path);
}
}
if ($GLOBALS["CLASS_UNIX"]->file_time_min("/etc/artica-postfix/UNIVTLSE_STAT_DB_SIZE") > 60) {
$UNIVTLSE_STAT_DB_SIZE = $GLOBALS["CLASS_UNIX"]->DIRSIZE_KO("/var/lib/ftpunivtlse1fr");
@unlink("/etc/artica-postfix/UNIVTLSE_STAT_DB_SIZE");
@file_put_contents("/etc/artica-postfix/UNIVTLSE_STAT_DB_SIZE", $UNIVTLSE_STAT_DB_SIZE);
}
$C = 0;
if ($GLOBALS["CLASS_UNIX"]->file_time_min("/etc/artica-postfix/UNIVTLSE_STAT_DB_ITEMS") > 60) {
while (list($table, $path) = each($FILEDBSC)) {
$unix = new unix();
$C = $C + intval($unix->COUNT_LINES_OF_FILE($path));
}
@unlink("/etc/artica-postfix/UNIVTLSE_STAT_DB_ITEMS");
@file_put_contents("/etc/artica-postfix/UNIVTLSE_STAT_DB_ITEMS", $C);
}
$STATUS = unserialize(@file_get_contents("/etc/artica-postfix/TLSE_LAST_DOWNLOAD"));
if (!isset($STATUS["LAST_DOWNLOAD"])) {
$STATUS["LAST_DOWNLOAD"] = array();
}
$GLOBALS["MAIN_ARRAY"]["TLSE_PRC"] = $prc;
$GLOBALS["MAIN_ARRAY"]["TLSE_STAT_SIZE"] = trim(@file_get_contents("/etc/artica-postfix/UNIVTLSE_STAT_DB_SIZE"));
$GLOBALS["MAIN_ARRAY"]["TLSE_STAT_ITEMS"] = trim(@file_get_contents("/etc/artica-postfix/UNIVTLSE_STAT_DB_ITEMS"));
$GLOBALS["MAIN_ARRAY"]["TLSE_LAST_SINCE"] = $GLOBALS["CLASS_UNIX"]->distanceOfTimeInWords($STATUS["LAST_DOWNLOAD"]["TIME"], time());
$GLOBALS["MAIN_ARRAY"]["TLSE_LAST_CAT"] = $STATUS["LAST_DOWNLOAD"]["CATEGORY"];
$GLOBALS["MAIN_ARRAY"]["TLSE_LAST_SIZE"] = $STATUS["LAST_DOWNLOAD"]["SIZE"];
$GLOBALS["MAIN_ARRAY"]["TLSE_LAST_CHECK"] = $STATUS["LAST_CHECK"];
$GLOBALS["MAIN_ARRAY"]["TLSE_ARRAY"] = $TLSE_COUNTZ;
if ($GLOBALS["VERBOSE"]) {
print_r($GLOBALS["MAIN_ARRAY"]);
}
}
function xstart()
{
$T1 = time();
$curl = new ccurl();
$unix = new unix();
$GLOBALS["MYPID"] = getmypid();
$pidfile = "/etc/artica-postfix/pids/windowupdate.partial.processor.pid";
$pid = $unix->get_pid_from_file($pidfile);
if ($unix->process_exists($pid, basename(__FILE__))) {
die;
}
$pids = $unix->PIDOF_PATTERN_ALL(basename(__FILE__), true);
if (count($pids) > 0) {
while (list($i, $line) = each($pids)) {
events("Already executed PID:{$i}... aborting ", __LINE__);
}
die;
}
$TEMPDIR = $unix->TEMP_DIR() . "/WindowsUpdates";
$rm = $unix->find_program("rm");
@file_put_contents($pidfile, $GLOBALS["MYPID"]);
if (is_dir($TEMPDIR)) {
shell_exec("{$rm} -rf {$TEMPDIR}");
@mkdir($TEMPDIR);
}
$GLOBALS["WindowsUpdateMaxToPartialQueue"] = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateMaxToPartialQueue"));
$GLOBALS["WindowsUpdateCachingDir"] = @file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateCachingDir");
if ($GLOBALS["WindowsUpdateCachingDir"] == null) {
$GLOBALS["WindowsUpdateCachingDir"] = "/home/squid/WindowsUpdate";
}
$WindowsUpdateDownTimeout = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateDownTimeout"));
if ($WindowsUpdateDownTimeout == 0) {
$WindowsUpdateDownTimeout = 600;
}
$WindowsUpdateBandwidthMaxFailed = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateBandwidthMaxFailed"));
if ($WindowsUpdateBandwidthMaxFailed == 0) {
$WindowsUpdateBandwidthMaxFailed = 50;
}
$WindowsUpdateMaxPartition = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateMaxPartition"));
if ($WindowsUpdateMaxPartition == 0) {
$WindowsUpdateMaxPartition = 80;
}
$fileSource = "{$GLOBALS["WindowsUpdateCachingDir"]}/Partials/Queue.log";
$LinesCount = $unix->COUNT_LINES_OF_FILE($fileSource);
if (!is_file($fileSource)) {
return;
}
$md5start = md5_file($fileSource);
$CheckPartitionPercentage = CheckPartitionPercentage();
if ($CheckPartitionPercentage > $WindowsUpdateMaxPartition) {
events("Failed: Storage Partition exceed {$WindowsUpdateMaxPartition}% Stopping retreivals", __LINE__);
@touch("/etc/squid3/WindowsUpdatePartitionExceed");
DirectorySize();
return;
}
$WindowsUpdateInProduction = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateInProduction"));
if ($WindowsUpdateInProduction == 0) {
if ($unix->IsProductionTime()) {
$time = $unix->file_time_min("/etc/artica-postfix/pids/WindowsUpdateInProduction");
if ($time > 15) {
@unlink("/etc/artica-postfix/pids/WindowsUpdateInProduction");
@touch("/etc/artica-postfix/pids/WindowsUpdateInProduction");
events("INFO: Aborting, No download during production time", __LINE__);
DirectorySize();
}
return;
}
}
if (is_file("/etc/squid3/WindowsUpdatePartitionExceed")) {
@unlink("/etc/squid3/WindowsUpdatePartitionExceed");
}
$handle = @fopen($fileSource, "r");
if (!$handle) {
events("Fopen failed on {$fileSource}", __LINE__);
return false;
}
$NEWBUFFER = array();
$URLALREADY = array();
$NewBuffer = array();
$FinalSize = 0;
$FF = 0;
$c = 0;
while (!feof($handle)) {
$buffer = trim(fgets($handle));
$c++;
if ($buffer == null) {
continue;
}
$TR = explode("|||", $buffer);
$LocalFile = $TR[0];
$URI = $TR[1];
$ExpectedSize = $TR[2];
if (!isset($TR[3])) {
$TR[3] = 1;
}
if (isset($URLALREADY[$URI])) {
continue;
}
$URLALREADY[$URI] = true;
$BaseNameOfFile = basename($URI);
events("INFO: {$BaseNameOfFile} " . xFormatBytes($ExpectedSize / 1024) . " Retry:{$TR[3]} [{$c}/{$LinesCount}]", __LINE__);
if (DownloadByRange($URI, $ExpectedSize, $LocalFile)) {
update_mysql($LocalFile, $URI);
continue;
}
$TR[3] = $TR[3] + 1;
if ($TR[3] > $WindowsUpdateBandwidthMaxFailed) {
events("Error: Max retry ({$WindowsUpdateBandwidthMaxFailed}) for filename: {$BaseNameOfFile}", __LINE__);
SaveToBlacklists($BaseNameOfFile);
RemoveTempOf($URI, $LocalFile);
continue;
}
$NewBuffer[] = @implode("|||", $TR);
}
if (count($NewBuffer) == 0) {
@unlink($fileSource);
return;
}
events("INFO: add " . count($NewBuffer) . " orders to queue", __LINE__);
@file_put_contents($fileSource, @implode("\n", $NewBuffer));
DirectorySize();
}
function xrun()
{
build_progress("{scanning} {files}", 20);
$BaseWorkDir = "/home/postfix/logrotate";
if (!($handle = opendir($BaseWorkDir))) {
echo "Failed open {$BaseWorkDir}\n";
return;
}
$MAIN = array();
while (false !== ($filename = readdir($handle))) {
if ($filename == ".") {
continue;
}
if ($filename == "..") {
continue;
}
$targetfile = "{$BaseWorkDir}/{$filename}";
if (strpos($filename, ".gz") == 0) {
continue;
}
$fileKey = str_replace(".gz", "", $filename);
$MAIN[$fileKey] = $targetfile;
}
if (count($MAIN) == 0) {
echo "No files to scan....\n";
build_progress("{scanning} {files} {failed}", 110);
return;
}
ksort($MAIN);
$sock = new sockets();
$PostfixHistorySearch = $sock->GET_INFO("PostfixHistorySearch");
if ($PostfixHistorySearch == null) {
echo "No search pattern, aborting...\n";
build_progress("{scanning} {files} {failed}", 110);
return;
}
if (!preg_match("#regex\\s+(.+)#", $PostfixHistorySearch, $re)) {
$PostfixHistorySearch = str_replace(".", "\\.", $PostfixHistorySearch);
$PostfixHistorySearch = str_replace("*", ".*?", $PostfixHistorySearch);
$PostfixHistorySearch = str_replace("[", "\\[", $PostfixHistorySearch);
$PostfixHistorySearch = str_replace("]", "\\]", $PostfixHistorySearch);
$PostfixHistorySearch = str_replace("(", "\\(", $PostfixHistorySearch);
$PostfixHistorySearch = str_replace(")", "\\)", $PostfixHistorySearch);
$PostfixHistorySearch = str_replace("/", "\\/", $PostfixHistorySearch);
} else {
$PostfixHistorySearch = $re[1];
}
$unix = new unix();
$zcat = $unix->find_program("zcat");
$grep = $unix->find_program("grep");
if (is_file("/usr/share/artica-postfix/ressources/logs/web/mail-history.log")) {
@unlink("/usr/share/artica-postfix/ressources/logs/web/mail-history.log");
}
$perc = 20;
while (list($zDate, $filepath) = each($MAIN)) {
$perc = $perc + 1;
if ($perc > 95) {
$perc = 95;
}
$size = @filesize("/usr/share/artica-postfix/ressources/logs/web/mail-history.log");
$size = $size / 1024;
$size = round($size, 2);
echo "Scanning {$filepath}\n";
build_progress("{scanning} {$zDate} for {$PostfixHistorySearch} ( {$size}KB )", $perc);
$cmd = "{$zcat} {$filepath} | {$grep} -E '{$PostfixHistorySearch}' >>/usr/share/artica-postfix/ressources/logs/web/mail-history.log 2>&1";
echo "{$cmd}\n";
shell_exec($cmd);
}
$countlines = $unix->COUNT_LINES_OF_FILE("/usr/share/artica-postfix/ressources/logs/web/mail-history.log");
$size = @filesize("/usr/share/artica-postfix/ressources/logs/web/mail-history.log");
$size = $size / 1024;
$size = round($size, 2);
build_progress("{scanning} {done} {$countlines} {lines} {$size}KB", 100);
@chmod("/usr/share/artica-postfix/ressources/logs/web/mail-history.log", 0755);
}
function update_milter_greylist()
{
$unix = new unix();
$mirror = "http://mirror.articatech.net/webfilters-databases";
if ($GLOBALS["VERBOSE"]) {
echo "Downloading {$mirror}/milter-greylist-database.txt\n";
}
$curl = new ccurl("{$mirror}/milter-greylist-database.txt");
$curl->NoHTTP_POST = true;
$temppath = $unix->TEMP_DIR();
if (!$curl->GetFile("{$temppath}/milter-greylist-database.txt")) {
postfix_admin_mysql(0, "Unable to get Milter-greylist index file", $curl->error);
return;
}
if (!is_file("{$temppath}/milter-greylist-database.txt")) {
postfix_admin_mysql(0, "Unable to get Milter-greylist index file (no such file)", $curl->error);
return;
}
$data = @file_get_contents("{$temppath}/milter-greylist-database.txt");
$MAIN = unserialize($data);
if ($GLOBALS["VERBOSE"]) {
echo $data . "\n";
}
if ($GLOBALS["VERBOSE"]) {
print_r($MAIN);
}
@unlink("{$temppath}/milter-greylist-database.txt");
$TIME = $MAIN["PATTERN"]["TIME"];
$MD5 = $MAIN["PATTERN"]["MD5"];
$sock = new sockets();
$MyTime = $sock->GET_INFO("MilterGreyListPatternTime");
if (!is_file("/etc/mail/milter-greylist-database.conf")) {
$MyTime = 0;
}
if ($TIME == $MyTime) {
if ($GLOBALS["VERBOSE"]) {
echo "{$TIME}=={$MyTime} No new update\n";
}
return;
}
$curl = new ccurl("{$mirror}/milter-greylist-database.gz");
$curl->NoHTTP_POST = true;
if (!$curl->GetFile("{$temppath}/milter-greylist-database.gz")) {
postfix_admin_mysql(0, "Unable to get milter-greylist-database.gz", $curl->error, __FILE__, __LINE__);
return;
}
$md5f = md5_file("{$temppath}/milter-greylist-database.gz");
if ($md5f != $MD5) {
@unlink("{$temppath}/milter-greylist-database.gz");
postfix_admin_mysql(0, "Unable to get milter-greylist-database.gz (corrupted)", $curl->error, __FILE__, __LINE__);
return;
}
if (!$unix->uncompress("{$temppath}/milter-greylist-database.gz", "{$temppath}/milter-greylist-database.conf")) {
@unlink("{$temppath}/milter-greylist-database.gz");
postfix_admin_mysql(0, "Unable to extract milter-greylist-database.gz (corrupted)", null, __FILE__, __LINE__);
return;
}
@unlink("{$temppath}/milter-greylist-database.gz");
@unlink("/etc/mail/milter-greylist-database.conf");
@copy("{$temppath}/milter-greylist-database.conf", "/etc/mail/milter-greylist-database.conf");
@unlink("{$temppath}/milter-greylist-database.conf");
postfix_admin_mysql(0, "Success updating new Milter-greylist database version {$TIME}", null, __FILE__, __LINE__);
$sock->SET_INFO("MilterGreyListPatternTime", $TIME);
$sock->SET_INFO("MilterGreyListPatternCount", $unix->COUNT_LINES_OF_FILE("/etc/mail/milter-greylist-database.conf"));
$main = new maincf_multi("master", "master");
$check_client_access = $main->check_client_access();
$postfix = $unix->find_program("postfix");
$php5 = $unix->LOCATE_PHP5_BIN();
$nohup = $unix->find_program("nohup");
shell_exec("{$php5} /usr/share/artica-postfix/exec.postfix.maincf.php --body-checks >/dev/null 2>&1 &");
shell_exec("{$postfix} stop");
shell_exec("{$postfix} start");
postfix_admin_mysql(1, "Restarting Milter-greylist service", null, __FILE__, __LINE__);
shell_exec("/etc/init.d/milter-greylist restart");
}
function xrun()
{
$unix = new unix();
$siege = $unix->find_program("siege");
$sock = new sockets();
$ARRAY = unserialize(base64_decode($sock->GET_INFO("SquidSiegeConfig")));
if (!is_numeric($ARRAY["GRAB_URLS"])) {
$ARRAY["GRAB_URLS"] = 0;
}
if (!is_numeric($ARRAY["USE_LOCAL_PROXY"])) {
$ARRAY["USE_LOCAL_PROXY"] = 1;
}
if (!is_numeric($ARRAY["SESSIONS"])) {
$ARRAY["SESSIONS"] = 150;
}
if (!is_numeric($ARRAY["MAX_TIME"])) {
$ARRAY["MAX_TIME"] = 30;
}
build_progress_disconnect("{starting}", 5);
if (!is_file($siege)) {
build_progress_disconnect("{please_wait} {installing} SIEGE", 50);
$unix->DEBIAN_INSTALL_PACKAGE("siege");
$siege = $unix->find_program("siege");
if (!is_file($siege)) {
build_progress_disconnect("{installing} SIEGE {failed}", 110);
}
}
$f[] = "internet = true";
if ($ARRAY["USE_LOCAL_PROXY"] == 1) {
$squid = new squidbee();
if ($squid->hasProxyTransparent == 1) {
$port = $squid->second_listen_port;
} else {
$port = $squid->listen_port;
}
$addr = "127.0.0.1";
} else {
$addr = $ARRAY["REMOTE_PROXY"];
$port = intval($ARRAY["REMOTE_PROXY_PORT"]);
}
if ($addr == null) {
build_progress_disconnect("{failed} No proxy address", 110);
return;
}
if ($port == 0) {
build_progress_disconnect("{failed} No proxy port", 110);
return;
}
if ($ARRAY["SESSIONS"] == 0) {
build_progress_disconnect("{failed} {simulate} 0 sessions", 110);
return;
}
$f[] = "proxy-host ={$addr}";
$f[] = "proxy-port = {$port}";
$f[] = "user-agent = Mozilla/5.0 (compatible; IE 11.0; Win32; Trident/7.0)";
$f[] = "file = /etc/siege/urls.txt";
$f[] = "concurrent = {$ARRAY["SESSIONS"]}";
$f[] = "time = {$ARRAY["MAX_TIME"]}S";
$f[] = "timeout = 5";
$f[] = "logfile = /var/log/siege.log";
if (trim($ARRAY["USERNAME"]) != null) {
$f[] = "username = {$ARRAY["USERNAME"]}";
$f[] = "password = {$ARRAY["PASSWORD"]}";
}
@file_put_contents("/root/.siegerc", @implode("\n", $f));
$filetemp = $unix->FILE_TEMP();
$nohup = $unix->find_program("nohup");
$URLS_NUMBER = $unix->COUNT_LINES_OF_FILE("/etc/siege/urls.txt");
if ($URLS_NUMBER < 20) {
@unlink("/etc/siege/urls.txt");
if ($ARRAY["GRAB_URLS"] == 1) {
import_urls();
} else {
@copy("/usr/share/artica-postfix/bin/install/squid/urls.txt", "/etc/siege/urls.txt");
}
$URLS_NUMBER = $unix->COUNT_LINES_OF_FILE("/etc/siege/urls.txt");
}
$FINAL["urls"] = $URLS_NUMBER;
$FINAL["START_TIME"] = time();
$ss[] = "{$nohup} {$siege} --concurrent={$ARRAY["SESSIONS"]}";
$ss[] = "--internet --file=/etc/siege/urls.txt --time={$ARRAY["MAX_TIME"]}S";
$ss[] = "--benchmark --rc=/root/.siegerc >{$filetemp} 2>&1 &";
$cmd = @implode(" ", $ss);
echo "{$cmd}\n";
build_progress_disconnect("{executing}", 50);
system($cmd);
sleep(2);
$pid = $unix->PIDOF($siege);
while ($unix->process_exists($pid)) {
$array_mem = getSystemMemInfo();
$MemFree = $array_mem["MemFree"];
$array_load = sys_getloadavg();
$internal_load = $array_load[0];
echo "Memory Free: " . round($MemFree / 1024) . " MB\n";
echo "Load: {$internal_load}\n";
build_progress_disconnect("{please_wait} Load:{$internal_load}", 50);
sleep(2);
$pid = $unix->PIDOF($siege);
}
build_progress_disconnect("{please_wait} {analyze}...", 90);
$array = explode("\n", @file_get_contents($filetemp));
@unlink($filetemp);
while (list($num, $val) = each($array)) {
echo "{$val}\n";
if (preg_match("#alert#", $val)) {
continue;
}
if (preg_match("#ERROR#", $val)) {
continue;
}
if (preg_match("#(.+?):\\s+(.+)#", $val, $re)) {
$FINAL[trim($re[1])] = trim($re[2]);
}
}
$FINAL["STOP_TIME"] = time();
build_progress_disconnect("{done}...", 99);
sleep(5);
build_progress_disconnect("{done}...", 100);
@file_put_contents("/usr/share/artica-postfix/ressources/logs/web/siege.report.txt", serialize($FINAL));
@chmod("/usr/share/artica-postfix/ressources/logs/web/siege.report.txt", 0755);
}
function _analyze_file($filepath, $zmd5)
{
if (!is_file($filepath)) {
events("{$filepath} no such file");
return false;
}
$sock = new sockets();
$unix = new unix();
$EnableImportWithSarg = $sock->GET_INFO("EnableImportWithSarg");
if (!is_numeric($EnableImportWithSarg)) {
$EnableImportWithSarg = 1;
}
$SargOutputDir = $sock->GET_INFO("SargOutputDir");
if ($SargOutputDir == null) {
$SargOutputDir = "/var/www/html/squid-reports";
}
$basename = basename($filepath);
$timeStart = time();
$unix = new unix();
$sock = new sockets();
$q = new mysql_squid_builder();
$TimeOfFile = strtotime(GetDateOfFile($filepath));
$ContainerDir = "/var/log/artica-postfix/squid/queues/" . date("Y-m-d-h", $TimeOfFile);
@mkdir($ContainerDir, 0755, true);
$handle = @fopen($filepath, "r");
if (!$handle) {
events("Failed to open file {$filepath}");
echo "Failed to open file\n";
return;
}
$c = 0;
$max = $unix->COUNT_LINES_OF_FILE($filepath);
$GLOBALS["BUFFER_FILE_ANALYZED"] = $filepath;
events("{$filepath} {$max} lines");
$ligne = mysql_fetch_array($q->QUERY_SQL("SELECT * FROM accesslogs_import WHERE zmd5='{$zmd5}'"));
$FileStatus = $ligne["status"];
if (!$GLOBALS["SIMULATE"]) {
if ($ligne["filename"] == null) {
echo "{$filepath}: {$zmd5} did not match expected md5\n";
@fclose($handle);
return;
}
}
if ($GLOBALS["VERBOSE"]) {
echo "Container: {$ContainerDir}\n";
echo "Status...: {$FileStatus}\n";
echo "Lines....: {$max}\n";
}
if (!$GLOBALS["FORCE"]) {
if ($FileStatus == 3) {
events("{$filepath} already analyzed, skip it...");
@fclose($handle);
return true;
}
}
if ($EnableImportWithSarg == 1) {
$u = null;
$nice = EXEC_NICE();
$sarg = $unix->find_program("sarg");
$php = $unix->LOCATE_PHP5_BIN();
$squid = new squidbee();
if ($squid->LDAP_AUTH == 1) {
$usersauth = true;
}
if ($squid->LDAP_EXTERNAL_AUTH == 1) {
$usersauth = true;
}
if ($usersauth) {
echo "Starting......: " . date("H:i:s") . " Sarg, user authentification enabled\n";
$u = " -i ";
}
if (is_file($sarg)) {
shell_exec("{$php} /usr/share/artica-postfix/exec.sarg.php --conf >/dev/null 2>&1");
exec("{$nice}{$sarg} {$u}-f /etc/squid3/sarg.conf -l {$filepath} -o \"{$SargOutputDir}\" 2>&1", $sargR);
while (list($index, $line) = each($sargR)) {
events("Sarg: {$line}\n");
}
}
}
$percent_ret = 0;
while (!feof($handle)) {
$c++;
$buffer = trim(fgets($handle));
if ($buffer == null) {
continue;
}
$array = parseline($buffer);
if (count($array) == 0) {
continue;
}
$ip = null;
$user = null;
$xtime = $array["TIME"];
$ip = $array["IPADDR"];
$user = $array["UID"];
$code_error = $array["ERRCODE"];
$size = $array["SIZE"];
$uri = $array["URI"];
$cached = $array["CACHED"];
// $q->QUERY_SQL("INSERT INTO accesslogs_import (zmd5,filename,zDate,size,status,percent) VALUES ('$md5','$filename','$date','$size',0,0)");
$HOSTNAME = $array["HOSTNAME"];
if (is_numeric($user)) {
echo "\n\n\n****************\n\nNumeric user:{$user}\n{$buffer}\n\n";
die;
}
if ($ip == null) {
if ($HOSTNAME != null) {
$ip = $HOSTNAME;
}
}
$GLOBALS["BUFFER_ANALYZED"] = $buffer;
$GLOBALS["squidtail"]->Builsql($ip, $user, $uri, $code_error, $size, $xtime, $cached, null, $xtime);
if ($GLOBALS["SIMULATE"]) {
continue;
}
$percent = $c / $max * 100;
$percent = round($percent);
if ($percent != $percent_ret) {
if ($GLOBALS["VERBOSE"]) {
echo "****************** {$percent}% ********************\n";
}
$percent_ret = $percent;
events("{$percent_ret}% " . count($GLOBALS["squidtail"]->GLOBAL_QUEUE) . " in memory - {$filepath}");
$q->QUERY_SQL("UPDATE accesslogs_import SET percent='{$percent}',status=1 WHERE zmd5='{$zmd5}'");
}
if (count($GLOBALS["squidtail"]->GLOBAL_QUEUE) > 2000) {
events("analyze_file()::{$basename}::{$percent}% GLOBAL_RTTSIZE......: " . count($GLOBALS["squidtail"]->GLOBAL_RTTSIZE) . " items...", __LINE__);
events("analyze_file()::{$basename}::{$percent}% GLOBAL_PAGEKEEPER...: " . count($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER) . " items...", __LINE__);
events("analyze_file()::{$basename}::{$percent}% GLOBAL_YOUTUBE......: " . count($GLOBALS["squidtail"]->GLOBAL_YOUTUBE) . " items...", __LINE__);
events("analyze_file()::{$basename}::{$percent}% GLOBAL_SQUIDUSERS...: " . count($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS) . " items...", __LINE__);
events("analyze_file()::{$basename}::{$percent}% GLOBAL_SEARCHWORDS..: " . count($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS) . " items...", __LINE__);
PURGE_GLOBAL_QUEUE($GLOBALS["squidtail"]->GLOBAL_QUEUE);
$GLOBALS["squidtail"]->GLOBAL_QUEUE = array();
if (count($GLOBALS["squidtail"]->GLOBAL_RTTSIZE) > 500) {
@mkdir("{$ContainerDir}/RTTSize", 0755, true);
@file_put_contents("{$ContainerDir}/RTTSize/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_RTTSIZE)), serialize($GLOBALS["squidtail"]->GLOBAL_RTTSIZE));
$GLOBALS["squidtail"]->GLOBAL_RTTSIZE = array();
$RTTSIZE = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER) > 500) {
@mkdir("{$ContainerDir}/PageKeeper", 0755, true);
@file_put_contents("{$ContainerDir}/PageKeeper/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER)), serialize($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER));
$GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER = array();
$PAGEKEEP = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_YOUTUBE) > 500) {
@mkdir("{$ContainerDir}/Youtube", 0755, true);
$md5 = md5(serialize($GLOBALS["squidtail"]->GLOBAL_YOUTUBE));
youtube_events("Saving queue:(2000) {$ContainerDir}/Youtube/" . $md5, __LINE__);
@file_put_contents("{$ContainerDir}/Youtube/" . $md5, serialize($GLOBALS["squidtail"]->GLOBAL_YOUTUBE));
$GLOBALS["squidtail"]->GLOBAL_YOUTUBE = array();
$YOUTUBE = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS) > 500) {
@mkdir("{$ContainerDir}/Members", 0755, true);
@file_put_contents("{$ContainerDir}/Members/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS)), serialize($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS));
$GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS = array();
}
if (count($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS) > 500) {
@mkdir("{$ContainerDir}/SearchWords", 0755, true);
@file_put_contents("{$ContainerDir}/SearchWords/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS)), serialize($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS));
$GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS = array();
}
}
// PURGE OVER 2000
}
// END GLOBAL LOOP
@fclose($handle);
PURGE_GLOBAL_QUEUE($GLOBALS["squidtail"]->GLOBAL_QUEUE);
events("analyze_file()::{$basename}:: Container.........: `{$ContainerDir}` ", __LINE__);
events("analyze_file()::{$basename}:: GLOBAL_RTTSIZE....: " . count($GLOBALS["squidtail"]->GLOBAL_RTTSIZE) . " items...", __LINE__);
events("analyze_file()::{$basename}:: GLOBAL_PAGEKEEPER.: " . count($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER) . " items...", __LINE__);
events("analyze_file()::{$basename}:: GLOBAL_YOUTUBE....: " . count($GLOBALS["squidtail"]->GLOBAL_YOUTUBE) . " items...", __LINE__);
events("analyze_file()::{$basename}:: GLOBAL_SQUIDUSERS.: " . count($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS) . " items...", __LINE__);
events("analyze_file()::{$basename}:: GLOBAL_SEARCHWORDS: " . count($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS) . " items...", __LINE__);
$q->QUERY_SQL("UPDATE accesslogs_import SET percent='100',status='3' WHERE zmd5='{$zmd5}'");
if (count($GLOBALS["squidtail"]->GLOBAL_RTTSIZE) > 0) {
@mkdir("{$ContainerDir}/RTTSize", 0755, true);
@file_put_contents("{$ContainerDir}/RTTSize/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_RTTSIZE)), serialize($GLOBALS["squidtail"]->GLOBAL_RTTSIZE));
$RTTSIZE = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER) > 0) {
@mkdir("{$ContainerDir}/PageKeeper", 0755, true);
@file_put_contents("{$ContainerDir}/PageKeeper/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER)), serialize($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER));
$PAGEKEEP = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_YOUTUBE) > 0) {
@mkdir("{$ContainerDir}/Youtube", 0755, true);
$md5 = md5(serialize($GLOBALS["squidtail"]->GLOBAL_YOUTUBE));
youtube_events("Saving queue: {$ContainerDir}/Youtube/" . $md5, __LINE__);
@file_put_contents("{$ContainerDir}/Youtube/" . $md5, serialize($GLOBALS["squidtail"]->GLOBAL_YOUTUBE));
$YOUTUBE = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS) > 0) {
@mkdir("{$ContainerDir}/Members", 0755, true);
@file_put_contents("{$ContainerDir}/Members/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS)), serialize($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS));
}
if (count($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS) > 0) {
@mkdir("{$ContainerDir}/SearchWords", 0755, true);
@file_put_contents("{$ContainerDir}/SearchWords/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS)), serialize($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS));
}
$size = round($size / 1024, 2);
events("analyze_file()::{$basename}:: {$max}: lines parsed in " . $unix->distanceOfTimeInWords($timeStart, time()) . __LINE__);
if (system_is_overloaded(basename(__FILE__))) {
return;
}
$php5 = $unix->LOCATE_PHP5_BIN();
$nice = EXEC_NICE();
$nohup = $unix->find_program("nohup");
$cmd = "{$nohup} {$php5} " . __FILE__ . " --squid >/dev/null 2>&1 &";
events("analyze_file()::{$cmd}");
shell_exec($cmd);
if ($PAGEKEEP) {
$cmd = "{$nohup} {$php5} /usr/share/artica-postfix/exec.squid.stats.php --thumbs-parse >/dev/null 2>&1 &";
events(__FUNCTION__ . ":: {$cmd}", __LINE__);
shell_exec($cmd);
}
if ($YOUTUBE) {
$cmd = "{$nohup} {$php5} " . __FILE__ . " --youtube >/dev/null 2>&1 &";
events(__FUNCTION__ . ":: {$cmd}", __LINE__);
shell_exec($cmd);
}
if ($RTTSIZE) {
$cmd = "{$nohup} {$php5} /usr/share/artica-postfix/exec.squid-users-rttsize.php --now >/dev/null 2>&1 &";
events(__FUNCTION__ . ":: {$cmd}", __LINE__);
shell_exec($cmd);
}
}
function parse($filename)
{
$unix = new unix();
$LastScannLine = 0;
$GLOBALS["MYSQL_CATZ"] = new mysql_catz();
$GLOBALS["SQUID_FAMILY_CLASS"] = new squid_familysite();
if (!isset($GLOBALS["MYHOSTNAME"])) {
$unix = new unix();
$GLOBALS["MYHOSTNAME"] = $unix->hostname_g();
}
$filesource = dirname(__FILE__) . "/ressources/conf/upload/{$filename}";
if (!is_file($filesource)) {
build_progress(110, "{$filename} no such file");
return;
}
$tmpfile = $unix->FILE_TEMP();
if (!@copy($filesource, $tmpfile)) {
@unlink($filesource);
build_progress(110, "{$filename} -> {$tmpfile} {failed}");
return;
}
@unlink($filesource);
$SumOflines = $unix->COUNT_LINES_OF_FILE($tmpfile);
echo "{$tmpfile} {$SumOflines} lines\n";
$handle = @fopen($tmpfile, "r");
if (!$handle) {
echo "Fopen failed on {$tmpfile}\n";
build_progress(110, "{$tmpfile} {failed}");
@unlink($tmpfile);
return false;
}
$c = 0;
$d = 0;
$e = 0;
$prc = 0;
$prc_text = 0;
$mysql_first_time = 0;
$SITES = array();
$NOT_CATEGORIZED = array();
$CATEGORIZED = array();
$RQS = array();
$IPClass = new IP();
$FIRSTTIME = 0;
$LASTTIME = 0;
$TIME1 = time();
while (!feof($handle)) {
$c++;
$d++;
$e++;
$prc = $c / $SumOflines;
$prc = round($prc * 100);
$buffer = trim(fgets($handle));
if ($buffer == null) {
continue;
}
$stats_sites = count($SITES);
$stats_categorized = count($CATEGORIZED);
$stats_not_categorized = count($NOT_CATEGORIZED);
if (!isset($GLOBALS["LAST_PRC"])) {
build_progress($prc, "{$c}/{$SumOflines} {please_wait}");
$GLOBALS["LAST_PRC"] = $prc;
}
if ($prc > 5) {
if ($prc < 95) {
if ($GLOBALS["LAST_PRC"] != $prc) {
$array_load = sys_getloadavg();
$internal_load = $array_load[0];
$mem = round(memory_get_usage() / 1024 / 1000, 2);
echo "Load: {$internal_load}, Memory {$mem}MB\n";
echo "Categorized: " . FormatNumber($stats_categorized) . "\n";
echo "Unknown....: " . FormatNumber($stats_not_categorized) . "\n";
echo "Web sites..: " . FormatNumber($stats_sites) . "\n";
build_progress($prc, FormatNumber($c) . "/" . FormatNumber($SumOflines) . " {please_wait} - {$mem}MB {memory}");
$GLOBALS["LAST_PRC"] = $prc;
}
}
}
$array = parseAccessLine($buffer);
if (count($array) == 0) {
continue;
}
$TIME = $array["TIME"];
$LASTTIME = $TIME;
if ($FIRSTTIME == 0) {
$FIRSTTIME = $TIME;
}
$CATEGORY = $array["CATEGORY"];
$FAMILYSITE = $array["FAMILYSITE"];
$SIZE = intval($array["SIZE"]);
if ($IPClass->isIPAddress($FAMILYSITE)) {
if (!isset($IPADDRESSES[$FAMILYSITE]["RQS"])) {
$IPADDRESSES[$FAMILYSITE]["RQS"] = 1;
$IPADDRESSES[$FAMILYSITE]["SIZE"] = 0;
$IPADDRESSES[$FAMILYSITE]["CATEGORY"] = $CATEGORY;
} else {
$IPADDRESSES[$FAMILYSITE]["RQS"] = $IPADDRESSES[$FAMILYSITE]["RQS"] + 1;
$IPADDRESSES[$FAMILYSITE]["SIZE"] = $IPADDRESSES[$FAMILYSITE]["SIZE"] + $SIZE;
}
continue;
}
if (!isset($SITES[$FAMILYSITE])) {
$SITES[$FAMILYSITE] = 0;
}
if (!isset($RQS[$FAMILYSITE])) {
$RQS[$FAMILYSITE] = 0;
}
$SITES[$FAMILYSITE] = $SITES[$FAMILYSITE] + $SIZE;
$RQS[$FAMILYSITE] = $RQS[$FAMILYSITE] + 1;
if ($CATEGORY != null) {
$CATEGORIZED[$FAMILYSITE] = $CATEGORY;
continue;
}
$NOT_CATEGORIZED[$FAMILYSITE] = true;
}
fclose($handle);
@unlink($tmpfile);
build_progress(91, "{building_report}");
$TIME2 = time();
$stats_sites = count($SITES);
$stats_categorized = count($CATEGORIZED);
$stats_not_categorized = count($NOT_CATEGORIZED);
$ARRAY["DURATION"] = $unix->distanceOfTimeInWords($TIME1, $TIME2);
$ARRAY["SumOflines"] = $SumOflines;
$ARRAY["stats_sites"] = $stats_sites;
$ARRAY["stats_ip"] = count($IPADDRESSES);
$ARRAY["firsttime"] = $FIRSTTIME;
$ARRAY["lasttime"] = $LASTTIME;
$ARRAY["stats_categorized"] = $stats_categorized;
$ARRAY["stats_not_categorized"] = $stats_not_categorized;
build_progress(92, "{building_report}");
$CSV1[] = array("website", "size", "requests");
while (list($familysite, $ligne) = each($NOT_CATEGORIZED)) {
$CSV1[] = array($familysite, $SITES[$familysite], $RQS[$familysite]);
}
build_progress(95, "{building_report}");
$CSV2[] = array("website", "category", "size", "requests");
while (list($familysite, $category) = each($CATEGORIZED)) {
$CSV2[] = array($familysite, $category, $SITES[$familysite], $RQS[$familysite]);
}
build_progress(97, "{building_report}");
$CSV3[] = array("Public IP addresses", "category", "size", "requests");
while (list($ip, $ARRAYIPS) = each($IPADDRESSES)) {
$category = $ARRAYIPS["CATEGORY"];
$size = $ARRAYIPS["SIZE"];
$RQS = $ARRAYIPS["RQS"];
$CSV3[] = array($ip, $category, $size, $RQS);
}
build_progress(99, "{saving_reports}");
outputCSV($CSV1, "/usr/share/artica-postfix/ressources/logs/notcategorized.csv");
outputCSV($CSV2, "/usr/share/artica-postfix/ressources/logs/categorized.csv");
outputCSV($CSV3, "/usr/share/artica-postfix/ressources/logs/ipcategorized.csv");
@file_put_contents("/usr/share/artica-postfix/ressources/logs/categorized.array", serialize($ARRAY));
@chmod("/usr/share/artica-postfix/ressources/logs/notcategorized.csv", 0755);
@chmod("/usr/share/artica-postfix/ressources/logs/ipcategorized.csv", 0755);
@chmod("/usr/share/artica-postfix/ressources/logs/categorized.csv", 0755);
@chmod("/usr/share/artica-postfix/ressources/logs/categorized.array", 0755);
build_progress(100, "{done}");
}
function sargToFile($filePath)
{
if (!is_file($filePath)) {
progress("Fatal {$filePath} no such file", 10);
return;
}
$unix = new unix();
$sarg_bin = $unix->find_program("sarg");
$linesNumber = $unix->COUNT_LINES_OF_FILE($filePath);
$basename = basename($filePath);
progress("Open {$filePath} {$linesNumber} lines", 10);
$sock = new sockets();
$SargOutputDir = $sock->GET_INFO("SargOutputDir");
if ($SargOutputDir == null) {
$SargOutputDir = "/var/www/html/squid-reports";
}
$nice = EXEC_NICE();
$usersauth = false;
$t = time();
$squid = new squidbee();
if ($squid->is_auth()) {
$usersauth = true;
}
if ($usersauth) {
events("User authentification enabled");
$u = " -i ";
} else {
events("User authentification disabled");
}
$t = time();
$cmd = "{$nice}{$sarg_bin} {$u}-f /etc/squid3/sarg.conf -l \"{$filePath}\" -o \"{$SargOutputDir}\" -x -z 2>&1";
progress("Open {$cmd}", 10);
exec($cmd, $results);
while (list($index, $line) = each($results)) {
if (preg_match("#SARG: OPTION:#", $line)) {
continue;
}
events($line);
}
if ($basename == "sarg.log") {
continue;
}
$took = $unix->distanceOfTimeInWords($t, time(), true);
sarg_admin_events("{$basename} generated took: {$took}\n" . @implode("\n", $results), __FUNCTION__, __FILE__, __LINE__, "sarg");
build_index_page();
}
function parse_db($id)
{
$unix = new unix();
$pidfile = "/etc/artica-postfix/" . basename(__FILE__) . ".{$id}.pid";
$pid = trim(@file_get_contents($pidfile));
if ($unix->process_exists($pid)) {
$pid = getmypid();
echo "[{$pid}]:: Process {$pid} already running...\n";
die;
}
$pid = getmypid();
file_put_contents($pidfile, $pid);
$sql = "SELECT * FROM emailing_db_paths WHERE ID={$id}";
$q = new mysql();
$ligne = mysql_fetch_array($q->QUERY_SQL($sql, "artica_backup"));
$unzipbin = $unix->find_program("unzip");
if ($unzipbin == null) {
echo "[{$pid}]:: unzip tool no such file or directory...\n";
update_status($id, "110", 1, "unzip tool no such file or directory");
die;
}
$zip_path = $ligne["filepath"];
if (!is_file($zip_path)) {
echo "[{$pid}]:: {$zip_path} no such file or directory\n";
update_status($id, "110", 1, "zip db no such file or directory");
die;
}
$tmp_path = "/tmp/emailing-import/{$id}";
writeevent("using {$tmp_path}", $id);
@mkdir($tmp_path, 666, true);
writeevent("Uncompress {$zip_path}", $id);
echo "{$unzipbin} -o {$zip_path} -d {$tmp_path}\n";
shell_exec("{$unzipbin} -o {$zip_path} -d {$tmp_path}");
$files = $unix->DirFiles($tmp_path);
if (!is_array($files)) {
update_status($id, "110", 1, "{$zip_path} corrupted or no files stored");
die;
}
while (list($filename, $file_name) = each($files)) {
writeevent("parsing {$file_name}", $pid);
$max = $unix->COUNT_LINES_OF_FILE("{$tmp_path}/{$file_name}");
writeevent("parsing {$file_name} {$max} entries", $pid);
$handle = @fopen("{$tmp_path}/{$file_name}", "r");
$ligne["databasename"] = format_mysql_table($ligne["databasename"]);
$q = new mysql();
$q->CheckTableEmailingContacts("emailing_{$ligne["databasename"]}");
$sql = "INSERT INTO emailing_{$ligne["databasename"]} (`gender`,`firstname`, `lastname`,`email`,`phone`,`city`,`cp`,`postaladdress`,`domain`) VALUES";
if ($handle) {
while (!feof($handle)) {
$tw = $tw + 1;
$count = $count + 1;
unset($re);
$buffer = fgets($handle, 4096);
if (trim($buffer) == null) {
continue;
}
$buffer = str_replace('"', "", $buffer);
$buffer = str_replace(';', ",", $buffer);
$lines = explode(",", addslashes($buffer));
if (!is_array($lines)) {
if (preg_match("#.+?@.+#", $buffer)) {
for ($i = 0; $i < 8; $i++) {
$lines[$i] = "";
}
$lines[3] = trim($buffer);
}
}
if (count($lines) < 2) {
if (preg_match("#.+?@.+#", $buffer)) {
for ($i = 0; $i < 8; $i++) {
$lines[$i] = "";
}
$lines[3] = trim($buffer);
}
}
$lines[3] = str_replace(";", ".", $lines[3]);
$lines[3] = str_replace("?", ".", $lines[3]);
$lines[3] = str_replace("@.", "@", $lines[3]);
$lines[3] = str_replace('^', '@', $lines[3]);
$lines[3] = str_replace(',', '.', $lines[3]);
if (trim($lines[3]) == null) {
writeevent("failed 3:[{$buffer}] [" . __LINE__ . "]", $pid);
$GLOBALS["FAILED_CONTACTS"] = $GLOBALS["FAILED_CONTACTS"] + 1;
unset($lines);
continue;
}
if (!preg_match("#(.+?)@(.+?)\\.(.+)#", $lines[3], $re)) {
writeevent("failed 3:{$lines[3]} bad email address [" . __LINE__ . "]", $pid);
$GLOBALS["FAILED_CONTACTS"] = $GLOBALS["FAILED_CONTACTS"] + 1;
unset($lines);
continue;
}
$domain = $re[2];
if (substr($domain, strlen($domain) - 1, 1) == '.') {
$domain = substr($domain, 0, strlen($domain) - 1);
}
if (preg_match("#^\\..+#", $domain)) {
writeevent("failed 3:{$lines[3]} bad domain {$domain} [" . __LINE__ . "]", $pid);
$GLOBALS["FAILED_CONTACTS"] = $GLOBALS["FAILED_CONTACTS"] + 1;
unset($lines);
continue;
}
if ($GLOBALS["EMAILS"][$lines[3]]) {
writeevent("failed 3:{$lines[3]} already exists [" . __LINE__ . "]", $pid);
$GLOBALS["FAILED_CONTACTS"] = $GLOBALS["FAILED_CONTACTS"] + 1;
continue;
}
if (isBlockedMail($lines[3])) {
writeevent("failed 3:{$lines[3]} is blacklisted [" . __LINE__ . "]", $pid);
$GLOBALS["FAILED_CONTACTS"] = $GLOBALS["FAILED_CONTACTS"] + 1;
continue;
}
$sqla[] = "('{$lines[0]}',\n\t\t\t\t'{$lines[1]}',\n\t\t\t\t'{$lines[2]}',\n\t\t\t\t'{$lines[3]}',\n\t\t\t\t'{$lines[4]}',\n\t\t\t\t'{$lines[5]}',\n\t\t\t\t'{$lines[6]}',\n\t\t\t\t'{$lines[7]}','{$domain}')";
$GLOBALS["SUCCESS_CONTACTS"] = $GLOBALS["SUCCESS_CONTACTS"] + 1;
$GLOBALS["EMAILS"][$lines[3]] = true;
if (count($GLOBALS["EMAILS"]) > 10000) {
$GLOBALS["EMAILS"] = array();
}
unset($lines);
if ($tw > 100) {
$fullsql = $sql . "\n" . @implode(",", $sqla);
$q = new mysql();
$q->QUERY_SQL($fullsql, "artica_backup");
if (!$q->ok) {
writeevent($id, "{$q->mysql_error}\n{$fullsql}");
}
unset($sqla);
$purc = $count / $max;
$purc = $purc * 100;
$purc = round($purc, 0);
update_status($id, $purc, 0);
$tw = 0;
}
}
fclose($handle);
if (is_array($sqla)) {
$fullsql = $sql . "\n" . @implode(",", $sqla);
$q->QUERY_SQL($fullsql, "artica_backup");
if (!$q->ok) {
writeevent($id, "{$q->mysql_error}\n{$fullsql}");
}
}
}
}
if ($GLOBALS["SUCCESS_CONTACTS"] > 0) {
writeevent("Failed.:{$GLOBALS["FAILED_CONTACTS"]}", $id);
writeevent("Success:{$GLOBALS["SUCCESS_CONTACTS"]}", $id);
update_status($id, 100, 1, null);
} else {
update_status($id, 110, 0, null);
}
}
