function readsourcecode($file, &$strings)
{
global $CFG;
$lines = file($file);
foreach ($lines as $line) {
parseline($strings, $line, $file);
}
return $strings;
}
function ExplodeFile($filepath)
{
$unix = new unix();
$LastScannLine = 0;
$GLOBALS["MYSQL_CATZ"] = new mysql_catz();
$GLOBALS["SQUID_FAMILY_CLASS"] = new squid_familysite();
if (!isset($GLOBALS["MYHOSTNAME"])) {
$unix = new unix();
$GLOBALS["MYHOSTNAME"] = $unix->hostname_g();
}
$GLOBALS["SEQUENCE"] = md5_file($filepath);
$handle = @fopen($filepath, "r");
if (!$handle) {
echo "Fopen failed on {$filepath}\n";
return false;
}
$countlines = 0;
$c = 0;
$d = 0;
$e = 0;
$prc = 0;
$prc_text = 0;
$mysql_first_time = 0;
while (!feof($handle)) {
$c++;
$d++;
$e++;
if ($countlines > 0) {
$prc = $c / $countlines;
$prc = round($prc * 100);
if (!isset($GLOBALS["LAST_PRC"])) {
if ($GLOBALS["PROGRESS"]) {
echo "{$prc}%\n";
}
$GLOBALS["LAST_PRC"] = $prc;
} else {
if ($GLOBALS["LAST_PRC"] != $prc) {
if ($GLOBALS["PROGRESS"]) {
echo "{$prc}%\n";
}
$GLOBALS["LAST_PRC"] = $prc;
}
}
if ($prc > 10) {
if ($prc < 99) {
if ($prc > $prc_text) {
$array_load = sys_getloadavg();
$internal_load = $array_load[0];
$mem = round(memory_get_usage() / 1024 / 1000, 2);
$prc_design = FormatNumber($c) . "/" . FormatNumber($countlines);
$prc_text = $prc;
}
}
}
}
if ($d > 50) {
$iSeek = ftell($handle);
@file_put_contents("{$filepath}.last", $iSeek);
if ($GLOBALS["VERBOSE"]) {
$prc_design = FormatNumber($c) . "/" . FormatNumber($countlines);
echo "{$prc}% {$prc_design}\n";
}
$d = 0;
}
if ($e > 500) {
$mem = round(memory_get_usage() / 1024 / 1000, 2);
$prc_design = FormatNumber($c) . "/" . FormatNumber($countlines);
$e = 0;
}
$buffer = trim(fgets($handle));
if ($buffer == null) {
continue;
}
$array = parseline($buffer);
if (count($array) == 0) {
continue;
}
if ($mysql_first_time == 0) {
if (date("Y", $array["TIME"]) > 2001) {
$mysql_first_time = $array["TIME"];
}
}
if ($array["CATEGORY"] == null) {
$MAIN[$array["SITENAME"]]["COUNT"]++;
//echo date("Y-m-d H:i:s",$array["TIME"]);
//echo "{$array["SITENAME"]} {$array["SIZE"]} {$array["UID"]} {$array["IPADDR"]} {$array["CATEGORY"]} {$array["FAMILYSITE"]}\n";
}
}
krsort($MAIN);
while (list($sitename, $SUBARRAY) = each($MAIN)) {
echo "{$sitename} {$SUBARRAY["COUNT"]}\n";
}
}
}
function send_notification($fromaddress, $toaddress, $subject, $messagetext)
{
echo "Sending notification to {$toaddress}.\n";
$smtpsession = "mail from: <{$fromaddress}>\n" . "rcpt to: <{$toaddress}>\n" . "data\n" . "Subject: {$subject}\n" . "{$messagetext}\n" . ".\n" . "quit\n";
$pipe = popen('telnet smtp.srv.ualberta.ca 25', 'w');
if ($pipe == false) {
echo "Error occurred opening pipe connection to smtp server.\n";
} else {
$bytes = fwrite($pipe, $smtpsession);
if ($bytes = false) {
echo "Error writing to smtp server.\n";
}
pclose($pipe);
}
echo "Mail sent.\n";
}
while ($line = fgets(STDIN)) {
parseline($line);
}
if (isset($argv[1]) && $argv[1] == '-n') {
foreach ($coursesbyemail as $email => $courses) {
$messagebody = generate_messagebody($firstnamebyemail[$email], $lastnamebyemail[$email], $coursesbyemail[$email]);
print_notification('*****@*****.**', $email, 'eClass course archiving', $messagebody);
}
} else {
foreach ($coursesbyemail as $email => $courses) {
$messagebody = generate_messagebody($firstnamebyemail[$email], $lastnamebyemail[$email], $coursesbyemail[$email]);
send_notification('*****@*****.**', $email, 'eClass course archiving', $messagebody);
}
}
function _analyze_file($filepath, $zmd5)
{
if (!is_file($filepath)) {
events("{$filepath} no such file");
return false;
}
$sock = new sockets();
$unix = new unix();
$EnableImportWithSarg = $sock->GET_INFO("EnableImportWithSarg");
if (!is_numeric($EnableImportWithSarg)) {
$EnableImportWithSarg = 1;
}
$SargOutputDir = $sock->GET_INFO("SargOutputDir");
if ($SargOutputDir == null) {
$SargOutputDir = "/var/www/html/squid-reports";
}
$basename = basename($filepath);
$timeStart = time();
$unix = new unix();
$sock = new sockets();
$q = new mysql_squid_builder();
$TimeOfFile = strtotime(GetDateOfFile($filepath));
$ContainerDir = "/var/log/artica-postfix/squid/queues/" . date("Y-m-d-h", $TimeOfFile);
@mkdir($ContainerDir, 0755, true);
$handle = @fopen($filepath, "r");
if (!$handle) {
events("Failed to open file {$filepath}");
echo "Failed to open file\n";
return;
}
$c = 0;
$max = $unix->COUNT_LINES_OF_FILE($filepath);
$GLOBALS["BUFFER_FILE_ANALYZED"] = $filepath;
events("{$filepath} {$max} lines");
$ligne = mysql_fetch_array($q->QUERY_SQL("SELECT * FROM accesslogs_import WHERE zmd5='{$zmd5}'"));
$FileStatus = $ligne["status"];
if (!$GLOBALS["SIMULATE"]) {
if ($ligne["filename"] == null) {
echo "{$filepath}: {$zmd5} did not match expected md5\n";
@fclose($handle);
return;
}
}
if ($GLOBALS["VERBOSE"]) {
echo "Container: {$ContainerDir}\n";
echo "Status...: {$FileStatus}\n";
echo "Lines....: {$max}\n";
}
if (!$GLOBALS["FORCE"]) {
if ($FileStatus == 3) {
events("{$filepath} already analyzed, skip it...");
@fclose($handle);
return true;
}
}
if ($EnableImportWithSarg == 1) {
$u = null;
$nice = EXEC_NICE();
$sarg = $unix->find_program("sarg");
$php = $unix->LOCATE_PHP5_BIN();
$squid = new squidbee();
if ($squid->LDAP_AUTH == 1) {
$usersauth = true;
}
if ($squid->LDAP_EXTERNAL_AUTH == 1) {
$usersauth = true;
}
if ($usersauth) {
echo "Starting......: " . date("H:i:s") . " Sarg, user authentification enabled\n";
$u = " -i ";
}
if (is_file($sarg)) {
shell_exec("{$php} /usr/share/artica-postfix/exec.sarg.php --conf >/dev/null 2>&1");
exec("{$nice}{$sarg} {$u}-f /etc/squid3/sarg.conf -l {$filepath} -o \"{$SargOutputDir}\" 2>&1", $sargR);
while (list($index, $line) = each($sargR)) {
events("Sarg: {$line}\n");
}
}
}
$percent_ret = 0;
while (!feof($handle)) {
$c++;
$buffer = trim(fgets($handle));
if ($buffer == null) {
continue;
}
$array = parseline($buffer);
if (count($array) == 0) {
continue;
}
$ip = null;
$user = null;
$xtime = $array["TIME"];
$ip = $array["IPADDR"];
$user = $array["UID"];
$code_error = $array["ERRCODE"];
$size = $array["SIZE"];
$uri = $array["URI"];
$cached = $array["CACHED"];
// $q->QUERY_SQL("INSERT INTO accesslogs_import (zmd5,filename,zDate,size,status,percent) VALUES ('$md5','$filename','$date','$size',0,0)");
$HOSTNAME = $array["HOSTNAME"];
if (is_numeric($user)) {
echo "\n\n\n****************\n\nNumeric user:{$user}\n{$buffer}\n\n";
die;
}
if ($ip == null) {
if ($HOSTNAME != null) {
$ip = $HOSTNAME;
}
}
$GLOBALS["BUFFER_ANALYZED"] = $buffer;
$GLOBALS["squidtail"]->Builsql($ip, $user, $uri, $code_error, $size, $xtime, $cached, null, $xtime);
if ($GLOBALS["SIMULATE"]) {
continue;
}
$percent = $c / $max * 100;
$percent = round($percent);
if ($percent != $percent_ret) {
if ($GLOBALS["VERBOSE"]) {
echo "****************** {$percent}% ********************\n";
}
$percent_ret = $percent;
events("{$percent_ret}% " . count($GLOBALS["squidtail"]->GLOBAL_QUEUE) . " in memory - {$filepath}");
$q->QUERY_SQL("UPDATE accesslogs_import SET percent='{$percent}',status=1 WHERE zmd5='{$zmd5}'");
}
if (count($GLOBALS["squidtail"]->GLOBAL_QUEUE) > 2000) {
events("analyze_file()::{$basename}::{$percent}% GLOBAL_RTTSIZE......: " . count($GLOBALS["squidtail"]->GLOBAL_RTTSIZE) . " items...", __LINE__);
events("analyze_file()::{$basename}::{$percent}% GLOBAL_PAGEKEEPER...: " . count($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER) . " items...", __LINE__);
events("analyze_file()::{$basename}::{$percent}% GLOBAL_YOUTUBE......: " . count($GLOBALS["squidtail"]->GLOBAL_YOUTUBE) . " items...", __LINE__);
events("analyze_file()::{$basename}::{$percent}% GLOBAL_SQUIDUSERS...: " . count($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS) . " items...", __LINE__);
events("analyze_file()::{$basename}::{$percent}% GLOBAL_SEARCHWORDS..: " . count($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS) . " items...", __LINE__);
PURGE_GLOBAL_QUEUE($GLOBALS["squidtail"]->GLOBAL_QUEUE);
$GLOBALS["squidtail"]->GLOBAL_QUEUE = array();
if (count($GLOBALS["squidtail"]->GLOBAL_RTTSIZE) > 500) {
@mkdir("{$ContainerDir}/RTTSize", 0755, true);
@file_put_contents("{$ContainerDir}/RTTSize/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_RTTSIZE)), serialize($GLOBALS["squidtail"]->GLOBAL_RTTSIZE));
$GLOBALS["squidtail"]->GLOBAL_RTTSIZE = array();
$RTTSIZE = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER) > 500) {
@mkdir("{$ContainerDir}/PageKeeper", 0755, true);
@file_put_contents("{$ContainerDir}/PageKeeper/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER)), serialize($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER));
$GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER = array();
$PAGEKEEP = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_YOUTUBE) > 500) {
@mkdir("{$ContainerDir}/Youtube", 0755, true);
$md5 = md5(serialize($GLOBALS["squidtail"]->GLOBAL_YOUTUBE));
youtube_events("Saving queue:(2000) {$ContainerDir}/Youtube/" . $md5, __LINE__);
@file_put_contents("{$ContainerDir}/Youtube/" . $md5, serialize($GLOBALS["squidtail"]->GLOBAL_YOUTUBE));
$GLOBALS["squidtail"]->GLOBAL_YOUTUBE = array();
$YOUTUBE = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS) > 500) {
@mkdir("{$ContainerDir}/Members", 0755, true);
@file_put_contents("{$ContainerDir}/Members/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS)), serialize($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS));
$GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS = array();
}
if (count($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS) > 500) {
@mkdir("{$ContainerDir}/SearchWords", 0755, true);
@file_put_contents("{$ContainerDir}/SearchWords/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS)), serialize($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS));
$GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS = array();
}
}
// PURGE OVER 2000
}
// END GLOBAL LOOP
@fclose($handle);
PURGE_GLOBAL_QUEUE($GLOBALS["squidtail"]->GLOBAL_QUEUE);
events("analyze_file()::{$basename}:: Container.........: `{$ContainerDir}` ", __LINE__);
events("analyze_file()::{$basename}:: GLOBAL_RTTSIZE....: " . count($GLOBALS["squidtail"]->GLOBAL_RTTSIZE) . " items...", __LINE__);
events("analyze_file()::{$basename}:: GLOBAL_PAGEKEEPER.: " . count($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER) . " items...", __LINE__);
events("analyze_file()::{$basename}:: GLOBAL_YOUTUBE....: " . count($GLOBALS["squidtail"]->GLOBAL_YOUTUBE) . " items...", __LINE__);
events("analyze_file()::{$basename}:: GLOBAL_SQUIDUSERS.: " . count($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS) . " items...", __LINE__);
events("analyze_file()::{$basename}:: GLOBAL_SEARCHWORDS: " . count($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS) . " items...", __LINE__);
$q->QUERY_SQL("UPDATE accesslogs_import SET percent='100',status='3' WHERE zmd5='{$zmd5}'");
if (count($GLOBALS["squidtail"]->GLOBAL_RTTSIZE) > 0) {
@mkdir("{$ContainerDir}/RTTSize", 0755, true);
@file_put_contents("{$ContainerDir}/RTTSize/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_RTTSIZE)), serialize($GLOBALS["squidtail"]->GLOBAL_RTTSIZE));
$RTTSIZE = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER) > 0) {
@mkdir("{$ContainerDir}/PageKeeper", 0755, true);
@file_put_contents("{$ContainerDir}/PageKeeper/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER)), serialize($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER));
$PAGEKEEP = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_YOUTUBE) > 0) {
@mkdir("{$ContainerDir}/Youtube", 0755, true);
$md5 = md5(serialize($GLOBALS["squidtail"]->GLOBAL_YOUTUBE));
youtube_events("Saving queue: {$ContainerDir}/Youtube/" . $md5, __LINE__);
@file_put_contents("{$ContainerDir}/Youtube/" . $md5, serialize($GLOBALS["squidtail"]->GLOBAL_YOUTUBE));
$YOUTUBE = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS) > 0) {
@mkdir("{$ContainerDir}/Members", 0755, true);
@file_put_contents("{$ContainerDir}/Members/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS)), serialize($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS));
}
if (count($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS) > 0) {
@mkdir("{$ContainerDir}/SearchWords", 0755, true);
@file_put_contents("{$ContainerDir}/SearchWords/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS)), serialize($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS));
}
$size = round($size / 1024, 2);
events("analyze_file()::{$basename}:: {$max}: lines parsed in " . $unix->distanceOfTimeInWords($timeStart, time()) . __LINE__);
if (system_is_overloaded(basename(__FILE__))) {
return;
}
$php5 = $unix->LOCATE_PHP5_BIN();
$nice = EXEC_NICE();
$nohup = $unix->find_program("nohup");
$cmd = "{$nohup} {$php5} " . __FILE__ . " --squid >/dev/null 2>&1 &";
events("analyze_file()::{$cmd}");
shell_exec($cmd);
if ($PAGEKEEP) {
$cmd = "{$nohup} {$php5} /usr/share/artica-postfix/exec.squid.stats.php --thumbs-parse >/dev/null 2>&1 &";
events(__FUNCTION__ . ":: {$cmd}", __LINE__);
shell_exec($cmd);
}
if ($YOUTUBE) {
$cmd = "{$nohup} {$php5} " . __FILE__ . " --youtube >/dev/null 2>&1 &";
events(__FUNCTION__ . ":: {$cmd}", __LINE__);
shell_exec($cmd);
}
if ($RTTSIZE) {
$cmd = "{$nohup} {$php5} /usr/share/artica-postfix/exec.squid-users-rttsize.php --now >/dev/null 2>&1 &";
events(__FUNCTION__ . ":: {$cmd}", __LINE__);
shell_exec($cmd);
}
}
} else {
// if there are commands in the client's
// content data, then let's process them.
if (strlen($content) > 0) {
do {
// TODO: This seems like a really odd-construct to me - looping
// until we receive an positive from the parseline routine. shouldn't
// we instead loop through all the "lines" in the content, ending
// the loop naturally when we've exceeded all of them?
$i = strpos($content, "\r\n");
if ($i == 0) {
break;
}
$bm_row = substr($content, 0, $i);
// get the current bookmark and send it to be parsed.
if (parseline($bm_row, $ID)) {
my_echo("*Z");
end_script("'parseline' returned a positive value.");
}
// this bookmark is finished, so pop it off our list.
$content = substr($content, strlen($content) - (strlen($content) - $i - 1));
} while (true);
// with our updates finished, we update the token and move on.
my_mysql_query("update syncit_person set token = token+1, lastchanged = now() where personid='{$ID}'");
$res = my_mysql_query("select token from syncit_person where personid='{$ID}'");
// get the new server token for client updating.
$data = mysql_fetch_assoc($res);
$stoken = $data["token"];
// return new server token.
my_echo("*T," . $stoken);
}
