function _youtube_dayz($sourcetable)
{
$q = new mysql_squid_builder();
if (!$q->TABLE_EXISTS($sourcetable)) {
return true;
}
$sql = "SELECT zDate,ipaddr,hostname,uid,MAC,account,youtubeid,SUM(hits) as hits FROM {$sourcetable}\n\tGROUP BY zDate,ipaddr,hostname,uid,MAC,account,youtubeid";
$results = $q->QUERY_SQL($sql);
if (!$q->ok) {
writelogs_squid("Fatal: {$q->mysql_error} on `{$sourcetable}`", __FUNCTION__, __FILE__, __LINE__, "stats");
return;
}
youtube_events("_youtube_dayz(): {$sourcetable} " . mysql_num_rows($results) . " rows");
$f = array();
while ($ligne = @mysql_fetch_array($results, MYSQL_ASSOC)) {
$f[] = "('{$ligne["zDate"]}','{$ligne["hits"]}','{$ligne["ipaddr"]}','{$ligne["hostname"]}','{$ligne["uid"]}','{$ligne["MAC"]}','{$ligne["youtubeid"]}')";
}
$prefix = "INSERT IGNORE INTO youtube_dayz (zDate,hits,ipaddr,hostname,uid,MAC,youtubeid) VALUES ";
if (count($f) > 0) {
$q->QUERY_SQL($prefix . @implode(",", $f));
if (!$q->ok) {
writelogs_squid("Fatal: {$q->mysql_error} on `{$sourcetable}`", __FUNCTION__, __FILE__, __LINE__, "stats");
return;
}
}
return true;
}
function youtube_infos($VIDEOID)
{
if (isset($GLOBALS["youtubeid"][$VIDEOID])) {
return true;
}
$uri = "https://gdata.youtube.com/feeds/api/videos/{$VIDEOID}?v=2&alt=jsonc";
if ($GLOBALS["VERBOSE"]) {
echo "{$VIDEOID}:: {$uri} -> \n";
}
$curl = new ccurl($uri);
$error = null;
if (!$curl->GetFile("/tmp/jsonc.inc")) {
youtube_events("gdata.youtube.com = Failed = > {$curl->error}", __LINE__);
return false;
}
$infox = @file_get_contents("/tmp/jsonc.inc");
$infos = json_decode($infox);
$uploaded = $infos->data->uploaded;
$title = $infos->data->title;
if ($title == null) {
$error = $infos->error->message;
if ($error == null) {
if ($GLOBALS["VERBOSE"]) {
echo "data->title NULL ({$error})\n";
var_dump($infos);
}
return false;
} else {
$title = $error;
}
}
$category = $infos->data->category;
if ($category == null) {
if ($error != null) {
$category = $error;
}
}
$q = new mysql_squid_builder();
$ligne = mysql_fetch_array($q->QUERY_SQL("SELECT `youtubeid` FROM youtube_objects WHERE `youtubeid`='{$VIDEOID}'"));
if (!$q->ok) {
if (strpos("youtube_objects' doesn't exist", $q->mysql_error) > 0) {
$ligne = mysql_fetch_array($q->QUERY_SQL("SELECT `youtubeid` FROM youtube_objects WHERE `youtubeid`='{$VIDEOID}'"));
}
if ($GLOBALS["VERBOSE"]) {
echo "{$q->mysql_error}\n";
}
return;
}
if ($ligne["youtubeid"] != null) {
$GLOBALS["youtubeid"][$VIDEOID] = true;
if ($GLOBALS["VERBOSE"]) {
echo "{$VIDEOID} already exists in table\n";
}
return true;
}
$tumbnail = $infos->data->thumbnail->sqDefault;
$curl = new ccurl($tumbnail);
$curl->GetFile("/tmp/thumbnail");
$CATZ["Autos & Vehicles"] = "automobile/cars";
$CATZ["Film & Animation"] = "movies";
$CATZ["Gaming"] = "games";
$CATZ["Education"] = "recreation/schools";
$CATZ["Music"] = "music";
$CATZ["News & Politics"] = "news";
$CATZ["People & Blogs"] = "hobby/pets";
$CATZ["Science & Technology"] = "sciences";
$CATZ["Sports"] = "recreation/sports";
$CATZ["Travel & Events"] = "recreation/travel";
if (isset($CATZ[$category])) {
$category = $CATZ[$category];
}
$date = strtotime($uploaded);
$zDate = date("Y-m-d H:i:s");
$infox_enc = base64_encode($infox);
$title = addslashes($title);
$category = addslashes($category);
$duration = $infos->data->duration;
$thumbnail = addslashes(@file_get_contents("/tmp/thumbnail"));
$sql = "INSERT INTO youtube_objects (youtubeid,category,title,content,uploaded,duration,thumbnail) \n\tVALUES('{$VIDEOID}','{$category}','{$title}','{$infox_enc}','{$zDate}','{$duration}','{$thumbnail}')";
$q->QUERY_SQL($sql, "artica");
if (!$q->ok) {
if (strpos("youtube_objects' doesn", " {$q->mysql_error}") > 0) {
$q->CheckTables();
$ligne = mysql_fetch_array($q->QUERY_SQL("SELECT `youtubeid` FROM youtube_objects WHERE `youtubeid`='{$VIDEOID}'"));
}
}
if (!$q->ok) {
if ($GLOBALS["VERBOSE"]) {
echo "{$q->mysql_error}\n";
}
return false;
}
return true;
}
function _analyze_file($filepath, $zmd5)
{
if (!is_file($filepath)) {
events("{$filepath} no such file");
return false;
}
$sock = new sockets();
$unix = new unix();
$EnableImportWithSarg = $sock->GET_INFO("EnableImportWithSarg");
if (!is_numeric($EnableImportWithSarg)) {
$EnableImportWithSarg = 1;
}
$SargOutputDir = $sock->GET_INFO("SargOutputDir");
if ($SargOutputDir == null) {
$SargOutputDir = "/var/www/html/squid-reports";
}
$basename = basename($filepath);
$timeStart = time();
$unix = new unix();
$sock = new sockets();
$q = new mysql_squid_builder();
$TimeOfFile = strtotime(GetDateOfFile($filepath));
$ContainerDir = "/var/log/artica-postfix/squid/queues/" . date("Y-m-d-h", $TimeOfFile);
@mkdir($ContainerDir, 0755, true);
$handle = @fopen($filepath, "r");
if (!$handle) {
events("Failed to open file {$filepath}");
echo "Failed to open file\n";
return;
}
$c = 0;
$max = $unix->COUNT_LINES_OF_FILE($filepath);
$GLOBALS["BUFFER_FILE_ANALYZED"] = $filepath;
events("{$filepath} {$max} lines");
$ligne = mysql_fetch_array($q->QUERY_SQL("SELECT * FROM accesslogs_import WHERE zmd5='{$zmd5}'"));
$FileStatus = $ligne["status"];
if (!$GLOBALS["SIMULATE"]) {
if ($ligne["filename"] == null) {
echo "{$filepath}: {$zmd5} did not match expected md5\n";
@fclose($handle);
return;
}
}
if ($GLOBALS["VERBOSE"]) {
echo "Container: {$ContainerDir}\n";
echo "Status...: {$FileStatus}\n";
echo "Lines....: {$max}\n";
}
if (!$GLOBALS["FORCE"]) {
if ($FileStatus == 3) {
events("{$filepath} already analyzed, skip it...");
@fclose($handle);
return true;
}
}
if ($EnableImportWithSarg == 1) {
$u = null;
$nice = EXEC_NICE();
$sarg = $unix->find_program("sarg");
$php = $unix->LOCATE_PHP5_BIN();
$squid = new squidbee();
if ($squid->LDAP_AUTH == 1) {
$usersauth = true;
}
if ($squid->LDAP_EXTERNAL_AUTH == 1) {
$usersauth = true;
}
if ($usersauth) {
echo "Starting......: " . date("H:i:s") . " Sarg, user authentification enabled\n";
$u = " -i ";
}
if (is_file($sarg)) {
shell_exec("{$php} /usr/share/artica-postfix/exec.sarg.php --conf >/dev/null 2>&1");
exec("{$nice}{$sarg} {$u}-f /etc/squid3/sarg.conf -l {$filepath} -o \"{$SargOutputDir}\" 2>&1", $sargR);
while (list($index, $line) = each($sargR)) {
events("Sarg: {$line}\n");
}
}
}
$percent_ret = 0;
while (!feof($handle)) {
$c++;
$buffer = trim(fgets($handle));
if ($buffer == null) {
continue;
}
$array = parseline($buffer);
if (count($array) == 0) {
continue;
}
$ip = null;
$user = null;
$xtime = $array["TIME"];
$ip = $array["IPADDR"];
$user = $array["UID"];
$code_error = $array["ERRCODE"];
$size = $array["SIZE"];
$uri = $array["URI"];
$cached = $array["CACHED"];
// $q->QUERY_SQL("INSERT INTO accesslogs_import (zmd5,filename,zDate,size,status,percent) VALUES ('$md5','$filename','$date','$size',0,0)");
$HOSTNAME = $array["HOSTNAME"];
if (is_numeric($user)) {
echo "\n\n\n****************\n\nNumeric user:{$user}\n{$buffer}\n\n";
die;
}
if ($ip == null) {
if ($HOSTNAME != null) {
$ip = $HOSTNAME;
}
}
$GLOBALS["BUFFER_ANALYZED"] = $buffer;
$GLOBALS["squidtail"]->Builsql($ip, $user, $uri, $code_error, $size, $xtime, $cached, null, $xtime);
if ($GLOBALS["SIMULATE"]) {
continue;
}
$percent = $c / $max * 100;
$percent = round($percent);
if ($percent != $percent_ret) {
if ($GLOBALS["VERBOSE"]) {
echo "****************** {$percent}% ********************\n";
}
$percent_ret = $percent;
events("{$percent_ret}% " . count($GLOBALS["squidtail"]->GLOBAL_QUEUE) . " in memory - {$filepath}");
$q->QUERY_SQL("UPDATE accesslogs_import SET percent='{$percent}',status=1 WHERE zmd5='{$zmd5}'");
}
if (count($GLOBALS["squidtail"]->GLOBAL_QUEUE) > 2000) {
events("analyze_file()::{$basename}::{$percent}% GLOBAL_RTTSIZE......: " . count($GLOBALS["squidtail"]->GLOBAL_RTTSIZE) . " items...", __LINE__);
events("analyze_file()::{$basename}::{$percent}% GLOBAL_PAGEKEEPER...: " . count($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER) . " items...", __LINE__);
events("analyze_file()::{$basename}::{$percent}% GLOBAL_YOUTUBE......: " . count($GLOBALS["squidtail"]->GLOBAL_YOUTUBE) . " items...", __LINE__);
events("analyze_file()::{$basename}::{$percent}% GLOBAL_SQUIDUSERS...: " . count($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS) . " items...", __LINE__);
events("analyze_file()::{$basename}::{$percent}% GLOBAL_SEARCHWORDS..: " . count($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS) . " items...", __LINE__);
PURGE_GLOBAL_QUEUE($GLOBALS["squidtail"]->GLOBAL_QUEUE);
$GLOBALS["squidtail"]->GLOBAL_QUEUE = array();
if (count($GLOBALS["squidtail"]->GLOBAL_RTTSIZE) > 500) {
@mkdir("{$ContainerDir}/RTTSize", 0755, true);
@file_put_contents("{$ContainerDir}/RTTSize/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_RTTSIZE)), serialize($GLOBALS["squidtail"]->GLOBAL_RTTSIZE));
$GLOBALS["squidtail"]->GLOBAL_RTTSIZE = array();
$RTTSIZE = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER) > 500) {
@mkdir("{$ContainerDir}/PageKeeper", 0755, true);
@file_put_contents("{$ContainerDir}/PageKeeper/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER)), serialize($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER));
$GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER = array();
$PAGEKEEP = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_YOUTUBE) > 500) {
@mkdir("{$ContainerDir}/Youtube", 0755, true);
$md5 = md5(serialize($GLOBALS["squidtail"]->GLOBAL_YOUTUBE));
youtube_events("Saving queue:(2000) {$ContainerDir}/Youtube/" . $md5, __LINE__);
@file_put_contents("{$ContainerDir}/Youtube/" . $md5, serialize($GLOBALS["squidtail"]->GLOBAL_YOUTUBE));
$GLOBALS["squidtail"]->GLOBAL_YOUTUBE = array();
$YOUTUBE = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS) > 500) {
@mkdir("{$ContainerDir}/Members", 0755, true);
@file_put_contents("{$ContainerDir}/Members/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS)), serialize($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS));
$GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS = array();
}
if (count($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS) > 500) {
@mkdir("{$ContainerDir}/SearchWords", 0755, true);
@file_put_contents("{$ContainerDir}/SearchWords/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS)), serialize($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS));
$GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS = array();
}
}
// PURGE OVER 2000
}
// END GLOBAL LOOP
@fclose($handle);
PURGE_GLOBAL_QUEUE($GLOBALS["squidtail"]->GLOBAL_QUEUE);
events("analyze_file()::{$basename}:: Container.........: `{$ContainerDir}` ", __LINE__);
events("analyze_file()::{$basename}:: GLOBAL_RTTSIZE....: " . count($GLOBALS["squidtail"]->GLOBAL_RTTSIZE) . " items...", __LINE__);
events("analyze_file()::{$basename}:: GLOBAL_PAGEKEEPER.: " . count($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER) . " items...", __LINE__);
events("analyze_file()::{$basename}:: GLOBAL_YOUTUBE....: " . count($GLOBALS["squidtail"]->GLOBAL_YOUTUBE) . " items...", __LINE__);
events("analyze_file()::{$basename}:: GLOBAL_SQUIDUSERS.: " . count($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS) . " items...", __LINE__);
events("analyze_file()::{$basename}:: GLOBAL_SEARCHWORDS: " . count($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS) . " items...", __LINE__);
$q->QUERY_SQL("UPDATE accesslogs_import SET percent='100',status='3' WHERE zmd5='{$zmd5}'");
if (count($GLOBALS["squidtail"]->GLOBAL_RTTSIZE) > 0) {
@mkdir("{$ContainerDir}/RTTSize", 0755, true);
@file_put_contents("{$ContainerDir}/RTTSize/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_RTTSIZE)), serialize($GLOBALS["squidtail"]->GLOBAL_RTTSIZE));
$RTTSIZE = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER) > 0) {
@mkdir("{$ContainerDir}/PageKeeper", 0755, true);
@file_put_contents("{$ContainerDir}/PageKeeper/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER)), serialize($GLOBALS["squidtail"]->GLOBAL_PAGEKEEPER));
$PAGEKEEP = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_YOUTUBE) > 0) {
@mkdir("{$ContainerDir}/Youtube", 0755, true);
$md5 = md5(serialize($GLOBALS["squidtail"]->GLOBAL_YOUTUBE));
youtube_events("Saving queue: {$ContainerDir}/Youtube/" . $md5, __LINE__);
@file_put_contents("{$ContainerDir}/Youtube/" . $md5, serialize($GLOBALS["squidtail"]->GLOBAL_YOUTUBE));
$YOUTUBE = true;
}
if (count($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS) > 0) {
@mkdir("{$ContainerDir}/Members", 0755, true);
@file_put_contents("{$ContainerDir}/Members/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS)), serialize($GLOBALS["squidtail"]->GLOBAL_SQUIDUSERS));
}
if (count($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS) > 0) {
@mkdir("{$ContainerDir}/SearchWords", 0755, true);
@file_put_contents("{$ContainerDir}/SearchWords/" . md5(serialize($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS)), serialize($GLOBALS["squidtail"]->GLOBAL_SEARCHWORDS));
}
$size = round($size / 1024, 2);
events("analyze_file()::{$basename}:: {$max}: lines parsed in " . $unix->distanceOfTimeInWords($timeStart, time()) . __LINE__);
if (system_is_overloaded(basename(__FILE__))) {
return;
}
$php5 = $unix->LOCATE_PHP5_BIN();
$nice = EXEC_NICE();
$nohup = $unix->find_program("nohup");
$cmd = "{$nohup} {$php5} " . __FILE__ . " --squid >/dev/null 2>&1 &";
events("analyze_file()::{$cmd}");
shell_exec($cmd);
if ($PAGEKEEP) {
$cmd = "{$nohup} {$php5} /usr/share/artica-postfix/exec.squid.stats.php --thumbs-parse >/dev/null 2>&1 &";
events(__FUNCTION__ . ":: {$cmd}", __LINE__);
shell_exec($cmd);
}
if ($YOUTUBE) {
$cmd = "{$nohup} {$php5} " . __FILE__ . " --youtube >/dev/null 2>&1 &";
events(__FUNCTION__ . ":: {$cmd}", __LINE__);
shell_exec($cmd);
}
if ($RTTSIZE) {
$cmd = "{$nohup} {$php5} /usr/share/artica-postfix/exec.squid-users-rttsize.php --now >/dev/null 2>&1 &";
events(__FUNCTION__ . ":: {$cmd}", __LINE__);
shell_exec($cmd);
}
}
