function WEBTESTS(){
$www=$_REQUEST["WEBTESTS"];
$q=new mysql_squid_builder();
$www=$q->WebsiteStrip($www);
if($www==null){echo "corrupted\n";return;}
$catz=str_replace(",", "\n- ", $q->GET_CATEGORIES($www,true));
echo "\nFinal:\n\"".$q->GET_CATEGORIES($www,true)."\"\n";
}
function CategorizeAWebSite($www, $category)
{
$md5 = md5($www . $category);
$q = new mysql_squid_builder();
$q->CheckTables();
$uuid = $GLOBALS["UUID"];
$category_table = $q->category_transform_name($category);
$sql_add = "INSERT IGNORE INTO categorize (zmd5,zDate,category,pattern,uuid) VALUES('{$md5}',NOW(),'{$category}','{$www}','{$uuid}')";
$sql_add2 = "INSERT IGNORE INTO category_{$category_table} (zmd5,zDate,category,pattern,uuid) VALUES('{$md5}',NOW(),'{$category}','{$www}','{$uuid}')";
$q->QUERY_SQL($sql_add);
if (!$q->ok) {
echo $q->mysql_error . "\n{$sql_add}\n";
return false;
}
$q->QUERY_SQL($sql_add2);
if (!$q->ok) {
echo $q->mysql_error . "\n{$sql_add2}\n";
return false;
}
$categories = $q->GET_CATEGORIES($www, true);
if ($categories != null) {
$sql = "UPDATE visited_sites SET category='{$categories}' WHERE sitename='{$www}'";
$q->QUERY_SQL($sql);
if (!$q->ok) {
echo $q->mysql_error . "\n{$sql}\n";
return false;
}
}
}
function visited_sites()
{
$unix = new unix();
$pidfile = "/etc/artica-postfix/pids/squid.visited_sites_rescan.pid";
$pid = @file_get_contents($pidfile);
if ($pid < 100) {
$pid = null;
}
$t = time();
if ($unix->process_exists($pid, basename(__FILE__))) {
$time = $unix->PROCCESS_TIME_MIN($pid);
if ($GLOBALS["VERBOSE"]) {
echo "Already executed pid {$pid} since {$time}mn\n";
}
die;
}
$mypid = getmypid();
@file_put_contents($pidfile, $mypid);
stats_admin_events(2, "Starting table visited_sites", "", __FILE__, __LINE__);
progress("Starting table visited_sites", 5);
$q = new mysql_squid_builder();
$sql = "SELECT sitename FROM visited_sites WHERE LENGTH(category)=0";
$results = $q->QUERY_SQL($sql);
$num_rows = mysql_num_rows($results);
$took = $unix->distanceOfTimeInWords($t, time());
stats_admin_events(2, "Query done {$num_rows} websites to scan", "took:{$took}", __FILE__, __LINE__);
if ($num_rows == 0) {
progress(null, 100);
return;
}
progress("Query done {$num_rows} websites to scan", 10);
$c = 0;
$t = 0;
$d = 0;
while ($ligne = @mysql_fetch_array($results, MYSQL_ASSOC)) {
$sitenameOrg = $ligne["sitename"];
$sitename = strtolower(trim($sitenameOrg));
if (badCharacters($sitename)) {
$q->categorize_reaffected($sitename);
$sitenameOrg = mysql_escape_string2($sitenameOrg);
$q->QUERY_SQL("UPDATE visited_sites SET category='reaffected' WHERE `sitename`='{$sitenameOrg}'");
if (!$q->ok) {
progress("Fatal", 100);
die;
}
$d++;
$c++;
continue;
}
if (strpos($sitename, ".") == 0) {
$q->categorize_reaffected($sitename);
$sitenameOrg = mysql_escape_string2($sitenameOrg);
$q->QUERY_SQL("UPDATE visited_sites SET category='reaffected' WHERE `sitename`='{$sitenameOrg}'");
if (!$q->ok) {
progress("Fatal", 100);
die;
}
$d++;
$c++;
continue;
}
if (preg_match("#[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $sitename)) {
$sitename = gethostbyaddr($sitename);
if (preg_match("#[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $sitename)) {
$c++;
continue;
}
}
$ipaddr = gethostbyname($sitename);
if (!preg_match("#[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $ipaddr)) {
$q->categorize_reaffected($sitenameOrg);
$q->QUERY_SQL("UPDATE visited_sites SET category='reaffected' WHERE `sitename`='{$sitenameOrg}'");
if (!$q->ok) {
progress("Fatal", 100);
die;
}
$d++;
$c++;
continue;
}
$cat = $q->GET_CATEGORIES($sitename);
if ($cat != null) {
$d++;
$q->QUERY_SQL("UPDATE visited_sites SET category='{$cat}' WHERE `sitename`='{$sitenameOrg}'");
if (!$q->ok) {
progress("Fatal", 100);
die;
}
}
$c++;
if ($c > 50) {
$t = $t + $c;
$purc = $t / $num_rows;
$purc = round($purc, 2) * 100;
$c = 0;
if ($purc > 10) {
progress("{$sitename} {$t}/{$num_rows}", $purc);
}
}
}
progress($sitename, 100);
$took = $unix->distanceOfTimeInWords($t, time());
if ($d > 0) {
stats_admin_events(2, "{$d} New categorized websites...", "took:{$took}", __FILE__, __LINE__);
}
}
function GetDomains($i)
{
$unix = new unix();
$tmpfile = $unix->FILE_TEMP();
$curl = new ccurl("http://0675.com.cn/newwebsite_20120101_list._page0.html");
$curl->NoHTTP_POST = true;
echo "Get page {$i}\n";
if (!$curl->GetFile($tmpfile)) {
echo "http://0675.com.cn/newwebsite_20120101_list._page{$i}.html -> error: \n" . $curl->error . "\n";
return;
}
$datas = @file($tmpfile);
$size = strlen(@implode("", $datas)) / 1024;
echo "Page[{$i}]:: {$tmpfile} Size: {$size} Ko\n";
while (list($num, $ligne) = each($datas)) {
if (preg_match("#<div class=.*?newdomain.*?>(.*)#", $ligne)) {
echo "Sure line {$num}";
$newdata = str_replace("</li>", "", $ligne);
$newdata = str_replace("</div>", "", $newdata);
$f = explode("<li>", $newdata);
}
}
$q = new mysql_squid_builder();
while (list($num, $www) = each($f)) {
if (preg_match("#^\\.(.+)#", $www, $re)) {
$www = $re[1];
}
if (strpos($www, ",") > 0) {
continue;
}
if (strpos($www, " ") > 0) {
continue;
}
if (strpos($www, ":") > 0) {
continue;
}
if (strpos($www, "%") > 0) {
continue;
}
if (strpos($www, ">") > 0) {
continue;
}
if (strpos($www, "<") > 0) {
continue;
}
if (preg_match("#^www\\.(.+)#", $www, $re)) {
$www = $re[1];
}
$articacats = trim($q->GET_CATEGORIES($www, true, false));
if ($articacats != null) {
echo "\"{$www}\" SUCCESS - {$articacats} -\n";
continue;
}
echo "\"{$www}\" FAILED\n";
}
}
function ParseSitesInfos_artica_category($sitename)
{
$q = new mysql_squid_builder();
return $q->GET_CATEGORIES($sitename, true);
}
function site_infos_categories()
{
$q = new mysql_squid_builder();
$tpl = new templates();
$page = CurrentPageName();
include_once dirname(__FILE__) . "/ressources/class.dansguardian.inc";
$delete = $tpl->javascript_parse_text("{delete}");
$from = $tpl->javascript_parse_text("{from}");
$do_you_want_to_recompile_database = $tpl->javascript_parse_text("{do_you_want_to_recompile_database}");
$success = $tpl->javascript_parse_text("{sucess}");
$t = time();
$catz = $q->GET_CATEGORIES($_GET["search-stats-categories"]);
if ($catz != null) {
if (strpos($catz, ",") > 0) {
$cats = explode(",", $catz);
} else {
$cats[] = $catz;
}
}
$catNum = count($cats);
if ($catNum == 0) {
$html = "\n\t\t\n\t\t<table style='width:99%'>\n\t\t\t<tbody><tr><td width=1%>\n\t\t\t\t<img src='img/warning-panneau-32.png'></td>\n\t\t\t\t<td width=99% style='font-size:13px;font-weight:bold'>{this_website_hasnocat}</td>\n\t\t\t\t</tr></tbody></table>";
echo $tpl->_ENGINE_parse_body("{$html}");
return;
}
$dd = new dansguardian_rules();
$catstext = $tpl->_ENGINE_parse_body("{this_website_is_categorized_inXX}");
$catstext = str_replace("XX", count($cats), $catstext);
https:
//192.168.1.106:9000/squid.categories.php?query=yes&category=p**n&search=megavideo.&strictSearch=0
$html = "<table style='width:99%'>\n\t\t<tbody>\n\t\t<tr>\n\t\t\t<td colspan=2 style='font-size:13px'>{$catstext}</td>\n\t\t</tr>\n\t";
while (list($a, $b) = each($cats)) {
if ($b == null) {
continue;
}
$img = $dd->array_pics[$b];
if ($img == null) {
$img = "20-categories-personnal.png";
}
$id = md5($b);
$html = $html . "<tr id='tt{$id}'>\n\t\t\t<td width=1%><img src='img/{$img}'></td>\n\t\t\t<td style='font-size:13px;font-weight:bold'>\n\t\t\t<a href=\"javascript:blur();\" \n\t\t\tOnClick=\"javascript:Loadjs('squid.categorize.php?www={$_GET["search-stats-categories"]}&day=&week=&month=');\" \n\t\t\tstyle='font-size:13px;text-decoration:underline;;font-weight:bold'>{$b}</td>\n\t\t\t</tr>";
}
$html = $html . "</tbody></table>\n\t\n\t<script>\n\tvar MEMCAT{$t}='';\n\tvar MEMID{$t}='';\n\t\n\t\tvar x_DeleteCategoryFromSite{$t}= function (obj) {\n\t\t\tvar results=obj.responseText;\n\t\t\tif(results.length>0){alert(results);return;}\n\t\t\t\$('#tt'+ MEMID{$t}).remove();\n\t\t\tif(confirm('{$do_you_want_to_recompile_database} ?')){\n\t\t\t\tvar XHR = new XHRConnection();\n\t\t\t\tXHR.appendData('action_compile_category','yes');\n\t\t\t\tXHR.appendData('category',MEMCAT{$t});\n\t\t\t\tXHR.sendAndLoad('{$page}', 'POST');\t\t\t\n\t\t\t\talert('{$success}');\n\t\t\t}\n\t\t\t\n\t\t}\t\t\n\t\n\t\n\t\tfunction DeleteCategoryFromSite{$t}(category,www,id){\n\t\t\tMEMID{$t}=id;\n\t\t\tif(confirm('{$delete} '+category+' {$from} '+www+' ?')){\n\t\t\t\tvar XHR = new XHRConnection();\n\t\t\t\tMEMCAT{$t}=category;\n\t\t\t\tXHR.appendData('action_delete_from_category','yes');\n\t\t\t\tXHR.appendData('sitename',www);\n\t\t\t\tXHR.appendData('category',category);\n\t\t\t\tXHR.sendAndLoad('{$page}', 'POST',x_DeleteCategoryFromSite{$t});\n\t\t\t}\n\t\t\n\t\t}\n\t</script>\n\t";
echo $tpl->_ENGINE_parse_body("{$html}");
return;
}
echo "No changes\n";
die;
}
$table_hours = $q->LIST_TABLES_HOURS();
$table_days = $q->LIST_TABLES_DAYS();
$table_week = $q->LIST_TABLES_WEEKS();
$ALREADY = array();
$MAX = mysql_num_rows($results);
$D = 0;
while ($ligne = @mysql_fetch_array($results, MYSQL_ASSOC)) {
if (isset($ALREADY[$ligne["sitename"]])) {
$q->QUERY_SQL("DELETE FROM categorize_changes WHERE `zmd5`='{$ligne["zmd5"]}'");
continue;
}
$website = $ligne["sitename"];
$categories = $q->GET_CATEGORIES($website, true);
$ALREADY[$ligne["sitename"]] = true;
reset($table_hours);
reset($table_days);
reset($table_week);
$categories = addslashes($categories);
$D++;
$t = time();
$a = 0;
$b = 0;
$c = 0;
while (list($num, $table) = each($table_hours)) {
$a++;
if ($GLOBALS["VERBOSE"]) {
echo "Update {$table}\n";
}
function popup_categories()
{
$tpl = new templates();
if (trim($_GET["www"]) == null) {
$error_no_website_selected = $tpl->javascript_parse_text("{error_no_website_selected}");
echo "\n\t\t<script>\n\t\t\talert('{$error_no_website_selected}');\n\t\t\tYahooWinBrowseHide();\n\t\t</script>\n\t\t";
return;
}
$www = trim(strtolower($_GET["www"]));
if (preg_match("#www\\.(.+?)\$#i", $www, $re)) {
$www = $re[1];
}
$q = new mysql_squid_builder();
$cats = explode(",", $q->GET_CATEGORIES($www));
if (is_array($cats)) {
while (list($num, $ligne) = each($cats)) {
$ligne = trim($ligne);
if ($ligne == null) {
continue;
}
$hash_community[$ligne] = true;
}
}
$www_encoded = base64_encode($_GET["www"]);
$count = count($hash);
$count_community = count($hash_community);
$html = "\n\t<div style='font-size:13px'>{dansguardian_categorize_explain}</div>\n\t<hr>\n\t<div style='font-size:13px;color:red'>{$count_community} {categoryies_community}</div>\n\t<hr>\n\t<div style='height:490px;overflow:auto;margin:9px'>\n\t<table style='width:100%'>\n\t";
$dans = new dansguardian_rules();
while (list($num, $val) = each($dans->array_blacksites)) {
$md = md5($num);
$field_enabled = 0;
if ($hash_community[$num]) {
$field_enabled = 1;
}
$field = Field_checkbox("{$md}", 1, $field_enabled, "DansCommunityCategory('{$md}','{$num}','{$www_encoded}')");
if ($dans->array_pics[$num] != null) {
$pic = "<img src='img/{$dans->array_pics[$num]}'>";
} else {
$pic = " ";
}
$color = "black";
if ($hash[$num]) {
$field = "<img src='img/check2.gif'>";
$color = "red";
}
$html = $html . "\n\t\t<tr " . CellRollOver() . ">\n\t\t\t<td width=1%>{$pic}</td>\n\t\t\t<td><strong style='font-size:11px;color:{$color}'>{$val}</td>\n\t\t\t<td>{$field}</td>\n\t\t\t<td><span style='color:{$color}'>{$num}</span></td>\n\t\t\t\n\t\t</tr> \n\t\t\n\t\t";
}
$html = $html . "</table>";
$tpl = new templates();
echo $tpl->_ENGINE_parse_body($html);
}
function GetCategory($www)
{
$q = new mysql_squid_builder();
return $q->GET_CATEGORIES($www);
}
function categorize($day = null)
{
$unix = new unix();
$pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . ".{$day}." . __FUNCTION__ . ".pid";
$pid = @file_get_contents($pidfile);
if ($unix->process_exists($pid, basename(__FILE__))) {
if ($GLOBALS["VERBOSE"]) {
echo "Already executed pid {$pid}\n";
}
return;
}
if ($day == null) {
return;
}
if (system_is_overloaded()) {
echo "Overloaded system, aborting task\n";
writelogs_squid("Overloaded system, aborting task", __FUNCTION__, __FILE__, __LINE__, "categorize");
return;
}
$daySource = $day;
$mypid = getmypid();
@file_put_contents($pidfile, $mypid);
$q = new mysql_squid_builder();
$time = strtotime("{$day} 00:00:00");
$day = str_replace("-", "", $day);
$table = "{$day}_hour";
$table_blocked = "{$day}_blocked";
$table_month = date("Ym", $time) . "_day";
$table_week = date("YW", $time) . "_week";
$table_week_blocked = date("YW", $time) . "_blocked_week";
$ipClass = new IP();
echo "{$daySource} time: {$time} Table day={$table}, table_blocked={$table_blocked}, table_month={$table_month}, table_week={$table_week}\n";
events("{$daySource} time: {$time} Table day={$table}, table_blocked={$table_blocked}, table_month={$table_month}, table_week={$table_week}");
$t = time();
$f = 0;
if (!$q->TABLE_EXISTS($table)) {
echo $table . " no such table\n";
return;
}
$sql = "SELECT sitename,category FROM {$table} GROUP BY sitename,category HAVING LENGTH(category)=0";
events("{$sql}");
$results = $q->QUERY_SQL($sql);
if (!$q->ok) {
writelogs_squid("Re-categorized table {$table} Query failed: `{$sql}` ({$q->mysql_error})", __FUNCTION__, __FILE__, __LINE__, "categorize");
}
if (!$q->TABLE_EXISTS($table_month)) {
if (!$q->CreateMonthTable($table_month)) {
writelogs_squid("failed Create {$table_month} table {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "categorize");
}
}
if (!$q->TABLE_EXISTS($table_week)) {
if (!$q->CreateWeekTable($table_week)) {
writelogs_squid("failed Create {$table_week} table {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "categorize");
}
}
$L = 0;
$q->QUERY_SQL("DELETE FROM `catztemp` WHERE `category`=''");
while ($ligne = @mysql_fetch_array($results, MYSQL_ASSOC)) {
$website = trim($ligne["sitename"]);
if (preg_match("#^www\\.(.+)#", $website, $re)) {
$q->QUERY_SQL("UPDATE {$table} SET sitename='{$re[1]}' WHERE sitename='{$website}'");
$q->QUERY_SQL("UPDATE {$table_month} SET sitename='{$re[1]}' WHERE sitename='{$website}'");
$q->QUERY_SQL("UPDATE {$table_week} SET sitename='{$re[1]}' WHERE sitename='{$website}'");
$q->QUERY_SQL("UPDATE {$table_blocked} SET website='{$re[1]}' WHERE sitename='{$website}'");
$q->QUERY_SQL("UPDATE {$table_week_blocked} SET website='{$re[1]}' WHERE sitename='{$website}'");
$website = $re[1];
}
if ($website == null) {
continue;
}
if ($ipClass->isValid($website)) {
$website = gethostbyaddr($website);
}
if (isset($GLOBALS[__FUNCTION__][$website])) {
$category = $GLOBALS[__FUNCTION__][$website];
}
$category = $q->GET_CATEGORIES($website);
if ($category == null) {
if ($ipClass->isValid($website)) {
$category = "ipaddr";
}
}
events("{$day}] {$website} = {$category}");
$GLOBALS[__FUNCTION__][$website] = $category;
if ($L > 500) {
if (system_is_overloaded()) {
ufdbguard_admin_events("Fatal: Overloaded system, die();", __FUNCTION__, __FILE__, __LINE__, "stats");
return;
}
$L = 0;
}
if ($category == null) {
continue;
}
$f++;
events("Update {$table} {$website} = {$category}");
$q->QUERY_SQL("UPDATE {$table} SET category='{$category}' WHERE sitename='{$website}'");
if (!$q->ok) {
writelogs_squid("Re-categorized table {$table} failed {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "categorize");
}
events("Update {$table_month} {$website} = {$category}");
$q->QUERY_SQL("UPDATE {$table_month} SET category='{$category}' WHERE sitename='{$website}'");
if (!$q->ok) {
writelogs_squid("Re-categorized table {$table_month} failed {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "categorize");
}
events("Update {$table_week} {$website} = {$category}");
$q->QUERY_SQL("UPDATE {$table_week} SET category='{$category}' WHERE sitename='{$website}'");
if (!$q->ok) {
writelogs_squid("Re-categorized table {$table_week} failed {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "categorize");
}
events("Update {$table_blocked} {$website} = {$category}");
$q->QUERY_SQL("UPDATE {$table_blocked} SET category='{$category}' WHERE website='{$website}'");
if (!$q->ok) {
writelogs_squid("Re-categorized table {$table_blocked} failed {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "categorize");
}
events("Update {$table_week_blocked} {$website} = {$category}");
if ($q->CreateWeekBlockedTable($table_week_blocked)) {
}
$q->QUERY_SQL("UPDATE {$table_week_blocked} SET category='{$category}' WHERE website='{$website}'");
if (!$q->ok) {
writelogs_squid("Re-categorized table {$table_week_blocked} failed {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "categorize");
}
}
$took = $unix->distanceOfTimeInWords($t, time());
if ($f > 0) {
ufdbguard_admin_events("Re-categorized table {$table} with {$f} websites ({$took})", __FUNCTION__, __FILE__, __LINE__, "statistics");
}
if ($GLOBALS["VERBOSE"]) {
echo "recategorize_singleday({$day}) FINISH\n";
}
}
function import_translated()
{
$q = new mysql_squid_builder();
$MAINZ = unserialize(@file_get_contents("/root/translated"));
$translated_done = unserialize(@file_get_contents("/root/translated_done"));
$max = count($MAINZ);
$gg = new generic_categorize();
$i = 1;
while (list($www, $category) = each($MAINZ)) {
echo "{$i}/{$max} {$www} ";
$i++;
if (isset($translated_done[$www])) {
echo "\n";
continue;
}
$category_artica = $gg->GetCategories($www);
if ($category_artica != null) {
echo "-> ARTICA {$category_artica}\n";
$q->categorize($www, $category_artica);
$translated_done[$www] = true;
@file_put_contents("/root/translated_done", serialize($translated_done));
continue;
}
$category_artica = $q->GET_CATEGORIES($www, true, true, true, true);
if ($category_artica != null) {
echo "-> ARTICA {$category}<>{$category_artica}\n";
$translated_done[$www] = true;
@file_put_contents("/root/translated_done", serialize($translated_done));
continue;
}
echo "{$category}\n";
$q->categorize($www, $category);
$translated_done[$www] = true;
@file_put_contents("/root/translated_done", serialize($translated_done));
}
}
function visited_list()
{
$tpl = new templates();
$MyPage = CurrentPageName();
$q = new mysql_squid_builder();
$family = $q->GetFamilySites($_GET["www"]);
$search = '%';
$table = "visited_sites";
$page = 1;
$COUNT_ROWS = $q->COUNT_ROWS($table);
$FORCE_FILTER = "familysite='{$family}'";
if ($COUNT_ROWS == 0) {
json_error_show("No data...");
}
if (isset($_POST["sortname"])) {
if ($_POST["sortname"] != null) {
$ORDER = "ORDER BY {$_POST["sortname"]} {$_POST["sortorder"]}";
}
}
if (isset($_POST['page'])) {
$page = $_POST['page'];
}
$_POST["query"] = trim($_POST["query"]);
if ($_POST["query"] != null) {
$_POST["query"] = "*{$_POST["query"]}*";
$_POST["query"] = str_replace("**", "*", $_POST["query"]);
$_POST["query"] = str_replace("**", "*", $_POST["query"]);
$_POST["query"] = str_replace("*", "%", $_POST["query"]);
$search = $_POST["query"];
$searchstring = "AND (`{$_POST["qtype"]}` LIKE '{$search}')";
$sql = "SELECT COUNT(*) as TCOUNT FROM `{$table}` WHERE {$FORCE_FILTER} {$searchstring}";
writelogs($sql, __FUNCTION__, __FILE__, __LINE__);
$ligne = mysql_fetch_array($q->QUERY_SQL($sql, "artica_backup"));
$total = $ligne["TCOUNT"];
} else {
$sql = "SELECT COUNT(*) as TCOUNT FROM `{$table}` WHERE {$FORCE_FILTER}";
writelogs($sql, __FUNCTION__, __FILE__, __LINE__);
$ligne = mysql_fetch_array($q->QUERY_SQL($sql, "artica_backup"));
$total = $ligne["TCOUNT"];
}
if (isset($_POST['rp'])) {
$rp = $_POST['rp'];
}
$pageStart = ($page - 1) * $rp;
$limitSql = "LIMIT {$pageStart}, {$rp}";
$sql = "SELECT * FROM `{$table}` WHERE {$searchstring} {$FORCE_FILTER} {$ORDER} {$limitSql}";
writelogs($sql, __FUNCTION__, __FILE__, __LINE__);
$results = $q->QUERY_SQL($sql);
$data = array();
$data['page'] = $page;
$data['total'] = $total;
$data['rows'] = array();
if (mysql_num_rows($results) == 0) {
$data['rows'][] = array('id' => $ligne[time()], 'cell' => array($sql, "", "", ""));
}
while ($ligne = mysql_fetch_assoc($results)) {
$categories = $q->GET_CATEGORIES($ligne['sitename']);
$FamilySite = $q->GetFamilySites($ligne['sitename']);
$categorize = imgtootltip("add-database-32.png", $ligne['sitename'], "javascript:Loadjs('squid.categorize.php?www={$ligne['sitename']}&day={$_GET["day"]}&week={$_GET["week"]}&month={$_GET["month"]}');");
$thumbs = $q->GET_THUMBNAIL($ligne['sitename'], 48);
$ahref = "Loadjs('squid.traffic.statistics.days.php?today-zoom=yes&type=req&familysite={$FamilySite}&day={$_GET["day"]}')";
$data['rows'][] = array('id' => $ligne['sitename'], 'cell' => array($thumbs, "<div style='margin-top:10px'><a href=\"javascript:blur();\" OnClick=\"javascript:{$ahref}\" style='font-size:14px;text-decoration:underline'>{$ligne['sitename']}</a></div>", "<div style='font-size:14px;margin-top:10px'>{$categories}</span>", $categorize));
}
echo json_encode($data);
}
function build_report($ID, $nopid = false)
{
if (!is_numeric($ID)) {
ufdbguard_admin_events("Not a numeric ID", __FUNCTION__, __FILE__, __LINE__, "reports");
return;
}
$t = time();
$unix = new unix();
$tablename = "WebTrackMem{$ID}";
$tableBlock = "WebTrackMeB{$ID}";
if (!$nopid) {
$pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".{$ID}.pid";
$pid = @file_get_contents($pidfile);
if ($pid < 100) {
$pid = null;
}
if ($unix->process_exists($pid, basename(__FILE__))) {
ufdbguard_admin_events("Already executed pid {$pid}", __FUNCTION__, __FILE__, __LINE__, "reports");
if ($GLOBALS["VERBOSE"]) {
echo "Already executed pid {$pid}\n";
}
return;
}
}
if ($GLOBALS["VERBOSE"]) {
echo "Building report {$ID}\n";
}
$q = new mysql_squid_builder();
if ($q->TABLE_EXISTS($tablename)) {
$q->DELETE_TABLE($tablename);
}
if ($q->TABLE_EXISTS($tableBlock)) {
$q->DELETE_TABLE($tableBlock);
}
if (!$q->CreateMemberReportTable($tablename)) {
ufdbguard_admin_events("could not create table {$tablename}", __FUNCTION__, __FILE__, __LINE__, "reports");
return;
}
if (!$q->CreateMemberReportBlockTable($tableBlock)) {
ufdbguard_admin_events("could not create table {$tablename}", __FUNCTION__, __FILE__, __LINE__, "reports");
return;
}
$rp = new squid_report($ID);
$LIST_TABLES_dansguardian_events = $q->LIST_TABLES_dansguardian_events();
progress(10, $ID);
$counttables = count($LIST_TABLES_dansguardian_events);
$prefix = "INSERT IGNORE INTO {$tablename} (`zMD5`,`sitename`,`familysite`,`{$rp->userfield}`,`zDate`,`size`,`hits`,`category`) VALUES ";
while (list($sourcetable, $ligne) = each($LIST_TABLES_dansguardian_events)) {
$c++;
$sql = $rp->BuildQuery($sourcetable);
$results = $q->QUERY_SQL($sql);
if (!$q->ok) {
ufdbguard_admin_events("{$q->mysql_error}\n{$sql}", __FUNCTION__, __FILE__, __LINE__, "reports");
return;
}
if ($GLOBALS["VERBOSE"]) {
echo "Parsing {$sourcetable} \n{$sql}\n-> `" . mysql_num_rows($results) . "` rows\n";
}
if (mysql_num_rows($results) == 0) {
continue;
}
$purc = round($c / $counttables, 2) * 100;
progress($purc, $ID);
$f = array();
while ($ligne = @mysql_fetch_array($results, MYSQL_ASSOC)) {
$md5 = md5(serialize($ligne));
$sitename = $ligne["sitename"];
$familysite = $q->GetFamilySites($sitename);
if (!isset($GLOBALS["CATEGORY"][$sitename])) {
$GLOBALS["CATEGORY"][$sitename] = $q->GET_CATEGORIES($sitename);
}
$category = $GLOBALS["CATEGORY"][$sitename];
$source = addslashes($ligne["source"]);
$zDate = $ligne["zDate"];
$size = $ligne["size"];
$hits = $ligne["hits"];
$category = addslashes($category);
$f[] = "('{$md5}','{$sitename}','{$familysite}','{$source}','{$zDate}','{$size}','{$hits}','{$category}')";
}
if (count($f) == 0) {
continue;
}
$q->QUERY_SQL($prefix . @implode(",", $f));
if (!$q->ok) {
ufdbguard_admin_events("{$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "reports");
return;
}
if (system_is_overloaded(__FILE__)) {
sleep(5);
}
}
if ($rp->csv == 1) {
SaveCSV($ID);
}
$LIST_TABLES_BLOCKED = $q->LIST_TABLES_BLOCKED();
$prefix = "INSERT IGNORE INTO {$tableBlock} (`zMD5`,`zDate`,`hits`,`website`,`category`,`rulename`,`event`,`why`,`explain`,`blocktype`,`{$rp->userfield}`) VALUES ";
while (list($sourcetable, $ligne) = each($LIST_TABLES_BLOCKED)) {
$c++;
if ($GLOBALS["VERBOSE"]) {
echo "Parsing {$sourcetable}\n";
}
$sql = $rp->BuildQueryBlock($sourcetable);
if (!$q->FIELD_EXISTS("{$sourcetable}", "uid")) {
$q->QUERY_SQL("ALTER TABLE `{$sourcetable}` ADD `uid` VARCHAR( 128 ) NOT NULL ,ADD INDEX ( `uid` )");
}
$results = $q->QUERY_SQL($sql);
if (!$q->ok) {
ufdbguard_admin_events("{$q->mysql_error}\n{$sql}", __FUNCTION__, __FILE__, __LINE__, "reports");
return;
}
if (mysql_num_rows($results) == 0) {
continue;
}
$purc = round($c / $counttables, 2) * 100;
progress($purc, $ID);
$f = array();
while ($ligne = @mysql_fetch_array($results, MYSQL_ASSOC)) {
$zMD5 = md5(serialize($ligne));
$sitename = $ligne["sitename"];
$category = $ligne["category"];
if ($category == null) {
if (!isset($GLOBALS["CATEGORY"][$sitename])) {
$GLOBALS["CATEGORY"][$sitename] = $q->GET_CATEGORIES($sitename);
}
$category = $GLOBALS["CATEGORY"][$sitename];
}
$source = addslashes($ligne["source"]);
$zDate = $ligne["zDate"];
$hits = $ligne["hits"];
$rulename = $ligne["rulename"];
$event = $ligne["event"];
$why = $ligne["why"];
$explain = $ligne["explain"];
$blocktype = $ligne["blocktype"];
$category = addslashes($category);
$f[] = "('{$zMD5}','{$zDate}','{$hits}','{$sitename}','{$category}','{$rulename}','{$event}','{$why}','{$explain}','{$blocktype}','{$source}')";
}
if (system_is_overloaded(__FILE__)) {
sleep(5);
}
}
$myisamchk = $unix->find_program("myisamchk");
$myisampack = $unix->find_program("myisampack");
$mysql_data = $unix->MYSQL_DATA_DIR();
echo "OPTIMIZE TABLE {$tableBlock}\n";
$q->QUERY_SQL("OPTIMIZE TABLE {$tableBlock}");
echo "OPTIMIZE TABLE {$tablename}\n";
$q->QUERY_SQL("OPTIMIZE TABLE {$tablename}");
echo "LOCK TABLE {$tablename}\n";
$q->QUERY_SQL("LOCK TABLE {$tablename} WRITE");
echo "LOCK TABLE {$tableBlock}\n";
$q->QUERY_SQL("LOCK TABLE {$tableBlock} WRITE");
$q->QUERY_SQL("FLUSH TABLE {$tableBlock}");
$q->QUERY_SQL("FLUSH TABLE {$tablename}");
echo "myisamchk {$tablename}\n";
shell_exec("{$myisamchk} -cFU {$mysql_data}/squidlogs/{$tablename}.MYI");
echo "myisamchk {$tableBlock}\n";
shell_exec("{$myisamchk} -cFU {$mysql_data}/squidlogs/{$tableBlock}.MYI");
echo "myisampack {$tablename}\n";
shell_exec("{$myisampack} -f {$mysql_data}/squidlogs/{$tablename}.MYI");
echo "myisampack {$tableBlock}\n";
shell_exec("{$myisampack} -f {$mysql_data}/squidlogs/{$tableBlock}.MYI");
$q->QUERY_SQL("FLUSH TABLE {$tablename}");
$q->QUERY_SQL("FLUSH TABLE {$tableBlock}");
$rp->set_duration($unix->distanceOfTimeInWords($t, time(), true));
progress(100, $ID);
}
function MoveCategorizedWebsite($md5 = null, $nextCategory = null, $table = null)
{
$q = new mysql_squid_builder();
$sock = new sockets();
if ($md5 == null) {
$md5 = $_POST["MoveCategorizedWebsite"];
}
if ($nextCategory == null) {
$nextCategory = trim($_POST["NextCategory"]);
}
if ($table == null) {
$table = trim($_POST["TABLE"]);
}
if ($nextCategory == null) {
echo "Next category = Null\n";
return;
}
if ($table == null) {
echo "Table = Null\n";
return;
}
if ($md5 == null) {
echo "md5 = Null\n";
return;
}
if (!isset($GLOBALS["uuid"])) {
$GLOBALS["uuid"] = base64_decode($sock->getFrameWork("cmd.php?system-unique-id=yes"));
}
$uuid = $GLOBALS["uuid"];
$sql = "SELECT * FROM {$table} WHERE zmd5='{$md5}'";
$ligne = mysql_fetch_array($q->QUERY_SQL($sql));
if (!$q->ok) {
echo $q->mysql_error;
return;
}
$www = $ligne["pattern"];
$sql = "INSERT IGNORE INTO categorize_delete (sitename,category,zmd5) VALUES ('{$ligne["pattern"]}','{$ligne["category"]}','{$ligne["zmd5"]}')";
$q->QUERY_SQL($sql);
if (!$q->ok) {
echo $q->mysql_error;
return;
}
$q->QUERY_SQL("UPDATE {$table} SET enabled=0 WHERE zmd5='{$md5}'");
if (!$q->ok) {
echo $q->mysql_error;
return;
}
$sock->getFrameWork("squid.php?export-deleted-categories=yes");
$newmd5 = md5($www . $nextCategory);
$q->QUERY_SQL("INSERT IGNORE INTO categorize_changes (zmd5,sitename,category) VALUES('{$newmd5}','{$www}','{$nextCategory}')");
if (!$q->ok) {
echo $q->mysql_error;
return;
}
$q->CreateCategoryTable($nextCategory);
$category_table = $q->category_transform_name($nextCategory);
$q->QUERY_SQL("INSERT IGNORE INTO categorize (zmd5,zDate,category,pattern,uuid) VALUES('{$newmd5}',NOW(),'{$nextCategory}','{$www}','{$uuid}')");
if (!$q->ok) {
echo $q->mysql_error;
return;
}
$q->QUERY_SQL("INSERT IGNORE INTO category_{$category_table} (zmd5,zDate,category,pattern,uuid,enabled) VALUES('{$md5}',NOW(),'{$nextCategory}','{$www}','{$uuid}',1)");
if (!$q->ok) {
echo $q->mysql_error;
return;
}
$cats = addslashes($q->GET_CATEGORIES($www, true));
$q->QUERY_SQL("UPDATE visited_sites SET category='{$cats}' WHERE sitename='{$www}'");
if (!$q->ok) {
echo $q->mysql_error . "\n";
echo $sql . "\n";
}
$sock = new sockets();
$sock->getFrameWork("cmd.php?export-community-categories=yes");
$sock->getFrameWork("squid.php?re-categorize=yes");
}
function CheckWebsites($data, $category)
{
$founds = array();
if (!preg_match_all("#Domain:(.*?)<#s", $data, $founds)) {
echo "NO MATCH scan 1.....\n";
}
$count = count($founds[1]);
if ($count == 0) {
if (!preg_match_all("#Name:(.*?)<#s", $data, $founds)) {
echo "NO MATCH scan 2.....\n";
}
}
$count = count($founds[1]);
if ($count == 0) {
return;
}
echo strlen($curl->data) . " bytes lenth...found: {$count} items\n";
$q = new mysql_squid_builder();
if ($count > 0) {
while (list($id, $www) = each($founds[1])) {
$www = trim($www);
$www = str_replace("\t", "", $www);
$www = str_replace(chr(194), "", $www);
$www = str_replace(chr(32), "", $www);
$www = str_replace(chr(160), "", $www);
if (!preg_match("#([a-z0-9\\-_\\.]+)\\.([a-z]+)\$#i", $www, $re)) {
echo "{$www} skiped L." . __LINE__ . "\n";
continue;
}
if (strpos($www, ",") > 0) {
echo "{$www} skiped L." . __LINE__ . "\n";
continue;
}
if (strpos($www, " ") > 0) {
echo "{$www} skiped L." . __LINE__ . "\n";
continue;
}
if (strpos($www, ":") > 0) {
echo "{$www} skiped L." . __LINE__ . "\n";
continue;
}
if (strpos($www, "%") > 0) {
echo "{$www} skiped L." . __LINE__ . "\n";
continue;
}
if (preg_match("#^www\\.(.+)#", $www, $re)) {
$www = $re[1];
}
$articacats = trim($q->GET_CATEGORIES($www, true, true));
if ($articacats != null) {
echo "\"{$www}\" already in {$articacats}\n";
}
$newsWeb[] = $www;
}
}
while (list($id, $www) = each($newsWeb)) {
echo "Adding \"{$www}\" -> {$category}\n";
$q->ADD_CATEGORYZED_WEBSITE($www, $category);
}
}
function websites_uid_not_categorised($uid = null, $tablename = null, $aspid = false)
{
if (isset($GLOBALS["websites_uid_not_categorised_{$uid}"])) {
return;
}
$unix = new unix();
$pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".{$uid}.pid";
if ($aspid) {
$pid = @file_get_contents($pidfile);
$myfile = basename(__FILE__);
if ($unix->process_exists($pid, $myfile)) {
ufdbguard_admin_events("Task already running PID: {$pid}, aborting current task", __FUNCTION__, __FILE__, __LINE__, "stats");
return;
}
}
$mypid = getmypid();
@file_put_contents($pidfile, $mypid);
$q = new mysql_squid_builder();
if ($uid != null) {
$uidtable = $q->uid_to_tablename($uid);
$tablename = "www_{$uidtable}";
}
if (!$q->FIELD_EXISTS($tablename, "category")) {
$q->QUERY_SQL("ALTER TABLE `{$tablename}` ADD `category` varchar(255), ADD INDEX (`category`)");
}
$sql = "SELECT familysite,`category` FROM `{$tablename}` GROUP BY familysite,`category` HAVING `category` IS NULL ";
$results = $q->QUERY_SQL($sql);
if (!$q->ok) {
if ($GLOBALS["VERBOSE"]) {
echo "############# ERROR #########\n{$q->mysql_error}\\Line:" . __LINE__ . "\n#############\n";
}
return false;
}
$c = 0;
$mysql_num_rows = mysql_num_rows($results);
if ($mysql_num_rows == 0) {
if ($GLOBALS["VERBOSE"]) {
echo "{$sql} (No rows)\n";
}
return true;
}
while ($ligne = @mysql_fetch_array($results, MYSQL_ASSOC)) {
$sitename = $ligne["familysite"];
$IpClass = new IP();
if ($IpClass->isValid($sitename)) {
if (isset($GLOBALS["IPCACHE"][$sitename])) {
$t = time();
$sitename = gethostbyaddr($sitename);
events("{$tablename}: {$ligne["familysite"]} -> {$sitename} " . $unix->distanceOfTimeInWords($t, time()) . " gethostbyaddr() LINE:" . __LINE__);
$GLOBALS["IPCACHE"][$sitename] = $sitename;
}
}
$category = $q->GET_CATEGORIES($sitename);
if ($IpClass->isValid($sitename)) {
if ($category == null) {
$category = "ipaddr";
}
$q->categorize($sitename, $category);
}
events("{$tablename}: {$ligne["familysite"]} -> {$sitename} [{$category}] LINE:" . __LINE__);
if (strlen($category) > 0) {
$category = mysql_escape_string2($category);
$ligne["familysite"] = mysql_escape_string2($ligne["familysite"]);
$sql = "UPDATE `{$tablename}` SET `category`='{$category}' WHERE familysite='{$ligne["familysite"]}'";
$q->QUERY_SQL($sql);
if (!$q->ok) {
ufdbguard_admin_events("{$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "stats");
return;
}
}
}
}
function _not_categorized_months($table)
{
$t = time();
$unix = new unix();
$sql = "SELECT familysite FROM {$table} WHERE category='' GROUP BY familysite";
$q = new mysql_squid_builder();
$results = $q->QUERY_SQL($sql);
if (mysql_num_rows($results) == 0) {
return true;
}
$c = 0;
while ($ligne = mysql_fetch_assoc($results)) {
$category = $q->GET_CATEGORIES($ligne["familysite"]);
echo "{$table} {$ligne["familysite"]} -> `{$category}`\n";
if ($category == null) {
continue;
}
$q->QUERY_SQL("UPDATE {$table} SET category='{$category}' WHERE familysite='{$ligne["familysite"]}'");
$c++;
}
if ($c > 0) {
stats_admin_events(2, "{$table} {$c} websites categorized took:" . $unix->distanceOfTimeInWords($t, time()), null, __FILE__, __LINE__);
}
}
function CategorizeAll_perform()
{
if ($_GET["pattern"] == null) {
return;
}
$sock = new sockets();
$uuid = base64_decode($sock->getFrameWork("cmd.php?system-unique-id=yes"));
if ($uuid == null) {
echo "UUID=NULL; Aborting";
return;
}
$pattern = " AND sitename LIKE '%{$_GET["pattern"]}%' ";
$pattern = str_replace("*", "%", $pattern);
$sql = "SELECT sitename FROM `visited_sites` WHERE LENGTH( `category` )=0 {$pattern}";
$category = $_GET["CategorizeAll_category"];
if ($category == null) {
return;
}
$q = new mysql_squid_builder();
$results = $q->QUERY_SQL($sql, "artica_events");
if (!$q->ok) {
echo $q->mysql_error . "\n" . basename(__FILE__) . "\nLine" . __LINE__;
return;
}
$category_table = "category_" . $q->category_transform_name($category);
if (!$q->TABLE_EXISTS($category_table)) {
$q->CreateCategoryTable($category);
}
while ($ligne = mysql_fetch_array($results, MYSQL_ASSOC)) {
$website = $ligne["sitename"];
if ($website == null) {
return;
}
$www = trim(strtolower($website));
if (preg_match("#^www\\.(.+?)\$#i", trim($www), $re)) {
$www = $re[1];
}
$md5 = md5($www . $category);
$enabled = 1;
if ($www == null) {
echo "Alert: website is null...\n";
return;
}
$sql_add = "INSERT INTO {$category_table} (zmd5,zDate,category,pattern,uuid,enabled) VALUES('{$md5}',NOW(),'{$category}','{$www}','{$uuid}',1)";
$q->QUERY_SQL($sql_add);
if (!$q->ok) {
echo $q->mysql_error . "\n" . basename(__FILE__) . "\nLine" . __LINE__ . "\n";
echo $sql;
return;
}
$sql = "INSERT IGNORE INTO categorize (zmd5,zDate,category,pattern,uuid) VALUES('{$md5}',NOW(),'{$category}','{$www}','{$uuid}')";
$q->QUERY_SQL($sql);
if (!$q->ok) {
echo $q->mysql_error . "\n" . basename(__FILE__) . "\nLine" . __LINE__ . "\n";
echo $sql;
return;
}
if ($_GET["day"] != null) {
$time = strtotime($_GET["day"] . " 00:00:00");
$tableSrc = date('Ymd') . "_hour";
$categories = $q->GET_CATEGORIES($www, true);
$q->QUERY_SQL("UPDATE {$tableSrc} SET category='{$categories}' WHERE sitename='{$www}'");
}
}
$sql = "UPDATE `visited_sites` SET `category`='{$category}' WHERE LENGTH( `category` )=0 {$pattern}";
$q->QUERY_SQL($sql);
if (!$q->ok) {
echo $q->mysql_error . "\n" . basename(__FILE__) . "\nLine" . __LINE__ . "\n";
echo $sql;
return;
}
$sock = new sockets();
$sock->getFrameWork("cmd.php?export-community-categories=yes");
}
function popup_categories_sql()
{
//ini_set('html_errors',0);ini_set('display_errors', 1);ini_set('error_reporting', E_ALL);ini_set('error_prepend_string','');ini_set('error_append_string','');
$tpl = new templates();
$MyPage = CurrentPageName();
if ($GLOBALS["VERBOSE"]) {
echo __FUNCTION__ . "::" . __LINE__ . ":: mysql_squid_builder()\n<br>";
}
$q = new mysql_squid_builder();
$OnlyEnabled = false;
if (isset($_GET["OnlyEnabled"])) {
$OnlyEnabled = true;
}
$search = '%';
$table = "webfilters_categories_caches";
$page = 1;
$ORDER = "ORDER BY categorykey ASC";
if (!$q->TABLE_EXISTS($table)) {
if ($GLOBALS["VERBOSE"]) {
echo "<H2>Create create_webfilters_categories_caches()</H2>\n";
}
$q->create_webfilters_categories_caches();
}
$FORCE_FILTER = null;
if (trim($_GET["group"]) != null) {
$FORCE_FILTER = " AND master_category='{$_GET["group"]}'";
}
if ($GLOBALS["VERBOSE"]) {
echo "<h2>" . __FUNCTION__ . "::" . __LINE__ . ":: q->COUNT_ROWS({$table})</h2>\n<br>";
}
if ($q->COUNT_ROWS($table) == 0) {
$ss = new dansguardian_rules();
$ss->CategoriesTableCache();
}
$www = trim(strtolower($_GET["www"]));
$ArticaDBZ = new mysql_catz();
$CategoriesFound = $ArticaDBZ->GET_CATEGORIES($www);
$catArDB = explode(",", $CategoriesFound);
writelogs("ArticaDB({$www}) = " . @implode(",", $catArDB), __FUNCTION__, __FILE__, __LINE__);
if (is_array($catArDB)) {
while (list($num, $ligne) = each($catArDB)) {
$ligne = trim($ligne);
if ($ligne == null) {
continue;
}
$hash_ARTICA[$ligne] = true;
}
}
if (preg_match("#www\\.(.+?)\$#i", $www, $re)) {
$www = $re[1];
}
$q = new mysql_squid_builder();
$CategoriesFound = $q->GET_CATEGORIES($www, true, true, true, true);
$cats = explode(",", $CategoriesFound);
$www_encoded = base64_encode($_GET["www"]);
$COUNT_ROWS = $q->COUNT_ROWS($table);
$hash_community = array();
if (is_array($cats)) {
while (list($num, $ligne) = each($cats)) {
$ligne = trim($ligne);
if ($ligne == null) {
continue;
}
$hash_community[$ligne] = true;
}
}
if ($COUNT_ROWS == 0) {
$data['page'] = $page;
$data['total'] = $total;
$data['rows'] = array();
echo json_encode($data);
return;
}
if (isset($_POST["sortname"])) {
if ($_POST["sortname"] != null) {
$ORDER = "ORDER BY {$_POST["sortname"]} {$_POST["sortorder"]}";
}
}
if (isset($_POST['page'])) {
$page = $_POST['page'];
}
$_POST["query"] = trim($_POST["query"]);
if ($_POST["query"] != null) {
$_POST["query"] = "*{$_POST["query"]}*";
$_POST["query"] = str_replace("**", "*", $_POST["query"]);
$_POST["query"] = str_replace("**", "*", $_POST["query"]);
$_POST["query"] = str_replace("*", "%", $_POST["query"]);
$search = $_POST["query"];
$searchstring = "AND (`{$_POST["qtype"]}` LIKE '{$search}')";
$sql = "SELECT COUNT(*) as TCOUNT FROM `{$table}` WHERE 1 {$FORCE_FILTER} {$searchstring}";
$ligne = mysql_fetch_array($q->QUERY_SQL($sql, "artica_backup"));
$total = $ligne["TCOUNT"];
} else {
$total = $COUNT_ROWS;
}
if (isset($_POST['rp'])) {
$rp = $_POST['rp'];
}
$pageStart = ($page - 1) * $rp;
$limitSql = "LIMIT {$pageStart}, {$rp}";
if ($OnlyEnabled) {
$limitSql = null;
}
$sql = "SELECT * FROM `{$table}` WHERE 1 {$searchstring} {$FORCE_FILTER} {$ORDER} {$limitSql}";
writelogs($sql, __FUNCTION__, __FILE__, __LINE__);
$results = $q->QUERY_SQL($sql);
$data = array();
$data['page'] = $page;
$data['total'] = $total;
$data['rows'] = array();
if (mysql_num_rows($results) == 0) {
$data['rows'][] = array('id' => $ligne[time()], 'cell' => array($sql, "", "", ""));
}
$c = 0;
while ($ligne = mysql_fetch_assoc($results)) {
if ($ligne['categorykey'] == "phishtank") {
continue;
}
$DISABLED = false;
if ($ligne["picture"] == null) {
$ligne["picture"] = "20-categories-personnal.png";
}
$TextInterne = null;
$img = "img/{$ligne["picture"]}";
$val = 0;
if ($hash_community[$ligne['categorykey']]) {
$val = 1;
}
if ($hash_ARTICA[$ligne['categorykey']]) {
$TextInterne = $tpl->_ENGINE_parse_body("<div style='color:#D01313;font-size:11px;font-style:italic'>{categorized_in_articadb}</div>");
$val = 1;
$DISABLED = true;
}
$md = md5($ligne['categorykey']);
if ($OnlyEnabled) {
if ($val == 0) {
if ($TextInterne == null) {
continue;
}
}
}
$c++;
$js = "DansCommunityCategory('{$md}','{$ligne["categorykey"]}','{$www_encoded}')";
$disable = Field_checkbox($md, 1, $val, "{$js}", null, $DISABLED);
$ligne['description'] = utf8_encode($ligne['description']);
$data['rows'][] = array('id' => $ligne['categorykey'], 'cell' => array("<img src='{$img}'>", "{$ligne['categorykey']}", $TextInterne . $ligne['description'], $disable));
}
if ($OnlyEnabled) {
$data['total'] = $c;
}
echo json_encode($data);
}
function test_category_perform(){
$www=$_REQUEST["WEBTESTS"];
$tpl=new templates();
$q=new mysql_squid_builder();
$www=$q->WebsiteStrip($www);
if($www==null){
echo $tpl->_ENGINE_parse_body("<p class=text-error>{corrupted_request}: «{$_REQUEST["WEBTESTS"]}»</p>");
return;
}
$dans=new dansguardian_rules();
$dans->LoadBlackListes();
$catz=$q->GET_CATEGORIES($www,true);
if($catz==null){
echo $tpl->_ENGINE_parse_body("<p class=text-error>{unknown}: «{$_REQUEST["WEBTESTS"]}»</p>");
return;
}
if(strpos(" $catz", ",")>0){$CATs=explode(",", $catz);}else{$CATs[]=$catz;}
$sql="SELECT * FROM personal_categories";
if(!$q->TABLE_EXISTS("personal_categories")){json_error_show("personal_categories no such table!",1);}
$results=$q->QUERY_SQL($sql);
if(!$q->ok){json_error_show("Mysql Error [".__LINE__."]: $q->mysql_error",1);}
while($ligne=mysql_fetch_array($results,MYSQL_ASSOC)){$PERSONALSCATS[$ligne["category"]]=$ligne["category_description"];}
while (list ($num, $categoryname) = each ($CATs) ){
if(!isset($dans->array_blacksites[$categoryname])){
if(isset($dans->array_blacksites[str_replace("_","-",$categoryname)])){$categoryname=str_replace("_","-",$categoryname);}
if(isset($dans->array_blacksites[str_replace("_","/",$categoryname)])){$categoryname=str_replace("_","/",$categoryname);}
}
if($dans->array_pics[$categoryname]<>null){$pic="<img src='img/{$dans->array_pics[$categoryname]}'>";}else{$pic=" ";}
$text_category=$dans->array_blacksites[$categoryname];
if(isset($PERSONALSCATS[$categoryname])){
$text_category=utf8_encode($PERSONALSCATS[$categoryname]);
if($pic==" "){$pic="<img src='img/20-categories-personnal.png'>";}
}
$js="javascript:Loadjs('squid.categories.php?category=$categoryname&t=$t')";
$categoryText[]=$tpl->_ENGINE_parse_body("
<tr>
<td width=1% nowrap>$pic</td>
<td valign='top'>
<div style='font-size:18px';font-weight:bold'>
<a href=\"javascript:blur();\" OnClick=\"$js\" style='text-decoration:underline'>$categoryname</a>:</div>
<div style='font-size:16px;width:100%;font-weight:normal'>{$text_category}</div>
</td>
</tr>
");
}
$found=$tpl->_ENGINE_parse_body("{found}");
echo "<div style='width:95%:padding-left:50px;padding-top:20px' class=text-info><div style='font-size:18px'>$found</div><table>".@implode("\n", $categoryText)."</table></div>";
}
function recategorize_tables($zdate, $recategorize = 0, $report_id)
{
$xtime = strtotime("{$zdate} 00:00:00");
$tday = date("Ymd", $xtime);
$table_hour = "{$tday}_hour";
$q = new mysql_squid_builder();
$qr = new mysql_squid_reports($report_id);
$q->QUERY_SQL("UPDATE {$table_hour} SET category='' WHERE category=','");
$sql = "SELECT familysite FROM {$table_hour} GROUP BY familysite";
if ($recategorize == 1) {
$sql = "SELECT familysite FROM {$table_hour} WHERE LENGTH(category)=0 GROUP BY familysite";
}
$results = $q->QUERY_SQL("SELECT familysite FROM {$table_hour} GROUP BY familysite");
if (!$q->ok) {
echo $q->mysql_error . "\n";
}
echo "Recategorize table {$table_hour} " . mysql_num_rows($results) . "\n";
while ($ligne = @mysql_fetch_array($results, MYSQL_ASSOC)) {
$familysite = $ligne["familysite"];
$categories = trim($q->GET_CATEGORIES($familysite));
if ($categories == null) {
$qr->QUERY_SQL("INSERT IGNORE INTO nocatz (familysite) VALUE ('{$familysite}')");
continue;
}
echo "{$familysite} = `{$categories}`\n";
$q->QUERY_SQL("UPDATE {$table_hour} SET `category`='{$categories}' WHERE `familysite`='{$familysite}'");
}
}
function analyze()
{
if ($GLOBALS["VERBOSE"]) {
echo "analyze<br>\n";
}
$page = CurrentPageName();
$tpl = new templates();
$q = new mysql_squid_builder();
$uri = $_GET["uri"];
$curl = new ccurl($uri);
if ($GLOBALS["VERBOSE"]) {
echo "{$uri}<br>\n";
}
$filename = md5($uri);
if (!$curl->GetFile("ressources/logs/web/{$filename}")) {
echo "<H2>" . $curl->error . "</H2>";
return;
}
$ARRY = array();
if ($GLOBALS["VERBOSE"]) {
echo "Open ressources/logs/web/{$filename}<br>\n";
}
$datas = @file_get_contents("ressources/logs/web/{$filename}");
$tb = explode("\n", $datas);
@unlink("ressources/logs/web/{$filename}");
if (preg_match("#google\\..+?#", $uri)) {
if (preg_match_all('#<h3 class="r"><a href="(.+?)"#', $datas, $re)) {
while (list($num, $uri) = each($re[1])) {
if (preg_match("#^(?:[^/]+://)?([^/:]+)#", $uri, $ri)) {
$sitename = $ri[1];
if (substr($sitename, 0, 1) == "#") {
continue;
}
if (preg_match("#^www\\.(.+)#", $sitename, $ri)) {
$sitename = $ri[1];
}
if (preg_match("#\\.php\\?#", $sitename)) {
continue;
}
if (preg_match("#\\.php\$#", $sitename)) {
continue;
}
$ARRY[$sitename] = $sitename;
}
}
}
}
while (list($num, $line) = each($tb)) {
if (preg_match("#<a\\s+href=(.*)\\.([a-z]+)#i", $line, $re)) {
$uri = trim($re[1] . "." . $re[2]);
$uri = str_replace("\"", "", $uri);
$uri = str_replace("'", "", $uri);
if (strpos($uri, ">") > 0) {
$uri = substr($uri, 0, strpos($uri, ">"));
}
if (preg_match("#^(?:[^/]+://)?([^/:]+)#", $uri, $re)) {
$sitename = $re[1];
if (substr($sitename, 0, 1) == "#") {
continue;
}
if (preg_match("#^www\\.(.+)#", $sitename, $ri)) {
$sitename = $ri[1];
}
if (preg_match("#\\.php\\?#", $sitename)) {
continue;
}
if (preg_match("#\\.php\\s+[a-z]#", $sitename)) {
continue;
}
if (preg_match("#\\.php\$#", $sitename)) {
continue;
}
if (strpos($sitename, ".") == 0) {
continue;
}
if (strpos($sitename, "{") > 0) {
continue;
}
if (strpos($sitename, "}") > 0) {
continue;
}
if (strpos($sitename, "\$") > 0) {
continue;
}
$ARRY[trim(strtolower($sitename))] = trim(strtolower($sitename));
}
}
}
$html = "\n\t";
$f = 0;
$s = 0;
$t = 0;
while (list($num, $line) = each($ARRY)) {
if (strlen($num) < 3) {
continue;
}
if ($num == "javascript") {
continue;
}
if (strpos($num, ".") == 0) {
continue;
}
if (strpos($num, "{") > 0) {
continue;
}
if (strpos($num, " ") > 0) {
continue;
}
if (strpos($num, "}") > 0) {
continue;
}
if (strpos($num, "\$") > 0) {
continue;
}
$tz = explode(".", $num);
if ($tz[count($tz) - 1] == "php") {
continue;
}
if ($tz[count($tz) - 1] == "html") {
continue;
}
if ($tz[count($tz) - 1] == "htm") {
continue;
}
if (preg_match("#\\.php\\?#", $num)) {
continue;
}
if (preg_match("#\\.php\$#", $num)) {
continue;
}
$t++;
if ($t > 200) {
break;
}
$cats = $q->GET_CATEGORIES($num);
$mustcat = " ";
$color = "black";
if ($classtr == "oddRow") {
$classtr = null;
} else {
$classtr = "oddRow";
}
if ($cats == null) {
$tt = explode(".", $num);
$familysite = $tt[count($tt) - 2] . "." . $tt[count($tt) - 1];
$q->QUERY_SQL("INSERT IGNORE INTO visited_sites (sitename,familysite) VALUES ('{$num}','{$familysite}')");
$f++;
$cats = " ";
$mustcat = imgtootltip("add-database-32.png", "{categorize}", "Loadjs('squid.categorize.php?www={$num}')");
$color = "#CC0A0A";
} else {
$s++;
}
$html = $html . "\n\t\t<tr class={$classtr}>\n\t\t<td style='font-size:14px;color:{$color}'>{$num}</td>\n\t\t<td style='font-size:14px'>{$cats}</td>\n\t\t<td width=1%>{$mustcat}</td>\n\t\t</tr>\n\t\t";
}
//javascript:Loadjs('squid.categorize.php?www=api161.thefilter.com&day=&week=');
$purc = round($s / $t * 100, 2);
$html = "<center><table cellspacing='0' cellpadding='0' border='0' class='tableView' style='width:100%'>\n\t<thead class='thead'>\n\t<tr>\n\t<th>{$t} {websites} {$f} {failed} {$s} {success} {$purc}%</th>\n\t<th>{category}</th>\n\t<th> </th>\n\t</tr>\n</thead>\n<tbody>{$html}</tbody></table><span id='webalyzer-lock'></span>";
echo $tpl->_ENGINE_parse_body($html);
}
function CategorizeAll_perform(){
if($_GET["pattern"]==null){return;}
$sock=new sockets();
$uuid=base64_decode($sock->getFrameWork("cmd.php?system-unique-id=yes"));
if($uuid==null){echo "UUID=NULL; Aborting";return;}
$pattern=" AND sitename LIKE '%{$_GET["pattern"]}%' ";
$pattern=str_replace("*","%",$pattern);
$sql="SELECT sitename FROM `visited_sites` WHERE LENGTH( `category` )=0 $pattern";
$category=$_GET["CategorizeAll_category"];
if($category==null){return;}
if($category=="teans"){$category="teens";}
$q=new mysql_squid_builder();
$results=$q->QUERY_SQL($sql,"artica_events");
if(!$q->ok){
echo $q->mysql_error."\n".basename(__FILE__)."\nLine".__LINE__;
return;
}
if($_GET["week"]<>null){$_GET["day"]=$_GET["week"];}
$category_table="category_".$q->category_transform_name($category);
if(!$q->TABLE_EXISTS($category_table)){$q->CreateCategoryTable($category);}
while($ligne=mysql_fetch_array($results,MYSQL_ASSOC)){
$website=$ligne["sitename"];
if($website==null){return;}
$www=trim(strtolower($website));
if(preg_match("#^www\.(.+?)$#i",trim($www),$re)){$www=$re[1];}
$md5=md5($category.$www);
$enabled=1;
if($www==null){echo "Alert: website is null...\n";return;}
$sql_add="INSERT INTO $category_table (zmd5,zDate,category,pattern,uuid,enabled) VALUES('$md5',NOW(),'$category','$www','$uuid',1)";
$ligne=mysql_fetch_array($q->QUERY_SQL("SELECT category,pattern FROM `$category_table` WHERE zmd5='$md5'"));
if($ligne["category"]==null){$sql_add="UPDATE $category_table SET `category`='$category',pattern='$www' WHERE zmd5='$md5'";}
if($ligne["pattern"]==null){$sql_add="UPDATE $category_table SET `category`='$category',pattern='$www' WHERE zmd5='$md5'";}
$q->QUERY_SQL($sql_add);
if(!$q->ok){echo $q->mysql_error."\n".basename(__FILE__)."\nLine".__LINE__."\n";echo $sql;return;}
$sql="INSERT IGNORE INTO categorize (zmd5,zDate,category,pattern,uuid) VALUES('$md5',NOW(),'$category','$www','$uuid')";
$q->QUERY_SQL($sql);
if(!$q->ok){echo $q->mysql_error."\n".basename(__FILE__)."\nLine".__LINE__."\n";echo $sql;return;}
$sql="UPDATE `visited_sites` SET `category`='$category' WHERE LENGTH( `category` )=0 $pattern";
$q->QUERY_SQL($sql);
if(!$q->ok){echo $q->mysql_error."\n".basename(__FILE__)."\nLine".__LINE__."\n";echo $sql;return;}
if($_GET["day"]<>null){
if($pattern<>null){
$time=strtotime($_GET["day"]." 00:00:00");
$tableSrc=date('Ymd',$time)."_hour";
$categories=$q->GET_CATEGORIES($www,true);
if($categories==null){$categories=$category;}
$sql="UPDATE $tableSrc SET category='$categories' WHERE LENGTH( `category` )=0 $pattern";
$q->QUERY_SQL($sql);
writelogs($sql,__FUNCTION__,__FILE__,__LINE__);
if(!$q->ok){echo $q->mysql_error."\n".basename(__FILE__)."\nLine".__LINE__."\n";echo $sql;}
$tableWeek=date("YW",$time)."_week";
$sql="UPDATE $tableWeek SET category='$categories' WHERE LENGTH( `category` )=0 $pattern";
$q->QUERY_SQL($sql);
writelogs($sql,__FUNCTION__,__FILE__,__LINE__);
if(!$q->ok){echo $q->mysql_error."\n".basename(__FILE__)."\nLine".__LINE__."\n";echo $sql;}
$tableSrcDay=date('Ym',$time)."_day";
if($q->TABLE_EXISTS($tableSrcDay)){
$sql="UPDATE $tableSrcDay SET category='$categories' WHERE LENGTH( `category` )=0 $pattern";
writelogs($sql,__FUNCTION__,__FILE__,__LINE__);
$q->QUERY_SQL($sql);
if(!$q->ok){echo $q->mysql_error."\n".basename(__FILE__)."\nLine".__LINE__."\n";echo $sql;}
}
}else{
echo "Pattern is null!!\n";
}
}
}
$sock=new sockets();
$sock->getFrameWork("cmd.php?export-community-categories=yes");
}
function page()
{
$t = time();
$familysite = $_GET["familysite"];
$tpl = new templates();
$q = new mysql_squid_builder();
$page = CurrentPageName();
$categories = $q->GET_CATEGORIES($familysite);
$familysiteenc = urlencode($familysite);
$thumbs = $q->GET_THUMBNAIL($familysite, 320);
if (strpos($categories, ",") > 0) {
$categoriesZ = explode(",", $categories);
$tt[] = "<ul>";
while (list($num, $ligne) = each($categoriesZ)) {
$tt[] = "<li>{$ligne}</li>";
}
$tt[] = "</ul>";
$categories = @implode("", $tt);
}
$t = time();
$html = "\n\t<table style='width:100%' class=TableRemove>\n\t<tr>\n\t<td valign='top'>\n\t\t<div id='{$t}-img'>\n\t\t{$thumbs}\n\t\t</div>\n\t\t<center style='margin-top:10px'>" . button("{regenerate_thumbnail}", "LoadAjax('{$t}-img','{$page}?rethumbnail={$familysiteenc}');") . "</center>\t\n\t</td>\n\t<td valign='top' style='padding-left:20px'>\t\t\n\t\t<H3>{$familysite}</H3>\n\t\t<p><strong style='font-size:16px'>{this_website_was_categorized_in}:{$categories}</strong></p>\n\t\t<H3>{details}</H3>\n\t\t<div id='{$t}'></div>\n\t</td>\n\t</tr>\n\t</table>\n\t<script>\n\t\tLoadAjax('{$t}','{$page}?whois={$familysiteenc}');\n\t</script>\n\t\n\t";
echo $tpl->_ENGINE_parse_body($html);
}
if ($IpClass->isIPAddress($ligne["sitename"])) {
$q->QUERY_SQL("DELETE FROM webtests WHERE sitename='{$ligne["sitename"]}'");
$ligne["sitename"] = gethostbyaddr($ligne["sitename"]);
}
$familysite = $q->GetFamilySites($ligne["sitename"]);
if ($familysite == $ligne["sitename"]) {
$familysite = null;
}
if ($GLOBALS["OUTPUT"]) {
echo "{$www} -> STAMP\n";
}
$q->QUERY_SQL("UPDATE webtests SET checked=1 WHERE sitename='{$ligne["sitename"]}'");
if ($GLOBALS["OUTPUT"]) {
echo "{$www} -> GET_CATEGORIES\n";
}
$articacats = trim($q->GET_CATEGORIES($ligne["sitename"], true, false));
if ($GLOBALS["OUTPUT"]) {
echo "{$ligne["sitename"]} -> \"{$articacats}\"\n";
}
if ($articacats != null) {
$q->categorize($ligne["sitename"], $articacats);
$unix->_syslog("{$percent}) {$ligne["sitename"]} = {$articacats}", __FILE__);
continue;
}
if ($familysite != null) {
$articacats = trim($q->GET_CATEGORIES($familysite, true, false));
if ($GLOBALS["OUTPUT"]) {
echo "{$ligne["sitename"]} {$familysite} -> {$articacats}\n";
}
if ($articacats != null) {
$q->categorize($ligne["sitename"], $articacats);
