function WEBTESTS(){ $www=$_REQUEST["WEBTESTS"]; $q=new mysql_squid_builder(); $www=$q->WebsiteStrip($www); if($www==null){echo "corrupted\n";return;} $catz=str_replace(",", "\n- ", $q->GET_CATEGORIES($www,true)); echo "\nFinal:\n\"".$q->GET_CATEGORIES($www,true)."\"\n"; }
function CategorizeAWebSite($www, $category) { $md5 = md5($www . $category); $q = new mysql_squid_builder(); $q->CheckTables(); $uuid = $GLOBALS["UUID"]; $category_table = $q->category_transform_name($category); $sql_add = "INSERT IGNORE INTO categorize (zmd5,zDate,category,pattern,uuid) VALUES('{$md5}',NOW(),'{$category}','{$www}','{$uuid}')"; $sql_add2 = "INSERT IGNORE INTO category_{$category_table} (zmd5,zDate,category,pattern,uuid) VALUES('{$md5}',NOW(),'{$category}','{$www}','{$uuid}')"; $q->QUERY_SQL($sql_add); if (!$q->ok) { echo $q->mysql_error . "\n{$sql_add}\n"; return false; } $q->QUERY_SQL($sql_add2); if (!$q->ok) { echo $q->mysql_error . "\n{$sql_add2}\n"; return false; } $categories = $q->GET_CATEGORIES($www, true); if ($categories != null) { $sql = "UPDATE visited_sites SET category='{$categories}' WHERE sitename='{$www}'"; $q->QUERY_SQL($sql); if (!$q->ok) { echo $q->mysql_error . "\n{$sql}\n"; return false; } } }
function visited_sites() { $unix = new unix(); $pidfile = "/etc/artica-postfix/pids/squid.visited_sites_rescan.pid"; $pid = @file_get_contents($pidfile); if ($pid < 100) { $pid = null; } $t = time(); if ($unix->process_exists($pid, basename(__FILE__))) { $time = $unix->PROCCESS_TIME_MIN($pid); if ($GLOBALS["VERBOSE"]) { echo "Already executed pid {$pid} since {$time}mn\n"; } die; } $mypid = getmypid(); @file_put_contents($pidfile, $mypid); stats_admin_events(2, "Starting table visited_sites", "", __FILE__, __LINE__); progress("Starting table visited_sites", 5); $q = new mysql_squid_builder(); $sql = "SELECT sitename FROM visited_sites WHERE LENGTH(category)=0"; $results = $q->QUERY_SQL($sql); $num_rows = mysql_num_rows($results); $took = $unix->distanceOfTimeInWords($t, time()); stats_admin_events(2, "Query done {$num_rows} websites to scan", "took:{$took}", __FILE__, __LINE__); if ($num_rows == 0) { progress(null, 100); return; } progress("Query done {$num_rows} websites to scan", 10); $c = 0; $t = 0; $d = 0; while ($ligne = @mysql_fetch_array($results, MYSQL_ASSOC)) { $sitenameOrg = $ligne["sitename"]; $sitename = strtolower(trim($sitenameOrg)); if (badCharacters($sitename)) { $q->categorize_reaffected($sitename); $sitenameOrg = mysql_escape_string2($sitenameOrg); $q->QUERY_SQL("UPDATE visited_sites SET category='reaffected' WHERE `sitename`='{$sitenameOrg}'"); if (!$q->ok) { progress("Fatal", 100); die; } $d++; $c++; continue; } if (strpos($sitename, ".") == 0) { $q->categorize_reaffected($sitename); $sitenameOrg = mysql_escape_string2($sitenameOrg); $q->QUERY_SQL("UPDATE visited_sites SET category='reaffected' WHERE `sitename`='{$sitenameOrg}'"); if (!$q->ok) { progress("Fatal", 100); die; } $d++; $c++; continue; } if (preg_match("#[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $sitename)) { $sitename = gethostbyaddr($sitename); if (preg_match("#[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $sitename)) { $c++; continue; } } $ipaddr = gethostbyname($sitename); if (!preg_match("#[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$#", $ipaddr)) { $q->categorize_reaffected($sitenameOrg); $q->QUERY_SQL("UPDATE visited_sites SET category='reaffected' WHERE `sitename`='{$sitenameOrg}'"); if (!$q->ok) { progress("Fatal", 100); die; } $d++; $c++; continue; } $cat = $q->GET_CATEGORIES($sitename); if ($cat != null) { $d++; $q->QUERY_SQL("UPDATE visited_sites SET category='{$cat}' WHERE `sitename`='{$sitenameOrg}'"); if (!$q->ok) { progress("Fatal", 100); die; } } $c++; if ($c > 50) { $t = $t + $c; $purc = $t / $num_rows; $purc = round($purc, 2) * 100; $c = 0; if ($purc > 10) { progress("{$sitename} {$t}/{$num_rows}", $purc); } } } progress($sitename, 100); $took = $unix->distanceOfTimeInWords($t, time()); if ($d > 0) { stats_admin_events(2, "{$d} New categorized websites...", "took:{$took}", __FILE__, __LINE__); } }
function GetDomains($i) { $unix = new unix(); $tmpfile = $unix->FILE_TEMP(); $curl = new ccurl("http://0675.com.cn/newwebsite_20120101_list._page0.html"); $curl->NoHTTP_POST = true; echo "Get page {$i}\n"; if (!$curl->GetFile($tmpfile)) { echo "http://0675.com.cn/newwebsite_20120101_list._page{$i}.html -> error: \n" . $curl->error . "\n"; return; } $datas = @file($tmpfile); $size = strlen(@implode("", $datas)) / 1024; echo "Page[{$i}]:: {$tmpfile} Size: {$size} Ko\n"; while (list($num, $ligne) = each($datas)) { if (preg_match("#<div class=.*?newdomain.*?>(.*)#", $ligne)) { echo "Sure line {$num}"; $newdata = str_replace("</li>", "", $ligne); $newdata = str_replace("</div>", "", $newdata); $f = explode("<li>", $newdata); } } $q = new mysql_squid_builder(); while (list($num, $www) = each($f)) { if (preg_match("#^\\.(.+)#", $www, $re)) { $www = $re[1]; } if (strpos($www, ",") > 0) { continue; } if (strpos($www, " ") > 0) { continue; } if (strpos($www, ":") > 0) { continue; } if (strpos($www, "%") > 0) { continue; } if (strpos($www, ">") > 0) { continue; } if (strpos($www, "<") > 0) { continue; } if (preg_match("#^www\\.(.+)#", $www, $re)) { $www = $re[1]; } $articacats = trim($q->GET_CATEGORIES($www, true, false)); if ($articacats != null) { echo "\"{$www}\" SUCCESS - {$articacats} -\n"; continue; } echo "\"{$www}\" FAILED\n"; } }
function ParseSitesInfos_artica_category($sitename) { $q = new mysql_squid_builder(); return $q->GET_CATEGORIES($sitename, true); }
function site_infos_categories() { $q = new mysql_squid_builder(); $tpl = new templates(); $page = CurrentPageName(); include_once dirname(__FILE__) . "/ressources/class.dansguardian.inc"; $delete = $tpl->javascript_parse_text("{delete}"); $from = $tpl->javascript_parse_text("{from}"); $do_you_want_to_recompile_database = $tpl->javascript_parse_text("{do_you_want_to_recompile_database}"); $success = $tpl->javascript_parse_text("{sucess}"); $t = time(); $catz = $q->GET_CATEGORIES($_GET["search-stats-categories"]); if ($catz != null) { if (strpos($catz, ",") > 0) { $cats = explode(",", $catz); } else { $cats[] = $catz; } } $catNum = count($cats); if ($catNum == 0) { $html = "\n\t\t\n\t\t<table style='width:99%'>\n\t\t\t<tbody><tr><td width=1%>\n\t\t\t\t<img src='img/warning-panneau-32.png'></td>\n\t\t\t\t<td width=99% style='font-size:13px;font-weight:bold'>{this_website_hasnocat}</td>\n\t\t\t\t</tr></tbody></table>"; echo $tpl->_ENGINE_parse_body("{$html}"); return; } $dd = new dansguardian_rules(); $catstext = $tpl->_ENGINE_parse_body("{this_website_is_categorized_inXX}"); $catstext = str_replace("XX", count($cats), $catstext); https: //192.168.1.106:9000/squid.categories.php?query=yes&category=p**n&search=megavideo.&strictSearch=0 $html = "<table style='width:99%'>\n\t\t<tbody>\n\t\t<tr>\n\t\t\t<td colspan=2 style='font-size:13px'>{$catstext}</td>\n\t\t</tr>\n\t"; while (list($a, $b) = each($cats)) { if ($b == null) { continue; } $img = $dd->array_pics[$b]; if ($img == null) { $img = "20-categories-personnal.png"; } $id = md5($b); $html = $html . "<tr id='tt{$id}'>\n\t\t\t<td width=1%><img src='img/{$img}'></td>\n\t\t\t<td style='font-size:13px;font-weight:bold'>\n\t\t\t<a href=\"javascript:blur();\" \n\t\t\tOnClick=\"javascript:Loadjs('squid.categorize.php?www={$_GET["search-stats-categories"]}&day=&week=&month=');\" \n\t\t\tstyle='font-size:13px;text-decoration:underline;;font-weight:bold'>{$b}</td>\n\t\t\t</tr>"; } $html = $html . "</tbody></table>\n\t\n\t<script>\n\tvar MEMCAT{$t}='';\n\tvar MEMID{$t}='';\n\t\n\t\tvar x_DeleteCategoryFromSite{$t}= function (obj) {\n\t\t\tvar results=obj.responseText;\n\t\t\tif(results.length>0){alert(results);return;}\n\t\t\t\$('#tt'+ MEMID{$t}).remove();\n\t\t\tif(confirm('{$do_you_want_to_recompile_database} ?')){\n\t\t\t\tvar XHR = new XHRConnection();\n\t\t\t\tXHR.appendData('action_compile_category','yes');\n\t\t\t\tXHR.appendData('category',MEMCAT{$t});\n\t\t\t\tXHR.sendAndLoad('{$page}', 'POST');\t\t\t\n\t\t\t\talert('{$success}');\n\t\t\t}\n\t\t\t\n\t\t}\t\t\n\t\n\t\n\t\tfunction DeleteCategoryFromSite{$t}(category,www,id){\n\t\t\tMEMID{$t}=id;\n\t\t\tif(confirm('{$delete} '+category+' {$from} '+www+' ?')){\n\t\t\t\tvar XHR = new XHRConnection();\n\t\t\t\tMEMCAT{$t}=category;\n\t\t\t\tXHR.appendData('action_delete_from_category','yes');\n\t\t\t\tXHR.appendData('sitename',www);\n\t\t\t\tXHR.appendData('category',category);\n\t\t\t\tXHR.sendAndLoad('{$page}', 'POST',x_DeleteCategoryFromSite{$t});\n\t\t\t}\n\t\t\n\t\t}\n\t</script>\n\t"; echo $tpl->_ENGINE_parse_body("{$html}"); return; }
echo "No changes\n"; die; } $table_hours = $q->LIST_TABLES_HOURS(); $table_days = $q->LIST_TABLES_DAYS(); $table_week = $q->LIST_TABLES_WEEKS(); $ALREADY = array(); $MAX = mysql_num_rows($results); $D = 0; while ($ligne = @mysql_fetch_array($results, MYSQL_ASSOC)) { if (isset($ALREADY[$ligne["sitename"]])) { $q->QUERY_SQL("DELETE FROM categorize_changes WHERE `zmd5`='{$ligne["zmd5"]}'"); continue; } $website = $ligne["sitename"]; $categories = $q->GET_CATEGORIES($website, true); $ALREADY[$ligne["sitename"]] = true; reset($table_hours); reset($table_days); reset($table_week); $categories = addslashes($categories); $D++; $t = time(); $a = 0; $b = 0; $c = 0; while (list($num, $table) = each($table_hours)) { $a++; if ($GLOBALS["VERBOSE"]) { echo "Update {$table}\n"; }
function popup_categories() { $tpl = new templates(); if (trim($_GET["www"]) == null) { $error_no_website_selected = $tpl->javascript_parse_text("{error_no_website_selected}"); echo "\n\t\t<script>\n\t\t\talert('{$error_no_website_selected}');\n\t\t\tYahooWinBrowseHide();\n\t\t</script>\n\t\t"; return; } $www = trim(strtolower($_GET["www"])); if (preg_match("#www\\.(.+?)\$#i", $www, $re)) { $www = $re[1]; } $q = new mysql_squid_builder(); $cats = explode(",", $q->GET_CATEGORIES($www)); if (is_array($cats)) { while (list($num, $ligne) = each($cats)) { $ligne = trim($ligne); if ($ligne == null) { continue; } $hash_community[$ligne] = true; } } $www_encoded = base64_encode($_GET["www"]); $count = count($hash); $count_community = count($hash_community); $html = "\n\t<div style='font-size:13px'>{dansguardian_categorize_explain}</div>\n\t<hr>\n\t<div style='font-size:13px;color:red'>{$count_community} {categoryies_community}</div>\n\t<hr>\n\t<div style='height:490px;overflow:auto;margin:9px'>\n\t<table style='width:100%'>\n\t"; $dans = new dansguardian_rules(); while (list($num, $val) = each($dans->array_blacksites)) { $md = md5($num); $field_enabled = 0; if ($hash_community[$num]) { $field_enabled = 1; } $field = Field_checkbox("{$md}", 1, $field_enabled, "DansCommunityCategory('{$md}','{$num}','{$www_encoded}')"); if ($dans->array_pics[$num] != null) { $pic = "<img src='img/{$dans->array_pics[$num]}'>"; } else { $pic = " "; } $color = "black"; if ($hash[$num]) { $field = "<img src='img/check2.gif'>"; $color = "red"; } $html = $html . "\n\t\t<tr " . CellRollOver() . ">\n\t\t\t<td width=1%>{$pic}</td>\n\t\t\t<td><strong style='font-size:11px;color:{$color}'>{$val}</td>\n\t\t\t<td>{$field}</td>\n\t\t\t<td><span style='color:{$color}'>{$num}</span></td>\n\t\t\t\n\t\t</tr> \n\t\t\n\t\t"; } $html = $html . "</table>"; $tpl = new templates(); echo $tpl->_ENGINE_parse_body($html); }
function GetCategory($www) { $q = new mysql_squid_builder(); return $q->GET_CATEGORIES($www); }
function categorize($day = null) { $unix = new unix(); $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . ".{$day}." . __FUNCTION__ . ".pid"; $pid = @file_get_contents($pidfile); if ($unix->process_exists($pid, basename(__FILE__))) { if ($GLOBALS["VERBOSE"]) { echo "Already executed pid {$pid}\n"; } return; } if ($day == null) { return; } if (system_is_overloaded()) { echo "Overloaded system, aborting task\n"; writelogs_squid("Overloaded system, aborting task", __FUNCTION__, __FILE__, __LINE__, "categorize"); return; } $daySource = $day; $mypid = getmypid(); @file_put_contents($pidfile, $mypid); $q = new mysql_squid_builder(); $time = strtotime("{$day} 00:00:00"); $day = str_replace("-", "", $day); $table = "{$day}_hour"; $table_blocked = "{$day}_blocked"; $table_month = date("Ym", $time) . "_day"; $table_week = date("YW", $time) . "_week"; $table_week_blocked = date("YW", $time) . "_blocked_week"; $ipClass = new IP(); echo "{$daySource} time: {$time} Table day={$table}, table_blocked={$table_blocked}, table_month={$table_month}, table_week={$table_week}\n"; events("{$daySource} time: {$time} Table day={$table}, table_blocked={$table_blocked}, table_month={$table_month}, table_week={$table_week}"); $t = time(); $f = 0; if (!$q->TABLE_EXISTS($table)) { echo $table . " no such table\n"; return; } $sql = "SELECT sitename,category FROM {$table} GROUP BY sitename,category HAVING LENGTH(category)=0"; events("{$sql}"); $results = $q->QUERY_SQL($sql); if (!$q->ok) { writelogs_squid("Re-categorized table {$table} Query failed: `{$sql}` ({$q->mysql_error})", __FUNCTION__, __FILE__, __LINE__, "categorize"); } if (!$q->TABLE_EXISTS($table_month)) { if (!$q->CreateMonthTable($table_month)) { writelogs_squid("failed Create {$table_month} table {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "categorize"); } } if (!$q->TABLE_EXISTS($table_week)) { if (!$q->CreateWeekTable($table_week)) { writelogs_squid("failed Create {$table_week} table {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "categorize"); } } $L = 0; $q->QUERY_SQL("DELETE FROM `catztemp` WHERE `category`=''"); while ($ligne = @mysql_fetch_array($results, MYSQL_ASSOC)) { $website = trim($ligne["sitename"]); if (preg_match("#^www\\.(.+)#", $website, $re)) { $q->QUERY_SQL("UPDATE {$table} SET sitename='{$re[1]}' WHERE sitename='{$website}'"); $q->QUERY_SQL("UPDATE {$table_month} SET sitename='{$re[1]}' WHERE sitename='{$website}'"); $q->QUERY_SQL("UPDATE {$table_week} SET sitename='{$re[1]}' WHERE sitename='{$website}'"); $q->QUERY_SQL("UPDATE {$table_blocked} SET website='{$re[1]}' WHERE sitename='{$website}'"); $q->QUERY_SQL("UPDATE {$table_week_blocked} SET website='{$re[1]}' WHERE sitename='{$website}'"); $website = $re[1]; } if ($website == null) { continue; } if ($ipClass->isValid($website)) { $website = gethostbyaddr($website); } if (isset($GLOBALS[__FUNCTION__][$website])) { $category = $GLOBALS[__FUNCTION__][$website]; } $category = $q->GET_CATEGORIES($website); if ($category == null) { if ($ipClass->isValid($website)) { $category = "ipaddr"; } } events("{$day}] {$website} = {$category}"); $GLOBALS[__FUNCTION__][$website] = $category; if ($L > 500) { if (system_is_overloaded()) { ufdbguard_admin_events("Fatal: Overloaded system, die();", __FUNCTION__, __FILE__, __LINE__, "stats"); return; } $L = 0; } if ($category == null) { continue; } $f++; events("Update {$table} {$website} = {$category}"); $q->QUERY_SQL("UPDATE {$table} SET category='{$category}' WHERE sitename='{$website}'"); if (!$q->ok) { writelogs_squid("Re-categorized table {$table} failed {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "categorize"); } events("Update {$table_month} {$website} = {$category}"); $q->QUERY_SQL("UPDATE {$table_month} SET category='{$category}' WHERE sitename='{$website}'"); if (!$q->ok) { writelogs_squid("Re-categorized table {$table_month} failed {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "categorize"); } events("Update {$table_week} {$website} = {$category}"); $q->QUERY_SQL("UPDATE {$table_week} SET category='{$category}' WHERE sitename='{$website}'"); if (!$q->ok) { writelogs_squid("Re-categorized table {$table_week} failed {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "categorize"); } events("Update {$table_blocked} {$website} = {$category}"); $q->QUERY_SQL("UPDATE {$table_blocked} SET category='{$category}' WHERE website='{$website}'"); if (!$q->ok) { writelogs_squid("Re-categorized table {$table_blocked} failed {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "categorize"); } events("Update {$table_week_blocked} {$website} = {$category}"); if ($q->CreateWeekBlockedTable($table_week_blocked)) { } $q->QUERY_SQL("UPDATE {$table_week_blocked} SET category='{$category}' WHERE website='{$website}'"); if (!$q->ok) { writelogs_squid("Re-categorized table {$table_week_blocked} failed {$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "categorize"); } } $took = $unix->distanceOfTimeInWords($t, time()); if ($f > 0) { ufdbguard_admin_events("Re-categorized table {$table} with {$f} websites ({$took})", __FUNCTION__, __FILE__, __LINE__, "statistics"); } if ($GLOBALS["VERBOSE"]) { echo "recategorize_singleday({$day}) FINISH\n"; } }
function import_translated() { $q = new mysql_squid_builder(); $MAINZ = unserialize(@file_get_contents("/root/translated")); $translated_done = unserialize(@file_get_contents("/root/translated_done")); $max = count($MAINZ); $gg = new generic_categorize(); $i = 1; while (list($www, $category) = each($MAINZ)) { echo "{$i}/{$max} {$www} "; $i++; if (isset($translated_done[$www])) { echo "\n"; continue; } $category_artica = $gg->GetCategories($www); if ($category_artica != null) { echo "-> ARTICA {$category_artica}\n"; $q->categorize($www, $category_artica); $translated_done[$www] = true; @file_put_contents("/root/translated_done", serialize($translated_done)); continue; } $category_artica = $q->GET_CATEGORIES($www, true, true, true, true); if ($category_artica != null) { echo "-> ARTICA {$category}<>{$category_artica}\n"; $translated_done[$www] = true; @file_put_contents("/root/translated_done", serialize($translated_done)); continue; } echo "{$category}\n"; $q->categorize($www, $category); $translated_done[$www] = true; @file_put_contents("/root/translated_done", serialize($translated_done)); } }
function visited_list() { $tpl = new templates(); $MyPage = CurrentPageName(); $q = new mysql_squid_builder(); $family = $q->GetFamilySites($_GET["www"]); $search = '%'; $table = "visited_sites"; $page = 1; $COUNT_ROWS = $q->COUNT_ROWS($table); $FORCE_FILTER = "familysite='{$family}'"; if ($COUNT_ROWS == 0) { json_error_show("No data..."); } if (isset($_POST["sortname"])) { if ($_POST["sortname"] != null) { $ORDER = "ORDER BY {$_POST["sortname"]} {$_POST["sortorder"]}"; } } if (isset($_POST['page'])) { $page = $_POST['page']; } $_POST["query"] = trim($_POST["query"]); if ($_POST["query"] != null) { $_POST["query"] = "*{$_POST["query"]}*"; $_POST["query"] = str_replace("**", "*", $_POST["query"]); $_POST["query"] = str_replace("**", "*", $_POST["query"]); $_POST["query"] = str_replace("*", "%", $_POST["query"]); $search = $_POST["query"]; $searchstring = "AND (`{$_POST["qtype"]}` LIKE '{$search}')"; $sql = "SELECT COUNT(*) as TCOUNT FROM `{$table}` WHERE {$FORCE_FILTER} {$searchstring}"; writelogs($sql, __FUNCTION__, __FILE__, __LINE__); $ligne = mysql_fetch_array($q->QUERY_SQL($sql, "artica_backup")); $total = $ligne["TCOUNT"]; } else { $sql = "SELECT COUNT(*) as TCOUNT FROM `{$table}` WHERE {$FORCE_FILTER}"; writelogs($sql, __FUNCTION__, __FILE__, __LINE__); $ligne = mysql_fetch_array($q->QUERY_SQL($sql, "artica_backup")); $total = $ligne["TCOUNT"]; } if (isset($_POST['rp'])) { $rp = $_POST['rp']; } $pageStart = ($page - 1) * $rp; $limitSql = "LIMIT {$pageStart}, {$rp}"; $sql = "SELECT * FROM `{$table}` WHERE {$searchstring} {$FORCE_FILTER} {$ORDER} {$limitSql}"; writelogs($sql, __FUNCTION__, __FILE__, __LINE__); $results = $q->QUERY_SQL($sql); $data = array(); $data['page'] = $page; $data['total'] = $total; $data['rows'] = array(); if (mysql_num_rows($results) == 0) { $data['rows'][] = array('id' => $ligne[time()], 'cell' => array($sql, "", "", "")); } while ($ligne = mysql_fetch_assoc($results)) { $categories = $q->GET_CATEGORIES($ligne['sitename']); $FamilySite = $q->GetFamilySites($ligne['sitename']); $categorize = imgtootltip("add-database-32.png", $ligne['sitename'], "javascript:Loadjs('squid.categorize.php?www={$ligne['sitename']}&day={$_GET["day"]}&week={$_GET["week"]}&month={$_GET["month"]}');"); $thumbs = $q->GET_THUMBNAIL($ligne['sitename'], 48); $ahref = "Loadjs('squid.traffic.statistics.days.php?today-zoom=yes&type=req&familysite={$FamilySite}&day={$_GET["day"]}')"; $data['rows'][] = array('id' => $ligne['sitename'], 'cell' => array($thumbs, "<div style='margin-top:10px'><a href=\"javascript:blur();\" OnClick=\"javascript:{$ahref}\" style='font-size:14px;text-decoration:underline'>{$ligne['sitename']}</a></div>", "<div style='font-size:14px;margin-top:10px'>{$categories}</span>", $categorize)); } echo json_encode($data); }
function build_report($ID, $nopid = false) { if (!is_numeric($ID)) { ufdbguard_admin_events("Not a numeric ID", __FUNCTION__, __FILE__, __LINE__, "reports"); return; } $t = time(); $unix = new unix(); $tablename = "WebTrackMem{$ID}"; $tableBlock = "WebTrackMeB{$ID}"; if (!$nopid) { $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".{$ID}.pid"; $pid = @file_get_contents($pidfile); if ($pid < 100) { $pid = null; } if ($unix->process_exists($pid, basename(__FILE__))) { ufdbguard_admin_events("Already executed pid {$pid}", __FUNCTION__, __FILE__, __LINE__, "reports"); if ($GLOBALS["VERBOSE"]) { echo "Already executed pid {$pid}\n"; } return; } } if ($GLOBALS["VERBOSE"]) { echo "Building report {$ID}\n"; } $q = new mysql_squid_builder(); if ($q->TABLE_EXISTS($tablename)) { $q->DELETE_TABLE($tablename); } if ($q->TABLE_EXISTS($tableBlock)) { $q->DELETE_TABLE($tableBlock); } if (!$q->CreateMemberReportTable($tablename)) { ufdbguard_admin_events("could not create table {$tablename}", __FUNCTION__, __FILE__, __LINE__, "reports"); return; } if (!$q->CreateMemberReportBlockTable($tableBlock)) { ufdbguard_admin_events("could not create table {$tablename}", __FUNCTION__, __FILE__, __LINE__, "reports"); return; } $rp = new squid_report($ID); $LIST_TABLES_dansguardian_events = $q->LIST_TABLES_dansguardian_events(); progress(10, $ID); $counttables = count($LIST_TABLES_dansguardian_events); $prefix = "INSERT IGNORE INTO {$tablename} (`zMD5`,`sitename`,`familysite`,`{$rp->userfield}`,`zDate`,`size`,`hits`,`category`) VALUES "; while (list($sourcetable, $ligne) = each($LIST_TABLES_dansguardian_events)) { $c++; $sql = $rp->BuildQuery($sourcetable); $results = $q->QUERY_SQL($sql); if (!$q->ok) { ufdbguard_admin_events("{$q->mysql_error}\n{$sql}", __FUNCTION__, __FILE__, __LINE__, "reports"); return; } if ($GLOBALS["VERBOSE"]) { echo "Parsing {$sourcetable} \n{$sql}\n-> `" . mysql_num_rows($results) . "` rows\n"; } if (mysql_num_rows($results) == 0) { continue; } $purc = round($c / $counttables, 2) * 100; progress($purc, $ID); $f = array(); while ($ligne = @mysql_fetch_array($results, MYSQL_ASSOC)) { $md5 = md5(serialize($ligne)); $sitename = $ligne["sitename"]; $familysite = $q->GetFamilySites($sitename); if (!isset($GLOBALS["CATEGORY"][$sitename])) { $GLOBALS["CATEGORY"][$sitename] = $q->GET_CATEGORIES($sitename); } $category = $GLOBALS["CATEGORY"][$sitename]; $source = addslashes($ligne["source"]); $zDate = $ligne["zDate"]; $size = $ligne["size"]; $hits = $ligne["hits"]; $category = addslashes($category); $f[] = "('{$md5}','{$sitename}','{$familysite}','{$source}','{$zDate}','{$size}','{$hits}','{$category}')"; } if (count($f) == 0) { continue; } $q->QUERY_SQL($prefix . @implode(",", $f)); if (!$q->ok) { ufdbguard_admin_events("{$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "reports"); return; } if (system_is_overloaded(__FILE__)) { sleep(5); } } if ($rp->csv == 1) { SaveCSV($ID); } $LIST_TABLES_BLOCKED = $q->LIST_TABLES_BLOCKED(); $prefix = "INSERT IGNORE INTO {$tableBlock} (`zMD5`,`zDate`,`hits`,`website`,`category`,`rulename`,`event`,`why`,`explain`,`blocktype`,`{$rp->userfield}`) VALUES "; while (list($sourcetable, $ligne) = each($LIST_TABLES_BLOCKED)) { $c++; if ($GLOBALS["VERBOSE"]) { echo "Parsing {$sourcetable}\n"; } $sql = $rp->BuildQueryBlock($sourcetable); if (!$q->FIELD_EXISTS("{$sourcetable}", "uid")) { $q->QUERY_SQL("ALTER TABLE `{$sourcetable}` ADD `uid` VARCHAR( 128 ) NOT NULL ,ADD INDEX ( `uid` )"); } $results = $q->QUERY_SQL($sql); if (!$q->ok) { ufdbguard_admin_events("{$q->mysql_error}\n{$sql}", __FUNCTION__, __FILE__, __LINE__, "reports"); return; } if (mysql_num_rows($results) == 0) { continue; } $purc = round($c / $counttables, 2) * 100; progress($purc, $ID); $f = array(); while ($ligne = @mysql_fetch_array($results, MYSQL_ASSOC)) { $zMD5 = md5(serialize($ligne)); $sitename = $ligne["sitename"]; $category = $ligne["category"]; if ($category == null) { if (!isset($GLOBALS["CATEGORY"][$sitename])) { $GLOBALS["CATEGORY"][$sitename] = $q->GET_CATEGORIES($sitename); } $category = $GLOBALS["CATEGORY"][$sitename]; } $source = addslashes($ligne["source"]); $zDate = $ligne["zDate"]; $hits = $ligne["hits"]; $rulename = $ligne["rulename"]; $event = $ligne["event"]; $why = $ligne["why"]; $explain = $ligne["explain"]; $blocktype = $ligne["blocktype"]; $category = addslashes($category); $f[] = "('{$zMD5}','{$zDate}','{$hits}','{$sitename}','{$category}','{$rulename}','{$event}','{$why}','{$explain}','{$blocktype}','{$source}')"; } if (system_is_overloaded(__FILE__)) { sleep(5); } } $myisamchk = $unix->find_program("myisamchk"); $myisampack = $unix->find_program("myisampack"); $mysql_data = $unix->MYSQL_DATA_DIR(); echo "OPTIMIZE TABLE {$tableBlock}\n"; $q->QUERY_SQL("OPTIMIZE TABLE {$tableBlock}"); echo "OPTIMIZE TABLE {$tablename}\n"; $q->QUERY_SQL("OPTIMIZE TABLE {$tablename}"); echo "LOCK TABLE {$tablename}\n"; $q->QUERY_SQL("LOCK TABLE {$tablename} WRITE"); echo "LOCK TABLE {$tableBlock}\n"; $q->QUERY_SQL("LOCK TABLE {$tableBlock} WRITE"); $q->QUERY_SQL("FLUSH TABLE {$tableBlock}"); $q->QUERY_SQL("FLUSH TABLE {$tablename}"); echo "myisamchk {$tablename}\n"; shell_exec("{$myisamchk} -cFU {$mysql_data}/squidlogs/{$tablename}.MYI"); echo "myisamchk {$tableBlock}\n"; shell_exec("{$myisamchk} -cFU {$mysql_data}/squidlogs/{$tableBlock}.MYI"); echo "myisampack {$tablename}\n"; shell_exec("{$myisampack} -f {$mysql_data}/squidlogs/{$tablename}.MYI"); echo "myisampack {$tableBlock}\n"; shell_exec("{$myisampack} -f {$mysql_data}/squidlogs/{$tableBlock}.MYI"); $q->QUERY_SQL("FLUSH TABLE {$tablename}"); $q->QUERY_SQL("FLUSH TABLE {$tableBlock}"); $rp->set_duration($unix->distanceOfTimeInWords($t, time(), true)); progress(100, $ID); }
function MoveCategorizedWebsite($md5 = null, $nextCategory = null, $table = null) { $q = new mysql_squid_builder(); $sock = new sockets(); if ($md5 == null) { $md5 = $_POST["MoveCategorizedWebsite"]; } if ($nextCategory == null) { $nextCategory = trim($_POST["NextCategory"]); } if ($table == null) { $table = trim($_POST["TABLE"]); } if ($nextCategory == null) { echo "Next category = Null\n"; return; } if ($table == null) { echo "Table = Null\n"; return; } if ($md5 == null) { echo "md5 = Null\n"; return; } if (!isset($GLOBALS["uuid"])) { $GLOBALS["uuid"] = base64_decode($sock->getFrameWork("cmd.php?system-unique-id=yes")); } $uuid = $GLOBALS["uuid"]; $sql = "SELECT * FROM {$table} WHERE zmd5='{$md5}'"; $ligne = mysql_fetch_array($q->QUERY_SQL($sql)); if (!$q->ok) { echo $q->mysql_error; return; } $www = $ligne["pattern"]; $sql = "INSERT IGNORE INTO categorize_delete (sitename,category,zmd5) VALUES ('{$ligne["pattern"]}','{$ligne["category"]}','{$ligne["zmd5"]}')"; $q->QUERY_SQL($sql); if (!$q->ok) { echo $q->mysql_error; return; } $q->QUERY_SQL("UPDATE {$table} SET enabled=0 WHERE zmd5='{$md5}'"); if (!$q->ok) { echo $q->mysql_error; return; } $sock->getFrameWork("squid.php?export-deleted-categories=yes"); $newmd5 = md5($www . $nextCategory); $q->QUERY_SQL("INSERT IGNORE INTO categorize_changes (zmd5,sitename,category) VALUES('{$newmd5}','{$www}','{$nextCategory}')"); if (!$q->ok) { echo $q->mysql_error; return; } $q->CreateCategoryTable($nextCategory); $category_table = $q->category_transform_name($nextCategory); $q->QUERY_SQL("INSERT IGNORE INTO categorize (zmd5,zDate,category,pattern,uuid) VALUES('{$newmd5}',NOW(),'{$nextCategory}','{$www}','{$uuid}')"); if (!$q->ok) { echo $q->mysql_error; return; } $q->QUERY_SQL("INSERT IGNORE INTO category_{$category_table} (zmd5,zDate,category,pattern,uuid,enabled) VALUES('{$md5}',NOW(),'{$nextCategory}','{$www}','{$uuid}',1)"); if (!$q->ok) { echo $q->mysql_error; return; } $cats = addslashes($q->GET_CATEGORIES($www, true)); $q->QUERY_SQL("UPDATE visited_sites SET category='{$cats}' WHERE sitename='{$www}'"); if (!$q->ok) { echo $q->mysql_error . "\n"; echo $sql . "\n"; } $sock = new sockets(); $sock->getFrameWork("cmd.php?export-community-categories=yes"); $sock->getFrameWork("squid.php?re-categorize=yes"); }
function CheckWebsites($data, $category) { $founds = array(); if (!preg_match_all("#Domain:(.*?)<#s", $data, $founds)) { echo "NO MATCH scan 1.....\n"; } $count = count($founds[1]); if ($count == 0) { if (!preg_match_all("#Name:(.*?)<#s", $data, $founds)) { echo "NO MATCH scan 2.....\n"; } } $count = count($founds[1]); if ($count == 0) { return; } echo strlen($curl->data) . " bytes lenth...found: {$count} items\n"; $q = new mysql_squid_builder(); if ($count > 0) { while (list($id, $www) = each($founds[1])) { $www = trim($www); $www = str_replace("\t", "", $www); $www = str_replace(chr(194), "", $www); $www = str_replace(chr(32), "", $www); $www = str_replace(chr(160), "", $www); if (!preg_match("#([a-z0-9\\-_\\.]+)\\.([a-z]+)\$#i", $www, $re)) { echo "{$www} skiped L." . __LINE__ . "\n"; continue; } if (strpos($www, ",") > 0) { echo "{$www} skiped L." . __LINE__ . "\n"; continue; } if (strpos($www, " ") > 0) { echo "{$www} skiped L." . __LINE__ . "\n"; continue; } if (strpos($www, ":") > 0) { echo "{$www} skiped L." . __LINE__ . "\n"; continue; } if (strpos($www, "%") > 0) { echo "{$www} skiped L." . __LINE__ . "\n"; continue; } if (preg_match("#^www\\.(.+)#", $www, $re)) { $www = $re[1]; } $articacats = trim($q->GET_CATEGORIES($www, true, true)); if ($articacats != null) { echo "\"{$www}\" already in {$articacats}\n"; } $newsWeb[] = $www; } } while (list($id, $www) = each($newsWeb)) { echo "Adding \"{$www}\" -> {$category}\n"; $q->ADD_CATEGORYZED_WEBSITE($www, $category); } }
function websites_uid_not_categorised($uid = null, $tablename = null, $aspid = false) { if (isset($GLOBALS["websites_uid_not_categorised_{$uid}"])) { return; } $unix = new unix(); $pidfile = "/etc/artica-postfix/pids/" . basename(__FILE__) . "." . __FUNCTION__ . ".{$uid}.pid"; if ($aspid) { $pid = @file_get_contents($pidfile); $myfile = basename(__FILE__); if ($unix->process_exists($pid, $myfile)) { ufdbguard_admin_events("Task already running PID: {$pid}, aborting current task", __FUNCTION__, __FILE__, __LINE__, "stats"); return; } } $mypid = getmypid(); @file_put_contents($pidfile, $mypid); $q = new mysql_squid_builder(); if ($uid != null) { $uidtable = $q->uid_to_tablename($uid); $tablename = "www_{$uidtable}"; } if (!$q->FIELD_EXISTS($tablename, "category")) { $q->QUERY_SQL("ALTER TABLE `{$tablename}` ADD `category` varchar(255), ADD INDEX (`category`)"); } $sql = "SELECT familysite,`category` FROM `{$tablename}` GROUP BY familysite,`category` HAVING `category` IS NULL "; $results = $q->QUERY_SQL($sql); if (!$q->ok) { if ($GLOBALS["VERBOSE"]) { echo "############# ERROR #########\n{$q->mysql_error}\\Line:" . __LINE__ . "\n#############\n"; } return false; } $c = 0; $mysql_num_rows = mysql_num_rows($results); if ($mysql_num_rows == 0) { if ($GLOBALS["VERBOSE"]) { echo "{$sql} (No rows)\n"; } return true; } while ($ligne = @mysql_fetch_array($results, MYSQL_ASSOC)) { $sitename = $ligne["familysite"]; $IpClass = new IP(); if ($IpClass->isValid($sitename)) { if (isset($GLOBALS["IPCACHE"][$sitename])) { $t = time(); $sitename = gethostbyaddr($sitename); events("{$tablename}: {$ligne["familysite"]} -> {$sitename} " . $unix->distanceOfTimeInWords($t, time()) . " gethostbyaddr() LINE:" . __LINE__); $GLOBALS["IPCACHE"][$sitename] = $sitename; } } $category = $q->GET_CATEGORIES($sitename); if ($IpClass->isValid($sitename)) { if ($category == null) { $category = "ipaddr"; } $q->categorize($sitename, $category); } events("{$tablename}: {$ligne["familysite"]} -> {$sitename} [{$category}] LINE:" . __LINE__); if (strlen($category) > 0) { $category = mysql_escape_string2($category); $ligne["familysite"] = mysql_escape_string2($ligne["familysite"]); $sql = "UPDATE `{$tablename}` SET `category`='{$category}' WHERE familysite='{$ligne["familysite"]}'"; $q->QUERY_SQL($sql); if (!$q->ok) { ufdbguard_admin_events("{$q->mysql_error}", __FUNCTION__, __FILE__, __LINE__, "stats"); return; } } } }
function _not_categorized_months($table) { $t = time(); $unix = new unix(); $sql = "SELECT familysite FROM {$table} WHERE category='' GROUP BY familysite"; $q = new mysql_squid_builder(); $results = $q->QUERY_SQL($sql); if (mysql_num_rows($results) == 0) { return true; } $c = 0; while ($ligne = mysql_fetch_assoc($results)) { $category = $q->GET_CATEGORIES($ligne["familysite"]); echo "{$table} {$ligne["familysite"]} -> `{$category}`\n"; if ($category == null) { continue; } $q->QUERY_SQL("UPDATE {$table} SET category='{$category}' WHERE familysite='{$ligne["familysite"]}'"); $c++; } if ($c > 0) { stats_admin_events(2, "{$table} {$c} websites categorized took:" . $unix->distanceOfTimeInWords($t, time()), null, __FILE__, __LINE__); } }
function CategorizeAll_perform() { if ($_GET["pattern"] == null) { return; } $sock = new sockets(); $uuid = base64_decode($sock->getFrameWork("cmd.php?system-unique-id=yes")); if ($uuid == null) { echo "UUID=NULL; Aborting"; return; } $pattern = " AND sitename LIKE '%{$_GET["pattern"]}%' "; $pattern = str_replace("*", "%", $pattern); $sql = "SELECT sitename FROM `visited_sites` WHERE LENGTH( `category` )=0 {$pattern}"; $category = $_GET["CategorizeAll_category"]; if ($category == null) { return; } $q = new mysql_squid_builder(); $results = $q->QUERY_SQL($sql, "artica_events"); if (!$q->ok) { echo $q->mysql_error . "\n" . basename(__FILE__) . "\nLine" . __LINE__; return; } $category_table = "category_" . $q->category_transform_name($category); if (!$q->TABLE_EXISTS($category_table)) { $q->CreateCategoryTable($category); } while ($ligne = mysql_fetch_array($results, MYSQL_ASSOC)) { $website = $ligne["sitename"]; if ($website == null) { return; } $www = trim(strtolower($website)); if (preg_match("#^www\\.(.+?)\$#i", trim($www), $re)) { $www = $re[1]; } $md5 = md5($www . $category); $enabled = 1; if ($www == null) { echo "Alert: website is null...\n"; return; } $sql_add = "INSERT INTO {$category_table} (zmd5,zDate,category,pattern,uuid,enabled) VALUES('{$md5}',NOW(),'{$category}','{$www}','{$uuid}',1)"; $q->QUERY_SQL($sql_add); if (!$q->ok) { echo $q->mysql_error . "\n" . basename(__FILE__) . "\nLine" . __LINE__ . "\n"; echo $sql; return; } $sql = "INSERT IGNORE INTO categorize (zmd5,zDate,category,pattern,uuid) VALUES('{$md5}',NOW(),'{$category}','{$www}','{$uuid}')"; $q->QUERY_SQL($sql); if (!$q->ok) { echo $q->mysql_error . "\n" . basename(__FILE__) . "\nLine" . __LINE__ . "\n"; echo $sql; return; } if ($_GET["day"] != null) { $time = strtotime($_GET["day"] . " 00:00:00"); $tableSrc = date('Ymd') . "_hour"; $categories = $q->GET_CATEGORIES($www, true); $q->QUERY_SQL("UPDATE {$tableSrc} SET category='{$categories}' WHERE sitename='{$www}'"); } } $sql = "UPDATE `visited_sites` SET `category`='{$category}' WHERE LENGTH( `category` )=0 {$pattern}"; $q->QUERY_SQL($sql); if (!$q->ok) { echo $q->mysql_error . "\n" . basename(__FILE__) . "\nLine" . __LINE__ . "\n"; echo $sql; return; } $sock = new sockets(); $sock->getFrameWork("cmd.php?export-community-categories=yes"); }
function popup_categories_sql() { //ini_set('html_errors',0);ini_set('display_errors', 1);ini_set('error_reporting', E_ALL);ini_set('error_prepend_string','');ini_set('error_append_string',''); $tpl = new templates(); $MyPage = CurrentPageName(); if ($GLOBALS["VERBOSE"]) { echo __FUNCTION__ . "::" . __LINE__ . ":: mysql_squid_builder()\n<br>"; } $q = new mysql_squid_builder(); $OnlyEnabled = false; if (isset($_GET["OnlyEnabled"])) { $OnlyEnabled = true; } $search = '%'; $table = "webfilters_categories_caches"; $page = 1; $ORDER = "ORDER BY categorykey ASC"; if (!$q->TABLE_EXISTS($table)) { if ($GLOBALS["VERBOSE"]) { echo "<H2>Create create_webfilters_categories_caches()</H2>\n"; } $q->create_webfilters_categories_caches(); } $FORCE_FILTER = null; if (trim($_GET["group"]) != null) { $FORCE_FILTER = " AND master_category='{$_GET["group"]}'"; } if ($GLOBALS["VERBOSE"]) { echo "<h2>" . __FUNCTION__ . "::" . __LINE__ . ":: q->COUNT_ROWS({$table})</h2>\n<br>"; } if ($q->COUNT_ROWS($table) == 0) { $ss = new dansguardian_rules(); $ss->CategoriesTableCache(); } $www = trim(strtolower($_GET["www"])); $ArticaDBZ = new mysql_catz(); $CategoriesFound = $ArticaDBZ->GET_CATEGORIES($www); $catArDB = explode(",", $CategoriesFound); writelogs("ArticaDB({$www}) = " . @implode(",", $catArDB), __FUNCTION__, __FILE__, __LINE__); if (is_array($catArDB)) { while (list($num, $ligne) = each($catArDB)) { $ligne = trim($ligne); if ($ligne == null) { continue; } $hash_ARTICA[$ligne] = true; } } if (preg_match("#www\\.(.+?)\$#i", $www, $re)) { $www = $re[1]; } $q = new mysql_squid_builder(); $CategoriesFound = $q->GET_CATEGORIES($www, true, true, true, true); $cats = explode(",", $CategoriesFound); $www_encoded = base64_encode($_GET["www"]); $COUNT_ROWS = $q->COUNT_ROWS($table); $hash_community = array(); if (is_array($cats)) { while (list($num, $ligne) = each($cats)) { $ligne = trim($ligne); if ($ligne == null) { continue; } $hash_community[$ligne] = true; } } if ($COUNT_ROWS == 0) { $data['page'] = $page; $data['total'] = $total; $data['rows'] = array(); echo json_encode($data); return; } if (isset($_POST["sortname"])) { if ($_POST["sortname"] != null) { $ORDER = "ORDER BY {$_POST["sortname"]} {$_POST["sortorder"]}"; } } if (isset($_POST['page'])) { $page = $_POST['page']; } $_POST["query"] = trim($_POST["query"]); if ($_POST["query"] != null) { $_POST["query"] = "*{$_POST["query"]}*"; $_POST["query"] = str_replace("**", "*", $_POST["query"]); $_POST["query"] = str_replace("**", "*", $_POST["query"]); $_POST["query"] = str_replace("*", "%", $_POST["query"]); $search = $_POST["query"]; $searchstring = "AND (`{$_POST["qtype"]}` LIKE '{$search}')"; $sql = "SELECT COUNT(*) as TCOUNT FROM `{$table}` WHERE 1 {$FORCE_FILTER} {$searchstring}"; $ligne = mysql_fetch_array($q->QUERY_SQL($sql, "artica_backup")); $total = $ligne["TCOUNT"]; } else { $total = $COUNT_ROWS; } if (isset($_POST['rp'])) { $rp = $_POST['rp']; } $pageStart = ($page - 1) * $rp; $limitSql = "LIMIT {$pageStart}, {$rp}"; if ($OnlyEnabled) { $limitSql = null; } $sql = "SELECT * FROM `{$table}` WHERE 1 {$searchstring} {$FORCE_FILTER} {$ORDER} {$limitSql}"; writelogs($sql, __FUNCTION__, __FILE__, __LINE__); $results = $q->QUERY_SQL($sql); $data = array(); $data['page'] = $page; $data['total'] = $total; $data['rows'] = array(); if (mysql_num_rows($results) == 0) { $data['rows'][] = array('id' => $ligne[time()], 'cell' => array($sql, "", "", "")); } $c = 0; while ($ligne = mysql_fetch_assoc($results)) { if ($ligne['categorykey'] == "phishtank") { continue; } $DISABLED = false; if ($ligne["picture"] == null) { $ligne["picture"] = "20-categories-personnal.png"; } $TextInterne = null; $img = "img/{$ligne["picture"]}"; $val = 0; if ($hash_community[$ligne['categorykey']]) { $val = 1; } if ($hash_ARTICA[$ligne['categorykey']]) { $TextInterne = $tpl->_ENGINE_parse_body("<div style='color:#D01313;font-size:11px;font-style:italic'>{categorized_in_articadb}</div>"); $val = 1; $DISABLED = true; } $md = md5($ligne['categorykey']); if ($OnlyEnabled) { if ($val == 0) { if ($TextInterne == null) { continue; } } } $c++; $js = "DansCommunityCategory('{$md}','{$ligne["categorykey"]}','{$www_encoded}')"; $disable = Field_checkbox($md, 1, $val, "{$js}", null, $DISABLED); $ligne['description'] = utf8_encode($ligne['description']); $data['rows'][] = array('id' => $ligne['categorykey'], 'cell' => array("<img src='{$img}'>", "{$ligne['categorykey']}", $TextInterne . $ligne['description'], $disable)); } if ($OnlyEnabled) { $data['total'] = $c; } echo json_encode($data); }
function test_category_perform(){ $www=$_REQUEST["WEBTESTS"]; $tpl=new templates(); $q=new mysql_squid_builder(); $www=$q->WebsiteStrip($www); if($www==null){ echo $tpl->_ENGINE_parse_body("<p class=text-error>{corrupted_request}: «{$_REQUEST["WEBTESTS"]}»</p>"); return; } $dans=new dansguardian_rules(); $dans->LoadBlackListes(); $catz=$q->GET_CATEGORIES($www,true); if($catz==null){ echo $tpl->_ENGINE_parse_body("<p class=text-error>{unknown}: «{$_REQUEST["WEBTESTS"]}»</p>"); return; } if(strpos(" $catz", ",")>0){$CATs=explode(",", $catz);}else{$CATs[]=$catz;} $sql="SELECT * FROM personal_categories"; if(!$q->TABLE_EXISTS("personal_categories")){json_error_show("personal_categories no such table!",1);} $results=$q->QUERY_SQL($sql); if(!$q->ok){json_error_show("Mysql Error [".__LINE__."]: $q->mysql_error",1);} while($ligne=mysql_fetch_array($results,MYSQL_ASSOC)){$PERSONALSCATS[$ligne["category"]]=$ligne["category_description"];} while (list ($num, $categoryname) = each ($CATs) ){ if(!isset($dans->array_blacksites[$categoryname])){ if(isset($dans->array_blacksites[str_replace("_","-",$categoryname)])){$categoryname=str_replace("_","-",$categoryname);} if(isset($dans->array_blacksites[str_replace("_","/",$categoryname)])){$categoryname=str_replace("_","/",$categoryname);} } if($dans->array_pics[$categoryname]<>null){$pic="<img src='img/{$dans->array_pics[$categoryname]}'>";}else{$pic=" ";} $text_category=$dans->array_blacksites[$categoryname]; if(isset($PERSONALSCATS[$categoryname])){ $text_category=utf8_encode($PERSONALSCATS[$categoryname]); if($pic==" "){$pic="<img src='img/20-categories-personnal.png'>";} } $js="javascript:Loadjs('squid.categories.php?category=$categoryname&t=$t')"; $categoryText[]=$tpl->_ENGINE_parse_body(" <tr> <td width=1% nowrap>$pic</td> <td valign='top'> <div style='font-size:18px';font-weight:bold'> <a href=\"javascript:blur();\" OnClick=\"$js\" style='text-decoration:underline'>$categoryname</a>:</div> <div style='font-size:16px;width:100%;font-weight:normal'>{$text_category}</div> </td> </tr> "); } $found=$tpl->_ENGINE_parse_body("{found}"); echo "<div style='width:95%:padding-left:50px;padding-top:20px' class=text-info><div style='font-size:18px'>$found</div><table>".@implode("\n", $categoryText)."</table></div>"; }
function recategorize_tables($zdate, $recategorize = 0, $report_id) { $xtime = strtotime("{$zdate} 00:00:00"); $tday = date("Ymd", $xtime); $table_hour = "{$tday}_hour"; $q = new mysql_squid_builder(); $qr = new mysql_squid_reports($report_id); $q->QUERY_SQL("UPDATE {$table_hour} SET category='' WHERE category=','"); $sql = "SELECT familysite FROM {$table_hour} GROUP BY familysite"; if ($recategorize == 1) { $sql = "SELECT familysite FROM {$table_hour} WHERE LENGTH(category)=0 GROUP BY familysite"; } $results = $q->QUERY_SQL("SELECT familysite FROM {$table_hour} GROUP BY familysite"); if (!$q->ok) { echo $q->mysql_error . "\n"; } echo "Recategorize table {$table_hour} " . mysql_num_rows($results) . "\n"; while ($ligne = @mysql_fetch_array($results, MYSQL_ASSOC)) { $familysite = $ligne["familysite"]; $categories = trim($q->GET_CATEGORIES($familysite)); if ($categories == null) { $qr->QUERY_SQL("INSERT IGNORE INTO nocatz (familysite) VALUE ('{$familysite}')"); continue; } echo "{$familysite} = `{$categories}`\n"; $q->QUERY_SQL("UPDATE {$table_hour} SET `category`='{$categories}' WHERE `familysite`='{$familysite}'"); } }
function analyze() { if ($GLOBALS["VERBOSE"]) { echo "analyze<br>\n"; } $page = CurrentPageName(); $tpl = new templates(); $q = new mysql_squid_builder(); $uri = $_GET["uri"]; $curl = new ccurl($uri); if ($GLOBALS["VERBOSE"]) { echo "{$uri}<br>\n"; } $filename = md5($uri); if (!$curl->GetFile("ressources/logs/web/{$filename}")) { echo "<H2>" . $curl->error . "</H2>"; return; } $ARRY = array(); if ($GLOBALS["VERBOSE"]) { echo "Open ressources/logs/web/{$filename}<br>\n"; } $datas = @file_get_contents("ressources/logs/web/{$filename}"); $tb = explode("\n", $datas); @unlink("ressources/logs/web/{$filename}"); if (preg_match("#google\\..+?#", $uri)) { if (preg_match_all('#<h3 class="r"><a href="(.+?)"#', $datas, $re)) { while (list($num, $uri) = each($re[1])) { if (preg_match("#^(?:[^/]+://)?([^/:]+)#", $uri, $ri)) { $sitename = $ri[1]; if (substr($sitename, 0, 1) == "#") { continue; } if (preg_match("#^www\\.(.+)#", $sitename, $ri)) { $sitename = $ri[1]; } if (preg_match("#\\.php\\?#", $sitename)) { continue; } if (preg_match("#\\.php\$#", $sitename)) { continue; } $ARRY[$sitename] = $sitename; } } } } while (list($num, $line) = each($tb)) { if (preg_match("#<a\\s+href=(.*)\\.([a-z]+)#i", $line, $re)) { $uri = trim($re[1] . "." . $re[2]); $uri = str_replace("\"", "", $uri); $uri = str_replace("'", "", $uri); if (strpos($uri, ">") > 0) { $uri = substr($uri, 0, strpos($uri, ">")); } if (preg_match("#^(?:[^/]+://)?([^/:]+)#", $uri, $re)) { $sitename = $re[1]; if (substr($sitename, 0, 1) == "#") { continue; } if (preg_match("#^www\\.(.+)#", $sitename, $ri)) { $sitename = $ri[1]; } if (preg_match("#\\.php\\?#", $sitename)) { continue; } if (preg_match("#\\.php\\s+[a-z]#", $sitename)) { continue; } if (preg_match("#\\.php\$#", $sitename)) { continue; } if (strpos($sitename, ".") == 0) { continue; } if (strpos($sitename, "{") > 0) { continue; } if (strpos($sitename, "}") > 0) { continue; } if (strpos($sitename, "\$") > 0) { continue; } $ARRY[trim(strtolower($sitename))] = trim(strtolower($sitename)); } } } $html = "\n\t"; $f = 0; $s = 0; $t = 0; while (list($num, $line) = each($ARRY)) { if (strlen($num) < 3) { continue; } if ($num == "javascript") { continue; } if (strpos($num, ".") == 0) { continue; } if (strpos($num, "{") > 0) { continue; } if (strpos($num, " ") > 0) { continue; } if (strpos($num, "}") > 0) { continue; } if (strpos($num, "\$") > 0) { continue; } $tz = explode(".", $num); if ($tz[count($tz) - 1] == "php") { continue; } if ($tz[count($tz) - 1] == "html") { continue; } if ($tz[count($tz) - 1] == "htm") { continue; } if (preg_match("#\\.php\\?#", $num)) { continue; } if (preg_match("#\\.php\$#", $num)) { continue; } $t++; if ($t > 200) { break; } $cats = $q->GET_CATEGORIES($num); $mustcat = " "; $color = "black"; if ($classtr == "oddRow") { $classtr = null; } else { $classtr = "oddRow"; } if ($cats == null) { $tt = explode(".", $num); $familysite = $tt[count($tt) - 2] . "." . $tt[count($tt) - 1]; $q->QUERY_SQL("INSERT IGNORE INTO visited_sites (sitename,familysite) VALUES ('{$num}','{$familysite}')"); $f++; $cats = " "; $mustcat = imgtootltip("add-database-32.png", "{categorize}", "Loadjs('squid.categorize.php?www={$num}')"); $color = "#CC0A0A"; } else { $s++; } $html = $html . "\n\t\t<tr class={$classtr}>\n\t\t<td style='font-size:14px;color:{$color}'>{$num}</td>\n\t\t<td style='font-size:14px'>{$cats}</td>\n\t\t<td width=1%>{$mustcat}</td>\n\t\t</tr>\n\t\t"; } //javascript:Loadjs('squid.categorize.php?www=api161.thefilter.com&day=&week='); $purc = round($s / $t * 100, 2); $html = "<center><table cellspacing='0' cellpadding='0' border='0' class='tableView' style='width:100%'>\n\t<thead class='thead'>\n\t<tr>\n\t<th>{$t} {websites} {$f} {failed} {$s} {success} {$purc}%</th>\n\t<th>{category}</th>\n\t<th> </th>\n\t</tr>\n</thead>\n<tbody>{$html}</tbody></table><span id='webalyzer-lock'></span>"; echo $tpl->_ENGINE_parse_body($html); }
function CategorizeAll_perform(){ if($_GET["pattern"]==null){return;} $sock=new sockets(); $uuid=base64_decode($sock->getFrameWork("cmd.php?system-unique-id=yes")); if($uuid==null){echo "UUID=NULL; Aborting";return;} $pattern=" AND sitename LIKE '%{$_GET["pattern"]}%' "; $pattern=str_replace("*","%",$pattern); $sql="SELECT sitename FROM `visited_sites` WHERE LENGTH( `category` )=0 $pattern"; $category=$_GET["CategorizeAll_category"]; if($category==null){return;} if($category=="teans"){$category="teens";} $q=new mysql_squid_builder(); $results=$q->QUERY_SQL($sql,"artica_events"); if(!$q->ok){ echo $q->mysql_error."\n".basename(__FILE__)."\nLine".__LINE__; return; } if($_GET["week"]<>null){$_GET["day"]=$_GET["week"];} $category_table="category_".$q->category_transform_name($category); if(!$q->TABLE_EXISTS($category_table)){$q->CreateCategoryTable($category);} while($ligne=mysql_fetch_array($results,MYSQL_ASSOC)){ $website=$ligne["sitename"]; if($website==null){return;} $www=trim(strtolower($website)); if(preg_match("#^www\.(.+?)$#i",trim($www),$re)){$www=$re[1];} $md5=md5($category.$www); $enabled=1; if($www==null){echo "Alert: website is null...\n";return;} $sql_add="INSERT INTO $category_table (zmd5,zDate,category,pattern,uuid,enabled) VALUES('$md5',NOW(),'$category','$www','$uuid',1)"; $ligne=mysql_fetch_array($q->QUERY_SQL("SELECT category,pattern FROM `$category_table` WHERE zmd5='$md5'")); if($ligne["category"]==null){$sql_add="UPDATE $category_table SET `category`='$category',pattern='$www' WHERE zmd5='$md5'";} if($ligne["pattern"]==null){$sql_add="UPDATE $category_table SET `category`='$category',pattern='$www' WHERE zmd5='$md5'";} $q->QUERY_SQL($sql_add); if(!$q->ok){echo $q->mysql_error."\n".basename(__FILE__)."\nLine".__LINE__."\n";echo $sql;return;} $sql="INSERT IGNORE INTO categorize (zmd5,zDate,category,pattern,uuid) VALUES('$md5',NOW(),'$category','$www','$uuid')"; $q->QUERY_SQL($sql); if(!$q->ok){echo $q->mysql_error."\n".basename(__FILE__)."\nLine".__LINE__."\n";echo $sql;return;} $sql="UPDATE `visited_sites` SET `category`='$category' WHERE LENGTH( `category` )=0 $pattern"; $q->QUERY_SQL($sql); if(!$q->ok){echo $q->mysql_error."\n".basename(__FILE__)."\nLine".__LINE__."\n";echo $sql;return;} if($_GET["day"]<>null){ if($pattern<>null){ $time=strtotime($_GET["day"]." 00:00:00"); $tableSrc=date('Ymd',$time)."_hour"; $categories=$q->GET_CATEGORIES($www,true); if($categories==null){$categories=$category;} $sql="UPDATE $tableSrc SET category='$categories' WHERE LENGTH( `category` )=0 $pattern"; $q->QUERY_SQL($sql); writelogs($sql,__FUNCTION__,__FILE__,__LINE__); if(!$q->ok){echo $q->mysql_error."\n".basename(__FILE__)."\nLine".__LINE__."\n";echo $sql;} $tableWeek=date("YW",$time)."_week"; $sql="UPDATE $tableWeek SET category='$categories' WHERE LENGTH( `category` )=0 $pattern"; $q->QUERY_SQL($sql); writelogs($sql,__FUNCTION__,__FILE__,__LINE__); if(!$q->ok){echo $q->mysql_error."\n".basename(__FILE__)."\nLine".__LINE__."\n";echo $sql;} $tableSrcDay=date('Ym',$time)."_day"; if($q->TABLE_EXISTS($tableSrcDay)){ $sql="UPDATE $tableSrcDay SET category='$categories' WHERE LENGTH( `category` )=0 $pattern"; writelogs($sql,__FUNCTION__,__FILE__,__LINE__); $q->QUERY_SQL($sql); if(!$q->ok){echo $q->mysql_error."\n".basename(__FILE__)."\nLine".__LINE__."\n";echo $sql;} } }else{ echo "Pattern is null!!\n"; } } } $sock=new sockets(); $sock->getFrameWork("cmd.php?export-community-categories=yes"); }
function page() { $t = time(); $familysite = $_GET["familysite"]; $tpl = new templates(); $q = new mysql_squid_builder(); $page = CurrentPageName(); $categories = $q->GET_CATEGORIES($familysite); $familysiteenc = urlencode($familysite); $thumbs = $q->GET_THUMBNAIL($familysite, 320); if (strpos($categories, ",") > 0) { $categoriesZ = explode(",", $categories); $tt[] = "<ul>"; while (list($num, $ligne) = each($categoriesZ)) { $tt[] = "<li>{$ligne}</li>"; } $tt[] = "</ul>"; $categories = @implode("", $tt); } $t = time(); $html = "\n\t<table style='width:100%' class=TableRemove>\n\t<tr>\n\t<td valign='top'>\n\t\t<div id='{$t}-img'>\n\t\t{$thumbs}\n\t\t</div>\n\t\t<center style='margin-top:10px'>" . button("{regenerate_thumbnail}", "LoadAjax('{$t}-img','{$page}?rethumbnail={$familysiteenc}');") . "</center>\t\n\t</td>\n\t<td valign='top' style='padding-left:20px'>\t\t\n\t\t<H3>{$familysite}</H3>\n\t\t<p><strong style='font-size:16px'>{this_website_was_categorized_in}:{$categories}</strong></p>\n\t\t<H3>{details}</H3>\n\t\t<div id='{$t}'></div>\n\t</td>\n\t</tr>\n\t</table>\n\t<script>\n\t\tLoadAjax('{$t}','{$page}?whois={$familysiteenc}');\n\t</script>\n\t\n\t"; echo $tpl->_ENGINE_parse_body($html); }
if ($IpClass->isIPAddress($ligne["sitename"])) { $q->QUERY_SQL("DELETE FROM webtests WHERE sitename='{$ligne["sitename"]}'"); $ligne["sitename"] = gethostbyaddr($ligne["sitename"]); } $familysite = $q->GetFamilySites($ligne["sitename"]); if ($familysite == $ligne["sitename"]) { $familysite = null; } if ($GLOBALS["OUTPUT"]) { echo "{$www} -> STAMP\n"; } $q->QUERY_SQL("UPDATE webtests SET checked=1 WHERE sitename='{$ligne["sitename"]}'"); if ($GLOBALS["OUTPUT"]) { echo "{$www} -> GET_CATEGORIES\n"; } $articacats = trim($q->GET_CATEGORIES($ligne["sitename"], true, false)); if ($GLOBALS["OUTPUT"]) { echo "{$ligne["sitename"]} -> \"{$articacats}\"\n"; } if ($articacats != null) { $q->categorize($ligne["sitename"], $articacats); $unix->_syslog("{$percent}) {$ligne["sitename"]} = {$articacats}", __FILE__); continue; } if ($familysite != null) { $articacats = trim($q->GET_CATEGORIES($familysite, true, false)); if ($GLOBALS["OUTPUT"]) { echo "{$ligne["sitename"]} {$familysite} -> {$articacats}\n"; } if ($articacats != null) { $q->categorize($ligne["sitename"], $articacats);