function add_btag($tag, $reg) { $api_url = 'https://www.hotslogs.com/API/Players/'; $battle_tag = implode('_', explode('#', $tag)); $u = $api_url . $reg . '/' . $battle_tag; return scrape($u); }
function shouldScrapeByTerm($term) { $cc = new cURL(); $output = $cc->get('http://www.nyu.edu/registrar/listings/'); $output = substring($output, 'Course data was last updated on ', '<form method="post" name="selForm" action="results.html">', true); $date = substr($output, 0, strlen($output) - 5); $scrapeDate = date('Y-m-d H:i:s', strtotime($date)); $sql = 'SELECT * FROM `nyu_courses_settings`'; $result = mysql_query($sql); $row = mysql_fetch_assoc($result); if (!(stripos($term, "Fall") === false)) { $column = "latest_fall_scrape"; } else { if (!(stripos($term, "Spring") === false)) { $column = "latest_spring_scrape"; } else { if (!(stripos($term, "Winter") === false)) { $column = "latest_ws_scrape"; } else { if (!(stripos($term, "Summer") === false)) { $column = "latest_ws_scrape"; } } } } if ($row[$column] != $scrapeDate) { scrape($term, $scrapeDate); deleteOldData($term, $scrapeDate); $sql = "UPDATE `nyu_courses_settings` SET `{$column}` = '{$scrapeDate}'"; mysql_query($sql); } }
function scrape($s, $offset) { // locate para $p_start = strpos($s, "<p>", $offset); $p_end = strpos($s, "</p>", $p_start); if ($p_start === false || $p_end === false) { return false; } // extract para $p = substr($s, $p_start, $p_end - $p_start); if ($p === false) { return false; } // get the right link $link = find_apt_link($p, 0); // if link found return it if ($link !== false) { return $link; } else { return scrape($s, $p_end + 4); } }
array_push($results, $result); } //Clean loop variables $option = null; $result = null; unset($option); unset($result); if ($i % 100 === 0) { scraperwiki::save_sqlite(array('path'), $results); $results = array(); } } //Clean up $query = null; $xpath = null; $ch = null; unset($query); unset($xpath); unset($ch); //Final save to database scraperwiki::save_sqlite(array('path'), $results); $results = null; unset($results); echo "Peak memory usage: " . memory_get_peak_usage() . "\n"; } $case_types = array(67, 57, 65, 66, 62, 61, 55, 12, 11, 8, 63, 58, 20, 37, 34, 42, 21, 39, 35, 5, 24, 27, 54, 6, 29, 7, 64, 53, 52, 16, 51, 15, 28, 14, 1, 25, 31, 3, 19, 33, 2, 10, 23, 32, 60, 56, 74, 36, 40, 68, 9, 26, 43, 73, 4, 22, 17, 50); for ($year = date('Y'); $year < date('Y') + 1; $year++) { foreach ($case_types as $case_type) { scrape($year, $case_type); } }
function updatedata() { global $CURRENTPATH, $TABLE_PREFIX, $btit_settings; require_once $CURRENTPATH . '/getscrape.php'; global $update_interval; if ($update_interval == 0) { return; } $now = time(); $res = get_result("SELECT last_time as lt FROM {$TABLE_PREFIX}tasks WHERE task='update'", true, $btit_settings['cache_duration']); $row = $res[0]; if (!$row) { do_sqlquery("INSERT INTO {$TABLE_PREFIX}tasks (task, last_time) VALUES ('update',{$now})"); return; } $ts = $row['lt']; if ($ts + $update_interval > $now) { return; } do_sqlquery("UPDATE {$TABLE_PREFIX}tasks SET last_time={$now} WHERE task='update' AND last_time = {$ts}"); if (!mysqli_affected_rows($GLOBALS["___mysqli_ston"])) { return; } $res = get_result("SELECT announce_url FROM {$TABLE_PREFIX}files WHERE external='yes' ORDER BY lastupdate ASC LIMIT 1", true, $btit_settings['cache_duration']); if (!$res || count($res) == 0) { return; } // get the url to scrape, take 5 torrent at a time (try to getting multiscrape) $row = $res[0]; $resurl = get_result("SELECT info_hash FROM {$TABLE_PREFIX}files WHERE external='yes' AND announce_url='" . $row['announce_url'] . "' ORDER BY lastupdate ASC LIMIT 5", true, $btit_settings['cache_duration']); if (!$resurl || count($resurl) == 0) { return $combinedinfohash = array(); } foreach ($resurl as $id => $rhash) { $combinedinfohash[] = $rhash['info_hash']; } //scrape($row["announce_url"],$row["info_hash"]); scrape($row[0], implode("','", $combinedinfohash)); }
function scrape($session, $session_id, $zone, $bench, $appeal_date, $page) { echo "Loading data ...\n"; $header = array('X-Prototype-Version: 1.4.0', 'X-Requested-With: XMLHttpRequest', 'SOAPAction: ""', 'Content-type: application/x-www-form-urlencoded text/xml; charset=UTF-8', 'request-type: SOAP'); $soap = '<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"><Body xmlns="http://schemas.xmlsoap.org/soap/envelope/"><GetUpdatedObjects xmlns="http://schemas.eclipse.org/birt"><Operation><Target><Id>Document</Id><Type>Document</Type></Target><Operator>GetPage</Operator><Oprand><Name>Appeal No</Name><Value></Value></Oprand><Oprand><Name>__isdisplay__Appeal No</Name><Value></Value></Oprand><Oprand><Name>Serial No</Name><Value></Value></Oprand><Oprand><Name>__isdisplay__Serial No</Name><Value></Value></Oprand><Oprand><Name>Assessee Name</Name><Value>_</Value></Oprand><Oprand><Name>__isdisplay__Assessee Name</Name><Value></Value></Oprand><Oprand><Name>searchWhat</Name><Value>searchByDate</Value></Oprand><Oprand><Name>__isdisplay__searchWhat</Name><Value>searchByDate</Value></Oprand><Oprand><Name>AssType</Name><Value>null</Value></Oprand><Oprand><Name>__isdisplay__AssType</Name><Value>DontKnow</Value></Oprand><Oprand><Name>appealDate</Name><Value>' . $appeal_date . '</Value></Oprand><Oprand><Name>__isdisplay__appealDate</Name><Value>' . $appeal_date . '</Value></Oprand><Oprand><Name>Bench</Name><Value>' . $bench . '</Value></Oprand><Oprand><Name>__isdisplay__Bench</Name><Value>' . $bench . '</Value></Oprand><Oprand><Name>__page</Name><Value>' . $page . '</Value></Oprand><Oprand><Name>__svg</Name><Value>false</Value></Oprand><Oprand><Name>__page</Name><Value>' . $page . '</Value></Oprand><Oprand><Name>__taskid</Name><Value></Value></Oprand></Operation></GetUpdatedObjects></Body></soap:Envelope>'; $ch = curl_init(); curl_setopt_array($ch, array(CURLOPT_URL => 'http://www.itatonline.in:8080/itat/jsp/runBirt2.jsp?subAction=showReoprt&__report=CaseDetails1_' . $zone . '.rptdesign&searchWhat=searchByDate&Serial%20No=&Appeal%20No=&Assessee%20Name=&AssType=null&appealDate=' . urlencode($appeal_date) . '&Bench=' . $bench . '&__sessionId=' . $session_id, CURLOPT_COOKIE => $session, CURLOPT_HTTPHEADER => $header, CURLOPT_POST => true, CURLOPT_POSTFIELDS => $soap, CURLOPT_RETURNTRANSFER => true)); $dom = new DOMDocument(); @$dom->loadHTML('<html><body>' . htmlspecialchars_decode(curl_exec($ch)) . '</body></html>'); $xpath = new DOMXPath($dom); $pages = $xpath->query('//updatedata/data/page'); if ($pages->item(0)) { $results_page = intval($pages->item(0)->childNodes->item(0)->nodeValue); } else { echo 'No data found for ' . $zone . ' : ' . $bench . ' on ' . $appeal_date . " ...\n"; return false; //no data } $total_pages = intval($pages->item(0)->childNodes->item(1)->nodeValue); $query = $xpath->query('//tr[@class="style_9"]'); curl_close($ch); $header = null; $soap = null; $xpath = null; $pages = null; $dom = null; $ch = null; unset($header); unset($soap); unset($xpath); unset($pages); unset($dom); unset($ch); echo "Parsing results ...\n"; $results = array(); $n = $query->length; for ($i = 0; $i < $n; $i++) { $row = $query->item($i); @($result = array('appeal_number' => trim($row->childNodes->item(2)->nodeValue), 'assessee_name' => trim($row->childNodes->item(4)->nodeValue), 'filed_by' => trim($row->childNodes->item(6)->nodeValue), 'zone' => $zone, 'bench' => $bench, 'filing_date' => trim($row->childNodes->item(10)->nodeValue))); if ($result['appeal_number']) { array_push($results, $result); } } scraperwiki::save_sqlite(array('appeal_number'), $results); echo "Peak memory usage: " . memory_get_peak_usage() . "\n"; $query = null; $results = null; unset($query); unset($results); if ($results_page + 1 < $total_pages) { scrape($session, $session_id, $zone, $bench, $appeal_date, $results_page + 1); } }
for ($i = 0; $i < $n; $i++) { $row = $query->item($i); if ($row->getElementsByTagName('a')->item(0)) { $url = $row->getElementsByTagName('a')->item(0)->getAttribute('href'); } else { $url = null; } array_push($results, @($result = array('name' => trim($row->childNodes->item(0)->nodeValue), 'url' => $url, 'source' => trim($row->childNodes->item(2)->nodeValue), 'date' => trim($row->childNodes->item(4)->nodeValue)))); if ($result['name'] == 'There are currently no firms listed under this section.') { $results = array(); } //Clean loop variables $option = null; $result = null; unset($option); unset($result); } //Clean up $options = null; $ch = null; unset($options); unset($ch); scraperwiki::save_sqlite(array('name'), $results); $results = null; unset($results); echo "Peak memory usage: " . memory_get_peak_usage() . "\n"; } $pages = array('0-9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'); foreach ($pages as $page) { scrape($page); }
unset($result); if ($i % 100 === 0) { scraperwiki::save_sqlite(array('unique_id'), $results); $results = array(); } } //Clean up $query = null; $xpath = null; $ch = null; unset($query); unset($xpath); unset($ch); //Final save to database scraperwiki::save_sqlite(array('unique_id'), $results); $results = null; unset($results); echo "Peak memory usage: " . memory_get_peak_usage() . "\n"; if ($n < 10) { return true; } else { return false; } } $i = 1; $a = scrape($i); $i++; while ($a === false) { $a = scrape($i); $i++; }
function do_sanity($ts = 0) { // Lets try upping the max_execution_time and memory_limit if we can (Code from Pet/FM) if (@ini_get("max_execution_time") < 300) { @ini_set("max_execution_time", 300); } if (trim(@ini_get("memory_limit"), "M") < 128) { @ini_set("memory_limit", "128M"); } global $clean_interval, $BASEURL, $btit_settings, $XBTT_USE, $db_prefix, $autopruneusers, $email_on_prune, $days_members, $days_not_comfirm, $days_to_email, $PRIVATE_ANNOUNCE, $TORRENTSDIR, $CURRENTPATH, $LIVESTATS, $LOG_HISTORY, $TABLE_PREFIX, $DOXPATH, $DBDT; $THIS_BASEPATH = dirname(__FILE__); $days = 14; $time = time() - $days * 86400; mysqli_query($GLOBALS["___mysqli_ston"], "DELETE FROM {$TABLE_PREFIX}bugs WHERE status != 'na' AND added < {$time}") or sqlerr(__FILE__, __LINE__); //delete bots after 48 hours mysqli_query($GLOBALS["___mysqli_ston"], "DELETE FROM {$TABLE_PREFIX}bots WHERE visit < (NOW() - INTERVAL 2880 MINUTE)"); //end bots //delete last up/downloads after 48 hours mysqli_query($GLOBALS["___mysqli_ston"], "DELETE FROM {$TABLE_PREFIX}downloads WHERE date < (NOW() - INTERVAL 2880 MINUTE)"); //end last up/downloads //show images in shoutbox if ($btit_settings["endtch"] == TRUE) { $shout = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT * FROM {$TABLE_PREFIX}chat ORDER BY id DESC LIMIT 1"); $shoutrow = mysqli_fetch_assoc($shout); $fp = $btit_settings["fix_chat"]; if ($shoutrow["count"] >= $btit_settings["don_chat"]) { if ($btit_settings["ran_chat"] == TRUE) { do_sqlquery("INSERT INTO {$TABLE_PREFIX}chat (uid, time, name, text) VALUES (0," . time() . ", 'System','[img]{$BASEURL}/images/shouts/shout.php[/img]')"); } else { do_sqlquery("INSERT INTO {$TABLE_PREFIX}chat (uid, time, name, text) VALUES (0," . time() . ", 'System','[img]{$BASEURL}/images/shouts/" . $fp . "[/img]')"); } } } //show images in shoutbox end //happy hour $happy_r = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT UNIX_TIMESTAMP(value_s) AS happy , value_i AS happys from {$TABLE_PREFIX}avps where arg='happyhour'") or sqlerr(__FILE__, __LINE__); $happy_a = mysqli_fetch_array($happy_r); $curDate = time(); $happyTime = $happy_a["happy"] + 3600; if ($happy_a["happys"] == 0) { $happyHour = happyHour(); mysqli_query($GLOBALS["___mysqli_ston"], "UPDATE {$TABLE_PREFIX}avps set value_s=" . sqlesc($happyHour) . ", value_i='1' WHERE arg='happyhour' LIMIT 1 ") or sqlerr(__FILE__, __LINE__); } elseif ($happy_a["happys"] == 1 && $curDate > $happyTime) { mysqli_query($GLOBALS["___mysqli_ston"], "UPDATE {$TABLE_PREFIX}avps set value_i='0' WHERE arg='happyhour' LIMIT 1 "); } $switch = do_sqlquery("SELECT * FROM `{$TABLE_PREFIX}files` WHERE `external`='no'", true); $switch_happy = mysqli_fetch_array($switch); if ($switch_happy["happy_hour"] == "yes") { if (ishappyHour("check") && $happyTime > "0:00") { do_sqlquery("ALTER TABLE `{$TABLE_PREFIX}files` CHANGE `happy` `happy` ENUM( 'yes', 'no' ) NULL DEFAULT 'yes'") or sqlerr(); do_sqlquery("UPDATE `{$TABLE_PREFIX}files` SET `happy`='yes' WHERE `external`='no'", true); } else { do_sqlquery("ALTER TABLE `{$TABLE_PREFIX}files` CHANGE `happy` `happy` ENUM( 'yes', 'no' ) NULL DEFAULT 'no'") or sqlerr(); do_sqlquery("UPDATE `{$TABLE_PREFIX}files` SET `happy`='no' WHERE `external`='no'", true); } } // happy hour // featured if ($btit_settings["auto_feat"] == TRUE) { $feat = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT info_hash , leechers , image from {$TABLE_PREFIX}files where image!='' ORDER BY leechers DESC limit 1"); $resfeat = mysqli_fetch_array($feat); do_sqlquery("INSERT INTO {$TABLE_PREFIX}featured (fid,torrent_id) VALUES ('','{$resfeat['info_hash']}')"); } //featured //sb if ($XBTT_USE) { $ressb = do_sqlquery("SELECT uid FROM xbt_files_users as u INNER JOIN xbt_files as x ON u.fid=x.fid WHERE u.left = '0' AND x.flags='0' AND u.active='1'"); if (mysqli_num_rows($ressb) > 0) { while ($arrsb = mysqli_fetch_assoc($ressb)) { $x = $arrsb["uid"]; quickQuery("UPDATE `{$TABLE_PREFIX}users` SET `seedbonus`=`seedbonus`+'" . number_format(($ts > 0 ? time() - $ts : $clean_interval) / 3600 * $GLOBALS["bonus"], 6, ".", "") . "' WHERE `id` = '{$x}'"); } } } else { $ressb = do_sqlquery("SELECT pid FROM {$TABLE_PREFIX}peers WHERE status = 'seeder'"); if (mysqli_num_rows($ressb) > 0) { while ($arrsb = mysqli_fetch_assoc($ressb)) { $x = $arrsb['pid']; quickQuery("UPDATE `{$TABLE_PREFIX}users` SET `seedbonus`=`seedbonus`+'" . number_format(($ts > 0 ? time() - $ts : $clean_interval) / 3600 * $GLOBALS["bonus"], 6, ".", "") . "' WHERE `pid`= '{$x}'"); } } } //sb //warn $query = do_sqlquery("SELECT * FROM `{$TABLE_PREFIX}users` WHERE warn='yes'"); while ($conf = mysqli_fetch_assoc($query)) { if (mysqli_num_rows($query) > 0) { $expire_dat = $conf['warnadded']; $expire2 = strtotime($expire_dat); $nown = strtotime("now"); if ($nown >= $expire2) { $subj = sqlesc("Your Warning time is expired !!"); $msg = sqlesc("You are not longer Warned , please be carefull to not make the same mistake again !!"); send_pm(0, $conf[id], $subj, $msg); mysqli_query($GLOBALS["___mysqli_ston"], "UPDATE {$TABLE_PREFIX}users SET warn='no' WHERE id='{$conf['id']}'") or sqlerr(); } } } //warn //remove boot after expiration require_once load_language("lang_userdetails.php"); $datetime = gmdate("Y-m-d H:i:s"); $bootedstats = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT * FROM {$TABLE_PREFIX}users WHERE addbooted < '{$datetime}' AND booted='yes'"); while ($arr = mysqli_fetch_assoc($bootedstats)) { if (mysqli_num_rows($bootedstats) > 0) { $sub = sqlesc($language["BOOT_SUB"]); $mess = sqlesc($language["BOOT_MSG"]); send_pm(0, $arr[id], $sub, $mess); mysqli_query($GLOBALS["___mysqli_ston"], "UPDATE {$TABLE_PREFIX}users SET booted='no' WHERE id='{$arr['id']}'") or sqlerr(); } } //remove boot after expiration // DT request hack start $reqprune = $btit_settings["req_prune"]; $request = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT id FROM {$TABLE_PREFIX}requests WHERE filledby > '0' AND fulfilled < DATE_SUB(NOW(), INTERVAL {$reqprune} DAY)"); $reqrow = mysqli_fetch_assoc($request); $reqid = $reqrow["id"]; if (mysqli_num_rows($request) > 0) { mysqli_query($GLOBALS["___mysqli_ston"], "DELETE FROM {$TABLE_PREFIX}requests WHERE filledby > 0 AND id = {$reqid}"); mysqli_query($GLOBALS["___mysqli_ston"], "DELETE FROM {$TABLE_PREFIX}addedrequests WHERE requestid = {$reqid}"); } // DT request hack end if ($autopruneusers) { $timeout = $days_members * 60 * 60 * 24; $timeout2 = $days_not_comfirm * 60 * 60 * 24; if ($GLOBALS["FORUMLINK"] == "smf") { do_sqlquery("DELETE u,smfm FROM {$TABLE_PREFIX}users u INNER JOIN {$db_prefix}members smfm ON smfm.ID_MEMBER=u.smf_fid INNER JOIN {$TABLE_PREFIX}users_level ul ON ul.id=u.id_level WHERE (u.id_level = '2' AND UNIX_TIMESTAMP(u.lastconnect)<(UNIX_TIMESTAMP()-{$timeout2}) AND ul.auto_prune='yes') OR (UNIX_TIMESTAMP(lastconnect)<(UNIX_TIMESTAMP()-{$timeout}) AND ul.auto_prune='yes')"); } else { do_sqlquery("DELETE u FROM {$TABLE_PREFIX}users u INNER JOIN {$TABLE_PREFIX}users_level ul ON ul.id=u.id_level WHERE (u.id_level = '2' AND UNIX_TIMESTAMP(u.lastconnect)<(UNIX_TIMESTAMP()-{$timeout2}) AND ul.auto_prune='yes') OR (UNIX_TIMESTAMP(lastconnect)<(UNIX_TIMESTAMP()-{$timeout}) AND ul.auto_prune='yes')"); } if ($email_on_prune) { $timeout = $days_to_email * 60 * 60 * 24; $res = get_result("SELECT email, lastconnect FROM {$TABLE_PREFIX}users u INNER JOIN {$TABLE_PREFIX}users_level ul ON ul.id=u.id_level WHERE UNIX_TIMESTAMP()>=(UNIX_TIMESTAMP(lastconnect)+{$timeout}-{$clean_interval}/2) AND UNIX_TIMESTAMP()<(UNIX_TIMESTAMP(lastconnect)+{$timeout}+{$clean_interval}/2) AND UNIX_TIMESTAMP(lastconnect)<(UNIX_TIMESTAMP()-{$timeout}) AND ul.auto_prune='yes'", true); foreach ($res as $id => $rusers) { send_mail($rusers["email"], $language["EMAIL_INACTIVE_SUBJ"], $language["EMAIL_INACTIVE_MSG"] . "\n\n" . $BASEURL . "/index.php"); } } } // Autoprune torrents if ($btit_settings["autotprune"] == TRUE) { quickQuery("UPDATE `{$TABLE_PREFIX}files` `f` " . ($XBTT_USE ? "LEFT JOIN `xbt_files` `xf` ON `f`.`bin_hash`=`xf`.`info_hash`" : "") . " SET `f`.`dead_time`=UNIX_TIMESTAMP() WHERE ((" . ($XBTT_USE ? "`xf`.`seeders`>0 OR `xf`.`leechers`>0" : "`f`.`seeds`>0 OR `f`.`leechers`>0") . ") OR `f`.`dead_time`=0) AND `f`.`external`='no'"); $res = get_result("SELECT `info_hash`, `bin_hash` FROM `{$TABLE_PREFIX}files` WHERE `dead_time`<=" . (time() - $btit_settings["autotprundedays"] * 86400) . " AND `dead_time`!=0 AND `external`='no'"); if (count($res) > 0) { foreach ($res as $row) { quickQuery("DELETE FROM `{$TABLE_PREFIX}files` WHERE `info_hash`='" . mysqli_real_escape_string($DBDT, $row["info_hash"]) . "'"); quickQuery("DELETE FROM `{$TABLE_PREFIX}timestamps` WHERE `info_hash`='" . mysqli_real_escape_string($DBDT, $row["info_hash"]) . "'"); quickQuery("DELETE FROM `{$TABLE_PREFIX}comments` WHERE `info_hash`='" . mysqli_real_escape_string($DBDT, $row["info_hash"]) . "'"); quickQuery("DELETE FROM `{$TABLE_PREFIX}ratings` WHERE `infohash`='" . mysqli_real_escape_string($DBDT, $row["info_hash"]) . "'"); quickQuery("DELETE FROM `{$TABLE_PREFIX}peers` WHERE `infohash`='" . mysqli_real_escape_string($DBDT, $row["info_hash"]) . "'"); quickQuery("DELETE FROM `{$TABLE_PREFIX}history` WHERE `infohash`='" . mysqli_real_escape_string($DBDT, $row["info_hash"]) . "'"); if ($XBTT_USE) { quickQuery("UPDATE `xbt_files` SET `flags`=1 WHERE `info_hash`='" . mysqli_real_escape_string($DBDT, $row["bin_hash"]) . "'"); } } } } // Autoprune torrents // timed registration $expire_datetr = $btit_settings["regi_d"]; $expire_timetr = $btit_settings["regi_t"]; $expire_datetrs = $expire_datetr . " " . $expire_timetr . ":00:00"; $expiretr = strtotime($expire_datetrs); $nowtr = strtotime("now"); if ($nowtr >= $expiretr) { do_sqlquery("UPDATE `{$TABLE_PREFIX}settings` SET `value`='true' WHERE `key`='regi'", true); } // end timed registration // Anti Hit and Run V2 based on CobraCRK's Anti Hit&Run Mod v1 Enhanced By IntelPentium4 & fatepower // converted ( and improved ) to XBTIT 2 by DiemThuy Nov 2008 if (!$XBTT_USE) { // Get current time $timenow = time(); // Get last time that dosanity() was run $timeres = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT last_time FROM {$TABLE_PREFIX}anti_hit_run_tasks WHERE task='sanity'"); if (mysqli_num_rows($timeres) > 0) { $timearr = mysqli_fetch_array($timeres); $lastrecordedtime = intval($timearr['last_time']); } else { $lastrecordedtime = $timenow - $clean_interval; } // Update Active Seeders' Seeding Clock $res = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT pid, infohash FROM {$TABLE_PREFIX}peers WHERE status = 'seeder'"); if (mysqli_num_rows($res) > 0) { while ($arr = mysqli_fetch_assoc($res)) { $x = $arr['pid']; $t = $arr['infohash']; $pl = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT id FROM {$TABLE_PREFIX}users WHERE pid='" . $x . "'"); if (mysqli_num_rows($pl) > 0) { $ccc = mysqli_result($pl, 0, "id"); } else { $ccc = "Unknown"; } mysqli_query($GLOBALS["___mysqli_ston"], "UPDATE {$TABLE_PREFIX}history SET seed = seed+" . $timenow . "-" . $lastrecordedtime . " WHERE uid = {$ccc} AND infohash='{$t}'"); } } //Update table anti_hit_run_tasks with new time info. $hunden = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT last_time FROM {$TABLE_PREFIX}anti_hit_run_tasks WHERE task='sanity'"); $manneplutt = mysqli_fetch_row($hunden); if (!$manneplutt) { mysqli_query($GLOBALS["___mysqli_ston"], "INSERT INTO {$TABLE_PREFIX}anti_hit_run_tasks (task, last_time) VALUES ('sanity',{$timenow})"); } else { $ts = $manneplutt[0]; mysqli_query($GLOBALS["___mysqli_ston"], "UPDATE {$TABLE_PREFIX}anti_hit_run_tasks SET last_time={$timenow} WHERE task='sanity' AND last_time = {$ts}"); } // Rank who has no anti-hit punishment rule should be excluded $levels = mysqli_query($GLOBALS["___mysqli_ston"], "select id from {$TABLE_PREFIX}users_level order by id"); while ($SingleLevel = mysqli_fetch_array($levels)) { $hasAntiHitRecord = mysqli_query($GLOBALS["___mysqli_ston"], "select id_level from {$TABLE_PREFIX}anti_hit_run where id_level=" . $SingleLevel["id"]); if (mysqli_num_rows($hasAntiHitRecord) == 0) { @mysqli_query($GLOBALS["___mysqli_ston"], "UPDATE `{$TABLE_PREFIX}history`,`users` set hitchecked= 2 where history.uid=users.id and users.id_level=" . $SingleLevel["id"] . " and completed='yes' and hitchecked='0'"); } } $hit_parameters = mysqli_query($GLOBALS["___mysqli_ston"], "select * from {$TABLE_PREFIX}anti_hit_run order by id_level"); while ($hit = mysqli_fetch_array($hit_parameters)) { // Punishment $r = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT DISTINCT uid,infohash FROM {$TABLE_PREFIX}history history inner join {$TABLE_PREFIX}users users on history.uid=users.id WHERE users.id_level=" . $hit["id_level"] . " AND active='no' AND completed='yes' AND hit='no' AND hitchecked= 0 AND date < ( UNIX_TIMESTAMP( ) - (86400 * " . $hit["tolerance_days_before_punishment"] . ")) AND history.downloaded>(1048576 * " . $hit["min_download_size"] . ") AND seed<( 3600 * " . $hit["min_seed_hours"] . ") AND (history.uploaded/history.downloaded)<" . $hit["min_ratio"]); while ($x = mysqli_fetch_array($r)) { @mysqli_query($GLOBALS["___mysqli_ston"], "Update {$TABLE_PREFIX}history SET hit='yes' WHERE uid=" . $x[uid] . " AND infohash='" . $x[infohash] . "' AND hitchecked=0"); if (mysqli_affected_rows($GLOBALS["___mysqli_ston"]) > 0) { if ($hit["reward"] == 'yes') { $reward = "\n\n[color=red]If you want to get the lost amount back , you must seed for at least " . $hit["min_seed_hours"] . " hour(s) or until the file\\'s ratio becomes greater than " . $hit["min_ratio"] . " then your total upload will incremented by " . $hit["upload_punishment"] . " MB !! \n\n\\ [/color][color=purple]This is a automatic system message , so DO NOT reply ![/color]"; } else { $reward = " "; } @mysqli_query($GLOBALS["___mysqli_ston"], "Update {$TABLE_PREFIX}history SET hitchecked= 1 ,punishment_amount=" . $hit["upload_punishment"] . " WHERE uid=" . $x[uid] . " AND infohash='" . $x[infohash] . "' AND hitchecked=0"); @mysqli_query($GLOBALS["___mysqli_ston"], "Update {$TABLE_PREFIX}users SET uploaded=(case when uploaded-(1048576 * " . $hit["upload_punishment"] . ")<0 then 0 else uploaded-(1048576 * " . $hit["upload_punishment"] . ") end) WHERE id={$x['uid']}"); send_pm(0, $x[uid], sqlesc("Auto Hit an Run warning"), sqlesc("You did Hit and Run on the following torrent :\n\n [url]" . $BASEURL . "/index.php?page=details&id={$x['infohash']}[/url] !\n\n\\We did take away " . $hit["upload_punishment"] . " MB as punishment\n\nBe carefull to not make the mistake once more ! " . $reward . "")); // DT reputation system start $reput = do_sqlquery("SELECT * FROM {$TABLE_PREFIX}reputation_settings WHERE id =1"); $setrep = mysqli_fetch_array($reput); $plus = $setrep["rep_hit"]; if ($setrep["rep_is_online"] == FALSE) { //do nothing } else { @mysqli_query($GLOBALS["___mysqli_ston"], "UPDATE {$TABLE_PREFIX}users SET reputation = reputation - '{$plus}' WHERE id='{$x['uid']}'"); } // DT reputation system end // warn at hit and run if ($hit["warn"] == 'yes') { $id = (int) $x[uid]; $warnreason = "Auto Hit & Run Warning"; $warnaddedby = "System"; $added = warn_expiration(mktime(date('H') + 2, date('i'), date('s'), date('m'), date('d') + addslashes($hit["days1"]), date('Y'))); quickQuery('UPDATE ' . $TABLE_PREFIX . 'users SET warn="yes",warns=warns+1,warnreason="' . $warnreason . '",warnaddedby="' . $warnaddedby . '",warnadded="' . $added . '" WHERE id=' . $id); } // end warn at hit and run // boot at hit and run if ($hit["boot"] == 'yes') { $id = (int) $x[uid]; $whybooted = "Auto Hit & Run Ban"; $whobooted = "System"; $addbooted = booted_expiration(mktime(date('H') + 2, date('i'), date('s'), date('m'), date('d') + addslashes($hit["days2"]), date('Y'))); quickQuery('UPDATE ' . $TABLE_PREFIX . 'users SET booted="yes", whybooted="' . $whybooted . '",whobooted="' . $whobooted . '",addbooted="' . $addbooted . '" WHERE id=' . $id); } // end boot at hit and run //Dox Hack Start $r = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT id, filename, added FROM {$TABLE_PREFIX}dox WHERE added < '" . date('Y-m-d', strtotime('-' . $btit_settings["dox_del"] . ' weeks')) . "'"); while ($del = mysqli_fetch_array($r)) { $MANNE = "{$BASEURL}/{$DOXPATH}"; @unlink("{$MANNE}/{$del['filename']}"); quickQuery("DELETE FROM {$TABLE_PREFIX}dox WHERE id={$del['id']}"); } //Dox Hack End // boot after warn at hit and run if ($hit["warnboot"] == 'yes') { $diem = do_sqlquery("SELECT warns FROM {$TABLE_PREFIX}users WHERE id={$x['uid']}"); $thuy = mysqli_fetch_array($diem); if ($thuy["warns"] >= $hit["days3"]) { } $id = (int) $x[uid]; $whybooted = "Auto Hit & Run Ban after beeing warned"; $whobooted = "System"; $addbooted = booted_expiration(mktime(date('H') + 2, date('i'), date('s'), date('m'), date('d') + addslashes($hit["days2"]), date('Y'))); quickQuery('UPDATE ' . $TABLE_PREFIX . 'users SET booted="yes", whybooted="' . $whybooted . '",whobooted="' . $whobooted . '",addbooted="' . $addbooted . '" WHERE id=' . $id); } // end boot after warn at hit and run } } mysqli_free_result($r) || is_object($r) && get_class($r) == "mysqli_result" ? true : false; // Reward if ($hit["reward"] == 'yes') { $rr = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT DISTINCT uid,infohash,punishment_amount FROM {$TABLE_PREFIX}history history inner join {$TABLE_PREFIX}users users on history.uid=users.id WHERE users.id_level=" . $hit["id_level"] . " AND hit='yes' AND completed='yes' AND hitchecked= 1 AND (seed>=( 3600 * " . $hit["min_seed_hours"] . ") or (history.uploaded/history.downloaded)>=" . $hit["min_ratio"] . ")"); while ($xr = mysqli_fetch_array($rr)) { @mysqli_query($GLOBALS["___mysqli_ston"], "Update {$TABLE_PREFIX}history SET hitchecked= 3 WHERE uid=" . $xr[uid] . " AND infohash='" . $xr[infohash] . "' AND hitchecked=1"); if (mysqli_affected_rows($GLOBALS["___mysqli_ston"]) > 0) { @mysqli_query($GLOBALS["___mysqli_ston"], "Update {$TABLE_PREFIX}users SET uploaded=uploaded+(1048576 * " . $xr["punishment_amount"] . ") WHERE id={$xr['uid']}"); send_pm(0, $xr[uid], sqlesc("Thanks (Punishement Removed)"), sqlesc("Thank you very much for seeding back the following torrent:\n\n [url]" . $BASEURL . "/index.php?page=details&id={$xr['infohash']}[/url] !\n\n [color=green]The punishment is now removed and you total upload amount increased by " . $xr["punishment_amount"] . " MB! [/color]\n\n [color=purple]This is a automatic system message , so DO NOT reply ![/color]")); // DT reputation system start if ($setrep["rep_is_online"] == FALSE) { //do nothing } else { @mysqli_query($GLOBALS["___mysqli_ston"], "UPDATE {$TABLE_PREFIX}users SET reputation = reputation + '{$plus}' WHERE id='{$x['uid']}'"); } // DT reputation system end // warn at hit and run if ($hit["warn"] == 'yes') { quickQuery('UPDATE ' . $TABLE_PREFIX . 'users SET warn="no", warns=warns-1 WHERE id=' . $xr[uid]); } // end warn at hit and run // boot at hit and run if ($hit["boot"] == 'yes') { quickQuery('UPDATE ' . $TABLE_PREFIX . 'users SET booted="no" WHERE id=' . $xr[uid]); } // end boot at hit and run } } mysqli_free_result($rr) || is_object($rr) && get_class($rr) == "mysqli_result" ? true : false; } // Who are fine should not be punished @mysqli_query($GLOBALS["___mysqli_ston"], "Update {$TABLE_PREFIX}history,users SET hitchecked= 1 WHERE history.uid=users.id AND users.id_level = users.id_level=" . $hit["id_level"] . " AND completed='yes' AND date < ( UNIX_TIMESTAMP( ) - (86400 * " . $hit["tolerance_days_before_punishment"] . ")) AND hitchecked= 0"); } mysqli_free_result($levels) || is_object($levels) && get_class($levels) == "mysqli_result" ? true : false; mysqli_free_result($hasAntiHitRecord) || is_object($hasAntiHitRecord) && get_class($hasAntiHitRecord) == "mysqli_result" ? true : false; mysqli_free_result($hit_parameters) || is_object($hit_parameters) && get_class($hit_parameters) == "mysqli_result" ? true : false; } // End of Anti Hit and Run //Invalid Login System Hack Start mysqli_query($GLOBALS["___mysqli_ston"], "DELETE FROM {$TABLE_PREFIX}bannedip WHERE comment='max_number_of_invalid_logins_reached'"); //invalid Login System Hack Stop //start freeleech $queryd = do_sqlquery("SELECT free_expire_date, free FROM `{$TABLE_PREFIX}files` WHERE `external`='no'", true); $configd = mysqli_fetch_array($queryd); $expire_dated = $configd['free_expire_date']; $expired = strtotime($expire_dated); $nowd = strtotime("now"); if ($nowd >= $expired && $configd['free'] == 'yes') { do_sqlquery("UPDATE `{$TABLE_PREFIX}files` SET `free`='no',free_expire_date='0000-00-00 00:00:00' WHERE `external`='no'", true); do_sqlquery("ALTER TABLE `{$TABLE_PREFIX}files` CHANGE `free` `free` ENUM( 'yes', 'no' ) NULL DEFAULT 'no'", true); // xbtt if ($XBTT_USE) { do_sqlquery("UPDATE xbt_files SET down_multi=0, flags=2"); do_sqlquery("ALTER TABLE xbt_files CHANGE `down_multi` `down_multi` INT NULL DEFAULT '0'", true); } } // end freeleech $query = do_sqlquery("SELECT * FROM `{$TABLE_PREFIX}lottery_config` WHERE `id`=1", true); $config = mysqli_fetch_array($query); $expire_date = $config['lot_expire_date']; $expire = strtotime($expire_date); $now = strtotime("now"); if ($now >= $expire) { $number_winners = $config['lot_number_winners']; $number_to_win = $config['lot_number_to_win']; $minupload = $config['lot_amount']; $res = do_sqlquery("SELECT `id`, `user` FROM `{$TABLE_PREFIX}lottery_tickets` ORDER BY RAND(NOW()) LIMIT " . $number_winners . "", true); //select number of winners $total = mysqli_num_rows(do_sqlquery("SELECT * FROM `{$TABLE_PREFIX}lottery_tickets`", true)); //select total selled tickets $pot = $total * $minupload; //selled tickets * ticket price $pot += $number_to_win; // ticket prize + minimum win $win = $pot / $number_winners; // prize for each winner $subject = sqlesc("You have won a prize with the lottery"); //subject in pm $msg = sqlesc("Congratulations you have won a prize with our Lottery. Your prize has been added to your account. You won " . makesize($win) . ""); //next 3 rows are the msg for PM $sender = $config['sender_id']; // Sender id, in my case 0 //print the winners and send them PM en give them price while ($row = mysqli_fetch_array($res)) { $ras = do_sqlquery("SELECT `smf_fid`, `id`, `username` FROM `{$TABLE_PREFIX}users` WHERE `id`=" . $row['user'] . "", true); $raw = mysqli_fetch_array($ras); $rec = sqlesc("{$raw['id']}"); $lotid = $raw["id"]; $lotname = $raw["username"]; do_sqlquery("UPDATE `{$TABLE_PREFIX}users` SET `uploaded`=uploaded+" . $win . " WHERE `id`=" . $row['user'] . "", true); $smf = mysqli_fetch_assoc(do_sqlquery("SELECT smf_fid, username FROM `{$TABLE_PREFIX}users` WHERE `id`=" . $row["user"] . "", true)); send_pm($sender, $rec, $subject, $msg); // begin - announce winner in shoutbox do_sqlquery("INSERT INTO {$TABLE_PREFIX}chat (uid, time, name, text) VALUES (0," . time() . ", 'System','[color=red]Lottery winner : [/color][url={$BASEURL}/index.php?page=userdetails&id={$lotid}]" . $lotname . " did win " . makesize($win) . "[/url]')"); // end - announce winner in shoutbox do_sqlquery("INSERT INTO `{$TABLE_PREFIX}lottery_winners` (`id`, `win_user`, `windate`, `price`) VALUES ('', '" . $raw['username'] . "', '" . $expire_date . "', '" . $win . "')"); } do_sqlquery("TRUNCATE TABLE `{$TABLE_PREFIX}lottery_tickets`", true); do_sqlquery("UPDATE `{$TABLE_PREFIX}lottery_config` SET `lot_status`='closed' WHERE `id`=1", true); } // lottery auto start if ($btit_settings["autolot"] == TRUE) { $date_end = lastOfMonth(); $klaar = $config["lot_status"]; if ($klaar == 'closed') { $expire_date = $date_end; $expire_time = 23; $val1 = $expire_date . " " . $expire_time . ":59:59"; $val2 = 1; $val3 = 15 * 1024 * 1024 * 1024; // Gb $val4 = 500 * 1024 * 1024; // Mb $val5 = 'yes'; $val6 = 1; $val7 = 2; do_sqlquery("UPDATE `{$TABLE_PREFIX}lottery_config` SET `lot_expire_date`='" . $val1 . "', `lot_number_winners`='" . $val2 . "', `lot_number_to_win`='" . $val3 . "', `lot_amount`='" . $val4 . "', `lot_status`='" . $val5 . "', `limit_buy`='" . $val6 . "', `sender_id`=" . $val7 . " WHERE `id`=1", true); } // lottery end } // SANITY FOR TORRENTS $results = do_sqlquery("SELECT info_hash, seeds, leechers, dlbytes, filename FROM {$TABLE_PREFIX}files WHERE external='no'"); $i = 0; while ($row = mysqli_fetch_row($results)) { list($hash, $seeders, $leechers, $bytes, $filename) = $row; $timeout = time() - intval($GLOBALS["report_interval"] * 2); // for testing purpose -- begin $resupd = do_sqlquery("SELECT * FROM {$TABLE_PREFIX}peers where lastupdate < " . $timeout . " AND infohash='{$hash}'"); if (mysqli_num_rows($resupd) > 0) { while ($resupdate = mysqli_fetch_array($resupd)) { $uploaded = max(0, $resupdate["uploaded"]); $downloaded = max(0, $resupdate["downloaded"]); $pid = $resupdate["pid"]; $ip = $resupdate["ip"]; // update user->peer stats only if not livestat if (!$LIVESTATS) { if ($PRIVATE_ANNOUNCE) { quickQuery("UPDATE {$TABLE_PREFIX}users SET uploaded=uploaded+{$uploaded}, downloaded=downloaded+{$downloaded} WHERE pid='{$pid}' AND id>1 LIMIT 1"); } else { // ip quickQuery("UPDATE {$TABLE_PREFIX}users SET uploaded=uploaded+{$uploaded}, downloaded=downloaded+{$downloaded} WHERE cip='{$ip}' AND id>1 LIMIT 1"); } } // update dead peer to non active in history table if ($LOG_HISTORY) { $resuser = do_sqlquery("SELECT id FROM {$TABLE_PREFIX}users WHERE " . ($PRIVATE_ANNOUNCE ? "pid='{$pid}'" : "cip='{$ip}'") . " ORDER BY lastconnect DESC LIMIT 1"); $curu = @mysqli_fetch_row($resuser); quickquery("UPDATE {$TABLE_PREFIX}history SET active='no' WHERE uid={$curu['0']} AND infohash='{$hash}'"); } } } // for testing purpose -- end quickQuery("DELETE FROM {$TABLE_PREFIX}peers where lastupdate < " . $timeout . " AND infohash='{$hash}'"); quickQuery("UPDATE {$TABLE_PREFIX}files SET lastcycle='" . time() . "' WHERE info_hash='{$hash}'"); $results2 = do_sqlquery("SELECT status, COUNT(status) from {$TABLE_PREFIX}peers WHERE infohash='{$hash}' GROUP BY status"); $counts = array(); while ($row = mysqli_fetch_row($results2)) { $counts[$row[0]] = 0 + $row[1]; } quickQuery("UPDATE {$TABLE_PREFIX}files SET leechers=" . (isset($counts["leecher"]) ? $counts["leecher"] : 0) . ",seeds=" . (isset($counts["seeder"]) ? $counts["seeder"] : 0) . " WHERE info_hash=\"{$hash}\""); if ($bytes < 0) { quickQuery("UPDATE {$TABLE_PREFIX}files SET dlbytes=0 WHERE info_hash=\"{$hash}\""); } } //Auto Seedbox Start do_sqlquery("UPDATE {$TABLE_PREFIX}files SET `seedbox`='0' "); $sid = do_sqlquery("select * FROM {$TABLE_PREFIX}peers WHERE `ip` =" . $btit_settings["seedip"]); while ($sow = mysqli_fetch_array($sid)) { do_sqlquery("UPDATE {$TABLE_PREFIX}files SET `seedbox`='1' WHERE `info_hash`='{$sow['infohash']}'"); } //Auto Seedbox End // END TORRENT'S SANITY //DT Uploader Medals global $btit_settings; do_sqlquery("UPDATE {$TABLE_PREFIX}users SET `up_med`='0' "); $time_B = 86400 * $btit_settings['UPD']; $time_E = strtotime(now); $time_D = $time_E - $time_B; $res = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT uploader,count( * ) AS Count FROM {$TABLE_PREFIX}files WHERE UNIX_TIMESTAMP(data) > " . $time_D . " GROUP by uploader"); while ($fetch_U = mysqli_fetch_array($res)) { if ($fetch_U['Count'] >= $btit_settings['UPB'] and $fetch_U['Count'] < $btit_settings['UPS']) { do_sqlquery("UPDATE {$TABLE_PREFIX}users SET `up_med`='1' WHERE `id`='{$fetch_U['uploader']}'"); } if ($fetch_U['Count'] >= $btit_settings['UPS'] and $fetch_U['Count'] < $btit_settings['UPG']) { do_sqlquery("UPDATE {$TABLE_PREFIX}users SET `up_med`='2' WHERE `id`='{$fetch_U['uploader']}'"); } if ($fetch_U['Count'] >= $btit_settings['UPG']) { do_sqlquery("UPDATE {$TABLE_PREFIX}users SET `up_med`='3' WHERE `id`='{$fetch_U['uploader']}'"); } } //DT end Uploader Medals // high speed report if ($btit_settings["highswitch"] == TRUE) { if ($GLOBALS["XBTT_USE"]) { $resch = do_sqlquery("SELECT `uid` `id`, `up_rate` FROM `xbt_files_users` WHERE `up_rate` >= (" . $btit_settings["highspeed"] . "*1024) AND `active`=1"); } else { $resch = do_sqlquery("SELECT `p`.`upload_difference`, `p`.`announce_interval`, `u`.`id` FROM `{$TABLE_PREFIX}peers` `p` LEFT JOIN `{$TABLE_PREFIX}users` `u` ON " . ($PRIVATE_ANNOUNCE ? "`p`.`pid`=`u`.`pid`" : "`p`.`ip`=`u`.`cip`") . " WHERE (`p`.`upload_difference`/`p`.`announce_interval`) >= (" . $btit_settings["highspeed"] . "*1024)"); } if (@mysqli_num_rows($resch) > 0) { while ($rowch = mysqli_fetch_assoc($resch)) { if (!is_null($rowch["id"])) { if ($GLOBALS["XBTT_USE"]) { $transferrate = "Upload speed " . round($rowch["up_rate"] / 1024, 2) . " KB/sec ?!"; } else { $transferrate = "Upload speed " . round(round($rowch['upload_difference'] / $rowch['announce_interval']) / 1024, 2) . " KB/sec ?!"; } $high = $rowch["id"]; if ($btit_settings["highonce"] == TRUE) { $once = do_sqlquery("SELECT `id` FROM `{$TABLE_PREFIX}reports` WHERE `addedby` = 0 AND `votedfor` = {$high} AND `type` = 'user' AND reason LIKE 'Upload speed%'"); if (@mysqli_num_rows($once) === FALSE) { do_sqlquery("INSERT INTO `{$TABLE_PREFIX}reports` (`addedby`,`votedfor`,`type`,`reason`) VALUES ('0','{$high}','user', '{$transferrate}')"); } } if ($btit_settings["highonce"] == FALSE) { do_sqlquery("INSERT INTO `{$TABLE_PREFIX}reports` (`addedby`,`votedfor`,`type`,`reason`) VALUES ('0','{$high}','user', '{$transferrate}')"); } } } } } // end high speed report // DT reputation system start $reput = do_sqlquery("SELECT * FROM {$TABLE_PREFIX}reputation_settings WHERE id =1"); $setrep = mysqli_fetch_array($reput); if ($setrep["rep_is_online"] == FALSE or $setrep["rep_en_sys"] == FALSE) { //do nothing } else { // demote $rep_sub = sqlesc("You are Demoted!"); $rep_msg = sqlesc($setrep["rep_dm_text"]); $rep_demotelist = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT id FROM {$TABLE_PREFIX}users WHERE reputation < " . $setrep["rep_dm"] . " AND id_level = " . $setrep["rep_pr_id"]); while ($rep_demote = mysqli_fetch_assoc($rep_demotelist)) { mysqli_query($GLOBALS["___mysqli_ston"], "UPDATE {$TABLE_PREFIX}users SET id_level=" . $setrep["rep_dm_id"] . " WHERE id=" . $rep_demote["id"]); send_pm(0, $rep_demote[id], $rep_sub, $rep_msg); } // promote $rep_subj = sqlesc("You are Promoted!"); $rep_mesg = sqlesc($setrep["rep_pm_text"]); $rep_promotelist = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT id FROM {$TABLE_PREFIX}users WHERE reputation > " . $setrep["rep_pr"] . " AND id_level = " . $setrep["rep_dm_id"]); while ($rep_promote = mysqli_fetch_assoc($rep_promotelist)) { mysqli_query($GLOBALS["___mysqli_ston"], "UPDATE {$TABLE_PREFIX}users SET id_level=" . $setrep["rep_pr_id"] . " WHERE id=" . $rep_promote["id"]); send_pm(0, $rep_promote[id], $rep_subj, $rep_mesg); } } // DT reputation system start // Client Log for XBT if ($GLOBALS["XBTT_USE"]) { $timeout = time() - intval($GLOBALS["report_interval"] * 2); quickQuery("DELETE FROM `xbt_announce_log` WHERE `mtime`<={$timeout}"); $res = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT `u`.`id`, INET_NTOA(`al`.`ipa`) `ip`, `al`.`port`, LOWER(HEX(`al`.`peer_id`)) `peer_id`, `u`.`clientinfo` FROM `xbt_announce_log` `al` LEFT JOIN `{$TABLE_PREFIX}users` `u` ON `al`.`uid`=`u`.`id` WHERE `al`.`event`=2 GROUP BY `al`.`peer_id` ORDER BY `u`.`id` ASC"); if (@mysqli_num_rows($res) > 0) { $old_clients = array(); $current_clients = array(); while ($row = mysqli_fetch_assoc($res)) { $client = getagent("", $row["peer_id"]); if (!empty($row["clientinfo"])) { if (!array_key_exists($row["id"], $old_clients)) { $old_clients[$row["id"]] = unserialize($row["clientinfo"]); } if (!array_key_exists($row["id"], $current_clients)) { $current_clients[$row["id"]] = unserialize($row["clientinfo"]); } } else { if (!array_key_exists($row["id"], $old_clients)) { $old_clients[$row["id"]] = array(); } if (!array_key_exists($row["id"], $current_clients)) { $current_clients[$row["id"]] = array(); } } if (!in_array($client . "[X]" . $row["port"], $current_clients[$row["id"]])) { if (count($current_clients[$row["id"]]) == 20) { unset($current_clients[$row["id"]][0]); unset($current_clients[$row["id"]][1]); $newlist = array(); foreach ($current_clients[$row["id"]] as $v) { $newlist[] = $v; } $current_clients[$row["id"]] = $newlist; } $current_clients[$row["id"]][] = $client . "[X]" . $row["port"]; $current_clients[$row["id"]][] = time() . "[X]" . $row["ip"]; } } foreach ($current_clients as $k => $v) { $s_old_clients = serialize($old_clients[$k]); $s_current_clients = serialize($current_clients[$k]); if ($s_old_clients != $s_current_clients) { mysqli_query($GLOBALS["___mysqli_ston"], "UPDATE `{$TABLE_PREFIX}users` SET `clientinfo`='" . mysqli_real_escape_string($DBDT, $s_current_clients) . "' WHERE `id`={$k}"); } } } } // Client Log for XBT // banbutton $timeout = $btit_settings["bandays"] * 86400; @mysqli_query($GLOBALS["___mysqli_ston"], "DELETE FROM `{$TABLE_PREFIX}signup_ip_block` WHERE (UNIX_TIMESTAMP() - `added`) > {$timeout}"); // end banbutton # global language, $clean_interval, $reload_cfg_interval; global $language, $clean_interval, $reload_cfg_interval; require dirname(__FILE__) . '/khez.php'; quickQuery('OPTIMIZE TABLE `' . $TABLE_PREFIX . 'khez_configs`;'); # hacks can start here ==Khez== // warn-ban system with acp by DT global $XBTT_USE; $resset = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT * FROM {$TABLE_PREFIX}low_ratio_ban_settings WHERE id ='1'"); $art = mysqli_fetch_assoc($resset); $resban = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT * FROM {$TABLE_PREFIX}low_ratio_ban "); while ($ban = mysqli_fetch_assoc($resban)) { if ($art["wb_sys"] == TRUE) { if ($XBTT_USE) { $udownloaded = "u.downloaded+IFNULL(x.downloaded,0)"; $uuploaded = "u.uploaded+IFNULL(x.uploaded,0)"; $utables = "{$TABLE_PREFIX}users u LEFT JOIN xbt_users x ON x.uid=u.id"; } else { $udownloaded = "u.downloaded"; $uuploaded = "u.uploaded"; $utables = "{$TABLE_PREFIX}users u"; } $min_dl = $ban["wb_down"] * 1024 * 1024 * 1024; // find bad users 1 $demotelist = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT id FROM {$utables} WHERE {$udownloaded} > " . $min_dl . " AND {$uuploaded}/{$udownloaded} <= " . $ban["wb_one"] . " AND id_level=" . $ban["wb_rank"] . " AND rat_warn_level = 0 "); while ($demote = mysqli_fetch_assoc($demotelist)) { // warn bad users 1 do_sqlquery("UPDATE {$TABLE_PREFIX}users SET rat_warn_level = 1 , rat_warn_time = NOW() WHERE id=" . $demote["id"]); // DT reputation system start $reput = do_sqlquery("SELECT * FROM {$TABLE_PREFIX}reputation_settings WHERE id =1"); $setrep = mysqli_fetch_array($reput); $plus = $setrep["rep_minrep"]; if ($setrep["rep_is_online"] == FALSE) { //do nothing } else { @mysqli_query($GLOBALS["___mysqli_ston"], "UPDATE {$TABLE_PREFIX}users SET reputation = reputation - '{$plus}' WHERE id='{$demote['id']}'"); } // DT reputation system end // send pm bad users 1 $sub = sqlesc("Low Ratio Warning!"); $msg = sqlesc($art["wb_text_one"]); send_pm(0, $demote[id], $sub, $msg); // add warn symbol 1 if ($ban["wb_warn"] == TRUE) { $id = $demote["id"]; $warnreason = "Low Ratio Warning"; $warnaddedby = "System"; $added = warn_expiration(mktime(date('H') + 2, date('i'), date('s'), date('m'), date('d') + addslashes($ban['wb_days_one']), date('Y'))); quickQuery('UPDATE ' . $TABLE_PREFIX . 'users SET warn="yes",warns=warns+1,warnreason="' . $warnreason . '",warnaddedby="' . $warnaddedby . '",warnadded="' . $added . '" WHERE id=' . $id); } } // time date stuff $time_AA = 86400 * $ban['wb_days_one']; $time_BB = strtotime(now); $time_CC = $time_BB - $time_AA; // find bad users 2 $demotelistt = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT id,rat_warn_time FROM {$utables} WHERE {$udownloaded} > " . $min_dl . " AND {$uuploaded}/{$udownloaded} <= " . $ban["wb_two"] . " AND id_level=" . $ban["wb_rank"] . " AND rat_warn_level = 1 "); while ($demotee = mysqli_fetch_assoc($demotelistt)) { $time_DD = strtotime($demotee["rat_warn_time"]); if ($time_DD <= $time_CC) { // warn bad users 2 do_sqlquery("UPDATE {$TABLE_PREFIX}users SET rat_warn_level = 2 , rat_warn_time = NOW() WHERE id=" . $demotee["id"]); // DT reputation system start $reput = do_sqlquery("SELECT * FROM {$TABLE_PREFIX}reputation_settings WHERE id =1"); $setrep = mysqli_fetch_array($reput); $plus = $setrep["rep_minrep"]; if ($setrep["rep_is_online"] == FALSE) { //do nothing } else { @mysqli_query($GLOBALS["___mysqli_ston"], "UPDATE {$TABLE_PREFIX}users SET reputation = reputation - '{$plus}' WHERE id='{$demotee['id']}'"); } // DT reputation system end // send pm bad users 2 $sub = sqlesc("Low Ratio Warning Two!"); $msg = sqlesc($art["wb_text_two"]); send_pm(0, $demotee[id], $sub, $msg); // add warn symbol 2 if ($ban["wb_warn"] == TRUE) { $warnreason = "Low Ratio Warning"; $warnaddedby = "System"; $added = warn_expiration(mktime(date('H') + 2, date('i'), date('s'), date('m'), date('d') + addslashes($ban['wb_days_two']), date('Y'))); quickQuery('UPDATE ' . $TABLE_PREFIX . 'users SET warn="yes",warns=warns+1,warnreason="' . $warnreason . '",warnaddedby="' . $warnaddedby . '",warnadded="' . $added . '" WHERE id=' . $id); } } } // unwarn user who did improve $unwarnone = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT id,rat_warn_time FROM {$utables} WHERE {$udownloaded} > " . $min_dl . " AND {$uuploaded}/{$udownloaded} > " . $ban["wb_one"] . " AND id_level=" . $ban["wb_rank"] . " AND rat_warn_level = 1 "); while ($unwarna = mysqli_fetch_assoc($unwarnone)) { $iid = $unwarna["id"]; quickQuery('UPDATE ' . $TABLE_PREFIX . 'users SET rat_warn_level=rat_warn_level-1 WHERE id=' . $iid); } // time date stuff $time_EE = 86400 * $ban['wb_days_two']; $time_FF = $time_BB - $time_EE; // find bad users 3 $demotelisttt = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT id,rat_warn_time FROM {$utables} WHERE {$udownloaded} > " . $min_dl . " AND {$uuploaded}/{$udownloaded} <= " . $ban["wb_three"] . " AND id_level=" . $ban["wb_rank"] . " AND rat_warn_level = 2 "); while ($demoteee = mysqli_fetch_assoc($demotelisttt)) { $time_GG = strtotime($demoteee["rat_warn_time"]); if ($time_GG <= $time_FF) { // warn bad users 3 do_sqlquery("UPDATE {$TABLE_PREFIX}users SET rat_warn_level = 3 , rat_warn_time = NOW() WHERE id=" . $demoteee["id"]); // DT reputation system start $reput = do_sqlquery("SELECT * FROM {$TABLE_PREFIX}reputation_settings WHERE id =1"); $setrep = mysqli_fetch_array($reput); $plus = $setrep["rep_minrep"]; if ($setrep["rep_is_online"] == FALSE) { //do nothing } else { @mysqli_query($GLOBALS["___mysqli_ston"], "UPDATE {$TABLE_PREFIX}users SET reputation = reputation - '{$plus}' WHERE id='{$demoteee['id']}'"); } // DT reputation system end // send pm bad users 3 $sub = sqlesc("Final Low Ratio Warning!"); $msg = sqlesc($art["wb_text_fin"]); send_pm(0, $demoteee[id], $sub, $msg); // add warn symbol 3 if ($ban["wb_warn"] == TRUE) { $id = $demoteee["id"]; $warnreason = "Low Ratio Warning"; $warnaddedby = "System"; $added = warn_expiration(mktime(date('H') + 2, date('i'), date('s'), date('m'), date('d') + addslashes($ban['wb_days_fin']), date('Y'))); quickQuery('UPDATE ' . $TABLE_PREFIX . 'users SET warn="yes",warns=warns+1,warnreason="' . $warnreason . '",warnaddedby="' . $warnaddedby . '",warnadded="' . $added . '" WHERE id=' . $id); } } } // unwarn user who did improve 2 $unwarntwo = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT id,rat_warn_time FROM {$utables} WHERE {$udownloaded} > " . $min_dl . " AND {$uuploaded}/{$udownloaded} > " . $ban["wb_two"] . " AND id_level=" . $ban["wb_rank"] . " AND rat_warn_level = 2 "); while ($unwarnb = mysqli_fetch_assoc($unwarntwo)) { $oid = $unwarnb["id"]; quickQuery('UPDATE ' . $TABLE_PREFIX . 'users SET rat_warn_level=rat_warn_level-2 WHERE id=' . $oid); } // time date stuff $time_HH = 86400 * $ban['wb_days_fin']; $time_II = $time_BB - $time_HH; // find bad users 4 $demotelistttt = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT id,rat_warn_time FROM {$utables} WHERE {$udownloaded} > " . $min_dl . " AND {$uuploaded}/{$udownloaded} <= " . $ban["wb_fin"] . " AND id_level=" . $ban["wb_rank"] . " AND rat_warn_level = 3 "); while ($demoteeee = mysqli_fetch_assoc($demotelistttt)) { $time_JJ = strtotime($demoteeee["rat_warn_time"]); if ($time_JJ <= $time_II) { // ban bad users 4 if ($btit_settings["en_sys"] == TRUE) { do_sqlquery("UPDATE {$TABLE_PREFIX}users SET rat_warn_level = 4 ,rat_warn_time = NOW(), id_level=" . $btit_settings["dm_id"] . " WHERE id=" . $demoteeee["id"]); } else { do_sqlquery("UPDATE {$TABLE_PREFIX}users SET rat_warn_level = 4 ,rat_warn_time = NOW(), bandt='yes' WHERE id=" . $demoteeee["id"]); } } } // unwarn user who did improve last $unwarnthree = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT id,rat_warn_time FROM {$utables} WHERE {$udownloaded} > " . $min_dl . " AND {$uuploaded}/{$udownloaded} > " . $ban["wb_three"] . " AND id_level=" . $ban["wb_rank"] . " AND rat_warn_level = 3 "); while ($unwarnc = mysqli_fetch_assoc($unwarnthree)) { $lid = $unwarnc["id"]; quickQuery('UPDATE ' . $TABLE_PREFIX . 'users SET rat_warn_level=rat_warn_level-3 WHERE id=' . $lid); } } } // warn-ban system with acp end // optimize peers table quickQuery("OPTIMIZE TABLE {$TABLE_PREFIX}peers"); // delete readposts when topic don't exist or deleted *** should be done by delete, just in case quickQuery("DELETE readposts FROM {$TABLE_PREFIX}readposts LEFT JOIN topics ON readposts.topicid = topics.id WHERE topics.id IS NULL"); // delete readposts when users was deleted *** should be done by delete, just in case quickQuery("DELETE readposts FROM {$TABLE_PREFIX}readposts LEFT JOIN users ON readposts.userid = users.id WHERE users.id IS NULL"); // deleting orphan image in captcha folder (if image code is enabled) $CAPTCHA_FOLDER = realpath("{$CURRENTPATH}/../{$CAPTCHA_FOLDER}"); if ($dir = @opendir($CAPTCHA_FOLDER . "/")) { while (false !== ($file = @readdir($dir))) { if ($ext = substr(strrchr($file, "."), 1) == "png") { unlink("{$CAPTCHA_FOLDER}/{$file}"); } } @closedir($dir); } quickQuery("UPDATE `{$TABLE_PREFIX}users` SET `birthday_bonus`=0 WHERE DAYOFMONTH(`dob`)!=" . date('j')); $res = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT `u`.`id`, `u`.`dob`,`l`.`language_url` FROM `{$TABLE_PREFIX}users` `u` LEFT JOIN `language` `l` ON `u`.`language`=`l`.`id` WHERE DAYOFMONTH(`u`.`dob`)=" . date('j') . " AND MONTH(`u`.`dob`)=" . date('n') . " AND `u`.`dob`!=CURDATE() AND `u`.`birthday_bonus`=0 ORDER BY `l`.`language_url` ASC"); if (@mysqli_num_rows($res) > 0) { global $THIS_BASEPATH; $firstrun = 1; $englang = "language/english"; $templang = $englang; require_once $THIS_BASEPATH . "/" . $englang . "/lang_main.php"; while ($row = mysqli_fetch_assoc($res)) { if ($row["language_url"] != $templang) { if ($firstrun != 1) { // Reset the language to English before loading the new language require_once $THIS_BASEPATH . "/" . $englang . "/lang_main.php"; } // Load the new language etc. require_once $THIS_BASEPATH . "/" . $row["language_url"] . "/lang_main.php"; $templang = $row["language_url"]; $firstrun = 0; } $dob = explode("-", $row["dob"]); $age = userage($dob[0], $dob[1], $dob[2]); $bonus = round($age * $btit_settings["birthday_bonus"] * 1073741824); $query1 = "UPDATE `{$TABLE_PREFIX}users` SET `uploaded`=`uploaded`+{$bonus}, `birthday_bonus`=1 WHERE `id`=" . $row["id"]; quickQuery($query1); send_pm(0, $row["id"], addslashes($language["HB_SUBJECT"]), addslashes($language["HB_MESSAGE_1"] . makesize($bonus) . $language["HB_MESSAGE_2"] . $btit_settings["birthday_bonus"] . $language["HB_MESSAGE_3"])); } } //timed rank $datetimedt = date("Y-m-d H:i:s"); $rankstats = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT * FROM {$TABLE_PREFIX}users WHERE timed_rank < '{$datetimedt}' AND rank_switch='yes'"); while ($arrdt = mysqli_fetch_assoc($rankstats)) { if (mysqli_num_rows($rankstats) > 0) { $res6 = mysqli_query($GLOBALS["___mysqli_ston"], "SELECT level FROM {$TABLE_PREFIX}users_level WHERE id ='{$arrdt['old_rank']}'"); $arr6 = mysqli_fetch_assoc($res6); $oldrank = $arr6[level]; $subj = sqlesc("Your timed rank is expired !"); $msg = sqlesc("Your timed rank is expired !\n\n Your rank did changed back to " . $oldrank . "\n\n [color=red]This is a automatic system message , so DO NOT reply ![/color]"); send_pm(0, $arrdt["id"], $subj, $msg); mysqli_query($GLOBALS["___mysqli_ston"], "UPDATE {$TABLE_PREFIX}users SET rank_switch='no', id_level = old_rank WHERE id='{$arrdt['id']}'") or sqlerr(); } } //timed rank end //begin invitation system by dodge global $INV_EXPIRES; $deadtime = $INV_EXPIRES * 86400; $user = do_sqlquery("SELECT inviter FROM {$TABLE_PREFIX}invitations WHERE time_invited < DATE_SUB(NOW(), INTERVAL {$deadtime} SECOND)"); @($arr = mysqli_fetch_assoc($user)); if (mysqli_num_rows($user) > 0) { mysqli_query($GLOBALS["___mysqli_ston"], "UPDATE {$TABLE_PREFIX}users SET invitations=invitations+1 WHERE id = '" . $arr["inviter"] . "'"); mysqli_query($GLOBALS["___mysqli_ston"], "DELETE FROM {$TABLE_PREFIX}invitations WHERE inviter = '" . $arr["inviter"] . "' AND time_invited < DATE_SUB(NOW(), INTERVAL {$deadtime} SECOND)"); } //end invitation system do_updateranks(); // auto ext update $num_torrents_to_update = 5; $torrents = get_result("SELECT `announces`, `info_hash` FROM `{$TABLE_PREFIX}files` WHERE `external`='yes' ORDER BY `lastupdate` DESC LIMIT " . $num_torrents_to_update); if (count($torrents) > 0) { require_once "getscrape.php"; for ($i = 0; $i < count($torrents); $i++) { $announces = @unserialize($torrents[$i]['announces']) ? unserialize($torrents[$i]['announces']) : array(); if (count($announces) > 0) { $keys = array_keys($announces); $random = mt_rand(0, count($keys) - 1); $url = $keys[$random]; scrape($url, $torrents[$i]['info_hash']); } } } // auto ext update // OK We're finished, let's reset max_execution_time and memory_limit back to the php.ini defaults @ini_restore("max_execution_time"); @ini_restore("memory_limit"); }
function scrape($cookie, $viewstate, $event_validation, $page, $from, $to, $last_row_number) { echo "Loading data (" . $from['day'] . '/' . $from['month'] . '/' . $from['year'] . ' -> ' . $to['day'] . '/' . $to['month'] . '/' . $to['year'] . ") ...\n"; $dom = new DOMDocument(); if (!$page) { @$dom->loadHTML(first_page($cookie, $viewstate, $event_validation, $from, $to)); } else { @$dom->loadHTML(not_first_page($cookie, $viewstate, $event_validation, $page)); } $xpath = new DOMXPath($dom); @($records_found = intval(preg_replace('/\\D/', '', $xpath->query('//*[@id="Label5"]')->item(0)->nodeValue))); $viewstate = $xpath->query('//*[@id="__VIEWSTATE"]')->item(0)->getAttribute('value'); $event_validation = $xpath->query('//*[@id="__EVENTVALIDATION"]'); if ($event_validation->length) { $event_validation = $event_validation->item(0)->getAttribute('value'); } $query = $xpath->query('//table[@id="DataGrid1"]/tr'); $dom = null; $xpath = null; $ch = null; unset($dom); unset($xpath); unset($ch); echo "Parsing results ...\n"; $results = array(); $n = $query->length; for ($i = 1; $i < $n; $i++) { $row = $query->item($i); preg_match('/filename=(.*?)">/i', $row->ownerDocument->saveXML($row), $href); @($result = array('case_number' => trim($row->childNodes->item(1)->nodeValue), 'path' => $href[1], 'petitioner' => trim(substr($row->childNodes->item(2)->nodeValue, 0, strpos($row->childNodes->item(2)->nodeValue, ' Vs. '))), 'respondent' => trim(substr($row->childNodes->item(2)->nodeValue, strpos($row->childNodes->item(2)->nodeValue, ' Vs. ') + 5)), 'judge' => trim(trim($row->childNodes->item(3)->nodeValue, 'Coram: ')), 'judgment_date' => trim($row->childNodes->item(4)->nodeValue), 'unique_id' => preg_replace('/\\s+/', '', $row->childNodes->item(0)->nodeValue . '-' . $row->childNodes->item(1)->nodeValue . '-' . $from['day'] . $from['month'] . $from['year'] . '-' . $to['day'] . $to['month'] . $to['year']))); if ($result['case_number'] || $result['path'] || $result['petitioner'] || $result['respondent'] || $result['judgment_date']) { $new_last_row_number = intval($row->childNodes->item(0)->nodeValue); array_push($results, $result); } } scraperwiki::save_sqlite(array('unique_id'), $results); echo "Peak memory usage: " . memory_get_peak_usage() . "\n"; $query = null; $results = null; unset($query); unset($results); if (!isset($new_last_row_number)) { $new_last_row_number = 0; } if ($last_row_number !== $new_last_row_number) { scrape($cookie, $viewstate, $event_validation, $page + 1, $from, $to, $new_last_row_number); } }
$xpath = null; unset($xpath); $n = $query->length; echo "Parsing Results ...\n"; $results = array(); for ($i = 0; $i < $n; $i++) { $row = $query->item($i)->getElementsByTagName('td'); @($result = array('id' => preg_replace('/\\s/', '', $year . '-' . $row->item(2)->nodeValue), 'year' => trim($year), 'country_rank' => trim($row->item(0)->nodeValue), 'company' => trim($row->item(1)->nodeValue), 'global_rank' => trim($row->item(2)->nodeValue), 'city' => trim($row->item(3)->nodeValue), 'country' => trim($country), 'revenue' => trim($row->item(4)->nodeValue))); if (!empty($result['company'])) { array_push($results, $result); } //Clean loop variables $option = null; $result = null; unset($option); unset($result); } //Clean up $options = null; $ch = null; unset($options); unset($ch); scraperwiki::save_sqlite(array('id'), $results); $results = null; unset($results); echo "Peak memory usage: " . memory_get_peak_usage() . "\n"; } $countries = array('Australia', 'Austria', 'Belgium', 'BelgiumNetherlands', 'Brazil', 'Britain', 'BritainNetherlands', 'Canada', 'China', 'Colombia', 'Denmark', 'Finland', 'France', 'Germany', 'Hungary', 'India', 'Ireland', 'Israel', 'Italy', 'Japan', 'Luxembourg', 'Malaysia', 'Mexico', 'Netherlands', 'Norway', 'Poland', 'Portugal', 'Russia', 'SaudiArabia', 'Singapore', 'SouthKorea', 'Spain', 'Sweden', 'Switzerland', 'Taiwan', 'Thailand', 'Turkey', 'US', 'UnitedArabEmirates', 'Venezuela'); foreach ($countries as $country) { scrape(date('Y'), $country); }
//Clean loop variables $option = null; $result = null; unset($option); unset($result); } //Clean up $options = null; unset($options); //Save to database scraperwiki::save_sqlite(array('unique_id'), $results); $results = null; unset($results); echo "Peak memory usage: " . memory_get_peak_usage() . "\n"; } $data = scraperwiki::scrape('http://securities.stanford.edu/fmi/xsl/SCACPUDB/recordlist.xsl?-db=SCACPUDB&-lay=Search&-sortfield.1=FIC_DateFiled&-sortfield.2=LitigationName&-sortorder.1=ascend&-max=1&-findall=&-lay.response=ListGral&-encoding=UTF-8&-grammar=fmresultset&-skip=0'); echo "Loading data ...\n"; $dom = new DOMDocument(); @$dom->loadHTML($data); $xpath = new DOMXPath($dom); $dom = null; unset($dom); preg_match('/\\d+\\s+of\\s+(.*?)\\s+\\|/', $xpath->query('//td[@align="right"]')->item(0)->nodeValue, $total_results_match); $total_results = intval($total_results_match[1]); $xpath = null; unset($xpath); $results_per_page = 1000; $n = $total_results / $results_per_page + 1; for ($page = 1; $page < $n; $page++) { scrape($page, $results_per_page); }
unset($option); unset($result); if ($i % 100 === 0) { //scraperwiki::save_sqlite(array('unique_id'), $results); $results = array(); } } //Clean up $options = null; $ch = null; unset($options); unset($ch); //Final save to database //scraperwiki::save_sqlite(array('unique_id'), $results); $results = null; unset($results); } for ($year = date('Y'); $year < date('Y') + 1; $year++) { for ($month = date('m') - 1; $month < date('m') + 1; $month++) { $end_day = cal_days_in_month(CAL_GREGORIAN, $month, $year); $from = array(1, $month, $year); $to = array($end_day, $month, $year); $details = scrape($from, $to, 'get_results_total', null, 'NA', 'NA'); $cookie = $details[0]; $pages = ceil($details[1] / 50); scrape($from, $to, 'first_page', $cookie, 0, $pages); for ($i = 1; $i < $pages; $i++) { scrape($from, $to, 'next_page', $cookie, $i, $pages); } } }
$q = "SELECT searches.* FROM searches HAVING searches.updated_at IS NULL OR searches.updated_at < unix_timestamp(DATE_SUB(NOW(), INTERVAL 1 HOUR))"; $stmt = $db->prepare($q); $p = array(date('Y-m-d H:i:s')); $stmt->execute($p); $data = $stmt->fetchAll(PDO::FETCH_ASSOC); $res = array(); //TODO reduce scraping frequency when there are rarely any new ads foreach ($data as $search) { if (isset($search['lang']) && $search['lang']) { setLeBonMailLocale(getSearchLocale($search['lang'])); } else { setLeBonMailLocale(array('fr_FR', 'fr_FR.utf8')); } if ($search['annonces'] == null || $search['updated_at'] == null || $search['updated_at'] < time() - 3600 * $app->config('cache_duration')) { $app->getLog()->info('scraping for ' . $search['url']); $annonces = scrape($search['url'], $app->getLog()); $app->getLog()->info('Got ' . count($annonces) . ' adds'); if (is_array($annonces) && !empty($annonces)) { $p = array(serialize($annonces), time(), serialize($annonces[0]), $search['hash']); $q = "UPDATE searches SET annonces=?, updated_at=?, last=? WHERE hash=?"; $stmt = $db->prepare($q); $stmt->execute($p); } else { $p = array(time(), $search['hash']); $q = "UPDATE searches SET updated_at=? WHERE hash=?"; $stmt = $db->prepare($q); $stmt->execute($p); } } else { $annonces = unserialize($search['annonces']); }
function updatedata() { global $db; require_once INCL_PATH . 'getscrape.php'; global $update_interval; if (0 + $update_interval == 0) { return; } $now = vars::$timestamp; $res = @$db->query("SELECT last_time FROM tasks WHERE task='update'"); $row = @$res->fetch_array(MYSQLI_BOTH); if (!$row) { $db->query("INSERT INTO tasks (task, last_time) VALUES ('update', " . $now . ")"); return; } $ts = $row[0]; if ($ts + $update_interval > $now) { return; } $db->query("UPDATE tasks SET last_time = " . $now . " WHERE task = 'update' AND last_time = " . $ts); if (!$db->affected_rows) { return; } // new control time is lastupdate (before the current one) - update interval $ts = $ts - $update_interval; $res = @$db->query("SELECT announce_url FROM namemap WHERE external = 'yes' AND UNIX_TIMESTAMP(lastupdate) < " . $ts . " ORDER BY lastupdate ASC LIMIT 1"); if (!$res || $res->num_rows == 0) { return; } // get the url to scrape, take 5 torrent at a time (try to getting multiscrape) $row = $res->fetch_row(); $resurl = @$db->query("SELECT info_hash FROM namemap WHERE external = 'yes' AND UNIX_TIMESTAMP(lastupdate) < " . $ts . " AND announce_url = '" . $row[0] . "' ORDER BY lastupdate DESC LIMIT 5"); if (!$resurl || $resurl->num_rows == 0) { return $combinedinfohash = array(); } while ($rhash = $resurl->fetch_row()) { $combinedinfohash[] = $rhash[0]; } scrape($row[0], implode("','", $combinedinfohash)); }
$results = null; unset($query); unset($results); } scrape(); function scrape() { echo "Loading data ...\n"; $dom = new DOMDocument(); @$dom->loadHTML(scraperwiki::scrape('http://www.bmfbovespa.com.br/en-us/markets/equities/companies/companies-with-tag-along-rights.aspx?idioma=en-us')); $xpath = new DOMXPath($dom); $dom = null; unset($dom); $rows = $xpath->query('//div[@class="tabela"]/table/tbody/tr'); $n = $rows->length; $xpath = null; unset($xpath); $results = array(); for ($i = 0; $i < $n; $i++) { $row = $rows->item($i)->getElementsByTagName('td'); array_push($results, array('unique_id' => preg_replace('/\\s+/', '', trim(@$row->item(0)->nodeValue) . '-' . trim(@$row->item(2)->nodeValue)), 'name' => trim(@$row->item(0)->nodeValue), 'corporate_resolution' => trim(@$row->item(1)->nodeValue), 'event_date' => trim(@$row->item(2)->nodeValue), 'tag_voting_pct' => trim(@$row->item(3)->nodeValue), 'tag_non_voting_pct' => trim(@$row->item(4)->nodeValue), 'listing_segment' => trim(@$row->item(5)->nodeValue))); } scraperwiki::save_sqlite(array('unique_id'), $results); echo "Peak memory usage: " . memory_get_peak_usage() . "\n"; $query = null; $results = null; unset($query); unset($results); } scrape();
array_push($results, $result); //Clean loop variables $option = null; $result = null; unset($option); unset($result); if ($i % 100 === 0 && count($results) > 1) { //If $results gets above 100 during the loop, then save and clear scraperwiki::save_sqlite(array('unique_id'), $results); $results = array(); } } //Clean up $query = null; $xpath = null; $ch = null; unset($query); unset($xpath); unset($ch); //Final save to database scraperwiki::save_sqlite(array('unique_id'), $results); $results = null; unset($results); echo "Peak memory usage: " . memory_get_peak_usage() . "\n"; } $court_act_codes = array(396 => 'Absorbed Employees Act, 1965', 53 => 'Adovcates Act 1961', 68 => 'Ancient Monuments & Archeological Sites & Remains Act,1958', 90 => 'Andhra Pradesh (Agricultural Produce & Livestock)Markets Act, 1966', 376 => 'Apprentices Act, 1961', 80 => 'Arbitration Act', 82 => 'ARBITRATION AND CONCILIATION ACT, 1996', 15 => 'B.T. & A.L. Act', 402 => 'Bank & Financial Institution Act, 1993', 39 => 'Banking Regulation Act', 400 => 'C.C.S (Classification,Control & Appeal) Rules,1965', 398 => 'C.C.S(C.C.A.) Rules,1965.', 1 => 'C.P.C.', 399 => 'Cental Civil Services (Conduct) Rules, 1964', 67 => 'Central Civil Services (Revised Pay) Rules, 1997', 34 => 'Central Excise & Salt Act', 85 => 'Central Excise act 1944 under rule 59Q', 401 => 'Central Industrial Tribunals Act', 18 => 'City & Industrial Development Act', 312 => 'Code of Communidade', 73 => 'Commission of Inquiry Act, 1952', 355 => 'Company Act, 1956', 79 => 'Company Laws', 54 => 'Constitution of India', 360 => 'Consumer Protection Act', 369 => 'Contempt of Court Act', 27 => 'Contract Labour (Reg. & Abolition) Act', 356 => 'Court Fees Act, 1870', 201 => 'Criminal Procedure Code', 35 => 'Customs Act, 1962', 392 => 'Customs Tariff Act 1975', 302 => 'Devasthan Regulation (Regu. de Mazanias)', 51 => 'Displace Persons (C&R) Act 1954', 319 => 'Divorce case under the Local Law', 72 => 'Dock Workers (Regulation of Employment) Act, 1948', 30 => 'Employees State Insurance Act, 1948', 28 => 'Employment Provident Fund & Misc. P. Act', 57 => 'Essential Commodities Act, 1955', 71 => 'Evidence Act', 26 => 'Factories Act', 84 => 'FAMILY ACT CODE', 365 => 'Foreign Exchange Managment Act, 1999', 388 => 'Forest (Conservation) Act 1980', 359 => 'Forest Act, 1927', 351 => 'G.D.D. Agricultural Produce Marketing (Regulation) Rules, 1969', 350 => 'G.D.D. Medical Education Service Rules, 1979', 340 => 'G.D.D. Tourist Trade Act', 322 => 'G.D.D.Abolition of Proprietorship of Lands in Diu Act,1971', 323 => 'G.D.D.Abolition of Proprietorship of Lands in Diu(D.C.O.R.)Rules,1972', 324 => 'G.D.D.Absorbed Employees Conditions of Service(Amendment)Rules,1976', 336 => 'G.D.D.Agricultural Tenancy Act and Rules with Amendments upto 1992', 326 => 'G.D.D.Civil Service (Judicial Branch) Rules-Amendment', 327 => 'G.D.D.Entertainment Tax Act, 196', 352 => 'G.D.D.Industrial Development Act,1965', 329 => 'G.D.D.Irrigation Act, 1973', 330 => 'G.D.D.Land Revenue Code, 1968 Vol.I', 331 => 'G.D.D.Mundkar Act and Rules', 332 => 'G.D.D.Municipalities (Amendment) Act, 1978', 328 => 'G.D.D.Public Gambling Act, 1976', 333 => 'G.D.D.Village Panchayats, Regulation and Rules', 88 => 'Gazetted Posts Recruitment Rules, 1998', 354 => 'Goa Childrens Act, 2003. U/s 6', 383 => 'Goa Civil Court Act, 1965', 346 => 'Goa Civil Services (Retirement) Rules, 2000', 307 => 'Goa D.D Excise Duty Act 1964', 306 => 'Goa D.D Housing Boards Act 1968', 320 => 'Goa D.D Land Revenue Code 1966', 305 => 'Goa D.D Lease Rent & Evic Cont. Act 1968', 303 => 'Goa D.D Mundkars(Pro. from Evic.)Act1975', 309 => 'Goa D.D Municipalities Act 1968', 315 => 'Goa D.D Regist.of Tourism Trd.Act 1982', 308 => 'Goa D.D Sales Tax Act 1964', 317 => 'Goa D.D School Education Act 1984', 313 => 'Goa D.D Shops & Establishment Act 1973', 304 => 'Goa D.D Town & Country Planning Act 1974', 301 => 'Goa D.D. Adminstrative Tribunal act 1965', 310 => 'Goa Daman Diu Panchayat Act 1993', 335 => 'Goa Govt.Seniority Rules, 1967', 366 => 'Goa maintenance of public order & safety act 1988', 343 => 'Goa Police Subordinate Services(Discipline & Appeal)Rules, 1975', 377 => 'Goa Public Health Act', 349 => 'Goa Right to Information Act, 1997', 373 => 'Goa Rural Improvement and Welfare Cess Act', 334 => 'Goa Sales Tax Act, with Amendment upto 1993', 394 => 'Goa School Education Act', 342 => 'Goa State Reorganization Act, 1987', 385 => 'Goa Tax on Entry of Goods Act, 2000', 314 => 'Goa Tax on Luxuries(Hotel &Lodg)Act 1988', 348 => 'Goa Tourist Places (Protection & Maintenance) Act, 2001', 347 => 'Goa University Act, 1994', 364 => 'Goa Vexatious Litigation (Prevention) Act., 2007', 321 => 'Goa. D.D Agricultural Tenancy Act 1964', 316 => 'Grant-in-Aid Code of Sec.School.Coll.etc', 6 => 'Guardian and Wards Act', 8 => 'Hindu Marriage Act', 380 => 'IMMORAL TRAFFIC (PREVENTION) ACT (ITPA), 1956', 406 => 'Import and Export (Control) Act, 1947', 32 => 'Income Tax Act, 1961', 5 => 'Indian Arbitration Act', 11 => 'Indian Citizenship Act', 74 => 'Indian Constitution Act', 75 => 'Indian Contract Act', 9 => 'Indian Divorce Act', 362 => 'Indian Easements Act', 59 => 'Indian Electricity Act', 83 => 'INDIAN LIMITATION ACT OF 1963', 46 => 'Indian Medical Council Act- 1956', 202 => 'Indian Penal Code', 3 => 'Indian Railways Act,1890', 81 => 'Indian Registration Act, 1908', 7 => 'Indian Succession Act', 404 => 'Indian Telegraph Act, 1885', 20 => 'Industrial Dispute Act, 1947.', 31 => 'Industrial Employment (Standing O.) Act', 318 => 'Inventory Matters', 13 => 'Land Acquisition Act', 397 => 'Limitation Act', 44 => 'M.R.T.U. & P.U.L.P. Act', 311 => 'Mah. Co-op. Societies Act to Goa D.D', 86 => 'Maharashtra Co-op. Soc. Act', 371 => 'Maharashtra Co-operative Societies Act 1960', 38 => 'Major Port Trusts Act', 358 => 'Mamlatdar\'s Courts Act, 1906', 70 => 'Merchant Shipping Act, 1958', 16 => 'Mines & Minerals (Reg. & Devolp) Act', 21 => 'Minimum Wages Act, 1948.', 36 => 'Motor Vehicle Act', 4 => 'Motor Vehicles Act,1939', 370 => 'Multi State Co-Operative Socities Act 2002', 203 => 'N.D.P.S. Act,1985', 40 => 'Negotiable Instruments Act', 0 => 'Unknown', 405 => 'Patents Act, 1970', 25 => 'Payment of Wages Act', 353 => 'PORTUGUESE CIVIL CODE', 43 => 'Presidency Small Causes Court Act', 375 => 'Prevention of Food Adulteration Act, 1954', 372 => 'Protection of Women from Domestic Violence Act, 05', 409 => 'Provision of the Families Laws of Goa', 17 => 'Public Premises (Eviction) Act', 37 => 'Railways Act', 52 => 'Rent Act', 391 => 'Representation of the People Act, 1951', 10 => 'Reqn.& Acqn.of Immoveable Property Act', 367 => 'Right to Information Act', 337 => 'Rules of P&C of Business of Goa State L.Assembly(Second Edition)1967', 47 => 'S.C. & S.T. Orders (Amendment) Act.', 19 => 'Secondary School Code', 386 => 'Securitisation & Reconstruction of Financial Assests & Enforcements of Securities Interests Act 2002', 87 => 'Service Matter', 41 => 'Sick Textile Undertakings Act', 381 => 'SLUM AREAS (IMPROVEMENT & CLEARANCE) ACT, 1956', 407 => 'Special Economic Zone Act, 2005', 42 => 'Specific Relief Act', 50 => 'Standards of Weights & Measures Act-1976', 357 => 'Suits Valuation Act, 1887', 78 => 'The Air Pollution Control Act, 1981', 63 => 'The Central Excise Act, 1944', 64 => 'The Central Excise Rules, 1944', 403 => 'The Cinematograph Act, 1952', 387 => 'The City of Panaji Corporation Act, 2002', 77 => 'The Contempt of Courts Act, 1971', 12 => 'The Copy Right Act', 393 => 'The Dock Workers (Regulation of Employment) (inapplicability to Major Ports) Act, 1997', 65 => 'The Drugs & Cosmetics Act, 1940', 66 => 'The Drugs & Cosmetics Rules, 1945', 58 => 'The Environment Protectin Act, 1986', 379 => 'THE GOA ANIMAL PRESERVATION ACT, 1995', 378 => 'The Goa Cess on Fluid Milk (Control) Act, 2000', 408 => 'The Goa Co-operative Societies Act, 2001', 345 => 'The Goa Panchayat Raj Act, 1993', 339 => 'The Goa Panchayat Raj Act, 1994', 341 => 'The Goa Preservation of Trees Act, 1984', 344 => 'The Goa Public Men\'s Corruption(Investigation & Inquiries)Act,1988', 395 => 'The Goa Tax on Infrastructure Act, 2009', 338 => 'The Goa University Act, 1984', 374 => 'The Goa, Daman & Diu Fire Force Act, 1986', 368 => 'The Human Right Act, 1993', 390 => 'The Indian Fisheries Act,1897 and The Goa Fisheries Rules,1981', 76 => 'The Inland Vessels Act, 1917', 89 => 'The Maharashtra Agricultural Produce Marketing(Regulation)Act,1963', 361 => 'The Marine Insurance Act', 62 => 'The Medical & Toilet Preparations (Excise Duties) Rules, 1956', 61 => 'The Medical & Toilet Preparations(Excise Duties) Act, 1955', 22 => 'The Payment of Bonus Act, 1965.', 23 => 'The Payment of Gratuity Act', 60 => 'The Ports Act, 1908', 363 => 'The Prevention of Corruption Act', 411 => 'THE PROBATION OF OFFENDERS ACT', 325 => 'The State Agricultural Credit Corporations Act, 1968', 382 => 'The State Financial Corporation Act, 1951', 56 => 'The University Grants Commission Act, 1956', 389 => 'The Wakf Act, 1995', 29 => 'Trade Unions Act, 1926', 45 => 'Transfer of Property Act', 55 => 'Unlawful Activities (prevention) Act, 1967', 14 => 'Urban & Agricultural Land Ceiling Act', 49 => 'Urban Land CC & RD Act 1976', 69 => 'Water(Prevention & Control of Pollution) Act, 1974', 33 => 'Wealth Tax Act, 1957', 384 => 'Wild Life Protection Act, 1972', 24 => 'Workmen\'s Compensation Act'); for ($year = date("Y"); $year < date("Y") + 1; $year++) { foreach ($court_act_codes as $code => $court_act) { scrape($year, $code, $court_act); } }
unset($option); unset($result); } //Clean up $options = null; $ch = null; unset($options); unset($ch); //Save to database scraperwiki::save_sqlite(array('case_number'), $results); $results = null; unset($results); echo "Peak memory usage: " . memory_get_peak_usage() . "\n"; } $ch = curl_init(); curl_setopt_array($ch, array(CURLOPT_URL => 'http://lobis.nic.in/ori/title1.php?dc=19&fflag=1', CURLOPT_RETURNTRANSFER => true, CURLOPT_HEADER => true, CURLOPT_POSTFIELDS => http_build_query(array('radiosel' => 'O', 'p_name' => '%', 'frdate' => date("d/m/Y", strtotime("-10 days")), 'todate' => date("d/m/Y"), 'Submit' => 'Submit')), CURLOPT_CONNECTTIMEOUT => 0, CURLOPT_TIMEOUT => 1000)); echo "Loading data ...\n"; $dom = new DOMDocument(); $data = curl_exec($ch); preg_match('/Set-Cookie: (.*?);/i', $data, $cookie); @$dom->loadHTML($data); $xpath = new DOMXPath($dom); $dom = null; unset($dom); $total_results = intval(preg_replace('/\\D+/', '', $xpath->query('//p[@class="style2"]')->item(0)->nodeValue)); $xpath = null; unset($xpath); for ($i = 0; $i < $total_results / 10; $i++) { //10 results per page scrape($cookie[1], $i * 10); }
} preg_match('/(.*?)Advocate :/i', preg_replace('/Vs./i', '', $row[15]), $respondent); @($result = array('case_number' => trim(preg_replace('/\\d+\\/\\d+/', ' $0', $row[5])), 'path' => $path[1], 'petitioner' => trim($row[13]), 'respondent' => trim($respondent[1]), 'status' => trim(preg_replace('/\\[|\\]/', '', $row[7])))); $result['unique_id'] = preg_replace('/\\s+/', '', $result['path'] . '-' . $result['case_number']); array_push($results, $result); $case_number = null; $row = null; $result = null; unset($case_number); unset($row); unset($result); } //Clean up and save $query = null; unset($query); scraperwiki::save_sqlite(array('unique_id'), $results); echo "Peak memory usage: " . memory_get_peak_usage() . "\n"; } $year = date('Y'); $dom = new DOMDocument(); @$dom->loadHTML(scraperwiki::scrape('http://delhihighcourt.nic.in/dhc_case_status_list_new.asp?ayear=&pyear=' . $year . '&SNo=2&SRecNo=0&dno=&dyear=&ctype=&cno=&cyear=&party=%&adv=')); $xpath = new DOMXPath($dom); $dom = null; unset($dom); $query = $xpath->query('//tr[@bgcolor="#ebebeb"]'); @($total_pages = intval($xpath->query('//div[@class="archivelink"]')->item(0)->nodeValue)); $xpath = null; unset($xpath); for ($this_page = 0; $this_page < $total_pages; $this_page++) { scrape($year, $this_page, $total_pages); }
$ret = $html->find($path)->plaintext; break; default: $ret = $html->find($path); break; } // clean up memory $html->clear(); unset($dirty_html); unset($clean_html); unset($html); return $ret; } // ----------------------------------------------------------------------------- $url = !empty($_REQUEST['url']) ? $_REQUEST['url'] : 'http://www.openrecommender.org'; $path = !empty($_REQUEST['path']) ? $_REQUEST['path'] : 'a'; $parse = !empty($_REQUEST['parse']) ? $_REQUEST['parse'] : 'default'; $output = !empty($_REQUEST['output']) ? $_REQUEST['output'] : 'attr'; $debug = !empty($_REQUEST['debug']) ? $_REQUEST['debug'] : false; $text = scrape($url, $path, $parse); if ($debug) { echo "<hr/><pre>"; print_r($text); echo "</pre><hr/>"; } foreach ($text as $k => $v) { echo '<strong>' . $k . ':</strong> ' . $v; foreach ($v->{$output} as $val => $t) { echo ' | ' . $t . '<br/>'; } }
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, // EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // //////////////////////////////////////////////////////////////////////////////////// if (!defined("IN_BTIT")) { die("non direct access!"); } $id = AddSlashes(isset($_GET["id"]) ? $_GET["id"] : false); if (!isset($id) || !$id) { stderr($language["ERROR"], $language["ERROR_ID"] . ": {$id}", $GLOBALS["usepopup"]); } require_once load_language("lang_torrents.php"); if (isset($_GET["act"]) && $_GET["act"] == "update") { //die("<center>".$language["TORRENT_UPDATE"]."</center>"); require_once dirname(__FILE__) . "/include/getscrape.php"; scrape(urldecode($_GET["surl"]), $id); redirect("index.php?page=torrent-details&id={$id}"); exit; } if ($XBTT_USE) { $tseeds = "f.seeds+ifnull(x.seeders,0) as seeds"; $tleechs = "f.leechers+ifnull(x.leechers,0) as leechers"; $tcompletes = "f.finished+ifnull(x.completed,0) as finished"; $ttables = "{$TABLE_PREFIX}files f LEFT JOIN xbt_files x ON x.info_hash = f.bin_hash"; } else { $tseeds = "f.seeds as seeds"; $tleechs = "f.leechers as leechers"; $tcompletes = "f.finished as finished"; $ttables = "{$TABLE_PREFIX}files f"; } if (!$CURUSER || $CURUSER["view_torrents"] != "yes") {
function scrape($cookie, $viewstate, $event_validation, $page, $this_page_code, $prev_last_result_id, $from, $to) { echo "Loading data (" . $from['day'] . '/' . $from['month'] . '/' . $from['year'] . ' -> ' . $to['day'] . '/' . $to['month'] . '/' . $to['year'] . ") ...\n"; $dom = new DOMDocument(); if ($page == 1) { @$dom->loadHTML(first_page($cookie, $viewstate, $event_validation, $from, $to)); } else { @$dom->loadHTML(not_first_page($cookie, $viewstate, $event_validation, $this_page_code)); //When page number hits 25/50/75/etc..., the website strangely starts counting over from 1 again (i.e. page 1 can be page 26/51/76. //And if there's more than 25 pages, but less than 50, the page numbering can move around randomly. } $xpath = new DOMXPath($dom); $viewstate = $xpath->query('//*[@id="__VIEWSTATE"]')->item(0)->getAttribute('value'); $event_validation = $xpath->query('//*[@id="__EVENTVALIDATION"]'); if ($event_validation->length) { $event_validation = $event_validation->item(0)->getAttribute('value'); } if ($page == 25) { $next_page_code = '25'; } elseif ($page % 25 === 0) { $next_page_code = '26'; } elseif ($page < 25) { $next_page_code = sprintf("%02s", $page); } else { foreach ($xpath->query('//td[@colspan="5"]/font/a') as $a) { if (intval($a->nodeValue) == $page + 1) { preg_match('/\\$ctl19\\$ctl(.*?)\'/i', $a->getAttribute('href'), $asp_page_id); $next_page_code = $asp_page_id[1]; } } } $query = $xpath->query('//table[@id="DataGrid1"]/tr'); $dom = null; $xpath = null; $ch = null; unset($dom); unset($xpath); unset($ch); echo "Parsing results ...\n"; $results = array(); $last_result_id = -1; $n = $query->length; for ($i = 1; $i < $n - 1; $i++) { $row = $query->item($i); preg_match('/filename=(.*?)">/i', $row->ownerDocument->saveXML($row), $href); @($result = array('case_number' => trim($row->childNodes->item(1)->nodeValue), 'path' => $href[1], 'petitioner' => trim(substr($row->childNodes->item(2)->nodeValue, 0, strpos($row->childNodes->item(2)->nodeValue, ' Vs. '))), 'respondent' => trim(substr($row->childNodes->item(2)->nodeValue, strpos($row->childNodes->item(2)->nodeValue, ' Vs. ') + 5)), 'judge' => trim(trim($row->childNodes->item(3)->nodeValue, 'Coram: ')), 'judgment_date' => trim($row->childNodes->item(4)->nodeValue), 'unique_id' => preg_replace('/\\s+/', '', $row->childNodes->item(0)->nodeValue . '-' . $row->childNodes->item(1)->nodeValue . '-' . $from['day'] . $from['month'] . $from['year'] . '-' . $to['day'] . $to['month'] . $to['year']))); if ($result['case_number'] || $result['path'] || $result['petitioner'] || $result['respondent'] || $result['judgment_date']) { array_push($results, $result); } $last_result_id = intval($row->childNodes->item(0)->nodeValue); } scraperwiki::save_sqlite(array('unique_id'), $results); $results_count = count($results); echo "Peak memory usage: " . memory_get_peak_usage() . "\n"; $query = null; $results = null; unset($query); unset($results); if ($prev_last_result_id != $last_result_id) { @scrape($cookie, $viewstate, $event_validation, $page + 1, $next_page_code, $last_result_id, $from, $to); } }
unset($xpath); $results = array(); for ($i = 0; $i < $n; $i++) { $row = $rows->item($i); if ($a = @$row->getElementsByTagName('a')->item(0)) { $name = @$a->nodeValue; } else { $name = null; } if ($p = @$row->getElementsByTagName('p')->item(0)) { $address = @$p->nodeValue; } else { $address = null; } array_push($results, array('unique_id' => preg_replace('/\\s+/', '', $category . '-' . @$name), 'name' => trim(@$name), 'address' => trim(@$address), 'category' => trim($category))); } scraperwiki::save_sqlite(array('unique_id'), $results); echo "Peak memory usage: " . memory_get_peak_usage() . "\n"; $query = null; $results = null; unset($query); unset($results); } $sections = array('Self-Clearing Houses', 'Full-Clearing Agents', 'Custodians', 'Gross Settlement Agents'); $dom = new DOMDocument(); @$dom->loadHTML(scraperwiki::scrape('http://www.bmfbovespa.com.br/Agentes/agentes.aspx?idioma=en-us')); $xpath = new DOMXPath($dom); $viewstate = trim($xpath->query('//input[@name="__VIEWSTATE"]')->item(0)->getAttribute('value')); foreach ($sections as $section) { scrape($section, $viewstate); }
if (!defined("IN_BTIT")) { die("non direct access!"); } $id = AddSlashes(isset($_GET["id"]) ? $_GET["id"] : false); if (!isset($id) || !$id) { stderr($language["ERROR"], $language["ERROR_ID"] . ": {$id}", $GLOBALS["usepopup"]); } require_once load_language("lang_torrents.php"); if (isset($_GET["act"]) && $_GET["act"] == "update") { require_once dirname(__FILE__) . "/include/getscrape.php"; $torrent = get_result('SELECT announces FROM ' . $TABLE_PREFIX . 'files WHERE info_hash="' . mysqli_real_escape_string($DBDT, $_GET["id"]) . '"'); $urls = @unserialize($torrent[0]["announces"]) ? unserialize($torrent[0]["announces"]) : array(); $keys = array_keys($urls); $random = mt_rand(0, count($urls) - 1); $url = $keys[$random]; scrape($url, $id); redirect("index.php?page=torrent-details&id={$id}"); exit; } if (isset($_POST[delcomment])) { if ($CURUSER["delete_comments"] == "yes") { do_sqlquery("DELETE FROM {$TABLE_PREFIX}comments WHERE id IN (" . implode(", ", $_POST[delcomment]) . ")"); } } if ($XBTT_USE) { $tdt = "f.seeds+ifnull(x.seeders,0)"; $tseeds = "f.seeds+ifnull(x.seeders,0) as seeds"; $tleechs = "f.leechers+ifnull(x.leechers,0) as leechers"; $tcompletes = "f.finished+ifnull(x.completed,0) as finished"; $ttables = "{$TABLE_PREFIX}files f LEFT JOIN xbt_files x ON x.info_hash=f.bin_hash"; } else {
$xpath = null; unset($xpath); $n = $query->length; echo "Parsing Results ...\n"; $results = array(); for ($i = 0; $i < $n; $i++) { $row = $query->item($i); if ($row->childNodes->item(2)) { preg_match_all('/([^\\s+]+)/', $row->childNodes->item(2)->nodeValue, $case_number); @($result = array('district' => trim(preg_replace('/\\xC2\\xA0/', '', $row->childNodes->item(0)->nodeValue)), 'case_number' => $case_number[0][0] . ' ' . $case_number[0][2] . '/' . $case_number[0][4], 'filing_date' => trim(preg_replace('/\\xC2\\xA0/', '', $row->childNodes->item(4)->nodeValue)), 'petitioner' => trim(preg_replace('/\\xC2\\xA0/', '', $row->childNodes->item(5)->nodeValue)), 'respondent' => trim(preg_replace('/\\xC2\\xA0/', '', $row->childNodes->item(6)->nodeValue)), 'sub_type' => trim(preg_replace('/\\xC2\\xA0/', '', $row->childNodes->item(9)->nodeValue)), 'sub_sub_type' => trim(preg_replace('/\\xC2\\xA0/', '', $row->childNodes->item(10)->nodeValue)))); if ($i > 0 && $result['case_number']) { array_push($results, $result); } } $case_number = null; $row = null; $result = null; unset($case_number); unset($row); unset($result); } //Clean up and save $query = null; unset($query); scraperwiki::save_sqlite(array('case_number'), $results); echo "Peak memory usage: " . memory_get_peak_usage() . "\n"; } scrape('B'); scrape('D'); scrape('G');
// Returning the scraped data from the function } if (isset($_GET['url'])) { $beg = "<div class=\"productSpecs specSection\">"; $link = $_GET['url']; $page = scrape($link); $spec = fetchdata($page, $beg, "</div>"); echo $spec; } if (isset($_GET["amz"])) { $name = $_GET["amz"]; $name = str_replace(" ", "+", $name); $price = 0; $link = "#"; $amzlink = "http://www.amazon.in/s/ref=nb_sb_noss_2?field-keywords=" . $name; $adata = scrape($amzlink); echo $adata; /**$uldata=fetchdata($adata,'<ul id="s-results-list-atf"','</ul>'); $product=fetchdata($uldata,'<li id="result_1"','</li>'); if($product){ $link=fetchdata($product,'<a class="a-link-normal a-text-normal" href="','">'); $price=fetchdata($product,'<span class="a-size-base a-color-price s-price a-text-bold"><span class="currencyINR"> </span>','</span>'); } else{ $link-fetchdata($product,'href="','"'); $price=fetchdata($adata,'<span class="a-size-base a-color-price s-price a-text-bold"><span class="currencyINR"> </span>','</span>'); }
if (!$mf) { // failed to move file do_sqlquery("DELETE FROM {$TABLE_PREFIX}files WHERE info_hash=\"{$hash}\"", true); if ($XBTT_USE) { do_sqlquery("UPDATE xbt_files SET flags=1 WHERE info_hash=0x{$hash}", true); } stderr($language["ERROR"], $language["ERR_MOVING_TORR"]); } // try to chmod new moved file, on some server chmod without this could result 600, seems to be php bug @chmod($TORRENTSDIR . "/" . $hash . ".btf", 0766); // gold/silver torrent do_sqlquery("UPDATE {$TABLE_PREFIX}files SET gold='{$gold}' WHERE info_hash=\"{$hash}\""); // if ($announce!=$BASEURL."/announce.php") if (!in_array($announce, $TRACKER_ANNOUNCEURLS)) { require_once "./include/getscrape.php"; scrape($announce, $hash); $status = 2; write_log("Uploaded new torrent {$filename} - EXT ({$hash})", "add"); } else { if ($DHT_PRIVATE) { $alltorrent = bencode($array); $fd = fopen($TORRENTSDIR . "/" . $hash . ".btf", "rb+"); fwrite($fd, $alltorrent); fclose($fd); } // with pid system active or private flag (dht disabled), tell the user to download the new torrent write_log("Uploaded new torrent {$filename} ({$hash})", "add"); $status = 1; } // Announce new Uploaded torrents in ShoutBoX start global $BASEURL;
$minDelay = '0'; //sec $maxDelay = '5'; //sec $i = 0; $depth = 1; $stack = array(); $end = false; $source = "notfoundorg"; $word = 'atco'; //geef zoekwoord op dat domein moet bevatten //visualiseren van data //pay options? > data verzamelen!!! country based, ip, dates, months insights, ... //domain watcher service? //headerChecker("http://www.driesbultynck.be"); scrape($source); /* STANDARD FUNCTIONS */ function save() { //iedere link met 404 code saven in database //vanuit zelf ingegeven link //vanuit zelf zoekmachine + keyword //check voor update > indien geen 404 meer, zet als fixed en delete uit lijst } function delay() { global $minDelay, $maxDelay, $delay; if ($minDelay == '0') { //echo "werkt"; //wait(); } else {
function scrapeHomepage($homepage) { //geen https ondersteuning > xpath uitzoeken > Regex is nogal complex //set_time_limit(10); //goedkope oplossing domain > header check nog doen $newHomepage = 'http://www.' . $homepage; $html = file_get_contents($newHomepage); $dom = new DOMDocument(); @$dom->loadHTML($html); $x = new DOMXPath($dom); //lege velden nog uit filteren //foreach($x->query("//a[((contains(@href,'".$homepage."') or not(contains(@href, 'http://'))) and not(contains(@href, 'https://'))) and not(contains(@href, 'mailto:')) and not(contains(@href, '#'))]/@href") as $node){ foreach ($x->query("//a[contains(@href,'" . $homepage . "') and not(contains(@href, 'mailto:')) and not(contains(@href, '#'))]/@href") as $node) { if (!empty($node)) { $arrBaseUrls[] = $node->nodeValue; //echo $node->nodeValue.', '; } else { echo 'leeg'; } } echo count($arrBaseUrls) . ' pagina\'s gevonden op niveau 1<br/>'; //print_r($arrBaseUrls); scrape($arrBaseUrls, $homepage); }
echo "Parsing Results ...\n"; $results = array(); for ($i = 1; $i < $n; $i++) { $row = $query->item($i); preg_match('/href="(.*?)">/i', $row->ownerDocument->saveXML($row), $a); @($result = array('judgment_date' => trim($row->childNodes->item(0)->nodeValue), 'case_number' => trim($row->childNodes->item(1)->nodeValue), 'judge' => trim($row->childNodes->item(2)->nodeValue), 'petitioner' => trim($row->childNodes->item(3)->nodeValue), 'respondent' => trim($row->childNodes->item(4)->nodeValue), 'location' => trim($row->childNodes->item(5)->nodeValue), 'path' => $a[1], 'unique_id' => preg_replace('/\\D+/', '', $a[1]) . '-' . preg_replace('/\\s+/', '', $row->childNodes->item(1)->nodeValue))); array_push($results, $result); if ($i % 100 === 0) { scraperwiki::save_sqlite(array('unique_id'), $results); $results = array(); } //Clean loop variables $option = null; $result = null; unset($option); unset($result); } //Clean up $options = null; $ch = null; unset($options); unset($ch); //Save final remainder to database scraperwiki::save_sqlite(array('unique_id'), $results); $results = null; unset($results); echo "Peak memory usage: " . memory_get_peak_usage() . "\n"; } for ($i = 0; $i < 10; $i++) { scrape($i); }