// print $player_fullName.$player_id."\n"; // Prepare Overall Record $allrecord = array('matchId' => $matchId, 'venuefullName' => $venuefullName, 'venueCity' => $venueCity, 'season' => "ipl" . $iplSeries, 'tossWin' => $tossWin, 'result' => $result, 'teamA' => $teamA["team"]["shortName"], 'teamB' => $teamB["team"]["shortName"], 'teamToBat' => $teamToBat); scraperwiki::save(array('matchId', 'season'), $allrecord); } else { print "Empty JSON FOR" . $matchId . "\n"; } $matchNId++; } ?> <?php require 'scraperwiki/simple_html_dom.php'; $iplSeries = "2013"; //$html_content= scraperWiki::scrape("http://www.thatscricket.com/indian-premier-league/".$iplSeries."/"); ScraperWiki::attach("sqads", "src"); //$playerIds=scraperwiki::select("distinct player_id from src.swdata desc"); $matchNId = 1; while ($matchNId <= 59) { //Carrer Stats http://dynamic.pulselive.com/test/data/core/cricket/careerStats/1_careerStats.js?_1363343668516= //http://dynamic.pulselive.com/dynamic/data/core/cricket/2012/ipl2012/ipl2012-70/scoring.js?_1364576135590= $json_content = scraperWiki::scrape("http://dynamic.pulselive.com/dynamic/data/core/cricket/2012/ipl" . $iplSeries . "/ipl" . $iplSeries . "-" . sprintf("%02s", $matchNId) . "/scoring.js?_1364576135590="); //print "JSON".$json_content; if (strpos($json_content, "The page is not found") === FALSE) { //http://dynamic.pulselive.com/dynamic/data/core/cricket/2012/ipl2012/23_careerStats.js $json_content = str_replace("onScoring(", "", $json_content); $json_content = str_replace(");", "", $json_content); $myMatchData = null; $myMatchData = json_decode($json_content, true); //var_dump($myPlayerData); // PLAYER DATA
<div class="yui-u"> <div id="class_div"></div><div id="level_div"></div><div id="race_div"></div> </div> </div> </div> <div id="ft" role="contentinfo"><p>Last update on <?php echo $date; ?> </p></div> </div> </body> </html><?php $short_name = 'kon_members-1'; ScraperWiki::attach($short_name, 'src'); $keys = ScraperWiki::sqliteexecute("select * from swdata limit 0")->keys; $data = ScraperWiki::select("* from swdata"); $class = array("captain" => 0, "lore-master" => 0, "rune-keeper" => 0, "guardian" => 0, "minstrel" => 0, "hunter" => 0, "burglar" => 0, "champion" => 0, "warden" => 0); $level = array('0-9' => 0, '10-19' => 0, '20-29' => 0, '30-39' => 0, '40-49' => 0, '50-59' => 0, '60-65' => 0); $race = array('elf' => 0, 'race of man' => 0, 'hobbit' => 0, 'dwarf' => 0); $rank = array('leader' => 0, 'officer' => 0, 'member' => 0); $members = array(); foreach ($data as $i => $member) { $class[strtolower($member->class)]++; $race[strtolower($member->race)]++; $rank[strtolower($member->rank)]++; $lvl = intval($member->level); $range = $lvl - $lvl % 10; if ($lvl <= 10) { $st = 0; $en = 10; } elseif ($lvl >= 60) {
preg_match("/Dato: ([0-9]+)\\.([0-9]+)\\.([0-9]+)/", $titTag[0]->plaintext, $m); $date = sprintf("%s-%s-%s", $m[3], $m[2], $m[1]); $areas = array("indre_oslofjord" => array(103, 400, 135, 430), "ytre_oslofjord" => array(103, 420, 135, 444), "gulen" => array(17, 373, 41, 397), "trondheimsfjorden" => array(68, 279, 105, 312), "troms" => array(90, 62, 148, 116), "lofoten_vesterålen" => array(57, 106, 117, 182)); $data = array('date' => $date); foreach ($areas as $name => $area) { $good = 0; $bad = 0; for ($y = $area[1]; $y <= $area[3]; $y++) { for ($x = $area[0]; $x <= $area[2]; $x++) { $rgb = imagecolorat($img, $x, $y); $r = $rgb >> 16 & 0xff; $g = $rgb >> 8 & 0xff; $b = $rgb & 0xff; if ($r > 250 && $g > 240 && $b > 240) { $good++; } else { if ($r > 150 && $r < 175 && $g > 50 && $g < 80 && $b > 70 && $b < 90) { $bad++; } } } } if ($good > 4 || $bad > 4) { $visibility = round($good * (10 / ($good + $bad + 1))); } else { $visibility = 'null'; } $data[$name] = $visibility; } ScraperWiki::save(array('date'), $data);
function scrape($month, $year) { echo "Loading data ({$month} {$year}) ...\n"; $dom = new DOMDocument(); @$dom->loadHTML(ScraperWiki::scrape("http://ghconline.nic.in/JDMT{$month}{$year}.html")); $xpath = new DOMXPath($dom); $dom = null; unset($dom); $query = $xpath->query('//table[@class="DISCOVERY3"]/tr'); $xpath = null; unset($xpath); $n = $query->length; echo "Parsing Results ...\n"; $results = array(); for ($i = 1; $i < $n; $i++) { $row = $query->item($i); if ($year == 10 || $year == 11 && $month == 'Jan') { $subjects = preg_split('/VS/i', preg_replace('/\\xC2\\xA0/', ' ', $row->childNodes->item(4)->nodeValue)); @($result = array('unique_id' => preg_replace('/\\s+/', '', $row->childNodes->item(0)->nodeValue . '-' . $month . $year . '-' . $row->childNodes->item(2)->nodeValue), 'case_number' => $row->childNodes->item(2)->nodeValue, 'judgment_date' => $month . '. ' . preg_replace('/\\D+/', '', $row->childNodes->item(0)->nodeValue) . ' 20' . $year, 'path' => ltrim($row->getElementsByTagName('a')->item(0)->getAttribute('href'), '.'), 'petitioner' => trim($subjects[0]), 'respondent' => trim($subjects[1]), 'subject' => $row->childNodes->item(6)->nodeValue)); } else { $subjects = preg_split('/VS/i', preg_replace('/\\xC2\\xA0/', ' ', $row->childNodes->item(3)->nodeValue)); @($result = array('unique_id' => preg_replace('/\\s+/', '', $row->childNodes->item(0)->nodeValue . '-' . $month . $year . '-' . $row->childNodes->item(2)->nodeValue), 'case_number' => $row->childNodes->item(2)->nodeValue, 'judgment_date' => $month . '. ' . preg_replace('/\\D+/', '', $row->childNodes->item(1)->nodeValue) . ' 20' . $year, 'path' => ltrim($row->getElementsByTagName('a')->item(0)->getAttribute('href'), '.'), 'petitioner' => trim($subjects[0]), 'respondent' => trim($subjects[1]), 'subject' => $row->childNodes->item(4)->nodeValue)); } array_push($results, $result); //Clean loop variables $option = null; $result = null; unset($option); unset($result); } //Clean up $options = null; $ch = null; unset($options); unset($ch); //Final save to database $this_results = count($results); scraperwiki::save_sqlite(array('unique_id'), $results); $results = null; unset($results); }
#} #https://scraperwiki.com/scrapers/campbelltown_city_council_development_proposals/edit/ #print_r($reports); #if (isset($reports)){ #print "id" . $reports['id']; #$existingreports = array(); #$existingreports = scraperwiki::sqliteexecute("select * from reports where `id`='" . $reports['id'] . "'"); #print_r($existingreports); # if (sizeof($existingreports) == 0) # { #scraperwiki::sqliteexecute("drop table reports"); #scraperwiki::sqliteexecute("create table if not exists reports ('id' string, 'title' string, 'incidentdescription' string, 'incidentdate' string, 'incidentmode' string, 'incidentactive' string, 'incidentverified' string, 'locationid' string, 'locationname' string, 'locationlatitude' string, 'locationlongitude' string, 'categorytitle' string, 'categoryid' string, 'error' string, 'mediaid' string, 'mediatype' string, 'medialink' string, 'mediathumb' string, 'reporturl' string)"); foreach ($reports as $id => $values) { scraperwiki::sqliteexecute("insert or replace into reports values (:id, :title, :incidentdescription, :incidentdate, :incidentmode, :incidentactive, :incidentverified, :locationid, :locationname, :locationlatitude, :locationlongitude, :categorytitle, :categoryid, :error, :mediaid, :mediatype, :medialink, :mediathumb, :reporturl)", array("title" => $values["Title"], "id" => $id, "incidentdescription" => $values["Incidentdescription"], "incidentdate" => $values["Incidentdate"], "incidentmode" => $values["Incidentmode"], "incidentactive" => $values["Incidentactive"], "incidentverified" => $values["Incidentverified"], "locationid" => $values["Locationid"], "locationname" => $values["Locationname"], "locationlatitude" => $values["Locationlatitude"], "locationlongitude" => $values["Locationlongitude"], "categorytitle" => $values["Category"], "categoryid" => $values["CategoryID"], "error" => $values["Error"], "mediaid" => $values["MediaID"], "mediatype" => $values["MediaType"], "medialink" => $values["MediaLink"], "mediathumb" => $values["MediaThumb"], "reporturl" => $values["ReportURL"])); } # $unique_keys = array("id"); #$table = "reports"; #if (isset($reports)){ #scraperwiki::save_sqlite($unique_keys, $reports, $table); #, $table #} #} # else # { # print ("Skipping already saved record " . $reports['id'] . "\n"); # } #Scraperwiki::sqliteexecute("select * from reports where `id`='" . $report['id'] . "'"); #if ScraperWiki::sqliteexecute("delete from reports where `title`='No data'"); ScraperWiki::sqliteexecute("delete from reports where `title`='No data'"); print "done"; scraperwiki::sqlitecommit(); #}
function get_page($url, $table_name = "pages") { $rows = ScraperWiki::select("`text` from " . $table_name . " where url=?;", array($url)); #Check length, then return $rows[0]['text']; }