private function searchResults($page, $fields = array())
{
sleep(2);
if (App::runningInConsole()) {
fwrite(STDOUT, "=========================== " . PHP_EOL);
fwrite(STDOUT, "Results: " . $this->page_count . " : " . $page . PHP_EOL);
fwrite(STDOUT, "=========================== " . PHP_EOL);
}
$url = $this->url_base . "OffenderSearchResults.aspx";
if (!$page) {
$this->crawler = $this->client->request('GET', $url);
} else {
$this->crawler = $this->client->request('POST', $url, $fields);
}
$this->check_uri();
$links = $this->crawler->filter('#ctl00_ContentPlaceHolder1_pnlOffenders a[id*="lnkPage"]')->extract(array('_text', 'href'));
$pages = array();
$lastPage = true;
foreach ($links as $link) {
$next = false;
$prev = false;
$label = $link[0];
$link = $link[1];
if (empty($link)) {
$pages[] = array('target' => 'current', 'prev' => $prev, 'next' => $next, 'label' => $label);
} else {
list($j, $target) = explode("'", $link);
if (strpos($target, 'rptPager') !== false) {
if ($label == '[Previous 10]') {
$prev = true;
}
if ($label == '[Next 10]') {
$next = true;
$lastPage = false;
}
$pages[] = array('target' => $target, 'prev' => $prev, 'next' => $next, 'label' => $label);
}
}
}
//var_dump($pages);
if (App::runningInConsole()) {
fwrite(STDOUT, "=========== Check links ================ " . PHP_EOL);
}
if (empty($pages)) {
fwrite(STDOUT, "=========== ------ no links ================ " . PHP_EOL);
$this->crawlFrom($this->page_count);
} else {
$real_page_count = $this->crawler->filter('#ctl00_ContentPlaceHolder1_pnlOffenders a[disabled="disabled"]');
if (count($real_page_count)) {
$real_page_count = $real_page_count->text();
$this->page_count = $real_page_count;
} else {
$real_page_count = $this->page_count;
}
}
if (App::runningInConsole()) {
fwrite(STDOUT, "=========== Write files ================ : " . $this->page_count . " / " . $real_page_count . PHP_EOL);
}
// Write the contents back to the file
// store current page in database or some file
Storage::put('sexoffenders/_html/' . str_slug($this->state->state_name) . '/curr_page_num.html', $real_page_count);
$fields = $this->getInputHiddenFields();
for ($i = 0; $i <= 9; $i++) {
//
$this->records_crawled = $this->page_count * 10 + $i - 10;
$this->estimate_time('profiles');
$this->start_time = time();
//
$this->getProfile($fields, $i);
}
$this->page_count++;
$page++;
$lastPageNum = end($pages);
//var_dump($lastPageNum);
$isFinal = false;
if ($lastPageNum['target'] == "current") {
$isFinal = true;
}
if ($lastPageNum['label'] == "[Previous 10]") {
$isFinal = true;
}
if ($this->page_count > 19) {
if ($page == 11) {
$page = 1;
}
} else {
if ($page == 11) {
$page = 2;
}
}
$fields['__EVENTTARGET'] = 'ctl00$ContentPlaceHolder1$rptPager$ctl' . str_pad($page, 2, "0", STR_PAD_LEFT) . '$lnkPage';
if (!$lastPage) {
$this->searchResults($page, $fields);
} else {
if ($isFinal) {
$county = GaCounty::firstOrNew(['county_id' => $this->curr_county_id]);
$county->status = '1';
$county->pages = $this->page_count - 1;
$county->save();
$this->handleCounties();
} else {
$this->searchResults($page, $fields);
}
}
}
public function run()
{
DB::table('ga_counties')->delete();
GaCounty::create(["county_id" => "44", "name" => "APPLING", 'pages' => 4]);
GaCounty::create(["county_id" => "50", "name" => "ATKINSON", 'pages' => 3]);
GaCounty::create(["county_id" => "51", "name" => "BACON", 'pages' => 4]);
GaCounty::create(["county_id" => "52", "name" => "BAKER", 'pages' => 2]);
GaCounty::create(["county_id" => "53", "name" => "BALDWIN", 'pages' => 10]);
GaCounty::create(["county_id" => "54", "name" => "BANKS", 'pages' => 5]);
GaCounty::create(["county_id" => "55", "name" => "BARROW", 'pages' => 15]);
GaCounty::create(["county_id" => "56", "name" => "BARTOW", 'pages' => 21]);
GaCounty::create(["county_id" => "57", "name" => "BEN HILL", 'pages' => 7]);
GaCounty::create(["county_id" => "58", "name" => "BERRIEN", 'pages' => 7]);
GaCounty::create(["county_id" => "59", "name" => "BIBB", 'pages' => 28]);
GaCounty::create(["county_id" => "60", "name" => "BLECKLEY", 'pages' => 4]);
GaCounty::create(["county_id" => "61", "name" => "BRANTLEY", 'pages' => 6]);
GaCounty::create(["county_id" => "62", "name" => "BROOKS", 'pages' => 5]);
GaCounty::create(["county_id" => "63", "name" => "BRYAN", 'pages' => 5]);
GaCounty::create(["county_id" => "64", "name" => "BULLOCH", 'pages' => 8]);
GaCounty::create(["county_id" => "65", "name" => "BURKE", 'pages' => 9]);
GaCounty::create(["county_id" => "66", "name" => "BUTTS", 'pages' => 6]);
GaCounty::create(["county_id" => "67", "name" => "CALHOUN", 'pages' => 3]);
GaCounty::create(["county_id" => "68", "name" => "CAMDEN", 'pages' => 8]);
GaCounty::create(["county_id" => "69", "name" => "CANDLER", 'pages' => 6]);
GaCounty::create(["county_id" => "70", "name" => "CARROLL", 'pages' => 26]);
GaCounty::create(["county_id" => "71", "name" => "CATOOSA", 'pages' => 14]);
GaCounty::create(["county_id" => "72", "name" => "CHARLTON", 'pages' => 3]);
GaCounty::create(["county_id" => "73", "name" => "CHATHAM", 'pages' => 46]);
GaCounty::create(["county_id" => "74", "name" => "CHATTAHOOCHEE", 'pages' => 2]);
GaCounty::create(["county_id" => "75", "name" => "CHATTOOGA", 'pages' => 10]);
GaCounty::create(["county_id" => "76", "name" => "CHEROKEE", 'pages' => 22]);
GaCounty::create(["county_id" => "77", "name" => "CLARKE", 'pages' => 14]);
GaCounty::create(["county_id" => "78", "name" => "CLAY", 'pages' => 1]);
GaCounty::create(["county_id" => "79", "name" => "CLAYTON", 'pages' => 44]);
GaCounty::create(["county_id" => "80", "name" => "CLINCH", 'pages' => 2]);
GaCounty::create(["county_id" => "45", "name" => "COBB", 'pages' => 47]);
GaCounty::create(["county_id" => "81", "name" => "COFFEE", 'pages' => 9]);
GaCounty::create(["county_id" => "82", "name" => "COLQUITT", 'pages' => 12]);
GaCounty::create(["county_id" => "83", "name" => "COLUMBIA", 'pages' => 9]);
GaCounty::create(["county_id" => "84", "name" => "COOK", 'pages' => 7]);
GaCounty::create(["county_id" => "85", "name" => "COWETA", 'pages' => 16]);
GaCounty::create(["county_id" => "86", "name" => "CRAWFORD", 'pages' => 4]);
GaCounty::create(["county_id" => "87", "name" => "CRISP", 'pages' => 6]);
GaCounty::create(["county_id" => "88", "name" => "DADE", 'pages' => 4]);
GaCounty::create(["county_id" => "89", "name" => "DAWSON", 'pages' => 4]);
GaCounty::create(["county_id" => "90", "name" => "DECATUR", 'pages' => 10]);
GaCounty::create(["county_id" => "47", "name" => "DEKALB", 'pages' => 91]);
GaCounty::create(["county_id" => "91", "name" => "DODGE", 'pages' => 8]);
GaCounty::create(["county_id" => "92", "name" => "DOOLY", 'pages' => 4]);
GaCounty::create(["county_id" => "93", "name" => "DOUGHERTY", 'pages' => 27]);
GaCounty::create(["county_id" => "94", "name" => "DOUGLAS", 'pages' => 20]);
GaCounty::create(["county_id" => "95", "name" => "EARLY", 'pages' => 7]);
GaCounty::create(["county_id" => "96", "name" => "ECHOLS", 'pages' => 1]);
GaCounty::create(["county_id" => "97", "name" => "EFFINGHAM", 'pages' => 10]);
GaCounty::create(["county_id" => "98", "name" => "ELBERT", 'pages' => 6]);
GaCounty::create(["county_id" => "99", "name" => "EMANUEL", 'pages' => 7]);
GaCounty::create(["county_id" => "100", "name" => "EVANS", 'pages' => 3]);
GaCounty::create(["county_id" => "101", "name" => "FANNIN", 'pages' => 5]);
GaCounty::create(["county_id" => "102", "name" => "FAYETTE", 'pages' => 4]);
GaCounty::create(["county_id" => "103", "name" => "FLOYD", 'pages' => 23]);
GaCounty::create(["county_id" => "104", "name" => "FORSYTH", 'pages' => 12]);
GaCounty::create(["county_id" => "105", "name" => "FRANKLIN", 'pages' => 6]);
GaCounty::create(["county_id" => "48", "name" => "FULTON", 'pages' => 151]);
GaCounty::create(["county_id" => "106", "name" => "GILMER", 'pages' => 11]);
GaCounty::create(["county_id" => "107", "name" => "GLASCOCK", 'pages' => 1]);
GaCounty::create(["county_id" => "108", "name" => "GLYNN", 'pages' => 11]);
GaCounty::create(["county_id" => "109", "name" => "GORDON", 'pages' => 12]);
GaCounty::create(["county_id" => "110", "name" => "GRADY", 'pages' => 7]);
GaCounty::create(["county_id" => "111", "name" => "GREENE", 'pages' => 4]);
GaCounty::create(["county_id" => "46", "name" => "GWINNETT", 'pages' => 48]);
GaCounty::create(["county_id" => "112", "name" => "HABERSHAM", 'pages' => 8]);
GaCounty::create(["county_id" => "113", "name" => "HALL", 'pages' => 24]);
GaCounty::create(["county_id" => "114", "name" => "HANCOCK", 'pages' => 3]);
GaCounty::create(["county_id" => "115", "name" => "HARALSON", 'pages' => 9]);
GaCounty::create(["county_id" => "116", "name" => "HARRIS", 'pages' => 5]);
GaCounty::create(["county_id" => "117", "name" => "HART", 'pages' => 5]);
GaCounty::create(["county_id" => "118", "name" => "HEARD", 'pages' => 5]);
GaCounty::create(["county_id" => "119", "name" => "HENRY", 'pages' => 19]);
GaCounty::create(["county_id" => "120", "name" => "HOUSTON", 'pages' => 26]);
GaCounty::create(["county_id" => "121", "name" => "IRWIN", 'pages' => 4]);
GaCounty::create(["county_id" => "122", "name" => "JACKSON", 'pages' => 14]);
GaCounty::create(["county_id" => "123", "name" => "JASPER", 'pages' => 3]);
GaCounty::create(["county_id" => "124", "name" => "JEFF DAVIS", 'pages' => 4]);
GaCounty::create(["county_id" => "125", "name" => "JEFFERSON", 'pages' => 6]);
GaCounty::create(["county_id" => "126", "name" => "JENKINS", 'pages' => 2]);
GaCounty::create(["county_id" => "127", "name" => "JOHNSON", 'pages' => 4]);
GaCounty::create(["county_id" => "128", "name" => "JONES", 'pages' => 5]);
GaCounty::create(["county_id" => "129", "name" => "LAMAR", 'pages' => 3]);
GaCounty::create(["county_id" => "130", "name" => "LANIER", 'pages' => 4]);
GaCounty::create(["county_id" => "131", "name" => "LAURENS", 'pages' => 14]);
GaCounty::create(["county_id" => "132", "name" => "LEE", 'pages' => 4]);
GaCounty::create(["county_id" => "133", "name" => "LIBERTY", 'pages' => 13]);
GaCounty::create(["county_id" => "134", "name" => "LINCOLN", 'pages' => 3]);
GaCounty::create(["county_id" => "135", "name" => "LONG", 'pages' => 5]);
GaCounty::create(["county_id" => "136", "name" => "LOWNDES", 'pages' => 21]);
GaCounty::create(["county_id" => "137", "name" => "LUMPKIN", 'pages' => 6]);
GaCounty::create(["county_id" => "140", "name" => "MACON", 'pages' => 3]);
GaCounty::create(["county_id" => "141", "name" => "MADISON", 'pages' => 7]);
GaCounty::create(["county_id" => "142", "name" => "MARION", 'pages' => 4]);
GaCounty::create(["county_id" => "138", "name" => "MCDUFFIE", 'pages' => 7]);
GaCounty::create(["county_id" => "139", "name" => "MCINTOSH", 'pages' => 4]);
GaCounty::create(["county_id" => "143", "name" => "MERIWETHER", 'pages' => 8]);
GaCounty::create(["county_id" => "144", "name" => "MILLER", 'pages' => 2]);
GaCounty::create(["county_id" => "145", "name" => "MITCHELL", 'pages' => 9]);
GaCounty::create(["county_id" => "146", "name" => "MONROE", 'pages' => 5]);
GaCounty::create(["county_id" => "147", "name" => "MONTGOMERY", 'pages' => 3]);
GaCounty::create(["county_id" => "148", "name" => "MORGAN", 'pages' => 3]);
GaCounty::create(["county_id" => "149", "name" => "MURRAY", 'pages' => 10]);
GaCounty::create(["county_id" => "150", "name" => "MUSCOGEE", 'pages' => 40]);
GaCounty::create(["county_id" => "151", "name" => "NEWTON", 'pages' => 20]);
GaCounty::create(["county_id" => "152", "name" => "OCONEE", 'pages' => 2]);
GaCounty::create(["county_id" => "153", "name" => "OGLETHORPE", 'pages' => 4]);
GaCounty::create(["county_id" => "154", "name" => "PAULDING", 'pages' => 21]);
GaCounty::create(["county_id" => "155", "name" => "PEACH", 'pages' => 5]);
GaCounty::create(["county_id" => "156", "name" => "PICKENS", 'pages' => 6]);
GaCounty::create(["county_id" => "157", "name" => "PIERCE", 'pages' => 3]);
GaCounty::create(["county_id" => "158", "name" => "PIKE", 'pages' => 3]);
GaCounty::create(["county_id" => "159", "name" => "POLK", 'pages' => 11]);
GaCounty::create(["county_id" => "160", "name" => "PULASKI", 'pages' => 3]);
GaCounty::create(["county_id" => "161", "name" => "PUTNAM", 'pages' => 5]);
GaCounty::create(["county_id" => "162", "name" => "QUITMAN", 'pages' => 2]);
GaCounty::create(["county_id" => "163", "name" => "RABUN", 'pages' => 3]);
GaCounty::create(["county_id" => "164", "name" => "RANDOLPH", 'pages' => 3]);
GaCounty::create(["county_id" => "165", "name" => "RICHMOND", 'pages' => 43]);
GaCounty::create(["county_id" => "166", "name" => "ROCKDALE", 'pages' => 10]);
GaCounty::create(["county_id" => "167", "name" => "SCHLEY", 'pages' => 2]);
GaCounty::create(["county_id" => "168", "name" => "SCREVEN", 'pages' => 5]);
GaCounty::create(["county_id" => "169", "name" => "SEMINOLE", 'pages' => 3]);
GaCounty::create(["county_id" => "170", "name" => "SPALDING", 'pages' => 17]);
GaCounty::create(["county_id" => "171", "name" => "STEPHENS", 'pages' => 8]);
GaCounty::create(["county_id" => "172", "name" => "STEWART", 'pages' => 3]);
GaCounty::create(["county_id" => "173", "name" => "SUMTER", 'pages' => 8]);
GaCounty::create(["county_id" => "174", "name" => "TALBOT", 'pages' => 2]);
GaCounty::create(["county_id" => "175", "name" => "TALIAFERRO", 'pages' => 1]);
GaCounty::create(["county_id" => "176", "name" => "TATTNALL", 'pages' => 7]);
GaCounty::create(["county_id" => "177", "name" => "TAYLOR", 'pages' => 3]);
GaCounty::create(["county_id" => "178", "name" => "TELFAIR", 'pages' => 4]);
GaCounty::create(["county_id" => "179", "name" => "TERRELL", 'pages' => 4]);
GaCounty::create(["county_id" => "180", "name" => "THOMAS", 'pages' => 13]);
GaCounty::create(["county_id" => "181", "name" => "TIFT", 'pages' => 5]);
GaCounty::create(["county_id" => "182", "name" => "TOOMBS", 'pages' => 6]);
GaCounty::create(["county_id" => "183", "name" => "TOWNS", 'pages' => 2]);
GaCounty::create(["county_id" => "184", "name" => "TREUTLEN", 'pages' => 3]);
GaCounty::create(["county_id" => "185", "name" => "TROUP", 'pages' => 24]);
GaCounty::create(["county_id" => "186", "name" => "TURNER", 'pages' => 4]);
GaCounty::create(["county_id" => "187", "name" => "TWIGGS", 'pages' => 3]);
GaCounty::create(["county_id" => "188", "name" => "UNION", 'pages' => 4]);
GaCounty::create(["county_id" => "189", "name" => "UPSON", 'pages' => 12]);
GaCounty::create(["county_id" => "190", "name" => "WALKER", 'pages' => 14]);
GaCounty::create(["county_id" => "191", "name" => "WALTON", 'pages' => 17]);
GaCounty::create(["county_id" => "192", "name" => "WARE", 'pages' => 10]);
GaCounty::create(["county_id" => "193", "name" => "WARREN", 'pages' => 2]);
GaCounty::create(["county_id" => "194", "name" => "WASHINGTON", 'pages' => 5]);
GaCounty::create(["county_id" => "195", "name" => "WAYNE", 'pages' => 7]);
GaCounty::create(["county_id" => "196", "name" => "WEBSTER", 'pages' => 1]);
GaCounty::create(["county_id" => "197", "name" => "WHEELER", 'pages' => 4]);
GaCounty::create(["county_id" => "198", "name" => "WHITE", 'pages' => 6]);
GaCounty::create(["county_id" => "199", "name" => "WHITFIELD", 'pages' => 22]);
GaCounty::create(["county_id" => "200", "name" => "WILCOX", 'pages' => 4]);
GaCounty::create(["county_id" => "201", "name" => "WILKES", 'pages' => 3]);
GaCounty::create(["county_id" => "202", "name" => "WILKINSON", 'pages' => 4]);
GaCounty::create(["county_id" => "203", "name" => "WORTH", 'pages' => 7]);
}
