private function searchResults($page, $fields = array())
{
sleep(2);
if (App::runningInConsole()) {
fwrite(STDOUT, "=========================== " . PHP_EOL);
fwrite(STDOUT, "Results: " . $this->page_count . " : " . $page . PHP_EOL);
fwrite(STDOUT, "=========================== " . PHP_EOL);
}
$url = $this->url_base . "OffenderSearchResults.aspx";
if (!$page) {
$this->crawler = $this->client->request('GET', $url);
} else {
$this->crawler = $this->client->request('POST', $url, $fields);
}
$this->check_uri();
$links = $this->crawler->filter('#ctl00_ContentPlaceHolder1_pnlOffenders a[id*="lnkPage"]')->extract(array('_text', 'href'));
$pages = array();
$lastPage = true;
foreach ($links as $link) {
$next = false;
$prev = false;
$label = $link[0];
$link = $link[1];
if (empty($link)) {
$pages[] = array('target' => 'current', 'prev' => $prev, 'next' => $next, 'label' => $label);
} else {
list($j, $target) = explode("'", $link);
if (strpos($target, 'rptPager') !== false) {
if ($label == '[Previous 10]') {
$prev = true;
}
if ($label == '[Next 10]') {
$next = true;
$lastPage = false;
}
$pages[] = array('target' => $target, 'prev' => $prev, 'next' => $next, 'label' => $label);
}
}
}
//var_dump($pages);
if (App::runningInConsole()) {
fwrite(STDOUT, "=========== Check links ================ " . PHP_EOL);
}
if (empty($pages)) {
fwrite(STDOUT, "=========== ------ no links ================ " . PHP_EOL);
$this->crawlFrom($this->page_count);
} else {
$real_page_count = $this->crawler->filter('#ctl00_ContentPlaceHolder1_pnlOffenders a[disabled="disabled"]');
if (count($real_page_count)) {
$real_page_count = $real_page_count->text();
$this->page_count = $real_page_count;
} else {
$real_page_count = $this->page_count;
}
}
if (App::runningInConsole()) {
fwrite(STDOUT, "=========== Write files ================ : " . $this->page_count . " / " . $real_page_count . PHP_EOL);
}
// Write the contents back to the file
// store current page in database or some file
Storage::put('sexoffenders/_html/' . str_slug($this->state->state_name) . '/curr_page_num.html', $real_page_count);
$fields = $this->getInputHiddenFields();
for ($i = 0; $i <= 9; $i++) {
//
$this->records_crawled = $this->page_count * 10 + $i - 10;
$this->estimate_time('profiles');
$this->start_time = time();
//
$this->getProfile($fields, $i);
}
$this->page_count++;
$page++;
$lastPageNum = end($pages);
//var_dump($lastPageNum);
$isFinal = false;
if ($lastPageNum['target'] == "current") {
$isFinal = true;
}
if ($lastPageNum['label'] == "[Previous 10]") {
$isFinal = true;
}
if ($this->page_count > 19) {
if ($page == 11) {
$page = 1;
}
} else {
if ($page == 11) {
$page = 2;
}
}
$fields['__EVENTTARGET'] = 'ctl00$ContentPlaceHolder1$rptPager$ctl' . str_pad($page, 2, "0", STR_PAD_LEFT) . '$lnkPage';
if (!$lastPage) {
$this->searchResults($page, $fields);
} else {
if ($isFinal) {
$county = GaCounty::firstOrNew(['county_id' => $this->curr_county_id]);
$county->status = '1';
$county->pages = $this->page_count - 1;
$county->save();
$this->handleCounties();
} else {
$this->searchResults($page, $fields);
}
}
}
