function load_page($a_sURL, &$a_sResult, $a_bWait = false)
{
// first try a regullar page pull without headers and such
sleep(1);
$l_oCURL = curl_init();
curl_setopt($l_oCURL, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($l_oCURL, CURLOPT_URL, $a_sURL);
$a_sResult = curl_exec($l_oCURL);
$statuscode = curl_getinfo($l_oCURL, CURLINFO_HTTP_CODE);
curl_close($l_oCURL);
if ($a_sResult && $statuscode == 200) {
unset($l_oCURL);
return 200;
} else {
sleep(1);
unset($l_oCURL);
$a_sResult = '';
$l_oCURL = curl_init();
curl_setopt($l_oCURL, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($l_oCURL, CURLOPT_URL, $a_sURL);
$a_sResult = curl_exec($l_oCURL);
$statuscode = curl_getinfo($l_oCURL, CURLINFO_HTTP_CODE);
curl_close($l_oCURL);
}
if ($a_sResult && $statuscode == 200) {
unset($l_oCURL);
return 200;
} else {
unset($l_oCURL);
$l_hHandler = new CCURL($a_sURL);
$l_iCurrentTry = 3;
/* maximum tries to fetch page */
$l_iHTTPStatus = 0;
while ($l_iHTTPStatus != 200 && $l_iCurrentTry > 0) {
sleep(1);
/* sleep for a second before trying a page */
$l_iCurrentTry = $l_iCurrentTry - 1;
$l_hHandler->createCurl();
$l_iHTTPStatus = $l_hHandler->getHttpStatus();
}
unset($l_iHTTPStatus);
if ($l_hHandler->getHttpStatus() == 200) {
$a_sResult = (string) $l_hHandler->__tostring();
}
$l_iHTTPStatus = $l_hHandler->getHttpStatus();
unset($l_hHandler);
return (int) $l_iHTTPStatus;
}
}
private function load_content($a_sURL, &$a_sResult)
{
$l_hHandler = new CCURL($a_sURL);
$l_iCurrentTry = _CRAWLER_REQUEST_PAGE;
$l_iHTTPStatus = 0;
while ($l_iHTTPStatus != 200 && $l_iCurrentTry > 0) {
sleep(1);
$l_iCurrentTry = $l_iCurrentTry - 1;
$l_hHandler->createCurl();
$l_iHTTPStatus = $l_hHandler->getHttpStatus();
}
unset($l_iHTTPStatus);
if ($l_hHandler->getHttpStatus() == 200) {
$a_sResult = $l_hHandler->__tostring();
unset($l_hHandler);
return true;
} else {
unset($l_hHandler);
return false;
}
}