function getContent($url, $enableProxy = true, $logCrawl = true)
{
curl_setopt($this->_CURL_RESOURCE, CURLOPT_URL, $url);
curl_setopt($this->_CURL_RESOURCE, CURLOPT_FAILONERROR, $this->_CURLOPT_FAILONERROR);
@curl_setopt($this->_CURL_RESOURCE, CURLOPT_FOLLOWLOCATION, $this->_CURLOPT_FOLLOWLOCATION);
curl_setopt($this->_CURL_RESOURCE, CURLOPT_RETURNTRANSFER, $this->_CURLOPT_RETURNTRANSFER);
curl_setopt($this->_CURL_RESOURCE, CURLOPT_TIMEOUT, $this->_CURLOPT_TIMEOUT);
curl_setopt($this->_CURL_RESOURCE, CURLOPT_COOKIEJAR, $this->_CURLOPT_COOKIEJAR);
curl_setopt($this->_CURL_RESOURCE, CURLOPT_COOKIEFILE, $this->_CURLOPT_COOKIEFILE);
curl_setopt($this->_CURL_RESOURCE, CURLOPT_HEADER, $this->_CURLOPT_HEADER);
if (!empty($this->_CURLOPT_COOKIE)) {
curl_setopt($this->_CURL_RESOURCE, CURLOPT_COOKIE, $this->_CURLOPT_COOKIE);
}
if (!empty($this->_CURLOPT_REFERER)) {
curl_setopt($this->_CURL_RESOURCE, CURLOPT_REFERER, $this->_CURLOPT_REFERER);
}
if (strlen($this->_CURLOPT_POSTFIELDS) > 1) {
curl_setopt($this->_CURL_RESOURCE, CURLOPT_POST, $this->_CURLOPT_POST);
curl_setopt($this->_CURL_RESOURCE, CURLOPT_POSTFIELDS, $this->_CURLOPT_POSTFIELDS);
}
// user agent assignment
$this->_CURLOPT_USERAGENT = defined('SP_USER_AGENT') ? SP_USER_AGENT : $this->_CURLOPT_USERAGENT;
if (strlen($this->_CURLOPT_USERAGENT) > 0) {
curl_setopt($this->_CURL_RESOURCE, CURLOPT_USERAGENT, $this->_CURLOPT_USERAGENT);
}
if (strlen($this->_CURLOPT_USERPWD) > 2) {
curl_setopt($this->_CURL_RESOURCE, CURLOPT_USERPWD, $this->_CURLOPT_USERPWD);
}
// to use proxy if proxy enabled
if (SP_ENABLE_PROXY && $enableProxy) {
$proxyCtrler = new ProxyController();
if ($proxyInfo = $proxyCtrler->getRandomProxy()) {
curl_setopt($this->_CURL_RESOURCE, CURLOPT_PROXY, $proxyInfo['proxy'] . ":" . $proxyInfo['port']);
curl_setopt($this->_CURL_RESOURCE, CURLOPT_HTTPPROXYTUNNEL, CURLOPT_HTTPPROXYTUNNEL_VAL);
if (!empty($proxyInfo['proxy_auth'])) {
curl_setopt($this->_CURL_RESOURCE, CURLOPT_PROXYUSERPWD, $proxyInfo['proxy_username'] . ":" . $proxyInfo['proxy_password']);
}
} else {
showErrorMsg("No active proxies found!! Please check your proxy settings from Admin Panel.");
}
}
$ret['page'] = curl_exec($this->_CURL_RESOURCE);
$ret['error'] = curl_errno($this->_CURL_RESOURCE);
$ret['errmsg'] = curl_error($this->_CURL_RESOURCE);
// update crawl log in database for future reference
if ($logCrawl) {
$crawlLogCtrl = new CrawlLogController();
$crawlInfo['crawl_status'] = $ret['error'] ? 0 : 1;
$crawlInfo['ref_id'] = $crawlInfo['crawl_link'] = addslashes($url);
$crawlInfo['crawl_referer'] = addslashes($this->_CURLOPT_REFERER);
$crawlInfo['crawl_cookie'] = addslashes($this->_CURLOPT_COOKIE);
$crawlInfo['crawl_post_fields'] = addslashes($this->_CURLOPT_POSTFIELDS);
$crawlInfo['crawl_useragent'] = addslashes($this->_CURLOPT_USERAGENT);
$crawlInfo['proxy_id'] = $proxyInfo['id'];
$crawlInfo['log_message'] = addslashes($ret['errmsg']);
$ret['log_id'] = $crawlLogCtrl->createCrawlLog($crawlInfo);
}
// disable proxy if not working
if (SP_ENABLE_PROXY && $enableProxy && !empty($ret['error']) && !empty($proxyInfo['id'])) {
// deactivate proxy
if (PROXY_DEACTIVATE_CRAWL) {
$proxyCtrler->__changeStatus($proxyInfo['id'], 0);
}
// chekc with another proxy
if (CHECK_WITH_ANOTHER_PROXY_IF_FAILED) {
$ret = $this->getContent($url, $enableProxy);
}
}
return $ret;
}
