$controller->set('spTextPanel', $controller->getLanguageTexts('panel', $_SESSION['lang_code'])); $controller->spTextProxy = $controller->getLanguageTexts('proxy', $_SESSION['lang_code']); $controller->set('spTextProxy', $controller->spTextProxy); if ($_SERVER['REQUEST_METHOD'] == 'POST') { switch ($_POST['sec']) { case "create": $controller->createProxy($_POST); break; case "update": $controller->updateProxy($_POST); break; } } else { switch ($_GET['sec']) { case "Activate": $controller->__changeStatus($_GET['proxyId'], 1); $controller->listProxy($_GET); break; case "Inactivate": $controller->__changeStatus($_GET['proxyId'], 0); $controller->listProxy($_GET); break; case "delete": $controller->__deleteProxy($_GET['proxyId']); $controller->listProxy($_GET); break; case "edit": $controller->editProxy($_GET['proxyId']); break; case "new": $controller->newProxy($_GET);
function getContent($url, $enableProxy = true, $logCrawl = true) { curl_setopt($this->_CURL_RESOURCE, CURLOPT_URL, $url); curl_setopt($this->_CURL_RESOURCE, CURLOPT_FAILONERROR, $this->_CURLOPT_FAILONERROR); @curl_setopt($this->_CURL_RESOURCE, CURLOPT_FOLLOWLOCATION, $this->_CURLOPT_FOLLOWLOCATION); curl_setopt($this->_CURL_RESOURCE, CURLOPT_RETURNTRANSFER, $this->_CURLOPT_RETURNTRANSFER); curl_setopt($this->_CURL_RESOURCE, CURLOPT_TIMEOUT, $this->_CURLOPT_TIMEOUT); curl_setopt($this->_CURL_RESOURCE, CURLOPT_COOKIEJAR, $this->_CURLOPT_COOKIEJAR); curl_setopt($this->_CURL_RESOURCE, CURLOPT_COOKIEFILE, $this->_CURLOPT_COOKIEFILE); curl_setopt($this->_CURL_RESOURCE, CURLOPT_HEADER, $this->_CURLOPT_HEADER); if (!empty($this->_CURLOPT_COOKIE)) { curl_setopt($this->_CURL_RESOURCE, CURLOPT_COOKIE, $this->_CURLOPT_COOKIE); } if (!empty($this->_CURLOPT_REFERER)) { curl_setopt($this->_CURL_RESOURCE, CURLOPT_REFERER, $this->_CURLOPT_REFERER); } if (strlen($this->_CURLOPT_POSTFIELDS) > 1) { curl_setopt($this->_CURL_RESOURCE, CURLOPT_POST, $this->_CURLOPT_POST); curl_setopt($this->_CURL_RESOURCE, CURLOPT_POSTFIELDS, $this->_CURLOPT_POSTFIELDS); } // user agent assignment $this->_CURLOPT_USERAGENT = defined('SP_USER_AGENT') ? SP_USER_AGENT : $this->_CURLOPT_USERAGENT; if (strlen($this->_CURLOPT_USERAGENT) > 0) { curl_setopt($this->_CURL_RESOURCE, CURLOPT_USERAGENT, $this->_CURLOPT_USERAGENT); } if (strlen($this->_CURLOPT_USERPWD) > 2) { curl_setopt($this->_CURL_RESOURCE, CURLOPT_USERPWD, $this->_CURLOPT_USERPWD); } // to use proxy if proxy enabled if (SP_ENABLE_PROXY && $enableProxy) { $proxyCtrler = new ProxyController(); if ($proxyInfo = $proxyCtrler->getRandomProxy()) { curl_setopt($this->_CURL_RESOURCE, CURLOPT_PROXY, $proxyInfo['proxy'] . ":" . $proxyInfo['port']); curl_setopt($this->_CURL_RESOURCE, CURLOPT_HTTPPROXYTUNNEL, CURLOPT_HTTPPROXYTUNNEL_VAL); if (!empty($proxyInfo['proxy_auth'])) { curl_setopt($this->_CURL_RESOURCE, CURLOPT_PROXYUSERPWD, $proxyInfo['proxy_username'] . ":" . $proxyInfo['proxy_password']); } } else { showErrorMsg("No active proxies found!! Please check your proxy settings from Admin Panel."); } } $ret['page'] = curl_exec($this->_CURL_RESOURCE); $ret['error'] = curl_errno($this->_CURL_RESOURCE); $ret['errmsg'] = curl_error($this->_CURL_RESOURCE); // update crawl log in database for future reference if ($logCrawl) { $crawlLogCtrl = new CrawlLogController(); $crawlInfo['crawl_status'] = $ret['error'] ? 0 : 1; $crawlInfo['ref_id'] = $crawlInfo['crawl_link'] = addslashes($url); $crawlInfo['crawl_referer'] = addslashes($this->_CURLOPT_REFERER); $crawlInfo['crawl_cookie'] = addslashes($this->_CURLOPT_COOKIE); $crawlInfo['crawl_post_fields'] = addslashes($this->_CURLOPT_POSTFIELDS); $crawlInfo['crawl_useragent'] = addslashes($this->_CURLOPT_USERAGENT); $crawlInfo['proxy_id'] = $proxyInfo['id']; $crawlInfo['log_message'] = addslashes($ret['errmsg']); $ret['log_id'] = $crawlLogCtrl->createCrawlLog($crawlInfo); } // disable proxy if not working if (SP_ENABLE_PROXY && $enableProxy && !empty($ret['error']) && !empty($proxyInfo['id'])) { // deactivate proxy if (PROXY_DEACTIVATE_CRAWL) { $proxyCtrler->__changeStatus($proxyInfo['id'], 0); } // chekc with another proxy if (CHECK_WITH_ANOTHER_PROXY_IF_FAILED) { $ret = $this->getContent($url, $enableProxy); } } return $ret; }
function getContent($url, $enableProxy = true) { curl_setopt($this->_CURL_RESOURCE, CURLOPT_URL, $url); curl_setopt($this->_CURL_RESOURCE, CURLOPT_FAILONERROR, $this->_CURLOPT_FAILONERROR); @curl_setopt($this->_CURL_RESOURCE, CURLOPT_FOLLOWLOCATION, $this->_CURLOPT_FOLLOWLOCATION); curl_setopt($this->_CURL_RESOURCE, CURLOPT_RETURNTRANSFER, $this->_CURLOPT_RETURNTRANSFER); curl_setopt($this->_CURL_RESOURCE, CURLOPT_TIMEOUT, $this->_CURLOPT_TIMEOUT); curl_setopt($this->_CURL_RESOURCE, CURLOPT_COOKIEJAR, $this->_CURLOPT_COOKIEJAR); curl_setopt($this->_CURL_RESOURCE, CURLOPT_COOKIEFILE, $this->_CURLOPT_COOKIEFILE); curl_setopt($this->_CURL_RESOURCE, CURLOPT_HEADER, $this->_CURLOPT_HEADER); if (!empty($this->_CURLOPT_COOKIE)) { curl_setopt($this->_CURL_RESOURCE, CURLOPT_COOKIE, $this->_CURLOPT_COOKIE); } if (!empty($this->_CURLOPT_REFERER)) { curl_setopt($this->_CURL_RESOURCE, CURLOPT_REFERER, $this->_CURLOPT_REFERER); } if (strlen($this->_CURLOPT_POSTFIELDS) > 1) { curl_setopt($this->_CURL_RESOURCE, CURLOPT_POST, $this->_CURLOPT_POST); curl_setopt($this->_CURL_RESOURCE, CURLOPT_POSTFIELDS, $this->_CURLOPT_POSTFIELDS); } // user agent assignment $this->_CURLOPT_USERAGENT = defined('SP_USER_AGENT') ? SP_USER_AGENT : $this->_CURLOPT_USERAGENT; if (strlen($this->_CURLOPT_USERAGENT) > 0) { curl_setopt($this->_CURL_RESOURCE, CURLOPT_USERAGENT, $this->_CURLOPT_USERAGENT); } if (strlen($this->_CURLOPT_USERPWD) > 2) { curl_setopt($this->_CURL_RESOURCE, CURLOPT_USERPWD, $this->_CURLOPT_USERPWD); } // to use proxy if proxy enabled if (SP_ENABLE_PROXY && $enableProxy) { $proxyCtrler = new ProxyController(); if ($proxyInfo = $proxyCtrler->getRandomProxy()) { curl_setopt($this->_CURL_RESOURCE, CURLOPT_PROXY, $proxyInfo['proxy'] . ":" . $proxyInfo['port']); curl_setopt($this->_CURL_RESOURCE, CURLOPT_HTTPPROXYTUNNEL, CURLOPT_HTTPPROXYTUNNEL_VAL); if (!empty($proxyInfo['proxy_auth'])) { curl_setopt($this->_CURL_RESOURCE, CURLOPT_PROXYUSERPWD, $proxyInfo['proxy_username'] . ":" . $proxyInfo['proxy_password']); } } else { showErrorMsg("No active proxies found!! Please check your proxy settings from Admin Panel."); } } $ret['page'] = curl_exec($this->_CURL_RESOURCE); $ret['error'] = curl_errno($this->_CURL_RESOURCE); $ret['errmsg'] = curl_error($this->_CURL_RESOURCE); // disable proxy if not working if (SP_ENABLE_PROXY && $enableProxy && !empty($ret['error']) && !empty($proxyInfo['id'])) { // deactivate proxy if (PROXY_DEACTIVATE_CRAWL) { echo "Deactivating proxy - " . $proxyInfo['proxy'] . "....<br>\n"; $proxyCtrler->__changeStatus($proxyInfo['id'], 0); } // chekc with another proxy if (CHECK_WITH_ANOTHER_PROXY_IF_FAILED) { echo "Checking with another proxy....<br>\n"; $ret = $this->getContent($url, $enableProxy); } } return $ret; }