static function _fetchUrl(&$website, $url, $postdata, $checkConstraints = false, $pForceFetch = false, $verifyCertificate = null, $http_user = null, $http_pass = null, $sslVersion = 0)
{
//$agent = 'Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)';
$agent = 'Mozilla/5.0 (compatible; MainWP/' . MainWP_System::$version . '; +http://mainwp.com)';
if (!$pForceFetch) {
//todo: RS:
//check if offline
}
$identifier = null;
if ($checkConstraints) {
$semLock = '103218';
//SNSyncLock
//Lock
$identifier = MainWP_Utility::getLockIdentifier($semLock);
//Check the delays
//In MS
$minimumDelay = get_option('mainwp_minimumDelay') === false ? 200 : get_option('mainwp_minimumDelay');
if ($minimumDelay > 0) {
$minimumDelay = $minimumDelay / 1000;
}
$minimumIPDelay = get_option('mainwp_minimumIPDelay') === false ? 1000 : get_option('mainwp_minimumIPDelay');
if ($minimumIPDelay > 0) {
$minimumIPDelay = $minimumIPDelay / 1000;
}
MainWP_Utility::endSession();
$delay = true;
while ($delay) {
MainWP_Utility::lock($identifier);
if ($minimumDelay > 0) {
//Check last request overall
$lastRequest = MainWP_DB::Instance()->getLastRequestTimestamp();
if ($lastRequest > microtime(true) - $minimumDelay) {
//Delay!
MainWP_Utility::release($identifier);
usleep(($minimumDelay - (microtime(true) - $lastRequest)) * 1000 * 1000);
continue;
}
}
if ($minimumIPDelay > 0 && $website != null) {
//Get ip of this site url
$ip = MainWP_DB::Instance()->getWPIp($website->id);
if ($ip != null && $ip != '') {
//Check last request for this site
$lastRequest = MainWP_DB::Instance()->getLastRequestTimestamp($ip);
//Check last request for this subnet?
if ($lastRequest > microtime(true) - $minimumIPDelay) {
//Delay!
MainWP_Utility::release($identifier);
usleep(($minimumIPDelay - (microtime(true) - $lastRequest)) * 1000 * 1000);
continue;
}
}
}
$delay = false;
}
//Check the simultaneous requests
$maximumRequests = get_option('mainwp_maximumRequests') === false ? 4 : get_option('mainwp_maximumRequests');
$maximumIPRequests = get_option('mainwp_maximumIPRequests') === false ? 1 : get_option('mainwp_maximumIPRequests');
$first = true;
$delay = true;
while ($delay) {
if (!$first) {
MainWP_Utility::lock($identifier);
} else {
$first = false;
}
//Clean old open requests (may have timed out or something..)
MainWP_DB::Instance()->closeOpenRequests();
if ($maximumRequests > 0) {
$nrOfOpenRequests = MainWP_DB::Instance()->getNrOfOpenRequests();
if ($nrOfOpenRequests >= $maximumRequests) {
//Delay!
MainWP_Utility::release($identifier);
//Wait 200ms
usleep(200000);
continue;
}
}
if ($maximumIPRequests > 0 && $website != null) {
//Get ip of this site url
$ip = MainWP_DB::Instance()->getWPIp($website->id);
if ($ip != null && $ip != '') {
$nrOfOpenRequests = MainWP_DB::Instance()->getNrOfOpenRequests($ip);
if ($nrOfOpenRequests >= $maximumIPRequests) {
//Delay!
MainWP_Utility::release($identifier);
//Wait 200ms
usleep(200000);
continue;
}
}
}
$delay = false;
}
}
if ($website != null) {
//Log the start of this request!
MainWP_DB::Instance()->insertOrUpdateRequestLog($website->id, null, microtime(true), null);
}
if ($identifier != null) {
//Unlock
MainWP_Utility::release($identifier);
}
$dirs = self::getMainWPDir();
$cookieDir = $dirs[0] . 'cookies';
if (!@is_dir($cookieDir)) {
@mkdir($cookieDir, 0777, true);
}
if (!file_exists($cookieDir . '/.htaccess')) {
$file_htaccess = @fopen($cookieDir . '/.htaccess', 'w+');
@fwrite($file_htaccess, 'deny from all');
@fclose($file_htaccess);
}
if (!file_exists($cookieDir . '/index.php')) {
$file_index = @fopen($cookieDir . '/index.php', 'w+');
@fclose($file_index);
}
$ch = curl_init();
if ($website != null) {
$cookieFile = $cookieDir . '/' . sha1(sha1('mainwp' . LOGGED_IN_SALT . $website->id) . NONCE_SALT . 'WP_Cookie');
if (!file_exists($cookieFile)) {
@file_put_contents($cookieFile, '');
}
if (file_exists($cookieFile)) {
@curl_setopt($ch, CURLOPT_COOKIEJAR, $cookieFile);
@curl_setopt($ch, CURLOPT_COOKIEFILE, $cookieFile);
}
}
@curl_setopt($ch, CURLOPT_URL, $url);
@curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
@curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
@curl_setopt($ch, CURLOPT_POST, true);
@curl_setopt($ch, CURLOPT_POSTFIELDS, $postdata);
@curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
@curl_setopt($ch, CURLOPT_USERAGENT, $agent);
if (!empty($http_user) && !empty($http_pass)) {
@curl_setopt($ch, CURLOPT_USERPWD, "{$http_user}:{$http_pass}");
}
$ssl_verifyhost = false;
if ($verifyCertificate !== null) {
if ($verifyCertificate == 1) {
$ssl_verifyhost = true;
} else {
if ($verifyCertificate == 2) {
// use global setting
if (get_option('mainwp_sslVerifyCertificate') === false || get_option('mainwp_sslVerifyCertificate') == 1) {
$ssl_verifyhost = true;
}
}
}
} else {
if (get_option('mainwp_sslVerifyCertificate') === false || get_option('mainwp_sslVerifyCertificate') == 1) {
$ssl_verifyhost = true;
}
}
if ($ssl_verifyhost) {
@curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2);
@curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, true);
} else {
@curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
@curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
}
@curl_setopt($ch, CURLOPT_SSLVERSION, $sslVersion);
$timeout = 20 * 60 * 60;
//20 minutes
@curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
if (!ini_get('safe_mode')) {
@set_time_limit($timeout);
}
@ini_set('max_execution_time', $timeout);
MainWP_Utility::endSession();
$disabled_functions = ini_get('disable_functions');
if (empty($disabled_functions) || stristr($disabled_functions, 'curl_multi_exec') === false) {
$mh = @curl_multi_init();
@curl_multi_add_handle($mh, $ch);
$lastRun = 0;
do {
if (time() - $lastRun > 20) {
@set_time_limit($timeout);
//reset timer..
$lastRun = time();
}
@curl_multi_exec($mh, $running);
//Execute handlers
//$ready = curl_multi_select($mh);
while ($info = @curl_multi_info_read($mh)) {
$data = @curl_multi_getcontent($info['handle']);
$http_status = @curl_getinfo($info['handle'], CURLINFO_HTTP_CODE);
$err = @curl_error($info['handle']);
$real_url = @curl_getinfo($info['handle'], CURLINFO_EFFECTIVE_URL);
@curl_multi_remove_handle($mh, $info['handle']);
}
usleep(10000);
} while ($running > 0);
@curl_multi_close($mh);
} else {
$data = @curl_exec($ch);
$http_status = @curl_getinfo($ch, CURLINFO_HTTP_CODE);
$err = @curl_error($ch);
$real_url = @curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
}
$host = parse_url($real_url, PHP_URL_HOST);
$ip = gethostbyname($host);
if ($website != null) {
MainWP_DB::Instance()->insertOrUpdateRequestLog($website->id, $ip, null, microtime(true));
}
if ($data === false && $http_status == 0) {
MainWP_Logger::Instance()->debugForWebsite($website, 'fetchUrl', '[' . $url . '] HTTP Error: [status=0][' . $err . ']');
throw new MainWP_Exception('HTTPERROR', $err);
} else {
if (empty($data) && !empty($err)) {
MainWP_Logger::Instance()->debugForWebsite($website, 'fetchUrl', '[' . $url . '] HTTP Error: [status=' . $http_status . '][' . $err . ']');
throw new MainWP_Exception('HTTPERROR', $err);
} else {
if (preg_match('/<mainwp>(.*)<\\/mainwp>/', $data, $results) > 0) {
$result = $results[1];
$information = unserialize(base64_decode($result));
return $information;
} else {
MainWP_Logger::Instance()->debugForWebsite($website, 'fetchUrl', '[' . $url . '] Result was: [' . $data . ']');
throw new MainWP_Exception('NOMAINWP', $url);
}
}
}
}