public static function findProxy($url, $curProxy)
{
if (!array_key_exists($host, ProxyFinder::$cache)) {
$host = parse_url($url)['host'];
}
if (!array_key_exists($host, ProxyFinder::$cache)) {
ProxyFinder::$cache[$host] = ProxyFinder::getNextProxy(null);
}
if (ProxyFinder::$cache[$host] && $curProxy == ProxyFinder::$cache[$host]) {
ProxyFinder::$cache[$host] = ProxyFinder::getNextProxy($curProxy);
}
return ProxyFinder::$cache[$host];
}
/**
* Performs multiple curl requests
*
* @access private
* @throws RollingCurlException
* @param int $window_size Max number of simultaneous connections
* @return bool
*/
private function rolling_curl($window_size = null)
{
if ($window_size) {
$this->window_size = $window_size;
}
// make sure the rolling window isn't greater than the # of urls
//if (sizeof($this->requests) < $this->window_size)
// $this->window_size = sizeof($this->requests);
//if ($this->window_size < 2) {
// throw new RollingCurlException("Window size must be greater than 1");
//}
$master = curl_multi_init();
// start the first batch of requests
for ($i = 0; $i < min($this->window_size, count($this->requests)); $i++) {
$ch = curl_init();
$options = $this->get_options($this->requests[$i]);
curl_setopt_array($ch, $options);
curl_multi_add_handle($master, $ch);
// Add to our request Maps
$key = (string) $ch;
$this->requestMap[$key] = $i;
}
do {
while (($execrun = curl_multi_exec($master, $running)) == CURLM_CALL_MULTI_PERFORM) {
}
if ($execrun != CURLM_OK) {
break;
}
// a request was just completed -- find out which one
while ($done = curl_multi_info_read($master)) {
// get the info and content returned on the request
$info = curl_getinfo($done['handle']);
$key = (string) $done['handle'];
$request = $this->requests[$this->requestMap[$key]];
$options = $this->get_options($request);
//try with proxy
if ($info['http_code'] != 200 && ($proxy = ProxyFinder::findProxy($info['url'], $options[CURLOPT_PROXY]))) {
$options[CURLOPT_PROXY] = $proxy;
$request->options = $options;
global $logger;
$logger->info("try " . $info['url'] . " with proxy: {$proxy}");
$this->requests[] = $request;
$running = 1;
} else {
$output = curl_multi_getcontent($done['handle']);
// send the return values to the callback function.
$callback = $this->callback;
if (is_callable($callback)) {
unset($this->requestMap[$key]);
call_user_func($callback, $output, $info, $request);
}
}
// start a new request (it's important to do this before removing the old one)
if ($i < sizeof($this->requests) && isset($this->requests[$i]) && $i < count($this->requests)) {
$ch = curl_init();
$options = $this->get_options($this->requests[$i]);
curl_setopt_array($ch, $options);
curl_multi_add_handle($master, $ch);
// Add to our request Maps
$key = (string) $ch;
$this->requestMap[$key] = $i;
$i++;
}
// remove the curl handle that just completed
curl_multi_remove_handle($master, $done['handle']);
}
// Block for data in / output; error handling is done by curl_multi_exec
if ($running) {
curl_multi_select($master, $this->timeout);
}
} while ($running);
curl_multi_close($master);
return true;
}
