*/
// a little example that fetches a bunch of sites in parallel and echos the page title and response info for each request
function request_callback($response, $info)
{
// parse the page title out of the returned HTML
if (preg_match("~<title>(.*?)</title>~i", $response, $out)) {
$title = $out[1];
}
echo "<b>{$title}</b><br />";
print_r($info);
echo "<hr>";
}
require "RollingCurl.php";
// single curl request
$rc = new RollingCurl("request_callback");
$rc->request("http://www.msn.com");
$rc->execute();
// another single curl request
$rc = new RollingCurl("request_callback");
$rc->request("http://www.google.com");
$rc->execute();
echo "<hr>";
// top 20 sites according to alexa (11/5/09)
$urls = array("http://www.google.com", "http://www.facebook.com", "http://www.yahoo.com", "http://www.youtube.com", "http://www.live.com", "http://www.wikipedia.com", "http://www.blogger.com", "http://www.msn.com", "http://www.baidu.com", "http://www.yahoo.co.jp", "http://www.myspace.com", "http://www.qq.com", "http://www.google.co.in", "http://www.twitter.com", "http://www.google.de", "http://www.microsoft.com", "http://www.google.cn", "http://www.sina.com.cn", "http://www.wordpress.com", "http://www.google.co.uk");
$rc = new RollingCurl("request_callback");
$rc->window_size = 20;
foreach ($urls as $url) {
$request = new Request($url);
$rc->add($request);
}
$rc->execute();
public static function curl_multi_get($requests)
{
cache::$__LinkToFunc = array();
cache::$__TempResult = array();
cache::$__ApiTime = microtime(true);
$rc = new RollingCurl(function ($response, $info, $request) {
$url = $request->url;
if ($response == null) {
cache::log("CURL RESULT EMPTY for url: " . $url . " with HTTP code " . $info['http_code']);
/*$response = file_get_contents($url);
if( $response == false )
{
cache::log("FILE_GET_CONTENTS failed for url: ".$url);
}*/
}
cache::log(sprintf("Got multi result for %s in %s sec, len %s", $url, number_format(microtime(true) - cache::$__ApiTime, 3), strlen($response)));
$cbtime = microtime(true);
if (cache::$__LinkToFunc[$url]) {
call_user_func_array(cache::$__LinkToFunc[$url], array($response));
}
cache::log(sprintf("Ran callback for URL in %s sec", number_format(microtime(true) - $cbtime, 3)));
cache::$__TempResult[$url] = $response;
cache::put($url, $response);
});
foreach ($requests as $request) {
cache::$__LinkToFunc[$request['url']] = $request['func'];
$rc->request($request['url']);
}
$rc->window_size = count($requests);
$rc->execute();
cache::log(sprintf("Multi API request took %s sec", number_format(microtime(true) - cache::$__ApiTime, 3)));
return cache::$__TempResult;
}
/**
* Request execution overload
*
* @access public
*
* @throws AngryCurlException
*
* @param string $url Request URL
* @param enum(GET/POST) $method
* @param array $post_data
* @param array $headers
* @param array $options
*
* @return bool
*/
public function request($url, $method = "GET", $post_data = null, $headers = null, $options = null)
{
if ($this->n_proxy > 0 && $this->use_proxy_list) {
$options[CURLOPT_PROXY] = $this->array_proxy[mt_rand(0, $this->n_proxy - 1)];
// self::add_debug_msg("Using PROXY({$this->n_proxy}): ".$options[CURLOPT_PROXY]);
} elseif ($this->n_proxy < 1 && $this->use_proxy_list) {
throw new AngryCurlException("(!) Option 'use_proxy_list' is set, but no alive proxy available");
}
if ($this->n_useragent > 0 && $this->use_useragent_list) {
$options[CURLOPT_USERAGENT] = $this->array_useragent[mt_rand(0, $this->n_useragent - 1)];
// self::add_debug_msg("Using USERAGENT: ".$options[CURLOPT_USERAGENT]);
} elseif ($this->n_useragent < 1 && $this->use_useragent_list) {
throw new AngryCurlException("(!) Option 'use_useragent_list' is set, but no useragents available");
}
parent::request($url, $method, $post_data, $headers, $options);
return true;
}
/**
* Request execution overload
*
* @access public
*
* @param string $url Request URL
* @param enum(GET/POST) $method
* @param array $post_data
* @param array $headers
* @param array $options
*
* @return bool
*/
public function request($url, $method = "GET", $post_data = null, $headers = null, $options = null)
{
if ($this->n_proxy > 0 && $this->use_proxy_list) {
$options[CURLOPT_PROXY] = $this->array_proxy[mt_rand(0, $this->n_proxy - 1)];
// self::add_debug_msg("Using PROXY({$this->n_proxy}): ".$options[CURLOPT_PROXY]);
}
if ($this->n_useragent > 0 && $this->use_useragent_list) {
$options[CURLOPT_USERAGENT] = $this->array_useragent[rand(0, $this->n_useragent - 1)];
// self::add_debug_msg("Using USERAGENT: ".$options[CURLOPT_USERAGENT]);
}
parent::request($url, $method, $post_data, $headers, $options);
return true;
}
