*/ // a little example that fetches a bunch of sites in parallel and echos the page title and response info for each request function request_callback($response, $info) { // parse the page title out of the returned HTML if (preg_match("~<title>(.*?)</title>~i", $response, $out)) { $title = $out[1]; } echo "<b>{$title}</b><br />"; print_r($info); echo "<hr>"; } require "RollingCurl.php"; // single curl request $rc = new RollingCurl("request_callback"); $rc->request("http://www.msn.com"); $rc->execute(); // another single curl request $rc = new RollingCurl("request_callback"); $rc->request("http://www.google.com"); $rc->execute(); echo "<hr>"; // top 20 sites according to alexa (11/5/09) $urls = array("http://www.google.com", "http://www.facebook.com", "http://www.yahoo.com", "http://www.youtube.com", "http://www.live.com", "http://www.wikipedia.com", "http://www.blogger.com", "http://www.msn.com", "http://www.baidu.com", "http://www.yahoo.co.jp", "http://www.myspace.com", "http://www.qq.com", "http://www.google.co.in", "http://www.twitter.com", "http://www.google.de", "http://www.microsoft.com", "http://www.google.cn", "http://www.sina.com.cn", "http://www.wordpress.com", "http://www.google.co.uk"); $rc = new RollingCurl("request_callback"); $rc->window_size = 20; foreach ($urls as $url) { $request = new Request($url); $rc->add($request); } $rc->execute();
public static function curl_multi_get($requests) { cache::$__LinkToFunc = array(); cache::$__TempResult = array(); cache::$__ApiTime = microtime(true); $rc = new RollingCurl(function ($response, $info, $request) { $url = $request->url; if ($response == null) { cache::log("CURL RESULT EMPTY for url: " . $url . " with HTTP code " . $info['http_code']); /*$response = file_get_contents($url); if( $response == false ) { cache::log("FILE_GET_CONTENTS failed for url: ".$url); }*/ } cache::log(sprintf("Got multi result for %s in %s sec, len %s", $url, number_format(microtime(true) - cache::$__ApiTime, 3), strlen($response))); $cbtime = microtime(true); if (cache::$__LinkToFunc[$url]) { call_user_func_array(cache::$__LinkToFunc[$url], array($response)); } cache::log(sprintf("Ran callback for URL in %s sec", number_format(microtime(true) - $cbtime, 3))); cache::$__TempResult[$url] = $response; cache::put($url, $response); }); foreach ($requests as $request) { cache::$__LinkToFunc[$request['url']] = $request['func']; $rc->request($request['url']); } $rc->window_size = count($requests); $rc->execute(); cache::log(sprintf("Multi API request took %s sec", number_format(microtime(true) - cache::$__ApiTime, 3))); return cache::$__TempResult; }
/** * Request execution overload * * @access public * * @throws AngryCurlException * * @param string $url Request URL * @param enum(GET/POST) $method * @param array $post_data * @param array $headers * @param array $options * * @return bool */ public function request($url, $method = "GET", $post_data = null, $headers = null, $options = null) { if ($this->n_proxy > 0 && $this->use_proxy_list) { $options[CURLOPT_PROXY] = $this->array_proxy[mt_rand(0, $this->n_proxy - 1)]; // self::add_debug_msg("Using PROXY({$this->n_proxy}): ".$options[CURLOPT_PROXY]); } elseif ($this->n_proxy < 1 && $this->use_proxy_list) { throw new AngryCurlException("(!) Option 'use_proxy_list' is set, but no alive proxy available"); } if ($this->n_useragent > 0 && $this->use_useragent_list) { $options[CURLOPT_USERAGENT] = $this->array_useragent[mt_rand(0, $this->n_useragent - 1)]; // self::add_debug_msg("Using USERAGENT: ".$options[CURLOPT_USERAGENT]); } elseif ($this->n_useragent < 1 && $this->use_useragent_list) { throw new AngryCurlException("(!) Option 'use_useragent_list' is set, but no useragents available"); } parent::request($url, $method, $post_data, $headers, $options); return true; }
/** * Request execution overload * * @access public * * @param string $url Request URL * @param enum(GET/POST) $method * @param array $post_data * @param array $headers * @param array $options * * @return bool */ public function request($url, $method = "GET", $post_data = null, $headers = null, $options = null) { if ($this->n_proxy > 0 && $this->use_proxy_list) { $options[CURLOPT_PROXY] = $this->array_proxy[mt_rand(0, $this->n_proxy - 1)]; // self::add_debug_msg("Using PROXY({$this->n_proxy}): ".$options[CURLOPT_PROXY]); } if ($this->n_useragent > 0 && $this->use_useragent_list) { $options[CURLOPT_USERAGENT] = $this->array_useragent[rand(0, $this->n_useragent - 1)]; // self::add_debug_msg("Using USERAGENT: ".$options[CURLOPT_USERAGENT]); } parent::request($url, $method, $post_data, $headers, $options); return true; }