public static function multiGetFromCiteServer($mode, $sets, $style = 'chicago-note-bibliography') { require_once "../include/RollingCurl.inc.php"; $t = microtime(true); $setIDs = array(); $data = array(); $requestCallback = function ($response, $info) use($mode, &$setIDs, &$data) { if ($info['http_code'] != 200) { error_log("WARNING: HTTP {$info['http_code']} from citeserver {$mode} request: " . $response); return; } $response = json_decode($response); if (!$response) { error_log("WARNING: Invalid response from citeserver {$mode} request: " . $response); return; } $str = parse_url($info['url']); $str = parse_str($str['query']); if ($mode == 'citation') { $data[$setIDs[$setID]] = Zotero_Cite::processCitationResponse($response); } else { if ($mode == "bib") { $data[$setIDs[$setID]] = Zotero_Cite::processBibliographyResponse($response); } } }; $rc = new RollingCurl($requestCallback); // Number of simultaneous requests $rc->window_size = 20; foreach ($sets as $key => $items) { $json = self::getJSONFromItems($items); $server = Z_CONFIG::$CITATION_SERVERS[array_rand(Z_CONFIG::$CITATION_SERVERS)]; $url = "http://{$server}/?responseformat=json&style={$style}"; if ($mode == 'citation') { $url .= "&citations=1&bibliography=0"; } // Include array position in URL so that the callback can figure // out what request this was $url .= "&setID=" . $key; // TODO: support multiple items per set, if necessary if (!$items instanceof Zotero_Item) { throw new Exception("items is not a Zotero_Item"); } $setIDs[$key] = $items->libraryID . "/" . $items->key; $request = new RollingCurlRequest($url); $request->options = array(CURLOPT_POST => 1, CURLOPT_POSTFIELDS => $json, CURLOPT_HTTPHEADER => array("Expect:"), CURLOPT_CONNECTTIMEOUT => 1, CURLOPT_TIMEOUT => 4, CURLOPT_HEADER => 0, CURLOPT_RETURNTRANSFER => 1); $rc->add($request); } $rc->execute(); error_log(sizeOf($sets) . " {$mode} requests in " . round(microtime(true) - $t, 3)); return $data; }
/** * 抓取网页 * * @param int $window_size curl批处理句柄最大同时连接数 */ public function fetch_html($window_size = NULL) { $rc = new RollingCurl($this->callback); foreach ($this->urls as $url) { $request = new RollingCurlRequest($url, 'Get', NULL, NULL, $this->options); $rc->add($request); } if (!empty($window_size)) { $rc->execute($window_size); } else { $rc->execute($this->window_size); } }
/** * Default constructor. * * @access public * @param string $startUrl The starting url for crawling * @param string $dbUser The database username * @param string $dbPass The database password * @param string $dbName The database name * @param string $dbHost The database host * @return void */ public function __construct($startUrl, $dbUser, $dbPass, $dbName, $dbHost) { // register the Rolling Curl callback parent::__construct('parseHtml'); // validate the url is valid HTTP or HTTPS if (strpos($startUrl, 'http://') === false && strpos($startUrl, 'https://') === false) { throw new Exception('Your starting caterpillar URL must begin with http or https.'); } // store the base url $this->startUrl = rtrim($startUrl, '/') . '/'; // store the start time $this->startTime = date('Y-m-d H:i:s'); // store the base domain $info = parse_url($startUrl); $this->domain = $info['scheme'] . '://' . $info['host']; // open mysql connection $this->db = mysql_connect($dbHost, $dbUser, $dbPass); if ($this->db === false) { throw new Exception('The caterpillar database connection could not be established'); } // use user specified database if (!mysql_select_db($dbName)) { throw new Exception('An error occurred attempting to connect to the database ' . $dbName); } }
public function execute($window_size = null) { if (count($this->requests) == 0) { return false; } return parent::execute($window_size); }
function updateLinks() { global $logger; $logger->info('UPDATE LINKS'); $resPirate = array(); //parallel RollingCurl RollingCurl::$rc = new RollingCurl("main_callback"); // the window size determines how many simultaneous requests to allow. RollingCurl::$rc->window_size = 5; //List of tracker loaders $loaders = array(); $rutorMain = RUTORROOT; $loaders[] = new RutorLoader("{$rutorMain}/browse/0/1/0/2/"); //foreign movies $loaders[] = new RutorLoader("{$rutorMain}/browse/1/1/0/2/"); $loaders[] = new RutorLoader("{$rutorMain}/browse/2/1/0/2/"); $loaders[] = new RutorLoader("{$rutorMain}/browse/3/1/0/2/"); $loaders[] = new RutorLoader("{$rutorMain}/browse/4/1/0/2/"); $loaders[] = new RutorLoader("{$rutorMain}/browse/5/1/0/2/"); $loaders[] = new RutorLoader("{$rutorMain}/browse/6/1/0/2/"); $loaders[] = new RutorLoader("{$rutorMain}/browse/7/1/0/2/"); $loaders[] = new RutorLoader("{$rutorMain}/browse/8/1/0/2/"); $loaders[] = new RutorLoader("{$rutorMain}/browse/9/1/0/2/"); $loaders[] = new RutorLoader("{$rutorMain}/browse/10/1/0/2/"); $loaders[] = new RutorLoader("{$rutorMain}/browse/11/1/0/2/"); $loaders[] = new RutorLoader("{$rutorMain}/browse/12/1/0/2/"); $loaders[] = new RutorLoader("{$rutorMain}/browse/13/1/0/2/"); $loaders[] = new RutorLoader("{$rutorMain}/browse/0/7/0/2/"); //multiplication $loaders[] = new RutorLoader("{$rutorMain}/browse/0/5/0/2"); //russian movies $loaders[] = new RutorLoader("{$rutorMain}/browse/1/5/0/2"); $NNMData = array("prev_sd" => 0, "prev_a" => 0, "prev_my" => 0, "prev_n" => 0, "prev_shc" => 0, "prev_shf" => 1, "prev_sha" => 1, "prev_shs" => 0, "prev_shr" => 0, "prev_sht" => 0, "f[0]" => 270, "o" => 10, "s" => 2, "tm" => -1, "ta" => -1, "sns" => -1, "sds" => -1, "nm" => "", "pn" => ""); $loaders[] = new NNMLoader("http://nnm-club.me/forum/tracker.php", $NNMData); $NNMData["f[0]"] = 218; $loaders[] = new NNMLoader("http://nnm-club.me/forum/tracker.php", $NNMData); $NNMData["f[0]"] = 217; $NNMData["f[1]"] = 954; $loaders[] = new NNMLoader("http://nnm-club.me/forum/tracker.php", $NNMData); $pirateMain = PIRATEROOT; $loaders[] = new PirateLoader("{$pirateMain}/browse/201/0/7/0"); /* $resPirate1 = new Pirate; $resPirate1->getPirateBay("https://pirateproxy.sx/browse/207/0/7"); $resPirate2 = new Pirate; $resPirate2->getPirateBay("https://pirateproxy.sx/browse/207/1/7"); $resPirate3 = new Pirate; $resPirate3->getPirateBay("https://pirateproxy.sx/browse/201/0/7"); $resPirate = array_merge($resPirate1->result, $resPirate2->result, $resPirate3->result, $resRutor, $resSeedoff); flush(); */ foreach ($loaders as $loader) { $loader->setLogger($logger); $loader->load(); } RollingCurl::$rc->execute(); //result array with torrent infos $result = array(); foreach ($loaders as $loader) { $result = array_merge($result, $loader->getResult()); } $resSeedoff = array(); $resSeedoff = seedoff\getSeedoff(); $resSeedoff = array_merge($resSeedoff, seedoff\getSeedoff("http://www.seedoff.net/index.php?page=ajax&active=0&options=0&recommend=0&sticky=0&period=0&category=14&options=0&order=5&by=2&pages=2")); $resSeedoff = array_merge($resSeedoff, seedoff\getSeedoff("http://www.seedoff.net/index.php?page=ajax&active=0&options=0&recommend=0&sticky=0&period=0&category=14&options=0&order=5&by=2&pages=3")); $resSeedoff = array_merge($resSeedoff, seedoff\getSeedoff("http://www.seedoff.net/index.php?page=ajax&active=0&options=0&recommend=0&sticky=0&period=0&category=64&options=0&order=5&by=2&pages=1")); $result = array_merge($result, $resSeedoff); foreach ($result as $cur) { if (trySkip($cur)) { continue; } getIds($cur['title_approx'], $cur); $logger->info("add link: " . $cur['title_approx'] . "::" . $cur['description'] . "::" . $cur['link']); $res = addLink($cur); if ($res !== 0) { $logger->warning("link was not added: {$res}"); } usleep(100 * 1000); } $logger->info(count($result) . " links updated"); }
*/ // a little example that fetches a bunch of sites in parallel and echos the page title and response info for each request function request_callback($response, $info) { // parse the page title out of the returned HTML if (preg_match("~<title>(.*?)</title>~i", $response, $out)) { $title = $out[1]; } echo "<b>{$title}</b><br />"; print_r($info); echo "<hr>"; } require "RollingCurl.php"; // single curl request $rc = new RollingCurl("request_callback"); $rc->request("http://www.msn.com"); $rc->execute(); // another single curl request $rc = new RollingCurl("request_callback"); $rc->request("http://www.google.com"); $rc->execute(); echo "<hr>"; // top 20 sites according to alexa (11/5/09) $urls = array("http://www.google.com", "http://www.facebook.com", "http://www.yahoo.com", "http://www.youtube.com", "http://www.live.com", "http://www.wikipedia.com", "http://www.blogger.com", "http://www.msn.com", "http://www.baidu.com", "http://www.yahoo.co.jp", "http://www.myspace.com", "http://www.qq.com", "http://www.google.co.in", "http://www.twitter.com", "http://www.google.de", "http://www.microsoft.com", "http://www.google.cn", "http://www.sina.com.cn", "http://www.wordpress.com", "http://www.google.co.uk"); $rc = new RollingCurl("request_callback"); $rc->window_size = 20; foreach ($urls as $url) { $request = new Request($url); $rc->add($request); } $rc->execute();
/** * AngryCurl destructor * * @return void */ function __destruct() { self::add_debug_msg("# Finishing ..."); parent::__destruct(); }
if (!isset($fision[1])) { continue; } //$request = new RollingCurlRequest('http://'.preg_replace('@http[s]?://@i','',$ips[$i])); $request = new RollingCurlRequest('http://' . preg_replace('@http[s]?://@i', '', $fision[1])); $request->rank = $fision[0]; $request->outputfilehandler = $fh; if (isset($fision[5]) && $fision[5] !== '' && $fision[5] !== '-1') { $request->pagerank = $fision[5]; } $headers = array("Host: " . $fision[1]); $request->options = array(CURLOPT_HTTPHEADER => $headers); $rc->add($request); if (($i + 1) % $window_size === 0) { $rc->execute(); $rc = new RollingCurl("request_callback"); $rc->window_size = $window_size; if (($sleeptime = $window_size * 1100 - (($curtime = microtime(true)) - $time)) > 10000) { usleep($sleeptime); } $time = $curtime; } } if (($maxi + 1) % $window_size !== 0) { $rc->execute(); } fclose($fh); echo "rank, url, \"title\", \"keywords\", \"description\", PageRank CSV file of English websites in top Alexa 1M generated in {$file}.\n"; if ($snapshots) { require_once 'snapshots.php'; }
public function fetchAllOnce(array $urls, $isRedirect = false) { if (!$isRedirect) { $urls = array_unique($urls); } if (empty($urls)) { return; } ////////////////////////////////////////////////////// // parallel (HttpRequestPool) if ($this->method == self::METHOD_REQUEST_POOL) { $this->debug('Starting parallel fetch (HttpRequestPool)'); try { while (count($urls) > 0) { $this->debug('Processing set of ' . min($this->maxParallelRequests, count($urls))); $subset = array_splice($urls, 0, $this->maxParallelRequests); $pool = new HttpRequestPool(); foreach ($subset as $orig => $url) { if (!$isRedirect) { $orig = $url; } unset($this->redirectQueue[$orig]); $this->debug("...{$url}"); if (!$isRedirect && isset($this->requests[$url])) { $this->debug("......in memory"); /* } elseif ($this->isCached($url)) { $this->debug("......is cached"); if (!$this->minimiseMemoryUse) { $this->requests[$url] = $this->getCached($url); } */ } else { $this->debug("......adding to pool"); $req_url = $this->rewriteHashbangFragment ? $this->rewriteHashbangFragment($url) : $url; $req_url = $this->removeFragment($req_url); $httpRequest = new HttpRequest($req_url, HttpRequest::METH_GET, $this->requestOptions); // send cookies, if we have any if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) { $this->debug("......sending cookies: {$cookies}"); $httpRequest->addHeaders(array('Cookie' => $cookies)); } $this->requests[$orig] = array('headers' => null, 'body' => null, 'httpRequest' => $httpRequest); $this->requests[$orig]['original_url'] = $orig; $pool->attach($httpRequest); } } // did we get anything into the pool? if (count($pool) > 0) { $this->debug('Sending request...'); try { $pool->send(); } catch (HttpRequestPoolException $e) { // do nothing } $this->debug('Received responses'); foreach ($subset as $orig => $url) { if (!$isRedirect) { $orig = $url; } //if (!isset($this->requests[$url]['fromCache'])) { $request = $this->requests[$orig]['httpRequest']; //$this->requests[$orig]['headers'] = $this->headersToString($request->getResponseHeader()); // getResponseHeader() doesn't return status line, so, for consistency... $this->requests[$orig]['headers'] = substr($request->getRawResponseMessage(), 0, $request->getResponseInfo('header_size')); $this->requests[$orig]['body'] = $request->getResponseBody(); $this->requests[$orig]['effective_url'] = $request->getResponseInfo('effective_url'); $this->requests[$orig]['status_code'] = $status_code = $request->getResponseCode(); // is redirect? if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && $request->getResponseHeader('location')) { $redirectURL = $request->getResponseHeader('location'); if (!preg_match('!^https?://!i', $redirectURL)) { $redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url); } if ($this->validateURL($redirectURL)) { $this->debug('Redirect detected. Valid URL: ' . $redirectURL); // store any cookies $cookies = $request->getResponseHeader('set-cookie'); if ($cookies && !is_array($cookies)) { $cookies = array($cookies); } if ($cookies) { $this->cookieJar->storeCookies($url, $cookies); } $this->redirectQueue[$orig] = $redirectURL; } else { $this->debug('Redirect detected. Invalid URL: ' . $redirectURL); } } //die($url.' -multi- '.$request->getResponseInfo('effective_url')); $pool->detach($request); unset($this->requests[$orig]['httpRequest'], $request); /* if ($this->minimiseMemoryUse) { if ($this->cache($url)) { unset($this->requests[$url]); } } */ //} } } } } catch (HttpException $e) { $this->debug($e); return false; } } elseif ($this->method == self::METHOD_CURL_MULTI) { $this->debug('Starting parallel fetch (curl_multi_*)'); while (count($urls) > 0) { $this->debug('Processing set of ' . min($this->maxParallelRequests, count($urls))); $subset = array_splice($urls, 0, $this->maxParallelRequests); $pool = new RollingCurl(array($this, 'handleCurlResponse')); $pool->window_size = count($subset); foreach ($subset as $orig => $url) { if (!$isRedirect) { $orig = $url; } unset($this->redirectQueue[$orig]); $this->debug("...{$url}"); if (!$isRedirect && isset($this->requests[$url])) { $this->debug("......in memory"); /* } elseif ($this->isCached($url)) { $this->debug("......is cached"); if (!$this->minimiseMemoryUse) { $this->requests[$url] = $this->getCached($url); } */ } else { $this->debug("......adding to pool"); $req_url = $this->rewriteHashbangFragment ? $this->rewriteHashbangFragment($url) : $url; $req_url = $this->removeFragment($req_url); $headers = array(); // send cookies, if we have any if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) { $this->debug("......sending cookies: {$cookies}"); $headers[] = 'Cookie: ' . $cookies; } $httpRequest = new RollingCurlRequest($req_url, 'GET', null, $headers, array(CURLOPT_CONNECTTIMEOUT => $this->requestOptions['timeout'], CURLOPT_TIMEOUT => $this->requestOptions['timeout'])); $httpRequest->set_original_url($orig); $this->requests[$orig] = array('headers' => null, 'body' => null, 'httpRequest' => $httpRequest); $this->requests[$orig]['original_url'] = $orig; // TODO: is this needed anymore? $pool->add($httpRequest); } } // did we get anything into the pool? if (count($pool) > 0) { $this->debug('Sending request...'); $pool->execute(); // this will call handleCurlResponse() and populate $this->requests[$orig] $this->debug('Received responses'); foreach ($subset as $orig => $url) { if (!$isRedirect) { $orig = $url; } // $this->requests[$orig]['headers'] // $this->requests[$orig]['body'] // $this->requests[$orig]['effective_url'] $status_code = $this->requests[$orig]['status_code']; if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && isset($this->requests[$orig]['location'])) { $redirectURL = $this->requests[$orig]['location']; if (!preg_match('!^https?://!i', $redirectURL)) { $redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url); } if ($this->validateURL($redirectURL)) { $this->debug('Redirect detected. Valid URL: ' . $redirectURL); // store any cookies $cookies = $this->cookieJar->extractCookies($this->requests[$orig]['headers']); if (!empty($cookies)) { $this->cookieJar->storeCookies($url, $cookies); } $this->redirectQueue[$orig] = $redirectURL; } else { $this->debug('Redirect detected. Invalid URL: ' . $redirectURL); } } // die($url.' -multi- '.$request->getResponseInfo('effective_url')); unset($this->requests[$orig]['httpRequest']); } } } } else { $this->debug('Starting sequential fetch (file_get_contents)'); $this->debug('Processing set of ' . count($urls)); foreach ($urls as $orig => $url) { if (!$isRedirect) { $orig = $url; } unset($this->redirectQueue[$orig]); $this->debug("...{$url}"); if (!$isRedirect && isset($this->requests[$url])) { $this->debug("......in memory"); /* } elseif ($this->isCached($url)) { $this->debug("......is cached"); if (!$this->minimiseMemoryUse) { $this->requests[$url] = $this->getCached($url); } */ } else { $this->debug("Sending request for {$url}"); $this->requests[$orig]['original_url'] = $orig; $req_url = $this->rewriteHashbangFragment ? $this->rewriteHashbangFragment($url) : $url; $req_url = $this->removeFragment($req_url); // send cookies, if we have any $httpContext = $this->httpContext; if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) { $this->debug("......sending cookies: {$cookies}"); $httpContext['http']['header'] .= 'Cookie: ' . $cookies . "\r\n"; } if (false !== ($html = @file_get_contents($req_url, false, stream_context_create($httpContext)))) { $this->debug('Received response'); // get status code if (!isset($http_response_header[0]) || !preg_match('!^HTTP/\\d+\\.\\d+\\s+(\\d+)!', trim($http_response_header[0]), $match)) { $this->debug('Error: no status code found'); // TODO: handle error - no status code } else { $this->requests[$orig]['headers'] = $this->headersToString($http_response_header, false); $this->requests[$orig]['body'] = $html; $this->requests[$orig]['effective_url'] = $req_url; $this->requests[$orig]['status_code'] = $status_code = (int) $match[1]; unset($match); // handle redirect if (preg_match('/^Location:(.*?)$/m', $this->requests[$orig]['headers'], $match)) { $this->requests[$orig]['location'] = trim($match[1]); } if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && isset($this->requests[$orig]['location'])) { $redirectURL = $this->requests[$orig]['location']; if (!preg_match('!^https?://!i', $redirectURL)) { $redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url); } if ($this->validateURL($redirectURL)) { $this->debug('Redirect detected. Valid URL: ' . $redirectURL); // store any cookies $cookies = $this->cookieJar->extractCookies($this->requests[$orig]['headers']); if (!empty($cookies)) { $this->cookieJar->storeCookies($url, $cookies); } $this->redirectQueue[$orig] = $redirectURL; } else { $this->debug('Redirect detected. Invalid URL: ' . $redirectURL); } } } } else { $this->debug('Error retrieving URL'); //print_r($req_url); //print_r($http_response_header); //print_r($html); // TODO: handle error - failed to retrieve URL } } } } }
if ($problem) { echo "\nTHERE MAY BE A PROBLEM WITH THIS RESPONSE!\n"; echo $problem; $hasproblem = true; } } $results = array(); $verify = new RollingCurl(function ($response, $info, $request) use(&$results) { $png = file_get_contents(randpng($request->index)); if ($response == $png) { $results[$request->index] = 'pass'; } elseif (strpos($response, 'Integrity constraint violation') !== false) { $result = "retry\n"; } else { $result = "fail\n"; if ($response[0] == '<') { $result .= "\t\tReceived: " . $response . "...\n"; } else { $result .= "\t\tReceived: " . base64_encode(substr($response, 0, 50)) . "...\n"; } $result .= "\t\tSent: " . base64_encode(substr($png, 0, 50)) . "...\n"; $results[$request->index] = $result; } }); for ($x = 1; $x <= $files_appearing; $x++) { $results[$x] = 'unknown'; $request = new RollingCurlRequest('http://' . $username . ':' . $password . '@' . $owncloud_remote . '/webdav' . $testdir . '/sample_' . $x . '.png', 'GET'); $request->index = $x; $verify->add($request); } $verify->execute($window);
public function sendRequestCallBack($response, $info = '') { // 处理内容 $response = preg_replace("/<code[\\s\\S]*><!--/iU", "", $response); $response = preg_replace("/--><\\/code>/iU", "", $response); // 分析html,获取添加好友需要的数据 \phpQuery::newDocumentHTML($response); $pagelet_timeline_main_column = pq("#pagelet_timeline_main_column")->attr("data-gt"); $pagelet_timeline_main_column = json_decode($pagelet_timeline_main_column); $profile_owner = $pagelet_timeline_main_column->profile_owner; $requests = array(); foreach (pq("div.fsl.fwb.fcb > a") as $value) { $data_hovercard = pq($value)->attr("data-hovercard"); $data_gt = pq($value)->attr("data-gt"); $data_gt = json_decode($data_gt); preg_match("/\\?id=([0-9]*)&/iU", $data_hovercard, $matches); $to_friend = $matches[1]; // 发送好友请求 $query = array("to_friend" => $to_friend, "action" => "add_friend", "how_found" => "profile_friends", "ref_param" => "pb_friends_tl", "link_data[gt][coeff2_registry_key]" => $data_gt->coeff2_registry_key, "link_data[gt][coeff2_info]" => $data_gt->coeff2_info, "link_data[gt][coeff2_action]" => $data_gt->coeff2_action, "link_data[gt][coeff2_pv_signature]" => $data_gt->coeff2_pv_signature, "link_data[gt][profile_owner]" => $profile_owner, "link_data[gt][ref]" => "timeline:timeline", "outgoing_id" => '', "logging_location" => '', "no_flyout_on_click" => "true", "ego_log_data" => '', "http_referer" => '', "floc" => "friends_tab", "__user" => $this->user_id, "__a" => "1", "fb_dtsg" => $this->token, "__rev" => $this->version); $capt_opts = $this->curl_opts; $url = "https://www.facebook.com/ajax/add_friend/action.php?__pc=EXP1%3ADEFAULT"; $capt_opts[CURLOPT_POST] = true; $capt_opts[CURLOPT_POSTFIELDS] = $query; $request = new \RollingCurlRequest($url); $request->options = $capt_opts; $requests[] = $request; } if (empty($requests)) { return; } $rc = new \RollingCurl(); if (sizeof($requests) < 20) { $rc->window_size = sizeof($requests); } else { $rc->window_size = 20; } foreach ($requests as $value) { $rc->add($value); } $rc->execute(); }
public static function curl_multi_get($requests) { cache::$__LinkToFunc = array(); cache::$__TempResult = array(); cache::$__ApiTime = microtime(true); $rc = new RollingCurl(function ($response, $info, $request) { $url = $request->url; if ($response == null) { cache::log("CURL RESULT EMPTY for url: " . $url . " with HTTP code " . $info['http_code']); /*$response = file_get_contents($url); if( $response == false ) { cache::log("FILE_GET_CONTENTS failed for url: ".$url); }*/ } cache::log(sprintf("Got multi result for %s in %s sec, len %s", $url, number_format(microtime(true) - cache::$__ApiTime, 3), strlen($response))); $cbtime = microtime(true); if (cache::$__LinkToFunc[$url]) { call_user_func_array(cache::$__LinkToFunc[$url], array($response)); } cache::log(sprintf("Ran callback for URL in %s sec", number_format(microtime(true) - $cbtime, 3))); cache::$__TempResult[$url] = $response; cache::put($url, $response); }); foreach ($requests as $request) { cache::$__LinkToFunc[$request['url']] = $request['func']; $rc->request($request['url']); } $rc->window_size = count($requests); $rc->execute(); cache::log(sprintf("Multi API request took %s sec", number_format(microtime(true) - cache::$__ApiTime, 3))); return cache::$__TempResult; }
function __destruct() { parent::__destruct(); }
// 'dol.gov', // 'humanresources.about.com', // 'shrm.org', // 'diversityinc.com', // 'stevepavlina.com/blog/', // 'osha.gov', // 'hr.com', // 'ere.net', // 'cisin.com', // 'blr.com', // 'peopleadmin.com', // 'wageworks.com', // 'dalecarnegie.com', // 'doleta.gov', // 'mercer.com', // 'astd.org', // 'brightscope.com', // 'tmp.com', // 'trinet.com', //]; // DETECT CYCLE! $loader = new FileLoader(); $scanner = new Scanner($loader, __DIR__ . '/apps.json'); $category = new Category($loader, __DIR__ . '/apps.json'); require_once __DIR__ . '/lib/RollingCurl.php'; $rc = new RollingCurl(); foreach ($urls as $url) { $rc->add(new RollingCurlRequest("http://scanner.loc/worker.php?url=" . urlencode($url))); } $rc->execute(10); echo "All request sent";