public static function multiGetFromCiteServer($mode, $sets, $style = 'chicago-note-bibliography')
{
require_once "../include/RollingCurl.inc.php";
$t = microtime(true);
$setIDs = array();
$data = array();
$requestCallback = function ($response, $info) use($mode, &$setIDs, &$data) {
if ($info['http_code'] != 200) {
error_log("WARNING: HTTP {$info['http_code']} from citeserver {$mode} request: " . $response);
return;
}
$response = json_decode($response);
if (!$response) {
error_log("WARNING: Invalid response from citeserver {$mode} request: " . $response);
return;
}
$str = parse_url($info['url']);
$str = parse_str($str['query']);
if ($mode == 'citation') {
$data[$setIDs[$setID]] = Zotero_Cite::processCitationResponse($response);
} else {
if ($mode == "bib") {
$data[$setIDs[$setID]] = Zotero_Cite::processBibliographyResponse($response);
}
}
};
$rc = new RollingCurl($requestCallback);
// Number of simultaneous requests
$rc->window_size = 20;
foreach ($sets as $key => $items) {
$json = self::getJSONFromItems($items);
$server = Z_CONFIG::$CITATION_SERVERS[array_rand(Z_CONFIG::$CITATION_SERVERS)];
$url = "http://{$server}/?responseformat=json&style={$style}";
if ($mode == 'citation') {
$url .= "&citations=1&bibliography=0";
}
// Include array position in URL so that the callback can figure
// out what request this was
$url .= "&setID=" . $key;
// TODO: support multiple items per set, if necessary
if (!$items instanceof Zotero_Item) {
throw new Exception("items is not a Zotero_Item");
}
$setIDs[$key] = $items->libraryID . "/" . $items->key;
$request = new RollingCurlRequest($url);
$request->options = array(CURLOPT_POST => 1, CURLOPT_POSTFIELDS => $json, CURLOPT_HTTPHEADER => array("Expect:"), CURLOPT_CONNECTTIMEOUT => 1, CURLOPT_TIMEOUT => 4, CURLOPT_HEADER => 0, CURLOPT_RETURNTRANSFER => 1);
$rc->add($request);
}
$rc->execute();
error_log(sizeOf($sets) . " {$mode} requests in " . round(microtime(true) - $t, 3));
return $data;
}
/**
* 抓取网页
*
* @param int $window_size curl批处理句柄最大同时连接数
*/
public function fetch_html($window_size = NULL)
{
$rc = new RollingCurl($this->callback);
foreach ($this->urls as $url) {
$request = new RollingCurlRequest($url, 'Get', NULL, NULL, $this->options);
$rc->add($request);
}
if (!empty($window_size)) {
$rc->execute($window_size);
} else {
$rc->execute($this->window_size);
}
}
/**
* Default constructor.
*
* @access public
* @param string $startUrl The starting url for crawling
* @param string $dbUser The database username
* @param string $dbPass The database password
* @param string $dbName The database name
* @param string $dbHost The database host
* @return void
*/
public function __construct($startUrl, $dbUser, $dbPass, $dbName, $dbHost)
{
// register the Rolling Curl callback
parent::__construct('parseHtml');
// validate the url is valid HTTP or HTTPS
if (strpos($startUrl, 'http://') === false && strpos($startUrl, 'https://') === false) {
throw new Exception('Your starting caterpillar URL must begin with http or https.');
}
// store the base url
$this->startUrl = rtrim($startUrl, '/') . '/';
// store the start time
$this->startTime = date('Y-m-d H:i:s');
// store the base domain
$info = parse_url($startUrl);
$this->domain = $info['scheme'] . '://' . $info['host'];
// open mysql connection
$this->db = mysql_connect($dbHost, $dbUser, $dbPass);
if ($this->db === false) {
throw new Exception('The caterpillar database connection could not be established');
}
// use user specified database
if (!mysql_select_db($dbName)) {
throw new Exception('An error occurred attempting to connect to the database ' . $dbName);
}
}
public function execute($window_size = null)
{
if (count($this->requests) == 0) {
return false;
}
return parent::execute($window_size);
}
function updateLinks()
{
global $logger;
$logger->info('UPDATE LINKS');
$resPirate = array();
//parallel RollingCurl
RollingCurl::$rc = new RollingCurl("main_callback");
// the window size determines how many simultaneous requests to allow.
RollingCurl::$rc->window_size = 5;
//List of tracker loaders
$loaders = array();
$rutorMain = RUTORROOT;
$loaders[] = new RutorLoader("{$rutorMain}/browse/0/1/0/2/");
//foreign movies
$loaders[] = new RutorLoader("{$rutorMain}/browse/1/1/0/2/");
$loaders[] = new RutorLoader("{$rutorMain}/browse/2/1/0/2/");
$loaders[] = new RutorLoader("{$rutorMain}/browse/3/1/0/2/");
$loaders[] = new RutorLoader("{$rutorMain}/browse/4/1/0/2/");
$loaders[] = new RutorLoader("{$rutorMain}/browse/5/1/0/2/");
$loaders[] = new RutorLoader("{$rutorMain}/browse/6/1/0/2/");
$loaders[] = new RutorLoader("{$rutorMain}/browse/7/1/0/2/");
$loaders[] = new RutorLoader("{$rutorMain}/browse/8/1/0/2/");
$loaders[] = new RutorLoader("{$rutorMain}/browse/9/1/0/2/");
$loaders[] = new RutorLoader("{$rutorMain}/browse/10/1/0/2/");
$loaders[] = new RutorLoader("{$rutorMain}/browse/11/1/0/2/");
$loaders[] = new RutorLoader("{$rutorMain}/browse/12/1/0/2/");
$loaders[] = new RutorLoader("{$rutorMain}/browse/13/1/0/2/");
$loaders[] = new RutorLoader("{$rutorMain}/browse/0/7/0/2/");
//multiplication
$loaders[] = new RutorLoader("{$rutorMain}/browse/0/5/0/2");
//russian movies
$loaders[] = new RutorLoader("{$rutorMain}/browse/1/5/0/2");
$NNMData = array("prev_sd" => 0, "prev_a" => 0, "prev_my" => 0, "prev_n" => 0, "prev_shc" => 0, "prev_shf" => 1, "prev_sha" => 1, "prev_shs" => 0, "prev_shr" => 0, "prev_sht" => 0, "f[0]" => 270, "o" => 10, "s" => 2, "tm" => -1, "ta" => -1, "sns" => -1, "sds" => -1, "nm" => "", "pn" => "");
$loaders[] = new NNMLoader("http://nnm-club.me/forum/tracker.php", $NNMData);
$NNMData["f[0]"] = 218;
$loaders[] = new NNMLoader("http://nnm-club.me/forum/tracker.php", $NNMData);
$NNMData["f[0]"] = 217;
$NNMData["f[1]"] = 954;
$loaders[] = new NNMLoader("http://nnm-club.me/forum/tracker.php", $NNMData);
$pirateMain = PIRATEROOT;
$loaders[] = new PirateLoader("{$pirateMain}/browse/201/0/7/0");
/*
$resPirate1 = new Pirate;
$resPirate1->getPirateBay("https://pirateproxy.sx/browse/207/0/7");
$resPirate2 = new Pirate;
$resPirate2->getPirateBay("https://pirateproxy.sx/browse/207/1/7");
$resPirate3 = new Pirate;
$resPirate3->getPirateBay("https://pirateproxy.sx/browse/201/0/7");
$resPirate = array_merge($resPirate1->result, $resPirate2->result, $resPirate3->result, $resRutor, $resSeedoff);
flush();
*/
foreach ($loaders as $loader) {
$loader->setLogger($logger);
$loader->load();
}
RollingCurl::$rc->execute();
//result array with torrent infos
$result = array();
foreach ($loaders as $loader) {
$result = array_merge($result, $loader->getResult());
}
$resSeedoff = array();
$resSeedoff = seedoff\getSeedoff();
$resSeedoff = array_merge($resSeedoff, seedoff\getSeedoff("http://www.seedoff.net/index.php?page=ajax&active=0&options=0&recommend=0&sticky=0&period=0&category=14&options=0&order=5&by=2&pages=2"));
$resSeedoff = array_merge($resSeedoff, seedoff\getSeedoff("http://www.seedoff.net/index.php?page=ajax&active=0&options=0&recommend=0&sticky=0&period=0&category=14&options=0&order=5&by=2&pages=3"));
$resSeedoff = array_merge($resSeedoff, seedoff\getSeedoff("http://www.seedoff.net/index.php?page=ajax&active=0&options=0&recommend=0&sticky=0&period=0&category=64&options=0&order=5&by=2&pages=1"));
$result = array_merge($result, $resSeedoff);
foreach ($result as $cur) {
if (trySkip($cur)) {
continue;
}
getIds($cur['title_approx'], $cur);
$logger->info("add link: " . $cur['title_approx'] . "::" . $cur['description'] . "::" . $cur['link']);
$res = addLink($cur);
if ($res !== 0) {
$logger->warning("link was not added: {$res}");
}
usleep(100 * 1000);
}
$logger->info(count($result) . " links updated");
}
*/
// a little example that fetches a bunch of sites in parallel and echos the page title and response info for each request
function request_callback($response, $info)
{
// parse the page title out of the returned HTML
if (preg_match("~<title>(.*?)</title>~i", $response, $out)) {
$title = $out[1];
}
echo "<b>{$title}</b><br />";
print_r($info);
echo "<hr>";
}
require "RollingCurl.php";
// single curl request
$rc = new RollingCurl("request_callback");
$rc->request("http://www.msn.com");
$rc->execute();
// another single curl request
$rc = new RollingCurl("request_callback");
$rc->request("http://www.google.com");
$rc->execute();
echo "<hr>";
// top 20 sites according to alexa (11/5/09)
$urls = array("http://www.google.com", "http://www.facebook.com", "http://www.yahoo.com", "http://www.youtube.com", "http://www.live.com", "http://www.wikipedia.com", "http://www.blogger.com", "http://www.msn.com", "http://www.baidu.com", "http://www.yahoo.co.jp", "http://www.myspace.com", "http://www.qq.com", "http://www.google.co.in", "http://www.twitter.com", "http://www.google.de", "http://www.microsoft.com", "http://www.google.cn", "http://www.sina.com.cn", "http://www.wordpress.com", "http://www.google.co.uk");
$rc = new RollingCurl("request_callback");
$rc->window_size = 20;
foreach ($urls as $url) {
$request = new Request($url);
$rc->add($request);
}
$rc->execute();
/**
* AngryCurl destructor
*
* @return void
*/
function __destruct()
{
self::add_debug_msg("# Finishing ...");
parent::__destruct();
}
if (!isset($fision[1])) {
continue;
}
//$request = new RollingCurlRequest('http://'.preg_replace('@http[s]?://@i','',$ips[$i]));
$request = new RollingCurlRequest('http://' . preg_replace('@http[s]?://@i', '', $fision[1]));
$request->rank = $fision[0];
$request->outputfilehandler = $fh;
if (isset($fision[5]) && $fision[5] !== '' && $fision[5] !== '-1') {
$request->pagerank = $fision[5];
}
$headers = array("Host: " . $fision[1]);
$request->options = array(CURLOPT_HTTPHEADER => $headers);
$rc->add($request);
if (($i + 1) % $window_size === 0) {
$rc->execute();
$rc = new RollingCurl("request_callback");
$rc->window_size = $window_size;
if (($sleeptime = $window_size * 1100 - (($curtime = microtime(true)) - $time)) > 10000) {
usleep($sleeptime);
}
$time = $curtime;
}
}
if (($maxi + 1) % $window_size !== 0) {
$rc->execute();
}
fclose($fh);
echo "rank, url, \"title\", \"keywords\", \"description\", PageRank CSV file of English websites in top Alexa 1M generated in {$file}.\n";
if ($snapshots) {
require_once 'snapshots.php';
}
public function fetchAllOnce(array $urls, $isRedirect = false)
{
if (!$isRedirect) {
$urls = array_unique($urls);
}
if (empty($urls)) {
return;
}
//////////////////////////////////////////////////////
// parallel (HttpRequestPool)
if ($this->method == self::METHOD_REQUEST_POOL) {
$this->debug('Starting parallel fetch (HttpRequestPool)');
try {
while (count($urls) > 0) {
$this->debug('Processing set of ' . min($this->maxParallelRequests, count($urls)));
$subset = array_splice($urls, 0, $this->maxParallelRequests);
$pool = new HttpRequestPool();
foreach ($subset as $orig => $url) {
if (!$isRedirect) {
$orig = $url;
}
unset($this->redirectQueue[$orig]);
$this->debug("...{$url}");
if (!$isRedirect && isset($this->requests[$url])) {
$this->debug("......in memory");
/*
} elseif ($this->isCached($url)) {
$this->debug("......is cached");
if (!$this->minimiseMemoryUse) {
$this->requests[$url] = $this->getCached($url);
}
*/
} else {
$this->debug("......adding to pool");
$req_url = $this->rewriteHashbangFragment ? $this->rewriteHashbangFragment($url) : $url;
$req_url = $this->removeFragment($req_url);
$httpRequest = new HttpRequest($req_url, HttpRequest::METH_GET, $this->requestOptions);
// send cookies, if we have any
if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) {
$this->debug("......sending cookies: {$cookies}");
$httpRequest->addHeaders(array('Cookie' => $cookies));
}
$this->requests[$orig] = array('headers' => null, 'body' => null, 'httpRequest' => $httpRequest);
$this->requests[$orig]['original_url'] = $orig;
$pool->attach($httpRequest);
}
}
// did we get anything into the pool?
if (count($pool) > 0) {
$this->debug('Sending request...');
try {
$pool->send();
} catch (HttpRequestPoolException $e) {
// do nothing
}
$this->debug('Received responses');
foreach ($subset as $orig => $url) {
if (!$isRedirect) {
$orig = $url;
}
//if (!isset($this->requests[$url]['fromCache'])) {
$request = $this->requests[$orig]['httpRequest'];
//$this->requests[$orig]['headers'] = $this->headersToString($request->getResponseHeader());
// getResponseHeader() doesn't return status line, so, for consistency...
$this->requests[$orig]['headers'] = substr($request->getRawResponseMessage(), 0, $request->getResponseInfo('header_size'));
$this->requests[$orig]['body'] = $request->getResponseBody();
$this->requests[$orig]['effective_url'] = $request->getResponseInfo('effective_url');
$this->requests[$orig]['status_code'] = $status_code = $request->getResponseCode();
// is redirect?
if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && $request->getResponseHeader('location')) {
$redirectURL = $request->getResponseHeader('location');
if (!preg_match('!^https?://!i', $redirectURL)) {
$redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url);
}
if ($this->validateURL($redirectURL)) {
$this->debug('Redirect detected. Valid URL: ' . $redirectURL);
// store any cookies
$cookies = $request->getResponseHeader('set-cookie');
if ($cookies && !is_array($cookies)) {
$cookies = array($cookies);
}
if ($cookies) {
$this->cookieJar->storeCookies($url, $cookies);
}
$this->redirectQueue[$orig] = $redirectURL;
} else {
$this->debug('Redirect detected. Invalid URL: ' . $redirectURL);
}
}
//die($url.' -multi- '.$request->getResponseInfo('effective_url'));
$pool->detach($request);
unset($this->requests[$orig]['httpRequest'], $request);
/*
if ($this->minimiseMemoryUse) {
if ($this->cache($url)) {
unset($this->requests[$url]);
}
}
*/
//}
}
}
}
} catch (HttpException $e) {
$this->debug($e);
return false;
}
} elseif ($this->method == self::METHOD_CURL_MULTI) {
$this->debug('Starting parallel fetch (curl_multi_*)');
while (count($urls) > 0) {
$this->debug('Processing set of ' . min($this->maxParallelRequests, count($urls)));
$subset = array_splice($urls, 0, $this->maxParallelRequests);
$pool = new RollingCurl(array($this, 'handleCurlResponse'));
$pool->window_size = count($subset);
foreach ($subset as $orig => $url) {
if (!$isRedirect) {
$orig = $url;
}
unset($this->redirectQueue[$orig]);
$this->debug("...{$url}");
if (!$isRedirect && isset($this->requests[$url])) {
$this->debug("......in memory");
/*
} elseif ($this->isCached($url)) {
$this->debug("......is cached");
if (!$this->minimiseMemoryUse) {
$this->requests[$url] = $this->getCached($url);
}
*/
} else {
$this->debug("......adding to pool");
$req_url = $this->rewriteHashbangFragment ? $this->rewriteHashbangFragment($url) : $url;
$req_url = $this->removeFragment($req_url);
$headers = array();
// send cookies, if we have any
if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) {
$this->debug("......sending cookies: {$cookies}");
$headers[] = 'Cookie: ' . $cookies;
}
$httpRequest = new RollingCurlRequest($req_url, 'GET', null, $headers, array(CURLOPT_CONNECTTIMEOUT => $this->requestOptions['timeout'], CURLOPT_TIMEOUT => $this->requestOptions['timeout']));
$httpRequest->set_original_url($orig);
$this->requests[$orig] = array('headers' => null, 'body' => null, 'httpRequest' => $httpRequest);
$this->requests[$orig]['original_url'] = $orig;
// TODO: is this needed anymore?
$pool->add($httpRequest);
}
}
// did we get anything into the pool?
if (count($pool) > 0) {
$this->debug('Sending request...');
$pool->execute();
// this will call handleCurlResponse() and populate $this->requests[$orig]
$this->debug('Received responses');
foreach ($subset as $orig => $url) {
if (!$isRedirect) {
$orig = $url;
}
// $this->requests[$orig]['headers']
// $this->requests[$orig]['body']
// $this->requests[$orig]['effective_url']
$status_code = $this->requests[$orig]['status_code'];
if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && isset($this->requests[$orig]['location'])) {
$redirectURL = $this->requests[$orig]['location'];
if (!preg_match('!^https?://!i', $redirectURL)) {
$redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url);
}
if ($this->validateURL($redirectURL)) {
$this->debug('Redirect detected. Valid URL: ' . $redirectURL);
// store any cookies
$cookies = $this->cookieJar->extractCookies($this->requests[$orig]['headers']);
if (!empty($cookies)) {
$this->cookieJar->storeCookies($url, $cookies);
}
$this->redirectQueue[$orig] = $redirectURL;
} else {
$this->debug('Redirect detected. Invalid URL: ' . $redirectURL);
}
}
// die($url.' -multi- '.$request->getResponseInfo('effective_url'));
unset($this->requests[$orig]['httpRequest']);
}
}
}
} else {
$this->debug('Starting sequential fetch (file_get_contents)');
$this->debug('Processing set of ' . count($urls));
foreach ($urls as $orig => $url) {
if (!$isRedirect) {
$orig = $url;
}
unset($this->redirectQueue[$orig]);
$this->debug("...{$url}");
if (!$isRedirect && isset($this->requests[$url])) {
$this->debug("......in memory");
/*
} elseif ($this->isCached($url)) {
$this->debug("......is cached");
if (!$this->minimiseMemoryUse) {
$this->requests[$url] = $this->getCached($url);
}
*/
} else {
$this->debug("Sending request for {$url}");
$this->requests[$orig]['original_url'] = $orig;
$req_url = $this->rewriteHashbangFragment ? $this->rewriteHashbangFragment($url) : $url;
$req_url = $this->removeFragment($req_url);
// send cookies, if we have any
$httpContext = $this->httpContext;
if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) {
$this->debug("......sending cookies: {$cookies}");
$httpContext['http']['header'] .= 'Cookie: ' . $cookies . "\r\n";
}
if (false !== ($html = @file_get_contents($req_url, false, stream_context_create($httpContext)))) {
$this->debug('Received response');
// get status code
if (!isset($http_response_header[0]) || !preg_match('!^HTTP/\\d+\\.\\d+\\s+(\\d+)!', trim($http_response_header[0]), $match)) {
$this->debug('Error: no status code found');
// TODO: handle error - no status code
} else {
$this->requests[$orig]['headers'] = $this->headersToString($http_response_header, false);
$this->requests[$orig]['body'] = $html;
$this->requests[$orig]['effective_url'] = $req_url;
$this->requests[$orig]['status_code'] = $status_code = (int) $match[1];
unset($match);
// handle redirect
if (preg_match('/^Location:(.*?)$/m', $this->requests[$orig]['headers'], $match)) {
$this->requests[$orig]['location'] = trim($match[1]);
}
if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && isset($this->requests[$orig]['location'])) {
$redirectURL = $this->requests[$orig]['location'];
if (!preg_match('!^https?://!i', $redirectURL)) {
$redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url);
}
if ($this->validateURL($redirectURL)) {
$this->debug('Redirect detected. Valid URL: ' . $redirectURL);
// store any cookies
$cookies = $this->cookieJar->extractCookies($this->requests[$orig]['headers']);
if (!empty($cookies)) {
$this->cookieJar->storeCookies($url, $cookies);
}
$this->redirectQueue[$orig] = $redirectURL;
} else {
$this->debug('Redirect detected. Invalid URL: ' . $redirectURL);
}
}
}
} else {
$this->debug('Error retrieving URL');
//print_r($req_url);
//print_r($http_response_header);
//print_r($html);
// TODO: handle error - failed to retrieve URL
}
}
}
}
}
if ($problem) {
echo "\nTHERE MAY BE A PROBLEM WITH THIS RESPONSE!\n";
echo $problem;
$hasproblem = true;
}
}
$results = array();
$verify = new RollingCurl(function ($response, $info, $request) use(&$results) {
$png = file_get_contents(randpng($request->index));
if ($response == $png) {
$results[$request->index] = 'pass';
} elseif (strpos($response, 'Integrity constraint violation') !== false) {
$result = "retry\n";
} else {
$result = "fail\n";
if ($response[0] == '<') {
$result .= "\t\tReceived: " . $response . "...\n";
} else {
$result .= "\t\tReceived: " . base64_encode(substr($response, 0, 50)) . "...\n";
}
$result .= "\t\tSent: " . base64_encode(substr($png, 0, 50)) . "...\n";
$results[$request->index] = $result;
}
});
for ($x = 1; $x <= $files_appearing; $x++) {
$results[$x] = 'unknown';
$request = new RollingCurlRequest('http://' . $username . ':' . $password . '@' . $owncloud_remote . '/webdav' . $testdir . '/sample_' . $x . '.png', 'GET');
$request->index = $x;
$verify->add($request);
}
$verify->execute($window);
public function sendRequestCallBack($response, $info = '')
{
// 处理内容
$response = preg_replace("/<code[\\s\\S]*><!--/iU", "", $response);
$response = preg_replace("/--><\\/code>/iU", "", $response);
// 分析html,获取添加好友需要的数据
\phpQuery::newDocumentHTML($response);
$pagelet_timeline_main_column = pq("#pagelet_timeline_main_column")->attr("data-gt");
$pagelet_timeline_main_column = json_decode($pagelet_timeline_main_column);
$profile_owner = $pagelet_timeline_main_column->profile_owner;
$requests = array();
foreach (pq("div.fsl.fwb.fcb > a") as $value) {
$data_hovercard = pq($value)->attr("data-hovercard");
$data_gt = pq($value)->attr("data-gt");
$data_gt = json_decode($data_gt);
preg_match("/\\?id=([0-9]*)&/iU", $data_hovercard, $matches);
$to_friend = $matches[1];
// 发送好友请求
$query = array("to_friend" => $to_friend, "action" => "add_friend", "how_found" => "profile_friends", "ref_param" => "pb_friends_tl", "link_data[gt][coeff2_registry_key]" => $data_gt->coeff2_registry_key, "link_data[gt][coeff2_info]" => $data_gt->coeff2_info, "link_data[gt][coeff2_action]" => $data_gt->coeff2_action, "link_data[gt][coeff2_pv_signature]" => $data_gt->coeff2_pv_signature, "link_data[gt][profile_owner]" => $profile_owner, "link_data[gt][ref]" => "timeline:timeline", "outgoing_id" => '', "logging_location" => '', "no_flyout_on_click" => "true", "ego_log_data" => '', "http_referer" => '', "floc" => "friends_tab", "__user" => $this->user_id, "__a" => "1", "fb_dtsg" => $this->token, "__rev" => $this->version);
$capt_opts = $this->curl_opts;
$url = "https://www.facebook.com/ajax/add_friend/action.php?__pc=EXP1%3ADEFAULT";
$capt_opts[CURLOPT_POST] = true;
$capt_opts[CURLOPT_POSTFIELDS] = $query;
$request = new \RollingCurlRequest($url);
$request->options = $capt_opts;
$requests[] = $request;
}
if (empty($requests)) {
return;
}
$rc = new \RollingCurl();
if (sizeof($requests) < 20) {
$rc->window_size = sizeof($requests);
} else {
$rc->window_size = 20;
}
foreach ($requests as $value) {
$rc->add($value);
}
$rc->execute();
}
public static function curl_multi_get($requests)
{
cache::$__LinkToFunc = array();
cache::$__TempResult = array();
cache::$__ApiTime = microtime(true);
$rc = new RollingCurl(function ($response, $info, $request) {
$url = $request->url;
if ($response == null) {
cache::log("CURL RESULT EMPTY for url: " . $url . " with HTTP code " . $info['http_code']);
/*$response = file_get_contents($url);
if( $response == false )
{
cache::log("FILE_GET_CONTENTS failed for url: ".$url);
}*/
}
cache::log(sprintf("Got multi result for %s in %s sec, len %s", $url, number_format(microtime(true) - cache::$__ApiTime, 3), strlen($response)));
$cbtime = microtime(true);
if (cache::$__LinkToFunc[$url]) {
call_user_func_array(cache::$__LinkToFunc[$url], array($response));
}
cache::log(sprintf("Ran callback for URL in %s sec", number_format(microtime(true) - $cbtime, 3)));
cache::$__TempResult[$url] = $response;
cache::put($url, $response);
});
foreach ($requests as $request) {
cache::$__LinkToFunc[$request['url']] = $request['func'];
$rc->request($request['url']);
}
$rc->window_size = count($requests);
$rc->execute();
cache::log(sprintf("Multi API request took %s sec", number_format(microtime(true) - cache::$__ApiTime, 3)));
return cache::$__TempResult;
}
function __destruct()
{
parent::__destruct();
}
// 'dol.gov',
// 'humanresources.about.com',
// 'shrm.org',
// 'diversityinc.com',
// 'stevepavlina.com/blog/',
// 'osha.gov',
// 'hr.com',
// 'ere.net',
// 'cisin.com',
// 'blr.com',
// 'peopleadmin.com',
// 'wageworks.com',
// 'dalecarnegie.com',
// 'doleta.gov',
// 'mercer.com',
// 'astd.org',
// 'brightscope.com',
// 'tmp.com',
// 'trinet.com',
//];
// DETECT CYCLE!
$loader = new FileLoader();
$scanner = new Scanner($loader, __DIR__ . '/apps.json');
$category = new Category($loader, __DIR__ . '/apps.json');
require_once __DIR__ . '/lib/RollingCurl.php';
$rc = new RollingCurl();
foreach ($urls as $url) {
$rc->add(new RollingCurlRequest("http://scanner.loc/worker.php?url=" . urlencode($url)));
}
$rc->execute(10);
echo "All request sent";
