public function request(array $params)
{
$request = new \HttpRequest($this->getTransmissionURL(), "POST");
$request->addBody(json_encode($params));
$this->client->attach($request);
return $this->client->send();
}
public function addFeeds($urls)
{
$pool = new HttpRequestPool();
foreach ($urls as $url) {
$pool->attach($r = $this->setupRequest($url));
}
$pool->send();
foreach ($pool as $request) {
$this->handleResponse($request);
}
}
protected final function socketPerform()
{
$result = @parent::socketPerform();
foreach ($this->getFinishedRequests() as $r) {
$this->detach($r);
// handle response of finished request
}
return $result;
}
public function __construct()
{
parent::__construct(new HttpRequest('http://pear.php.net', HTTP_METH_HEAD), new HttpRequest('http://pecl.php.net', HTTP_METH_HEAD));
// HttpRequestPool methods socketPerform() and socketSelect() are
// protected; one could use this approach to do something else
// while the requests are being executed
print "Executing requests";
for ($i = 0; $this->socketPerform(); $i++) {
$i % 10 or print ".";
if (!$this->socketSelect()) {
throw new HttpException("Socket error!");
}
}
print "\nDone!\n";
}
/**
* Sends a request pool.
*
* @param \ArrayObject $requestList Request list
* @return \HttpRequestPool
* @throws \Jyxo\Webdav\Exception On error
*/
private function sendPool(\ArrayObject $requestList)
{
try {
// Clean the pool
$this->pool->reset();
// Attach requests
foreach ($requestList as $request) {
$this->pool->attach($request);
}
// Send
$this->pool->send();
return $this->pool;
} catch (\HttpException $e) {
// Find the innermost exception
$inner = $e;
while (null !== $inner->innerException) {
$inner = $inner->innerException;
}
throw new Exception($inner->getMessage(), 0, $inner);
}
}
<?php
try {
$p = new HttpRequestPool();
// if you want to set _any_ options of the HttpRequest object,
// you need to do so *prior attaching* to the request pool!
$p->attach(new HttpRequest('http://pear.php.net', HTTP_METH_HEAD));
$p->attach(new HttpRequest('http://pecl.php.net', HTTP_METH_HEAD));
} catch (HttpException $e) {
print $e;
exit;
}
try {
$p->send();
// HttpRequestPool implements an iterator over attached HttpRequest objects
foreach ($p as $r) {
echo "Checking ", $r->getUrl(), " reported ", $r->getResponseCode(), "\n";
}
} catch (HttpException $e) {
print $e;
}
#!/usr/bin/env php
<?php
try {
$pool = new HttpRequestPool(new HttpRequest('http://www.google.com/', HttpRequest::METH_HEAD), new HttpRequest('http://www.php.net/', HttpRequest::METH_HEAD));
$pool->send();
foreach ($pool as $request) {
printf("%s is %s (%d)\n", $request->getUrl(), $request->getResponseCode() ? 'alive' : 'not alive', $request->getResponseCode());
}
} catch (HttpException $e) {
echo $e;
}
public function fetchAllOnce(array $urls, $isRedirect = false)
{
if (!$isRedirect) {
$urls = array_unique($urls);
}
if (empty($urls)) {
return;
}
//////////////////////////////////////////////////////
// parallel (HttpRequestPool)
if ($this->method == self::METHOD_REQUEST_POOL) {
$this->debug('Starting parallel fetch (HttpRequestPool)');
try {
while (count($urls) > 0) {
$this->debug('Processing set of ' . min($this->maxParallelRequests, count($urls)));
$subset = array_splice($urls, 0, $this->maxParallelRequests);
$pool = new HttpRequestPool();
foreach ($subset as $orig => $url) {
if (!$isRedirect) {
$orig = $url;
}
unset($this->redirectQueue[$orig]);
$this->debug("...{$url}");
if (!$isRedirect && isset($this->requests[$url])) {
$this->debug("......in memory");
/*
} elseif ($this->isCached($url)) {
$this->debug("......is cached");
if (!$this->minimiseMemoryUse) {
$this->requests[$url] = $this->getCached($url);
}
*/
} else {
$this->debug("......adding to pool");
$req_url = $this->rewriteHashbangFragment ? $this->rewriteHashbangFragment($url) : $url;
$req_url = $this->removeFragment($req_url);
$httpRequest = new HttpRequest($req_url, HttpRequest::METH_GET, $this->requestOptions);
// send cookies, if we have any
if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) {
$this->debug("......sending cookies: {$cookies}");
$httpRequest->addHeaders(array('Cookie' => $cookies));
}
$this->requests[$orig] = array('headers' => null, 'body' => null, 'httpRequest' => $httpRequest);
$this->requests[$orig]['original_url'] = $orig;
$pool->attach($httpRequest);
}
}
// did we get anything into the pool?
if (count($pool) > 0) {
$this->debug('Sending request...');
try {
$pool->send();
} catch (HttpRequestPoolException $e) {
// do nothing
}
$this->debug('Received responses');
foreach ($subset as $orig => $url) {
if (!$isRedirect) {
$orig = $url;
}
//if (!isset($this->requests[$url]['fromCache'])) {
$request = $this->requests[$orig]['httpRequest'];
//$this->requests[$orig]['headers'] = $this->headersToString($request->getResponseHeader());
// getResponseHeader() doesn't return status line, so, for consistency...
$this->requests[$orig]['headers'] = substr($request->getRawResponseMessage(), 0, $request->getResponseInfo('header_size'));
$this->requests[$orig]['body'] = $request->getResponseBody();
$this->requests[$orig]['effective_url'] = $request->getResponseInfo('effective_url');
$this->requests[$orig]['status_code'] = $status_code = $request->getResponseCode();
// is redirect?
if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && $request->getResponseHeader('location')) {
$redirectURL = $request->getResponseHeader('location');
if (!preg_match('!^https?://!i', $redirectURL)) {
$redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url);
}
if ($this->validateURL($redirectURL)) {
$this->debug('Redirect detected. Valid URL: ' . $redirectURL);
// store any cookies
$cookies = $request->getResponseHeader('set-cookie');
if ($cookies && !is_array($cookies)) {
$cookies = array($cookies);
}
if ($cookies) {
$this->cookieJar->storeCookies($url, $cookies);
}
$this->redirectQueue[$orig] = $redirectURL;
} else {
$this->debug('Redirect detected. Invalid URL: ' . $redirectURL);
}
}
//die($url.' -multi- '.$request->getResponseInfo('effective_url'));
$pool->detach($request);
unset($this->requests[$orig]['httpRequest'], $request);
/*
if ($this->minimiseMemoryUse) {
if ($this->cache($url)) {
unset($this->requests[$url]);
}
}
*/
//}
}
}
}
} catch (HttpException $e) {
$this->debug($e);
return false;
}
} elseif ($this->method == self::METHOD_CURL_MULTI) {
$this->debug('Starting parallel fetch (curl_multi_*)');
while (count($urls) > 0) {
$this->debug('Processing set of ' . min($this->maxParallelRequests, count($urls)));
$subset = array_splice($urls, 0, $this->maxParallelRequests);
$pool = new RollingCurl(array($this, 'handleCurlResponse'));
$pool->window_size = count($subset);
foreach ($subset as $orig => $url) {
if (!$isRedirect) {
$orig = $url;
}
unset($this->redirectQueue[$orig]);
$this->debug("...{$url}");
if (!$isRedirect && isset($this->requests[$url])) {
$this->debug("......in memory");
/*
} elseif ($this->isCached($url)) {
$this->debug("......is cached");
if (!$this->minimiseMemoryUse) {
$this->requests[$url] = $this->getCached($url);
}
*/
} else {
$this->debug("......adding to pool");
$req_url = $this->rewriteHashbangFragment ? $this->rewriteHashbangFragment($url) : $url;
$req_url = $this->removeFragment($req_url);
$headers = array();
// send cookies, if we have any
if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) {
$this->debug("......sending cookies: {$cookies}");
$headers[] = 'Cookie: ' . $cookies;
}
$httpRequest = new RollingCurlRequest($req_url, 'GET', null, $headers, array(CURLOPT_CONNECTTIMEOUT => $this->requestOptions['timeout'], CURLOPT_TIMEOUT => $this->requestOptions['timeout']));
$httpRequest->set_original_url($orig);
$this->requests[$orig] = array('headers' => null, 'body' => null, 'httpRequest' => $httpRequest);
$this->requests[$orig]['original_url'] = $orig;
// TODO: is this needed anymore?
$pool->add($httpRequest);
}
}
// did we get anything into the pool?
if (count($pool) > 0) {
$this->debug('Sending request...');
$pool->execute();
// this will call handleCurlResponse() and populate $this->requests[$orig]
$this->debug('Received responses');
foreach ($subset as $orig => $url) {
if (!$isRedirect) {
$orig = $url;
}
// $this->requests[$orig]['headers']
// $this->requests[$orig]['body']
// $this->requests[$orig]['effective_url']
$status_code = $this->requests[$orig]['status_code'];
if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && isset($this->requests[$orig]['location'])) {
$redirectURL = $this->requests[$orig]['location'];
if (!preg_match('!^https?://!i', $redirectURL)) {
$redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url);
}
if ($this->validateURL($redirectURL)) {
$this->debug('Redirect detected. Valid URL: ' . $redirectURL);
// store any cookies
$cookies = $this->cookieJar->extractCookies($this->requests[$orig]['headers']);
if (!empty($cookies)) {
$this->cookieJar->storeCookies($url, $cookies);
}
$this->redirectQueue[$orig] = $redirectURL;
} else {
$this->debug('Redirect detected. Invalid URL: ' . $redirectURL);
}
}
// die($url.' -multi- '.$request->getResponseInfo('effective_url'));
unset($this->requests[$orig]['httpRequest']);
}
}
}
} else {
$this->debug('Starting sequential fetch (file_get_contents)');
$this->debug('Processing set of ' . count($urls));
foreach ($urls as $orig => $url) {
if (!$isRedirect) {
$orig = $url;
}
unset($this->redirectQueue[$orig]);
$this->debug("...{$url}");
if (!$isRedirect && isset($this->requests[$url])) {
$this->debug("......in memory");
/*
} elseif ($this->isCached($url)) {
$this->debug("......is cached");
if (!$this->minimiseMemoryUse) {
$this->requests[$url] = $this->getCached($url);
}
*/
} else {
$this->debug("Sending request for {$url}");
$this->requests[$orig]['original_url'] = $orig;
$req_url = $this->rewriteHashbangFragment ? $this->rewriteHashbangFragment($url) : $url;
$req_url = $this->removeFragment($req_url);
// send cookies, if we have any
$httpContext = $this->httpContext;
if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) {
$this->debug("......sending cookies: {$cookies}");
$httpContext['http']['header'] .= 'Cookie: ' . $cookies . "\r\n";
}
if (false !== ($html = @file_get_contents($req_url, false, stream_context_create($httpContext)))) {
$this->debug('Received response');
// get status code
if (!isset($http_response_header[0]) || !preg_match('!^HTTP/\\d+\\.\\d+\\s+(\\d+)!', trim($http_response_header[0]), $match)) {
$this->debug('Error: no status code found');
// TODO: handle error - no status code
} else {
$this->requests[$orig]['headers'] = $this->headersToString($http_response_header, false);
$this->requests[$orig]['body'] = $html;
$this->requests[$orig]['effective_url'] = $req_url;
$this->requests[$orig]['status_code'] = $status_code = (int) $match[1];
unset($match);
// handle redirect
if (preg_match('/^Location:(.*?)$/m', $this->requests[$orig]['headers'], $match)) {
$this->requests[$orig]['location'] = trim($match[1]);
}
if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && isset($this->requests[$orig]['location'])) {
$redirectURL = $this->requests[$orig]['location'];
if (!preg_match('!^https?://!i', $redirectURL)) {
$redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url);
}
if ($this->validateURL($redirectURL)) {
$this->debug('Redirect detected. Valid URL: ' . $redirectURL);
// store any cookies
$cookies = $this->cookieJar->extractCookies($this->requests[$orig]['headers']);
if (!empty($cookies)) {
$this->cookieJar->storeCookies($url, $cookies);
}
$this->redirectQueue[$orig] = $redirectURL;
} else {
$this->debug('Redirect detected. Invalid URL: ' . $redirectURL);
}
}
}
} else {
$this->debug('Error retrieving URL');
//print_r($req_url);
//print_r($http_response_header);
//print_r($html);
// TODO: handle error - failed to retrieve URL
}
}
}
}
}
/**
* Overrides HttpRequestPool::socketPerform()
*
* @return bool
*/
protected function socketPerform()
{
$rs = parent::socketPerform();
foreach ($this->getFinishedRequests() as $r) {
$this->detach($r);
if (206 != ($rc = $r->getResponseCode())) {
throw new HttpException("Unexpected response code: {$rc}");
}
file_put_contents(sprintf("%s.%04d", $this->tmp, $r->id), $r->getResponseBody());
if (-1 != ($offset = $this->getRangeOffset())) {
$this->attachNew($offset);
}
}
return $rs;
}
/**
* Test HttpRequestPool.
*/
function test_HttpRequestPool()
{
$http = new HttpRequestPool();
$http->send();
}
$request->setOptions(['redirect' => 3]);
$requests[] = $request;
} catch (\HttpException $e) {
unset($config['urls'][$key]);
}
}
if (sizeof($requests) == 0) {
echo "Unable to create HTTP client" . PHP_EOL;
exit;
}
// We know we have to iterate at least once, so use do... while
do {
// Get the current batch of requests
$threads = array_splice($requests, 0, $config['threads']);
// Create a new HttpRequestPool and attach the requests
$pool = new \HttpRequestPool();
foreach ($threads as $request) {
$pool->attach($request);
}
// Send the requests and wait till they are all complete
try {
$pool->send();
while (sizeof($pool->getFinishedRequests()) < sizeof($threads)) {
}
foreach ($pool as $request) {
// Display progress
if ($request->getResponseCode() == 200) {
echo '.';
} else {
unset($config['urls'][$request->key]);
echo '!';
