public function request(array $params) { $request = new \HttpRequest($this->getTransmissionURL(), "POST"); $request->addBody(json_encode($params)); $this->client->attach($request); return $this->client->send(); }
public function addFeeds($urls) { $pool = new HttpRequestPool(); foreach ($urls as $url) { $pool->attach($r = $this->setupRequest($url)); } $pool->send(); foreach ($pool as $request) { $this->handleResponse($request); } }
protected final function socketPerform() { $result = @parent::socketPerform(); foreach ($this->getFinishedRequests() as $r) { $this->detach($r); // handle response of finished request } return $result; }
public function __construct() { parent::__construct(new HttpRequest('http://pear.php.net', HTTP_METH_HEAD), new HttpRequest('http://pecl.php.net', HTTP_METH_HEAD)); // HttpRequestPool methods socketPerform() and socketSelect() are // protected; one could use this approach to do something else // while the requests are being executed print "Executing requests"; for ($i = 0; $this->socketPerform(); $i++) { $i % 10 or print "."; if (!$this->socketSelect()) { throw new HttpException("Socket error!"); } } print "\nDone!\n"; }
/** * Sends a request pool. * * @param \ArrayObject $requestList Request list * @return \HttpRequestPool * @throws \Jyxo\Webdav\Exception On error */ private function sendPool(\ArrayObject $requestList) { try { // Clean the pool $this->pool->reset(); // Attach requests foreach ($requestList as $request) { $this->pool->attach($request); } // Send $this->pool->send(); return $this->pool; } catch (\HttpException $e) { // Find the innermost exception $inner = $e; while (null !== $inner->innerException) { $inner = $inner->innerException; } throw new Exception($inner->getMessage(), 0, $inner); } }
<?php try { $p = new HttpRequestPool(); // if you want to set _any_ options of the HttpRequest object, // you need to do so *prior attaching* to the request pool! $p->attach(new HttpRequest('http://pear.php.net', HTTP_METH_HEAD)); $p->attach(new HttpRequest('http://pecl.php.net', HTTP_METH_HEAD)); } catch (HttpException $e) { print $e; exit; } try { $p->send(); // HttpRequestPool implements an iterator over attached HttpRequest objects foreach ($p as $r) { echo "Checking ", $r->getUrl(), " reported ", $r->getResponseCode(), "\n"; } } catch (HttpException $e) { print $e; }
#!/usr/bin/env php <?php try { $pool = new HttpRequestPool(new HttpRequest('http://www.google.com/', HttpRequest::METH_HEAD), new HttpRequest('http://www.php.net/', HttpRequest::METH_HEAD)); $pool->send(); foreach ($pool as $request) { printf("%s is %s (%d)\n", $request->getUrl(), $request->getResponseCode() ? 'alive' : 'not alive', $request->getResponseCode()); } } catch (HttpException $e) { echo $e; }
public function fetchAllOnce(array $urls, $isRedirect = false) { if (!$isRedirect) { $urls = array_unique($urls); } if (empty($urls)) { return; } ////////////////////////////////////////////////////// // parallel (HttpRequestPool) if ($this->method == self::METHOD_REQUEST_POOL) { $this->debug('Starting parallel fetch (HttpRequestPool)'); try { while (count($urls) > 0) { $this->debug('Processing set of ' . min($this->maxParallelRequests, count($urls))); $subset = array_splice($urls, 0, $this->maxParallelRequests); $pool = new HttpRequestPool(); foreach ($subset as $orig => $url) { if (!$isRedirect) { $orig = $url; } unset($this->redirectQueue[$orig]); $this->debug("...{$url}"); if (!$isRedirect && isset($this->requests[$url])) { $this->debug("......in memory"); /* } elseif ($this->isCached($url)) { $this->debug("......is cached"); if (!$this->minimiseMemoryUse) { $this->requests[$url] = $this->getCached($url); } */ } else { $this->debug("......adding to pool"); $req_url = $this->rewriteHashbangFragment ? $this->rewriteHashbangFragment($url) : $url; $req_url = $this->removeFragment($req_url); $httpRequest = new HttpRequest($req_url, HttpRequest::METH_GET, $this->requestOptions); // send cookies, if we have any if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) { $this->debug("......sending cookies: {$cookies}"); $httpRequest->addHeaders(array('Cookie' => $cookies)); } $this->requests[$orig] = array('headers' => null, 'body' => null, 'httpRequest' => $httpRequest); $this->requests[$orig]['original_url'] = $orig; $pool->attach($httpRequest); } } // did we get anything into the pool? if (count($pool) > 0) { $this->debug('Sending request...'); try { $pool->send(); } catch (HttpRequestPoolException $e) { // do nothing } $this->debug('Received responses'); foreach ($subset as $orig => $url) { if (!$isRedirect) { $orig = $url; } //if (!isset($this->requests[$url]['fromCache'])) { $request = $this->requests[$orig]['httpRequest']; //$this->requests[$orig]['headers'] = $this->headersToString($request->getResponseHeader()); // getResponseHeader() doesn't return status line, so, for consistency... $this->requests[$orig]['headers'] = substr($request->getRawResponseMessage(), 0, $request->getResponseInfo('header_size')); $this->requests[$orig]['body'] = $request->getResponseBody(); $this->requests[$orig]['effective_url'] = $request->getResponseInfo('effective_url'); $this->requests[$orig]['status_code'] = $status_code = $request->getResponseCode(); // is redirect? if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && $request->getResponseHeader('location')) { $redirectURL = $request->getResponseHeader('location'); if (!preg_match('!^https?://!i', $redirectURL)) { $redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url); } if ($this->validateURL($redirectURL)) { $this->debug('Redirect detected. Valid URL: ' . $redirectURL); // store any cookies $cookies = $request->getResponseHeader('set-cookie'); if ($cookies && !is_array($cookies)) { $cookies = array($cookies); } if ($cookies) { $this->cookieJar->storeCookies($url, $cookies); } $this->redirectQueue[$orig] = $redirectURL; } else { $this->debug('Redirect detected. Invalid URL: ' . $redirectURL); } } //die($url.' -multi- '.$request->getResponseInfo('effective_url')); $pool->detach($request); unset($this->requests[$orig]['httpRequest'], $request); /* if ($this->minimiseMemoryUse) { if ($this->cache($url)) { unset($this->requests[$url]); } } */ //} } } } } catch (HttpException $e) { $this->debug($e); return false; } } elseif ($this->method == self::METHOD_CURL_MULTI) { $this->debug('Starting parallel fetch (curl_multi_*)'); while (count($urls) > 0) { $this->debug('Processing set of ' . min($this->maxParallelRequests, count($urls))); $subset = array_splice($urls, 0, $this->maxParallelRequests); $pool = new RollingCurl(array($this, 'handleCurlResponse')); $pool->window_size = count($subset); foreach ($subset as $orig => $url) { if (!$isRedirect) { $orig = $url; } unset($this->redirectQueue[$orig]); $this->debug("...{$url}"); if (!$isRedirect && isset($this->requests[$url])) { $this->debug("......in memory"); /* } elseif ($this->isCached($url)) { $this->debug("......is cached"); if (!$this->minimiseMemoryUse) { $this->requests[$url] = $this->getCached($url); } */ } else { $this->debug("......adding to pool"); $req_url = $this->rewriteHashbangFragment ? $this->rewriteHashbangFragment($url) : $url; $req_url = $this->removeFragment($req_url); $headers = array(); // send cookies, if we have any if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) { $this->debug("......sending cookies: {$cookies}"); $headers[] = 'Cookie: ' . $cookies; } $httpRequest = new RollingCurlRequest($req_url, 'GET', null, $headers, array(CURLOPT_CONNECTTIMEOUT => $this->requestOptions['timeout'], CURLOPT_TIMEOUT => $this->requestOptions['timeout'])); $httpRequest->set_original_url($orig); $this->requests[$orig] = array('headers' => null, 'body' => null, 'httpRequest' => $httpRequest); $this->requests[$orig]['original_url'] = $orig; // TODO: is this needed anymore? $pool->add($httpRequest); } } // did we get anything into the pool? if (count($pool) > 0) { $this->debug('Sending request...'); $pool->execute(); // this will call handleCurlResponse() and populate $this->requests[$orig] $this->debug('Received responses'); foreach ($subset as $orig => $url) { if (!$isRedirect) { $orig = $url; } // $this->requests[$orig]['headers'] // $this->requests[$orig]['body'] // $this->requests[$orig]['effective_url'] $status_code = $this->requests[$orig]['status_code']; if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && isset($this->requests[$orig]['location'])) { $redirectURL = $this->requests[$orig]['location']; if (!preg_match('!^https?://!i', $redirectURL)) { $redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url); } if ($this->validateURL($redirectURL)) { $this->debug('Redirect detected. Valid URL: ' . $redirectURL); // store any cookies $cookies = $this->cookieJar->extractCookies($this->requests[$orig]['headers']); if (!empty($cookies)) { $this->cookieJar->storeCookies($url, $cookies); } $this->redirectQueue[$orig] = $redirectURL; } else { $this->debug('Redirect detected. Invalid URL: ' . $redirectURL); } } // die($url.' -multi- '.$request->getResponseInfo('effective_url')); unset($this->requests[$orig]['httpRequest']); } } } } else { $this->debug('Starting sequential fetch (file_get_contents)'); $this->debug('Processing set of ' . count($urls)); foreach ($urls as $orig => $url) { if (!$isRedirect) { $orig = $url; } unset($this->redirectQueue[$orig]); $this->debug("...{$url}"); if (!$isRedirect && isset($this->requests[$url])) { $this->debug("......in memory"); /* } elseif ($this->isCached($url)) { $this->debug("......is cached"); if (!$this->minimiseMemoryUse) { $this->requests[$url] = $this->getCached($url); } */ } else { $this->debug("Sending request for {$url}"); $this->requests[$orig]['original_url'] = $orig; $req_url = $this->rewriteHashbangFragment ? $this->rewriteHashbangFragment($url) : $url; $req_url = $this->removeFragment($req_url); // send cookies, if we have any $httpContext = $this->httpContext; if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) { $this->debug("......sending cookies: {$cookies}"); $httpContext['http']['header'] .= 'Cookie: ' . $cookies . "\r\n"; } if (false !== ($html = @file_get_contents($req_url, false, stream_context_create($httpContext)))) { $this->debug('Received response'); // get status code if (!isset($http_response_header[0]) || !preg_match('!^HTTP/\\d+\\.\\d+\\s+(\\d+)!', trim($http_response_header[0]), $match)) { $this->debug('Error: no status code found'); // TODO: handle error - no status code } else { $this->requests[$orig]['headers'] = $this->headersToString($http_response_header, false); $this->requests[$orig]['body'] = $html; $this->requests[$orig]['effective_url'] = $req_url; $this->requests[$orig]['status_code'] = $status_code = (int) $match[1]; unset($match); // handle redirect if (preg_match('/^Location:(.*?)$/m', $this->requests[$orig]['headers'], $match)) { $this->requests[$orig]['location'] = trim($match[1]); } if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && isset($this->requests[$orig]['location'])) { $redirectURL = $this->requests[$orig]['location']; if (!preg_match('!^https?://!i', $redirectURL)) { $redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url); } if ($this->validateURL($redirectURL)) { $this->debug('Redirect detected. Valid URL: ' . $redirectURL); // store any cookies $cookies = $this->cookieJar->extractCookies($this->requests[$orig]['headers']); if (!empty($cookies)) { $this->cookieJar->storeCookies($url, $cookies); } $this->redirectQueue[$orig] = $redirectURL; } else { $this->debug('Redirect detected. Invalid URL: ' . $redirectURL); } } } } else { $this->debug('Error retrieving URL'); //print_r($req_url); //print_r($http_response_header); //print_r($html); // TODO: handle error - failed to retrieve URL } } } } }
/** * Overrides HttpRequestPool::socketPerform() * * @return bool */ protected function socketPerform() { $rs = parent::socketPerform(); foreach ($this->getFinishedRequests() as $r) { $this->detach($r); if (206 != ($rc = $r->getResponseCode())) { throw new HttpException("Unexpected response code: {$rc}"); } file_put_contents(sprintf("%s.%04d", $this->tmp, $r->id), $r->getResponseBody()); if (-1 != ($offset = $this->getRangeOffset())) { $this->attachNew($offset); } } return $rs; }
/** * Test HttpRequestPool. */ function test_HttpRequestPool() { $http = new HttpRequestPool(); $http->send(); }
$request->setOptions(['redirect' => 3]); $requests[] = $request; } catch (\HttpException $e) { unset($config['urls'][$key]); } } if (sizeof($requests) == 0) { echo "Unable to create HTTP client" . PHP_EOL; exit; } // We know we have to iterate at least once, so use do... while do { // Get the current batch of requests $threads = array_splice($requests, 0, $config['threads']); // Create a new HttpRequestPool and attach the requests $pool = new \HttpRequestPool(); foreach ($threads as $request) { $pool->attach($request); } // Send the requests and wait till they are all complete try { $pool->send(); while (sizeof($pool->getFinishedRequests()) < sizeof($threads)) { } foreach ($pool as $request) { // Display progress if ($request->getResponseCode() == 200) { echo '.'; } else { unset($config['urls'][$request->key]); echo '!';