/** * Awaits all the requests attaching callbacks before. * * @return mixed */ public function unwrap() { //Deals with whole pool cache if ($this->getCachePool() && $this->getCacheKey() && ($cacheItem = $this->getCachePool()->getItem($this->getCacheKey())) && $cacheItem->isHit()) { return $cacheItem->get(); } $promises = $this->flushPromises(); if (empty($promises)) { return []; } foreach ($promises as $key => $promise) { $promises[$key] = $this->attachTransforms($promise); } $res = \GuzzleHttp\Promise\unwrap($promises); $res = array_merge($this->cacheData, $res); //saves pool cache if ($this->getCacheKey()) { $cacheItem = $this->getCachePool()->getItem($this->getCacheKey()); $cacheItem->set($res); } // clearing so the pool can be reused. $this->promises = []; $this->cacheData = []; $this->queries = []; return $res; }
public function testUnwrapsPromisesWithKeys() { $promises = ['foo' => new FulfilledPromise('a'), 'bar' => new FulfilledPromise('b')]; $this->assertEquals(['foo' => 'a', 'bar' => 'b'], \GuzzleHttp\Promise\unwrap($promises)); }
public static function purgeReValidation() { \GuzzleHttp\Promise\unwrap(static::$waitingRevalidate); }
public function batch($requests, $options = []) { $this->lastError = null; $this->lastRequest = null; $this->lastResponse = null; if (empty($this->accessKey)) { $this->lastError = "No access key set. Please set an access key before attempting to fetch forecast data."; } if (!$this->httpClient) { $this->httpClient = new \GuzzleHttp\Client(); } $baseURL = rtrim(self::ENDPOINT, '/') . '/' . $this->accessKey . '/'; $httpRequestPromises = []; foreach ($requests as $idx => $request) { $url = $baseURL . $request['latitude'] . ',' . $request['longitude']; if (!empty($request['time'])) { $url .= ',' . $request['time']; } if ($options) { $url .= '?' . http_build_query($options); } $requests[$idx]['url'] = $url; $httpRequestPromises[$idx] = $this->httpClient->getAsync($url, ['allow_redirects' => true, 'timeout' => $this->timeout, 'connect_timeout' => $this->connectTimeout, 'headers' => ['User-Agent' => 'Tave/ForecastIOWrapper']]); } try { $httpResults = \GuzzleHttp\Promise\unwrap($httpRequestPromises); foreach ($httpResults as $idx => $httpResponse) { if ($httpResponse instanceof \GuzzleHttp\Exception\RequestException) { $this->lastRequest = $requests[$idx]; $this->lastResponse = $httpResponse; $this->lastError = $httpResults[$resultIdx]->getMessage(); $requests[$idx]['error'] = $this->lastError; } else { $requests[$idx]['forecast'] = json_decode($httpResponse->getBody(), true); } } return $requests; } catch (\GuzzleHttp\Exception\RequestException $e) { $this->lastRequest = $e->getRequest(); $this->lastResponse = $e->hasResponse() ? $e->getResponse() : null; $this->lastError = $e->getMessage(); return false; } }
require __DIR__ . '/vendor/autoload.php'; use GuzzleHttp\Client; use GuzzleHttp\TransferStats; use GuzzleHttp\HandlerStack; $stats = function (TransferStats $stats) { echo $stats->getTransferTime() . " " . $stats->getEffectiveUri() . "\n"; }; // if you are going to use multiple clients, and want to make requests async among all of them // you have to pass a handler into the client, so they all run in the same loop $handler = HandlerStack::create(); $usernameClient = new Client(['base_uri' => 'http://localhost:8000/', 'timeout' => 10, 'on_stats' => $stats, 'handler' => $handler]); $passwordClient = new Client(['base_uri' => 'http://localhost:8001/', 'timeout' => 10, 'on_stats' => $stats, 'handler' => $handler]); $emailClient = new Client(['base_uri' => 'http://localhost:8002/', 'timeout' => 10, 'on_stats' => $stats, 'handler' => $handler]); $start = microtime(true); $username1 = $usernameClient->post('generate-username'); $username2 = $usernameClient->post('generate-username'); $password = $passwordClient->post('generate-password'); $email = $emailClient->post('generate-email'); $stop = microtime(true); $time = $stop - $start; echo "4 requests in {$time} seconds\n"; $start = microtime(true); $promises = []; $promises['username1'] = $usernameClient->postAsync('generate-username'); $promises['username2'] = $usernameClient->postAsync('generate-username'); $promises['password'] = $passwordClient->postAsync('generate-password'); $promises['email'] = $emailClient->postAsync('generate-email'); $results = GuzzleHttp\Promise\unwrap($promises); $stop = microtime(true); $time = $stop - $start; echo "4 requests in {$time} seconds\n";
/** * @return array */ protected function getResponses() { /** * @var Response $response */ $responses = \GuzzleHttp\Promise\unwrap($this->promises); $temp = []; foreach ($responses as $key => $response) { $temp[$key] = $this->getDecoder()->decode($response->getBody()->__toString()); } return $temp; }
/** * @param $threads */ private function crawlPages($threads) { $promises = []; for ($i = 0; $threads < count($promises) || $i < $threads; $i++) { if (!($url = array_shift($this->queue))) { break; } if (isset($this->scanned[$url])) { continue; } $this->scanned[$url] = true; if ($this->getUrlDepth($url) > $this->config->get('maxDepth')) { continue; } $this->log('Start scan page ' . $url, 1); try { $promise = $this->loadPage($url, true); } catch (TooManyRedirectsException $e) { $this->log("Failed to load resource {$url} with error: " . $e->getMessage() . "\n", 2); continue; } $promise->then(function (ResponseInterface $response) use($url, $threads) { if ($lastModified = $response->getHeaderLine("Last-Modified")) { $lastModified = \DateTime::createFromFormat("D, d M Y H:i:s O", $lastModified); $lastModified = $lastModified ? $lastModified->format('Y-m-d\\TH:i:sP') : null; } else { $lastModified = null; } if (200 !== (int) $response->getStatusCode()) { $this->log('Error! Url: ' . $url . ' Status: ' . $response->getStatusCode()); return false; } /** * @var $pageUrl Url */ $pageUrl = $this->config->get('url'); $linksAmount = $this->storage->countLinks($pageUrl->getUrl()); if ($linksAmount >= $this->config->get('limits.linksTotal')) { $this->queue = []; return false; } $html = (string) $response->getBody(); $result = $this->parser->load($html); $pageInfo = ['canonical' => $result['meta']['canonical'], 'metaRobots' => $result['meta']['robots'], "status" => $response->getStatusCode(), "lastModified" => $lastModified]; if (false === stripos($pageInfo['metaRobots'], 'noindex') && $this->checkContent($html)) { if ($this->config->get('download.enable')) { file_put_contents($this->config->get('download.directory') . '/' . urlencode($url) . '.tmp.html', $html); } $this->storage->addLink($pageUrl->getUrl(), $url, $pageInfo); $this->log('Link ' . $url . ' added', 2); } if (false !== stripos($pageInfo['metaRobots'], 'nofollow')) { $this->log("Page has meta tag nofollow\n", 2); return false; } $links = $this->getCorrectLinks($result['links']); foreach ($links as $link) { $preparedLink = $this->prepare($link); if (isset($this->scanned[$preparedLink])) { continue; } elseif (in_array($preparedLink, $this->queue)) { continue; } elseif ($this->getUrlDepth($preparedLink) > $this->config->get('maxDepth')) { continue; } $this->queue[] = $preparedLink; } return true; }, function (\Exception $e) use($url, $threads) { if (0 === $e->getCode()) { $this->queue[] = $url; } elseif (in_array((int) $e->getCode(), [429, 503])) { $this->queue[] = $url; } $this->log("Failed to load resource {$url} with error: " . $e->getMessage() . "\n", 2); }); $promises[] = $promise; } try { \GuzzleHttp\Promise\unwrap($promises); } catch (TooManyRedirectsException $e) { $this->log("Client error while resolving promise: " . $e->getMessage()); } catch (ClientException $e) { $this->log("Client error while resolving promise: " . $e->getMessage()); } catch (ServerException $e) { $this->log("Server error while resolving promise: " . $e->getMessage()); if (in_array((int) $e->getCode(), [429, 503])) { $threads = $this->config->get('limits.threads'); if ($threads > 1) { $this->config->set('limits.threads', $threads - 1); } $sleepTimeout = $this->config->get('timeout.sleep'); $this->config->set('limits.sleep', $sleepTimeout + 0.5); $this->log(sprintf('Sleep for 5 minutes. Threads limit is decremented. Now is %d', $threads)); sleep(300); } } catch (ConnectException $e) { $this->log("Connection error while resolving promise: " . $e->getMessage()); } $this->executeCallback(true); }