protected function startCrawlingQueue() { while ($this->crawlQueue->hasPendingUrls()) { $pool = new Pool($this->client, $this->getCrawlRequests(), ['concurrency' => $this->concurrency, 'options' => [RequestOptions::CONNECT_TIMEOUT => 10, RequestOptions::TIMEOUT => 10, RequestOptions::ALLOW_REDIRECTS => false], 'fulfilled' => function (ResponseInterface $response, int $index) { $this->handleResponse($response, $index); $crawlUrl = $this->crawlQueue->getPendingUrlAtIndex($index); if ($crawlUrl->url->host !== $this->baseUrl->host) { return; } $this->addAllLinksToCrawlQueue((string) $response->getBody(), $crawlUrl = $this->crawlQueue->getPendingUrlAtIndex($index)->url); }, 'rejected' => function (RequestException $exception, int $index) { $this->handleResponse($exception->getResponse(), $index); }]); $promise = $pool->promise(); $promise->wait(); $this->crawlQueue->removeProcessedUrlsFromPending(); } }