protected function setUp() { $this->logger = $this->getLogger(); $this->requests = [strtotime('-1 minute') => 'http://example/org/1', strtotime('-2 minutes') => 'http://example/org/2', strtotime('-5 minutes') => 'http://example/org/3', strtotime('-6 minutes') => 'http://example/org/4', strtotime('-7 minutes') => 'http://example/org/5', strtotime('-8 minutes') => 'http://example/org/6', strtotime('-9 minutes') => 'http://example/org/7']; foreach ($this->requests as $time => $url) { $this->logger->logRequest($url, new \DateTime('@' . $time)); } }
/** * @inheritdoc */ public function crawl($url) { $this->response = null; if ($this->rateLimit->limitReached()) { throw new RateLimitException($url, sprintf('Reached the rate limit of %s', $this->rateLimit->getLimit()), $this->rateLimit->getRetryDate()); } $this->logger->logRequest($url, new \DateTime()); list($this->url, $this->response) = $this->client->fetch($url, $this->getUserAgent($url)); if ($this->response->getStatusCode() === 429) { throw new RateLimitException($url, sprintf('Server replied with response %d (Too Many Requests)', 429), $this->getRetryAfterDate()); } if ($this->islastResponseNotFound()) { throw new NotFoundException($url, $this->response); } if (!$this->islastResponseOk()) { throw new UnexpectedResponseException($url, $this->response); } $body = $this->response->getBody(); $contents = $body->getContents(); // rewind stream, in case we need to use the last response if ($body->isSeekable()) { $body->rewind(); } return $contents; }