/** * @inheritdoc */ public function limitReached() { if (!$this->enabled) { return false; } $date = new \DateTime(sprintf('-%d seconds', $this->timeUnit)); $requests = $this->logger->getRequestsSince($date); return sizeof($requests) >= $this->maxAmount; }
/** * @inheritdoc */ public function crawl($url) { $this->response = null; if ($this->rateLimit->limitReached()) { throw new RateLimitException($url, sprintf('Reached the rate limit of %s', $this->rateLimit->getLimit()), $this->rateLimit->getRetryDate()); } $this->logger->logRequest($url, new \DateTime()); list($this->url, $this->response) = $this->client->fetch($url, $this->getUserAgent($url)); if ($this->response->getStatusCode() === 429) { throw new RateLimitException($url, sprintf('Server replied with response %d (Too Many Requests)', 429), $this->getRetryAfterDate()); } if ($this->islastResponseNotFound()) { throw new NotFoundException($url, $this->response); } if (!$this->islastResponseOk()) { throw new UnexpectedResponseException($url, $this->response); } $body = $this->response->getBody(); $contents = $body->getContents(); // rewind stream, in case we need to use the last response if ($body->isSeekable()) { $body->rewind(); } return $contents; }
/** * Tests fetching of all logged requests. */ public function testAllLoggedRequests() { $this->assertEquals($this->getLoggedRequests($this->requests), $this->logger->getRequestsSince()); }