/**
* @inheritdoc
*/
public function limitReached()
{
if (!$this->enabled) {
return false;
}
$date = new \DateTime(sprintf('-%d seconds', $this->timeUnit));
$requests = $this->logger->getRequestsSince($date);
return sizeof($requests) >= $this->maxAmount;
}
/**
* @inheritdoc
*/
public function crawl($url)
{
$this->response = null;
if ($this->rateLimit->limitReached()) {
throw new RateLimitException($url, sprintf('Reached the rate limit of %s', $this->rateLimit->getLimit()), $this->rateLimit->getRetryDate());
}
$this->logger->logRequest($url, new \DateTime());
list($this->url, $this->response) = $this->client->fetch($url, $this->getUserAgent($url));
if ($this->response->getStatusCode() === 429) {
throw new RateLimitException($url, sprintf('Server replied with response %d (Too Many Requests)', 429), $this->getRetryAfterDate());
}
if ($this->islastResponseNotFound()) {
throw new NotFoundException($url, $this->response);
}
if (!$this->islastResponseOk()) {
throw new UnexpectedResponseException($url, $this->response);
}
$body = $this->response->getBody();
$contents = $body->getContents();
// rewind stream, in case we need to use the last response
if ($body->isSeekable()) {
$body->rewind();
}
return $contents;
}
/**
* Tests fetching of all logged requests.
*/
public function testAllLoggedRequests()
{
$this->assertEquals($this->getLoggedRequests($this->requests), $this->logger->getRequestsSince());
}
