protected function setUp()
{
$this->logger = $this->getLogger();
$this->requests = [strtotime('-1 minute') => 'http://example/org/1', strtotime('-2 minutes') => 'http://example/org/2', strtotime('-5 minutes') => 'http://example/org/3', strtotime('-6 minutes') => 'http://example/org/4', strtotime('-7 minutes') => 'http://example/org/5', strtotime('-8 minutes') => 'http://example/org/6', strtotime('-9 minutes') => 'http://example/org/7'];
foreach ($this->requests as $time => $url) {
$this->logger->logRequest($url, new \DateTime('@' . $time));
}
}
/**
* @inheritdoc
*/
public function crawl($url)
{
$this->response = null;
if ($this->rateLimit->limitReached()) {
throw new RateLimitException($url, sprintf('Reached the rate limit of %s', $this->rateLimit->getLimit()), $this->rateLimit->getRetryDate());
}
$this->logger->logRequest($url, new \DateTime());
list($this->url, $this->response) = $this->client->fetch($url, $this->getUserAgent($url));
if ($this->response->getStatusCode() === 429) {
throw new RateLimitException($url, sprintf('Server replied with response %d (Too Many Requests)', 429), $this->getRetryAfterDate());
}
if ($this->islastResponseNotFound()) {
throw new NotFoundException($url, $this->response);
}
if (!$this->islastResponseOk()) {
throw new UnexpectedResponseException($url, $this->response);
}
$body = $this->response->getBody();
$contents = $body->getContents();
// rewind stream, in case we need to use the last response
if ($body->isSeekable()) {
$body->rewind();
}
return $contents;
}
