/**
* @inheritdoc
*/
public function scrape(ScraperEntity $scraper, $url, $continue = true)
{
$url = $this->normalizeUrl($url);
try {
$html = $this->crawler->crawl($url);
// put it in a bag
$item = new ScrapedItemBag($scraper, $url, $html);
// scrape the item and the next urls
$this->scrapeItem($item);
if ($continue) {
$this->scrapeNext($scraper);
}
} catch (RateLimitException $e) {
$this->handleRateLimitException($scraper, $url, $e);
throw $e;
} catch (UnexpectedResponseException $e) {
// we didn't get a 200 OK response, let the application know
$this->eventDispatcher->dispatch(ScraperEvents::SCRAPE_URL_NOT_OK, new ScrapeResponseEvent($scraper, $url, $e->getResponse()));
throw $e;
} catch (CrawlException $e) {
// something bad happened, let the calling command handle this
throw $e;
}
}
