/** * @param ScraperEntity $scraperEntity * @param bool $disableLimit * * @return ScraperInterface */ protected function createScraper(ScraperEntity $scraperEntity, $disableLimit = false) { if (!array_key_exists($scraperEntity->getId(), $this->scrapers)) { $scraper = $this->factory->createScraper($scraperEntity); if ($disableLimit) { $limit = $scraper->getCrawler()->getRateLimit(); if ($limit instanceof EnablingRateLimitInterface) { $limit->disable(); } } $this->scrapers[$scraperEntity->getId()] = $scraper; } return $this->scrapers[$scraperEntity->getId()]; }
/** * @inheritdoc */ public function findSourceByScraper(Scraper $scraper, $originalId) { // look for mapping $params = ['scraper' => $scraper->getId(), 'originalId' => $originalId]; return $this->getRepository()->findOneBy($params); }
/** * Returns a unique hash for a scraper. * * @param Scraper $scraper * * @return string */ protected function getScraperHash(Scraper $scraper) { return md5('scraper' . $scraper->getId()); }
/** * Returns a cached copy of the parser for the given scraper. * * @param ScraperEntity $scraper * * @return ParserInterface */ protected function getParser(ScraperEntity $scraper) { if (!isset($this->parsers[$scraper->getId()])) { $this->parsers[$scraper->getId()] = $this->createParser($scraper); } return $this->parsers[$scraper->getId()]; }