/** * @inheritdoc */ protected function execute(InputInterface $input, OutputInterface $output) { $async = $input->getOption('async'); $noLimit = $input->getOption('no-limit'); $scrapers = $this->findScrapers($input->getArgument('scraper')); foreach ($scrapers as $scraperEntity) { $date = new \DateTime(sprintf('-%d hours', $scraperEntity->getRevisitFrequency())); $builder = $this->sourceManager->getRepository()->queryByScraperAndUnvisitedSince($scraperEntity, $date); foreach ($builder->getQuery()->iterate() as list($source)) { /* @var SourceInterface $source */ try { $output->writeln(sprintf('Revisiting <info>%s</info>', $source->getOriginalUrl())); $this->revisitor->revisit($source, $async, $noLimit); } catch (CrawlException $e) { $output->writeln(sprintf('<error>%s</error>', $e->getMessage())); } } } }
/** * @inheritdoc */ public function execute(array $payload) { /** @var SourceInterface $source */ list($source) = $payload; try { $this->revisitor->revisit($source, true); return true; } catch (RateLimitException $e) { $re = new RescheduleException(); if ($date = $e->getRetryDate()) { $re->setRescheduleDate($date); } throw $re; } catch (CrawlException $e) { $this->logger->error($e->getMessage(), ['url' => $e->getUrl()]); return false; } }