/** * @inheritdoc */ public function execute(array $payload) { /** @var ScraperEntity $entity */ /** @var string $url */ list($entity, $url) = $payload; $scraper = $this->factory->createScraper($entity); $scraper->setAsync(true); try { $scraper->scrape($entity, $url); return true; } catch (RateLimitException $e) { $re = new RescheduleException(); if ($date = $e->getRetryDate()) { $re->setRescheduleDate($date); } throw $re; } catch (CrawlException $e) { $this->logger->error($e->getMessage(), ['url' => $e->getUrl()]); return false; } }
/** * @inheritdoc */ public function execute(array $payload) { /** @var SourceInterface $source */ list($source) = $payload; try { $this->revisitor->revisit($source, true); return true; } catch (RateLimitException $e) { $re = new RescheduleException(); if ($date = $e->getRetryDate()) { $re->setRescheduleDate($date); } throw $re; } catch (CrawlException $e) { $this->logger->error($e->getMessage(), ['url' => $e->getUrl()]); return false; } }
public function testExecuteJobAndReschedule() { $action = 'test'; $executor = $this->getMockForAbstractClass(ExecutorInterface::class); $executor->expects($this->any())->method('getName')->will($this->returnValue($action)); $this->manager->addExecutor($executor); $data = []; $job = new Job(1234, json_encode($data)); $stats = ['tube' => $action, 'releases' => 0, 'pri' => PheanstalkInterface::DEFAULT_PRIORITY]; $this->pheanstalk->expects($this->once())->method('statsJob')->with($job)->will($this->returnValue($stats)); $this->pheanstalk->expects($this->once())->method('release')->with($job, PheanstalkInterface::DEFAULT_PRIORITY, 10); $executor->expects($this->once())->method('execute')->will($this->throwException(RescheduleException::create('10 seconds'))); $this->assertFalse($this->manager->executeJob($job)); }