/** * Discover a URL from a location header. * * @param \LastCall\Crawler\Event\CrawlerResponseEvent $event * @param $eventName * @param \Symfony\Component\EventDispatcher\EventDispatcherInterface $dispatcher */ public function onResponse(CrawlerResponseEvent $event, $eventName, EventDispatcherInterface $dispatcher) { $response = $event->getResponse(); if ($this->isRedirectResponse($response)) { $this->processUris($event, $dispatcher, [$response->getHeaderLine('Location')], 'redirect'); } }
/** * Normalize and dispatch a discovery event for an array of URIs. * * @param \LastCall\Crawler\Event\CrawlerResponseEvent $event * @param \Symfony\Component\EventDispatcher\EventDispatcherInterface $dispatcher * @param string $context * @param string[] $urls */ protected function processUris(CrawlerResponseEvent $event, EventDispatcherInterface $dispatcher, array $urls, $context = 'unknown') { if (empty($urls)) { return; } $resolve = Normalizations::resolve($event->getRequest()->getUri()); $uris = []; foreach ($urls as $url) { $uri = new Uri($url); $uri = $resolve($uri); $uri = $this->normalizer->normalize($uri); $uris[(string) $uri] = $uri; } $uris = array_values($uris); $discoveryEvent = new CrawlerUrisDiscoveredEvent($event->getRequest(), $event->getResponse(), $uris, $context); $dispatcher->dispatch(CrawlerEvents::URIS_DISCOVERED, $discoveryEvent); foreach ($discoveryEvent->getAdditionalRequests() as $request) { $event->addAdditionalRequest($request); } }
private function getRedirectUri(CrawlerResponseEvent $event) { return $event->getResponse()->getHeaderLine('Location'); }
public function __construct(RequestInterface $request, ResponseInterface $response) { parent::__construct($request, $response); }
public function __construct(RequestInterface $request, ResponseInterface $response, array $discoveredUris, $context = 'unknown') { parent::__construct($request, $response); $this->discoveredUris = $discoveredUris; $this->context = $context; }