public function register(Container $pimple)
{
// Transforms URLs into a standard form.
$pimple['normalizer'] = function () use($pimple) {
return new Normalizer([Normalizations::lowercaseHostname(), Normalizations::capitalizeEscaped(), Normalizations::decodeUnreserved(), Normalizations::dropFragment()]);
};
}
/**
* Normalize and dispatch a discovery event for an array of URIs.
*
* @param \LastCall\Crawler\Event\CrawlerResponseEvent $event
* @param \Symfony\Component\EventDispatcher\EventDispatcherInterface $dispatcher
* @param string $context
* @param string[] $urls
*/
protected function processUris(CrawlerResponseEvent $event, EventDispatcherInterface $dispatcher, array $urls, $context = 'unknown')
{
if (empty($urls)) {
return;
}
$resolve = Normalizations::resolve($event->getRequest()->getUri());
$uris = [];
foreach ($urls as $url) {
$uri = new Uri($url);
$uri = $resolve($uri);
$uri = $this->normalizer->normalize($uri);
$uris[(string) $uri] = $uri;
}
$uris = array_values($uris);
$discoveryEvent = new CrawlerUrisDiscoveredEvent($event->getRequest(), $event->getResponse(), $uris, $context);
$dispatcher->dispatch(CrawlerEvents::URIS_DISCOVERED, $discoveryEvent);
foreach ($discoveryEvent->getAdditionalRequests() as $request) {
$event->addAdditionalRequest($request);
}
}