/**
  * Consume a message, extracts the URL from it and crawls the webpage.
  *
  * @param \PhpAmqpLib\Message\AMQPMessage $message
  */
 public function crawlUrl(AMQPMessage $message)
 {
     $data = json_decode($message->body, true);
     $crawlJob = new CrawlJob($data['url'], $data['base_url'], $data['blacklist'], $data['metadata'], $data['whitelist']);
     if (false === $crawlJob->isAllowedToCrawl()) {
         $this->indexer->deleteDocument($message);
         $this->queue->rejectMessage($message);
         $this->markAsSkipped($crawlJob, 'info', 'Not allowed to crawl');
         return;
     }
     if ($this->indexer->isUrlIndexedAndNotExpired($crawlJob->getUrl(), $crawlJob->getMetadata())) {
         $this->queue->rejectMessage($message);
         $this->markAsSkipped($crawlJob, 'info', 'Not expired yet');
         return;
     }
     try {
         $this->spider->getRequestHandler()->getClient()->setUserAgent($this->userAgent);
         $this->spider->getRequestHandler()->getClient()->getConfig()->set('request.params', ['redirect.disable' => true]);
         $this->spider->crawl($crawlJob);
         $this->logMessage('info', sprintf("Crawling %s", $crawlJob->getUrl()), $crawlJob->getUrl(), $data['metadata']['core']);
         $this->queue->acknowledge($message);
     } catch (ClientErrorResponseException $e) {
         switch ($e->getResponse()->getStatusCode()) {
             case 301:
                 $this->indexer->deleteDocument($message);
                 $this->queue->rejectMessage($message);
                 $this->markAsSkipped($crawlJob, 'warning', $e->getMessage());
                 $newCrawlJob = new CrawlJob($e->getResponse()->getInfo('redirect_url'), $crawlJob->getBaseUrl(), $crawlJob->getBlacklist(), $crawlJob->getMetadata(), $crawlJob->getWhitelist());
                 $this->queue->publishJob($newCrawlJob);
                 break;
             case 403:
             case 401:
             case 500:
                 $this->queue->rejectMessage($message);
                 $this->markAsSkipped($crawlJob, 'warning', 'status: ' . $e->getResponse()->getStatusCode());
                 break;
             case 404:
             case 418:
                 $this->indexer->deleteDocument($message);
                 $this->logMessage('warning', sprintf("Deleted %s", $crawlJob->getUrl()), $crawlJob->getUrl(), $data['metadata']['core']);
                 $this->queue->rejectMessage($message);
                 break;
             default:
                 $this->queue->rejectMessageAndRequeue($message);
                 $this->markAsFailed($crawlJob, $e->getResponse()->getStatusCode());
                 break;
         }
     } catch (Exception $e) {
         $this->queue->rejectMessage($message);
         $this->markAsFailed($crawlJob, $e->getMessage());
     }
     unset($crawlJob, $message, $data);
     gc_collect_cycles();
 }
 /**
  * Writes the found URL as a job on the queue.
  *
  * And URL is only persisted to the queue when it not has been indexed yet.
  *
  * @param \Symfony\Component\EventDispatcher\GenericEvent $event
  */
 public function onDiscoverUrl(Event $event)
 {
     $crawlJob = $event->getSubject()->getCurrentCrawlJob();
     foreach ($event['uris'] as $uri) {
         if ($position = strpos($uri, '#')) {
             $uri = new Uri(substr($uri, 0, $position));
         }
         $isBlacklisted = UrlCheck::isUrlBlacklisted($uri->normalize()->toString(), $crawlJob->getBlacklist());
         if ($isBlacklisted) {
             $this->eventDispatcher->dispatch("spider.crawl.blacklisted", new Event($this, ['uri' => $uri]));
             continue;
             //url blacklisted, so go to next one
         }
         if (!$this->indexer->isUrlIndexedandNotExpired(UrlCheck::fixUrl($uri->toString()), $crawlJob->getMetadata())) {
             $job = new CrawlJob(UrlCheck::fixUrl($uri->normalize()->toString()), (new Uri($crawlJob->getUrl()))->normalize()->toString(), $crawlJob->getBlacklist(), $crawlJob->getMetadata(), $crawlJob->getWhitelist());
             if ($job->isAllowedToCrawl()) {
                 $this->queue->publishJob($job);
             }
         }
     }
 }