PHP Spatie\Crawler CrawlQueue::hasAlreadyBeenProcessed Examples

hasAlreadyBeenProcessed() public method

public hasAlreadyBeenProcessed ( CrawlUrl $url )
$url	CrawlUrl

Example #1

File: Crawler.php Project: spatie/crawler

 protected function getCrawlRequests() : Generator
 {
     $i = 0;
     while ($crawlUrl = $this->crawlQueue->getPendingUrlAtIndex($i)) {
         if (!$this->crawlProfile->shouldCrawl($crawlUrl->url)) {
             $i++;
             continue;
         }
         if ($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl)) {
             $i++;
             continue;
         }
         $this->crawlObserver->willCrawl($crawlUrl->url);
         $this->crawlQueue->markAsProcessed($crawlUrl);
         (yield new Request('GET', (string) $crawlUrl->url));
         $i++;
     }
 }