Exemplo n.º 1
0
 public function urlStatus($url)
 {
     $model = BuilderIndex::findUrl($url);
     if (!$model) {
         // add the url to the index
         BuilderIndex::addToIndex($url);
         // update the urls content
         $model = BuilderIndex::findUrl($url);
         $model->content = $this->getCrawler($url)->getContent();
         $model->crawled = 1;
         $model->status_code = 1;
         $model->last_indexed = time();
         $model->save();
         // add the pages links to the index
         foreach ($this->getCrawler($url)->getLinks() as $link) {
             if ($this->matchBaseUrl($link[1])) {
                 BuilderIndex::addToIndex($link[1], $link[0]);
             }
         }
     } else {
         if ($model->crawled !== 1) {
             $model->content = $this->getCrawler($url)->getContent();
             $model->crawled = 1;
             $model->status_code = 1;
             $model->last_indexed = time();
             $model->save();
             foreach ($this->getCrawler($url)->getLinks() as $link) {
                 if ($this->matchBaseUrl($link[1])) {
                     BuilderIndex::addToIndex($link[1], $link[0]);
                 }
             }
         }
     }
 }
Exemplo n.º 2
0
 public function urlStatus($url)
 {
     $model = BuilderIndex::findUrl($url);
     if (!$model) {
         // add the url to the index
         if ($this->filterUrlIsValid($url)) {
             BuilderIndex::addToIndex($url, $this->getCrawler($url)->getTitle());
             // update the urls content
             $model = BuilderIndex::findUrl($url);
             $model->content = $this->getCrawler($url)->getContent();
             $model->crawled = 1;
             $model->status_code = 1;
             $model->last_indexed = time();
             $model->language_info = $this->getCrawler($url)->getLanguageInfo();
             $model->save(false);
             // add the pages links to the index
             foreach ($this->getCrawler($url)->getLinks() as $link) {
                 if ($this->matchBaseUrl($link[1])) {
                     if ($this->filterUrlIsValid($link[1])) {
                         BuilderIndex::addToIndex($link[1], $link[0]);
                     }
                 }
             }
         }
     } else {
         if (!$this->filterUrlIsValid($url)) {
             $model->delete();
         } else {
             if ($model->crawled !== 1) {
                 $model->content = $this->getCrawler($url)->getContent();
                 $model->crawled = 1;
                 $model->status_code = 1;
                 $model->last_indexed = time();
                 $model->language_info = $this->getCrawler($url)->getLanguageInfo();
                 $model->save(false);
                 foreach ($this->getCrawler($url)->getLinks() as $link) {
                     if ($this->matchBaseUrl($link[1])) {
                         if ($this->filterUrlIsValid($link[1])) {
                             BuilderIndex::addToIndex($link[1], $link[0]);
                         }
                     }
                 }
             }
         }
     }
 }