public function urlStatus($url)
{
$model = BuilderIndex::findUrl($url);
if (!$model) {
// add the url to the index
BuilderIndex::addToIndex($url);
// update the urls content
$model = BuilderIndex::findUrl($url);
$model->content = $this->getCrawler($url)->getContent();
$model->crawled = 1;
$model->status_code = 1;
$model->last_indexed = time();
$model->save();
// add the pages links to the index
foreach ($this->getCrawler($url)->getLinks() as $link) {
if ($this->matchBaseUrl($link[1])) {
BuilderIndex::addToIndex($link[1], $link[0]);
}
}
} else {
if ($model->crawled !== 1) {
$model->content = $this->getCrawler($url)->getContent();
$model->crawled = 1;
$model->status_code = 1;
$model->last_indexed = time();
$model->save();
foreach ($this->getCrawler($url)->getLinks() as $link) {
if ($this->matchBaseUrl($link[1])) {
BuilderIndex::addToIndex($link[1], $link[0]);
}
}
}
}
}
public function urlStatus($url)
{
$model = BuilderIndex::findUrl($url);
if (!$model) {
// add the url to the index
if ($this->filterUrlIsValid($url)) {
BuilderIndex::addToIndex($url, $this->getCrawler($url)->getTitle());
// update the urls content
$model = BuilderIndex::findUrl($url);
$model->content = $this->getCrawler($url)->getContent();
$model->crawled = 1;
$model->status_code = 1;
$model->last_indexed = time();
$model->language_info = $this->getCrawler($url)->getLanguageInfo();
$model->save(false);
// add the pages links to the index
foreach ($this->getCrawler($url)->getLinks() as $link) {
if ($this->matchBaseUrl($link[1])) {
if ($this->filterUrlIsValid($link[1])) {
BuilderIndex::addToIndex($link[1], $link[0]);
}
}
}
}
} else {
if (!$this->filterUrlIsValid($url)) {
$model->delete();
} else {
if ($model->crawled !== 1) {
$model->content = $this->getCrawler($url)->getContent();
$model->crawled = 1;
$model->status_code = 1;
$model->last_indexed = time();
$model->language_info = $this->getCrawler($url)->getLanguageInfo();
$model->save(false);
foreach ($this->getCrawler($url)->getLinks() as $link) {
if ($this->matchBaseUrl($link[1])) {
if ($this->filterUrlIsValid($link[1])) {
BuilderIndex::addToIndex($link[1], $link[0]);
}
}
}
}
}
}
}
