<?php
include 'libs/general.php';
CronManager::init(__FILE__, 10);
try {
// GET URLs
$urls = Providers::get_lead_urls();
_w('Got ' . count($urls) . ' urls');
/// PROCESS URLS
foreach ($urls as $url) {
// Parse array to url
$url_w = urls::create_url($url);
_w('Getting url ' . $url_w);
// Init content analyzer
$ca = ContentAnalyzer::getAnalyzer($url_w);
// If content ignored
if (!$ca) {
_w('ignored');
// Skip
continue;
}
// _w($ca->getOgDescription());
// die;
_w('Createing general CA data');
_w('populating search table');
if (Providers::create_search_item($ca)) {
_w("search data inserted");
} else {
_w("unable to insert search data");
}
_w('setting status to indexed');
/**
* Extracting links
* @return array
*/
public function getLinks()
{
if (!$this->_all_links) {
$this->_all_links = urls::create_full_url_list(HTML::extract_links($this->_content), $this->_url);
}
return $this->_all_links;
}
