<?php include 'libs/general.php'; CronManager::init(__FILE__, 10); try { // GET URLs $urls = Providers::get_lead_urls(); _w('Got ' . count($urls) . ' urls'); /// PROCESS URLS foreach ($urls as $url) { // Parse array to url $url_w = urls::create_url($url); _w('Getting url ' . $url_w); // Init content analyzer $ca = ContentAnalyzer::getAnalyzer($url_w); // If content ignored if (!$ca) { _w('ignored'); // Skip continue; } // _w($ca->getOgDescription()); // die; _w('Createing general CA data'); _w('populating search table'); if (Providers::create_search_item($ca)) { _w("search data inserted"); } else { _w("unable to insert search data"); } _w('setting status to indexed');
/** * Extracting links * @return array */ public function getLinks() { if (!$this->_all_links) { $this->_all_links = urls::create_full_url_list(HTML::extract_links($this->_content), $this->_url); } return $this->_all_links; }