/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$this->time['start'] = Carbon::createFromTimestamp(time())->toDateTimeString();
$limit = $this->config->get('settings.cron.limit');
while ($limit > 0) {
$data = json_decode($this->redis->rPop('nodes-queue'), true);
if (is_null($data)) {
$this->info("There is no more node in redis queue to process");
break;
}
$limit--;
if (!isset($data['url'])) {
$this->counter['failed_url']++;
$this->logDebugInfo('Invalid url', $data);
continue;
}
$url = $data['url'];
$date = isset($data['date']) || $data['date'] ? $data['date'] : null;
$cachePage = storage_path('caches/' . md5($url) . '.html');
if (!file_exists($cachePage)) {
try {
$htmlContent = file_get_contents($url);
} catch (\Exception $e) {
$this->error("# Cannot get content of {$url}. [{$e->getMessage()}]");
$this->counter['failed_url']++;
continue;
}
file_put_contents($cachePage, html_entity_decode($htmlContent));
}
@$this->dom->loadHTMLFile($cachePage);
$title = $this->getTitle();
if (is_null($title)) {
$this->counter['empty_url']++;
$this->logDebugInfo('Empty content', $data);
continue;
}
$content = $this->getContent($this->pruneContent($this->dom->getElementsByTagName('body')->item(0)));
$this->saveNode($title, $content, $url, $date);
}
$this->printResultTable();
}
