/** * Execute the console command. * * @return mixed */ public function handle() { $this->time['start'] = Carbon::createFromTimestamp(time())->toDateTimeString(); $limit = $this->config->get('settings.cron.limit'); while ($limit > 0) { $data = json_decode($this->redis->rPop('nodes-queue'), true); if (is_null($data)) { $this->info("There is no more node in redis queue to process"); break; } $limit--; if (!isset($data['url'])) { $this->counter['failed_url']++; $this->logDebugInfo('Invalid url', $data); continue; } $url = $data['url']; $date = isset($data['date']) || $data['date'] ? $data['date'] : null; $cachePage = storage_path('caches/' . md5($url) . '.html'); if (!file_exists($cachePage)) { try { $htmlContent = file_get_contents($url); } catch (\Exception $e) { $this->error("# Cannot get content of {$url}. [{$e->getMessage()}]"); $this->counter['failed_url']++; continue; } file_put_contents($cachePage, html_entity_decode($htmlContent)); } @$this->dom->loadHTMLFile($cachePage); $title = $this->getTitle(); if (is_null($title)) { $this->counter['empty_url']++; $this->logDebugInfo('Empty content', $data); continue; } $content = $this->getContent($this->pruneContent($this->dom->getElementsByTagName('body')->item(0))); $this->saveNode($title, $content, $url, $date); } $this->printResultTable(); }