/** * Run crawler * * @param $url */ protected function runUrlCrawler($url) { $this->write('Start crawler'); $queue = new Queue(); ControlTime::init(); $this->write('WAIT', false, false); do { $urlForCrawler = $queue->getQueue() ? current($queue->getQueue()) : $url; $facade = new ParserHtmlFacade($urlForCrawler); $facade->setQueueObject($queue)->run(); $this->write(' .', false, false); } while (count($queue->getQueue()) > 0); $this->write('', true, false); $this->write("Indexed " . count($queue->getComplete()) . " pages"); $saver = new Saver($queue->getComplete()); $this->write("Save to file"); $saver->save(); $this->write('Finish crawler'); CliReturnCode::success(); }
/** * Set data to complete stack * * @param $count */ private function setToComplete($count) { $this->getQueueObject()->setToComplete($this->url, $count, ControlTime::getPrevData()['from_prev']); }