public function execute() { $documents = array(); foreach ($this->url_queue as $url) { if ($this->config['timeout']) { usleep($this->config['timeout']); } $download = new Downloader($url, $this->config['referer'], $this->config['user_agent']); $documents[] = array('url' => $url, 'doc' => $download->execute()); if ($download->get_status() == '404') { $this->count_404++; } else { $this->count_404 = 0; } if ($this->config['give_up_404'] && $this->count_404 >= $this->config['give_up_404']) { $this->last_error = 'Maximum 404s hit'; return false; } } return $documents; }