public function execute()
{
$documents = array();
foreach ($this->url_queue as $url) {
if ($this->config['timeout']) {
usleep($this->config['timeout']);
}
$download = new Downloader($url, $this->config['referer'], $this->config['user_agent']);
$documents[] = array('url' => $url, 'doc' => $download->execute());
if ($download->get_status() == '404') {
$this->count_404++;
} else {
$this->count_404 = 0;
}
if ($this->config['give_up_404'] && $this->count_404 >= $this->config['give_up_404']) {
$this->last_error = 'Maximum 404s hit';
return false;
}
}
return $documents;
}
