public function download($uri, $options = array())
{
//Make sure that the curl_options exists.
if (!isset($options['curl_options'])) {
$options['curl_options'] = array();
}
if (isset($options['user_agent'])) {
$options['curl_options'][CURLOPT_USERAGENT] = $options['user_agent'];
}
//Make sure that the content is returned.
$options['curl_options'][CURLOPT_RETURNTRANSFER] = true;
$options['curl_options'][CURLOPT_NOBODY] = false;
$info = Spider::getURIInfo($uri, $options['curl_options']);
if (!$info['content']) {
throw new Exception('Error downloading ' . $uri . ' ' . $info['content']);
}
if (in_array($info['http_code'], array(0, 404)) && isset($options['crawl_404_pages']) && !$options['crawl_404_pages']) {
throw new Exception('404 page ' . $uri . ' ' . $info['http_code']);
}
return $info['content'];
}
public function current()
{
$urlInfo = Spider::getURIInfo(parent::current(), $this->options);
return $urlInfo['effective_url'];
}
