public function __construct(Iterator $iterator, $options = array()) { $this->downloader = new Spider_Downloader(); $this->options = $options; //Don't throw exceptions on 404 robots.txt $this->options['crawl_404_pages'] = true; parent::__construct($iterator); }
public function __construct(Iterator $iterator, $baseuri) { $this->baseuri = $baseuri; parent::__construct($iterator); }
public function current() { $urlInfo = Spider::getURIInfo(parent::current(), $this->options); return $urlInfo['effective_url']; }
public function current() { return preg_replace('/#(.*)/', '', parent::current()); }