public function __construct($rootUrl, $timeToLive)
{
$this->crawledUrlEnds = array();
$this->rootUrl = Util::getHostPart($rootUrl);
$this->robots = $this->parseRobotsTxt($rootUrl);
$this->urlFrontier = array();
array_push($this->urlFrontier, $rootUrl);
$this->crawled = array();
$this->ignore = Database::getRetrievedLinks($rootUrl);
$this->startDate = new DateTime();
$this->timeToLive = $timeToLive;
}
