public function testParsingWithStartingBOM()
{
$dataPath = __DIR__ . '/../data/withBom.txt';
$parser = new \webignition\RobotsTxt\File\Parser();
$parser->setSource(file_get_contents($dataPath));
$file = $parser->getFile();
$this->assertEquals(1, count($file->getRecords()));
$this->assertEquals(2, count($file->getDirectivesFor('*')->getValues()));
$this->assertEquals(array('disallow:/', 'allow:/humans.txt'), $file->getDirectivesFor('*')->getValues());
}
/**
*
* @return string
*/
private function findSitemapUrlsFromRobotsTxt()
{
$robotsTxtParser = new \webignition\RobotsTxt\File\Parser();
$robotsTxtParser->setSource($this->getRobotsTxtContent());
$robotsTxtFile = $robotsTxtParser->getFile();
$urls = array();
if ($robotsTxtFile->directiveList()->containsField('sitemap')) {
$sitemapDirectives = $robotsTxtFile->directiveList()->filter(array('field' => 'sitemap'));
foreach ($sitemapDirectives->get() as $sitemapDirective) {
/* @var $sitemapDirective \webignition\RobotsTxt\Directive\Directive */
$sitemapUrl = new \webignition\Url\Url((string) $sitemapDirective->getValue());
if ($sitemapUrl->isRelative()) {
$absoluteUrlDeriver = new \webignition\AbsoluteUrlDeriver\AbsoluteUrlDeriver((string) $sitemapUrl, $this->getConfiguration()->getRootUrl());
$urls[] = (string) $absoluteUrlDeriver->getAbsoluteUrl();
} else {
$urls[] = (string) $sitemapUrl;
}
}
}
return $urls;
}
