/**
* @link https://help.yandex.ru/webmaster/controlling-robot/robots-txt.xml#clean-param
*
* @dataProvider generateDataForTest
* @covers RobotsTxtParser::isDisallowed
* @covers RobotsTxtParser::checkRule
* @param string $robotsTxtContent
*/
public function testCleanParam($robotsTxtContent, $message = NULL)
{
// init parser
$parser = new RobotsTxtParser($robotsTxtContent);
$rules = $parser->getRules();
$this->assertInstanceOf('RobotsTxtParser', $parser);
$this->assertObjectHasAttribute('rules', $parser);
$this->assertArrayHasKey('*', $rules);
$this->assertArrayHasKey('clean-param', $rules['*']);
$this->assertEquals(array('utm_source&utm_medium&utm.campaign'), $rules['*']['clean-param'], $message);
}
/**
* @dataProvider generateDataForTest
* @covers RobotsTxtParser::isDisallowed
* @covers RobotsTxtParser::checkRule
* @param string $robotsTxtContent
*/
public function testCrawlDelay($robotsTxtContent)
{
// init parser
$parser = new RobotsTxtParser($robotsTxtContent);
$rules = $parser->getRules();
$this->assertInstanceOf('RobotsTxtParser', $parser);
$this->assertObjectHasAttribute('rules', $parser);
$this->assertArrayHasKey('ahrefsbot', $rules);
$this->assertArrayHasKey('crawl-delay', $rules['ahrefsbot']);
$this->assertEquals(1.5, $rules['ahrefsbot']['crawl-delay']);
}
/**
* @dataProvider generateDataForTest
* @covers RobotsTxtParser::isDisallowed
* @covers RobotsTxtParser::checkRule
* @param string $robotsTxtContent
*/
public function testHost($robotsTxtContent)
{
// init parser
$parser = new RobotsTxtParser($robotsTxtContent);
$rules = $parser->getRules();
$this->assertInstanceOf('RobotsTxtParser', $parser);
$this->assertObjectHasAttribute('rules', $parser);
$this->assertArrayHasKey('*', $rules);
$this->assertArrayHasKey('host', $rules['*']);
$this->assertEquals('www.example.com', $rules['*']['host']);
}