PHP RobotsTxtParser::isAllowed示例

示例#1

文件： DisallowAll.php 项目： shounakgupte/Robots.txt-Parser-Class

 /**
  * @link https://github.com/t1gor/Robots.txt-Parser-Class/issues/22
  */
 public function testAllowWildcard()
 {
     // init parser
     $parser = new RobotsTxtParser("\n\t\t\tUser-agent: *\n\t\t\tAllow: /\n\t\t");
     // asserts
     $this->assertFalse($parser->isDisallowed("/index"));
     $this->assertFalse($parser->isDisallowed("/"));
     $this->assertTrue($parser->isAllowed("/index"));
     $this->assertTrue($parser->isAllowed("/"));
 }

示例#2

文件： wordpress-site-check.php 项目： sturple/wordpress-site-check

function get_robots_tester()
{
    $root = (!empty($_SERVER['HTTPS']) ? 'https' : 'http') . '://' . $_SERVER['HTTP_HOST'] . '/';
    if (class_exists('\\RobotsTxtParser')) {
        $parser = new RobotsTxtParser(file_get_contents($root . 'robots.txt'));
        $parser->setUserAgent('GoogleBot');
        $status = $parser->isAllowed('/') ? 'Enable' : 'Blocked';
        $output = '<tr>';
        $output .= '<td>' . 'Robots Checking ' . $root . 'robots.txt' . '</td><td>' . ' Enable ' . '</td>';
        $output .= '<td  style="color:' . ($parser->isAllowed('/') ? 'green' : 'red') . ';">' . $status . '</td>';
        $output .= '<td>' . ($parser->isAllowed('/') ? 'Passed' : 'FAILED') . '</td>';
        $output .= '</tr>';
        echo $output;
    }
}

示例#3

文件： robots_parser.php 项目： travy/web_crawler

 public function allowCrawl($url)
 {
     $file = robot_parser::getRobotFile($url);
     $parser = new RobotsTxtParser($file);
     return $parser->isAllowed($url);
 }

示例#4

文件： EmptyRulesShouldAllowEverythingTest.php 项目： shounakgupte/Robots.txt-Parser-Class

 /**
  * @covder RobotsTxtParser::checkRule
  * @link https://github.com/t1gor/Robots.txt-Parser-Class/issues/23
  */
 public function testEmptyRulesAllow()
 {
     $parser = new RobotsTxtParser('');
     $this->assertTrue($parser->isAllowed('/foo'));
     $this->assertFalse($parser->isDisallowed('/foo'));
 }