Esempio n. 1
0
 /**
  * @dataProvider getMatcherData
  */
 public function testMatcher(array $types, $content, array $expected)
 {
     $matcher = new Matcher($types);
     $matcher->build($content);
     $index = 0;
     while (null !== ($match = $matcher->fetch())) {
         $this->assertTrue(isset($expected[$match['position']]));
         $this->assertEquals($expected[$match['position']], $match['content']);
         $this->assertEquals($expected[$match['position']], $match['type']);
         $index++;
     }
     $this->assertEquals(count($expected), $index);
 }
Esempio n. 2
0
 /**
  * Tokenizes some content.
  *
  * @param string $string Content to tokenize
  *
  * @return array An array of tokens
  */
 public function tokenize($string)
 {
     $tokens = array();
     $start = 0;
     $length = strlen($string);
     $this->matcher->build($string);
     while ($length > $start) {
         $match = $this->matcher->fetch();
         if (null === $match) {
             $tokens[] = new Token(Token::CONTENT, substr($string, $start));
             return $tokens;
         }
         if ($match['position'] > $start) {
             $tokens[] = new Token(Token::CONTENT, substr($string, $start, $match['position'] - $start));
             $start = $match['position'];
         }
         $tokens[] = new Token($match['type'], $match['content']);
         $start += $match['length'];
     }
     return $tokens;
 }