/**
* @dataProvider getMatcherData
*/
public function testMatcher(array $types, $content, array $expected)
{
$matcher = new Matcher($types);
$matcher->build($content);
$index = 0;
while (null !== ($match = $matcher->fetch())) {
$this->assertTrue(isset($expected[$match['position']]));
$this->assertEquals($expected[$match['position']], $match['content']);
$this->assertEquals($expected[$match['position']], $match['type']);
$index++;
}
$this->assertEquals(count($expected), $index);
}
/**
* Tokenizes some content.
*
* @param string $string Content to tokenize
*
* @return array An array of tokens
*/
public function tokenize($string)
{
$tokens = array();
$start = 0;
$length = strlen($string);
$this->matcher->build($string);
while ($length > $start) {
$match = $this->matcher->fetch();
if (null === $match) {
$tokens[] = new Token(Token::CONTENT, substr($string, $start));
return $tokens;
}
if ($match['position'] > $start) {
$tokens[] = new Token(Token::CONTENT, substr($string, $start, $match['position'] - $start));
$start = $match['position'];
}
$tokens[] = new Token($match['type'], $match['content']);
$start += $match['length'];
}
return $tokens;
}