/** * @dataProvider getMatcherData */ public function testMatcher(array $types, $content, array $expected) { $matcher = new Matcher($types); $matcher->build($content); $index = 0; while (null !== ($match = $matcher->fetch())) { $this->assertTrue(isset($expected[$match['position']])); $this->assertEquals($expected[$match['position']], $match['content']); $this->assertEquals($expected[$match['position']], $match['type']); $index++; } $this->assertEquals(count($expected), $index); }
/** * Tokenizes some content. * * @param string $string Content to tokenize * * @return array An array of tokens */ public function tokenize($string) { $tokens = array(); $start = 0; $length = strlen($string); $this->matcher->build($string); while ($length > $start) { $match = $this->matcher->fetch(); if (null === $match) { $tokens[] = new Token(Token::CONTENT, substr($string, $start)); return $tokens; } if ($match['position'] > $start) { $tokens[] = new Token(Token::CONTENT, substr($string, $start, $match['position'] - $start)); $start = $match['position']; } $tokens[] = new Token($match['type'], $match['content']); $start += $match['length']; } return $tokens; }