/**
* @param string $content
* @param array $tokens
*
* @dataProvider provideTokens
*/
public function testTokenizer($content, array $tokens)
{
$resource = fopen(new Stream($content), "r");
$reader = new Tokenizer($resource);
foreach ($tokens as $token) {
$token['token'] = $this->toTokenCode($token['token']);
$this->assertEquals($token, $reader->next());
}
fclose($resource);
}
/**
*
*/
protected function next()
{
return $this->token = $this->tokenizer->next();
}