/**
*
* @param string $s
* @return array
*/
protected function tokenize($s)
{
$tokens = $this->lexer->tokenize($s);
// Remove white_space tokens
$tokensReal = array();
foreach ($tokens as $token) {
if ($token['name'] !== 'white_space') {
$tokensReal[] = $token;
}
}
return $tokensReal;
}
public function testBigTest()
{
$source = <<<'EOT'
<<false
true/Name/Name2
123
43445
+17
-98
0
//Name<</Other#20Name
34.5
-3.62
+123.6
4.
-.002
0.0
/
false<<
/ /
>>
>>
EOT;
$expectedTokens = array('white_space', 'dictionary_start', 'boolean', 'white_space', 'boolean', 'name', 'name', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'name', 'name', 'dictionary_start', 'name', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'name', 'white_space', 'boolean', 'dictionary_start', 'white_space', 'name', 'white_space', 'name', 'white_space', 'dictionary_end', 'white_space', 'dictionary_end', 'white_space');
$lexer = new Lexer();
$tokens = $lexer->tokenize($source);
$this->assertEquals($expectedTokens, $this->pullTokenNames($tokens));
}
