/**
*
* @param string $s
* @return array
*/
protected function tokenize($s)
{
$tokens = $this->lexer->tokenize($s);
// Remove white_space tokens
$tokensReal = array();
foreach ($tokens as $token) {
if ($token['name'] !== 'white_space') {
$tokensReal[] = $token;
}
}
return $tokensReal;
}
/**
*
* @param string $source
* @param int $startPos
* @return array
*/
public function getNextObjectFromString($source, $startPos)
{
$nextToken = $this->lexer->getNextTokenFromString($source, $startPos);
while ($nextToken->tokenName === 'white_space') {
$startPos = $nextToken->newPos;
$nextToken = $this->lexer->getNextTokenFromString($source, $startPos);
}
$value = '';
$endPos = 0;
switch ($nextToken->tokenName) {
case 'numeric':
case 'string':
case 'boolean':
case 'null':
// value is $token content, expected are white space (optional) + endobj
$value = substr($source, $startPos, $nextToken->newPos - $startPos);
$endPos = $nextToken->newPos;
break;
case 'dictionary_start':
// Dictionary. Find next balanced dictionary_end and use
// DictionaryParser to parse the tokens into a PdfDictionary
// object
$dictionaryIndent = 1;
$tmpPos = $nextToken->newPos;
$mytoken = $this->lexer->getNextTokenFromString($source, $tmpPos);
while ($mytoken->tokenName !== 'dictionary_end' || $dictionaryIndent > 0) {
if ($mytoken->tokenName === 'dictionary_start') {
$dictionaryIndent++;
} elseif ($mytoken->tokenName === 'dictionary_end') {
$dictionaryIndent--;
}
if ($dictionaryIndent === 0 && $mytoken->tokenName === 'dictionary_end') {
break;
}
$tmpPos = $mytoken->newPos;
$mytoken = $this->lexer->getNextTokenFromString($source, $tmpPos);
}
$value = $this->dictionaryParser->parse(substr($source, $startPos, $mytoken->newPos - $startPos));
$endPos = $mytoken->newPos;
break;
}
return array($endPos, $value);
}
public function testBigTest()
{
$source = <<<'EOT'
<<false
true/Name/Name2
123
43445
+17
-98
0
//Name<</Other#20Name
34.5
-3.62
+123.6
4.
-.002
0.0
/
false<<
/ /
>>
>>
EOT;
$expectedTokens = array('white_space', 'dictionary_start', 'boolean', 'white_space', 'boolean', 'name', 'name', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'name', 'name', 'dictionary_start', 'name', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'name', 'white_space', 'boolean', 'dictionary_start', 'white_space', 'name', 'white_space', 'name', 'white_space', 'dictionary_end', 'white_space', 'dictionary_end', 'white_space');
$lexer = new Lexer();
$tokens = $lexer->tokenize($source);
$this->assertEquals($expectedTokens, $this->pullTokenNames($tokens));
}
