/** * * @param string $s * @return array */ protected function tokenize($s) { $tokens = $this->lexer->tokenize($s); // Remove white_space tokens $tokensReal = array(); foreach ($tokens as $token) { if ($token['name'] !== 'white_space') { $tokensReal[] = $token; } } return $tokensReal; }
/** * * @param string $source * @param int $startPos * @return array */ public function getNextObjectFromString($source, $startPos) { $nextToken = $this->lexer->getNextTokenFromString($source, $startPos); while ($nextToken->tokenName === 'white_space') { $startPos = $nextToken->newPos; $nextToken = $this->lexer->getNextTokenFromString($source, $startPos); } $value = ''; $endPos = 0; switch ($nextToken->tokenName) { case 'numeric': case 'string': case 'boolean': case 'null': // value is $token content, expected are white space (optional) + endobj $value = substr($source, $startPos, $nextToken->newPos - $startPos); $endPos = $nextToken->newPos; break; case 'dictionary_start': // Dictionary. Find next balanced dictionary_end and use // DictionaryParser to parse the tokens into a PdfDictionary // object $dictionaryIndent = 1; $tmpPos = $nextToken->newPos; $mytoken = $this->lexer->getNextTokenFromString($source, $tmpPos); while ($mytoken->tokenName !== 'dictionary_end' || $dictionaryIndent > 0) { if ($mytoken->tokenName === 'dictionary_start') { $dictionaryIndent++; } elseif ($mytoken->tokenName === 'dictionary_end') { $dictionaryIndent--; } if ($dictionaryIndent === 0 && $mytoken->tokenName === 'dictionary_end') { break; } $tmpPos = $mytoken->newPos; $mytoken = $this->lexer->getNextTokenFromString($source, $tmpPos); } $value = $this->dictionaryParser->parse(substr($source, $startPos, $mytoken->newPos - $startPos)); $endPos = $mytoken->newPos; break; } return array($endPos, $value); }
public function testBigTest() { $source = <<<'EOT' <<false true/Name/Name2 123 43445 +17 -98 0 //Name<</Other#20Name 34.5 -3.62 +123.6 4. -.002 0.0 / false<< / / >> >> EOT; $expectedTokens = array('white_space', 'dictionary_start', 'boolean', 'white_space', 'boolean', 'name', 'name', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'name', 'name', 'dictionary_start', 'name', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'numeric', 'white_space', 'name', 'white_space', 'boolean', 'dictionary_start', 'white_space', 'name', 'white_space', 'name', 'white_space', 'dictionary_end', 'white_space', 'dictionary_end', 'white_space'); $lexer = new Lexer(); $tokens = $lexer->tokenize($source); $this->assertEquals($expectedTokens, $this->pullTokenNames($tokens)); }