Beispiel #1
0
 private function lex($input, ReflectionClass $reflectionClass)
 {
     $tokens = $this->tokenizer->tokenize($input);
     $tokens = array_filter($tokens, function ($pair) {
         return $pair[2] !== self::TOKEN_WHITESPACE && $pair[2] !== NULL;
     });
     $tokens = array_values($tokens);
     $expanded = [];
     foreach ($tokens as $token) {
         if ($token[2] === self::TOKEN_STRING) {
             $token[0] = stripslashes(substr($token[0], 1, -1));
             $expanded[] = $token;
         } elseif ($token[2] === self::TOKEN_KEYWORD) {
             $values = $this->processKeyword($token[0], $reflectionClass);
             if (is_array($values)) {
                 $count = count($values) - 1;
                 foreach ($values as $i => $value) {
                     $expanded[] = [$value, $token[1], $token[2]];
                     if ($i !== $count) {
                         $expanded[] = [',', -1, self::TOKEN_SEPARATOR];
                     }
                 }
             } else {
                 $expanded[] = [$values, $token[1], $token[2]];
             }
         } else {
             $expanded[] = $token;
         }
     }
     return $expanded;
 }
Beispiel #2
0
 /**
  * 
  * @param string $input
  * @return \Nette\Utils\TokenIterator
  */
 public static function scan($input)
 {
     $tokenizer = new Tokenizer(self::$patterns);
     $tokens = $tokenizer->tokenize($input);
     $iterator = new TokenIterator($tokens);
     return $iterator;
 }
 /**
  * @return array|NULL
  * @throws CompositeTypeException
  */
 public static function parse(string $input)
 {
     if ($input === NULL) {
         return NULL;
     }
     $tokenizer = new Tokenizer([self::T_OPEN => '\\s*\\(', self::T_CLOSE => '\\)\\s*', self::T_SEPARATOR => ',', self::T_QUOTED_VALUE => '\\s*"(?:""|\\\\"|[^"])*"\\s*', self::T_VALUE => '[^,()"\\\\]+']);
     try {
         $tokens = $tokenizer->tokenize($input);
     } catch (TokenizerException $e) {
         throw CompositeTypeException::tokenizerFailure($input, $e);
     }
     list($value, $offset, $type) = $tokens[0];
     if ($type !== self::T_OPEN) {
         throw CompositeTypeException::openFailed($input);
     }
     list($values, $position) = self::innerParse($input, $tokens, 1);
     if ($position !== count($tokens)) {
         throw CompositeTypeException::mismatchedParens($input);
     }
     // A completely empty field value (no characters at all between the commas or parentheses) represents a NULL.
     if ([] === array_filter($values, function ($value) {
         return $value !== NULL;
     })) {
         return NULL;
     }
     return $values;
 }
Beispiel #4
0
 private function lex($input, ReflectionClass $reflectionClass)
 {
     try {
         $tokens = $this->tokenizer->tokenize($input);
     } catch (TokenizerException $e) {
         throw new InvalidModifierDefinitionException('Unable to tokenize the modifier.', 0, $e);
     }
     $tokens = array_filter($tokens, function ($pair) {
         return $pair[2] !== self::TOKEN_WHITESPACE && $pair[2] !== null;
     });
     $tokens = array_values($tokens);
     $expanded = [];
     foreach ($tokens as $token) {
         if ($token[2] === self::TOKEN_STRING) {
             $token[0] = stripslashes(substr($token[0], 1, -1));
             $expanded[] = $token;
         } elseif ($token[2] === self::TOKEN_KEYWORD) {
             $values = $this->processKeyword($token[0], $reflectionClass);
             if (is_array($values)) {
                 $count = count($values) - 1;
                 foreach ($values as $i => $value) {
                     $expanded[] = [$value, $token[1], $token[2]];
                     if ($i !== $count) {
                         $expanded[] = [',', -1, self::TOKEN_SEPARATOR];
                     }
                 }
             } else {
                 $expanded[] = [$values, $token[1], $token[2]];
             }
         } else {
             $expanded[] = $token;
         }
     }
     return $expanded;
 }
Beispiel #5
0
 public function parse($input)
 {
     try {
         $tokens = new TokenIterator($this->tokenizer->tokenize($input));
         $tokens->ignored = [self::TOK_WHITESPACE];
     } catch (Nette\Utils\TokenizerException $e) {
         throw new Kdyby\Aop\ParserException("Input contains unexpected expressions", 0, $e);
     }
     return $this->doParse($tokens);
 }
 /**
  * @param NULL|string $input
  * @param callable $transform (mixed $partial) called for each item
  * @return array|NULL
  * @throws PostgresArrayException
  */
 public static function parse($input, callable $transform)
 {
     if ($input === NULL) {
         return NULL;
     }
     $tokenizer = new Tokenizer([self::T_OPEN => '\\s*\\{', self::T_CLOSE => '\\}\\s*', self::T_SEPARATOR => ',', self::T_QUOTED_VALUE => '\\s*"(?:\\\\"|[^"])*"\\s*', self::T_VALUE => '[^,{}"\\\\]+']);
     try {
         $tokens = $tokenizer->tokenize($input);
     } catch (TokenizerException $e) {
         throw PostgresArrayException::tokenizerFailure($e);
     }
     list($value, $offset, $type) = $tokens[0];
     if ($type !== self::T_OPEN) {
         throw PostgresArrayException::openFailed();
     }
     list($values, $position) = self::innerParse($tokens, $transform, 1);
     if ($position !== count($tokens)) {
         throw PostgresArrayException::mismatchedBrackets();
     }
     return $values;
 }
 public function __construct($input)
 {
     parent::__construct(array(self::T_WHITESPACE => '\\s+', self::T_COMMENT => '(?s)/\\*.*?\\*/', self::T_STRING => Parser::RE_STRING, self::T_KEYWORD => '(?:true|false|null|and|or|xor|clone|new|instanceof|return|continue|break|[A-Z_][A-Z0-9_]{2,})(?![\\w\\pL_])', self::T_CAST => '\\((?:expand|string|array|int|integer|float|bool|boolean|object)\\)', self::T_VARIABLE => '\\$[\\w\\pL_]+', self::T_NUMBER => '[+-]?[0-9]+(?:\\.[0-9]+)?(?:e[0-9]+)?', self::T_SYMBOL => '[\\w\\pL_]+(?:-[\\w\\pL_]+)*', self::T_CHAR => '::|=>|[^"\']'), 'u');
     $this->ignored = array(self::T_COMMENT, self::T_WHITESPACE);
     $this->tokenize($input);
 }
Beispiel #8
0
 private function error($message = "Unexpected '%s'")
 {
     list(, $line, $col) = self::$tokenizer->getOffset($this->n);
     $token = isset(self::$tokenizer->tokens[$this->n]) ? str_replace("\n", '<new line>', Strings::truncate(self::$tokenizer->tokens[$this->n], 40)) : 'end';
     throw new NeonException(str_replace('%s', $token, $message) . " on line {$line}, column {$col}.");
 }
Beispiel #9
0
 private function error($message = "Unexpected '%s'")
 {
     $last = isset($this->tokens[$this->n]) ? $this->tokens[$this->n] : NULL;
     list($line, $col) = Tokenizer::getCoordinates($this->input, $last ? $last[Tokenizer::OFFSET] : strlen($this->input));
     $token = $last ? str_replace("\n", '<new line>', Strings::truncate($last[0], 40)) : 'end';
     throw new NeonException(str_replace('%s', $token, $message) . " on line {$line}, column {$col}.");
 }
 /**
  * Tokenizer and preparser.
  * @return array
  */
 private function parseMacroArgs($input)
 {
     $this->tokenizer->tokenize($input);
     $inTernary = $lastSymbol = $prev = NULL;
     $tokens = $arrays = array();
     $n = -1;
     while (++$n < count($this->tokenizer->tokens)) {
         $token = $this->tokenizer->tokens[$n];
         $token['depth'] = $depth = count($arrays);
         if ($token['type'] === self::T_COMMENT) {
             continue;
             // remove comments
         } elseif ($token['type'] === self::T_WHITESPACE) {
             $tokens[] = $token;
             continue;
         } elseif ($token['type'] === self::T_SYMBOL && ($prev === NULL || in_array($prev['value'], array(',', '(', '[', '=', '=>', ':', '?')))) {
             $lastSymbol = count($tokens);
             // quoting pre-requirements
         } elseif (is_int($lastSymbol) && in_array($token['value'], array(',', ')', ']', '=', '=>', ':', '|'))) {
             $tokens[$lastSymbol]['value'] = "'" . $tokens[$lastSymbol]['value'] . "'";
             // quote symbols
             $lastSymbol = NULL;
         } else {
             $lastSymbol = NULL;
         }
         if ($token['value'] === '?') {
             // short ternary operators without :
             $inTernary = $depth;
         } elseif ($token['value'] === ':') {
             $inTernary = NULL;
         } elseif ($inTernary === $depth && ($token['value'] === ',' || $token['value'] === ')' || $token['value'] === ']')) {
             // close ternary
             $tokens[] = Tokenizer::createToken(':') + array('depth' => $depth);
             $tokens[] = Tokenizer::createToken('null') + array('depth' => $depth);
             $inTernary = NULL;
         }
         if ($token['value'] === '[') {
             // simplified array syntax [...]
             if ($arrays[] = $prev['value'] !== ']' && $prev['type'] !== self::T_SYMBOL && $prev['type'] !== self::T_VARIABLE) {
                 $tokens[] = Tokenizer::createToken('array') + array('depth' => $depth);
                 $token = Tokenizer::createToken('(');
             }
         } elseif ($token['value'] === ']') {
             if (array_pop($arrays) === TRUE) {
                 $token = Tokenizer::createToken(')');
             }
         } elseif ($token['value'] === '(') {
             // only count
             $arrays[] = '(';
         } elseif ($token['value'] === ')') {
             // only count
             array_pop($arrays);
         }
         $tokens[] = $prev = $token;
     }
     if (is_int($lastSymbol)) {
         $tokens[$lastSymbol]['value'] = "'" . $tokens[$lastSymbol]['value'] . "'";
         // quote symbols
     }
     if ($inTernary !== NULL) {
         // close ternary
         $tokens[] = Tokenizer::createToken(':') + array('depth' => count($arrays));
         $tokens[] = Tokenizer::createToken('null') + array('depth' => count($arrays));
     }
     return $tokens;
 }
 public function parse($s)
 {
     self::$tokenizer = self::$tokenizer ?: new Nette\Utils\Tokenizer(array(self::T_WHITESPACE => '\\s+', self::T_COMMENT => '(?s)/\\*.*?\\*/', self::T_STRING => Parser::RE_STRING, self::T_KEYWORD => '(?:true|false|null|and|or|xor|clone|new|instanceof|return|continue|break|endif|endforeach|endwhile|endswitch|[A-Z_][A-Z0-9_]{2,})(?![\\w\\pL_])', self::T_CAST => '\\((?:expand|string|array|int|integer|float|bool|boolean|object)\\)', self::T_VARIABLE => '\\$[\\w\\pL_]+', self::T_NUMBER => '[+-]?[0-9]+(?:\\.[0-9]+)?(?:e[0-9]+)?', self::T_SYMBOL => '[\\w\\pL_]+(?:-[\\w\\pL_]+)*', self::T_CHAR => '::|=>|->|\\+\\+|--|<<|>>|<=|>=|===|!==|==|!=|<>|&&|\\|\\||[^"\']'), 'u');
     return self::$tokenizer->tokenize($s);
 }