private function lex($input, ReflectionClass $reflectionClass)
{
$tokens = $this->tokenizer->tokenize($input);
$tokens = array_filter($tokens, function ($pair) {
return $pair[2] !== self::TOKEN_WHITESPACE && $pair[2] !== NULL;
});
$tokens = array_values($tokens);
$expanded = [];
foreach ($tokens as $token) {
if ($token[2] === self::TOKEN_STRING) {
$token[0] = stripslashes(substr($token[0], 1, -1));
$expanded[] = $token;
} elseif ($token[2] === self::TOKEN_KEYWORD) {
$values = $this->processKeyword($token[0], $reflectionClass);
if (is_array($values)) {
$count = count($values) - 1;
foreach ($values as $i => $value) {
$expanded[] = [$value, $token[1], $token[2]];
if ($i !== $count) {
$expanded[] = [',', -1, self::TOKEN_SEPARATOR];
}
}
} else {
$expanded[] = [$values, $token[1], $token[2]];
}
} else {
$expanded[] = $token;
}
}
return $expanded;
}
/**
*
* @param string $input
* @return \Nette\Utils\TokenIterator
*/
public static function scan($input)
{
$tokenizer = new Tokenizer(self::$patterns);
$tokens = $tokenizer->tokenize($input);
$iterator = new TokenIterator($tokens);
return $iterator;
}
/**
* @return array|NULL
* @throws CompositeTypeException
*/
public static function parse(string $input)
{
if ($input === NULL) {
return NULL;
}
$tokenizer = new Tokenizer([self::T_OPEN => '\\s*\\(', self::T_CLOSE => '\\)\\s*', self::T_SEPARATOR => ',', self::T_QUOTED_VALUE => '\\s*"(?:""|\\\\"|[^"])*"\\s*', self::T_VALUE => '[^,()"\\\\]+']);
try {
$tokens = $tokenizer->tokenize($input);
} catch (TokenizerException $e) {
throw CompositeTypeException::tokenizerFailure($input, $e);
}
list($value, $offset, $type) = $tokens[0];
if ($type !== self::T_OPEN) {
throw CompositeTypeException::openFailed($input);
}
list($values, $position) = self::innerParse($input, $tokens, 1);
if ($position !== count($tokens)) {
throw CompositeTypeException::mismatchedParens($input);
}
// A completely empty field value (no characters at all between the commas or parentheses) represents a NULL.
if ([] === array_filter($values, function ($value) {
return $value !== NULL;
})) {
return NULL;
}
return $values;
}
private function lex($input, ReflectionClass $reflectionClass)
{
try {
$tokens = $this->tokenizer->tokenize($input);
} catch (TokenizerException $e) {
throw new InvalidModifierDefinitionException('Unable to tokenize the modifier.', 0, $e);
}
$tokens = array_filter($tokens, function ($pair) {
return $pair[2] !== self::TOKEN_WHITESPACE && $pair[2] !== null;
});
$tokens = array_values($tokens);
$expanded = [];
foreach ($tokens as $token) {
if ($token[2] === self::TOKEN_STRING) {
$token[0] = stripslashes(substr($token[0], 1, -1));
$expanded[] = $token;
} elseif ($token[2] === self::TOKEN_KEYWORD) {
$values = $this->processKeyword($token[0], $reflectionClass);
if (is_array($values)) {
$count = count($values) - 1;
foreach ($values as $i => $value) {
$expanded[] = [$value, $token[1], $token[2]];
if ($i !== $count) {
$expanded[] = [',', -1, self::TOKEN_SEPARATOR];
}
}
} else {
$expanded[] = [$values, $token[1], $token[2]];
}
} else {
$expanded[] = $token;
}
}
return $expanded;
}
public function parse($input)
{
try {
$tokens = new TokenIterator($this->tokenizer->tokenize($input));
$tokens->ignored = [self::TOK_WHITESPACE];
} catch (Nette\Utils\TokenizerException $e) {
throw new Kdyby\Aop\ParserException("Input contains unexpected expressions", 0, $e);
}
return $this->doParse($tokens);
}
/**
* @param NULL|string $input
* @param callable $transform (mixed $partial) called for each item
* @return array|NULL
* @throws PostgresArrayException
*/
public static function parse($input, callable $transform)
{
if ($input === NULL) {
return NULL;
}
$tokenizer = new Tokenizer([self::T_OPEN => '\\s*\\{', self::T_CLOSE => '\\}\\s*', self::T_SEPARATOR => ',', self::T_QUOTED_VALUE => '\\s*"(?:\\\\"|[^"])*"\\s*', self::T_VALUE => '[^,{}"\\\\]+']);
try {
$tokens = $tokenizer->tokenize($input);
} catch (TokenizerException $e) {
throw PostgresArrayException::tokenizerFailure($e);
}
list($value, $offset, $type) = $tokens[0];
if ($type !== self::T_OPEN) {
throw PostgresArrayException::openFailed();
}
list($values, $position) = self::innerParse($tokens, $transform, 1);
if ($position !== count($tokens)) {
throw PostgresArrayException::mismatchedBrackets();
}
return $values;
}
public function __construct($input)
{
parent::__construct(array(self::T_WHITESPACE => '\\s+', self::T_COMMENT => '(?s)/\\*.*?\\*/', self::T_STRING => Parser::RE_STRING, self::T_KEYWORD => '(?:true|false|null|and|or|xor|clone|new|instanceof|return|continue|break|[A-Z_][A-Z0-9_]{2,})(?![\\w\\pL_])', self::T_CAST => '\\((?:expand|string|array|int|integer|float|bool|boolean|object)\\)', self::T_VARIABLE => '\\$[\\w\\pL_]+', self::T_NUMBER => '[+-]?[0-9]+(?:\\.[0-9]+)?(?:e[0-9]+)?', self::T_SYMBOL => '[\\w\\pL_]+(?:-[\\w\\pL_]+)*', self::T_CHAR => '::|=>|[^"\']'), 'u');
$this->ignored = array(self::T_COMMENT, self::T_WHITESPACE);
$this->tokenize($input);
}
private function error($message = "Unexpected '%s'")
{
list(, $line, $col) = self::$tokenizer->getOffset($this->n);
$token = isset(self::$tokenizer->tokens[$this->n]) ? str_replace("\n", '<new line>', Strings::truncate(self::$tokenizer->tokens[$this->n], 40)) : 'end';
throw new NeonException(str_replace('%s', $token, $message) . " on line {$line}, column {$col}.");
}
private function error($message = "Unexpected '%s'")
{
$last = isset($this->tokens[$this->n]) ? $this->tokens[$this->n] : NULL;
list($line, $col) = Tokenizer::getCoordinates($this->input, $last ? $last[Tokenizer::OFFSET] : strlen($this->input));
$token = $last ? str_replace("\n", '<new line>', Strings::truncate($last[0], 40)) : 'end';
throw new NeonException(str_replace('%s', $token, $message) . " on line {$line}, column {$col}.");
}
/**
* Tokenizer and preparser.
* @return array
*/
private function parseMacroArgs($input)
{
$this->tokenizer->tokenize($input);
$inTernary = $lastSymbol = $prev = NULL;
$tokens = $arrays = array();
$n = -1;
while (++$n < count($this->tokenizer->tokens)) {
$token = $this->tokenizer->tokens[$n];
$token['depth'] = $depth = count($arrays);
if ($token['type'] === self::T_COMMENT) {
continue;
// remove comments
} elseif ($token['type'] === self::T_WHITESPACE) {
$tokens[] = $token;
continue;
} elseif ($token['type'] === self::T_SYMBOL && ($prev === NULL || in_array($prev['value'], array(',', '(', '[', '=', '=>', ':', '?')))) {
$lastSymbol = count($tokens);
// quoting pre-requirements
} elseif (is_int($lastSymbol) && in_array($token['value'], array(',', ')', ']', '=', '=>', ':', '|'))) {
$tokens[$lastSymbol]['value'] = "'" . $tokens[$lastSymbol]['value'] . "'";
// quote symbols
$lastSymbol = NULL;
} else {
$lastSymbol = NULL;
}
if ($token['value'] === '?') {
// short ternary operators without :
$inTernary = $depth;
} elseif ($token['value'] === ':') {
$inTernary = NULL;
} elseif ($inTernary === $depth && ($token['value'] === ',' || $token['value'] === ')' || $token['value'] === ']')) {
// close ternary
$tokens[] = Tokenizer::createToken(':') + array('depth' => $depth);
$tokens[] = Tokenizer::createToken('null') + array('depth' => $depth);
$inTernary = NULL;
}
if ($token['value'] === '[') {
// simplified array syntax [...]
if ($arrays[] = $prev['value'] !== ']' && $prev['type'] !== self::T_SYMBOL && $prev['type'] !== self::T_VARIABLE) {
$tokens[] = Tokenizer::createToken('array') + array('depth' => $depth);
$token = Tokenizer::createToken('(');
}
} elseif ($token['value'] === ']') {
if (array_pop($arrays) === TRUE) {
$token = Tokenizer::createToken(')');
}
} elseif ($token['value'] === '(') {
// only count
$arrays[] = '(';
} elseif ($token['value'] === ')') {
// only count
array_pop($arrays);
}
$tokens[] = $prev = $token;
}
if (is_int($lastSymbol)) {
$tokens[$lastSymbol]['value'] = "'" . $tokens[$lastSymbol]['value'] . "'";
// quote symbols
}
if ($inTernary !== NULL) {
// close ternary
$tokens[] = Tokenizer::createToken(':') + array('depth' => count($arrays));
$tokens[] = Tokenizer::createToken('null') + array('depth' => count($arrays));
}
return $tokens;
}
public function parse($s)
{
self::$tokenizer = self::$tokenizer ?: new Nette\Utils\Tokenizer(array(self::T_WHITESPACE => '\\s+', self::T_COMMENT => '(?s)/\\*.*?\\*/', self::T_STRING => Parser::RE_STRING, self::T_KEYWORD => '(?:true|false|null|and|or|xor|clone|new|instanceof|return|continue|break|endif|endforeach|endwhile|endswitch|[A-Z_][A-Z0-9_]{2,})(?![\\w\\pL_])', self::T_CAST => '\\((?:expand|string|array|int|integer|float|bool|boolean|object)\\)', self::T_VARIABLE => '\\$[\\w\\pL_]+', self::T_NUMBER => '[+-]?[0-9]+(?:\\.[0-9]+)?(?:e[0-9]+)?', self::T_SYMBOL => '[\\w\\pL_]+(?:-[\\w\\pL_]+)*', self::T_CHAR => '::|=>|->|\\+\\+|--|<<|>>|<=|>=|===|!==|==|!=|<>|&&|\\|\\||[^"\']'), 'u');
return self::$tokenizer->tokenize($s);
}