/** * Gets the used variables from the provided format * @param string $format Data format * @return array Array with the variable strings */ private function getVariablesFromFormat($format) { $symbol = new NestedSymbol(self::SYMBOL_OPEN, self::SYMBOL_CLOSE, null, true); $tokenizer = new Tokenizer(); $tokenizer->addSymbol($symbol); $tokens = $tokenizer->tokenize($format); $variables = array(); $isVariableOpen = false; $variableFormat = null; foreach ($tokens as $token) { if ($token == self::SYMBOL_OPEN) { $isVariableOpen = true; continue; } if (!$isVariableOpen) { continue; } if ($token == self::SYMBOL_CLOSE) { $variables[$variableFormat] = new DataFormatVariable($variableFormat); $isVariableOpen = false; $variableFormat = null; } else { $variableFormat .= $token; } } return $variables; }
/** * Constructs a new condition tokenizer * @return null */ public function __construct() { $this->addSymbol(new SimpleSymbol(Condition::OPERATOR_AND)); $this->addSymbol(new SimpleSymbol(Condition::OPERATOR_OR)); $this->addSymbol(new ConditionSymbol($this)); parent::setWillTrimTokens(true); }
public function tokenize($string) { $tokens = parent::tokenize($string); if (count($tokens) !== 1) { return $tokens; } if (substr($string, 0, 1) == MathematicalExpression::NESTED_OPEN && substr($string, -1) == MathematicalExpression::NESTED_CLOSE) { $tokens = $this->tokenize(substr($string, 1, -1)); } return $tokens; }
/** * Tokenizes the provided string * @param string $string String to tokenize * @return array Array with the tokens of this string as value */ public function tokenize($string) { $tokens = array(); $parentTokens = parent::tokenize($string); $currentToken = ''; foreach ($parentTokens as $token) { if ($token === self::FIELD_SEPARATOR) { $tokens[] = $currentToken; $currentToken = ''; continue; } $currentToken .= ($currentToken ? ' ' : '') . $token; } if ($currentToken) { $tokens[] = $currentToken; } return $tokens; }
/** * Checks for this symbol in the string which is being tokenized. * @param string $inProcess Current part of the string which is being tokenized. * @param string $toProcess Remaining part of the string which has not yet been tokenized * @return null|array Null when the symbol was not found, an array with the processed tokens if the symbol was found. */ public function tokenize(&$process, $toProcess) { $processLength = strlen($process); if ($processLength < $this->symbolOpenLength || substr($process, $this->symbolOpenOffset) != $this->symbolOpen) { return null; } $positionOpen = $processLength - $this->symbolOpenLength; $positionClose = $this->getClosePosition($toProcess, $positionOpen); $lengthProcess = strlen($process) + $positionOpen; $before = substr($process, 0, $positionOpen); if (!$this->allowsSymbolsBeforeOpen && trim($before)) { return null; } $between = substr($toProcess, $positionOpen + $this->symbolOpenLength, $positionOpen + $positionClose - $lengthProcess); $process .= $between . $this->symbolClose; if ($this->tokenizer !== null) { $between = $this->tokenizer->tokenize($between); } if ($this->willIncludeSymbols) { return array($before, $this->symbolOpen, $between, $this->symbolClose); } return array($before, $between); }
/** * Tokenizes the tag content in argument tokens * * eg. img=150x100 alt="Image description" width=150 * array { * "img=150x100", * "alt=Image Description", * "width=150", * } * * @param string $tagContent * @return array */ private function tokenize($tagContent) { $positionApo = strpos($tagContent, '"'); if (!$positionApo) { return explode(' ', $tagContent); } $tokens = array(); $tokenizer = new Tokenizer(); $tokenizer->addSymbol(new SimpleSymbol(' ', false)); $tokenizer->addSymbol(new NestedSymbol('"', '"', null, true)); $isStringArgument = false; $previousToken = null; $parsedTokens = $tokenizer->tokenize($tagContent); foreach ($parsedTokens as $token) { if ($token == '"') { $isStringArgument = !$isStringArgument; continue; } if ($isStringArgument) { $tokens[count($tokens) - 1] .= $token; } else { $tokens[] = $token; } } return $tokens; }