Пример #1
0
 /**
  * {@inheritDoc}
  */
 public function lex($string)
 {
     // normalize line endings
     $string = strtr($string, array("\r\n" => "\n", "\r" => "\n"));
     $tokens = array();
     $position = 0;
     $originalString = $string;
     $originalLength = Util::stringLength($string);
     while (true) {
         $token = $this->extractToken($string);
         if ($token === null) {
             break;
         }
         if (!$this->shouldSkipToken($token)) {
             $tokens[] = $token;
         }
         $shift = Util::stringLength($token->getValue());
         $position += $shift;
         // update line + offset
         if ($position > 0) {
             $this->line = substr_count($originalString, "\n", 0, $position) + 1;
         }
         $string = Util::substring($string, $shift);
     }
     if ($position !== $originalLength) {
         throw new RecognitionException($this->line);
     }
     $tokens[] = new CommonToken(Parser::EOF_TOKEN_TYPE, '', $this->line);
     return new ArrayTokenStream($tokens);
 }
Пример #2
0
 /**
  * {@inheritDoc}
  */
 protected function extractToken($string)
 {
     $value = $type = null;
     foreach ($this->recognizers as $t => $recognizer) {
         if ($recognizer->match($string, $v)) {
             if ($value === null || Util::stringLength($v) > Util::stringLength($value)) {
                 $value = $v;
                 $type = $t;
             }
         }
     }
     if ($type !== null) {
         return new CommonToken($type, $value, $this->getCurrentLine());
     }
     return null;
 }
Пример #3
0
 /**
  * {@inheritDoc}
  */
 protected function extractToken($string)
 {
     if (empty($this->stateStack)) {
         throw new LogicException("You must set a starting state before lexing.");
     }
     $value = $type = $action = null;
     $state = $this->states[$this->stateStack[count($this->stateStack) - 1]];
     foreach ($state['recognizers'] as $t => $recognizer) {
         if ($recognizer->match($string, $v)) {
             if ($value === null || Util::stringLength($v) > Util::stringLength($value)) {
                 $value = $v;
                 $type = $t;
                 $action = $state['actions'][$type];
             }
         }
     }
     if ($type !== null) {
         if (is_string($action)) {
             // enter new state
             $this->stateStack[] = $action;
         } elseif ($action === self::POP_STATE) {
             array_pop($this->stateStack);
         }
         return new CommonToken($type, $value, $this->getCurrentLine());
     }
     return null;
 }