Exemplo n.º 1
0
 /**
  * Lexical scanner.
  * @param  string
  * @return void
  */
 private function tokenize($s)
 {
     if (!self::$regexp) {
         self::$regexp = '~' . implode('|', self::$patterns) . '~mA';
     }
     $s = str_replace("\r", '', $s);
     $s = strtr($s, "\t", ' ');
     $s = "\n" . $s . "\n";
     // first is required by "Indent", last is required by parse-error check
     $this->input = $s;
     $this->tokens = String::split($s, self::$regexp, PREG_SPLIT_NO_EMPTY);
     if (end($this->tokens) !== "\n") {
         // unable to parse
         $this->n = key($this->tokens);
         $this->error();
     }
 }
Exemplo n.º 2
0
 private function tokenize($s)
 {
     if (!self::$regexp) {
         self::$regexp = '~' . implode('|', self::$patterns) . '~mA';
     }
     $s = str_replace("\r", '', $s);
     $s = strtr($s, "\t", ' ');
     $s = "\n" . $s . "\n";
     $this->input = $s;
     $this->tokens = preg_split(self::$regexp, $s, -1, PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE);
     if (end($this->tokens) !== "\n") {
         $this->n = key($this->tokens);
         $this->error();
     }
 }