/** * This method return a tuple representing the token discovered. * * @access public * @param \Unicity\IO\Reader $reader the reader to be used * @return \Unicity\Lexer\Scanner\Tuple a tuple representing the token * discovered */ public function process(IO\Reader $reader) { $index = $reader->position(); $char = $reader->readChar($index, false); if ($char !== null && in_array($char, $this->whitespace)) { $lookahead = $index; do { $lookahead++; $next = $reader->readChar($lookahead, false); } while ($next !== null && in_array($next, $this->whitespace)); $token = $reader->readRange($index, $lookahead); $tuple = new Lexer\Scanner\Tuple(Lexer\Scanner\TokenType::whitespace(), new Common\String($token)); return $tuple; } return null; }
/** * This method return a tuple representing the token discovered. * * @access public * @param \Unicity\IO\Reader $reader the reader to be used * @return \Unicity\Lexer\Scanner\Tuple a tuple representing the token * discovered */ public function process(IO\Reader $reader) { $index = $reader->position(); $char = $reader->readChar($index, false); if ($char !== null && !$this->blacklist->hasValue($char)) { $lookahead = $index; do { $lookahead++; $next = $reader->readChar($lookahead, false); } while ($next !== null && !$this->blacklist->hasValue($next)); $token = $reader->readRange($index, $lookahead); $tuple = new Lexer\Scanner\Tuple(Lexer\Scanner\TokenType::keyword(), new Common\String($token)); return $tuple; } return null; }
/** * This method return a tuple representing the token discovered. * * @access public * @param \Unicity\IO\Reader $reader the reader to be used * @return \Unicity\Lexer\Scanner\Tuple a tuple representing the token * discovered */ public function process(IO\Reader $reader) { $index = $reader->position(); $char = $reader->readChar($index, false); if ($char !== null && preg_match('/^[_a-z]$/i', $char)) { $lookahead = $index; do { $lookahead++; $next = $reader->readChar($lookahead, false); } while ($next !== null && preg_match('/^[_a-z0-9]$/i', $next)); $token = $reader->readRange($index, $lookahead); $type = $this->keywords->hasValue($token) ? Lexer\Scanner\TokenType::keyword() : Lexer\Scanner\TokenType::identifier(); $tuple = new Lexer\Scanner\Tuple($type, new Common\String($token)); return $tuple; } return null; }
/** * This method return a tuple representing the token discovered. * * @access public * @param \Unicity\IO\Reader $reader the reader to be used * @return \Unicity\Lexer\Scanner\Tuple a tuple representing the token * discovered */ public function process(IO\Reader $reader) { $char = $reader->readChar($reader->position(), false); if ($char !== null && $char == $this->delimiter) { $tuple = new Lexer\Scanner\Tuple(Lexer\Scanner\TokenType::delimiter(), new Common\String($char)); $reader->skip(1); return $tuple; } return null; }
/** * This method return a tuple representing the token discovered. * * @access public * @param \Unicity\IO\Reader $reader the reader to be used * @return \Unicity\Lexer\Scanner\Tuple a tuple representing the token * discovered */ public function process(IO\Reader $reader) { $index = $reader->position(); $char = $reader->readChar($index, false); if ($char == $this->quotation) { $lookahead = $index + 1; $length = $reader->length() - 1; while ($lookahead <= $length) { if ($reader->readChar($lookahead, false) == $this->quotation) { if ($lookahead == $length || $reader->readChar($lookahead + 1, false) != $this->quotation) { $lookahead++; break; } $lookahead++; } $lookahead++; } $token = $reader->readRange($index, $lookahead); $tuple = new Lexer\Scanner\Tuple(Lexer\Scanner\TokenType::literal(), new Common\String($token)); return $tuple; } return null; }
/** * This method return a tuple representing the token discovered. * * @access public * @param \Unicity\IO\Reader $reader the reader to be used * @return \Unicity\Lexer\Scanner\Tuple a tuple representing the token * discovered */ public function process(IO\Reader $reader) { $index = $reader->position(); $char = $reader->readChar($index, false); if ($char !== null && $char >= '0' && $char <= '9') { // "integer" token, "real" token, or "hexadecimal" token $type = null; $lookahead = $index; if ($char == '0') { $lookahead++; $next = $reader->readChar($lookahead, false); if ($next == 'x' or $next == 'X') { do { $lookahead++; $next = $reader->readChar($lookahead, false); } while ($next !== null && $next >= '0' && $next <= '9'); $type = Lexer\Scanner\TokenType::hexadecimal(); } else { if ($next == '.') { do { $lookahead++; $next = $reader->readChar($lookahead, false); } while ($next !== null && $next >= '0' && $next <= '9'); $type = Lexer\Scanner\TokenType::real(); } else { $type = Lexer\Scanner\TokenType::integer(); } } } else { $next = null; do { $lookahead++; $next = $reader->readChar($lookahead, false); } while ($next !== null && $next >= '0' && $next <= '9'); if ($next == '.') { do { $lookahead++; $next = $reader->readChar($lookahead, false); } while ($next !== null && $next >= '0' && $next <= '9'); $type = Lexer\Scanner\TokenType::real(); } else { $type = Lexer\Scanner\TokenType::integer(); } } $token = $reader->readRange($index, $lookahead); $tuple = new Lexer\Scanner\Tuple($type, new Common\String($token)); return $tuple; } return null; }