/**
* This method return a tuple representing the token discovered.
*
* @access public
* @param \Unicity\IO\Reader $reader the reader to be used
* @return \Unicity\Lexer\Scanner\Tuple a tuple representing the token
* discovered
*/
public function process(IO\Reader $reader)
{
$char = $reader->readChar($reader->position(), false);
if ($char !== null && $char == $this->delimiter) {
$tuple = new Lexer\Scanner\Tuple(Lexer\Scanner\TokenType::delimiter(), new Common\String($char));
$reader->skip(1);
return $tuple;
}
return null;
}
/**
* This method return a tuple representing the token discovered.
*
* @access public
* @param \Unicity\IO\Reader $reader the reader to be used
* @return \Unicity\Lexer\Scanner\Tuple a tuple representing the token
* discovered
*/
public function process(IO\Reader $reader)
{
$index = $reader->position();
$char = $reader->readChar($index, false);
if ($char !== null && in_array($char, $this->whitespace)) {
$lookahead = $index;
do {
$lookahead++;
$next = $reader->readChar($lookahead, false);
} while ($next !== null && in_array($next, $this->whitespace));
$token = $reader->readRange($index, $lookahead);
$tuple = new Lexer\Scanner\Tuple(Lexer\Scanner\TokenType::whitespace(), new Common\String($token));
return $tuple;
}
return null;
}
/**
* This method return a tuple representing the token discovered.
*
* @access public
* @param \Unicity\IO\Reader $reader the reader to be used
* @return \Unicity\Lexer\Scanner\Tuple a tuple representing the token
* discovered
*/
public function process(IO\Reader $reader)
{
$index = $reader->position();
$char = $reader->readChar($index, false);
if ($char !== null && !$this->blacklist->hasValue($char)) {
$lookahead = $index;
do {
$lookahead++;
$next = $reader->readChar($lookahead, false);
} while ($next !== null && !$this->blacklist->hasValue($next));
$token = $reader->readRange($index, $lookahead);
$tuple = new Lexer\Scanner\Tuple(Lexer\Scanner\TokenType::keyword(), new Common\String($token));
return $tuple;
}
return null;
}
/**
* This destructor ensures that any resources are properly disposed.
*
* @access public
*/
public function __destruct()
{
parent::__destruct();
unset($this->buffer);
unset($this->length);
unset($this->position);
}
/**
* This method return a tuple representing the token discovered.
*
* @access public
* @param \Unicity\IO\Reader $reader the reader to be used
* @return \Unicity\Lexer\Scanner\Tuple a tuple representing the token
* discovered
*/
public function process(IO\Reader $reader)
{
$index = $reader->position();
$char = $reader->readChar($index, false);
if ($char !== null && preg_match('/^[_a-z]$/i', $char)) {
$lookahead = $index;
do {
$lookahead++;
$next = $reader->readChar($lookahead, false);
} while ($next !== null && preg_match('/^[_a-z0-9]$/i', $next));
$token = $reader->readRange($index, $lookahead);
$type = $this->keywords->hasValue($token) ? Lexer\Scanner\TokenType::keyword() : Lexer\Scanner\TokenType::identifier();
$tuple = new Lexer\Scanner\Tuple($type, new Common\String($token));
return $tuple;
}
return null;
}
/**
* This method return a tuple representing the token discovered.
*
* @access public
* @param \Unicity\IO\Reader $reader the reader to be used
* @return \Unicity\Lexer\Scanner\Tuple a tuple representing the token
* discovered
*/
public function process(IO\Reader $reader)
{
$index = $reader->position();
$char = $reader->readChar($index, false);
if ($char !== null && $char >= '0' && $char <= '9') {
// "integer" token, "real" token, or "hexadecimal" token
$type = null;
$lookahead = $index;
if ($char == '0') {
$lookahead++;
$next = $reader->readChar($lookahead, false);
if ($next == 'x' or $next == 'X') {
do {
$lookahead++;
$next = $reader->readChar($lookahead, false);
} while ($next !== null && $next >= '0' && $next <= '9');
$type = Lexer\Scanner\TokenType::hexadecimal();
} else {
if ($next == '.') {
do {
$lookahead++;
$next = $reader->readChar($lookahead, false);
} while ($next !== null && $next >= '0' && $next <= '9');
$type = Lexer\Scanner\TokenType::real();
} else {
$type = Lexer\Scanner\TokenType::integer();
}
}
} else {
$next = null;
do {
$lookahead++;
$next = $reader->readChar($lookahead, false);
} while ($next !== null && $next >= '0' && $next <= '9');
if ($next == '.') {
do {
$lookahead++;
$next = $reader->readChar($lookahead, false);
} while ($next !== null && $next >= '0' && $next <= '9');
$type = Lexer\Scanner\TokenType::real();
} else {
$type = Lexer\Scanner\TokenType::integer();
}
}
$token = $reader->readRange($index, $lookahead);
$tuple = new Lexer\Scanner\Tuple($type, new Common\String($token));
return $tuple;
}
return null;
}
/**
* This method return a tuple representing the token discovered.
*
* @access public
* @param \Unicity\IO\Reader $reader the reader to be used
* @return \Unicity\Lexer\Scanner\Tuple a tuple representing the token
* discovered
*/
public function process(IO\Reader $reader)
{
$index = $reader->position();
$char = $reader->readChar($index, false);
if ($char == $this->quotation) {
$lookahead = $index + 1;
$length = $reader->length() - 1;
while ($lookahead <= $length) {
if ($reader->readChar($lookahead, false) == $this->quotation) {
if ($lookahead == $length || $reader->readChar($lookahead + 1, false) != $this->quotation) {
$lookahead++;
break;
}
$lookahead++;
}
$lookahead++;
}
$token = $reader->readRange($index, $lookahead);
$tuple = new Lexer\Scanner\Tuple(Lexer\Scanner\TokenType::literal(), new Common\String($token));
return $tuple;
}
return null;
}
/**
* This method rewinds the iterator back to the starting position.
*
* @access public
*/
public function rewind()
{
$this->current = null;
$this->reader->seek(0);
}
/**
* This destructor ensures that any resources are properly disposed.
*
* @access public
*/
public function __destruct()
{
parent::__destruct();
unset($this->file);
unset($this->handle);
}
