function GetTokens($def)
{
$it = new InputStream($def);
$tokens = [];
while ($it->hasMore()) {
$it->skipWhiteSpace();
$value = $it->val();
switch ($value) {
case "{":
$tokens[] = NewToken($value, SyntaxMap::OpenBrace);
break;
case "}":
$tokens[] = NewToken($value, SyntaxMap::CloseBrace);
break;
case "(":
$tokens[] = NewToken($value, SyntaxMap::OpenParen);
break;
case ")":
$tokens[] = NewToken($value, SyntaxMap::CloseParen);
break;
case ",":
$tokens[] = NewToken($value, SyntaxMap::Comma);
break;
case ".":
$tokens[] = NewToken($value, SyntaxMap::Dot);
break;
case ";":
$tokens[] = NewToken($value, SyntaxMap::Semicolon);
break;
case "=":
$tokens[] = NewToken($value, SyntaxMap::AssignOp);
break;
case "?":
$tokens[] = NewToken($value, SyntaxMap::QuestionMark);
break;
case "\"":
$string = GetString($it);
$tokens[] = NewToken($string, SyntaxMap::String);
break;
case "/":
$it->next();
$value = $it->val();
if ($value == "*") {
GetComment($it);
} else {
if ($value == "/") {
GetSingleLineComment($it);
} else {
throw ParserException::unexpectedCharacter($value, $it);
}
}
break;
case "@":
$annotation = GetAnnotation($it);
$tokens[] = NewToken($annotation, SyntaxMap::Annotation);
break;
case ctype_digit($value):
$number = GetNumber($it);
$tokens[] = NewToken($number, SyntaxMap::Number);
break;
case ctype_alpha($value):
$value = GetIdentifier($it);
$v = strtolower($value);
if (in_array($v, SyntaxMap::$keywords)) {
$tokens[] = NewToken($value, SyntaxMap::Keyword);
} else {
if (in_array($v, SyntaxMap::$dataTypes)) {
$tokens[] = NewToken($value, SyntaxMap::DataType);
} else {
$tokens[] = NewToken($value, SyntaxMap::Identifier);
}
}
break;
default:
throw ParserException::unexpectedCharacter($value, $it);
}
if (count($tokens) > 0) {
$lastToken = $tokens[count($tokens) - 1];
if (!isset($lastToken->line)) {
$lastToken->line = $it->line;
}
}
$it->next();
}
return $tokens;
}
