/** * @param string $string * @return array * @throws Exception */ public function lex($string) { $tokens = array(); $offset = 0; while (isset($string[$offset])) { foreach (self::$tokenMap as $regex => $token) { if (preg_match($regex, $string, $matches, null, $offset)) { $tokens[] = TokenObject::create($token, trim($matches[0])); $offset += strlen($matches[0]); continue 2; } } throw new Exception(sprintf('Unexpected character: >%s< offset >%d<', $string[$offset], $offset)); } $tokens[] = TokenObject::create(Token::EOF, 'eof'); return $tokens; }