$tokenizer->filename = $filename; return $tokenizer; } /** * Tokenize a string * * @param string $str String to tokenize * @return Tokenizer */ public static function tokenizeString($str) { /* PHP sometimes warns with "Unexpected character in input" * when tokenizing a file, especially when accidentally * tokenizing images. */ $tokens = @token_get_all($str); $tokenizer = new Tokenizer($tokens); return $tokenizer; } /** * Returns true if the currentToken index exists * * Used by the Iterator interface */ public function valid() { return isset($this->tokens[$this->currentToken]); } } Tokenizer::initialize();