$tokenizer->filename = $filename;
return $tokenizer;
}
/**
* Tokenize a string
*
* @param string $str String to tokenize
* @return Tokenizer
*/
public static function tokenizeString($str)
{
/* PHP sometimes warns with "Unexpected character in input"
* when tokenizing a file, especially when accidentally
* tokenizing images.
*/
$tokens = @token_get_all($str);
$tokenizer = new Tokenizer($tokens);
return $tokenizer;
}
/**
* Returns true if the currentToken index exists
*
* Used by the Iterator interface
*/
public function valid()
{
return isset($this->tokens[$this->currentToken]);
}
}
Tokenizer::initialize();
