public function testAdditionalTokenizerFeatures()
{
// These tests all pass with the old tokenizer, they were developed wile
// working on the patch
$tokenizer = new Doctrine_Query_Tokenizer();
$delimiters = array(' ', '+', '-', '*', '/', '<', '>', '=', '>=', '<=', '&', '|');
$res = $tokenizer->bracketExplode("(age < 20 AND age > 18) AND email LIKE '*****@*****.**'", ' AND ', '(', ')');
$this->assertEqual($res, array("(age < 20 AND age > 18)", "email LIKE '*****@*****.**'"));
$res = $tokenizer->sqlExplode("sentence OR 'term'", ' OR ');
$this->assertEqual($res, array("sentence", "'term'"));
$res = $tokenizer->clauseExplode("'a + b'+c", $delimiters);
$this->assertEqual($res, array(array("'a + b'", '+'), array('c', '')));
$res = $tokenizer->quoteExplode('"a"."b"', ' ');
$this->assertEqual($res, array('"a"."b"'));
}
