/**
* Unit test the Tokenizer - Tokenize and generate a HTML-compatible print-out of a comprehensive set of test cases
*/
static function UnitTestTokenizer()
{
// Comprehensive test cases for tokenizing
$tests = <<<EOD
String: "what about regular expressions, like for SSN (^\\d{3}-\\d{2}-\\d{4}) or US phone# ((?:\\(\\d{3}\\)\\s*\\d{3}-\\d{4})"
String: "Can strings contain embedded \\"quoted passages\\" (and parentheses + other characters?)?"
String: "can single quoted strings" . 'contain nested \\'quoted sections\\'?';
Parens: upcase('hello');
Numbers: 42 72.35 -15 +37 42A .5 0.7
And_Or: (this and that or the other); Sandles, sorting; (a && b || c)
Words: hi there, my name is C3PO!
UnaryOps: ++a, --b !b
BinaryOps: (a + b * c / d)
Comparators: > >= < <= == != gt ge lt le eq ne (target large gents built agile less equal)
Assign: = += -= *= /=
SGQA: 1X6X12 1X6X12ber1 1X6X12ber1_lab1 3583X84X249 12X3X5lab1_ber#1 1X6X12.NAOK 1X6X12ber1.NAOK 1X6X12ber1_lab1.NAOK 3583X84X249.NAOK 12X3X5lab1_ber#1.NAOK
Errors: Apt # 10C; (2 > 0) ? 'hi' : 'there'; array[30]; >>> <<< /* this is not a comment */ // neither is this
Words: q5pointChoice q5pointChoice.bogus q5pointChoice.code q5pointChoice.mandatory q5pointChoice.NAOK q5pointChoice.qid q5pointChoice.question q5pointChoice.relevance q5pointChoice.shown q5pointChoice.type
EOD;
$em = new ExpressionManager();
foreach (explode("\n", $tests) as $test) {
$tokens = $em->RDP_Tokenize($test);
print '<b>' . $test . '</b><hr/>';
print '<code>';
print implode("<br/>\n", explode("\n", print_r($tokens, TRUE)));
print '</code><hr/>';
}
}
