/** * Unit test the Tokenizer - Tokenize and generate a HTML-compatible print-out of a comprehensive set of test cases */ static function UnitTestTokenizer() { // Comprehensive test cases for tokenizing $tests = <<<EOD String: "what about regular expressions, like for SSN (^\\d{3}-\\d{2}-\\d{4}) or US phone# ((?:\\(\\d{3}\\)\\s*\\d{3}-\\d{4})" String: "Can strings contain embedded \\"quoted passages\\" (and parentheses + other characters?)?" String: "can single quoted strings" . 'contain nested \\'quoted sections\\'?'; Parens: upcase('hello'); Numbers: 42 72.35 -15 +37 42A .5 0.7 And_Or: (this and that or the other); Sandles, sorting; (a && b || c) Words: hi there, my name is C3PO! UnaryOps: ++a, --b !b BinaryOps: (a + b * c / d) Comparators: > >= < <= == != gt ge lt le eq ne (target large gents built agile less equal) Assign: = += -= *= /= SGQA: 1X6X12 1X6X12ber1 1X6X12ber1_lab1 3583X84X249 12X3X5lab1_ber#1 1X6X12.NAOK 1X6X12ber1.NAOK 1X6X12ber1_lab1.NAOK 3583X84X249.NAOK 12X3X5lab1_ber#1.NAOK Errors: Apt # 10C; (2 > 0) ? 'hi' : 'there'; array[30]; >>> <<< /* this is not a comment */ // neither is this Words: q5pointChoice q5pointChoice.bogus q5pointChoice.code q5pointChoice.mandatory q5pointChoice.NAOK q5pointChoice.qid q5pointChoice.question q5pointChoice.relevance q5pointChoice.shown q5pointChoice.type EOD; $em = new ExpressionManager(); foreach (explode("\n", $tests) as $test) { $tokens = $em->RDP_Tokenize($test); print '<b>' . $test . '</b><hr/>'; print '<code>'; print implode("<br/>\n", explode("\n", print_r($tokens, TRUE))); print '</code><hr/>'; } }