function testAssessmentTests(array $files, $validate = false)
{
$loaded = 0;
$totalSpent = 0;
foreach ($files as $f) {
$start = microtime();
$testDoc = new XmlDocument();
$testDoc->load($f, $validate);
$end = microtime();
$spent = spentTime($start, $end);
$totalSpent += $spent;
output("Test '" . pathinfo($f, PATHINFO_BASENAME) . "' loaded in " . sprintf("%.8f", $spent) . " seconds.");
$partCount = count($testDoc->getDocumentComponent()->getComponentsByClassName('testPart'));
$sectionCount = count($testDoc->getDocumentComponent()->getComponentsByClassName('assessmentSection'));
$itemCount = count($testDoc->getDocumentComponent()->getComponentsByClassName('assessmentItemRef'));
outputDescription("{$partCount} testPart(s), {$sectionCount} assessmentSection(s), {$itemCount} assessmentItemRef(s)");
outputDescription("Memory usage is " . memory_get_usage() / pow(1024, 2) . " MB");
output('');
$loaded++;
}
outputAverage($totalSpent / $loaded);
}
function doGrading($usercode, $TC)
{
$files = array();
if ($TC['showonly'] !== FALSE) {
$desired = explode(" ", $TC['showonly']);
foreach ($TC as $name => $value) {
if (substr($name, 0, 4) == "show") {
$TC[$name] = in_array(substr($name, 4), $desired) ? "Y" : "N";
}
}
}
if ($TC['answer'] !== FALSE) {
$TC['answer'] = ensureNewlineTerminated($TC['answer']);
}
$TC["inplace"] = booleanize($TC["inplace"]);
extract($TC);
// same as $showinput = $TC["showinput"], etc
$mainFile = "";
$er = FALSE;
$mainFile .= "from _UTILITIES import *\n";
$lang = 'en_US';
if (array_key_exists("lang", $_REQUEST) && preg_match("~^[a-zA-Z_]*\$~", $_REQUEST["lang"])) {
$lang = $_REQUEST["lang"];
}
$mainFile .= "_setLanguage('{$lang}')\n";
$inputMaker = inputMaker($TC);
$noInput = $inputMaker === FALSE;
$mainFile .= ($inputMaker === FALSE ? "_stdin=''" : $inputMaker) . "\n_stdincopy = open('stdincopy', 'w', encoding='utf-8')\nprint(_stdin, file=_stdincopy, end='')\n_stdincopy.close()\n";
if ($precode !== FALSE) {
$mainFile .= softSafeDereference($precode) . "\n";
}
$files['stdincopy'] = NULL;
$mainFile .= "import _GRADER\n";
$mainFile .= "_G = _GRADER\n";
global $inputInUse, $facultative;
if (!$inputInUse && ($inplace || $solver !== FALSE)) {
if ($solver !== FALSE) {
$files['solver'] = $solver;
}
if ($solver !== FALSE) {
$mainFile .= "_GRADER.globalsInitAndEcho(globals())\n";
} else {
$mainFile .= "_GRADER.globalsInitAndEcho(globals(), False)\n";
}
$files['graderreply'] = NULL;
$files['graderpre'] = NULL;
$files['solverstdout'] = NULL;
// run the solver before usercode, lest they mess up our globals.
$mainFile .= "_GRADER.runSolverWithTests()\n";
$testcode = "";
if ($rawtests !== FALSE) {
$testcode .= $rawtests . "\n";
}
if ($autotests != FALSE) {
$autotests = softSafeDereference($autotests);
$python_ident_regex = '(\\p{L}|\\p{Nl}|_)(\\p{L}|\\p{N}|\\p{Mn}|\\p{Mc}|\\p{Pc})*';
// note: this is close to, but not technically, 100% the
// same as the formal Python definition of an identifier
$py_regex_parens = 2;
foreach (explode("\n", $autotests) as $autotestline) {
if (preg_match('|^(\\s*)(\\S.*)$|', $autotestline, $matches) === 0) {
continue;
}
//skip blank lines
$indentation = $matches[1];
$command = trim($matches[2]);
if (1 == preg_match('@^' . $python_ident_regex . '$@u', $command)) {
//varname
$testcode .= $indentation . "_G.checkVar('{$command}')\n";
} elseif (1 == preg_match('@^(' . $python_ident_regex . ')\\s*\\((.*)\\)$@u', $command, $pieces)) {
if (strpos($pieces[2], $pieces[1]) === FALSE) {
// looks like a non-self-nested function call
$testcode .= $indentation . "_G.autotestCall('" . $pieces[1] . "',[" . $pieces[2 + $py_regex_parens] . "])\n";
} else {
// something more complex
$testcode .= $indentation . "_G.sayRunning(\"" . $command . "\")\n";
$testcode .= $indentation . "_G.autotestCompare(\"" . $command . "\", {$command})\n";
}
} else {
$testcode .= $autotestline . "\n";
}
// just leave it alone
}
}
$files['testcode'] = $testcode === FALSE ? "" : softSafeDereference($testcode) . "\n";
}
$mainFile .= '
_orig_std = (_sys.stdin, _sys.stdout)
_user_stdout = _StringIO()
_sys.stdout = _TeeOut(_user_stdout, _orig_std[1])
_sys.stdin = _StringIO(_stdin)
exec(compile(open(\'usercode\', encoding="utf-8").read(), ' . '\'usercode\', \'exec\'))
';
if (!$inputInUse) {
// lesson 18, part 2: may do this even if facultative
if ($inplace) {
$mainFile .= "exec(compile(open('testcode', encoding='utf-8').read()," . " 'testcode', 'exec'))\n";
$mainFile .= "_G.say('Y', 'noend')\n";
// success if none of the tests crash
}
}
// we've got all the user stdout necessary for testing
$mainFile .= '
__user_stdout = _user_stdout.getvalue()
_user_stdout.close()
(_sys.stdin, _sys.stdout) = _orig_std
';
if (!$facultative && !$inputInUse) {
if ($answer !== FALSE) {
$mainFile .= "_G._solver_stdout = " . pythonEscape(softSafeDereference($answer)) . "\n";
}
if ($grader !== '*nograder*' && ($answer !== FALSE || $solver !== FALSE)) {
$mainFile .= "_G.stdoutGrading(_stdin,__user_stdout,_G._solver_stdout, " . pythonEscape(softSafeDereference($grader)) . " )\n";
$files['stdoutgraderreply'] = NULL;
}
}
$testDescription = FALSE;
$files["usercode"] = $usercode;
global $usertni;
if ($inputInUse && $usertni) {
$mainFile .= "\n" . "exec(compile(open('usertests', encoding='utf-8').read(), " . "'usertests', 'exec'))\n";
global $userinput;
$files['usertests'] = $userinput;
}
$files["mainfile"] = $mainFile;
$userResult = safepython($files, "mainfile", "", $cpulimit);
extract($userResult);
// start printing stuff out now.
$m = '';
if ($testDescription != FALSE) {
$m .= $testDescription;
}
if (!$inputInUse && $inplace && trim($outdata['graderpre']) != '') {
$m .= '<i>' . __t('Before running your code:') . '</i> ' . $outdata['graderpre'] . '<br/>';
}
if ($showinput == "Y" && !$inputInUse && !$noInput && ($hideemptyinput == "N" || $outdata['stdincopy'] != "")) {
$m .= __t("Input:") . preBox($outdata['stdincopy']);
}
global $submit_code_stderr, $submit_code_errnice;
$submit_code_stderr = $stderr;
$submit_code_errnice = stderrNiceify($stderr);
if (userIsAdmin() && $stderrlen > 0) {
$m .= JQpopUp("Debug: view unsanitized", preBox($stderr, $stderrlen));
}
if ($stderr == '') {
$errnice = '';
} else {
$errnice = '<p>' . __t('Error messages: ') . preBoxHinted(stderrNiceify($stderr), $stderrlen) . '</p>';
}
if ($ok) {
$m .= "<p>" . __t('Program executed without crashing.') . "</p>";
} elseif (firstLine($safeexecOut) == 'Command exited with non-zero status (1)') {
$m .= "<p>" . __t("Program crashed.") . "</p>";
} else {
$m .= "<p>" . __t("Program crashed — ") . firstLine($safeexecOut) . ".</p>";
}
if (1 === 2) {
// these lines are just to trick gettext
__t("Memory Limit Exceeded") . __t("Time Limit Exceeded") . __t("Command exited with non-zero status") . __t("Command terminated by signal") . __t("Output Limit Exceeded") . __t("Invalid Function") . __t("Internal Error");
}
if ($showsafeexec == "Y") {
$m .= "Sandbox messages:" . preBox($safeexecOut);
}
$simpleOutputDescription = outputDescription(NULL, array('showoutput' => $showoutput, 'stdoutlen' => $stdoutlen, 'hideemptyoutput' => $hideemptyoutput, 'stdout' => $stdout, 'ok' => $ok));
if ($desirederror !== FALSE) {
$m .= $simpleOutputDescription;
$lines = explode("\n", trim($userResult["stderr"]));
$goodFail = count($lines) > 0 && $lines[count($lines) - 1] == $desirederror;
$m .= $errnice;
return $goodFail ? tcpass($m) : tcfail($m);
}
if (!$ok || $facultative) {
// we don't care what's in stdout
$graderreply = trim(getSoft($outdata, 'graderreply', ''));
if (!$ok && !$inputInUse && $graderreply != '') {
$m .= "<i>" . __t("The grader said:") . "</i>" . "<div>" . $graderreply . "</div>";
} elseif ($inplace && $solver === FALSE & $graderreply != '') {
$m .= "<i>" . __t("Automatic tests:") . "</i>" . "<div>" . substr($graderreply, 0, -1) . "</div>";
}
$m .= $errnice . $simpleOutputDescription;
return $ok ? tcpass($m) : tcfail($m);
}
if ($inplace) {
// don't care what's in stdout, unless solverstdout != ''
$GR = $outdata['graderreply'];
$inplaceresult = substr($GR, -1);
$inplacereply = substr($GR, 0, -1);
if ($inplacereply != '') {
$inplacereply = "<i>" . __t("The grader said:") . "</i>" . "<div>{$inplacereply}</div>";
}
if ($inplaceresult == 'Y') {
if ($outdata['solverstdout'] == '') {
return tcpass($m . $inplacereply . $errnice . $simpleOutputDescription);
}
} elseif ($inplaceresult == 'N') {
return tcfail($m . $inplacereply . $errnice . $simpleOutputDescription);
}
$m .= $inplacereply;
// carry on and let the stdout grader do its thing
}
// the user's code did not crash. what did the stdout grader say?
$outGraderReply = $outdata['stdoutgraderreply'];
if ($outGraderReply == "" || !($outGraderReply[0] == "Y" || $outGraderReply[0] == "N")) {
throw new PyboxException("Grader error 2 [" . $outGraderReply . '|' . $outdata['graderreply'] . "|" . ord(substr($outdata['graderreply'], -1)) . "| {$m} ]");
}
$outinfo = array('stdout' => $stdout, 'stdoutlen' => $stdoutlen, 'requiredStdout' => getSoft($outdata, 'solverstdout', $answer), 'showoutput' => $showoutput, 'showexpected' => $showexpected, 'grader' => $grader);
$m .= outputDescription($outGraderReply[0] == "Y", $outinfo) . $errnice;
if (strlen(trim($outGraderReply)) > 1) {
$m .= "<p>" . __t("Result of grading: ") . substr($outGraderReply, 1) . "</p>";
}
return $outGraderReply[0] == "Y" ? tcpass($m) : tcfail($m);
}