/** * Parse test results from phpunit json report * * @param string $path Path to test * @param string $test_results String containing phpunit json report * * @return array */ public function parseTestResults($path, $test_results) { $report = $this->getJsonReport($test_results); // coverage is for all testcases in the executed $path $coverage = array(); if ($this->enableCoverage !== false) { $coverage = $this->readCoverage(); } $results = array(); foreach ($report as $event) { if ('test' != $event->event) { continue; } $status = ArcanistUnitTestResult::RESULT_PASS; $user_data = ''; if ('fail' == $event->status) { $status = ArcanistUnitTestResult::RESULT_FAIL; $user_data .= $event->message . "\n"; foreach ($event->trace as $trace) { $user_data .= sprintf("\n%s:%s", $trace->file, $trace->line); } } else { if ('error' == $event->status) { if (strpos($event->message, 'Skipped Test') !== false) { $status = ArcanistUnitTestResult::RESULT_SKIP; $user_data .= $event->message; } else { if (strpos($event->message, 'Incomplete Test') !== false) { $status = ArcanistUnitTestResult::RESULT_SKIP; $user_data .= $event->message; } else { $status = ArcanistUnitTestResult::RESULT_BROKEN; $user_data .= $event->message; foreach ($event->trace as $trace) { $user_data .= sprintf("\n%s:%s", $trace->file, $trace->line); } } } } } $name = preg_replace('/ \\(.*\\)/', '', $event->test); $result = new ArcanistUnitTestResult(); $result->setName($name); $result->setResult($status); $result->setDuration($event->time); $result->setCoverage($coverage); $result->setUserData($user_data); $results[] = $result; } return $results; }
public function run() { $results = array(); $build_start = microtime(true); $config_manager = $this->getConfigurationManager(); if ($this->getEnableCoverage() !== false) { $command = $config_manager->getConfigFromAnySource('unit.engine.tap.cover'); } else { $command = $config_manager->getConfigFromAnySource('unit.engine.tap.command'); } $timeout = $config_manager->getConfigFromAnySource('unit.engine.tap.timeout'); if (!$timeout) { $timeout = 15; } $future = new ExecFuture('%C', $command); $future->setTimeout($timeout); $result = new ArcanistUnitTestResult(); $result->setName($command ? $command : 'unknown'); try { list($stdout, $stderr) = $future->resolvex(); $result->setResult(ArcanistUnitTestResult::RESULT_PASS); if ($this->getEnableCoverage() !== false) { $coverage = $this->readCoverage('coverage/cobertura-coverage.xml'); $result->setCoverage($coverage); } } catch (CommandException $exc) { $result->setResult(ArcanistUnitTestResult::RESULT_FAIL); if ($future->getWasKilledByTimeout()) { print "Process stdout:\n" . $exc->getStdout() . "\nProcess stderr:\n" . $exc->getStderr() . "\nExceeded timeout of {$timeout} secs.\nMake unit tests faster."; } else { $result->setUserdata($exc->getStdout() . $exc->getStderr()); } } $result->setDuration(microtime(true) - $build_start); $results[] = $result; return $results; }
function generateCoverageResults() { // Find all executables, not just the test executables. // We need to do this because the test executables may not // include all of the possible code (linker decides to omit // it from the image) so we see a skewed representation of // the source lines. $fp = popen("find {$this->repo_root} -type f -name watchman -o " . "-name \\*.a -o -name \\*.so -o -name \\*.dylib", "r"); while (true) { $line = fgets($fp); if ($line === false) { break; } $obj_files[] = trim($line); } // Parse line information from the objects foreach ($obj_files as $object) { DwarfLineInfo::loadObject($object); } $CG = new CallgrindFile((string) $this->cg_file); $CG->parse(); $source_files = array(); foreach ($CG->getSourceFiles() as $filename) { if (Filesystem::isDescendant($filename, $this->repo_root) && !preg_match("/(thirdparty|tests)/", $filename)) { $source_files[$filename] = $filename; } } $cov = array(); foreach ($source_files as $filename) { $relsrc = substr($filename, strlen($this->repo_root) + 1); $cov[$relsrc] = CallgrindFile::mergeSourceLineData($filename, array($CG)); } $res = new ArcanistUnitTestResult(); $res->setName('coverage'); $res->setUserData("Collected"); $res->setResult(ArcanistUnitTestResult::RESULT_PASS); $res->setCoverage($cov); // Stash it for review with our `arc cov` command $wc = ArcanistWorkingCopyIdentity::newFromPath($this->repo_root); $api = ArcanistRepositoryAPI::newAPIFromWorkingCopyIdentity($wc); $api->writeScratchFile('wman-cov.json', json_encode($cov)); return array($res); }
/** * Parses the test results from xUnit. * * @param string The name of the xUnit results file. * @param string The name of the coverage file if one was provided by * `buildTestFuture`. This is passed through to * `parseCoverageResult`. * @return array Test results. */ private function parseTestResult($xunit_tmp, $coverage) { $xunit_dom = new DOMDocument(); $xunit_dom->loadXML(Filesystem::readFile($xunit_tmp)); $results = array(); $tests = $xunit_dom->getElementsByTagName('test'); foreach ($tests as $test) { $name = $test->getAttribute('name'); $time = $test->getAttribute('time'); $status = ArcanistUnitTestResult::RESULT_UNSOUND; switch ($test->getAttribute('result')) { case 'Pass': $status = ArcanistUnitTestResult::RESULT_PASS; break; case 'Fail': $status = ArcanistUnitTestResult::RESULT_FAIL; break; case 'Skip': $status = ArcanistUnitTestResult::RESULT_SKIP; break; } $userdata = ''; $reason = $test->getElementsByTagName('reason'); $failure = $test->getElementsByTagName('failure'); if ($reason->length > 0 || $failure->length > 0) { $node = $reason->length > 0 ? $reason : $failure; $message = $node->item(0)->getElementsByTagName('message'); if ($message->length > 0) { $userdata = $message->item(0)->nodeValue; } $stacktrace = $node->item(0)->getElementsByTagName('stack-trace'); if ($stacktrace->length > 0) { $userdata .= "\n" . $stacktrace->item(0)->nodeValue; } } $result = new ArcanistUnitTestResult(); $result->setName($name); $result->setResult($status); $result->setDuration($time); $result->setUserData($userdata); if ($coverage != null) { $result->setCoverage($this->parseCoverageResult($coverage)); } $results[] = $result; } return $results; }
private final function resultTest($test_result, $reason) { $coverage = $this->endCoverage(); $result = new ArcanistUnitTestResult(); $result->setCoverage($coverage); $result->setNamespace(get_class($this)); $result->setName($this->runningTest); $result->setLink($this->getLink($this->runningTest)); $result->setResult($test_result); $result->setDuration(microtime(true) - $this->testStartTime); $result->setUserData($reason); $this->results[] = $result; if ($this->renderer) { echo $this->renderer->renderUnitResult($result); } }
/** * Mark the current running test as skipped. * * @param string Description for why this test was skipped. * @return void * @task internal */ private final function skipTest($reason) { $coverage = $this->endCoverage(); $result = new ArcanistUnitTestResult(); $result->setCoverage($coverage); $result->setName($this->runningTest); $result->setResult(ArcanistUnitTestResult::RESULT_SKIP); $result->setDuration(microtime(true) - $this->testStartTime); $result->setUserData($reason); $this->results[] = $result; }
public function parseTestResults($path, $xunit_tmp, $cover_tmp) { // xunit xsd: https://gist.github.com/959290 $xunit_dom = new DOMDocument(); $xunit_dom->loadXML(Filesystem::readFile($xunit_tmp)); // coverage is for all testcases in the executed $path $coverage = array(); if ($this->getEnableCoverage() !== false) { $coverage = $this->readCoverage($cover_tmp); } $results = array(); $testcases = $xunit_dom->getElementsByTagName("testcase"); foreach ($testcases as $testcase) { $classname = $testcase->getAttribute("classname"); $name = $testcase->getAttribute("name"); $time = $testcase->getAttribute("time"); $status = ArcanistUnitTestResult::RESULT_PASS; $user_data = ""; // A skipped test is a test which was ignored using framework // mechanizms (e.g. @skip decorator) $skipped = $testcase->getElementsByTagName("skipped"); if ($skipped->length > 0) { $status = ArcanistUnitTestResult::RESULT_SKIP; $messages = array(); for ($ii = 0; $ii < $skipped->length; $ii++) { $messages[] = trim($skipped->item($ii)->nodeValue, " \n"); } $user_data .= implode("\n", $messages); } // Failure is a test which the code has explicitly failed by using // the mechanizms for that purpose. e.g., via an assertEquals $failures = $testcase->getElementsByTagName("failure"); if ($failures->length > 0) { $status = ArcanistUnitTestResult::RESULT_FAIL; $messages = array(); for ($ii = 0; $ii < $failures->length; $ii++) { $messages[] = trim($failures->item($ii)->nodeValue, " \n"); } $user_data .= implode("\n", $messages) . "\n"; } // An errored test is one that had an unanticipated problem. e.g., an // unchecked throwable, or a problem with an implementation of the // test. $errors = $testcase->getElementsByTagName("error"); if ($errors->length > 0) { $status = ArcanistUnitTestResult::RESULT_BROKEN; $messages = array(); for ($ii = 0; $ii < $errors->length; $ii++) { $messages[] = trim($errors->item($ii)->nodeValue, " \n"); } $user_data .= implode("\n", $messages) . "\n"; } $result = new ArcanistUnitTestResult(); $result->setName($classname . "." . $name); $result->setResult($status); $result->setDuration($time); $result->setCoverage($coverage); $result->setUserData($user_data); $results[] = $result; } return $results; }
/** * Parse test results from phpunit json report * * @param string $path Path to test * @param string $test_results String containing phpunit json report * * @return array */ public function parseTestResults($path, $test_results) { if (!$test_results) { $result = id(new ArcanistUnitTestResult())->setName($path)->setUserData($this->stderr)->setResult(ArcanistUnitTestResult::RESULT_BROKEN); return array($result); } $report = $this->getJsonReport($test_results); // coverage is for all testcases in the executed $path $coverage = array(); if ($this->enableCoverage !== false) { $coverage = $this->readCoverage(); } $last_test_finished = true; $results = array(); foreach ($report as $event) { switch (idx($event, 'event')) { case 'test': break; case 'testStart': $last_test_finished = false; // fall through // fall through default: continue 2; // switch + loop } $status = ArcanistUnitTestResult::RESULT_PASS; $user_data = ''; if ('fail' == idx($event, 'status')) { $status = ArcanistUnitTestResult::RESULT_FAIL; $user_data .= idx($event, 'message') . "\n"; foreach (idx($event, 'trace') as $trace) { $user_data .= sprintf("\n%s:%s", idx($trace, 'file'), idx($trace, 'line')); } } else { if ('error' == idx($event, 'status')) { if (strpos(idx($event, 'message'), 'Skipped Test') !== false) { $status = ArcanistUnitTestResult::RESULT_SKIP; $user_data .= idx($event, 'message'); } else { if (strpos(idx($event, 'message'), 'Incomplete Test') !== false) { $status = ArcanistUnitTestResult::RESULT_SKIP; $user_data .= idx($event, 'message'); } else { $status = ArcanistUnitTestResult::RESULT_BROKEN; $user_data .= idx($event, 'message'); foreach (idx($event, 'trace') as $trace) { $user_data .= sprintf("\n%s:%s", idx($trace, 'file'), idx($trace, 'line')); } } } } } $name = preg_replace('/ \\(.*\\)/s', '', idx($event, 'test')); $result = new ArcanistUnitTestResult(); $result->setName($name); $result->setResult($status); $result->setDuration(idx($event, 'time')); $result->setCoverage($coverage); $result->setUserData($user_data); $results[] = $result; $last_test_finished = true; } if (!$last_test_finished) { $results[] = id(new ArcanistUnitTestResult())->setName(idx($event, 'test'))->setUserData($this->stderr)->setResult(ArcanistUnitTestResult::RESULT_BROKEN); } return $results; }
private function processCoverageResults($project_root, $results) { $time_start = microtime_float(); // generate annotated source files to find out which lines have // coverage // limit files to only those "*.py" files in getPaths() $pythonPathsStr = join(" ", $this->getPythonPaths()); $future = new ExecFuture("coverage annotate {$pythonPathsStr}"); $future->setCWD($project_root); try { $future->resolvex(); } catch (CommandException $exc) { if ($exc->getError() > 1) { // 'nose' returns 1 when tests are failing/broken. throw $exc; } } // store all the coverage results for this project $coverageArray = array(); $lines_total = 0; $lines_executable = 0; $lines_not_executable = 0; $lines_covered = 0; $lines_not_covered = 0; // walk through project directory, searching for all ",cover" files // that coverage.py left behind foreach (new RecursiveIteratorIterator(new RecursiveDirectoryIterator(".")) as $path) { // paths are given as "./path/to/file.py,cover", so match the // "path/to/file.py" part if (!preg_match(":^\\./(.*),cover\$:", $path, $matches)) { continue; } $srcFilePath = $matches[1]; $coverageStr = ""; foreach (file($path) as $coverLine) { /* python coverage > executed ! missing(not executed) - excluded phab coverage N Not executable. This is a comment or whitespace which should be ignored when computing test coverage. C Covered. This line has test coverage. U Uncovered. This line is executable but has no test coverage. X Unreachable. If your coverage analysis can detect unreachable code, you can report it here. */ $lines_total++; switch ($coverLine[0]) { case '>': $lines_covered++; $lines_executable++; $coverageStr .= 'C'; break; case '!': $lines_not_covered++; $lines_executable++; $coverageStr .= 'U'; break; case ' ': $lines_not_executable++; $coverageStr .= 'N'; break; case '-': $coverageStr .= 'X'; break; default: break; } } // delete the ,cover file unlink($path); // only add to coverage report if the path was originally // specified by arc if (in_array($srcFilePath, $this->getPaths())) { $coverageArray[$srcFilePath] = $coverageStr; } } $lines_percentage = bcdiv($lines_covered, $lines_executable, 4) * 100; $time_end = microtime_float(); $time = $time_end - $time_start; // python does not support per-test coverage results so just put all the coverage // in a single 'coverage test' $result = new ArcanistUnitTestResult(); $result->setNamespace('coverage'); $result->setName('coverage'); $result->setResult('pass'); $result->setDuration($time); $result->setUserData("coverage: {$lines_percentage}% executable: {$lines_executable} / covered: {$lines_covered}"); $result->setCoverage($coverageArray); $results[] = $result; return $results; }