/**
* Parse test results from phpunit json report
*
* @param string $path Path to test
* @param string $test_results String containing phpunit json report
*
* @return array
*/
public function parseTestResults($path, $test_results)
{
$report = $this->getJsonReport($test_results);
// coverage is for all testcases in the executed $path
$coverage = array();
if ($this->enableCoverage !== false) {
$coverage = $this->readCoverage();
}
$results = array();
foreach ($report as $event) {
if ('test' != $event->event) {
continue;
}
$status = ArcanistUnitTestResult::RESULT_PASS;
$user_data = '';
if ('fail' == $event->status) {
$status = ArcanistUnitTestResult::RESULT_FAIL;
$user_data .= $event->message . "\n";
foreach ($event->trace as $trace) {
$user_data .= sprintf("\n%s:%s", $trace->file, $trace->line);
}
} else {
if ('error' == $event->status) {
if (strpos($event->message, 'Skipped Test') !== false) {
$status = ArcanistUnitTestResult::RESULT_SKIP;
$user_data .= $event->message;
} else {
if (strpos($event->message, 'Incomplete Test') !== false) {
$status = ArcanistUnitTestResult::RESULT_SKIP;
$user_data .= $event->message;
} else {
$status = ArcanistUnitTestResult::RESULT_BROKEN;
$user_data .= $event->message;
foreach ($event->trace as $trace) {
$user_data .= sprintf("\n%s:%s", $trace->file, $trace->line);
}
}
}
}
}
$name = preg_replace('/ \\(.*\\)/', '', $event->test);
$result = new ArcanistUnitTestResult();
$result->setName($name);
$result->setResult($status);
$result->setDuration($event->time);
$result->setCoverage($coverage);
$result->setUserData($user_data);
$results[] = $result;
}
return $results;
}
public function run()
{
$results = array();
$build_start = microtime(true);
$config_manager = $this->getConfigurationManager();
if ($this->getEnableCoverage() !== false) {
$command = $config_manager->getConfigFromAnySource('unit.engine.tap.cover');
} else {
$command = $config_manager->getConfigFromAnySource('unit.engine.tap.command');
}
$timeout = $config_manager->getConfigFromAnySource('unit.engine.tap.timeout');
if (!$timeout) {
$timeout = 15;
}
$future = new ExecFuture('%C', $command);
$future->setTimeout($timeout);
$result = new ArcanistUnitTestResult();
$result->setName($command ? $command : 'unknown');
try {
list($stdout, $stderr) = $future->resolvex();
$result->setResult(ArcanistUnitTestResult::RESULT_PASS);
if ($this->getEnableCoverage() !== false) {
$coverage = $this->readCoverage('coverage/cobertura-coverage.xml');
$result->setCoverage($coverage);
}
} catch (CommandException $exc) {
$result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
if ($future->getWasKilledByTimeout()) {
print "Process stdout:\n" . $exc->getStdout() . "\nProcess stderr:\n" . $exc->getStderr() . "\nExceeded timeout of {$timeout} secs.\nMake unit tests faster.";
} else {
$result->setUserdata($exc->getStdout() . $exc->getStderr());
}
}
$result->setDuration(microtime(true) - $build_start);
$results[] = $result;
return $results;
}
function generateCoverageResults()
{
// Find all executables, not just the test executables.
// We need to do this because the test executables may not
// include all of the possible code (linker decides to omit
// it from the image) so we see a skewed representation of
// the source lines.
$fp = popen("find {$this->repo_root} -type f -name watchman -o " . "-name \\*.a -o -name \\*.so -o -name \\*.dylib", "r");
while (true) {
$line = fgets($fp);
if ($line === false) {
break;
}
$obj_files[] = trim($line);
}
// Parse line information from the objects
foreach ($obj_files as $object) {
DwarfLineInfo::loadObject($object);
}
$CG = new CallgrindFile((string) $this->cg_file);
$CG->parse();
$source_files = array();
foreach ($CG->getSourceFiles() as $filename) {
if (Filesystem::isDescendant($filename, $this->repo_root) && !preg_match("/(thirdparty|tests)/", $filename)) {
$source_files[$filename] = $filename;
}
}
$cov = array();
foreach ($source_files as $filename) {
$relsrc = substr($filename, strlen($this->repo_root) + 1);
$cov[$relsrc] = CallgrindFile::mergeSourceLineData($filename, array($CG));
}
$res = new ArcanistUnitTestResult();
$res->setName('coverage');
$res->setUserData("Collected");
$res->setResult(ArcanistUnitTestResult::RESULT_PASS);
$res->setCoverage($cov);
// Stash it for review with our `arc cov` command
$wc = ArcanistWorkingCopyIdentity::newFromPath($this->repo_root);
$api = ArcanistRepositoryAPI::newAPIFromWorkingCopyIdentity($wc);
$api->writeScratchFile('wman-cov.json', json_encode($cov));
return array($res);
}
/**
* Parses the test results from xUnit.
*
* @param string The name of the xUnit results file.
* @param string The name of the coverage file if one was provided by
* `buildTestFuture`. This is passed through to
* `parseCoverageResult`.
* @return array Test results.
*/
private function parseTestResult($xunit_tmp, $coverage)
{
$xunit_dom = new DOMDocument();
$xunit_dom->loadXML(Filesystem::readFile($xunit_tmp));
$results = array();
$tests = $xunit_dom->getElementsByTagName('test');
foreach ($tests as $test) {
$name = $test->getAttribute('name');
$time = $test->getAttribute('time');
$status = ArcanistUnitTestResult::RESULT_UNSOUND;
switch ($test->getAttribute('result')) {
case 'Pass':
$status = ArcanistUnitTestResult::RESULT_PASS;
break;
case 'Fail':
$status = ArcanistUnitTestResult::RESULT_FAIL;
break;
case 'Skip':
$status = ArcanistUnitTestResult::RESULT_SKIP;
break;
}
$userdata = '';
$reason = $test->getElementsByTagName('reason');
$failure = $test->getElementsByTagName('failure');
if ($reason->length > 0 || $failure->length > 0) {
$node = $reason->length > 0 ? $reason : $failure;
$message = $node->item(0)->getElementsByTagName('message');
if ($message->length > 0) {
$userdata = $message->item(0)->nodeValue;
}
$stacktrace = $node->item(0)->getElementsByTagName('stack-trace');
if ($stacktrace->length > 0) {
$userdata .= "\n" . $stacktrace->item(0)->nodeValue;
}
}
$result = new ArcanistUnitTestResult();
$result->setName($name);
$result->setResult($status);
$result->setDuration($time);
$result->setUserData($userdata);
if ($coverage != null) {
$result->setCoverage($this->parseCoverageResult($coverage));
}
$results[] = $result;
}
return $results;
}
private final function resultTest($test_result, $reason)
{
$coverage = $this->endCoverage();
$result = new ArcanistUnitTestResult();
$result->setCoverage($coverage);
$result->setNamespace(get_class($this));
$result->setName($this->runningTest);
$result->setLink($this->getLink($this->runningTest));
$result->setResult($test_result);
$result->setDuration(microtime(true) - $this->testStartTime);
$result->setUserData($reason);
$this->results[] = $result;
if ($this->renderer) {
echo $this->renderer->renderUnitResult($result);
}
}
/**
* Mark the current running test as skipped.
*
* @param string Description for why this test was skipped.
* @return void
* @task internal
*/
private final function skipTest($reason)
{
$coverage = $this->endCoverage();
$result = new ArcanistUnitTestResult();
$result->setCoverage($coverage);
$result->setName($this->runningTest);
$result->setResult(ArcanistUnitTestResult::RESULT_SKIP);
$result->setDuration(microtime(true) - $this->testStartTime);
$result->setUserData($reason);
$this->results[] = $result;
}
public function parseTestResults($path, $xunit_tmp, $cover_tmp)
{
// xunit xsd: https://gist.github.com/959290
$xunit_dom = new DOMDocument();
$xunit_dom->loadXML(Filesystem::readFile($xunit_tmp));
// coverage is for all testcases in the executed $path
$coverage = array();
if ($this->getEnableCoverage() !== false) {
$coverage = $this->readCoverage($cover_tmp);
}
$results = array();
$testcases = $xunit_dom->getElementsByTagName("testcase");
foreach ($testcases as $testcase) {
$classname = $testcase->getAttribute("classname");
$name = $testcase->getAttribute("name");
$time = $testcase->getAttribute("time");
$status = ArcanistUnitTestResult::RESULT_PASS;
$user_data = "";
// A skipped test is a test which was ignored using framework
// mechanizms (e.g. @skip decorator)
$skipped = $testcase->getElementsByTagName("skipped");
if ($skipped->length > 0) {
$status = ArcanistUnitTestResult::RESULT_SKIP;
$messages = array();
for ($ii = 0; $ii < $skipped->length; $ii++) {
$messages[] = trim($skipped->item($ii)->nodeValue, " \n");
}
$user_data .= implode("\n", $messages);
}
// Failure is a test which the code has explicitly failed by using
// the mechanizms for that purpose. e.g., via an assertEquals
$failures = $testcase->getElementsByTagName("failure");
if ($failures->length > 0) {
$status = ArcanistUnitTestResult::RESULT_FAIL;
$messages = array();
for ($ii = 0; $ii < $failures->length; $ii++) {
$messages[] = trim($failures->item($ii)->nodeValue, " \n");
}
$user_data .= implode("\n", $messages) . "\n";
}
// An errored test is one that had an unanticipated problem. e.g., an
// unchecked throwable, or a problem with an implementation of the
// test.
$errors = $testcase->getElementsByTagName("error");
if ($errors->length > 0) {
$status = ArcanistUnitTestResult::RESULT_BROKEN;
$messages = array();
for ($ii = 0; $ii < $errors->length; $ii++) {
$messages[] = trim($errors->item($ii)->nodeValue, " \n");
}
$user_data .= implode("\n", $messages) . "\n";
}
$result = new ArcanistUnitTestResult();
$result->setName($classname . "." . $name);
$result->setResult($status);
$result->setDuration($time);
$result->setCoverage($coverage);
$result->setUserData($user_data);
$results[] = $result;
}
return $results;
}
/**
* Parse test results from phpunit json report
*
* @param string $path Path to test
* @param string $test_results String containing phpunit json report
*
* @return array
*/
public function parseTestResults($path, $test_results)
{
if (!$test_results) {
$result = id(new ArcanistUnitTestResult())->setName($path)->setUserData($this->stderr)->setResult(ArcanistUnitTestResult::RESULT_BROKEN);
return array($result);
}
$report = $this->getJsonReport($test_results);
// coverage is for all testcases in the executed $path
$coverage = array();
if ($this->enableCoverage !== false) {
$coverage = $this->readCoverage();
}
$last_test_finished = true;
$results = array();
foreach ($report as $event) {
switch (idx($event, 'event')) {
case 'test':
break;
case 'testStart':
$last_test_finished = false;
// fall through
// fall through
default:
continue 2;
// switch + loop
}
$status = ArcanistUnitTestResult::RESULT_PASS;
$user_data = '';
if ('fail' == idx($event, 'status')) {
$status = ArcanistUnitTestResult::RESULT_FAIL;
$user_data .= idx($event, 'message') . "\n";
foreach (idx($event, 'trace') as $trace) {
$user_data .= sprintf("\n%s:%s", idx($trace, 'file'), idx($trace, 'line'));
}
} else {
if ('error' == idx($event, 'status')) {
if (strpos(idx($event, 'message'), 'Skipped Test') !== false) {
$status = ArcanistUnitTestResult::RESULT_SKIP;
$user_data .= idx($event, 'message');
} else {
if (strpos(idx($event, 'message'), 'Incomplete Test') !== false) {
$status = ArcanistUnitTestResult::RESULT_SKIP;
$user_data .= idx($event, 'message');
} else {
$status = ArcanistUnitTestResult::RESULT_BROKEN;
$user_data .= idx($event, 'message');
foreach (idx($event, 'trace') as $trace) {
$user_data .= sprintf("\n%s:%s", idx($trace, 'file'), idx($trace, 'line'));
}
}
}
}
}
$name = preg_replace('/ \\(.*\\)/s', '', idx($event, 'test'));
$result = new ArcanistUnitTestResult();
$result->setName($name);
$result->setResult($status);
$result->setDuration(idx($event, 'time'));
$result->setCoverage($coverage);
$result->setUserData($user_data);
$results[] = $result;
$last_test_finished = true;
}
if (!$last_test_finished) {
$results[] = id(new ArcanistUnitTestResult())->setName(idx($event, 'test'))->setUserData($this->stderr)->setResult(ArcanistUnitTestResult::RESULT_BROKEN);
}
return $results;
}
private function processCoverageResults($project_root, $results)
{
$time_start = microtime_float();
// generate annotated source files to find out which lines have
// coverage
// limit files to only those "*.py" files in getPaths()
$pythonPathsStr = join(" ", $this->getPythonPaths());
$future = new ExecFuture("coverage annotate {$pythonPathsStr}");
$future->setCWD($project_root);
try {
$future->resolvex();
} catch (CommandException $exc) {
if ($exc->getError() > 1) {
// 'nose' returns 1 when tests are failing/broken.
throw $exc;
}
}
// store all the coverage results for this project
$coverageArray = array();
$lines_total = 0;
$lines_executable = 0;
$lines_not_executable = 0;
$lines_covered = 0;
$lines_not_covered = 0;
// walk through project directory, searching for all ",cover" files
// that coverage.py left behind
foreach (new RecursiveIteratorIterator(new RecursiveDirectoryIterator(".")) as $path) {
// paths are given as "./path/to/file.py,cover", so match the
// "path/to/file.py" part
if (!preg_match(":^\\./(.*),cover\$:", $path, $matches)) {
continue;
}
$srcFilePath = $matches[1];
$coverageStr = "";
foreach (file($path) as $coverLine) {
/*
python coverage
> executed
! missing(not executed)
- excluded
phab coverage
N Not executable. This is a comment or whitespace which should be ignored when computing test coverage.
C Covered. This line has test coverage.
U Uncovered. This line is executable but has no test coverage.
X Unreachable. If your coverage analysis can detect unreachable code, you can report it here.
*/
$lines_total++;
switch ($coverLine[0]) {
case '>':
$lines_covered++;
$lines_executable++;
$coverageStr .= 'C';
break;
case '!':
$lines_not_covered++;
$lines_executable++;
$coverageStr .= 'U';
break;
case ' ':
$lines_not_executable++;
$coverageStr .= 'N';
break;
case '-':
$coverageStr .= 'X';
break;
default:
break;
}
}
// delete the ,cover file
unlink($path);
// only add to coverage report if the path was originally
// specified by arc
if (in_array($srcFilePath, $this->getPaths())) {
$coverageArray[$srcFilePath] = $coverageStr;
}
}
$lines_percentage = bcdiv($lines_covered, $lines_executable, 4) * 100;
$time_end = microtime_float();
$time = $time_end - $time_start;
// python does not support per-test coverage results so just put all the coverage
// in a single 'coverage test'
$result = new ArcanistUnitTestResult();
$result->setNamespace('coverage');
$result->setName('coverage');
$result->setResult('pass');
$result->setDuration($time);
$result->setUserData("coverage: {$lines_percentage}% executable: {$lines_executable} / covered: {$lines_covered}");
$result->setCoverage($coverageArray);
$results[] = $result;
return $results;
}