/**
  * Parse test results from phpunit json report
  *
  * @param string $path Path to test
  * @param string $test_results String containing phpunit json report
  *
  * @return array
  */
 public function parseTestResults($path, $test_results)
 {
     $report = $this->getJsonReport($test_results);
     // coverage is for all testcases in the executed $path
     $coverage = array();
     if ($this->enableCoverage !== false) {
         $coverage = $this->readCoverage();
     }
     $results = array();
     foreach ($report as $event) {
         if ('test' != $event->event) {
             continue;
         }
         $status = ArcanistUnitTestResult::RESULT_PASS;
         $user_data = '';
         if ('fail' == $event->status) {
             $status = ArcanistUnitTestResult::RESULT_FAIL;
             $user_data .= $event->message . "\n";
             foreach ($event->trace as $trace) {
                 $user_data .= sprintf("\n%s:%s", $trace->file, $trace->line);
             }
         } else {
             if ('error' == $event->status) {
                 if (strpos($event->message, 'Skipped Test') !== false) {
                     $status = ArcanistUnitTestResult::RESULT_SKIP;
                     $user_data .= $event->message;
                 } else {
                     if (strpos($event->message, 'Incomplete Test') !== false) {
                         $status = ArcanistUnitTestResult::RESULT_SKIP;
                         $user_data .= $event->message;
                     } else {
                         $status = ArcanistUnitTestResult::RESULT_BROKEN;
                         $user_data .= $event->message;
                         foreach ($event->trace as $trace) {
                             $user_data .= sprintf("\n%s:%s", $trace->file, $trace->line);
                         }
                     }
                 }
             }
         }
         $name = preg_replace('/ \\(.*\\)/', '', $event->test);
         $result = new ArcanistUnitTestResult();
         $result->setName($name);
         $result->setResult($status);
         $result->setDuration($event->time);
         $result->setCoverage($coverage);
         $result->setUserData($user_data);
         $results[] = $result;
     }
     return $results;
 }
Esempio n. 2
0
 public function run()
 {
     $results = array();
     $build_start = microtime(true);
     $config_manager = $this->getConfigurationManager();
     if ($this->getEnableCoverage() !== false) {
         $command = $config_manager->getConfigFromAnySource('unit.engine.tap.cover');
     } else {
         $command = $config_manager->getConfigFromAnySource('unit.engine.tap.command');
     }
     $timeout = $config_manager->getConfigFromAnySource('unit.engine.tap.timeout');
     if (!$timeout) {
         $timeout = 15;
     }
     $future = new ExecFuture('%C', $command);
     $future->setTimeout($timeout);
     $result = new ArcanistUnitTestResult();
     $result->setName($command ? $command : 'unknown');
     try {
         list($stdout, $stderr) = $future->resolvex();
         $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
         if ($this->getEnableCoverage() !== false) {
             $coverage = $this->readCoverage('coverage/cobertura-coverage.xml');
             $result->setCoverage($coverage);
         }
     } catch (CommandException $exc) {
         $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
         if ($future->getWasKilledByTimeout()) {
             print "Process stdout:\n" . $exc->getStdout() . "\nProcess stderr:\n" . $exc->getStderr() . "\nExceeded timeout of {$timeout} secs.\nMake unit tests faster.";
         } else {
             $result->setUserdata($exc->getStdout() . $exc->getStderr());
         }
     }
     $result->setDuration(microtime(true) - $build_start);
     $results[] = $result;
     return $results;
 }
Esempio n. 3
0
 function generateCoverageResults()
 {
     // Find all executables, not just the test executables.
     // We need to do this because the test executables may not
     // include all of the possible code (linker decides to omit
     // it from the image) so we see a skewed representation of
     // the source lines.
     $fp = popen("find {$this->repo_root} -type f -name watchman -o " . "-name \\*.a -o -name \\*.so -o -name \\*.dylib", "r");
     while (true) {
         $line = fgets($fp);
         if ($line === false) {
             break;
         }
         $obj_files[] = trim($line);
     }
     // Parse line information from the objects
     foreach ($obj_files as $object) {
         DwarfLineInfo::loadObject($object);
     }
     $CG = new CallgrindFile((string) $this->cg_file);
     $CG->parse();
     $source_files = array();
     foreach ($CG->getSourceFiles() as $filename) {
         if (Filesystem::isDescendant($filename, $this->repo_root) && !preg_match("/(thirdparty|tests)/", $filename)) {
             $source_files[$filename] = $filename;
         }
     }
     $cov = array();
     foreach ($source_files as $filename) {
         $relsrc = substr($filename, strlen($this->repo_root) + 1);
         $cov[$relsrc] = CallgrindFile::mergeSourceLineData($filename, array($CG));
     }
     $res = new ArcanistUnitTestResult();
     $res->setName('coverage');
     $res->setUserData("Collected");
     $res->setResult(ArcanistUnitTestResult::RESULT_PASS);
     $res->setCoverage($cov);
     // Stash it for review with our `arc cov` command
     $wc = ArcanistWorkingCopyIdentity::newFromPath($this->repo_root);
     $api = ArcanistRepositoryAPI::newAPIFromWorkingCopyIdentity($wc);
     $api->writeScratchFile('wman-cov.json', json_encode($cov));
     return array($res);
 }
Esempio n. 4
0
 /**
  * Parses the test results from xUnit.
  *
  * @param  string  The name of the xUnit results file.
  * @param  string  The name of the coverage file if one was provided by
  *                 `buildTestFuture`. This is passed through to
  *                 `parseCoverageResult`.
  * @return array   Test results.
  */
 private function parseTestResult($xunit_tmp, $coverage)
 {
     $xunit_dom = new DOMDocument();
     $xunit_dom->loadXML(Filesystem::readFile($xunit_tmp));
     $results = array();
     $tests = $xunit_dom->getElementsByTagName('test');
     foreach ($tests as $test) {
         $name = $test->getAttribute('name');
         $time = $test->getAttribute('time');
         $status = ArcanistUnitTestResult::RESULT_UNSOUND;
         switch ($test->getAttribute('result')) {
             case 'Pass':
                 $status = ArcanistUnitTestResult::RESULT_PASS;
                 break;
             case 'Fail':
                 $status = ArcanistUnitTestResult::RESULT_FAIL;
                 break;
             case 'Skip':
                 $status = ArcanistUnitTestResult::RESULT_SKIP;
                 break;
         }
         $userdata = '';
         $reason = $test->getElementsByTagName('reason');
         $failure = $test->getElementsByTagName('failure');
         if ($reason->length > 0 || $failure->length > 0) {
             $node = $reason->length > 0 ? $reason : $failure;
             $message = $node->item(0)->getElementsByTagName('message');
             if ($message->length > 0) {
                 $userdata = $message->item(0)->nodeValue;
             }
             $stacktrace = $node->item(0)->getElementsByTagName('stack-trace');
             if ($stacktrace->length > 0) {
                 $userdata .= "\n" . $stacktrace->item(0)->nodeValue;
             }
         }
         $result = new ArcanistUnitTestResult();
         $result->setName($name);
         $result->setResult($status);
         $result->setDuration($time);
         $result->setUserData($userdata);
         if ($coverage != null) {
             $result->setCoverage($this->parseCoverageResult($coverage));
         }
         $results[] = $result;
     }
     return $results;
 }
Esempio n. 5
0
 private final function resultTest($test_result, $reason)
 {
     $coverage = $this->endCoverage();
     $result = new ArcanistUnitTestResult();
     $result->setCoverage($coverage);
     $result->setNamespace(get_class($this));
     $result->setName($this->runningTest);
     $result->setLink($this->getLink($this->runningTest));
     $result->setResult($test_result);
     $result->setDuration(microtime(true) - $this->testStartTime);
     $result->setUserData($reason);
     $this->results[] = $result;
     if ($this->renderer) {
         echo $this->renderer->renderUnitResult($result);
     }
 }
Esempio n. 6
0
 /**
  * Mark the current running test as skipped.
  *
  * @param string  Description for why this test was skipped.
  * @return void
  * @task internal
  */
 private final function skipTest($reason)
 {
     $coverage = $this->endCoverage();
     $result = new ArcanistUnitTestResult();
     $result->setCoverage($coverage);
     $result->setName($this->runningTest);
     $result->setResult(ArcanistUnitTestResult::RESULT_SKIP);
     $result->setDuration(microtime(true) - $this->testStartTime);
     $result->setUserData($reason);
     $this->results[] = $result;
 }
Esempio n. 7
0
 public function parseTestResults($path, $xunit_tmp, $cover_tmp)
 {
     // xunit xsd: https://gist.github.com/959290
     $xunit_dom = new DOMDocument();
     $xunit_dom->loadXML(Filesystem::readFile($xunit_tmp));
     // coverage is for all testcases in the executed $path
     $coverage = array();
     if ($this->getEnableCoverage() !== false) {
         $coverage = $this->readCoverage($cover_tmp);
     }
     $results = array();
     $testcases = $xunit_dom->getElementsByTagName("testcase");
     foreach ($testcases as $testcase) {
         $classname = $testcase->getAttribute("classname");
         $name = $testcase->getAttribute("name");
         $time = $testcase->getAttribute("time");
         $status = ArcanistUnitTestResult::RESULT_PASS;
         $user_data = "";
         // A skipped test is a test which was ignored using framework
         // mechanizms (e.g. @skip decorator)
         $skipped = $testcase->getElementsByTagName("skipped");
         if ($skipped->length > 0) {
             $status = ArcanistUnitTestResult::RESULT_SKIP;
             $messages = array();
             for ($ii = 0; $ii < $skipped->length; $ii++) {
                 $messages[] = trim($skipped->item($ii)->nodeValue, " \n");
             }
             $user_data .= implode("\n", $messages);
         }
         // Failure is a test which the code has explicitly failed by using
         // the mechanizms for that purpose. e.g., via an assertEquals
         $failures = $testcase->getElementsByTagName("failure");
         if ($failures->length > 0) {
             $status = ArcanistUnitTestResult::RESULT_FAIL;
             $messages = array();
             for ($ii = 0; $ii < $failures->length; $ii++) {
                 $messages[] = trim($failures->item($ii)->nodeValue, " \n");
             }
             $user_data .= implode("\n", $messages) . "\n";
         }
         // An errored test is one that had an unanticipated problem. e.g., an
         // unchecked throwable, or a problem with an implementation of the
         // test.
         $errors = $testcase->getElementsByTagName("error");
         if ($errors->length > 0) {
             $status = ArcanistUnitTestResult::RESULT_BROKEN;
             $messages = array();
             for ($ii = 0; $ii < $errors->length; $ii++) {
                 $messages[] = trim($errors->item($ii)->nodeValue, " \n");
             }
             $user_data .= implode("\n", $messages) . "\n";
         }
         $result = new ArcanistUnitTestResult();
         $result->setName($classname . "." . $name);
         $result->setResult($status);
         $result->setDuration($time);
         $result->setCoverage($coverage);
         $result->setUserData($user_data);
         $results[] = $result;
     }
     return $results;
 }
 /**
  * Parse test results from phpunit json report
  *
  * @param string $path Path to test
  * @param string $test_results String containing phpunit json report
  *
  * @return array
  */
 public function parseTestResults($path, $test_results)
 {
     if (!$test_results) {
         $result = id(new ArcanistUnitTestResult())->setName($path)->setUserData($this->stderr)->setResult(ArcanistUnitTestResult::RESULT_BROKEN);
         return array($result);
     }
     $report = $this->getJsonReport($test_results);
     // coverage is for all testcases in the executed $path
     $coverage = array();
     if ($this->enableCoverage !== false) {
         $coverage = $this->readCoverage();
     }
     $last_test_finished = true;
     $results = array();
     foreach ($report as $event) {
         switch (idx($event, 'event')) {
             case 'test':
                 break;
             case 'testStart':
                 $last_test_finished = false;
                 // fall through
             // fall through
             default:
                 continue 2;
                 // switch + loop
         }
         $status = ArcanistUnitTestResult::RESULT_PASS;
         $user_data = '';
         if ('fail' == idx($event, 'status')) {
             $status = ArcanistUnitTestResult::RESULT_FAIL;
             $user_data .= idx($event, 'message') . "\n";
             foreach (idx($event, 'trace') as $trace) {
                 $user_data .= sprintf("\n%s:%s", idx($trace, 'file'), idx($trace, 'line'));
             }
         } else {
             if ('error' == idx($event, 'status')) {
                 if (strpos(idx($event, 'message'), 'Skipped Test') !== false) {
                     $status = ArcanistUnitTestResult::RESULT_SKIP;
                     $user_data .= idx($event, 'message');
                 } else {
                     if (strpos(idx($event, 'message'), 'Incomplete Test') !== false) {
                         $status = ArcanistUnitTestResult::RESULT_SKIP;
                         $user_data .= idx($event, 'message');
                     } else {
                         $status = ArcanistUnitTestResult::RESULT_BROKEN;
                         $user_data .= idx($event, 'message');
                         foreach (idx($event, 'trace') as $trace) {
                             $user_data .= sprintf("\n%s:%s", idx($trace, 'file'), idx($trace, 'line'));
                         }
                     }
                 }
             }
         }
         $name = preg_replace('/ \\(.*\\)/s', '', idx($event, 'test'));
         $result = new ArcanistUnitTestResult();
         $result->setName($name);
         $result->setResult($status);
         $result->setDuration(idx($event, 'time'));
         $result->setCoverage($coverage);
         $result->setUserData($user_data);
         $results[] = $result;
         $last_test_finished = true;
     }
     if (!$last_test_finished) {
         $results[] = id(new ArcanistUnitTestResult())->setName(idx($event, 'test'))->setUserData($this->stderr)->setResult(ArcanistUnitTestResult::RESULT_BROKEN);
     }
     return $results;
 }
 private function processCoverageResults($project_root, $results)
 {
     $time_start = microtime_float();
     // generate annotated source files to find out which lines have
     // coverage
     // limit files to only those "*.py" files in getPaths()
     $pythonPathsStr = join(" ", $this->getPythonPaths());
     $future = new ExecFuture("coverage annotate {$pythonPathsStr}");
     $future->setCWD($project_root);
     try {
         $future->resolvex();
     } catch (CommandException $exc) {
         if ($exc->getError() > 1) {
             // 'nose' returns 1 when tests are failing/broken.
             throw $exc;
         }
     }
     // store all the coverage results for this project
     $coverageArray = array();
     $lines_total = 0;
     $lines_executable = 0;
     $lines_not_executable = 0;
     $lines_covered = 0;
     $lines_not_covered = 0;
     // walk through project directory, searching for all ",cover" files
     // that coverage.py left behind
     foreach (new RecursiveIteratorIterator(new RecursiveDirectoryIterator(".")) as $path) {
         // paths are given as "./path/to/file.py,cover", so match the
         // "path/to/file.py" part
         if (!preg_match(":^\\./(.*),cover\$:", $path, $matches)) {
             continue;
         }
         $srcFilePath = $matches[1];
         $coverageStr = "";
         foreach (file($path) as $coverLine) {
             /*
                 python coverage
                 > executed
                 ! missing(not executed)
                 - excluded
             
                 phab coverage
                 N Not executable. This is a comment or whitespace which should be ignored when computing test coverage.
                 C Covered. This line has test coverage.
                 U Uncovered. This line is executable but has no test coverage.
                 X Unreachable. If your coverage analysis can detect unreachable code, you can report it here.
             */
             $lines_total++;
             switch ($coverLine[0]) {
                 case '>':
                     $lines_covered++;
                     $lines_executable++;
                     $coverageStr .= 'C';
                     break;
                 case '!':
                     $lines_not_covered++;
                     $lines_executable++;
                     $coverageStr .= 'U';
                     break;
                 case ' ':
                     $lines_not_executable++;
                     $coverageStr .= 'N';
                     break;
                 case '-':
                     $coverageStr .= 'X';
                     break;
                 default:
                     break;
             }
         }
         // delete the ,cover file
         unlink($path);
         // only add to coverage report if the path was originally
         // specified by arc
         if (in_array($srcFilePath, $this->getPaths())) {
             $coverageArray[$srcFilePath] = $coverageStr;
         }
     }
     $lines_percentage = bcdiv($lines_covered, $lines_executable, 4) * 100;
     $time_end = microtime_float();
     $time = $time_end - $time_start;
     // python does not support per-test coverage results so just put all the coverage
     // in a single 'coverage test'
     $result = new ArcanistUnitTestResult();
     $result->setNamespace('coverage');
     $result->setName('coverage');
     $result->setResult('pass');
     $result->setDuration($time);
     $result->setUserData("coverage: {$lines_percentage}% executable: {$lines_executable} / covered: {$lines_covered}");
     $result->setCoverage($coverageArray);
     $results[] = $result;
     return $results;
 }