/**
  * Runs the test suite.
  */
 public function run()
 {
     $results = array();
     $command = '(mkdir -p build && cd build && cmake ..)';
     $command .= '&& make -C build all';
     $command .= '&& make -C build test';
     // Execute the test command & time it.
     $timeStart = microtime(true);
     $future = new ExecFuture($command);
     do {
         $future->read();
         sleep(0.5);
     } while (!$future->isReady());
     list($error, $stdout, $stderr) = $future->resolve();
     $timeEnd = microtime(true);
     // Create a unit test result structure.
     $result = new ArcanistUnitTestResult();
     $result->setNamespace('DerpVision');
     $result->setName('Core');
     $result->setDuration($timeEnd - $timeStart);
     if ($error == 0) {
         $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
     } else {
         $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
         $result->setUserData($stdout . $stderr);
     }
     $results[] = $result;
     return $results;
 }
 private function parseOutput($output)
 {
     $results = array();
     $json = json_decode($output, true);
     foreach ($json['examples'] as $example) {
         $result = new ArcanistUnitTestResult();
         $result->setName($example['full_description']);
         if (array_key_exists('run_time', $example)) {
             $result->setDuration($example['run_time']);
         }
         switch ($example['status']) {
             case 'passed':
                 $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
                 break;
             case 'failed':
                 $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
                 $result->setUserData($example['exception']['message']);
                 break;
             case 'pending':
                 $result->setResult(ArcanistUnitTestResult::RESULT_SKIP);
                 break;
         }
         $results[] = $result;
     }
     return $results;
 }
Exemplo n.º 3
0
 private function parseTestResults($test, $err, $stdout, $stderr)
 {
     $result = new ArcanistUnitTestResult();
     $result->setName($test);
     $result->setUserData($stdout . $stderr);
     $result->setResult($err == 0 ? ArcanistUnitTestResult::RESULT_PASS : ArcanistUnitTestResult::RESULT_FAIL);
     if (preg_match("/# ELAPSED: (\\d+)ms/", $stderr, $M)) {
         $result->setDuration($M[1] / 1000);
     }
     return $result;
 }
 /**
  * Parse test results from phpunit json report
  *
  * @param string $path Path to test
  * @param string $test_results String containing phpunit json report
  *
  * @return array
  */
 public function parseTestResults($path, $test_results)
 {
     $report = $this->getJsonReport($test_results);
     // coverage is for all testcases in the executed $path
     $coverage = array();
     if ($this->enableCoverage !== false) {
         $coverage = $this->readCoverage();
     }
     $results = array();
     foreach ($report as $event) {
         if ('test' != $event->event) {
             continue;
         }
         $status = ArcanistUnitTestResult::RESULT_PASS;
         $user_data = '';
         if ('fail' == $event->status) {
             $status = ArcanistUnitTestResult::RESULT_FAIL;
             $user_data .= $event->message . "\n";
             foreach ($event->trace as $trace) {
                 $user_data .= sprintf("\n%s:%s", $trace->file, $trace->line);
             }
         } else {
             if ('error' == $event->status) {
                 if (strpos($event->message, 'Skipped Test') !== false) {
                     $status = ArcanistUnitTestResult::RESULT_SKIP;
                     $user_data .= $event->message;
                 } else {
                     if (strpos($event->message, 'Incomplete Test') !== false) {
                         $status = ArcanistUnitTestResult::RESULT_SKIP;
                         $user_data .= $event->message;
                     } else {
                         $status = ArcanistUnitTestResult::RESULT_BROKEN;
                         $user_data .= $event->message;
                         foreach ($event->trace as $trace) {
                             $user_data .= sprintf("\n%s:%s", $trace->file, $trace->line);
                         }
                     }
                 }
             }
         }
         $name = preg_replace('/ \\(.*\\)/', '', $event->test);
         $result = new ArcanistUnitTestResult();
         $result->setName($name);
         $result->setResult($status);
         $result->setDuration($event->time);
         $result->setCoverage($coverage);
         $result->setUserData($user_data);
         $results[] = $result;
     }
     return $results;
 }
Exemplo n.º 5
0
 public function runJs()
 {
     // First, check to see if karma is on $PATH:
     list($err, $stdout, $_) = exec_manual("which karma");
     if ($err != 0) {
         $result = new ArcanistUnitTestResult();
         $result->setName("Karma not found. Skipping js tests...");
         $result->setResult(ArcanistUnitTestResult::RESULT_SKIP);
         $result->setDuration(0);
         return array($result);
     }
     // Karma IS on the path.
     $old_dir = getcwd();
     $project_root = $this->getWorkingCopy()->getProjectRoot();
     chdir($project_root . '/client/js');
     exec_manual("karma start karma-conf-oneshot.js");
     chdir($old_dir);
     // Read from the text-results.xml file.
     $xml = file_get_contents($project_root . '/client/test-results.xml');
     $doc = new SimpleXMLElement($xml);
     // Destroy the test-results.xml file.
     unlink($project_root . '/client/test-results.xml');
     // Extract all the test cases.
     $results = array();
     foreach ($doc->testsuite as $suite) {
         $suite_name = $suite['name'];
         foreach ($suite->testcase as $case) {
             $case_name = $case['name'];
             $time = $case['time'];
             $fixture_name = substr($case['classname'], strlen($suite_name) + 1);
             // Did we fail?
             $failure = (string) $case->failure;
             // Convert each to a ArcanistUnitTestResult
             $result = new ArcanistUnitTestResult();
             $result->setName($fixture_name . ' ' . $case_name);
             $result->setResult($failure ? ArcanistUnitTestResult::RESULT_FAIL : ArcanistUnitTestResult::RESULT_PASS);
             $result->setUserData($failure);
             $result->setDuration($time);
             $results[] = $result;
         }
     }
     return $results;
 }
Exemplo n.º 6
0
 public function run()
 {
     $results = array();
     $build_start = microtime(true);
     $config_manager = $this->getConfigurationManager();
     if ($this->getEnableCoverage() !== false) {
         $command = $config_manager->getConfigFromAnySource('unit.engine.tap.cover');
     } else {
         $command = $config_manager->getConfigFromAnySource('unit.engine.tap.command');
     }
     $timeout = $config_manager->getConfigFromAnySource('unit.engine.tap.timeout');
     if (!$timeout) {
         $timeout = 15;
     }
     $future = new ExecFuture('%C', $command);
     $future->setTimeout($timeout);
     $result = new ArcanistUnitTestResult();
     $result->setName($command ? $command : 'unknown');
     try {
         list($stdout, $stderr) = $future->resolvex();
         $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
         if ($this->getEnableCoverage() !== false) {
             $coverage = $this->readCoverage('coverage/cobertura-coverage.xml');
             $result->setCoverage($coverage);
         }
     } catch (CommandException $exc) {
         $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
         if ($future->getWasKilledByTimeout()) {
             print "Process stdout:\n" . $exc->getStdout() . "\nProcess stderr:\n" . $exc->getStderr() . "\nExceeded timeout of {$timeout} secs.\nMake unit tests faster.";
         } else {
             $result->setUserdata($exc->getStdout() . $exc->getStderr());
         }
     }
     $result->setDuration(microtime(true) - $build_start);
     $results[] = $result;
     return $results;
 }
Exemplo n.º 7
0
 /**
  * Parse test results from Go test report
  * (e.g. `go test -v`)
  *
  * @param string $path Path to test
  * @param string $test_results String containing Go test output
  *
  * @return array
  */
 public function parseTestResults($path, $test_results)
 {
     $test_results = explode("\n", $test_results);
     $results = array();
     // We'll get our full test case name at the end and add it back in
     $test_case_name = '';
     // Temp store for test case results (in case we run multiple test cases)
     $test_case_results = array();
     foreach ($test_results as $i => $line) {
         if (strncmp($line, '--- PASS', 8) === 0) {
             // We have a passing test
             $meta = array();
             preg_match('/^--- PASS: (?P<test_name>.+) \\((?P<time>.+) seconds\\).*/', $line, $meta);
             $result = new ArcanistUnitTestResult();
             // For now set name without test case, we'll add it later
             $result->setName($meta['test_name']);
             $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
             $result->setDuration($meta['time']);
             $test_case_results[] = $result;
             continue;
         }
         if (strncmp($line, '--- FAIL', 8) === 0) {
             // We have a failing test
             $reason = trim($test_results[$i + 1]);
             $meta = array();
             preg_match('/^--- FAIL: (?P<test_name>.+) \\((?P<time>.+) seconds\\).*/', $line, $meta);
             $result = new ArcanistUnitTestResult();
             $result->setName($meta['test_name']);
             $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
             $result->setDuration($meta['time']);
             $result->setUserData($reason . "\n");
             $test_case_results[] = $result;
             continue;
         }
         if (strncmp($line, 'ok', 2) === 0) {
             $meta = array();
             preg_match('/^ok[\\s\\t]+(?P<test_name>\\w.*)[\\s\\t]+(?P<time>.*)s.*/', $line, $meta);
             $test_case_name = str_replace('/', '::', $meta['test_name']);
             // Our test case passed
             // check to make sure we were in verbose (-v) mode
             if (empty($test_case_results)) {
                 // We weren't in verbose mode
                 // create one successful result for the whole test case
                 $test_name = 'Go::TestCase::' . $test_case_name;
                 $result = new ArcanistUnitTestResult();
                 $result->setName($test_name);
                 $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
                 $result->setDuration($meta['time']);
                 $results[] = $result;
             } else {
                 $test_case_results = $this->fixNames($test_case_results, $test_case_name);
                 $results = array_merge($results, $test_case_results);
                 $test_case_results = array();
             }
             continue;
         }
         if (strncmp($line, "FAIL\t", 5) === 0) {
             $meta = array();
             preg_match('/^FAIL[\\s\\t]+(?P<test_name>\\w.*)[\\s\\t]+.*/', $line, $meta);
             $test_case_name = str_replace('/', '::', $meta['test_name']);
             $test_case_results = $this->fixNames($test_case_results, $test_case_name);
             $results = array_merge($results, $test_case_results);
             $test_case_results = array();
             continue;
         }
     }
     return $results;
 }
Exemplo n.º 8
0
 private function parseTestResults($targets, $status)
 {
     $code = $status[0];
     $output = $status[1];
     $lines = explode("\n", $output);
     if ($code == 4) {
         print "No tests affected...\n";
         return [];
     } else {
         if ($code == 1) {
             throw new Exception($output . "\n" . $status[2]);
         }
     }
     $query_command = $this->bazelCommand(["query", "-k", "%s"]);
     $future = new ExecFuture($query_command, 'tests(set(' . join(" ", $targets) . '))');
     $future->setCWD($this->project_root);
     $testTargets = explode("\n", trim($future->resolvex()[0]));
     $results = array();
     foreach ($testTargets as $test) {
         $data = $this->parseTestResultFile($test);
         $result = new ArcanistUnitTestResult();
         $result->setName($test);
         if (property_exists($data, "test_case")) {
             $testCase = $data->{"test_case"};
             if (property_exists($testCase, "run_duration_millis")) {
                 $result->setDuration($testCase->{"run_duration_millis"} / 1000);
             }
         }
         if ($data->{"test_passed"}) {
             $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
         } else {
             if ($data->{"status"} == 4) {
                 $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
             } else {
                 $result->setResult(ArcanistUnitTestResult::RESULT_BROKEN);
             }
         }
         $results[] = $result;
     }
     return $results;
 }
Exemplo n.º 9
0
 private final function resultTest($test_result, $reason)
 {
     $coverage = $this->endCoverage();
     $result = new ArcanistUnitTestResult();
     $result->setCoverage($coverage);
     $result->setNamespace(get_class($this));
     $result->setName($this->runningTest);
     $result->setLink($this->getLink($this->runningTest));
     $result->setResult($test_result);
     $result->setDuration(microtime(true) - $this->testStartTime);
     $result->setUserData($reason);
     $this->results[] = $result;
     if ($this->renderer) {
         echo $this->renderer->renderUnitResult($result);
     }
 }
 private function processCoverageResults($project_root, $results)
 {
     $time_start = microtime_float();
     // generate annotated source files to find out which lines have
     // coverage
     // limit files to only those "*.py" files in getPaths()
     $pythonPathsStr = join(" ", $this->getPythonPaths());
     $future = new ExecFuture("coverage annotate {$pythonPathsStr}");
     $future->setCWD($project_root);
     try {
         $future->resolvex();
     } catch (CommandException $exc) {
         if ($exc->getError() > 1) {
             // 'nose' returns 1 when tests are failing/broken.
             throw $exc;
         }
     }
     // store all the coverage results for this project
     $coverageArray = array();
     $lines_total = 0;
     $lines_executable = 0;
     $lines_not_executable = 0;
     $lines_covered = 0;
     $lines_not_covered = 0;
     // walk through project directory, searching for all ",cover" files
     // that coverage.py left behind
     foreach (new RecursiveIteratorIterator(new RecursiveDirectoryIterator(".")) as $path) {
         // paths are given as "./path/to/file.py,cover", so match the
         // "path/to/file.py" part
         if (!preg_match(":^\\./(.*),cover\$:", $path, $matches)) {
             continue;
         }
         $srcFilePath = $matches[1];
         $coverageStr = "";
         foreach (file($path) as $coverLine) {
             /*
                 python coverage
                 > executed
                 ! missing(not executed)
                 - excluded
             
                 phab coverage
                 N Not executable. This is a comment or whitespace which should be ignored when computing test coverage.
                 C Covered. This line has test coverage.
                 U Uncovered. This line is executable but has no test coverage.
                 X Unreachable. If your coverage analysis can detect unreachable code, you can report it here.
             */
             $lines_total++;
             switch ($coverLine[0]) {
                 case '>':
                     $lines_covered++;
                     $lines_executable++;
                     $coverageStr .= 'C';
                     break;
                 case '!':
                     $lines_not_covered++;
                     $lines_executable++;
                     $coverageStr .= 'U';
                     break;
                 case ' ':
                     $lines_not_executable++;
                     $coverageStr .= 'N';
                     break;
                 case '-':
                     $coverageStr .= 'X';
                     break;
                 default:
                     break;
             }
         }
         // delete the ,cover file
         unlink($path);
         // only add to coverage report if the path was originally
         // specified by arc
         if (in_array($srcFilePath, $this->getPaths())) {
             $coverageArray[$srcFilePath] = $coverageStr;
         }
     }
     $lines_percentage = bcdiv($lines_covered, $lines_executable, 4) * 100;
     $time_end = microtime_float();
     $time = $time_end - $time_start;
     // python does not support per-test coverage results so just put all the coverage
     // in a single 'coverage test'
     $result = new ArcanistUnitTestResult();
     $result->setNamespace('coverage');
     $result->setName('coverage');
     $result->setResult('pass');
     $result->setDuration($time);
     $result->setUserData("coverage: {$lines_percentage}% executable: {$lines_executable} / covered: {$lines_covered}");
     $result->setCoverage($coverageArray);
     $results[] = $result;
     return $results;
 }
Exemplo n.º 11
0
 /**
  * Mark the current running test as skipped.
  *
  * @param string  Description for why this test was skipped.
  * @return void
  * @task internal
  */
 private final function skipTest($reason)
 {
     $coverage = $this->endCoverage();
     $result = new ArcanistUnitTestResult();
     $result->setCoverage($coverage);
     $result->setName($this->runningTest);
     $result->setResult(ArcanistUnitTestResult::RESULT_SKIP);
     $result->setDuration(microtime(true) - $this->testStartTime);
     $result->setUserData($reason);
     $this->results[] = $result;
 }
Exemplo n.º 12
0
 public function parseTestResults($path, $xunit_tmp, $cover_tmp)
 {
     // xunit xsd: https://gist.github.com/959290
     $xunit_dom = new DOMDocument();
     $xunit_dom->loadXML(Filesystem::readFile($xunit_tmp));
     // coverage is for all testcases in the executed $path
     $coverage = array();
     if ($this->getEnableCoverage() !== false) {
         $coverage = $this->readCoverage($cover_tmp);
     }
     $results = array();
     $testcases = $xunit_dom->getElementsByTagName("testcase");
     foreach ($testcases as $testcase) {
         $classname = $testcase->getAttribute("classname");
         $name = $testcase->getAttribute("name");
         $time = $testcase->getAttribute("time");
         $status = ArcanistUnitTestResult::RESULT_PASS;
         $user_data = "";
         // A skipped test is a test which was ignored using framework
         // mechanizms (e.g. @skip decorator)
         $skipped = $testcase->getElementsByTagName("skipped");
         if ($skipped->length > 0) {
             $status = ArcanistUnitTestResult::RESULT_SKIP;
             $messages = array();
             for ($ii = 0; $ii < $skipped->length; $ii++) {
                 $messages[] = trim($skipped->item($ii)->nodeValue, " \n");
             }
             $user_data .= implode("\n", $messages);
         }
         // Failure is a test which the code has explicitly failed by using
         // the mechanizms for that purpose. e.g., via an assertEquals
         $failures = $testcase->getElementsByTagName("failure");
         if ($failures->length > 0) {
             $status = ArcanistUnitTestResult::RESULT_FAIL;
             $messages = array();
             for ($ii = 0; $ii < $failures->length; $ii++) {
                 $messages[] = trim($failures->item($ii)->nodeValue, " \n");
             }
             $user_data .= implode("\n", $messages) . "\n";
         }
         // An errored test is one that had an unanticipated problem. e.g., an
         // unchecked throwable, or a problem with an implementation of the
         // test.
         $errors = $testcase->getElementsByTagName("error");
         if ($errors->length > 0) {
             $status = ArcanistUnitTestResult::RESULT_BROKEN;
             $messages = array();
             for ($ii = 0; $ii < $errors->length; $ii++) {
                 $messages[] = trim($errors->item($ii)->nodeValue, " \n");
             }
             $user_data .= implode("\n", $messages) . "\n";
         }
         $result = new ArcanistUnitTestResult();
         $result->setName($classname . "." . $name);
         $result->setResult($status);
         $result->setDuration($time);
         $result->setCoverage($coverage);
         $result->setUserData($user_data);
         $results[] = $result;
     }
     return $results;
 }
 /**
  * Parse test results from phpunit json report
  *
  * @param string $path Path to test
  * @param string $test_results String containing phpunit json report
  *
  * @return array
  */
 public function parseTestResults($path, $test_results)
 {
     if (!$test_results) {
         $result = id(new ArcanistUnitTestResult())->setName($path)->setUserData($this->stderr)->setResult(ArcanistUnitTestResult::RESULT_BROKEN);
         return array($result);
     }
     $report = $this->getJsonReport($test_results);
     // coverage is for all testcases in the executed $path
     $coverage = array();
     if ($this->enableCoverage !== false) {
         $coverage = $this->readCoverage();
     }
     $last_test_finished = true;
     $results = array();
     foreach ($report as $event) {
         switch (idx($event, 'event')) {
             case 'test':
                 break;
             case 'testStart':
                 $last_test_finished = false;
                 // fall through
             // fall through
             default:
                 continue 2;
                 // switch + loop
         }
         $status = ArcanistUnitTestResult::RESULT_PASS;
         $user_data = '';
         if ('fail' == idx($event, 'status')) {
             $status = ArcanistUnitTestResult::RESULT_FAIL;
             $user_data .= idx($event, 'message') . "\n";
             foreach (idx($event, 'trace') as $trace) {
                 $user_data .= sprintf("\n%s:%s", idx($trace, 'file'), idx($trace, 'line'));
             }
         } else {
             if ('error' == idx($event, 'status')) {
                 if (strpos(idx($event, 'message'), 'Skipped Test') !== false) {
                     $status = ArcanistUnitTestResult::RESULT_SKIP;
                     $user_data .= idx($event, 'message');
                 } else {
                     if (strpos(idx($event, 'message'), 'Incomplete Test') !== false) {
                         $status = ArcanistUnitTestResult::RESULT_SKIP;
                         $user_data .= idx($event, 'message');
                     } else {
                         $status = ArcanistUnitTestResult::RESULT_BROKEN;
                         $user_data .= idx($event, 'message');
                         foreach (idx($event, 'trace') as $trace) {
                             $user_data .= sprintf("\n%s:%s", idx($trace, 'file'), idx($trace, 'line'));
                         }
                     }
                 }
             }
         }
         $name = preg_replace('/ \\(.*\\)/s', '', idx($event, 'test'));
         $result = new ArcanistUnitTestResult();
         $result->setName($name);
         $result->setResult($status);
         $result->setDuration(idx($event, 'time'));
         $result->setCoverage($coverage);
         $result->setUserData($user_data);
         $results[] = $result;
         $last_test_finished = true;
     }
     if (!$last_test_finished) {
         $results[] = id(new ArcanistUnitTestResult())->setName(idx($event, 'test'))->setUserData($this->stderr)->setResult(ArcanistUnitTestResult::RESULT_BROKEN);
     }
     return $results;
 }
Exemplo n.º 14
0
 public function run()
 {
     $projectRoot = $this->getWorkingCopy()->getProjectRoot();
     $cwd = getcwd();
     $buildDir = $this->findBuildDirectory($projectRoot, $cwd);
     $polliObjDir = $buildDir;
     if (is_dir($buildDir . DIRECTORY_SEPARATOR . "tools" . DIRECTORY_SEPARATOR . "polly" . DIRECTORY_SEPARATOR . "tools" . DIRECTORY_SEPARATOR . "polli")) {
         $polliObjDir = $buildDir . DIRECTORY_SEPARATOR . "tools" . DIRECTORY_SEPARATOR . "polly" . DIRECTORY_SEPARATOR . "tools" . DIRECTORY_SEPARATOR . "polli";
     }
     $polliTestDir = $polliObjDir . DIRECTORY_SEPARATOR . "test";
     if (is_dir($buildDir . DIRECTORY_SEPARATOR . "bin") && file_exists($buildDir . DIRECTORY_SEPARATOR . "bin" . DIRECTORY_SEPARATOR . "llvm-lit")) {
         $lit = $buildDir . DIRECTORY_SEPARATOR . "bin" . DIRECTORY_SEPARATOR . "llvm-lit";
         $cmd = "ninja -C " . $buildDir;
         print "Running ninja (" . $cmd . ")\n";
         exec($cmd);
     } else {
         $makeVars = $this->getMakeVars($buildDir);
         $lit = $this->findLitExecutable($makeVars);
     }
     print "Using lit executable '{$lit}'\n";
     // We have to modify the format string, because llvm-lit does not like a '' argument
     $cmd = '%s %s 2>&1';
     $litFuture = new ExecFuture($cmd, $lit, $polliTestDir);
     $out = "";
     $results = array();
     $lastTime = microtime(true);
     $ready = false;
     $dots = "";
     $numTests = 0;
     while (!$ready) {
         $ready = $litFuture->isReady();
         $newout = $litFuture->readStdout();
         if (strlen($newout) == 0) {
             usleep(100);
             continue;
         }
         $out .= $newout;
         if ($ready && strlen($out) > 0 && substr($out, -1) != "\n") {
             $out .= "\n";
         }
         while (($nlPos = strpos($out, "\n")) !== FALSE) {
             $line = substr($out, 0, $nlPos + 1);
             $out = substr($out, $nlPos + 1);
             $res = ArcanistUnitTestResult::RESULT_UNSOUND;
             if (substr($line, 0, 6) == "PASS: "******"FAIL: ") {
                 $res = ArcanistUnitTestResult::RESULT_FAIL;
             } elseif (substr($line, 0, 7) == "XPASS: "******"XFAIL: ") {
                 $res = ArcanistUnitTestResult::RESULT_PASS;
             } elseif (substr($line, 0, 13) == "UNSUPPORTED: ") {
                 $res = ArcanistUnitTestResult::RESULT_SKIP;
             } elseif (!$numTests && preg_match('/Testing: ([0-9]+) tests/', $line, $matches)) {
                 $numTests = (int) $matches[1];
             }
             if ($res == ArcanistUnitTestResult::RESULT_FAIL) {
                 print "";
             }
             if ($res != ArcanistUnitTestResult::RESULT_SKIP && $res != ArcanistUnitTestResult::RESULT_PASS) {
                 print "" . $line . self::progress($results, $numTests);
             }
             if ($res == ArcanistUnitTestResult::RESULT_UNSOUND) {
                 continue;
             }
             $result = new ArcanistUnitTestResult();
             $result->setName(trim(substr($line, strpos($line, ':') + 1)));
             $result->setResult($res);
             $newTime = microtime(true);
             $result->setDuration($newTime - $lastTime);
             $lastTime = $newTime;
             $results[] = $result;
             $dots .= ".";
             print "\r" . self::progress($results, $numTests);
         }
     }
     list($out1, $out2) = $litFuture->read();
     print $out1;
     if ($out2) {
         throw new Exception('There was error output, even though it should have been redirected to stdout.');
     }
     print "\n";
     $timeThreshold = 0.05;
     $interestingTests = array();
     foreach ($results as $result) {
         if ($result->getResult() != "pass") {
             $interestingTests[] = $result;
         }
         if ($result->getDuration() > $timeThreshold) {
             $interestingTests[] = $result;
         }
     }
     return $interestingTests;
 }
 public function runIntegrationTests($tests)
 {
     // Now find all the test programs
     $root = $this->getProjectRoot();
     $test_dir = $root . "/tests/integration/";
     if (!$tests) {
         $paths = glob($test_dir . "*.php");
         $paths[] = 'ruby/ruby-watchman/spec/ruby_watchman_spec.rb';
     } else {
         $paths = $tests;
     }
     foreach (array('/tmp/watchman-test.log', '/tmp/watchman-valgrind.log', '/tmp/watchman-valgrind.xml', '/tmp/watchman-callgrind.txt') as $log) {
         @unlink($log);
     }
     foreach ($paths as $path) {
         if (preg_match("/\\.php\$/", $path) && file_exists($path)) {
             // Don't pull in files starting with "_"; we're using
             // those as helpers for triggers
             $base = basename($path);
             if ($base[0] != '_') {
                 require_once $path;
             }
         }
     }
     // We test for this in a test case
     putenv("WATCHMAN_EMPTY_ENV_VAR=");
     $coverage = $this->getEnableCoverage();
     if (!$this->first_inst) {
         $this->first_inst = new WatchmanInstance($root, $coverage);
     }
     $first_inst = $this->first_inst;
     $instances = array($first_inst);
     // Helper for python or other language tests
     putenv("WATCHMAN_SOCK=" . $first_inst->getFullSockName());
     // Exercise the different serialization combinations
     $cli_matrix = array();
     // Find all the test cases that were declared
     $results = array();
     foreach (get_declared_classes() as $name) {
         $ref = new ReflectionClass($name);
         if (!$ref->isSubclassOf('WatchmanTestCase')) {
             continue;
         }
         // Good enough; let's use it
         $test_case = newv($name, array());
         $config = $test_case->getGlobalConfig();
         if ($config) {
             $instance = new WatchmanInstance($root, $coverage, $config);
             $instances[] = $instance;
         } else {
             $instance = $first_inst;
         }
         $test_case->setWatchmanInstance($instance);
         if (!$instance->getProcessID()) {
             $res = new ArcanistUnitTestResult();
             $res->setName('dead');
             $res->setUserData('died before test start');
             $res->setResult(ArcanistUnitTestResult::RESULT_FAIL);
             $results[] = array($res);
             break;
         }
         $test_case->setRoot($root);
         $test_case->setPaths($paths);
         $results[] = $test_case->run();
         if (!$test_case->needsLiveConnection()) {
             foreach ($cli_matrix as $mname => $args) {
                 $test_case->useCLI($args);
                 $cli_results = $test_case->run();
                 foreach ($cli_results as $res) {
                     $res->setName($res->getName() . " [CLI: {$mname}]");
                 }
                 $results[] = $cli_results;
             }
         }
     }
     foreach ($paths as $path) {
         if (!preg_match('/\\.rb$/', $path)) {
             continue;
         }
         if (!file_exists($path)) {
             // Was deleted in this (pending) rev
             continue;
         }
         $start = microtime(true);
         $future = new ExecFuture("PATH=\"{$root}:\$PATH\" \${MAKE:-make} rb-tests");
         $future->setTimeout(10);
         list($status, $out, $err) = $future->resolve();
         $end = microtime(true);
         $res = new ArcanistUnitTestResult();
         $res->setName($path);
         $res->setUserData($out . $err);
         $res->setDuration($end - $start);
         $res->setResult($status == 0 ? ArcanistUnitTestResult::RESULT_PASS : ArcanistUnitTestResult::RESULT_FAIL);
         $results[] = array($res);
     }
     foreach ($instances as $instance) {
         $results[] = $instance->generateValgrindTestResults();
     }
     $results = array_mergev($results);
     return $results;
 }
Exemplo n.º 16
0
 /**
  * Parses the test results from xUnit.
  *
  * @param  string  The name of the xUnit results file.
  * @param  string  The name of the coverage file if one was provided by
  *                 `buildTestFuture`. This is passed through to
  *                 `parseCoverageResult`.
  * @return array   Test results.
  */
 private function parseTestResult($xunit_tmp, $coverage)
 {
     $xunit_dom = new DOMDocument();
     $xunit_dom->loadXML(Filesystem::readFile($xunit_tmp));
     $results = array();
     $tests = $xunit_dom->getElementsByTagName('test');
     foreach ($tests as $test) {
         $name = $test->getAttribute('name');
         $time = $test->getAttribute('time');
         $status = ArcanistUnitTestResult::RESULT_UNSOUND;
         switch ($test->getAttribute('result')) {
             case 'Pass':
                 $status = ArcanistUnitTestResult::RESULT_PASS;
                 break;
             case 'Fail':
                 $status = ArcanistUnitTestResult::RESULT_FAIL;
                 break;
             case 'Skip':
                 $status = ArcanistUnitTestResult::RESULT_SKIP;
                 break;
         }
         $userdata = '';
         $reason = $test->getElementsByTagName('reason');
         $failure = $test->getElementsByTagName('failure');
         if ($reason->length > 0 || $failure->length > 0) {
             $node = $reason->length > 0 ? $reason : $failure;
             $message = $node->item(0)->getElementsByTagName('message');
             if ($message->length > 0) {
                 $userdata = $message->item(0)->nodeValue;
             }
             $stacktrace = $node->item(0)->getElementsByTagName('stack-trace');
             if ($stacktrace->length > 0) {
                 $userdata .= "\n" . $stacktrace->item(0)->nodeValue;
             }
         }
         $result = new ArcanistUnitTestResult();
         $result->setName($name);
         $result->setResult($status);
         $result->setDuration($time);
         $result->setUserData($userdata);
         if ($coverage != null) {
             $result->setCoverage($this->parseCoverageResult($coverage));
         }
         $results[] = $result;
     }
     return $results;
 }
 /**
  * Parse test results from provided input and return an array
  * of @{class:ArcanistUnitTestResult}.
  *
  * @param string $test_results String containing test results
  *
  * @return array ArcanistUnitTestResult
  */
 public function parseTestResults($test_results)
 {
     if (!strlen($test_results)) {
         throw new Exception(pht('%s argument to %s must not be empty', 'test_results', 'parseTestResults()'));
     }
     // xunit xsd: https://gist.github.com/959290
     $xunit_dom = new DOMDocument();
     $load_success = @$xunit_dom->loadXML($test_results);
     if (!$load_success) {
         $input_start = id(new PhutilUTF8StringTruncator())->setMaximumGlyphs(150)->truncateString($test_results);
         throw new Exception(sprintf("%s\n\n%s", pht('Failed to load XUnit report; Input starts with:'), $input_start));
     }
     $results = array();
     $testcases = $xunit_dom->getElementsByTagName('testcase');
     foreach ($testcases as $testcase) {
         $classname = $testcase->getAttribute('classname');
         $name = $testcase->getAttribute('name');
         $time = $testcase->getAttribute('time');
         $status = ArcanistUnitTestResult::RESULT_PASS;
         $user_data = '';
         // A skipped test is a test which was ignored using framework
         // mechanisms (e.g. @skip decorator)
         $skipped = $testcase->getElementsByTagName('skipped');
         if ($skipped->length > 0) {
             $status = ArcanistUnitTestResult::RESULT_SKIP;
             $messages = array();
             for ($ii = 0; $ii < $skipped->length; $ii++) {
                 $messages[] = trim($skipped->item($ii)->nodeValue, " \n");
             }
             $user_data .= implode("\n", $messages);
         }
         // Failure is a test which the code has explicitly failed by using
         // the mechanisms for that purpose. e.g., via an assertEquals
         $failures = $testcase->getElementsByTagName('failure');
         if ($failures->length > 0) {
             $status = ArcanistUnitTestResult::RESULT_FAIL;
             $messages = array();
             for ($ii = 0; $ii < $failures->length; $ii++) {
                 $messages[] = trim($failures->item($ii)->nodeValue, " \n");
             }
             $user_data .= implode("\n", $messages) . "\n";
         }
         // An errored test is one that had an unanticipated problem. e.g., an
         // unchecked throwable, or a problem with an implementation of the test.
         $errors = $testcase->getElementsByTagName('error');
         if ($errors->length > 0) {
             $status = ArcanistUnitTestResult::RESULT_BROKEN;
             $messages = array();
             for ($ii = 0; $ii < $errors->length; $ii++) {
                 $messages[] = trim($errors->item($ii)->nodeValue, " \n");
             }
             $user_data .= implode("\n", $messages) . "\n";
         }
         $result = new ArcanistUnitTestResult();
         $result->setName($classname . '.' . $name);
         $result->setResult($status);
         $result->setDuration((double) $time);
         $result->setUserData($user_data);
         $results[] = $result;
     }
     return $results;
 }
Exemplo n.º 18
0
 public function run()
 {
     $projectRoot = $this->getWorkingCopy()->getProjectRoot();
     $cwd = getcwd();
     $buildDir = $this->findBuildDirectory($projectRoot, $cwd);
     print "Using build directory '{$buildDir}'\n";
     $makeVars = $this->getMakeVars($buildDir);
     $lit = $this->findLitExecutable($makeVars);
     print "Using lit executable '{$lit}'\n";
     // We have to modify the format string, because llvm-lit does not like a '' argument
     $cmd = '%s ' . ($this->getEnableAsyncTests() ? '' : '-j1 ') . '%s 2>&1';
     $litFuture = new ExecFuture($cmd, $lit, $buildDir . "/test");
     $out = "";
     $results = array();
     $lastTime = microtime(true);
     $ready = false;
     $dots = "";
     $numTests = 0;
     while (!$ready) {
         $ready = $litFuture->isReady();
         $newout = $litFuture->readStdout();
         if (strlen($newout) == 0) {
             usleep(100);
             continue;
         }
         $out .= $newout;
         if ($ready && strlen($out) > 0 && substr($out, -1) != "\n") {
             $out .= "\n";
         }
         while (($nlPos = strpos($out, "\n")) !== FALSE) {
             $line = substr($out, 0, $nlPos + 1);
             $out = substr($out, $nlPos + 1);
             $res = ArcanistUnitTestResult::RESULT_UNSOUND;
             if (substr($line, 0, 6) == "PASS: "******"FAIL: ") {
                 $res = ArcanistUnitTestResult::RESULT_FAIL;
             } elseif (substr($line, 0, 7) == "XPASS: "******"XFAIL: ") {
                 $res = ArcanistUnitTestResult::RESULT_PASS;
             } elseif (substr($line, 0, 13) == "UNSUPPORTED: ") {
                 $res = ArcanistUnitTestResult::RESULT_SKIP;
             } elseif (!$numTests && preg_match('/Testing: ([0-9]+) tests/', $line, $matches)) {
                 $numTests = (int) $matches[1];
             }
             if ($res == ArcanistUnitTestResult::RESULT_FAIL) {
                 print "";
             }
             if ($res != ArcanistUnitTestResult::RESULT_SKIP && $res != ArcanistUnitTestResult::RESULT_PASS) {
                 print "\r" . $line . self::progress($results, $numTests);
             }
             if ($res == ArcanistUnitTestResult::RESULT_UNSOUND) {
                 continue;
             }
             $result = new ArcanistUnitTestResult();
             $result->setName(trim(substr($line, strpos($line, ':') + 1)));
             $result->setResult($res);
             $newTime = microtime(true);
             $result->setDuration($newTime - $lastTime);
             $lastTime = $newTime;
             $results[] = $result;
             $dots .= ".";
             print "\r" . self::progress($results, $numTests);
         }
     }
     list($out1, $out2) = $litFuture->read();
     print $out1;
     if ($out2) {
         throw new Exception('There was error output, even though it should have been redirected to stdout.');
     }
     print "\n";
     return $results;
 }
 /**
  * Parse test results from Go test report
  * (e.g. `go test -v`)
  *
  * @param string $path Path to test
  * @param string $stdout the Stdout of the command.
  * @param string $stderr the Stderr of the command.
  *
  * @return array
  */
 public function parseTestResults($path, $stdout, $stderr = '')
 {
     $test_results = $stderr . $stdout;
     $test_results = explode("\n", $test_results);
     $results = array();
     // We'll get our full test case name at the end and add it back in
     $test_case_name = '';
     // Temp store for test case results (in case we run multiple test cases)
     $test_case_results = array();
     for ($i = 0; $i < count($test_results); $i++) {
         $line = $test_results[$i];
         if (strlen($line) >= 18 && strncmp($line, '==================', 18) === 0 && strncmp($test_results[$i + 1], 'WARNING: DATA RACE', 18) === 0) {
             // We have a race condition
             $i++;
             // Advance to WARNING: DATA RACE
             $reason = '';
             $test_name = '';
             // loop to collect all data and move to the === line
             while (strncmp($test_results[$i], '==================', 18) !== 0) {
                 if (strncmp($test_results[$i], 'Goroutine', 9) === 0) {
                     $meta = array();
                     preg_match('/^.*\\.(?P<test_name>[^\\.]+)$/', $test_results[$i + 1], $meta);
                     $test_name = $meta['test_name'] . ' Race Detected';
                 }
                 $reason .= $test_results[$i++] . "\n";
                 // Are we out of lines?
                 if ($i > count($test_results)) {
                     return false;
                 }
             }
             $result = new ArcanistUnitTestResult();
             $result->setName($test_name);
             $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
             $result->setUserData($reason);
             $test_case_results[] = $result;
             continue;
         }
         if (strncmp($line, '--- PASS', 8) === 0) {
             // We have a passing test
             $meta = array();
             preg_match('/^--- PASS: (?P<test_name>.+) \\((?P<time>.+)\\s*s(?:econds)?\\).*/', $line, $meta);
             $result = new ArcanistUnitTestResult();
             // For now set name without test case, we'll add it later
             $result->setName($meta['test_name']);
             $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
             $result->setDuration((double) $meta['time']);
             $test_case_results[] = $result;
             continue;
         }
         if (strncmp($line, '--- FAIL', 8) === 0) {
             // We have a failing test
             $reason = trim($test_results[$i + 1]);
             $meta = array();
             preg_match('/^--- FAIL: (?P<test_name>.+) \\((?P<time>.+)\\s*s(?:econds)?\\).*/', $line, $meta);
             $result = new ArcanistUnitTestResult();
             $result->setName($meta['test_name']);
             $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
             $result->setDuration((double) $meta['time']);
             $result->setUserData($reason . "\n");
             $test_case_results[] = $result;
             continue;
         }
         if (strncmp($line, 'ok', 2) === 0) {
             $meta = array();
             preg_match('/^ok[\\s\\t]+(?P<test_name>\\w.*)[\\s\\t]+(?P<time>.*)s.*/', $line, $meta);
             $test_case_name = str_replace('/', '::', $meta['test_name']);
             // Our test case passed
             // check to make sure we were in verbose (-v) mode
             if (empty($test_case_results)) {
                 // We weren't in verbose mode
                 // create one successful result for the whole test case
                 $test_name = 'Go::TestCase::' . $test_case_name;
                 $result = new ArcanistUnitTestResult();
                 $result->setName($test_name);
                 $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
                 $result->setDuration((double) $meta['time']);
                 $results[] = $result;
             } else {
                 $test_case_results = $this->fixNames($test_case_results, $test_case_name);
                 $results = array_merge($results, $test_case_results);
                 $test_case_results = array();
             }
             continue;
         }
         if (strncmp($line, "FAIL\t", 5) === 0) {
             $meta = array();
             preg_match('/^FAIL[\\s\\t]+(?P<test_name>\\w.*)[\\s\\t]+.*/', $line, $meta);
             $test_case_name = str_replace('/', '::', $meta['test_name']);
             $test_case_results = $this->fixNames($test_case_results, $test_case_name);
             $results = array_merge($results, $test_case_results);
             $test_case_results = array();
             continue;
         }
     }
     return $results;
 }
 public function runIntegrationTests($tests)
 {
     $this->make('all');
     // Now find all the test programs
     $root = $this->getProjectRoot();
     $test_dir = $root . "/tests/integration/";
     if (!$tests) {
         $paths = glob($test_dir . "*.php");
         foreach (glob('python/tests/*.py') as $file) {
             $paths[] = $file;
         }
         $paths[] = 'ruby/ruby-watchman/spec/ruby_watchman_spec.rb';
     } else {
         $paths = $tests;
     }
     foreach (array('/tmp/watchman-test.log', '/tmp/watchman-valgrind.log', '/tmp/watchman-valgrind.xml', '/tmp/watchman-callgrind.txt') as $log) {
         @unlink($log);
     }
     foreach ($paths as $path) {
         if (preg_match("/\\.php\$/", $path) && file_exists($path)) {
             require_once $path;
         }
     }
     // We test for this in a test case
     putenv("WATCHMAN_EMPTY_ENV_VAR=");
     $coverage = $this->getEnableCoverage();
     $first_inst = new WatchmanInstance($root, $coverage);
     $instances = array($first_inst);
     // Helper for python or other language tests
     putenv("WATCHMAN_SOCK=" . $first_inst->getFullSockName());
     // Exercise the different serialization combinations
     $cli_matrix = array('bser/json' => '--server-encoding=bser --output-encoding=json', 'json/json' => '--server-encoding=json --output-encoding=json');
     // Find all the test cases that were declared
     $results = array();
     foreach (get_declared_classes() as $name) {
         $ref = new ReflectionClass($name);
         if (!$ref->isSubclassOf('WatchmanTestCase')) {
             continue;
         }
         // Good enough; let's use it
         $test_case = newv($name, array());
         $config = $test_case->getGlobalConfig();
         if ($config) {
             $instance = new WatchmanInstance($root, $coverage, $config);
             $instances[] = $instance;
         } else {
             $instance = $first_inst;
         }
         $test_case->setWatchmanInstance($instance);
         if (!$instance->getProcessID()) {
             $res = new ArcanistUnitTestResult();
             $res->setName('dead');
             $res->setUserData('died before test start');
             $res->setResult(ArcanistUnitTestResult::RESULT_FAIL);
             $results[] = array($res);
             break;
         }
         $test_case->setRoot($root);
         $test_case->setPaths($paths);
         $results[] = $test_case->run();
         if (!$test_case->needsLiveConnection()) {
             foreach ($cli_matrix as $mname => $args) {
                 $test_case->useCLI($args);
                 $cli_results = $test_case->run();
                 foreach ($cli_results as $res) {
                     $res->setName($res->getName() . " [CLI: {$mname}]");
                 }
                 $results[] = $cli_results;
             }
         }
     }
     // Also run the python tests if we built them
     foreach ($paths as $path) {
         if (!preg_match('/test.*\\.py$/', $path)) {
             continue;
         }
         if (!file_exists($path)) {
             // Was deleted in this (pending) rev
             continue;
         }
         if (!file_exists("python/pywatchman/bser.so")) {
             // Not enabled by the build
             continue;
         }
         // Note that this implicitly starts the instance if we haven't
         // yet done so.  This is important if the only test paths are
         // python paths
         if (!$first_inst->getProcessID()) {
             $res = new ArcanistUnitTestResult();
             $res->setName('dead');
             $res->setUserData('died before test start');
             $res->setResult(ArcanistUnitTestResult::RESULT_FAIL);
             $results[] = array($res);
             break;
         }
         // our Makefile contains the detected python, so just run the
         // rule from the makefile to pick it up
         $start = microtime(true);
         $future = new ExecFuture("PATH=\"{$root}:\$PATH\" PYTHONPATH={$root}/python " . "TESTNAME={$path} \${MAKE:-make} py-tests");
         $future->setTimeout(10);
         list($status, $out, $err) = $future->resolve();
         $end = microtime(true);
         $res = new ArcanistUnitTestResult();
         $res->setName($path);
         $res->setUserData($out . $err);
         $res->setDuration($end - $start);
         $res->setResult($status == 0 ? ArcanistUnitTestResult::RESULT_PASS : ArcanistUnitTestResult::RESULT_FAIL);
         $results[] = array($res);
     }
     foreach ($paths as $path) {
         if (!preg_match('/\\.rb$/', $path)) {
             continue;
         }
         if (!file_exists($path)) {
             // Was deleted in this (pending) rev
             continue;
         }
         $start = microtime(true);
         $future = new ExecFuture("PATH=\"{$root}:\$PATH\" \${MAKE:-make} rb-tests");
         $future->setTimeout(10);
         list($status, $out, $err) = $future->resolve();
         $end = microtime(true);
         $res = new ArcanistUnitTestResult();
         $res->setName($path);
         $res->setUserData($out . $err);
         $res->setDuration($end - $start);
         $res->setResult($status == 0 ? ArcanistUnitTestResult::RESULT_PASS : ArcanistUnitTestResult::RESULT_FAIL);
         $results[] = array($res);
     }
     foreach ($instances as $instance) {
         $results[] = $instance->generateValgrindTestResults();
     }
     $results = array_mergev($results);
     return $results;
 }