/**
  * Runs the test suite.
  */
 public function run()
 {
     $results = array();
     $command = '(mkdir -p build && cd build && cmake ..)';
     $command .= '&& make -C build all';
     $command .= '&& make -C build test';
     // Execute the test command & time it.
     $timeStart = microtime(true);
     $future = new ExecFuture($command);
     do {
         $future->read();
         sleep(0.5);
     } while (!$future->isReady());
     list($error, $stdout, $stderr) = $future->resolve();
     $timeEnd = microtime(true);
     // Create a unit test result structure.
     $result = new ArcanistUnitTestResult();
     $result->setNamespace('DerpVision');
     $result->setName('Core');
     $result->setDuration($timeEnd - $timeStart);
     if ($error == 0) {
         $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
     } else {
         $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
         $result->setUserData($stdout . $stderr);
     }
     $results[] = $result;
     return $results;
 }
 private function parseOutput($output)
 {
     $results = array();
     $json = json_decode($output, true);
     foreach ($json['examples'] as $example) {
         $result = new ArcanistUnitTestResult();
         $result->setName($example['full_description']);
         if (array_key_exists('run_time', $example)) {
             $result->setDuration($example['run_time']);
         }
         switch ($example['status']) {
             case 'passed':
                 $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
                 break;
             case 'failed':
                 $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
                 $result->setUserData($example['exception']['message']);
                 break;
             case 'pending':
                 $result->setResult(ArcanistUnitTestResult::RESULT_SKIP);
                 break;
         }
         $results[] = $result;
     }
     return $results;
 }
 private function parseOutput($output)
 {
     $results = array();
     $json = json_decode($output);
     foreach ($json as $feature) {
         if (property_exists($feature, 'elements')) {
             foreach ($feature->elements as $element) {
                 $passed = true;
                 $result = new ArcanistUnitTestResult();
                 $result->setName($feature->description);
                 foreach ($element->steps as $step) {
                     switch ($step->result->status) {
                         case 'passed':
                             break;
                         case 'failed':
                             $passed = false;
                             $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
                             $result->setUserData($step->result->error_message);
                             break;
                     }
                 }
                 if ($passed) {
                     $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
                 }
                 $results[] = $result;
             }
         }
     }
     return $results;
 }
 private function parseOutput($output)
 {
     $results = array();
     $lines = explode(PHP_EOL, $output);
     foreach ($lines as $index => $line) {
         preg_match('/^(not ok|ok)\\s+\\d+\\s+-?(.*)/', $line, $matches);
         if (count($matches) < 3) {
             continue;
         }
         $result = new ArcanistUnitTestResult();
         $result->setName(trim($matches[2]));
         switch (trim($matches[1])) {
             case 'ok':
                 $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
                 break;
             case 'not ok':
                 $exception_message = trim($lines[$index + 1]);
                 $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
                 $result->setUserData($exception_message);
                 break;
             default:
                 continue;
         }
         $results[] = $result;
     }
     return $results;
 }
 private function checkNonEmptyTestPlan()
 {
     $result = new ArcanistUnitTestResult();
     $result->setName("Test Plan");
     $lines = join(' ', $this->getMessage());
     $testPlanExists = preg_match('/\\sTest Plan:/', $lines);
     if (!$testPlanExists) {
         $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
         print 'Test Plan not found!';
         return array($result);
     }
     $platforms = $this->getPlatformTestsFromLines($this->getMessage());
     if ($platforms) {
         print "Found tests for the following platforms:\n";
         foreach ($platforms as $platform => $tests) {
             print "{$platform}:\n    ";
             print join("\n    ", $tests);
             print "\n";
         }
         $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
         print 'Test Plan found!';
         return array($result);
     }
     $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
     print "No tests found to run on CI! Check your repo's README for instructions\n";
     return array($result);
 }
 private function runTests()
 {
     $root = $this->getWorkingCopy()->getProjectRoot();
     $script = $this->getConfiguredScript();
     $path = $this->getConfiguredTestResultPath();
     foreach (glob($root . DIRECTORY_SEPARATOR . $path . "/*.xml") as $filename) {
         // Remove existing files so we cannot report old results
         $this->unlink($filename);
     }
     // Provide changed paths to process
     putenv("ARCANIST_DIFF_PATHS=" . implode(PATH_SEPARATOR, $this->getPaths()));
     $future = new ExecFuture('%C %s', $script, $path);
     $future->setCWD($root);
     $err = null;
     try {
         $future->resolvex();
     } catch (CommandException $exc) {
         $err = $exc;
     }
     $results = $this->parseTestResults($root . DIRECTORY_SEPARATOR . $path);
     if ($err) {
         $result = new ArcanistUnitTestResult();
         $result->setName('Unit Test Script');
         $result->setResult(ArcanistUnitTestResult::RESULT_BROKEN);
         $result->setUserData("ERROR: Command failed with code {$err->getError()}\nCOMMAND: `{$err->getCommand()}`");
         $results[] = $result;
     }
     return $results;
 }
Esempio n. 7
0
 public function run()
 {
     $working_copy = $this->getWorkingCopy();
     $this->projectRoot = $working_copy->getProjectRoot();
     $future = new ExecFuture('npm run coverage');
     $future->setCWD($this->projectRoot);
     list($err, $stdout, $stderr) = $future->resolve();
     $result = new ArcanistUnitTestResult();
     $result->setName("Node test engine");
     $result->setUserData($stdout);
     if ($err) {
         $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
     } else {
         $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
     }
     return array($result);
 }
Esempio n. 8
0
 private function runCommand($command)
 {
     exec($command, $output, $return_code);
     $result = new ArcanistUnitTestResult();
     $result->setName($command);
     $result->setResult($return_code == 0 ? ArcanistUnitTestResult::RESULT_PASS : ArcanistUnitTestResult::RESULT_FAIL);
     return array($result);
 }
 public function run()
 {
     // For a call to `arc call-conduit differential.updateunitresults` to
     // succeed we need at least one entry here.
     $result = new ArcanistUnitTestResult();
     $result->setName("dummy_placeholder_entry");
     $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
     return array($result);
 }
Esempio n. 10
0
 private function checkNonEmptyTestPlan()
 {
     $result = new ArcanistUnitTestResult();
     $lines = implode(' ', $this->getMessage());
     $test_plan_exists = preg_match('/\\sTest Plan:/', $lines);
     if (!$test_plan_exists) {
         $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
         $result->setName('Test Plan not found!');
         return array($result);
     }
     $test_plan_empty = preg_match('/\\sTest Plan:\\s*?Reviewers:/', $lines);
     if ($test_plan_empty) {
         $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
         $result->setName('Test Plan cannot be empty!');
         return array($result);
     }
     $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
     $result->setName('Test Plan found!');
     return array($result);
 }
Esempio n. 11
0
 private function parseTestResults($test, $err, $stdout, $stderr)
 {
     $result = new ArcanistUnitTestResult();
     $result->setName($test);
     $result->setUserData($stdout . $stderr);
     $result->setResult($err == 0 ? ArcanistUnitTestResult::RESULT_PASS : ArcanistUnitTestResult::RESULT_FAIL);
     if (preg_match("/# ELAPSED: (\\d+)ms/", $stderr, $M)) {
         $result->setDuration($M[1] / 1000);
     }
     return $result;
 }
Esempio n. 12
0
 public function run()
 {
     $working_copy = $this->getWorkingCopy();
     $this->project_root = $working_copy->getProjectRoot();
     // We only want to report results for tests that actually ran, so
     // we'll compare the test result files' timestamps to the start time
     // of the test run. This will probably break if multiple test runs
     // are happening in parallel, but if that's happening then we can't
     // count on the results files being intact anyway.
     $start_time = time();
     $maven_top_dirs = $this->findTopLevelMavenDirectories();
     // We'll figure out if any of the modified files we're testing are in
     // Maven directories. We won't want to run a bunch of Java tests for
     // changes to CSS files or whatever.
     $modified_paths = $this->getModifiedPaths();
     $maven_failed = false;
     foreach ($maven_top_dirs as $dir) {
         $dir_with_trailing_slash = $dir . '/';
         foreach ($modified_paths as $path) {
             if ($dir_with_trailing_slash === substr($path, 0, strlen($dir_with_trailing_slash))) {
                 $future = new ExecFuture('mvn test');
                 $future->setCWD($dir);
                 list($status, $stdout, $stderr) = $future->resolve();
                 if ($status) {
                     // Maven exits with a nonzero status if there were test failures
                     // or if there was a compilation error.
                     $maven_failed = true;
                     break 2;
                 }
                 break;
             }
         }
     }
     $testResults = $this->parseTestResultsSince($start_time);
     if ($maven_failed) {
         // If there wasn't a test failure, then synthesize one to represent
         // the failure of the test run as a whole, since it probably means the
         // code failed to compile.
         $found_failure = false;
         foreach ($testResults as $testResult) {
             if ($testResult->getResult() === ArcanistUnitTestResult::RESULT_FAIL || $testResult->getResult() === ArcanistUnitTestResult::RESULT_BROKEN) {
                 $found_failure = true;
                 break;
             }
         }
         if (!$found_failure) {
             $testResult = new ArcanistUnitTestResult();
             $testResult->setResult(ArcanistUnitTestResult::RESULT_BROKEN);
             $testResult->setName('mvn test');
             $testResults[] = $testResult;
         }
     }
     return $testResults;
 }
Esempio n. 13
0
 private function checkNonEmptyRevertPlan()
 {
     $result = new ArcanistUnitTestResult();
     $lines = implode(' ', $this->getMessage());
     $revert_plan_exists = preg_match('/\\sRevert Plan:/', $lines);
     if (!$revert_plan_exists) {
         $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
         $result->setName('Revert Plan not found!
         (See http://t.uber.com/revert for more info)');
         return array($result);
     }
     $revert_plan_empty = preg_match('/\\sRevert Plan:\\s*?$/', $lines);
     if ($revert_plan_empty) {
         $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
         $result->setName('Revert Plan cannot be empty!
         (See http://t.uber.com/revert for more info)');
         return array($result);
     }
     $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
     $result->setName('Revert Plan found!');
     return array($result);
 }
 public function run()
 {
     // Here we create a new unit test "jenkins_async_test" and promise we'll
     // update the results later.
     // Jenkins updates the results using `arc call-conduit
     // differential.updateunitresults` call. If you change the name here, also
     // make sure to change the name in Jenkins script that updates the test
     // result -- they have to be the same.
     $result = new ArcanistUnitTestResult();
     $result->setName("jenkins_async_test");
     $result->setResult(ArcanistUnitTestResult::RESULT_POSTPONED);
     return array($result);
 }
 /**
  * Parse test results from phpunit json report
  *
  * @param string $path Path to test
  * @param string $test_results String containing phpunit json report
  *
  * @return array
  */
 public function parseTestResults($path, $test_results)
 {
     $report = $this->getJsonReport($test_results);
     // coverage is for all testcases in the executed $path
     $coverage = array();
     if ($this->enableCoverage !== false) {
         $coverage = $this->readCoverage();
     }
     $results = array();
     foreach ($report as $event) {
         if ('test' != $event->event) {
             continue;
         }
         $status = ArcanistUnitTestResult::RESULT_PASS;
         $user_data = '';
         if ('fail' == $event->status) {
             $status = ArcanistUnitTestResult::RESULT_FAIL;
             $user_data .= $event->message . "\n";
             foreach ($event->trace as $trace) {
                 $user_data .= sprintf("\n%s:%s", $trace->file, $trace->line);
             }
         } else {
             if ('error' == $event->status) {
                 if (strpos($event->message, 'Skipped Test') !== false) {
                     $status = ArcanistUnitTestResult::RESULT_SKIP;
                     $user_data .= $event->message;
                 } else {
                     if (strpos($event->message, 'Incomplete Test') !== false) {
                         $status = ArcanistUnitTestResult::RESULT_SKIP;
                         $user_data .= $event->message;
                     } else {
                         $status = ArcanistUnitTestResult::RESULT_BROKEN;
                         $user_data .= $event->message;
                         foreach ($event->trace as $trace) {
                             $user_data .= sprintf("\n%s:%s", $trace->file, $trace->line);
                         }
                     }
                 }
             }
         }
         $name = preg_replace('/ \\(.*\\)/', '', $event->test);
         $result = new ArcanistUnitTestResult();
         $result->setName($name);
         $result->setResult($status);
         $result->setDuration($event->time);
         $result->setCoverage($coverage);
         $result->setUserData($user_data);
         $results[] = $result;
     }
     return $results;
 }
Esempio n. 16
0
 public function run()
 {
     $results = array();
     $build_start = microtime(true);
     $config_manager = $this->getConfigurationManager();
     if ($this->getEnableCoverage() !== false) {
         $command = $config_manager->getConfigFromAnySource('unit.engine.tap.cover');
     } else {
         $command = $config_manager->getConfigFromAnySource('unit.engine.tap.command');
     }
     $timeout = $config_manager->getConfigFromAnySource('unit.engine.tap.timeout');
     if (!$timeout) {
         $timeout = 15;
     }
     $future = new ExecFuture('%C', $command);
     $future->setTimeout($timeout);
     $result = new ArcanistUnitTestResult();
     $result->setName($command ? $command : 'unknown');
     try {
         list($stdout, $stderr) = $future->resolvex();
         $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
         if ($this->getEnableCoverage() !== false) {
             $coverage = $this->readCoverage('coverage/cobertura-coverage.xml');
             $result->setCoverage($coverage);
         }
     } catch (CommandException $exc) {
         $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
         if ($future->getWasKilledByTimeout()) {
             print "Process stdout:\n" . $exc->getStdout() . "\nProcess stderr:\n" . $exc->getStderr() . "\nExceeded timeout of {$timeout} secs.\nMake unit tests faster.";
         } else {
             $result->setUserdata($exc->getStdout() . $exc->getStderr());
         }
     }
     $result->setDuration(microtime(true) - $build_start);
     $results[] = $result;
     return $results;
 }
 public function run()
 {
     // If we are running asynchronously, mark all tests as postponed
     // and return those results.  Otherwise, run the tests and collect
     // the actual results.
     if ($this->getEnableAsyncTests()) {
         $results = array();
         $result = new ArcanistUnitTestResult();
         $result->setName("jcommon_build");
         $result->setResult(ArcanistUnitTestResult::RESULT_POSTPONED);
         $results[] = $result;
         return $results;
     } else {
         $server = new FacebookBuildServer();
         $server->startProjectBuilds(false);
         return array();
     }
 }
Esempio n. 18
0
 public function runJs()
 {
     // First, check to see if karma is on $PATH:
     list($err, $stdout, $_) = exec_manual("which karma");
     if ($err != 0) {
         $result = new ArcanistUnitTestResult();
         $result->setName("Karma not found. Skipping js tests...");
         $result->setResult(ArcanistUnitTestResult::RESULT_SKIP);
         $result->setDuration(0);
         return array($result);
     }
     // Karma IS on the path.
     $old_dir = getcwd();
     $project_root = $this->getWorkingCopy()->getProjectRoot();
     chdir($project_root . '/client/js');
     exec_manual("karma start karma-conf-oneshot.js");
     chdir($old_dir);
     // Read from the text-results.xml file.
     $xml = file_get_contents($project_root . '/client/test-results.xml');
     $doc = new SimpleXMLElement($xml);
     // Destroy the test-results.xml file.
     unlink($project_root . '/client/test-results.xml');
     // Extract all the test cases.
     $results = array();
     foreach ($doc->testsuite as $suite) {
         $suite_name = $suite['name'];
         foreach ($suite->testcase as $case) {
             $case_name = $case['name'];
             $time = $case['time'];
             $fixture_name = substr($case['classname'], strlen($suite_name) + 1);
             // Did we fail?
             $failure = (string) $case->failure;
             // Convert each to a ArcanistUnitTestResult
             $result = new ArcanistUnitTestResult();
             $result->setName($fixture_name . ' ' . $case_name);
             $result->setResult($failure ? ArcanistUnitTestResult::RESULT_FAIL : ArcanistUnitTestResult::RESULT_PASS);
             $result->setUserData($failure);
             $result->setDuration($time);
             $results[] = $result;
         }
     }
     return $results;
 }
 /**
  * Parse test results from Go test report
  * (e.g. `go test -v`)
  *
  * @param string $path Path to test
  * @param string $stdout the Stdout of the command.
  * @param string $stderr the Stderr of the command.
  *
  * @return array
  */
 public function parseTestResults($path, $stdout, $stderr = '')
 {
     $test_results = $stderr . $stdout;
     $test_results = explode("\n", $test_results);
     $results = array();
     // We'll get our full test case name at the end and add it back in
     $test_case_name = '';
     // Temp store for test case results (in case we run multiple test cases)
     $test_case_results = array();
     for ($i = 0; $i < count($test_results); $i++) {
         $line = $test_results[$i];
         if (strlen($line) >= 18 && strncmp($line, '==================', 18) === 0 && strncmp($test_results[$i + 1], 'WARNING: DATA RACE', 18) === 0) {
             // We have a race condition
             $i++;
             // Advance to WARNING: DATA RACE
             $reason = '';
             $test_name = '';
             // loop to collect all data and move to the === line
             while (strncmp($test_results[$i], '==================', 18) !== 0) {
                 if (strncmp($test_results[$i], 'Goroutine', 9) === 0) {
                     $meta = array();
                     preg_match('/^.*\\.(?P<test_name>[^\\.]+)$/', $test_results[$i + 1], $meta);
                     $test_name = $meta['test_name'] . ' Race Detected';
                 }
                 $reason .= $test_results[$i++] . "\n";
                 // Are we out of lines?
                 if ($i > count($test_results)) {
                     return false;
                 }
             }
             $result = new ArcanistUnitTestResult();
             $result->setName($test_name);
             $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
             $result->setUserData($reason);
             $test_case_results[] = $result;
             continue;
         }
         if (strncmp($line, '--- PASS', 8) === 0) {
             // We have a passing test
             $meta = array();
             preg_match('/^--- PASS: (?P<test_name>.+) \\((?P<time>.+)\\s*s(?:econds)?\\).*/', $line, $meta);
             $result = new ArcanistUnitTestResult();
             // For now set name without test case, we'll add it later
             $result->setName($meta['test_name']);
             $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
             $result->setDuration((double) $meta['time']);
             $test_case_results[] = $result;
             continue;
         }
         if (strncmp($line, '--- FAIL', 8) === 0) {
             // We have a failing test
             $reason = trim($test_results[$i + 1]);
             $meta = array();
             preg_match('/^--- FAIL: (?P<test_name>.+) \\((?P<time>.+)\\s*s(?:econds)?\\).*/', $line, $meta);
             $result = new ArcanistUnitTestResult();
             $result->setName($meta['test_name']);
             $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
             $result->setDuration((double) $meta['time']);
             $result->setUserData($reason . "\n");
             $test_case_results[] = $result;
             continue;
         }
         if (strncmp($line, 'ok', 2) === 0) {
             $meta = array();
             preg_match('/^ok[\\s\\t]+(?P<test_name>\\w.*)[\\s\\t]+(?P<time>.*)s.*/', $line, $meta);
             $test_case_name = str_replace('/', '::', $meta['test_name']);
             // Our test case passed
             // check to make sure we were in verbose (-v) mode
             if (empty($test_case_results)) {
                 // We weren't in verbose mode
                 // create one successful result for the whole test case
                 $test_name = 'Go::TestCase::' . $test_case_name;
                 $result = new ArcanistUnitTestResult();
                 $result->setName($test_name);
                 $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
                 $result->setDuration((double) $meta['time']);
                 $results[] = $result;
             } else {
                 $test_case_results = $this->fixNames($test_case_results, $test_case_name);
                 $results = array_merge($results, $test_case_results);
                 $test_case_results = array();
             }
             continue;
         }
         if (strncmp($line, "FAIL\t", 5) === 0) {
             $meta = array();
             preg_match('/^FAIL[\\s\\t]+(?P<test_name>\\w.*)[\\s\\t]+.*/', $line, $meta);
             $test_case_name = str_replace('/', '::', $meta['test_name']);
             $test_case_results = $this->fixNames($test_case_results, $test_case_name);
             $results = array_merge($results, $test_case_results);
             $test_case_results = array();
             continue;
         }
     }
     return $results;
 }
Esempio n. 20
0
 public function run()
 {
     $projectRoot = $this->getWorkingCopy()->getProjectRoot();
     $cwd = getcwd();
     $buildDir = $this->findBuildDirectory($projectRoot, $cwd);
     print "Using build directory '{$buildDir}'\n";
     $makeVars = $this->getMakeVars($buildDir);
     $lit = $this->findLitExecutable($makeVars);
     print "Using lit executable '{$lit}'\n";
     // We have to modify the format string, because llvm-lit does not like a '' argument
     $cmd = '%s ' . ($this->getEnableAsyncTests() ? '' : '-j1 ') . '%s 2>&1';
     $litFuture = new ExecFuture($cmd, $lit, $buildDir . "/test");
     $out = "";
     $results = array();
     $lastTime = microtime(true);
     $ready = false;
     $dots = "";
     $numTests = 0;
     while (!$ready) {
         $ready = $litFuture->isReady();
         $newout = $litFuture->readStdout();
         if (strlen($newout) == 0) {
             usleep(100);
             continue;
         }
         $out .= $newout;
         if ($ready && strlen($out) > 0 && substr($out, -1) != "\n") {
             $out .= "\n";
         }
         while (($nlPos = strpos($out, "\n")) !== FALSE) {
             $line = substr($out, 0, $nlPos + 1);
             $out = substr($out, $nlPos + 1);
             $res = ArcanistUnitTestResult::RESULT_UNSOUND;
             if (substr($line, 0, 6) == "PASS: "******"FAIL: ") {
                 $res = ArcanistUnitTestResult::RESULT_FAIL;
             } elseif (substr($line, 0, 7) == "XPASS: "******"XFAIL: ") {
                 $res = ArcanistUnitTestResult::RESULT_PASS;
             } elseif (substr($line, 0, 13) == "UNSUPPORTED: ") {
                 $res = ArcanistUnitTestResult::RESULT_SKIP;
             } elseif (!$numTests && preg_match('/Testing: ([0-9]+) tests/', $line, $matches)) {
                 $numTests = (int) $matches[1];
             }
             if ($res == ArcanistUnitTestResult::RESULT_FAIL) {
                 print "";
             }
             if ($res != ArcanistUnitTestResult::RESULT_SKIP && $res != ArcanistUnitTestResult::RESULT_PASS) {
                 print "\r" . $line . self::progress($results, $numTests);
             }
             if ($res == ArcanistUnitTestResult::RESULT_UNSOUND) {
                 continue;
             }
             $result = new ArcanistUnitTestResult();
             $result->setName(trim(substr($line, strpos($line, ':') + 1)));
             $result->setResult($res);
             $newTime = microtime(true);
             $result->setDuration($newTime - $lastTime);
             $lastTime = $newTime;
             $results[] = $result;
             $dots .= ".";
             print "\r" . self::progress($results, $numTests);
         }
     }
     list($out1, $out2) = $litFuture->read();
     print $out1;
     if ($out2) {
         throw new Exception('There was error output, even though it should have been redirected to stdout.');
     }
     print "\n";
     return $results;
 }
Esempio n. 21
0
 /**
  * Mark the current running test as skipped.
  *
  * @param string  Description for why this test was skipped.
  * @return void
  * @task internal
  */
 private final function skipTest($reason)
 {
     $coverage = $this->endCoverage();
     $result = new ArcanistUnitTestResult();
     $result->setCoverage($coverage);
     $result->setName($this->runningTest);
     $result->setResult(ArcanistUnitTestResult::RESULT_SKIP);
     $result->setDuration(microtime(true) - $this->testStartTime);
     $result->setUserData($reason);
     $this->results[] = $result;
 }
 /**
  * Parse test results from provided input and return an array
  * of @{class:ArcanistUnitTestResult}.
  *
  * @param string $test_results String containing test results
  *
  * @return array ArcanistUnitTestResult
  */
 public function parseTestResults($test_results)
 {
     if (!strlen($test_results)) {
         throw new Exception(pht('%s argument to %s must not be empty', 'test_results', 'parseTestResults()'));
     }
     // xunit xsd: https://gist.github.com/959290
     $xunit_dom = new DOMDocument();
     $load_success = @$xunit_dom->loadXML($test_results);
     if (!$load_success) {
         $input_start = id(new PhutilUTF8StringTruncator())->setMaximumGlyphs(150)->truncateString($test_results);
         throw new Exception(sprintf("%s\n\n%s", pht('Failed to load XUnit report; Input starts with:'), $input_start));
     }
     $results = array();
     $testcases = $xunit_dom->getElementsByTagName('testcase');
     foreach ($testcases as $testcase) {
         $classname = $testcase->getAttribute('classname');
         $name = $testcase->getAttribute('name');
         $time = $testcase->getAttribute('time');
         $status = ArcanistUnitTestResult::RESULT_PASS;
         $user_data = '';
         // A skipped test is a test which was ignored using framework
         // mechanisms (e.g. @skip decorator)
         $skipped = $testcase->getElementsByTagName('skipped');
         if ($skipped->length > 0) {
             $status = ArcanistUnitTestResult::RESULT_SKIP;
             $messages = array();
             for ($ii = 0; $ii < $skipped->length; $ii++) {
                 $messages[] = trim($skipped->item($ii)->nodeValue, " \n");
             }
             $user_data .= implode("\n", $messages);
         }
         // Failure is a test which the code has explicitly failed by using
         // the mechanisms for that purpose. e.g., via an assertEquals
         $failures = $testcase->getElementsByTagName('failure');
         if ($failures->length > 0) {
             $status = ArcanistUnitTestResult::RESULT_FAIL;
             $messages = array();
             for ($ii = 0; $ii < $failures->length; $ii++) {
                 $messages[] = trim($failures->item($ii)->nodeValue, " \n");
             }
             $user_data .= implode("\n", $messages) . "\n";
         }
         // An errored test is one that had an unanticipated problem. e.g., an
         // unchecked throwable, or a problem with an implementation of the test.
         $errors = $testcase->getElementsByTagName('error');
         if ($errors->length > 0) {
             $status = ArcanistUnitTestResult::RESULT_BROKEN;
             $messages = array();
             for ($ii = 0; $ii < $errors->length; $ii++) {
                 $messages[] = trim($errors->item($ii)->nodeValue, " \n");
             }
             $user_data .= implode("\n", $messages) . "\n";
         }
         $result = new ArcanistUnitTestResult();
         $result->setName($classname . '.' . $name);
         $result->setResult($status);
         $result->setDuration((double) $time);
         $result->setUserData($user_data);
         $results[] = $result;
     }
     return $results;
 }
Esempio n. 23
0
 function generateValgrindTestResults()
 {
     $this->terminateProcess();
     if ($this->coverage) {
         return $this->generateCoverageResults();
     }
     if (!$this->valgrind) {
         return array();
     }
     $definite_leaks = array();
     $possible_leaks = array();
     $errors = array();
     $descriptors = array();
     // valgrind seems to use an interesting definition of valid XML.
     // Tolerate having multiple documents in one file.
     // Confluence of weird bugs; hhvm has very low preg_match limits
     // so we have to grovel around to make sure that we read this
     // stuff in properly :-/
     $documents = array();
     $in_doc = false;
     $doc = null;
     foreach (file($this->vg_log . '.xml') as $line) {
         if ($in_doc) {
             $doc[] = $line;
             if (preg_match(',</valgrindoutput>,', $line)) {
                 $documents[] = implode("\n", $doc);
                 $doc = null;
             }
         } else {
             if (preg_match(',<valgrindoutput>,', $line)) {
                 $doc = array($line);
                 $in_doc = true;
             }
         }
     }
     libxml_use_internal_errors(true);
     foreach ($documents as $data) {
         libxml_clear_errors();
         $vg = @simplexml_load_string($data);
         if (is_object($vg)) {
             foreach ($vg->error as $err) {
                 $render = $this->renderVGResult($err);
                 switch ($err->kind) {
                     case 'Leak_DefinitelyLost':
                         $definite_leaks[] = $render;
                         break;
                     case 'Leak_PossiblyLost':
                         $possible_leaks[] = $render;
                         break;
                     default:
                         $errors[] = $render;
                 }
             }
             // These look like fd leak records, but they're not documented
             // as such.  These go away if we turn off track-fds
             foreach ($vg->stack as $stack) {
                 // Suppressing this for now: posix_spawn seems to confuse
                 // some valgrind's, particularly the version we run on travis,
                 // as it records open descriptors from the exec'ing child
                 // $descriptors[] = $this->renderVGStack($stack);
             }
         } else {
             $why = 'failed to parse xml';
             $lines = explode("\n", $data);
             foreach (libxml_get_errors() as $err) {
                 $slice = array_slice($lines, $err->line - 3, 6);
                 $slice = implode("\n", $slice);
                 $why .= sprintf("\n%s (line %d col %d) %s", $err->message, $err->line, $err->column, $slice);
             }
             printf("parsing valgrind output: %s\n", $why);
         }
     }
     $results = array();
     $res = new ArcanistUnitTestResult();
     $res->setName('valgrind possible leaks');
     $res->setUserData(implode("\n\n", $possible_leaks));
     $res->setResult(count($possible_leaks) ? ArcanistUnitTestResult::RESULT_SKIP : ArcanistUnitTestResult::RESULT_PASS);
     $results[] = $res;
     $res = new ArcanistUnitTestResult();
     $res->setName('descriptor leaks');
     $res->setUserData(implode("\n\n", $descriptors));
     $res->setResult(count($descriptors) ? ArcanistUnitTestResult::RESULT_FAIL : ArcanistUnitTestResult::RESULT_PASS);
     $results[] = $res;
     $res = new ArcanistUnitTestResult();
     $res->setName('valgrind leaks');
     $res->setUserData(implode("\n\n", $definite_leaks));
     $leak_res = count($definite_leaks) ? ArcanistUnitTestResult::RESULT_FAIL : ArcanistUnitTestResult::RESULT_PASS;
     if ($leak_res == ArcanistUnitTestResult::RESULT_FAIL && getenv('TRAVIS') == 'true') {
         // Travis has false positives at this time, downgrade
         $leak_res = ArcanistUnitTestResult::RESULT_SKIP;
     }
     $res->setResult($leak_res);
     $results[] = $res;
     $res = new ArcanistUnitTestResult();
     $res->setName('valgrind errors');
     $res->setUserData(implode("\n\n", $errors));
     $res->setResult(count($errors) ? ArcanistUnitTestResult::RESULT_FAIL : ArcanistUnitTestResult::RESULT_PASS);
     $results[] = $res;
     return $results;
 }
Esempio n. 24
0
 /**
  * Parse test results from Go test report
  * (e.g. `go test -v`)
  *
  * @param string $path Path to test
  * @param string $test_results String containing Go test output
  *
  * @return array
  */
 public function parseTestResults($path, $test_results)
 {
     $test_results = explode("\n", $test_results);
     $results = array();
     // We'll get our full test case name at the end and add it back in
     $test_case_name = '';
     // Temp store for test case results (in case we run multiple test cases)
     $test_case_results = array();
     foreach ($test_results as $i => $line) {
         if (strncmp($line, '--- PASS', 8) === 0) {
             // We have a passing test
             $meta = array();
             preg_match('/^--- PASS: (?P<test_name>.+) \\((?P<time>.+) seconds\\).*/', $line, $meta);
             $result = new ArcanistUnitTestResult();
             // For now set name without test case, we'll add it later
             $result->setName($meta['test_name']);
             $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
             $result->setDuration($meta['time']);
             $test_case_results[] = $result;
             continue;
         }
         if (strncmp($line, '--- FAIL', 8) === 0) {
             // We have a failing test
             $reason = trim($test_results[$i + 1]);
             $meta = array();
             preg_match('/^--- FAIL: (?P<test_name>.+) \\((?P<time>.+) seconds\\).*/', $line, $meta);
             $result = new ArcanistUnitTestResult();
             $result->setName($meta['test_name']);
             $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
             $result->setDuration($meta['time']);
             $result->setUserData($reason . "\n");
             $test_case_results[] = $result;
             continue;
         }
         if (strncmp($line, 'ok', 2) === 0) {
             $meta = array();
             preg_match('/^ok[\\s\\t]+(?P<test_name>\\w.*)[\\s\\t]+(?P<time>.*)s.*/', $line, $meta);
             $test_case_name = str_replace('/', '::', $meta['test_name']);
             // Our test case passed
             // check to make sure we were in verbose (-v) mode
             if (empty($test_case_results)) {
                 // We weren't in verbose mode
                 // create one successful result for the whole test case
                 $test_name = 'Go::TestCase::' . $test_case_name;
                 $result = new ArcanistUnitTestResult();
                 $result->setName($test_name);
                 $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
                 $result->setDuration($meta['time']);
                 $results[] = $result;
             } else {
                 $test_case_results = $this->fixNames($test_case_results, $test_case_name);
                 $results = array_merge($results, $test_case_results);
                 $test_case_results = array();
             }
             continue;
         }
         if (strncmp($line, "FAIL\t", 5) === 0) {
             $meta = array();
             preg_match('/^FAIL[\\s\\t]+(?P<test_name>\\w.*)[\\s\\t]+.*/', $line, $meta);
             $test_case_name = str_replace('/', '::', $meta['test_name']);
             $test_case_results = $this->fixNames($test_case_results, $test_case_name);
             $results = array_merge($results, $test_case_results);
             $test_case_results = array();
             continue;
         }
     }
     return $results;
 }
Esempio n. 25
0
 /**
  * Parses the test results from xUnit.
  *
  * @param  string  The name of the xUnit results file.
  * @param  string  The name of the coverage file if one was provided by
  *                 `buildTestFuture`. This is passed through to
  *                 `parseCoverageResult`.
  * @return array   Test results.
  */
 private function parseTestResult($xunit_tmp, $coverage)
 {
     $xunit_dom = new DOMDocument();
     $xunit_dom->loadXML(Filesystem::readFile($xunit_tmp));
     $results = array();
     $tests = $xunit_dom->getElementsByTagName('test');
     foreach ($tests as $test) {
         $name = $test->getAttribute('name');
         $time = $test->getAttribute('time');
         $status = ArcanistUnitTestResult::RESULT_UNSOUND;
         switch ($test->getAttribute('result')) {
             case 'Pass':
                 $status = ArcanistUnitTestResult::RESULT_PASS;
                 break;
             case 'Fail':
                 $status = ArcanistUnitTestResult::RESULT_FAIL;
                 break;
             case 'Skip':
                 $status = ArcanistUnitTestResult::RESULT_SKIP;
                 break;
         }
         $userdata = '';
         $reason = $test->getElementsByTagName('reason');
         $failure = $test->getElementsByTagName('failure');
         if ($reason->length > 0 || $failure->length > 0) {
             $node = $reason->length > 0 ? $reason : $failure;
             $message = $node->item(0)->getElementsByTagName('message');
             if ($message->length > 0) {
                 $userdata = $message->item(0)->nodeValue;
             }
             $stacktrace = $node->item(0)->getElementsByTagName('stack-trace');
             if ($stacktrace->length > 0) {
                 $userdata .= "\n" . $stacktrace->item(0)->nodeValue;
             }
         }
         $result = new ArcanistUnitTestResult();
         $result->setName($name);
         $result->setResult($status);
         $result->setDuration($time);
         $result->setUserData($userdata);
         if ($coverage != null) {
             $result->setCoverage($this->parseCoverageResult($coverage));
         }
         $results[] = $result;
     }
     return $results;
 }
 /**
  * Parse test results from phpunit json report
  *
  * @param string $path Path to test
  * @param string $test_results String containing phpunit json report
  *
  * @return array
  */
 public function parseTestResults($path, $test_results)
 {
     if (!$test_results) {
         $result = id(new ArcanistUnitTestResult())->setName($path)->setUserData($this->stderr)->setResult(ArcanistUnitTestResult::RESULT_BROKEN);
         return array($result);
     }
     $report = $this->getJsonReport($test_results);
     // coverage is for all testcases in the executed $path
     $coverage = array();
     if ($this->enableCoverage !== false) {
         $coverage = $this->readCoverage();
     }
     $last_test_finished = true;
     $results = array();
     foreach ($report as $event) {
         switch (idx($event, 'event')) {
             case 'test':
                 break;
             case 'testStart':
                 $last_test_finished = false;
                 // fall through
             // fall through
             default:
                 continue 2;
                 // switch + loop
         }
         $status = ArcanistUnitTestResult::RESULT_PASS;
         $user_data = '';
         if ('fail' == idx($event, 'status')) {
             $status = ArcanistUnitTestResult::RESULT_FAIL;
             $user_data .= idx($event, 'message') . "\n";
             foreach (idx($event, 'trace') as $trace) {
                 $user_data .= sprintf("\n%s:%s", idx($trace, 'file'), idx($trace, 'line'));
             }
         } else {
             if ('error' == idx($event, 'status')) {
                 if (strpos(idx($event, 'message'), 'Skipped Test') !== false) {
                     $status = ArcanistUnitTestResult::RESULT_SKIP;
                     $user_data .= idx($event, 'message');
                 } else {
                     if (strpos(idx($event, 'message'), 'Incomplete Test') !== false) {
                         $status = ArcanistUnitTestResult::RESULT_SKIP;
                         $user_data .= idx($event, 'message');
                     } else {
                         $status = ArcanistUnitTestResult::RESULT_BROKEN;
                         $user_data .= idx($event, 'message');
                         foreach (idx($event, 'trace') as $trace) {
                             $user_data .= sprintf("\n%s:%s", idx($trace, 'file'), idx($trace, 'line'));
                         }
                     }
                 }
             }
         }
         $name = preg_replace('/ \\(.*\\)/s', '', idx($event, 'test'));
         $result = new ArcanistUnitTestResult();
         $result->setName($name);
         $result->setResult($status);
         $result->setDuration(idx($event, 'time'));
         $result->setCoverage($coverage);
         $result->setUserData($user_data);
         $results[] = $result;
         $last_test_finished = true;
     }
     if (!$last_test_finished) {
         $results[] = id(new ArcanistUnitTestResult())->setName(idx($event, 'test'))->setUserData($this->stderr)->setResult(ArcanistUnitTestResult::RESULT_BROKEN);
     }
     return $results;
 }
Esempio n. 27
0
 public function run()
 {
     $projectRoot = $this->getWorkingCopy()->getProjectRoot();
     $cwd = getcwd();
     $buildDir = $this->findBuildDirectory($projectRoot, $cwd);
     $polliObjDir = $buildDir;
     if (is_dir($buildDir . DIRECTORY_SEPARATOR . "tools" . DIRECTORY_SEPARATOR . "polly" . DIRECTORY_SEPARATOR . "tools" . DIRECTORY_SEPARATOR . "polli")) {
         $polliObjDir = $buildDir . DIRECTORY_SEPARATOR . "tools" . DIRECTORY_SEPARATOR . "polly" . DIRECTORY_SEPARATOR . "tools" . DIRECTORY_SEPARATOR . "polli";
     }
     $polliTestDir = $polliObjDir . DIRECTORY_SEPARATOR . "test";
     if (is_dir($buildDir . DIRECTORY_SEPARATOR . "bin") && file_exists($buildDir . DIRECTORY_SEPARATOR . "bin" . DIRECTORY_SEPARATOR . "llvm-lit")) {
         $lit = $buildDir . DIRECTORY_SEPARATOR . "bin" . DIRECTORY_SEPARATOR . "llvm-lit";
         $cmd = "ninja -C " . $buildDir;
         print "Running ninja (" . $cmd . ")\n";
         exec($cmd);
     } else {
         $makeVars = $this->getMakeVars($buildDir);
         $lit = $this->findLitExecutable($makeVars);
     }
     print "Using lit executable '{$lit}'\n";
     // We have to modify the format string, because llvm-lit does not like a '' argument
     $cmd = '%s %s 2>&1';
     $litFuture = new ExecFuture($cmd, $lit, $polliTestDir);
     $out = "";
     $results = array();
     $lastTime = microtime(true);
     $ready = false;
     $dots = "";
     $numTests = 0;
     while (!$ready) {
         $ready = $litFuture->isReady();
         $newout = $litFuture->readStdout();
         if (strlen($newout) == 0) {
             usleep(100);
             continue;
         }
         $out .= $newout;
         if ($ready && strlen($out) > 0 && substr($out, -1) != "\n") {
             $out .= "\n";
         }
         while (($nlPos = strpos($out, "\n")) !== FALSE) {
             $line = substr($out, 0, $nlPos + 1);
             $out = substr($out, $nlPos + 1);
             $res = ArcanistUnitTestResult::RESULT_UNSOUND;
             if (substr($line, 0, 6) == "PASS: "******"FAIL: ") {
                 $res = ArcanistUnitTestResult::RESULT_FAIL;
             } elseif (substr($line, 0, 7) == "XPASS: "******"XFAIL: ") {
                 $res = ArcanistUnitTestResult::RESULT_PASS;
             } elseif (substr($line, 0, 13) == "UNSUPPORTED: ") {
                 $res = ArcanistUnitTestResult::RESULT_SKIP;
             } elseif (!$numTests && preg_match('/Testing: ([0-9]+) tests/', $line, $matches)) {
                 $numTests = (int) $matches[1];
             }
             if ($res == ArcanistUnitTestResult::RESULT_FAIL) {
                 print "";
             }
             if ($res != ArcanistUnitTestResult::RESULT_SKIP && $res != ArcanistUnitTestResult::RESULT_PASS) {
                 print "" . $line . self::progress($results, $numTests);
             }
             if ($res == ArcanistUnitTestResult::RESULT_UNSOUND) {
                 continue;
             }
             $result = new ArcanistUnitTestResult();
             $result->setName(trim(substr($line, strpos($line, ':') + 1)));
             $result->setResult($res);
             $newTime = microtime(true);
             $result->setDuration($newTime - $lastTime);
             $lastTime = $newTime;
             $results[] = $result;
             $dots .= ".";
             print "\r" . self::progress($results, $numTests);
         }
     }
     list($out1, $out2) = $litFuture->read();
     print $out1;
     if ($out2) {
         throw new Exception('There was error output, even though it should have been redirected to stdout.');
     }
     print "\n";
     $timeThreshold = 0.05;
     $interestingTests = array();
     foreach ($results as $result) {
         if ($result->getResult() != "pass") {
             $interestingTests[] = $result;
         }
         if ($result->getDuration() > $timeThreshold) {
             $interestingTests[] = $result;
         }
     }
     return $interestingTests;
 }
Esempio n. 28
0
 public function parseTestResults($path, $xunit_tmp, $cover_tmp)
 {
     // xunit xsd: https://gist.github.com/959290
     $xunit_dom = new DOMDocument();
     $xunit_dom->loadXML(Filesystem::readFile($xunit_tmp));
     // coverage is for all testcases in the executed $path
     $coverage = array();
     if ($this->getEnableCoverage() !== false) {
         $coverage = $this->readCoverage($cover_tmp);
     }
     $results = array();
     $testcases = $xunit_dom->getElementsByTagName("testcase");
     foreach ($testcases as $testcase) {
         $classname = $testcase->getAttribute("classname");
         $name = $testcase->getAttribute("name");
         $time = $testcase->getAttribute("time");
         $status = ArcanistUnitTestResult::RESULT_PASS;
         $user_data = "";
         // A skipped test is a test which was ignored using framework
         // mechanizms (e.g. @skip decorator)
         $skipped = $testcase->getElementsByTagName("skipped");
         if ($skipped->length > 0) {
             $status = ArcanistUnitTestResult::RESULT_SKIP;
             $messages = array();
             for ($ii = 0; $ii < $skipped->length; $ii++) {
                 $messages[] = trim($skipped->item($ii)->nodeValue, " \n");
             }
             $user_data .= implode("\n", $messages);
         }
         // Failure is a test which the code has explicitly failed by using
         // the mechanizms for that purpose. e.g., via an assertEquals
         $failures = $testcase->getElementsByTagName("failure");
         if ($failures->length > 0) {
             $status = ArcanistUnitTestResult::RESULT_FAIL;
             $messages = array();
             for ($ii = 0; $ii < $failures->length; $ii++) {
                 $messages[] = trim($failures->item($ii)->nodeValue, " \n");
             }
             $user_data .= implode("\n", $messages) . "\n";
         }
         // An errored test is one that had an unanticipated problem. e.g., an
         // unchecked throwable, or a problem with an implementation of the
         // test.
         $errors = $testcase->getElementsByTagName("error");
         if ($errors->length > 0) {
             $status = ArcanistUnitTestResult::RESULT_BROKEN;
             $messages = array();
             for ($ii = 0; $ii < $errors->length; $ii++) {
                 $messages[] = trim($errors->item($ii)->nodeValue, " \n");
             }
             $user_data .= implode("\n", $messages) . "\n";
         }
         $result = new ArcanistUnitTestResult();
         $result->setName($classname . "." . $name);
         $result->setResult($status);
         $result->setDuration($time);
         $result->setCoverage($coverage);
         $result->setUserData($user_data);
         $results[] = $result;
     }
     return $results;
 }
Esempio n. 29
0
 private final function resultTest($test_result, $reason)
 {
     $coverage = $this->endCoverage();
     $result = new ArcanistUnitTestResult();
     $result->setCoverage($coverage);
     $result->setNamespace(get_class($this));
     $result->setName($this->runningTest);
     $result->setLink($this->getLink($this->runningTest));
     $result->setResult($test_result);
     $result->setDuration(microtime(true) - $this->testStartTime);
     $result->setUserData($reason);
     $this->results[] = $result;
     if ($this->renderer) {
         echo $this->renderer->renderUnitResult($result);
     }
 }
Esempio n. 30
0
 private function parseTestResults($targets, $status)
 {
     $code = $status[0];
     $output = $status[1];
     $lines = explode("\n", $output);
     if ($code == 4) {
         print "No tests affected...\n";
         return [];
     } else {
         if ($code == 1) {
             throw new Exception($output . "\n" . $status[2]);
         }
     }
     $query_command = $this->bazelCommand(["query", "-k", "%s"]);
     $future = new ExecFuture($query_command, 'tests(set(' . join(" ", $targets) . '))');
     $future->setCWD($this->project_root);
     $testTargets = explode("\n", trim($future->resolvex()[0]));
     $results = array();
     foreach ($testTargets as $test) {
         $data = $this->parseTestResultFile($test);
         $result = new ArcanistUnitTestResult();
         $result->setName($test);
         if (property_exists($data, "test_case")) {
             $testCase = $data->{"test_case"};
             if (property_exists($testCase, "run_duration_millis")) {
                 $result->setDuration($testCase->{"run_duration_millis"} / 1000);
             }
         }
         if ($data->{"test_passed"}) {
             $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
         } else {
             if ($data->{"status"} == 4) {
                 $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
             } else {
                 $result->setResult(ArcanistUnitTestResult::RESULT_BROKEN);
             }
         }
         $results[] = $result;
     }
     return $results;
 }