public function runTests($test_paths, $source_path)
{
if (empty($test_paths)) {
return array();
}
$futures = array();
$tmpfiles = array();
foreach ($test_paths as $test_path) {
$xunit_tmp = new TempFile();
$cover_tmp = new TempFile();
$future = $this->buildTestFuture($test_path, $xunit_tmp, $cover_tmp);
$futures[$test_path] = $future;
$tmpfiles[$test_path] = array('xunit' => $xunit_tmp, 'cover' => $cover_tmp);
}
$results = array();
foreach (Futures($futures)->limit(4) as $test_path => $future) {
try {
list($stdout, $stderr) = $future->resolvex();
} catch (CommandException $exc) {
if ($exc->getError() > 1) {
// 'nose' returns 1 when tests are failing/broken.
throw $exc;
}
}
$xunit_tmp = $tmpfiles[$test_path]['xunit'];
$cover_tmp = $tmpfiles[$test_path]['cover'];
$this->parser = new ArcanistXUnitTestResultParser();
$results[] = $this->parseTestResults($source_path, $xunit_tmp, $cover_tmp);
}
return array_mergev($results);
}
public function willLintPaths(array $paths)
{
$program = false;
$ret_value = 0;
$last_line = system("which checkCpp", $ret_value);
if ($ret_value == 0) {
$program = $last_line;
} else {
if (file_exists(self::PROGRAM)) {
$program = self::PROGRAM;
}
}
if ($program) {
$futures = array();
foreach ($paths as $p) {
$futures[$p] = new ExecFuture("%s --lint %s 2>&1", $program, $this->getEngine()->getFilePathOnDisk($p));
}
foreach (Futures($futures)->limit(8) as $p => $f) {
list($stdout, $stderr) = $f->resolvex();
$raw = json_decode($stdout, true);
if (!is_array($raw)) {
throw new Exception("checkCpp returned invalid JSON!" . "Stdout: {$stdout} Stderr: {$stderr}");
}
foreach ($raw as $err) {
$this->addLintMessage(ArcanistLintMessage::newFromDictionary(array('path' => $err['file'], 'line' => $err['line'], 'char' => 0, 'name' => $err['name'], 'description' => $err['info'], 'code' => $this->getLinterName(), 'severity' => ArcanistLintSeverity::SEVERITY_WARNING)));
}
}
}
return;
}
public function willLintPaths(array $paths)
{
$futures = array();
foreach ($paths as $path) {
if (array_key_exists($path, $this->trees)) {
continue;
}
$futures[$path] = xhpast_get_parser_future($this->getData($path));
}
foreach (Futures($futures)->limit(8) as $path => $future) {
$this->willLintPath($path);
$this->trees[$path] = null;
try {
$this->trees[$path] = XHPASTTree::newFromDataAndResolvedExecFuture($this->getData($path), $future->resolve());
$root = $this->trees[$path]->getRootNode();
$root->buildSelectCache();
$root->buildTokenCache();
} catch (XHPASTSyntaxErrorException $ex) {
$this->raiseLintAtLine($ex->getErrorLine(), 1, self::LINT_PHP_SYNTAX_ERROR, 'This file contains a syntax error: ' . $ex->getMessage());
$this->stopAllLinters();
return;
} catch (Exception $ex) {
$this->raiseLintAtPath(self::LINT_UNABLE_TO_PARSE, 'XHPAST could not parse this file, probably because the AST is too ' . 'deep. Some lint issues may not have been detected. You may safely ' . 'ignore this warning.');
return;
}
}
}
private function renderGrepResults(array $results)
{
$drequest = $this->getDiffusionRequest();
require_celerity_resource('syntax-highlighting-css');
// NOTE: This can be wrong because we may find the string inside the
// comment. But it's correct in most cases and highlighting the whole file
// would be too expensive.
$futures = array();
$engine = PhabricatorSyntaxHighlighter::newEngine();
foreach ($results as $result) {
list($path, $line, $string) = $result;
$futures["{$path}:{$line}"] = $engine->getHighlightFuture($engine->getLanguageFromFilename($path), ltrim($string));
}
try {
Futures($futures)->limit(8)->resolveAll();
} catch (PhutilSyntaxHighlighterException $ex) {
}
$rows = array();
foreach ($results as $result) {
list($path, $line, $string) = $result;
$href = $drequest->generateURI(array('action' => 'browse', 'path' => $path, 'line' => $line));
try {
$string = $futures["{$path}:{$line}"]->resolve();
} catch (PhutilSyntaxHighlighterException $ex) {
}
$string = phutil_tag('pre', array('class' => 'PhabricatorMonospaced'), $string);
$path = Filesystem::readablePath($path, $drequest->getPath());
$rows[] = array(phutil_tag('a', array('href' => $href), $path), $line, $string);
}
$table = id(new AphrontTableView($rows))->setClassName('remarkup-code')->setHeaders(array(pht('Path'), pht('Line'), pht('String')))->setColumnClasses(array('', 'n', 'wide'))->setNoDataString(pht('The pattern you searched for was not found in the content of any ' . 'files.'));
return $table;
}
public function willLintPaths(array $paths)
{
$futures = array();
$ret_value = 0;
$last_line = system("which cpplint", $ret_value);
$CPP_LINT = false;
if ($ret_value == 0) {
$CPP_LINT = $last_line;
} else {
if (file_exists(self::CPPLINT)) {
$CPP_LINT = self::CPPLINT;
}
}
if ($CPP_LINT) {
foreach ($paths as $p) {
$lpath = $this->getEngine()->getFilePathOnDisk($p);
$lpath_file = file($lpath);
if (preg_match('/\\.(c)$/', $lpath) || preg_match('/-\\*-.*Mode: C[; ].*-\\*-/', $lpath_file[0]) || preg_match('/vim(:.*)*:\\s*(set\\s+)?filetype=c\\s*:/', $lpath_file[0])) {
$futures[$p] = new ExecFuture("%s %s %s 2>&1", $CPP_LINT, self::C_FLAG, $this->getEngine()->getFilePathOnDisk($p));
} else {
$futures[$p] = new ExecFuture("%s %s 2>&1", $CPP_LINT, $this->getEngine()->getFilePathOnDisk($p));
}
}
foreach (Futures($futures)->limit(8) as $p => $f) {
$this->rawLintOutput[$p] = $f->resolvex();
}
}
return;
}
public final function willLintPaths(array $paths)
{
$limit = $this->getFuturesLimit();
$this->futures = Futures(array())->limit($limit);
foreach ($this->buildFutures($paths) as $path => $future) {
$this->futures->addFuture($future, $path);
}
}
public function execute(HarbormasterBuild $build, HarbormasterBuildTarget $build_target)
{
$settings = $this->getSettings();
$variables = $build_target->getVariables();
$artifact = $build->loadArtifact($settings['hostartifact']);
$lease = $artifact->loadDrydockLease();
$this->platform = $lease->getAttribute('platform');
$command = $this->mergeVariables(array($this, 'escapeCommand'), $settings['command'], $variables);
$this->platform = null;
$interface = $lease->getInterface('command');
$future = $interface->getExecFuture('%C', $command);
$log_stdout = $build->createLog($build_target, 'remote', 'stdout');
$log_stderr = $build->createLog($build_target, 'remote', 'stderr');
$start_stdout = $log_stdout->start();
$start_stderr = $log_stderr->start();
$build_update = 5;
// Read the next amount of available output every second.
$futures = Futures(array($future));
foreach ($futures->setUpdateInterval(1) as $key => $future_iter) {
if ($future_iter === null) {
// Check to see if we should abort.
if ($build_update <= 0) {
$build->reload();
if ($this->shouldAbort($build, $build_target)) {
$future->resolveKill();
throw new HarbormasterBuildAbortedException();
} else {
$build_update = 5;
}
} else {
$build_update -= 1;
}
// Command is still executing.
// Read more data as it is available.
list($stdout, $stderr) = $future->read();
$log_stdout->append($stdout);
$log_stderr->append($stderr);
$future->discardBuffers();
} else {
// Command execution is complete.
// Get the return value so we can log that as well.
list($err) = $future->resolve();
// Retrieve the last few bits of information.
list($stdout, $stderr) = $future->read();
$log_stdout->append($stdout);
$log_stderr->append($stderr);
$future->discardBuffers();
break;
}
}
$log_stdout->finalize($start_stdout);
$log_stderr->finalize($start_stderr);
if ($err) {
throw new HarbormasterBuildFailureException();
}
}
public function run()
{
$this->projectRoot = $this->getWorkingCopy()->getProjectRoot();
$this->affectedTests = array();
foreach ($this->getPaths() as $path) {
$path = Filesystem::resolvePath($path, $this->projectRoot);
// TODO: add support for directories
// Users can call phpunit on the directory themselves
if (is_dir($path)) {
continue;
}
// Not sure if it would make sense to go further if
// it is not a .php file
if (substr($path, -4) != '.php') {
continue;
}
if (substr($path, -8) == 'Test.php') {
// Looks like a valid test file name.
$this->affectedTests[$path] = $path;
continue;
}
if ($test = $this->findTestFile($path)) {
$this->affectedTests[$path] = $test;
}
}
if (empty($this->affectedTests)) {
throw new ArcanistNoEffectException('No tests to run.');
}
$this->prepareConfigFile();
$futures = array();
$tmpfiles = array();
foreach ($this->affectedTests as $class_path => $test_path) {
if (!Filesystem::pathExists($test_path)) {
continue;
}
$json_tmp = new TempFile();
$clover_tmp = null;
$clover = null;
if ($this->getEnableCoverage() !== false) {
$clover_tmp = new TempFile();
$clover = csprintf('--coverage-clover %s', $clover_tmp);
}
$config = $this->configFile ? csprintf('-c %s', $this->configFile) : null;
$stderr = '-d display_errors=stderr';
$futures[$test_path] = new ExecFuture('%C %C %C --log-json %s %C %s', $this->phpunitBinary, $config, $stderr, $json_tmp, $clover, $test_path);
$tmpfiles[$test_path] = array('json' => $json_tmp, 'clover' => $clover_tmp);
}
$results = array();
foreach (Futures($futures)->limit(4) as $test => $future) {
list($err, $stdout, $stderr) = $future->resolve();
$results[] = $this->parseTestResults($test, $tmpfiles[$test]['json'], $tmpfiles[$test]['clover'], $stderr);
}
return array_mergev($results);
}
public function testMultipleTimeoutsTestShouldRunLessThan1Sec()
{
$futures = array();
for ($ii = 0; $ii < 4; $ii++) {
$futures[] = id(new ExecFuture('sleep 32000'))->setTimeout(0.01);
}
foreach (Futures($futures) as $future) {
list($err) = $future->resolve();
$this->assertEqual(true, $err > 0);
$this->assertEqual(true, $future->getWasKilledByTimeout());
}
}
public function run()
{
$paths = $this->getPaths();
$affected_tests = array();
foreach ($paths as $path) {
$absolute_path = Filesystem::resolvePath($path);
if (is_dir($absolute_path)) {
$absolute_test_path = Filesystem::resolvePath("tests/" . $path);
if (is_readable($absolute_test_path)) {
$affected_tests[] = $absolute_test_path;
}
}
if (is_readable($absolute_path)) {
$filename = basename($path);
$directory = dirname($path);
// assumes directory layout: tests/<package>/test_<module>.py
$relative_test_path = "tests/" . $directory . "/test_" . $filename;
$absolute_test_path = Filesystem::resolvePath($relative_test_path);
if (is_readable($absolute_test_path)) {
$affected_tests[] = $absolute_test_path;
}
}
}
if (empty($affected_tests)) {
return array();
}
$futures = array();
$tmpfiles = array();
foreach ($affected_tests as $test_path) {
$xunit_tmp = new TempFile();
$cover_tmp = new TempFile();
$future = $this->buildTestFuture($test_path, $xunit_tmp, $cover_tmp);
$futures[$test_path] = $future;
$tmpfiles[$test_path] = array('xunit' => $xunit_tmp, 'cover' => $cover_tmp);
}
$results = array();
foreach (Futures($futures)->limit(4) as $test_path => $future) {
try {
list($stdout, $stderr) = $future->resolvex();
} catch (CommandException $exc) {
if ($exc->getError() > 1) {
// 'nose' returns 1 when tests are failing/broken.
throw $exc;
}
}
$xunit_tmp = $tmpfiles[$test_path]['xunit'];
$cover_tmp = $tmpfiles[$test_path]['cover'];
$results[] = $this->parseTestResults($test_path, $xunit_tmp, $cover_tmp);
}
return array_mergev($results);
}
public function testManyWriters()
{
$root = phutil_get_library_root('phutil') . '/../';
$bin = $root . 'scripts/test/deferred_log.php';
$n_writers = 3;
$n_lines = 8;
$tmp = new TempFile();
$futures = array();
for ($ii = 0; $ii < $n_writers; $ii++) {
$futures[] = new ExecFuture('%s %d %s', $bin, $n_lines, (string) $tmp);
}
Futures($futures)->resolveAll();
$this->assertEqual(str_repeat("abcdefghijklmnopqrstuvwxyz\n", $n_writers * $n_lines), Filesystem::readFile($tmp));
}
public function willLintPaths(array $paths)
{
$script = $this->getConfiguredScript();
$root = $this->getEngine()->getWorkingCopy()->getProjectRoot();
$futures = array();
foreach ($paths as $path) {
$future = new ExecFuture('%C %s', $script, $path);
$future->setCWD($root);
$futures[$path] = $future;
}
foreach (Futures($futures)->limit(4) as $path => $future) {
list($stdout) = $future->resolvex();
$this->output[$path] = $stdout;
}
}
public function willLintPaths(array $paths)
{
$phpcs_bin = $this->getPhpcsPath();
$phpcs_options = $this->getPhpcsOptions();
$futures = array();
foreach ($paths as $path) {
$filepath = $this->getEngine()->getFilePathOnDisk($path);
$this->reports[$path] = new TempFile();
$futures[$path] = new ExecFuture('%C %C --report=xml --report-file=%s %s', $phpcs_bin, $phpcs_options, $this->reports[$path], $filepath);
}
foreach (Futures($futures)->limit(8) as $path => $future) {
$this->results[$path] = $future->resolve();
}
libxml_use_internal_errors(true);
}
public function willParseFiles(array $file_map)
{
$root = dirname(phutil_get_library_root('javelin-diviner'));
$bin = $root . '/jsast/jsast';
if (!Filesystem::pathExists($bin)) {
throw new Exception("You must build the 'jsast' binary before you can generate " . "Javelin documentation.");
}
$futures = array();
foreach ($file_map as $file => $data) {
$future = new ExecFuture($bin);
$future->write($data);
$futures[$file] = $future;
}
foreach (Futures($futures)->limit(8) as $file => $future) {
$this->trees[$file] = $future->resolveJSON();
}
}
public final function willLintPaths(array $paths)
{
$futures = array();
foreach ($paths as $path) {
if (!$this->shouldLintPath($path)) {
continue;
}
$changed = $this->changedLines[$path];
if (!isset($changed)) {
// do not run linter if there are no changes
continue;
}
$futures[$path] = $this->getFormatFuture($path, $changed);
}
foreach (Futures($futures)->limit(8) as $p => $f) {
$this->rawLintOutput[$p] = $f->resolvex();
}
}
public function willLintPaths(array $paths)
{
if ($this->haveSymbolsBinary === null) {
$binary = $this->getSymbolsBinaryPath();
$this->haveSymbolsBinary = Filesystem::pathExists($binary);
if (!$this->haveSymbolsBinary) {
return;
}
}
$futures = array();
foreach ($paths as $path) {
$future = $this->newSymbolsFuture($path);
$futures[$path] = $future;
}
foreach (Futures($futures)->limit(8) as $path => $future) {
$this->symbols[$path] = $future->resolvex();
}
}
public function willLintPaths(array $paths)
{
if (!$this->getBinaryPath()) {
return;
}
$root = dirname(phutil_get_library_root('phabricator'));
require_once $root . '/scripts/__init_script__.php';
$futures = array();
foreach ($paths as $path) {
if ($this->shouldIgnorePath($path)) {
continue;
}
$future = $this->newSymbolsFuture($path);
$futures[$path] = $future;
}
foreach (Futures($futures)->limit(8) as $path => $future) {
$this->symbols[$path] = $future->resolvex();
}
}
public function willLintPaths(array $paths)
{
$root = dirname(phutil_get_library_root('phabricator'));
require_once $root . '/scripts/__init_script__.php';
if ($this->haveSymbolsBinary === null) {
$binary = $this->getSymbolsBinaryPath();
$this->haveSymbolsBinary = Filesystem::pathExists($binary);
if (!$this->haveSymbolsBinary) {
return;
}
}
$futures = array();
foreach ($paths as $path) {
$future = $this->newSymbolsFuture($path);
$futures[$path] = $future;
}
foreach (Futures($futures)->limit(8) as $path => $future) {
$this->symbols[$path] = $future->resolvex();
}
}
public function willLintPaths(array $paths)
{
if (!file_exists(self::FLINT)) {
return;
}
$futures = array();
foreach ($paths as $p) {
$lpath = $this->getEngine()->getFilePathOnDisk($p);
$lpath_file = file($lpath);
if (preg_match('/\\.(c)$/', $lpath) || preg_match('/-\\*-.*Mode: C[; ].*-\\*-/', $lpath_file[0]) || preg_match('/vim(:.*)*:\\s*(set\\s+)?filetype=c\\s*:/', $lpath_file[0])) {
$futures[$p] = new ExecFuture("%s %s %s 2>&1", self::FLINT, self::C_FLAG, $this->getEngine()->getFilePathOnDisk($p));
} else {
$futures[$p] = new ExecFuture("%s %s 2>&1", self::FLINT, $this->getEngine()->getFilePathOnDisk($p));
}
}
foreach (Futures($futures)->limit(8) as $p => $f) {
$this->rawLintOutput[$p] = $f->resolvex();
}
return;
}
require_once $root . '/scripts/__init_script__.php';
if ($argc !== 1 || posix_isatty(STDIN)) {
echo phutil_console_format("usage: find . -type f -name '*.php' | ./generate_php_symbols.php\n");
exit(1);
}
$input = file_get_contents('php://stdin');
$input = trim($input);
$input = explode("\n", $input);
$data = array();
$futures = array();
foreach ($input as $file) {
$file = Filesystem::readablePath($file);
$data[$file] = Filesystem::readFile($file);
$futures[$file] = xhpast_get_parser_future($data[$file]);
}
foreach (Futures($futures)->limit(8) as $file => $future) {
$tree = XHPASTTree::newFromDataAndResolvedExecFuture($data[$file], $future->resolve());
$root = $tree->getRootNode();
$scopes = array();
$functions = $root->selectDescendantsOfType('n_FUNCTION_DECLARATION');
foreach ($functions as $function) {
$name = $function->getChildByIndex(2);
print_symbol($file, 'function', $name);
}
$classes = $root->selectDescendantsOfType('n_CLASS_DECLARATION');
foreach ($classes as $class) {
$class_name = $class->getChildByIndex(1);
print_symbol($file, 'class', $class_name);
$scopes[] = array($class, $class_name);
}
$interfaces = $root->selectDescendantsOfType('n_INTERFACE_DECLARATION');
private function loadRevisions(array $branches)
{
$ids = array();
$hashes = array();
foreach ($branches as $branch) {
if ($branch['revisionID']) {
$ids[] = $branch['revisionID'];
}
$hashes[] = array('gtcm', $branch['hash']);
$hashes[] = array('gttr', $branch['tree']);
}
$calls = array();
if ($ids) {
$calls[] = $this->getConduit()->callMethod('differential.query', array('ids' => $ids));
}
if ($hashes) {
$calls[] = $this->getConduit()->callMethod('differential.query', array('commitHashes' => $hashes));
}
$results = array();
foreach (Futures($calls) as $call) {
$results[] = $call->resolve();
}
return array_mergev($results);
}
private function loadMessagesForTags(array $tags)
{
assert_instances_of($tags, 'DiffusionRepositoryTag');
$drequest = $this->getDiffusionRequest();
$repository = $drequest->getRepository();
$futures = array();
foreach ($tags as $key => $tag) {
$futures[$key] = $repository->getLocalCommandFuture('cat-file tag %s', $tag->getName());
}
Futures($futures)->resolveAll();
foreach ($tags as $key => $tag) {
$future = $futures[$key];
list($err, $stdout) = $future->resolve();
$message = null;
if ($err) {
// Not all tags are actually "tag" objects: a "tag" object is only
// created if you provide a message or sign the tag. Tags created with
// `git tag x [commit]` are "lightweight tags" and `git cat-file tag`
// will fail on them. This is fine: they don't have messages.
} else {
$parts = explode("\n\n", $stdout, 2);
if (count($parts) == 2) {
$message = last($parts);
}
}
$tag->attachMessage($message);
}
return $tags;
}
phutil_require_module('arcanist', 'parser/phutilmodule');
$data = array();
$futures = array();
foreach (Filesystem::listDirectory($dir, $hidden_files = false) as $file) {
if (!preg_match('/.php$/', $file)) {
continue;
}
$data[$file] = Filesystem::readFile($dir . '/' . $file);
$futures[$file] = xhpast_get_parser_future($data[$file]);
}
$requirements = new PhutilModuleRequirements();
$requirements->addBuiltins($builtin);
$doc_parser = new PhutilDocblockParser();
$has_init = false;
$has_files = false;
foreach (Futures($futures) as $file => $future) {
try {
$tree = XHPASTTree::newFromDataAndResolvedExecFuture($data[$file], $future->resolve());
} catch (XHPASTSyntaxErrorException $ex) {
echo "Syntax Error! In '{$file}': " . $ex->getMessage() . "\n";
exit(1);
}
$root = $tree->getRootNode();
$requirements->setCurrentFile($file);
if ($file == '__init__.php') {
$has_init = true;
$calls = $root->selectDescendantsOfType('n_FUNCTION_CALL');
foreach ($calls as $call) {
$name = $call->getChildByIndex(0);
$call_name = $name->getConcreteString();
if ($call_name == 'phutil_require_source') {
protected function primeSubversionWorkingCopyData($paths)
{
$repository_api = $this->getRepositoryAPI();
$futures = array();
$targets = array();
foreach ($paths as $path => $mask) {
$futures[] = $repository_api->buildDiffFuture($path);
$targets[] = array('command' => 'diff', 'path' => $path);
$futures[] = $repository_api->buildInfoFuture($path);
$targets[] = array('command' => 'info', 'path' => $path);
}
foreach (Futures($futures)->limit(8) as $key => $future) {
$target = $targets[$key];
if ($target['command'] == 'diff') {
$repository_api->primeSVNDiffResult($target['path'], $future->resolve());
} else {
$repository_api->primeSVNInfoResult($target['path'], $future->resolve());
}
}
}
/**
* Run the xUnit test runner on each of the assemblies and parse the
* resulting XML.
*
* @param array Array of test assemblies.
* @return array Array of test results.
*/
private function testAssemblies(array $test_assemblies)
{
$results = array();
// Build the futures for running the tests.
$futures = array();
$outputs = array();
$coverages = array();
foreach ($test_assemblies as $test_assembly) {
list($future_r, $xunit_temp, $coverage) = $this->buildTestFuture($test_assembly['assembly']);
$futures[$test_assembly['assembly']] = $future_r;
$outputs[$test_assembly['assembly']] = $xunit_temp;
$coverages[$test_assembly['assembly']] = $coverage;
}
// Run all of the tests.
foreach (Futures($futures)->limit(8) as $test_assembly => $future) {
list($err, $stdout, $stderr) = $future->resolve();
if (file_exists($outputs[$test_assembly])) {
$result = $this->parseTestResult($outputs[$test_assembly], $coverages[$test_assembly]);
$results[] = $result;
unlink($outputs[$test_assembly]);
} else {
// FIXME: There's a bug in Mono which causes a segmentation fault
// when xUnit.NET runs; this causes the XML file to not appear
// (depending on when the segmentation fault occurs). See
// https://bugzilla.xamarin.com/show_bug.cgi?id=16379
// for more information.
// Since it's not possible for the user to correct this error, we
// ignore the fact the tests didn't run here.
}
}
return array_mergev($results);
}
/**
* Analyze the library, generating the file and symbol maps.
*
* @return void
*/
private function analyzeLibrary()
{
// Identify all the ".php" source files in the library.
$this->log("Finding source files...\n");
$source_map = $this->loadSourceFileMap();
$this->log("Found " . number_format(count($source_map)) . " files.\n");
// Load the symbol cache with existing parsed symbols. This allows us
// to remap libraries quickly by analyzing only changed files.
$this->log("Loading symbol cache...\n");
$symbol_cache = $this->loadSymbolCache();
// Build out the symbol analysis for all the files in the library. For
// each file, check if it's in cache. If we miss in the cache, do a fresh
// analysis.
$symbol_map = array();
$futures = array();
foreach ($source_map as $file => $hash) {
if (!empty($symbol_cache[$hash])) {
$symbol_map[$file] = $symbol_cache[$hash];
continue;
}
$futures[$file] = $this->buildSymbolAnalysisFuture($file);
}
$this->log("Found " . number_format(count($symbol_map)) . " files in cache.\n");
// Run the analyzer on any files which need analysis.
if ($futures) {
$limit = $this->subprocessLimit;
$count = number_format(count($futures));
$this->log("Analyzing {$count} files with {$limit} subprocesses...\n");
$progress = new PhutilConsoleProgressBar();
if ($this->quiet) {
$progress->setQuiet(true);
}
$progress->setTotal(count($futures));
foreach (Futures($futures)->limit($limit) as $file => $future) {
$result = $future->resolveJSON();
if (empty($result['error'])) {
$symbol_map[$file] = $result;
} else {
$progress->done(false);
throw new XHPASTSyntaxErrorException($result['line'], $file . ': ' . $result['error']);
}
$progress->update(1);
}
$progress->done();
}
$this->fileSymbolMap = $symbol_map;
// We're done building the cache, so write it out immediately. Note that
// we've only retained entries for files we found, so this implicitly cleans
// out old cache entries.
$this->writeSymbolCache($symbol_map, $source_map);
// Our map is up to date, so either show it on stdout or write it to disk.
$this->log("Building library map...\n");
$this->librarySymbolMap = $this->buildLibraryMap($symbol_map);
}
public function runUnitTests()
{
// Build any unit tests
$this->make('build-tests');
// Now find all the test programs
$root = $this->getProjectRoot();
$futures = array();
$cg_files = array();
$obj_files = array();
$coverage = $this->getEnableCoverage();
$tests = $this->getTestsToRun();
foreach ($tests as $relname => $test) {
if ($coverage) {
$cg = new TempFile();
$cg_files[$relname] = $cg;
$obj_files[] = $test;
$test = 'valgrind --tool=callgrind --collect-jumps=yes ' . '--callgrind-out-file=' . $cg . ' ' . $test;
}
$futures[$relname] = new ExecFuture($test);
}
$results = array();
// Use a smaller limit for SunOS because my test VM has crappy
// timing characteristics and that causes timer.t to fail
$limit = PHP_OS == 'SunOS' ? 1 : 4;
foreach (Futures($futures)->limit($limit) as $test => $future) {
list($err, $stdout, $stderr) = $future->resolve();
$results[$test] = $this->parseTestResults($test, $err, $stdout, $stderr);
}
if ($coverage) {
// Find all executables, not just the test executables.
// We need to do this because the test executables may not
// include all of the possible code (linker decides to omit
// it from the image) so we see a skewed representation of
// the source lines.
$fp = popen("find {$root} -type f -name \\*.a -o -name \\*.so -o -name \\*.dylib", "r");
while (true) {
$line = fgets($fp);
if ($line === false) {
break;
}
$obj_files[] = trim($line);
}
// Parse line information from the objects
foreach ($obj_files as $object) {
PhenomDwarfLineInfo::loadObject($object);
}
// Now process coverage data files
foreach ($cg_files as $relname => $cg) {
$CG = new PhenomCallgrindFile((string) $cg);
$CG->parse();
$source_files = array();
foreach ($CG->getSourceFiles() as $filename) {
if (Filesystem::isDescendant($filename, $root) && !preg_match("/(thirdparty|tests)/", $filename)) {
$source_files[$filename] = $filename;
}
}
$cov = array();
foreach ($source_files as $filename) {
$relsrc = substr($filename, strlen($root) + 1);
$cov[$relsrc] = PhenomCallgrindFile::mergeSourceLineData($filename, array($CG));
}
// Update the coverage data for that test.
// arcanist will merge the final results
$results[$relname]->setCoverage($cov);
}
}
return $results;
}
private function uploadFilesForChanges(array $changes)
{
assert_instances_of($changes, 'ArcanistDiffChange');
// Collect all the files we need to upload.
$need_upload = array();
foreach ($changes as $key => $change) {
if ($change->getFileType() != ArcanistDiffChangeType::FILE_BINARY) {
continue;
}
if ($this->getArgument('skip-binaries')) {
continue;
}
$name = basename($change->getCurrentPath());
$need_upload[] = array('type' => 'old', 'name' => $name, 'data' => $change->getOriginalFileData(), 'change' => $change);
$need_upload[] = array('type' => 'new', 'name' => $name, 'data' => $change->getCurrentFileData(), 'change' => $change);
}
if (!$need_upload) {
return;
}
// Determine mime types and file sizes. Update changes from "binary" to
// "image" if the file is an image. Set image metadata.
$type_image = ArcanistDiffChangeType::FILE_IMAGE;
foreach ($need_upload as $key => $spec) {
$change = $need_upload[$key]['change'];
$type = $spec['type'];
$size = strlen($spec['data']);
$change->setMetadata("{$type}:file:size", $size);
if ($spec['data'] === null) {
// This covers the case where a file was added or removed; we don't
// need to upload it. (This is distinct from an empty file, which we
// do upload.)
unset($need_upload[$key]);
continue;
}
$mime = $this->getFileMimeType($spec['data']);
if (preg_match('@^image/@', $mime)) {
$change->setFileType($type_image);
}
$change->setMetadata("{$type}:file:mime-type", $mime);
}
echo pht("Uploading %d files...", count($need_upload)) . "\n";
// Now we're ready to upload the actual file data. If possible, we'll just
// transmit a hash of the file instead of the actual file data. If the data
// already exists, Phabricator can share storage. Check if we can use
// "file.uploadhash" yet (i.e., if the server is up to date enough).
// TODO: Drop this check once we bump the protocol version.
$conduit_methods = $this->getConduit()->callMethodSynchronous('conduit.query', array());
$can_use_hash_upload = isset($conduit_methods['file.uploadhash']);
if ($can_use_hash_upload) {
$hash_futures = array();
foreach ($need_upload as $key => $spec) {
$hash_futures[$key] = $this->getConduit()->callMethod('file.uploadhash', array('name' => $spec['name'], 'hash' => sha1($spec['data'])));
}
foreach (Futures($hash_futures)->limit(8) as $key => $future) {
$type = $need_upload[$key]['type'];
$change = $need_upload[$key]['change'];
$name = $need_upload[$key]['name'];
$phid = null;
try {
$phid = $future->resolve();
} catch (Exception $e) {
// Just try uploading normally if the hash upload failed.
continue;
}
if ($phid) {
$change->setMetadata("{$type}:binary-phid", $phid);
unset($need_upload[$key]);
echo pht("Uploaded '%s' (%s).", $name, $type) . "\n";
}
}
}
$upload_futures = array();
foreach ($need_upload as $key => $spec) {
$upload_futures[$key] = $this->getConduit()->callMethod('file.upload', array('name' => $spec['name'], 'data_base64' => base64_encode($spec['data'])));
}
foreach (Futures($upload_futures)->limit(4) as $key => $future) {
$type = $need_upload[$key]['type'];
$change = $need_upload[$key]['change'];
$name = $need_upload[$key]['name'];
try {
$phid = $future->resolve();
$change->setMetadata("{$type}:binary-phid", $phid);
echo pht("Uploaded '%s' (%s).", $name, $type) . "\n";
} catch (Exception $e) {
echo "Failed to upload {$type} binary '{$name}'.\n";
if (!phutil_console_confirm('Continue?', $default_no = false)) {
throw new ArcanistUsageException('Aborted due to file upload failure. You can use --skip-binaries ' . 'to skip binary uploads.');
}
}
}
echo pht("Upload complete.") . "\n";
}
protected function buildUncommittedStatus()
{
$diff_options = $this->getDiffBaseOptions();
if ($this->repositoryHasNoCommits) {
$diff_base = self::GIT_MAGIC_ROOT_COMMIT;
} else {
$diff_base = 'HEAD';
}
// Find uncommitted changes.
$uncommitted_future = $this->buildLocalFuture(array('diff %C --raw %s --', $diff_options, $diff_base));
$untracked_future = $this->buildLocalFuture(array('ls-files --others --exclude-standard'));
// Unstaged changes
$unstaged_future = $this->buildLocalFuture(array('diff-files --name-only'));
$futures = array($uncommitted_future, $untracked_future);
Futures($futures)->resolveAll();
// We're clear to start the `git diff-files` now.
$unstaged_future->start();
$result = new PhutilArrayWithDefaultValue();
list($stdout) = $uncommitted_future->resolvex();
$uncommitted_files = $this->parseGitStatus($stdout);
foreach ($uncommitted_files as $path => $mask) {
$result[$path] |= $mask | self::FLAG_UNCOMMITTED;
}
list($stdout) = $untracked_future->resolvex();
$stdout = rtrim($stdout, "\n");
if (strlen($stdout)) {
$stdout = explode("\n", $stdout);
foreach ($stdout as $path) {
$result[$path] |= self::FLAG_UNTRACKED;
}
}
list($stdout, $stderr) = $unstaged_future->resolvex();
$stdout = rtrim($stdout, "\n");
if (strlen($stdout)) {
$stdout = explode("\n", $stdout);
foreach ($stdout as $path) {
$result[$path] |= self::FLAG_UNSTAGED;
}
}
return $result->toArray();
}
public function process()
{
$old = array();
$new = array();
$n = 0;
$this->old = array_reverse($this->old);
$this->new = array_reverse($this->new);
$whitelines = false;
$changed = false;
$skip_intra = array();
while (count($this->old) || count($this->new)) {
$o_desc = array_pop($this->old);
$n_desc = array_pop($this->new);
$oend = end($this->old);
if ($oend) {
$o_next = $oend['type'];
} else {
$o_next = null;
}
$nend = end($this->new);
if ($nend) {
$n_next = $nend['type'];
} else {
$n_next = null;
}
if ($o_desc) {
$o_type = $o_desc['type'];
} else {
$o_type = null;
}
if ($n_desc) {
$n_type = $n_desc['type'];
} else {
$n_type = null;
}
if ($o_type != null && $n_type == null) {
$old[] = $o_desc;
$new[] = null;
if ($n_desc) {
array_push($this->new, $n_desc);
}
$changed = true;
continue;
}
if ($n_type != null && $o_type == null) {
$old[] = null;
$new[] = $n_desc;
if ($o_desc) {
array_push($this->old, $o_desc);
}
$changed = true;
continue;
}
if ($this->whitespaceMode != self::WHITESPACE_SHOW_ALL) {
$similar = false;
switch ($this->whitespaceMode) {
case self::WHITESPACE_IGNORE_TRAILING:
if (rtrim($o_desc['text']) == rtrim($n_desc['text'])) {
if ($o_desc['type']) {
// If we're converting this into an unchanged line because of
// a trailing whitespace difference, mark it as a whitespace
// change so we can show "This file was modified only by
// adding or removing trailing whitespace." instead of
// "This file was not modified.".
$whitelines = true;
}
$similar = true;
}
break;
default:
// In this case, the lines are similar if there is no change type
// (that is, just trust the diff algorithm).
if (!$o_desc['type']) {
$similar = true;
}
break;
}
if ($similar) {
$o_desc['type'] = null;
$n_desc['type'] = null;
$skip_intra[count($old)] = true;
} else {
$changed = true;
}
} else {
$changed = true;
}
$old[] = $o_desc;
$new[] = $n_desc;
}
$this->old = $old;
$this->new = $new;
$unchanged = false;
if ($this->subparser) {
$unchanged = $this->subparser->isUnchanged();
$whitelines = $this->subparser->isWhitespaceOnly();
} else {
if (!$changed) {
$filetype = $this->changeset->getFileType();
if ($filetype == DifferentialChangeType::FILE_TEXT || $filetype == DifferentialChangeType::FILE_SYMLINK) {
$unchanged = true;
}
}
}
$this->specialAttributes = array(self::ATTR_UNCHANGED => $unchanged, self::ATTR_DELETED => array_filter($this->old) && !array_filter($this->new), self::ATTR_WHITELINES => $whitelines);
if ($this->isSubparser) {
// The rest of this function deals with formatting the diff for display;
// we can exit early if we're a subparser and avoid doing extra work.
return;
}
if ($this->subparser) {
// Use this parser's side-by-side line information -- notably, the
// change types -- but replace all the line text with the subparser's.
// This lets us render whitespace-only changes without marking them as
// different.
$old = $this->old;
$new = $this->new;
$old_text = ipull($this->subparser->old, 'text', 'line');
$new_text = ipull($this->subparser->new, 'text', 'line');
foreach ($old as $k => $desc) {
if (empty($desc)) {
continue;
}
$old[$k]['text'] = idx($old_text, $desc['line']);
}
foreach ($new as $k => $desc) {
if (empty($desc)) {
continue;
}
$new[$k]['text'] = idx($new_text, $desc['line']);
// If there's a corresponding "old" text and the line is marked as
// unchanged, test if there are internal whitespace changes between
// non-whitespace characters, e.g. spaces added to a string or spaces
// added around operators. If we find internal spaces, mark the line
// as changed.
//
// We only need to do this for "new" lines because any line that is
// missing either "old" or "new" text certainly can not have internal
// whitespace changes without also having non-whitespace changes,
// because characters had to be either added or removed to create the
// possibility of internal whitespace.
if (isset($old[$k]['text']) && empty($new[$k]['type'])) {
if (trim($old[$k]['text']) != trim($new[$k]['text'])) {
// The strings aren't the same when trimmed, so there are internal
// whitespace changes. Mark this line changed.
$old[$k]['type'] = '-';
$new[$k]['type'] = '+';
}
}
}
$this->old = $old;
$this->new = $new;
}
$min_length = min(count($this->old), count($this->new));
for ($ii = 0; $ii < $min_length; $ii++) {
if ($this->old[$ii] || $this->new[$ii]) {
if (isset($this->old[$ii]['text'])) {
$otext = $this->old[$ii]['text'];
} else {
$otext = '';
}
if (isset($this->new[$ii]['text'])) {
$ntext = $this->new[$ii]['text'];
} else {
$ntext = '';
}
if ($otext != $ntext && empty($skip_intra[$ii])) {
$this->intra[$ii] = ArcanistDiffUtils::generateIntralineDiff($otext, $ntext);
}
}
}
$lines_context = self::LINES_CONTEXT;
$max_length = max(count($this->old), count($this->new));
$old = $this->old;
$new = $this->new;
$visible = false;
$last = 0;
for ($cursor = -$lines_context; $cursor < $max_length; $cursor++) {
$offset = $cursor + $lines_context;
if (isset($old[$offset]) && $old[$offset]['type'] || isset($new[$offset]) && $new[$offset]['type']) {
$visible = true;
$last = $offset;
} else {
if ($cursor > $last + $lines_context) {
$visible = false;
}
}
if ($visible && $cursor > 0) {
$this->visible[$cursor] = 1;
}
}
// NOTE: Micro-optimize a couple of ipull()s here since it gives us a
// 10% performance improvement for certain types of large diffs like
// Phriction changes.
$old_corpus = array();
foreach ($this->old as $o) {
$old_corpus[] = $o['text'];
}
$old_corpus_block = implode("\n", $old_corpus);
$new_corpus = array();
foreach ($this->new as $n) {
$new_corpus[] = $n['text'];
}
$new_corpus_block = implode("\n", $new_corpus);
$old_future = $this->getHighlightFuture($old_corpus_block);
$new_future = $this->getHighlightFuture($new_corpus_block);
$futures = array('old' => $old_future, 'new' => $new_future);
foreach (Futures($futures) as $key => $future) {
try {
switch ($key) {
case 'old':
$this->oldRender = $this->processHighlightedSource($this->old, $future->resolve());
break;
case 'new':
$this->newRender = $this->processHighlightedSource($this->new, $future->resolve());
break;
}
} catch (Exception $ex) {
phlog($ex);
throw $ex;
}
}
$this->applyIntraline($this->oldRender, ipull($this->intra, 0), $old_corpus);
$this->applyIntraline($this->newRender, ipull($this->intra, 1), $new_corpus);
$generated = strpos($new_corpus_block, '@' . 'generated') !== false;
$this->specialAttributes[self::ATTR_GENERATED] = $generated;
}
