/**
* Runs the test suite.
*/
public function run()
{
$results = array();
$command = '(mkdir -p build && cd build && cmake ..)';
$command .= '&& make -C build all';
$command .= '&& make -C build test';
// Execute the test command & time it.
$timeStart = microtime(true);
$future = new ExecFuture($command);
do {
$future->read();
sleep(0.5);
} while (!$future->isReady());
list($error, $stdout, $stderr) = $future->resolve();
$timeEnd = microtime(true);
// Create a unit test result structure.
$result = new ArcanistUnitTestResult();
$result->setNamespace('DerpVision');
$result->setName('Core');
$result->setDuration($timeEnd - $timeStart);
if ($error == 0) {
$result->setResult(ArcanistUnitTestResult::RESULT_PASS);
} else {
$result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
$result->setUserData($stdout . $stderr);
}
$results[] = $result;
return $results;
}
/**
* Returns an array of the full canonical paths to all the Maven directories
* (directories containing pom.xml files) in the project.
*/
private function findMavenDirectories()
{
if (file_exists($this->project_root . "/.git")) {
// The fastest way to find all the pom.xml files is to let git scan
// its index.
$future = new ExecFuture('git ls-files \\*/pom.xml');
} else {
// Not a git repo. Do it the old-fashioned way.
$future = new ExecFuture('find . -name pom.xml -print');
}
// TODO: This will find *all* the pom.xml files in the working copy.
// Need to obey the optional paths argument to "arc unit" to let users
// run just a subset of tests.
$future->setCWD($this->project_root);
list($stdout) = $future->resolvex();
$poms = explode("\n", trim($stdout));
if (!$poms) {
throw new Exception("No pom.xml files found");
}
$maven_dirs = array_map(function ($pom) {
$maven_dir = dirname($pom);
return realpath($this->project_root . '/' . $maven_dir);
}, $poms);
return $maven_dirs;
}
private function runTests()
{
$root = $this->getWorkingCopy()->getProjectRoot();
$script = $this->getConfiguredScript();
$path = $this->getConfiguredTestResultPath();
foreach (glob($root . DIRECTORY_SEPARATOR . $path . "/*.xml") as $filename) {
// Remove existing files so we cannot report old results
$this->unlink($filename);
}
// Provide changed paths to process
putenv("ARCANIST_DIFF_PATHS=" . implode(PATH_SEPARATOR, $this->getPaths()));
$future = new ExecFuture('%C %s', $script, $path);
$future->setCWD($root);
$err = null;
try {
$future->resolvex();
} catch (CommandException $exc) {
$err = $exc;
}
$results = $this->parseTestResults($root . DIRECTORY_SEPARATOR . $path);
if ($err) {
$result = new ArcanistUnitTestResult();
$result->setName('Unit Test Script');
$result->setResult(ArcanistUnitTestResult::RESULT_BROKEN);
$result->setUserData("ERROR: Command failed with code {$err->getError()}\nCOMMAND: `{$err->getCommand()}`");
$results[] = $result;
}
return $results;
}
public function lintPath($path)
{
$sbt = $this->getSBTPath();
// Tell SBT to not use color codes so our regex life is easy.
// TODO: Should this be "clean compile" instead of "compile"?
$f = new ExecFuture("%s -Dsbt.log.noformat=true compile", $sbt);
list($err, $stdout, $stderr) = $f->resolve();
$lines = explode("\n", $stdout);
$messages = array();
foreach ($lines as $line) {
$matches = null;
if (!preg_match("/\\[(warn|error)\\] (.*?):(\\d+): (.*?)\$/", $line, $matches)) {
continue;
}
foreach ($matches as $key => $match) {
$matches[$key] = trim($match);
}
$message = new ArcanistLintMessage();
$message->setPath($matches[2]);
$message->setLine($matches[3]);
$message->setCode($this->getLinterName());
$message->setDescription($matches[4]);
$message->setSeverity($this->getMessageCodeSeverity($matches[1]));
$this->addLintMessage($message);
}
}
public function lintPath($path)
{
$working_copy = $this->getEngine()->getWorkingCopy();
$pyflakes_path = $working_copy->getConfig('lint.pyflakes.path');
$pyflakes_prefix = $working_copy->getConfig('lint.pyflakes.prefix');
// Default to just finding pyflakes in the users path
$pyflakes_bin = 'pyflakes';
$python_path = '';
// If a pyflakes path was specified, then just use that as the
// pyflakes binary and assume that the libraries will be imported
// correctly.
//
// If no pyflakes path was specified and a pyflakes prefix was
// specified, then use the binary from this prefix and add it to
// the PYTHONPATH environment variable so that the libs are imported
// correctly. This is useful when pyflakes is installed into a
// non-default location.
if ($pyflakes_path !== null) {
$pyflakes_bin = $pyflakes_path;
} else {
if ($pyflakes_prefix !== null) {
$pyflakes_bin = $pyflakes_prefix . '/bin/pyflakes';
$python_path = $pyflakes_prefix . '/lib/python2.6/site-packages:';
}
}
$options = $this->getPyFlakesOptions();
$f = new ExecFuture("/usr/bin/env PYTHONPATH=%s\$PYTHONPATH " . "{$pyflakes_bin} {$options}", $python_path);
$f->write($this->getData($path));
try {
list($stdout, $_) = $f->resolvex();
} catch (CommandException $e) {
// PyFlakes will return an exit code of 1 if warnings/errors
// are found but print nothing to stderr in this case. Therefore,
// if we see any output on stderr or a return code other than 1 or 0,
// pyflakes failed.
if ($e->getError() !== 1 || $e->getStderr() !== '') {
throw $e;
} else {
$stdout = $e->getStdout();
}
}
$lines = explode("\n", $stdout);
$messages = array();
foreach ($lines as $line) {
$matches = null;
if (!preg_match('/^(.*?):(\\d+): (.*)$/', $line, $matches)) {
continue;
}
foreach ($matches as $key => $match) {
$matches[$key] = trim($match);
}
$message = new ArcanistLintMessage();
$message->setPath($path);
$message->setLine($matches[2]);
$message->setCode($this->getLinterName());
$message->setDescription($matches[3]);
$message->setSeverity(ArcanistLintSeverity::SEVERITY_WARNING);
$this->addLintMessage($message);
}
}
public function lintPath($path)
{
$bin = $this->getLintPath();
$path = $this->rocksdbDir() . '/' . $path;
$f = new ExecFuture("%C {$path}", $bin);
list($err, $stdout, $stderr) = $f->resolve();
if ($err === 2) {
throw new Exception("cpplint failed to run correctly:\n" . $stderr);
}
$lines = explode("\n", $stderr);
$messages = array();
foreach ($lines as $line) {
$line = trim($line);
$matches = null;
$regex = '/^[^:]+:(\\d+):\\s*(.*)\\s*\\[(.*)\\] \\[(\\d+)\\]$/';
if (!preg_match($regex, $line, $matches)) {
continue;
}
foreach ($matches as $key => $match) {
$matches[$key] = trim($match);
}
$message = new ArcanistLintMessage();
$message->setPath($path);
$message->setLine($matches[1]);
$message->setCode($matches[3]);
$message->setName($matches[3]);
$message->setDescription($matches[2]);
$message->setSeverity(ArcanistLintSeverity::SEVERITY_WARNING);
$this->addLintMessage($message);
}
}
public function lintPath($path)
{
$rubyp = $this->getRubyPath();
$f = new ExecFuture("%s -wc", $rubyp);
$f->write($this->getData($path));
list($err, $stdout, $stderr) = $f->resolve();
if ($err === 0) {
return;
}
$lines = explode("\n", $stderr);
$messages = array();
foreach ($lines as $line) {
$matches = null;
if (!preg_match("/(.*?):(\\d+): (.*?)\$/", $line, $matches)) {
continue;
}
foreach ($matches as $key => $match) {
$matches[$key] = trim($match);
}
$code = head(explode(',', $matches[3]));
$message = new ArcanistLintMessage();
$message->setPath($path);
$message->setLine($matches[2]);
$message->setName($this->getLinterName() . " " . $code);
$message->setDescription($matches[3]);
$message->setSeverity($this->getMessageCodeSeverity($code));
$this->addLintMessage($message);
}
}
function ctags_check_executable()
{
$future = new ExecFuture('ctags --version');
$result = $future->resolve();
if (empty($result[1])) {
return false;
}
return true;
}
/**
* @group xhpast
*/
function xhpast_get_parser_future($data)
{
if (!xhpast_is_available()) {
throw new Exception(xhpast_get_build_instructions());
}
$future = new ExecFuture('%s', xhpast_get_binary_path());
$future->write($data);
return $future;
}
/**
* Construct an exec channel from a @{class:ExecFuture}. The future should
* **NOT** have been started yet (e.g., with `isReady()` or `start()`),
* because @{class:ExecFuture} closes stdin by default when futures start.
* If stdin has been closed, you will be unable to write on the channel.
*
* @param ExecFuture Future to use as an underlying I/O source.
* @task construct
*/
public function __construct(ExecFuture $future)
{
// Make an empty write to keep the stdin pipe open. By default, futures
// close this pipe when they start.
$future->write('', $keep_pipe = true);
// Start the future so that reads and writes work immediately.
$future->isReady();
$this->future = $future;
}
public function testTimeoutTestShouldRunLessThan1Sec()
{
// NOTE: This is partly testing that we choose appropriate select wait
// times; this test should run for significantly less than 1 second.
$future = new ExecFuture('sleep 32000');
list($err) = $future->setTimeout(0.01)->resolve();
$this->assertEqual(true, $err > 0);
$this->assertEqual(true, $future->getWasKilledByTimeout());
}
private function writeAndRead($write, $read)
{
$future = new ExecFuture('cat');
$future->write($write);
$lines = array();
foreach (new LinesOfALargeExecFuture($future) as $line) {
$lines[] = $line;
}
$this->assertEqual($read, $lines, pht('Write: %s', id(new PhutilUTF8StringTruncator())->setMaximumGlyphs(32)->truncateString($write)));
}
protected function configureFuture(ExecFuture $future)
{
if ($this->getTimeout()) {
$future->setTimeout($this->getTimeout());
}
if ($this->getByteLimit()) {
$future->setStdoutSizeLimit($this->getByteLimit());
$future->setStderrSizeLimit($this->getByteLimit());
}
}
private function writeAndRead($write, $read)
{
$future = new ExecFuture('cat');
$future->write($write);
$lines = array();
foreach (new LinesOfALargeExecFuture($future) as $line) {
$lines[] = $line;
}
$this->assertEqual($read, $lines, "Write: " . phutil_utf8_shorten($write, 32));
}
public function buildTestFuture($junit_tmp, $cover_tmp)
{
$paths = $this->getPaths();
$config_manager = $this->getConfigurationManager();
$coverage_command = $config_manager->getConfigFromAnySource('unit.golang.command');
$cmd_line = csprintf($coverage_command, $junit_tmp, $cover_tmp);
$future = new ExecFuture('%C', $cmd_line);
$future->setCWD($this->projectRoot);
return $future;
}
public function testNoHangOnExecFutureDestructionWithRunningChild()
{
$start = microtime(true);
$future = new ExecFuture('sleep 30');
$future->start();
unset($future);
$end = microtime(true);
// If ExecFuture::__destruct() hangs until the child closes, we won't make
// it here in time.
$this->assertEqual(true, $end - $start < 5);
}
private function runBootloaderTests(PhageAgentBootloader $boot)
{
$name = get_class($boot);
$exec = new ExecFuture('%C', $boot->getBootCommand());
$exec->write($boot->getBootSequence(), $keep_open = true);
$exec_channel = new PhutilExecChannel($exec);
$agent = new PhutilJSONProtocolChannel($exec_channel);
$agent->write(array('type' => 'EXEC', 'key' => 1, 'command' => 'echo phage'));
$this->agentExpect($agent, array('type' => 'RSLV', 'key' => 1, 'err' => 0, 'stdout' => "phage\n", 'stderr' => ''), "'echo phage' for {$name}");
$agent->write(array('type' => 'EXIT'));
}
public function willLintPaths(array $paths)
{
$working_copy = $this->getEngine()->getWorkingCopy();
$root = $working_copy->getProjectRoot();
$chunks = array_chunk($paths, 8);
foreach ($chunks as $chunk) {
$f = new ExecFuture("python %C --root=include --filter=-build/include_order " . "--verbose=2 %Ls", $root . '/thirdparty/cpplint.py', $chunk);
$f->start();
$this->futures[] = $f;
}
}
public function getHighlightFuture($source)
{
$language = idx($this->config, 'language');
if ($language) {
$language = $this->getPygmentsLexerNameFromLanguageName($language);
$future = new ExecFuture('pygmentize -O stripnl=False -f html -l %s', $language);
$future->write($source);
return new PhutilDefaultSyntaxHighlighterEnginePygmentsFuture($future, $source);
}
return id(new PhutilDefaultSyntaxHighlighter())->getHighlightFuture($source);
}
public function run()
{
$command = $this->getConfigurationManager()->getConfigFromAnySource('unit.engine.tap.command');
$future = new ExecFuture($command);
do {
list($stdout, $stderr) = $future->read();
echo $stdout;
echo $stderr;
sleep(0.5);
} while (!$future->isReady());
list($error, $stdout, $stderr) = $future->resolve();
return $this->parseOutput($stdout);
}
public function run()
{
$working_copy = $this->getWorkingCopy();
$this->projectRoot = $working_copy->getProjectRoot();
$junit_tmp = new TempFile();
$cover_tmp = new TempFile();
$future = $this->buildTestFuture($junit_tmp, $cover_tmp);
$future->resolvex();
$future = new ExecFuture('coverage xml -o %s', $cover_tmp);
$future->setCWD($this->projectRoot);
$future->resolvex();
return $this->parseTestResults($junit_tmp, $cover_tmp);
}
/**
* @phutil-external-symbol function jsShrink
*/
public function transformResource($path, $data)
{
$type = self::getResourceType($path);
switch ($type) {
case 'css':
$data = $this->replaceCSSPrintRules($path, $data);
$data = $this->replaceCSSVariables($path, $data);
$data = preg_replace_callback('@url\\s*\\((\\s*[\'"]?.*?)\\)@s', nonempty($this->translateURICallback, array($this, 'translateResourceURI')), $data);
break;
}
if (!$this->minify) {
return $data;
}
// Some resources won't survive minification (like Raphael.js), and are
// marked so as not to be minified.
if (strpos($data, '@' . 'do-not-minify') !== false) {
return $data;
}
switch ($type) {
case 'css':
// Remove comments.
$data = preg_replace('@/\\*.*?\\*/@s', '', $data);
// Remove whitespace around symbols.
$data = preg_replace('@\\s*([{}:;,])\\s*@', '\\1', $data);
// Remove unnecessary semicolons.
$data = preg_replace('@;}@', '}', $data);
// Replace #rrggbb with #rgb when possible.
$data = preg_replace('@#([a-f0-9])\\1([a-f0-9])\\2([a-f0-9])\\3@i', '#\\1\\2\\3', $data);
$data = trim($data);
break;
case 'js':
// If `jsxmin` is available, use it. jsxmin is the Javelin minifier and
// produces the smallest output, but is complicated to build.
if (Filesystem::binaryExists('jsxmin')) {
$future = new ExecFuture('jsxmin __DEV__:0');
$future->write($data);
list($err, $result) = $future->resolve();
if (!$err) {
$data = $result;
break;
}
}
// If `jsxmin` is not available, use `JsShrink`, which doesn't compress
// quite as well but is always available.
$root = dirname(phutil_get_library_root('phabricator'));
require_once $root . '/externals/JsShrink/jsShrink.php';
$data = jsShrink($data);
break;
}
return $data;
}
protected function resolveBlameFuture(ExecFuture $future)
{
list($err, $stdout) = $future->resolve();
if ($err) {
return null;
}
$result = array();
$lines = phutil_split_lines($stdout);
foreach ($lines as $line) {
list($commit) = explode(' ', $line, 2);
$result[] = $commit;
}
return $result;
}
public function testCloseExecWriteChannel()
{
$future = new ExecFuture('cat');
// If this test breaks, we want to explode, not hang forever.
$future->setTimeout(5);
$exec_channel = new PhutilExecChannel($future);
$exec_channel->write('quack');
$exec_channel->closeWriteChannel();
// If `closeWriteChannel()` did what it is supposed to, this will just
// echo "quack" and exit with no error code. If the channel did not close,
// this will time out after 5 seconds and throw.
$future->resolvex();
$this->assertTrue(true);
}
public function __construct($sockname, $repo_root, $args, array $command)
{
$this->command = json_encode($command);
$console = PhutilConsole::getConsole();
$console->writeLog("cli query: %s\n", $this->command);
$future = new ExecFuture("%C {$args} -U %s %s %s %s %s", "{$repo_root}/watchman", $sockname, '--no-pretty', '--no-spawn', '--no-local', '-j');
$cwd = getcwd();
if (!$cwd) {
throw new Exception("can't figure out my cwd!?");
}
$future->setCWD($cwd);
$future->write($this->command . "\n");
parent::__construct($future);
}
/**
* @group xhpast
*/
function xhpast_get_parser_future($data)
{
if (!xhpast_is_available()) {
try {
// Try to build XHPAST automatically. If we can't then just ask the user
// to build it themselves.
xhpast_build();
} catch (CommandException $e) {
throw new Exception(xhpast_get_build_instructions());
}
}
$future = new ExecFuture('%s', xhpast_get_binary_path());
$future->write($data);
return $future;
}
public function run()
{
$output = new TempFile();
$command = $this->getConfigurationManager()->getConfigFromAnySource('unit.engine.rspec.command');
if (!$command) {
$command = 'rspec';
}
$future = new ExecFuture($command . ' -f json -o ' . $output . ' -f Fuubar');
do {
list($stdout, $stderr) = $future->read();
echo $stdout;
sleep(0.5);
} while (!$future->isReady());
return $this->parseOutput(Filesystem::readFile($output));
}
public function willLintPaths(array $paths)
{
$script = $this->getConfiguredScript();
$root = $this->getEngine()->getWorkingCopy()->getProjectRoot();
$futures = array();
foreach ($paths as $path) {
$future = new ExecFuture('%C %s', $script, $path);
$future->setCWD($root);
$futures[$path] = $future;
}
foreach (Futures($futures)->limit(4) as $path => $future) {
list($stdout) = $future->resolvex();
$this->output[$path] = $stdout;
}
}
/**
* Constructs an @{class:ExecFuture} for XHPAST.
*
* @param wild Data to pass to the future.
* @return ExecFuture
*/
public static function getParserFuture($data)
{
if (!self::isAvailable()) {
try {
// Try to build XHPAST automatically. If we can't then just ask the
// user to build it themselves.
self::build();
} catch (CommandException $ex) {
throw new PhutilProxyException(self::getBuildInstructions(), $ex);
}
}
$future = new ExecFuture('%s', self::getPath());
$future->write($data);
return $future;
}
public function invoke()
{
parent::invoke();
$future = new \ExecFuture("composer --no-ansi update");
$future->start();
try {
do {
list($out) = $future->read();
echo $out;
} while (!$future->isReady());
} catch (\Exception $e) {
list($err) = $future->read();
throw new \Exception($err);
}
}
