/**
* Runs the test suite.
*/
public function run()
{
$results = array();
$command = '(mkdir -p build && cd build && cmake ..)';
$command .= '&& make -C build all';
$command .= '&& make -C build test';
// Execute the test command & time it.
$timeStart = microtime(true);
$future = new ExecFuture($command);
do {
$future->read();
sleep(0.5);
} while (!$future->isReady());
list($error, $stdout, $stderr) = $future->resolve();
$timeEnd = microtime(true);
// Create a unit test result structure.
$result = new ArcanistUnitTestResult();
$result->setNamespace('DerpVision');
$result->setName('Core');
$result->setDuration($timeEnd - $timeStart);
if ($error == 0) {
$result->setResult(ArcanistUnitTestResult::RESULT_PASS);
} else {
$result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
$result->setUserData($stdout . $stderr);
}
$results[] = $result;
return $results;
}
public function lintPath($path)
{
$sbt = $this->getSBTPath();
// Tell SBT to not use color codes so our regex life is easy.
// TODO: Should this be "clean compile" instead of "compile"?
$f = new ExecFuture("%s -Dsbt.log.noformat=true compile", $sbt);
list($err, $stdout, $stderr) = $f->resolve();
$lines = explode("\n", $stdout);
$messages = array();
foreach ($lines as $line) {
$matches = null;
if (!preg_match("/\\[(warn|error)\\] (.*?):(\\d+): (.*?)\$/", $line, $matches)) {
continue;
}
foreach ($matches as $key => $match) {
$matches[$key] = trim($match);
}
$message = new ArcanistLintMessage();
$message->setPath($matches[2]);
$message->setLine($matches[3]);
$message->setCode($this->getLinterName());
$message->setDescription($matches[4]);
$message->setSeverity($this->getMessageCodeSeverity($matches[1]));
$this->addLintMessage($message);
}
}
public function lintPath($path)
{
$rubyp = $this->getRubyPath();
$f = new ExecFuture("%s -wc", $rubyp);
$f->write($this->getData($path));
list($err, $stdout, $stderr) = $f->resolve();
if ($err === 0) {
return;
}
$lines = explode("\n", $stderr);
$messages = array();
foreach ($lines as $line) {
$matches = null;
if (!preg_match("/(.*?):(\\d+): (.*?)\$/", $line, $matches)) {
continue;
}
foreach ($matches as $key => $match) {
$matches[$key] = trim($match);
}
$code = head(explode(',', $matches[3]));
$message = new ArcanistLintMessage();
$message->setPath($path);
$message->setLine($matches[2]);
$message->setName($this->getLinterName() . " " . $code);
$message->setDescription($matches[3]);
$message->setSeverity($this->getMessageCodeSeverity($code));
$this->addLintMessage($message);
}
}
private function runTests($targets)
{
$future = new ExecFuture($this->bazelCommand(array_merge(["test", "--noshow_loading_progress", "--noshow_progress"], $targets)));
$future->setCWD($this->project_root);
$status = $future->resolve();
return $this->parseTestResults($targets, $status);
}
public function lintPath($path)
{
$bin = $this->getLintPath();
$path = $this->rocksdbDir() . '/' . $path;
$f = new ExecFuture("%C {$path}", $bin);
list($err, $stdout, $stderr) = $f->resolve();
if ($err === 2) {
throw new Exception("cpplint failed to run correctly:\n" . $stderr);
}
$lines = explode("\n", $stderr);
$messages = array();
foreach ($lines as $line) {
$line = trim($line);
$matches = null;
$regex = '/^[^:]+:(\\d+):\\s*(.*)\\s*\\[(.*)\\] \\[(\\d+)\\]$/';
if (!preg_match($regex, $line, $matches)) {
continue;
}
foreach ($matches as $key => $match) {
$matches[$key] = trim($match);
}
$message = new ArcanistLintMessage();
$message->setPath($path);
$message->setLine($matches[1]);
$message->setCode($matches[3]);
$message->setName($matches[3]);
$message->setDescription($matches[2]);
$message->setSeverity(ArcanistLintSeverity::SEVERITY_WARNING);
$this->addLintMessage($message);
}
}
function ctags_check_executable()
{
$future = new ExecFuture('ctags --version');
$result = $future->resolve();
if (empty($result[1])) {
return false;
}
return true;
}
public function run()
{
$working_copy = $this->getWorkingCopy();
$this->project_root = $working_copy->getProjectRoot();
// We only want to report results for tests that actually ran, so
// we'll compare the test result files' timestamps to the start time
// of the test run. This will probably break if multiple test runs
// are happening in parallel, but if that's happening then we can't
// count on the results files being intact anyway.
$start_time = time();
$maven_top_dirs = $this->findTopLevelMavenDirectories();
// We'll figure out if any of the modified files we're testing are in
// Maven directories. We won't want to run a bunch of Java tests for
// changes to CSS files or whatever.
$modified_paths = $this->getModifiedPaths();
$maven_failed = false;
foreach ($maven_top_dirs as $dir) {
$dir_with_trailing_slash = $dir . '/';
foreach ($modified_paths as $path) {
if ($dir_with_trailing_slash === substr($path, 0, strlen($dir_with_trailing_slash))) {
$future = new ExecFuture('mvn test');
$future->setCWD($dir);
list($status, $stdout, $stderr) = $future->resolve();
if ($status) {
// Maven exits with a nonzero status if there were test failures
// or if there was a compilation error.
$maven_failed = true;
break 2;
}
break;
}
}
}
$testResults = $this->parseTestResultsSince($start_time);
if ($maven_failed) {
// If there wasn't a test failure, then synthesize one to represent
// the failure of the test run as a whole, since it probably means the
// code failed to compile.
$found_failure = false;
foreach ($testResults as $testResult) {
if ($testResult->getResult() === ArcanistUnitTestResult::RESULT_FAIL || $testResult->getResult() === ArcanistUnitTestResult::RESULT_BROKEN) {
$found_failure = true;
break;
}
}
if (!$found_failure) {
$testResult = new ArcanistUnitTestResult();
$testResult->setResult(ArcanistUnitTestResult::RESULT_BROKEN);
$testResult->setName('mvn test');
$testResults[] = $testResult;
}
}
return $testResults;
}
public function run()
{
$command = $this->getConfigurationManager()->getConfigFromAnySource('unit.engine.tap.command');
$future = new ExecFuture($command);
do {
list($stdout, $stderr) = $future->read();
echo $stdout;
echo $stderr;
sleep(0.5);
} while (!$future->isReady());
list($error, $stdout, $stderr) = $future->resolve();
return $this->parseOutput($stdout);
}
/**
* @phutil-external-symbol function jsShrink
*/
public function transformResource($path, $data)
{
$type = self::getResourceType($path);
switch ($type) {
case 'css':
$data = $this->replaceCSSPrintRules($path, $data);
$data = $this->replaceCSSVariables($path, $data);
$data = preg_replace_callback('@url\\s*\\((\\s*[\'"]?.*?)\\)@s', nonempty($this->translateURICallback, array($this, 'translateResourceURI')), $data);
break;
}
if (!$this->minify) {
return $data;
}
// Some resources won't survive minification (like Raphael.js), and are
// marked so as not to be minified.
if (strpos($data, '@' . 'do-not-minify') !== false) {
return $data;
}
switch ($type) {
case 'css':
// Remove comments.
$data = preg_replace('@/\\*.*?\\*/@s', '', $data);
// Remove whitespace around symbols.
$data = preg_replace('@\\s*([{}:;,])\\s*@', '\\1', $data);
// Remove unnecessary semicolons.
$data = preg_replace('@;}@', '}', $data);
// Replace #rrggbb with #rgb when possible.
$data = preg_replace('@#([a-f0-9])\\1([a-f0-9])\\2([a-f0-9])\\3@i', '#\\1\\2\\3', $data);
$data = trim($data);
break;
case 'js':
// If `jsxmin` is available, use it. jsxmin is the Javelin minifier and
// produces the smallest output, but is complicated to build.
if (Filesystem::binaryExists('jsxmin')) {
$future = new ExecFuture('jsxmin __DEV__:0');
$future->write($data);
list($err, $result) = $future->resolve();
if (!$err) {
$data = $result;
break;
}
}
// If `jsxmin` is not available, use `JsShrink`, which doesn't compress
// quite as well but is always available.
$root = dirname(phutil_get_library_root('phabricator'));
require_once $root . '/externals/JsShrink/jsShrink.php';
$data = jsShrink($data);
break;
}
return $data;
}
protected function resolveBlameFuture(ExecFuture $future)
{
list($err, $stdout) = $future->resolve();
if ($err) {
return null;
}
$result = array();
$lines = phutil_split_lines($stdout);
foreach ($lines as $line) {
list($commit) = explode(' ', $line, 2);
$result[] = $commit;
}
return $result;
}
public function run()
{
$working_copy = $this->getWorkingCopy();
$this->projectRoot = $working_copy->getProjectRoot();
$future = new ExecFuture('npm run coverage');
$future->setCWD($this->projectRoot);
list($err, $stdout, $stderr) = $future->resolve();
$result = new ArcanistUnitTestResult();
$result->setName("Node test engine");
$result->setUserData($stdout);
if ($err) {
$result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
} else {
$result->setResult(ArcanistUnitTestResult::RESULT_PASS);
}
return array($result);
}
protected function resolveBlameFuture(ExecFuture $future)
{
list($err, $stdout) = $future->resolve();
if ($err) {
return null;
}
$result = array();
$matches = null;
$lines = phutil_split_lines($stdout);
foreach ($lines as $line) {
if (preg_match('/^\\s*(\\d+)/', $line, $matches)) {
$result[] = (int) $matches[1];
} else {
$result[] = null;
}
}
return $result;
}
public function transformResource($path, $data)
{
$type = self::getResourceType($path);
switch ($type) {
case 'css':
$data = preg_replace_callback('@url\\s*\\((\\s*[\'"]?/rsrc/.*?)\\)@s', array($this, 'translateResourceURI'), $data);
break;
}
if (!$this->minify) {
return $data;
}
// Some resources won't survive minification (like Raphael.js), and are
// marked so as not to be minified.
if (strpos($data, '@' . 'do-not-minify') !== false) {
return $data;
}
switch ($type) {
case 'css':
// Remove comments.
$data = preg_replace('@/\\*.*?\\*/@s', '', $data);
// Remove whitespace around symbols.
$data = preg_replace('@\\s*([{}:;,])\\s*@', '\\1', $data);
// Remove unnecessary semicolons.
$data = preg_replace('@;}@', '}', $data);
// Replace #rrggbb with #rgb when possible.
$data = preg_replace('@#([a-f0-9])\\1([a-f0-9])\\2([a-f0-9])\\3@i', '#\\1\\2\\3', $data);
$data = trim($data);
break;
case 'js':
$root = dirname(phutil_get_library_root('phabricator'));
$bin = $root . '/externals/javelin/support/jsxmin/jsxmin';
if (@file_exists($bin)) {
$future = new ExecFuture("{$bin} __DEV__:0");
$future->write($data);
list($err, $result) = $future->resolve();
if (!$err) {
$data = $result;
}
}
break;
}
return $data;
}
public function lintPath($path)
{
$flake8_bin = $this->getFlake8Path();
$options = $this->getFlake8Options();
$f = new ExecFuture("%C %C -", $flake8_bin, $options);
$f->write($this->getData($path));
list($err, $stdout, $stderr) = $f->resolve();
if ($err === 2) {
throw new Exception("flake8 failed to run correctly:\n" . $stderr);
}
$lines = explode("\n", $stdout);
$messages = array();
foreach ($lines as $line) {
$matches = null;
// stdin:2: W802 undefined name 'foo' # pyflakes
// stdin:3:1: E302 expected 2 blank lines, found 1 # pep8
if (!preg_match('/^(.*?):(\\d+):(?:(\\d+):)? (\\S+) (.*)$/', $line, $matches)) {
continue;
}
foreach ($matches as $key => $match) {
$matches[$key] = trim($match);
}
if (substr($matches[4], 0, 1) == 'E') {
$severity = ArcanistLintSeverity::SEVERITY_ERROR;
} else {
$severity = ArcanistLintSeverity::SEVERITY_WARNING;
}
$message = new ArcanistLintMessage();
$message->setPath($path);
$message->setLine($matches[2]);
if (!empty($matches[3])) {
$message->setChar($matches[3]);
}
$message->setCode($matches[4]);
$message->setName($this->getLinterName() . ' ' . $matches[3]);
$message->setDescription($matches[5]);
$message->setSeverity($severity);
$this->addLintMessage($message);
}
}
public function lintPath($path)
{
$this->checkBinaryConfiguration();
$data = $this->getData($path);
$future = new ExecFuture('%C', $this->getBinary());
$future->write($data);
list($err, $stdout, $stderr) = $future->resolve();
if (empty($stdout) && $err) {
throw new Exception(sprintf("%s\n\nSTDOUT\n%s\n\nSTDERR\n%s", pht($this->getLinterName() . ' failed to parse output!'), $stdout, $stderr));
}
if ($stdout !== $data) {
$lines = explode("\n", $data);
$formatted_lines = explode("\n", $stdout);
foreach ($lines as $line_idx => $line) {
if ($line != $formatted_lines[$line_idx]) {
$lines = array_slice($lines, $line_idx);
$formatted_lines = array_slice($formatted_lines, $line_idx);
break;
}
}
$this->raiseLintAtLine($line_idx + 1, 1, self::LINT_GO_UNFORMATTED, pht('%s was not formatted correctly. Please setup your ' . 'editor to run gofmt on save', $path), implode("\n", $lines), implode("\n", $formatted_lines));
}
}
public function markupContent($content, array $argv)
{
if (!Filesystem::binaryExists('cowsay')) {
return $this->markupError(pht('Unable to locate the `%s` binary. Install cowsay.', 'cowsay'));
}
$bin = idx($argv, 'think') ? 'cowthink' : 'cowsay';
$eyes = idx($argv, 'eyes', 'oo');
$tongue = idx($argv, 'tongue', ' ');
$cow = idx($argv, 'cow', 'default');
// NOTE: Strip this aggressively to prevent nonsense like
// `cow=/etc/passwd`. We could build a whiltelist with `cowsay -l`.
$cow = preg_replace('/[^a-z.-]+/', '', $cow);
$future = new ExecFuture('%s -e %s -T %s -f %s ', $bin, $eyes, $tongue, $cow);
$future->setTimeout(15);
$future->write($content);
list($err, $stdout, $stderr) = $future->resolve();
if ($err) {
return $this->markupError(pht('Execution of `%s` failed: %s', 'cowsay', $stderr));
}
if ($this->getEngine()->isTextMode()) {
return $stdout;
}
return phutil_tag('div', array('class' => 'PhabricatorMonospaced remarkup-cowsay'), $stdout);
}
protected final function launch()
{
$console = PhutilConsole::getConsole();
if ($this->debug) {
$console->writeOut("%s\n", pht('Starting Aphlict server in foreground...'));
} else {
Filesystem::writeFile($this->getPIDPath(), getmypid());
}
$command = csprintf('%s %s %Ls', $this->getNodeBinary(), $this->getAphlictScriptPath(), $this->getServerArgv());
if (!$this->debug) {
declare (ticks=1);
pcntl_signal(SIGINT, array($this, 'cleanup'));
pcntl_signal(SIGTERM, array($this, 'cleanup'));
}
register_shutdown_function(array($this, 'cleanup'));
if ($this->debug) {
$console->writeOut("%s\n\n \$ %s\n\n", pht('Launching server:'), $command);
$err = phutil_passthru('%C', $command);
$console->writeOut(">>> %s\n", pht('Server exited!'));
exit($err);
} else {
while (true) {
global $g_future;
$g_future = new ExecFuture('exec %C', $command);
$g_future->resolve();
// If the server exited, wait a couple of seconds and restart it.
unset($g_future);
sleep(2);
}
}
}
public function run()
{
if ($this->shouldRunSilently()) {
echo "Running daemon '{$this->daemon}' silently. Use '--trace' or " . "'--verbose' to produce debugging output.\n";
}
$root = phutil_get_library_root('phutil');
$root = dirname($root);
$exec_dir = $root . '/scripts/daemon/exec/';
// NOTE: PHP implements proc_open() by running 'sh -c'. On most systems this
// is bash, but on Ubuntu it's dash. When you proc_open() using bash, you
// get one new process (the command you ran). When you proc_open() using
// dash, you get two new processes: the command you ran and a parent
// "dash -c" (or "sh -c") process. This means that the child process's PID
// is actually the 'dash' PID, not the command's PID. To avoid this, use
// 'exec' to replace the shell process with the real process; without this,
// the child will call posix_getppid(), be given the pid of the 'sh -c'
// process, and send it SIGUSR1 to keepalive which will terminate it
// immediately. We also won't be able to do process group management because
// the shell process won't properly posix_setsid() so the pgid of the child
// won't be meaningful.
$exec_daemon = './exec_daemon.php';
$argv = $this->argv;
array_unshift($argv, 'exec', $exec_daemon, $this->daemon);
foreach ($argv as $k => $arg) {
$argv[$k] = escapeshellarg($arg);
}
$command = implode(' ', $argv);
while (true) {
$this->logMessage('INIT', 'Starting process.');
$future = new ExecFuture($command);
$future->setCWD($exec_dir);
$future->setStdoutSizeLimit($this->captureBufferSize);
$future->setStderrSizeLimit($this->captureBufferSize);
$this->deadline = time() + $this->deadlineTimeout;
$this->heartbeat = time() + self::HEARTBEAT_WAIT;
$future->isReady();
$this->childPID = $future->getPID();
do {
do {
if ($this->traceMemory) {
$memuse = number_format(memory_get_usage() / 1024, 1);
$this->logMessage('RAMS', 'Overseer Memory Usage: ' . $memuse . ' KB');
}
// We need a shortish timeout here so we can run the tick handler
// frequently in order to process signals.
$result = $future->resolve(1);
list($stdout, $stderr) = $future->read();
$stdout = trim($stdout);
$stderr = trim($stderr);
if (strlen($stdout)) {
$this->logMessage('STDO', $stdout, $stdout);
}
if (strlen($stderr)) {
$this->logMessage('STDE', $stderr, $stderr);
}
$future->discardBuffers();
if ($result !== null) {
list($err) = $result;
if ($err) {
$this->logMessage('FAIL', 'Process exited with error ' . $err . '.', $err);
} else {
$this->logMessage('DONE', 'Process exited successfully.');
}
break 2;
}
if ($this->heartbeat < time()) {
$this->heartbeat = time() + self::HEARTBEAT_WAIT;
if ($this->conduitURI) {
try {
$this->conduit = new ConduitClient($this->conduitURI);
$this->conduit->callMethodSynchronous('daemon.setstatus', array('daemonLogID' => $this->daemonLogID, 'status' => 'run'));
} catch (Exception $ex) {
}
}
}
} while (time() < $this->deadline);
$this->logMessage('HANG', 'Hang detected. Restarting process.');
$this->annihilateProcessGroup();
} while (false);
$this->logMessage('WAIT', 'Waiting to restart process.');
sleep($this->restartDelay);
}
}
private function runTests($targets)
{
$this->debugPrint("runTests(" . join($targets, ", ") . ")");
$future = new ExecFuture($this->bazelCommand("test", array_merge(["--verbose_failures", "--test_tag_filters=-broken", "--noshow_loading_progress", "--noshow_progress"], $targets)));
$future->setCWD($this->project_root);
$status = $future->resolve();
return $this->parseTestResults($targets, $status);
}
protected final function launch()
{
$console = PhutilConsole::getConsole();
if ($this->debug) {
$console->writeOut("%s\n", pht('Starting Aphlict server in foreground...'));
} else {
Filesystem::writeFile($this->getPIDPath(), getmypid());
}
$command = $this->getStartCommand($this->getServerArgv());
if (!$this->debug) {
declare (ticks=1);
pcntl_signal(SIGINT, array($this, 'cleanup'));
pcntl_signal(SIGTERM, array($this, 'cleanup'));
}
register_shutdown_function(array($this, 'cleanup'));
if ($this->debug) {
$console->writeOut("%s\n\n \$ %s\n\n", pht('Launching server:'), $command);
$err = phutil_passthru('%C', $command);
$console->writeOut(">>> %s\n", pht('Server exited!'));
exit($err);
} else {
while (true) {
global $g_future;
$g_future = new ExecFuture('exec %C', $command);
// Discard all output the subprocess produces: it writes to the log on
// disk, so we don't need to send the output anywhere and can just
// throw it away.
$g_future->setStdoutSizeLimit(0)->setStderrSizeLimit(0);
$g_future->resolve();
// If the server exited, wait a couple of seconds and restart it.
unset($g_future);
sleep(2);
}
}
}
declare (ticks=1);
pcntl_signal(SIGTERM, 'cleanup');
}
register_shutdown_function('cleanup');
// >>> pidfile -----------------------------------------------------------------
if ($g_pidfile) {
if (Filesystem::pathExists($g_pidfile)) {
$old_pid = (int) Filesystem::readFile($g_pidfile);
posix_kill($old_pid, SIGTERM);
sleep(1);
Filesystem::remove($g_pidfile);
}
Filesystem::writeFile($g_pidfile, getmypid());
}
// >>> run ---------------------------------------------------------------------
$command = csprintf('node %s %C', dirname(__FILE__) . '/aphlict_server.js', implode(' ', $server_argv));
if ($foreground) {
echo "Launching server:\n\n";
echo " \$ " . $command . "\n\n";
$err = phutil_passthru('%C', $command);
echo ">>> Server exited!\n";
exit($err);
} else {
while (true) {
$g_future = new ExecFuture('exec %C', $command);
$g_future->resolve();
// If the server exited, wait a couple of seconds and restart it.
unset($g_future);
sleep(2);
}
}
/**
* Handle resolving "arc:*" rules.
*/
private function resolveArcRule($rule, $name, $source)
{
switch ($name) {
case 'verbose':
$this->verbose = true;
$this->log("Enabled verbose mode.");
break;
case 'prompt':
$reason = "it is what you typed when prompted.";
$this->api->setBaseCommitExplanation($reason);
return phutil_console_prompt('Against which commit?');
case 'local':
case 'global':
case 'project':
case 'args':
case 'system':
// Push the other source on top of the list.
array_unshift($this->try, $name);
$this->log("Switching to source '{$name}'.");
return false;
case 'yield':
// Cycle this source to the end of the list.
$this->try[] = array_shift($this->try);
$this->log("Yielding processing of rules from '{$source}'.");
return false;
case 'halt':
// Dump the whole stack.
$this->try = array();
$this->log("Halting all rule processing.");
return false;
case 'skip':
return null;
case 'empty':
case 'upstream':
case 'outgoing':
case 'bookmark':
case 'amended':
case 'this':
return $this->api->resolveBaseCommitRule($rule, $source);
default:
$matches = null;
if (preg_match('/^exec\\((.*)\\)$/', $name, $matches)) {
$root = $this->api->getWorkingCopyIdentity()->getProjectRoot();
$future = new ExecFuture('%C', $matches[1]);
$future->setCWD($root);
list($err, $stdout) = $future->resolve();
if (!$err) {
return trim($stdout);
} else {
return null;
}
} else {
if (preg_match('/^nodiff\\((.*)\\)$/', $name, $matches)) {
return $this->api->resolveBaseCommitRule($rule, $source);
}
}
throw new ArcanistUsageException("Base commit rule '{$rule}' (from source '{$source}') " . "is not a recognized rule.");
}
}
protected final function launch($debug = false)
{
$console = PhutilConsole::getConsole();
if ($debug) {
$console->writeOut(pht("Starting Aphlict server in foreground...\n"));
} else {
Filesystem::writeFile($this->getPIDPath(), getmypid());
}
$server_uri = PhabricatorEnv::getEnvConfig('notification.server-uri');
$server_uri = new PhutilURI($server_uri);
$client_uri = PhabricatorEnv::getEnvConfig('notification.client-uri');
$client_uri = new PhutilURI($client_uri);
$user = PhabricatorEnv::getEnvConfig('notification.user');
$log = PhabricatorEnv::getEnvConfig('notification.log');
$server_argv = array();
$server_argv[] = csprintf('--port=%s', $client_uri->getPort());
$server_argv[] = csprintf('--admin=%s', $server_uri->getPort());
$server_argv[] = csprintf('--host=%s', $server_uri->getDomain());
if ($user) {
$server_argv[] = csprintf('--user=%s', $user);
}
if (!$debug) {
$server_argv[] = csprintf('--log=%s', $log);
}
$command = csprintf('%s %s %C', $this->getNodeBinary(), dirname(__FILE__) . '/../../../../support/aphlict/server/aphlict_server.js', implode(' ', $server_argv));
if (!$debug) {
declare (ticks=1);
pcntl_signal(SIGINT, array($this, 'cleanup'));
pcntl_signal(SIGTERM, array($this, 'cleanup'));
}
register_shutdown_function(array($this, 'cleanup'));
if ($debug) {
$console->writeOut("Launching server:\n\n \$ " . $command . "\n\n");
$err = phutil_passthru('%C', $command);
$console->writeOut(">>> Server exited!\n");
exit($err);
} else {
while (true) {
global $g_future;
$g_future = new ExecFuture('exec %C', $command);
$g_future->resolve();
// If the server exited, wait a couple of seconds and restart it.
unset($g_future);
sleep(2);
}
}
}
public function testReadBuffering()
{
$str_len_8 = 'abcdefgh';
$str_len_4 = 'abcd';
// This is a write/read with no read buffer.
$future = new ExecFuture('cat');
$future->write($str_len_8);
do {
$future->isReady();
list($read) = $future->read();
if (strlen($read)) {
break;
}
} while (true);
// We expect to get the entire string back in the read.
$this->assertEqual($str_len_8, $read);
$future->resolve();
// This is a write/read with a read buffer.
$future = new ExecFuture('cat');
$future->write($str_len_8);
// Set the read buffer size.
$future->setReadBufferSize(4);
do {
$future->isReady();
list($read) = $future->read();
if (strlen($read)) {
break;
}
} while (true);
// We expect to get the entire string back in the read.
$this->assertEqual($str_len_4, $read);
$future->resolve();
}
public function runIntegrationTests($tests)
{
// Now find all the test programs
$root = $this->getProjectRoot();
$test_dir = $root . "/tests/integration/";
if (!$tests) {
$paths = glob($test_dir . "*.php");
$paths[] = 'ruby/ruby-watchman/spec/ruby_watchman_spec.rb';
} else {
$paths = $tests;
}
foreach (array('/tmp/watchman-test.log', '/tmp/watchman-valgrind.log', '/tmp/watchman-valgrind.xml', '/tmp/watchman-callgrind.txt') as $log) {
@unlink($log);
}
foreach ($paths as $path) {
if (preg_match("/\\.php\$/", $path) && file_exists($path)) {
// Don't pull in files starting with "_"; we're using
// those as helpers for triggers
$base = basename($path);
if ($base[0] != '_') {
require_once $path;
}
}
}
// We test for this in a test case
putenv("WATCHMAN_EMPTY_ENV_VAR=");
$coverage = $this->getEnableCoverage();
if (!$this->first_inst) {
$this->first_inst = new WatchmanInstance($root, $coverage);
}
$first_inst = $this->first_inst;
$instances = array($first_inst);
// Helper for python or other language tests
putenv("WATCHMAN_SOCK=" . $first_inst->getFullSockName());
// Exercise the different serialization combinations
$cli_matrix = array();
// Find all the test cases that were declared
$results = array();
foreach (get_declared_classes() as $name) {
$ref = new ReflectionClass($name);
if (!$ref->isSubclassOf('WatchmanTestCase')) {
continue;
}
// Good enough; let's use it
$test_case = newv($name, array());
$config = $test_case->getGlobalConfig();
if ($config) {
$instance = new WatchmanInstance($root, $coverage, $config);
$instances[] = $instance;
} else {
$instance = $first_inst;
}
$test_case->setWatchmanInstance($instance);
if (!$instance->getProcessID()) {
$res = new ArcanistUnitTestResult();
$res->setName('dead');
$res->setUserData('died before test start');
$res->setResult(ArcanistUnitTestResult::RESULT_FAIL);
$results[] = array($res);
break;
}
$test_case->setRoot($root);
$test_case->setPaths($paths);
$results[] = $test_case->run();
if (!$test_case->needsLiveConnection()) {
foreach ($cli_matrix as $mname => $args) {
$test_case->useCLI($args);
$cli_results = $test_case->run();
foreach ($cli_results as $res) {
$res->setName($res->getName() . " [CLI: {$mname}]");
}
$results[] = $cli_results;
}
}
}
foreach ($paths as $path) {
if (!preg_match('/\\.rb$/', $path)) {
continue;
}
if (!file_exists($path)) {
// Was deleted in this (pending) rev
continue;
}
$start = microtime(true);
$future = new ExecFuture("PATH=\"{$root}:\$PATH\" \${MAKE:-make} rb-tests");
$future->setTimeout(10);
list($status, $out, $err) = $future->resolve();
$end = microtime(true);
$res = new ArcanistUnitTestResult();
$res->setName($path);
$res->setUserData($out . $err);
$res->setDuration($end - $start);
$res->setResult($status == 0 ? ArcanistUnitTestResult::RESULT_PASS : ArcanistUnitTestResult::RESULT_FAIL);
$results[] = array($res);
}
foreach ($instances as $instance) {
$results[] = $instance->generateValgrindTestResults();
}
$results = array_mergev($results);
return $results;
}
public function run()
{
if ($this->shouldRunSilently()) {
echo "Running daemon '{$this->daemon}' silently. Use '--trace' or " . "'--verbose' to produce debugging output.\n";
}
$root = phutil_get_library_root('phutil');
$root = dirname($root);
$exec_dir = $root . '/scripts/daemon/exec/';
// NOTE: PHP implements proc_open() by running 'sh -c'. On most systems this
// is bash, but on Ubuntu it's dash. When you proc_open() using bash, you
// get one new process (the command you ran). When you proc_open() using
// dash, you get two new processes: the command you ran and a parent
// "dash -c" (or "sh -c") process. This means that the child process's PID
// is actually the 'dash' PID, not the command's PID. To avoid this, use
// 'exec' to replace the shell process with the real process; without this,
// the child will call posix_getppid(), be given the pid of the 'sh -c'
// process, and send it SIGUSR1 to keepalive which will terminate it
// immediately. We also won't be able to do process group management because
// the shell process won't properly posix_setsid() so the pgid of the child
// won't be meaningful.
// Format the exec command, which looks something like:
//
// exec ./exec_daemon DaemonName --trace -- --no-discovery
$argv = array();
$argv[] = csprintf('exec ./exec_daemon.php %s', $this->daemon);
foreach ($this->argv as $k => $arg) {
$argv[] = csprintf('%s', $arg);
}
$argv[] = '--';
foreach ($this->moreArgs as $k => $arg) {
$argv[] = csprintf('%s', $arg);
}
$command = implode(' ', $argv);
while (true) {
$this->logMessage('INIT', 'Starting process.');
$future = new ExecFuture('%C', $command);
$future->setCWD($exec_dir);
$future->setStdoutSizeLimit($this->captureBufferSize);
$future->setStderrSizeLimit($this->captureBufferSize);
$this->deadline = time() + $this->deadlineTimeout;
$this->heartbeat = time() + self::HEARTBEAT_WAIT;
$future->isReady();
$this->childPID = $future->getPID();
do {
do {
if ($this->traceMemory) {
$memuse = number_format(memory_get_usage() / 1024, 1);
$this->logMessage('RAMS', 'Overseer Memory Usage: ' . $memuse . ' KB');
}
// We need a shortish timeout here so we can run the tick handler
// frequently in order to process signals.
$result = $future->resolve(1);
list($stdout, $stderr) = $future->read();
$stdout = trim($stdout);
$stderr = trim($stderr);
if (strlen($stdout)) {
$this->logMessage('STDO', $stdout);
}
if (strlen($stderr)) {
$this->logMessage('STDE', $stderr);
}
$future->discardBuffers();
if ($result !== null) {
list($err) = $result;
if ($err) {
$this->logMessage('FAIL', 'Process exited with error ' . $err . '.', $err);
} else {
$this->logMessage('DONE', 'Process exited successfully.');
}
break 2;
}
if ($this->heartbeat < time()) {
$this->heartbeat = time() + self::HEARTBEAT_WAIT;
$this->dispatchEvent(self::EVENT_DID_HEARTBEAT);
}
} while (time() < $this->deadline);
$this->logMessage('HANG', 'Hang detected. Restarting process.');
$this->annihilateProcessGroup();
} while (false);
if ($this->inGracefulShutdown) {
// If we just exited because of a graceful shutdown, break now.
break;
}
$this->logMessage('WAIT', 'Waiting to restart process.');
sleep(self::RESTART_WAIT);
if ($this->inGracefulShutdown) {
// If we were awakend by a graceful shutdown, break now.
break;
}
}
// This is a clean exit after a graceful shutdown.
$this->dispatchEvent(self::EVENT_WILL_EXIT);
exit(0);
}
public function getOpCodes()
{
$op_codes = array();
$file_a = new TempFile();
$file_b = new TempFile();
Filesystem::writeFile($file_a, implode('', $this->a));
Filesystem::writeFile($file_b, implode('', $this->b));
$diff_future = new ExecFuture('diff -U0 %s %s', $file_a, $file_b);
list($err, $stdout, $stderr) = $diff_future->resolve();
if (!in_array($err, array(0, 1))) {
throw new CommandException(pht('`diff` returned unexpected exit code %d', $err), $diff_future->getCommand(), $err, $stdout, $stderr);
}
foreach (phutil_split_lines($stdout) as $line) {
$matches = null;
$regexp = '/^@@ -(\\d+)(?:,(\\d+))? \\+(\\d+)(?:,(\\d+))? @@/';
if (!preg_match($regexp, $line, $matches)) {
continue;
}
// We need to:
// - normalize indices; the ones we get from `diff` are 1-indexed.
// - account for how `diff` represents empty hunks -- essentially, it
// attributes the change to the *previous line* (which may be 0!). This
// differs from Python's difflib.
//
// http://www.gnu.org/software/diffutils/manual/html_node/Detailed-Unified.html:
// If a hunk contains just one line, only its start line number
// appears. Otherwise its line numbers look like 'start,count'. An
// empty hunk is considered to start at the line that follows the
// hunk.
//
// If a hunk and its context contain two or more lines, its line
// numbers look like 'start,count'. Otherwise only its end line
// number appears. An empty hunk is considered to end at the line
// that precedes the hunk.
$i = (int) $matches[1] - 1;
$j = (int) $matches[3] - 1;
if (isset($matches[2]) && $matches[2] !== '') {
$li = (int) $matches[2];
} else {
$li = 1;
}
if (isset($matches[4]) && $matches[4] !== '') {
$lj = (int) $matches[4];
} else {
$lj = 1;
}
if (!($li || $lj)) {
throw new CommandException(pht('Malformed output from `diff`.'), $diff_future->getCommand(), $err, $stdout, $stderr);
}
if ($li === 0) {
$op_codes[] = array('insert', $i + 1, $i + 1, $j, $j + $lj);
} else {
if ($lj === 0) {
$op_codes[] = array('delete', $i, $i + $li, $j, $j);
} else {
$op_codes[] = array('replace', $i, $i + $li, $j, $j + $lj);
}
}
}
return $op_codes;
}
/**
* Determines what executables and lint paths to use. Between platforms
* this also changes whether the lint engine is run under .NET or Mono. It
* also ensures that all of the required binaries are available for the lint
* to run successfully.
*
* @return void
*/
private function loadEnvironment()
{
if ($this->loaded) {
return;
}
// Determine runtime engine (.NET or Mono).
if (phutil_is_windows()) {
$this->runtimeEngine = '';
} else {
if (Filesystem::binaryExists('mono')) {
$this->runtimeEngine = 'mono ';
} else {
throw new Exception(pht('Unable to find Mono and you are not on Windows!'));
}
}
// Determine cslint path.
$cslint = $this->cslintHintPath;
if ($cslint !== null && file_exists($cslint)) {
$this->cslintEngine = Filesystem::resolvePath($cslint);
} else {
if (Filesystem::binaryExists('cslint.exe')) {
$this->cslintEngine = 'cslint.exe';
} else {
throw new Exception(pht('Unable to locate %s.', 'cslint'));
}
}
// Determine cslint version.
$ver_future = new ExecFuture('%C -v', $this->runtimeEngine . $this->cslintEngine);
list($err, $stdout, $stderr) = $ver_future->resolve();
if ($err !== 0) {
throw new Exception(pht('You are running an old version of %s. Please ' . 'upgrade to version %s.', 'cslint', self::SUPPORTED_VERSION));
}
$ver = (int) $stdout;
if ($ver < self::SUPPORTED_VERSION) {
throw new Exception(pht('You are running an old version of %s. Please ' . 'upgrade to version %s.', 'cslint', self::SUPPORTED_VERSION));
} else {
if ($ver > self::SUPPORTED_VERSION) {
throw new Exception(pht('Arcanist does not support this version of %s (it is newer). ' . 'You can try upgrading Arcanist with `%s`.', 'cslint', 'arc upgrade'));
}
}
$this->loaded = true;
}
public function runIntegrationTests($tests)
{
$this->make('all');
// Now find all the test programs
$root = $this->getProjectRoot();
$test_dir = $root . "/tests/integration/";
if (!$tests) {
$paths = glob($test_dir . "*.php");
foreach (glob('python/tests/*.py') as $file) {
$paths[] = $file;
}
$paths[] = 'ruby/ruby-watchman/spec/ruby_watchman_spec.rb';
} else {
$paths = $tests;
}
foreach (array('/tmp/watchman-test.log', '/tmp/watchman-valgrind.log', '/tmp/watchman-valgrind.xml', '/tmp/watchman-callgrind.txt') as $log) {
@unlink($log);
}
foreach ($paths as $path) {
if (preg_match("/\\.php\$/", $path) && file_exists($path)) {
require_once $path;
}
}
// We test for this in a test case
putenv("WATCHMAN_EMPTY_ENV_VAR=");
$coverage = $this->getEnableCoverage();
$first_inst = new WatchmanInstance($root, $coverage);
$instances = array($first_inst);
// Helper for python or other language tests
putenv("WATCHMAN_SOCK=" . $first_inst->getFullSockName());
// Exercise the different serialization combinations
$cli_matrix = array('bser/json' => '--server-encoding=bser --output-encoding=json', 'json/json' => '--server-encoding=json --output-encoding=json');
// Find all the test cases that were declared
$results = array();
foreach (get_declared_classes() as $name) {
$ref = new ReflectionClass($name);
if (!$ref->isSubclassOf('WatchmanTestCase')) {
continue;
}
// Good enough; let's use it
$test_case = newv($name, array());
$config = $test_case->getGlobalConfig();
if ($config) {
$instance = new WatchmanInstance($root, $coverage, $config);
$instances[] = $instance;
} else {
$instance = $first_inst;
}
$test_case->setWatchmanInstance($instance);
if (!$instance->getProcessID()) {
$res = new ArcanistUnitTestResult();
$res->setName('dead');
$res->setUserData('died before test start');
$res->setResult(ArcanistUnitTestResult::RESULT_FAIL);
$results[] = array($res);
break;
}
$test_case->setRoot($root);
$test_case->setPaths($paths);
$results[] = $test_case->run();
if (!$test_case->needsLiveConnection()) {
foreach ($cli_matrix as $mname => $args) {
$test_case->useCLI($args);
$cli_results = $test_case->run();
foreach ($cli_results as $res) {
$res->setName($res->getName() . " [CLI: {$mname}]");
}
$results[] = $cli_results;
}
}
}
// Also run the python tests if we built them
foreach ($paths as $path) {
if (!preg_match('/test.*\\.py$/', $path)) {
continue;
}
if (!file_exists($path)) {
// Was deleted in this (pending) rev
continue;
}
if (!file_exists("python/pywatchman/bser.so")) {
// Not enabled by the build
continue;
}
// Note that this implicitly starts the instance if we haven't
// yet done so. This is important if the only test paths are
// python paths
if (!$first_inst->getProcessID()) {
$res = new ArcanistUnitTestResult();
$res->setName('dead');
$res->setUserData('died before test start');
$res->setResult(ArcanistUnitTestResult::RESULT_FAIL);
$results[] = array($res);
break;
}
// our Makefile contains the detected python, so just run the
// rule from the makefile to pick it up
$start = microtime(true);
$future = new ExecFuture("PATH=\"{$root}:\$PATH\" PYTHONPATH={$root}/python " . "TESTNAME={$path} \${MAKE:-make} py-tests");
$future->setTimeout(10);
list($status, $out, $err) = $future->resolve();
$end = microtime(true);
$res = new ArcanistUnitTestResult();
$res->setName($path);
$res->setUserData($out . $err);
$res->setDuration($end - $start);
$res->setResult($status == 0 ? ArcanistUnitTestResult::RESULT_PASS : ArcanistUnitTestResult::RESULT_FAIL);
$results[] = array($res);
}
foreach ($paths as $path) {
if (!preg_match('/\\.rb$/', $path)) {
continue;
}
if (!file_exists($path)) {
// Was deleted in this (pending) rev
continue;
}
$start = microtime(true);
$future = new ExecFuture("PATH=\"{$root}:\$PATH\" \${MAKE:-make} rb-tests");
$future->setTimeout(10);
list($status, $out, $err) = $future->resolve();
$end = microtime(true);
$res = new ArcanistUnitTestResult();
$res->setName($path);
$res->setUserData($out . $err);
$res->setDuration($end - $start);
$res->setResult($status == 0 ? ArcanistUnitTestResult::RESULT_PASS : ArcanistUnitTestResult::RESULT_FAIL);
$results[] = array($res);
}
foreach ($instances as $instance) {
$results[] = $instance->generateValgrindTestResults();
}
$results = array_mergev($results);
return $results;
}
