protected function didReceiveResult($result)
{
if ($this->profilerCallID !== null) {
$profiler = PhutilServiceProfiler::getInstance();
$profiler->endServiceCall($this->profilerCallID, array());
}
list($status, $body, $headers) = $result;
if ($status->isError()) {
throw $status;
}
$raw = $body;
$shield = 'for(;;);';
if (!strncmp($raw, $shield, strlen($shield))) {
$raw = substr($raw, strlen($shield));
}
$data = json_decode($raw, true);
if (!is_array($data)) {
throw new Exception("Host returned HTTP/200, but invalid JSON data in response to " . "a Conduit method call:\n{$raw}");
}
if ($data['error_code']) {
throw new ConduitClientException($data['error_code'], $data['error_info']);
}
$result = $data['result'];
$result = $this->client->didReceiveResponse($this->conduitMethod, $result);
return $result;
}
/**
* Execute this command.
*
* @return int Error code returned by the subprocess.
*
* @task command
*/
public function execute()
{
$command = $this->command;
$profiler = PhutilServiceProfiler::getInstance();
$call_id = $profiler->beginServiceCall(array('type' => 'exec', 'subtype' => 'passthru', 'command' => $command));
$spec = array(STDIN, STDOUT, STDERR);
$pipes = array();
if ($command instanceof PhutilCommandString) {
$unmasked_command = $command->getUnmaskedString();
} else {
$unmasked_command = $command;
}
$env = $this->env;
$cwd = $this->cwd;
$options = array();
if (phutil_is_windows()) {
// Without 'bypass_shell', things like launching vim don't work properly,
// and we can't execute commands with spaces in them, and all commands
// invoked from git bash fail horridly, and everything is a mess in
// general.
$options['bypass_shell'] = true;
}
$trap = new PhutilErrorTrap();
$proc = @proc_open($unmasked_command, $spec, $pipes, $cwd, $env, $options);
$errors = $trap->getErrorsAsString();
$trap->destroy();
if (!is_resource($proc)) {
throw new Exception(pht('Failed to passthru %s: %s', 'proc_open()', $errors));
}
$err = proc_close($proc);
$profiler->endServiceCall($call_id, array('err' => $err));
return $err;
}
protected function didReceiveResult($result)
{
if ($this->profilerCallID !== null) {
$profiler = PhutilServiceProfiler::getInstance();
$profiler->endServiceCall($this->profilerCallID, array());
}
list($status, $body, $headers) = $result;
if ($status->isError()) {
throw $status;
}
$raw = $body;
$shield = 'for(;;);';
if (!strncmp($raw, $shield, strlen($shield))) {
$raw = substr($raw, strlen($shield));
}
$data = null;
try {
$data = phutil_json_decode($raw);
} catch (PhutilJSONParserException $ex) {
throw new PhutilProxyException(pht('Host returned HTTP/200, but invalid JSON data in response to ' . 'a Conduit method call.'), $ex);
}
if ($data['error_code']) {
throw new ConduitClientException($data['error_code'], $data['error_info']);
}
$result = $data['result'];
$result = $this->client->didReceiveResponse($this->conduitMethod, $result);
return $result;
}
/**
* Execute a command which takes over stdin, stdout and stderr, similar to
* passthru(), but which preserves TTY semantics, escapes arguments, and is
* traceable.
*
* @param string sprintf()-style command pattern to execute.
* @param ... Arguments to sprintf pattern.
* @return int Return code.
* @group exec
*/
function phutil_passthru($cmd)
{
$args = func_get_args();
$command = call_user_func_array('csprintf', $args);
$profiler = PhutilServiceProfiler::getInstance();
$call_id = $profiler->beginServiceCall(array('type' => 'exec', 'subtype' => 'passthru', 'command' => $command));
$spec = array(STDIN, STDOUT, STDERR);
$pipes = array();
if (phutil_is_windows()) {
// Without 'bypass_shell', things like launching vim don't work properly,
// and we can't execute commands with spaces in them, and all commands
// invoked from git bash fail horridly, and everything is a mess in general.
$options = array('bypass_shell' => true);
$proc = @proc_open($command, $spec, $pipes, null, null, $options);
} else {
$proc = @proc_open($command, $spec, $pipes);
}
if ($proc === false) {
$err = 1;
} else {
$err = proc_close($proc);
}
$profiler->endServiceCall($call_id, array('err' => $err));
return $err;
}
public static function getInstance()
{
if (empty(self::$instance)) {
self::$instance = new PhutilServiceProfiler();
}
return self::$instance;
}
public static function dispatchEvent(PhutilEvent $event)
{
$instance = self::getInstance();
$listeners = idx($instance->listeners, $event->getType(), array());
$global_listeners = idx($instance->listeners, PhutilEventType::TYPE_ALL, array());
// Merge and deduplicate listeners (we want to send the event to each
// listener only once, even if it satisfies multiple criteria for the
// event).
$listeners = array_merge($listeners, $global_listeners);
$listeners = mpull($listeners, null, 'getListenerID');
$profiler = PhutilServiceProfiler::getInstance();
$profiler_id = $profiler->beginServiceCall(array('type' => 'event', 'kind' => $event->getType(), 'count' => count($listeners)));
$caught = null;
try {
foreach ($listeners as $listener) {
if ($event->isStopped()) {
// Do this first so if someone tries to dispatch a stopped event it
// doesn't go anywhere. Silly but less surprising.
break;
}
$listener->handleEvent($event);
}
} catch (Exception $ex) {
$profiler->endServiceCall($profiler_id, array());
throw $ex;
}
$profiler->endServiceCall($profiler_id, array());
}
/**
* Delete a blob from Amazon S3.
*/
public function deleteFile($handle)
{
AphrontWriteGuard::willWrite();
$s3 = $this->newS3API();
$profiler = PhutilServiceProfiler::getInstance();
$call_id = $profiler->beginServiceCall(array('type' => 's3', 'method' => 'deleteObject'));
$s3->deleteObject($this->getBucketName(), $handle);
$profiler->endServiceCall($call_id, array());
}
/**
* Delete a blob from Amazon S3.
*/
public function deleteFile($handle)
{
$s3 = $this->newS3API();
AphrontWriteGuard::willWrite();
$profiler = PhutilServiceProfiler::getInstance();
$call_id = $profiler->beginServiceCall(array('type' => 's3', 'method' => 'deleteObject'));
$s3->setParametersForDeleteObject($handle)->resolve();
$profiler->endServiceCall($call_id, array());
}
/**
* Execute a command which takes over stdin, stdout and stderr, similar to
* passthru(), but which preserves TTY semantics, escapes arguments, and is
* traceable.
*
* @param string sprintf()-style command pattern to execute.
* @param ... Arguments to sprintf pattern.
* @return int Return code.
* @group exec
*/
function phutil_passthru($cmd)
{
$args = func_get_args();
$command = call_user_func_array('csprintf', $args);
$profiler = PhutilServiceProfiler::getInstance();
$call_id = $profiler->beginServiceCall(array('type' => 'exec', 'subtype' => 'passthru', 'command' => $command));
$pipes = array();
$proc = proc_open($command, array(STDIN, STDOUT, STDERR), $pipes);
$err = proc_close($proc);
$profiler->endServiceCall($call_id, array('err' => $err));
return $err;
}
public function execute()
{
$profiler = PhutilServiceProfiler::getInstance();
$call_id = $profiler->beginServiceCall(array('type' => 'conduit', 'method' => $this->method));
try {
$result = $this->executeMethod();
} catch (Exception $ex) {
$profiler->endServiceCall($call_id, array());
throw $ex;
}
$profiler->endServiceCall($call_id, array());
return $result;
}
public static function initializeScriptEnvironment()
{
self::initializeCommonEnvironment();
// NOTE: This is dangerous in general, but we know we're in a script context
// and are not vulnerable to CSRF.
AphrontWriteGuard::allowDangerousUnguardedWrites(true);
// There are several places where we log information (about errors, events,
// service calls, etc.) for analysis via DarkConsole or similar. These are
// useful for web requests, but grow unboundedly in long-running scripts and
// daemons. Discard data as it arrives in these cases.
PhutilServiceProfiler::getInstance()->enableDiscardMode();
DarkConsoleErrorLogPluginAPI::enableDiscardMode();
DarkConsoleEventPluginAPI::enableDiscardMode();
}
private function doCacheTest(PhutilKeyValueCache $cache)
{
$key1 = 'test:' . mt_rand();
$key2 = 'test:' . mt_rand();
$default = 'cache-miss';
$value1 = 'cache-hit1';
$value2 = 'cache-hit2';
$cache->setProfiler(PhutilServiceProfiler::getInstance());
$test_info = get_class($cache);
// Test that we miss correctly on missing values.
$this->assertEqual($default, $cache->getKey($key1, $default), $test_info);
$this->assertEqual(array(), $cache->getKeys(array($key1, $key2)), $test_info);
// Test that we can set individual keys.
$cache->setKey($key1, $value1);
$this->assertEqual($value1, $cache->getKey($key1, $default), $test_info);
$this->assertEqual(array($key1 => $value1), $cache->getKeys(array($key1, $key2)), $test_info);
// Test that we can delete individual keys.
$cache->deleteKey($key1);
$this->assertEqual($default, $cache->getKey($key1, $default), $test_info);
$this->assertEqual(array(), $cache->getKeys(array($key1, $key2)), $test_info);
// Test that we can set multiple keys.
$cache->setKeys(array($key1 => $value1, $key2 => $value2));
$this->assertEqual($value1, $cache->getKey($key1, $default), $test_info);
$this->assertEqual(array($key1 => $value1, $key2 => $value2), $cache->getKeys(array($key1, $key2)), $test_info);
// Test that we can delete multiple keys.
$cache->deleteKeys(array($key1, $key2));
$this->assertEqual($default, $cache->getKey($key1, $default), $test_info);
$this->assertEqual(array(), $cache->getKeys(array($key1, $key2)), $test_info);
// NOTE: The TTL tests are necessarily slow (we must sleep() through the
// TTLs) and do not work with APC (it does not TTL until the next request)
// so they're disabled by default. If you're developing the cache stack,
// it may be useful to run them.
return;
// Test that keys expire when they TTL.
$cache->setKey($key1, $value1, 1);
$cache->setKey($key2, $value2, 5);
$this->assertEqual($value1, $cache->getKey($key1, $default));
$this->assertEqual($value2, $cache->getKey($key2, $default));
sleep(2);
$this->assertEqual($default, $cache->getKey($key1, $default));
$this->assertEqual($value2, $cache->getKey($key2, $default));
// Test that setting a 0 TTL overwrites a nonzero TTL.
$cache->setKey($key1, $value1, 1);
$this->assertEqual($value1, $cache->getKey($key1, $default));
$cache->setKey($key1, $value1, 0);
$this->assertEqual($value1, $cache->getKey($key1, $default));
sleep(2);
$this->assertEqual($value1, $cache->getKey($key1, $default));
}
public function processRequest()
{
$request = $this->getRequest();
$user = $request->getUser();
$cache = new PhabricatorKeyValueDatabaseCache();
$cache = new PhutilKeyValueCacheProfiler($cache);
$cache->setProfiler(PhutilServiceProfiler::getInstance());
$result = $cache->getKey('darkconsole:' . $this->key);
if (!$result) {
return new Aphront400Response();
}
try {
$result = phutil_json_decode($result);
} catch (PhutilJSONParserException $ex) {
return new Aphront400Response();
}
if ($result['vers'] != DarkConsoleCore::STORAGE_VERSION) {
return new Aphront400Response();
}
if ($result['user'] != $user->getPHID()) {
return new Aphront400Response();
}
$output = array();
$output['tabs'] = $result['tabs'];
$output['panel'] = array();
foreach ($result['data'] as $class => $data) {
try {
$obj = newv($class, array());
$obj->setData($data);
$obj->setRequest($request);
$panel = $obj->renderPanel();
// Because cookie names can now be prefixed, wipe out any cookie value
// with the session cookie name anywhere in its name.
$pattern = '(' . preg_quote(PhabricatorCookies::COOKIE_SESSION) . ')';
foreach ($_COOKIE as $cookie_name => $cookie_value) {
if (preg_match($pattern, $cookie_name)) {
$panel = PhutilSafeHTML::applyFunction('str_replace', $cookie_value, '(session-key)', $panel);
}
}
$output['panel'][$class] = $panel;
} catch (Exception $ex) {
$output['panel'][$class] = 'error';
}
}
return id(new AphrontAjaxResponse())->setContent($output);
}
public function getKey(AphrontRequest $request)
{
$plugins = $this->getPlugins();
foreach ($plugins as $plugin) {
$plugin->setRequest($request);
$plugin->willShutdown();
}
foreach ($plugins as $plugin) {
$plugin->didShutdown();
}
foreach ($plugins as $plugin) {
$plugin->setData($plugin->generateData());
}
$plugins = msort($plugins, 'getOrderKey');
$key = Filesystem::readRandomCharacters(24);
$tabs = array();
$data = array();
foreach ($plugins as $plugin) {
$class = get_class($plugin);
$tabs[] = array('class' => $class, 'name' => $plugin->getName(), 'color' => $plugin->getColor());
$data[$class] = $this->sanitizeForJSON($plugin->getData());
}
$storage = array('vers' => self::STORAGE_VERSION, 'tabs' => $tabs, 'data' => $data, 'user' => $request->getUser() ? $request->getUser()->getPHID() : null);
$cache = new PhabricatorKeyValueDatabaseCache();
$cache = new PhutilKeyValueCacheProfiler($cache);
$cache->setProfiler(PhutilServiceProfiler::getInstance());
// This encoding may fail if there are, e.g., database queries which
// include binary data. It would be a little cleaner to try to strip these,
// but just do something non-broken here if we end up with unrepresentable
// data.
$json = @json_encode($storage);
if (!$json) {
$json = '{}';
}
$cache->setKeys(array('darkconsole:' . $key => $json), $ttl = 60 * 60 * 6);
return $key;
}
}
echo phutil_console_wrap(phutil_console_format('This script will test that you have configured valid credentials for ' . 'access to a repository, so the Phabricator daemons can pull from it. ' . 'You should run this as the **same user you will run the daemons as**, ' . 'from the **same machine they will run from**. Doing this will help ' . 'detect various problems with your configuration, such as SSH issues.'));
list($whoami) = execx('whoami');
$whoami = trim($whoami);
$ok = phutil_console_confirm("Do you want to continue as '{$whoami}'?");
if (!$ok) {
die(1);
}
$callsign = $argv[1];
echo "Loading '{$callsign}' repository...\n";
$repository = id(new PhabricatorRepository())->loadOneWhere('callsign = %s', $argv[1]);
if (!$repository) {
throw new Exception("No such repository exists!");
}
$vcs = $repository->getVersionControlSystem();
PhutilServiceProfiler::installEchoListener();
echo "Trying to connect to the remote...\n";
switch ($vcs) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
$err = $repository->passthruRemoteCommand('--limit 1 log %s', $repository->getRemoteURI());
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
// Do an ls-remote on a nonexistent ref, which we expect to just return
// nothing.
$err = $repository->passthruRemoteCommand('ls-remote %s %s', $repository->getRemoteURI(), 'just-testing');
break;
default:
throw new Exception("Unsupported repository type.");
}
if ($err) {
echo phutil_console_format("<bg:red>** FAIL **</bg> Connection failed. The credentials for this " . "repository appear to be incorrectly configured.\n");
public function __construct(array $argv)
{
PhutilServiceProfiler::getInstance()->enableDiscardMode();
$original_argv = $argv;
$args = new PhutilArgumentParser($argv);
$args->setTagline('daemon overseer');
$args->setSynopsis(<<<EOHELP
**launch_daemon.php** [__options__] __daemon__
Launch and oversee an instance of __daemon__.
EOHELP
);
$args->parseStandardArguments();
$args->parsePartial(array(array('name' => 'trace-memory', 'help' => 'Enable debug memory tracing.'), array('name' => 'log', 'param' => 'file', 'help' => 'Send output to __file__.'), array('name' => 'daemonize', 'help' => 'Run in the background.'), array('name' => 'phd', 'param' => 'dir', 'help' => 'Write PID information to __dir__.'), array('name' => 'verbose', 'help' => 'Enable verbose activity logging.'), array('name' => 'load-phutil-library', 'param' => 'library', 'repeat' => true, 'help' => 'Load __library__.')));
$argv = array();
$more = $args->getUnconsumedArgumentVector();
$this->daemon = array_shift($more);
if (!$this->daemon) {
$args->printHelpAndExit();
}
if ($args->getArg('trace')) {
$this->traceMode = true;
$argv[] = '--trace';
}
if ($args->getArg('trace-memory')) {
$this->traceMode = true;
$this->traceMemory = true;
$argv[] = '--trace-memory';
}
if ($args->getArg('load-phutil-library')) {
foreach ($args->getArg('load-phutil-library') as $library) {
$argv[] = '--load-phutil-library=' . $library;
}
}
$log = $args->getArg('log');
if ($log) {
ini_set('error_log', $log);
$argv[] = '--log=' . $log;
}
$verbose = $args->getArg('verbose');
if ($verbose) {
$this->verbose = true;
$argv[] = '--verbose';
}
$this->daemonize = $args->getArg('daemonize');
$this->phddir = $args->getArg('phd');
$this->argv = $argv;
$this->moreArgs = coalesce($more, array());
error_log("Bringing daemon '{$this->daemon}' online...");
if (self::$instance) {
throw new Exception('You may not instantiate more than one Overseer per process.');
}
self::$instance = $this;
if ($this->daemonize) {
// We need to get rid of these or the daemon will hang when we TERM it
// waiting for something to read the buffers. TODO: Learn how unix works.
fclose(STDOUT);
fclose(STDERR);
ob_start();
$pid = pcntl_fork();
if ($pid === -1) {
throw new Exception('Unable to fork!');
} else {
if ($pid) {
exit(0);
}
}
}
if ($this->phddir) {
$desc = array('name' => $this->daemon, 'argv' => $this->moreArgs, 'pid' => getmypid(), 'start' => time());
Filesystem::writeFile($this->phddir . '/daemon.' . getmypid(), json_encode($desc));
}
$this->daemonID = $this->generateDaemonID();
$this->dispatchEvent(self::EVENT_DID_LAUNCH, array('argv' => array_slice($original_argv, 1), 'explicitArgv' => $this->moreArgs));
declare (ticks=1);
pcntl_signal(SIGUSR1, array($this, 'didReceiveKeepaliveSignal'));
pcntl_signal(SIGUSR2, array($this, 'didReceiveNotifySignal'));
pcntl_signal(SIGINT, array($this, 'didReceiveGracefulSignal'));
pcntl_signal(SIGTERM, array($this, 'didReceiveTerminalSignal'));
}
public function __construct(array $argv)
{
PhutilServiceProfiler::getInstance()->enableDiscardMode();
$args = new PhutilArgumentParser($argv);
$args->setTagline(pht('daemon overseer'));
$args->setSynopsis(<<<EOHELP
**launch_daemon.php** [__options__] __daemon__
Launch and oversee an instance of __daemon__.
EOHELP
);
$args->parseStandardArguments();
$args->parse(array(array('name' => 'trace-memory', 'help' => pht('Enable debug memory tracing.')), array('name' => 'verbose', 'help' => pht('Enable verbose activity logging.')), array('name' => 'label', 'short' => 'l', 'param' => 'label', 'help' => pht('Optional process label. Makes "%s" nicer, no behavioral effects.', 'ps'))));
$argv = array();
if ($args->getArg('trace')) {
$this->traceMode = true;
$argv[] = '--trace';
}
if ($args->getArg('trace-memory')) {
$this->traceMode = true;
$this->traceMemory = true;
$argv[] = '--trace-memory';
}
$verbose = $args->getArg('verbose');
if ($verbose) {
$this->verbose = true;
$argv[] = '--verbose';
}
$label = $args->getArg('label');
if ($label) {
$argv[] = '-l';
$argv[] = $label;
}
$this->argv = $argv;
if (function_exists('posix_isatty') && posix_isatty(STDIN)) {
fprintf(STDERR, pht('Reading daemon configuration from stdin...') . "\n");
}
$config = @file_get_contents('php://stdin');
$config = id(new PhutilJSONParser())->parse($config);
$this->libraries = idx($config, 'load');
$this->log = idx($config, 'log');
$this->daemonize = idx($config, 'daemonize');
$this->piddir = idx($config, 'piddir');
$this->config = $config;
if (self::$instance) {
throw new Exception(pht('You may not instantiate more than one Overseer per process.'));
}
self::$instance = $this;
$this->startEpoch = time();
// Check this before we daemonize, since if it's an issue the child will
// exit immediately.
if ($this->piddir) {
$dir = $this->piddir;
try {
Filesystem::assertWritable($dir);
} catch (Exception $ex) {
throw new Exception(pht("Specified daemon PID directory ('%s') does not exist or is " . "not writable by the daemon user!", $dir));
}
}
if (!idx($config, 'daemons')) {
throw new PhutilArgumentUsageException(pht('You must specify at least one daemon to start!'));
}
if ($this->log) {
// NOTE: Now that we're committed to daemonizing, redirect the error
// log if we have a `--log` parameter. Do this at the last moment
// so as many setup issues as possible are surfaced.
ini_set('error_log', $this->log);
}
if ($this->daemonize) {
// We need to get rid of these or the daemon will hang when we TERM it
// waiting for something to read the buffers. TODO: Learn how unix works.
fclose(STDOUT);
fclose(STDERR);
ob_start();
$pid = pcntl_fork();
if ($pid === -1) {
throw new Exception(pht('Unable to fork!'));
} else {
if ($pid) {
exit(0);
}
}
}
$this->modules = PhutilDaemonOverseerModule::getAllModules();
pcntl_signal(SIGUSR2, array($this, 'didReceiveNotifySignal'));
pcntl_signal(SIGHUP, array($this, 'didReceiveReloadSignal'));
pcntl_signal(SIGINT, array($this, 'didReceiveGracefulSignal'));
pcntl_signal(SIGTERM, array($this, 'didReceiveTerminalSignal'));
}
/**
* Close and free resources if necessary.
*
* @return void
* @task internal
*/
private function closeProcess()
{
foreach ($this->pipes as $pipe) {
if (isset($pipe)) {
@fclose($pipe);
}
}
$this->pipes = array(null, null, null);
if ($this->proc) {
@proc_close($this->proc);
$this->proc = null;
}
$this->stdin = null;
if ($this->profilerCallID !== null) {
$profiler = PhutilServiceProfiler::getInstance();
$profiler->endServiceCall($this->profilerCallID, array('err' => $this->result ? idx($this->result, 0) : null));
$this->profilerCallID = null;
}
}
public function __construct($cache_directory, $name)
{
$dir_cache = id(new PhutilDirectoryKeyValueCache())->setCacheDirectory($cache_directory);
$profiled_cache = id(new PhutilKeyValueCacheProfiler($dir_cache))->setProfiler(PhutilServiceProfiler::getInstance())->setName($name);
$this->cache = $profiled_cache;
}
public function isReady()
{
if (isset($this->result)) {
return true;
}
$uri = $this->getURI();
$domain = id(new PhutilURI($uri))->getDomain();
if (!$this->handle) {
$uri_object = new PhutilURI($uri);
$proxy = PhutilHTTPEngineExtension::buildHTTPProxyURI($uri_object);
$profiler = PhutilServiceProfiler::getInstance();
$this->profilerCallID = $profiler->beginServiceCall(array('type' => 'http', 'uri' => $uri, 'proxy' => (string) $proxy));
if (!self::$multi) {
self::$multi = curl_multi_init();
if (!self::$multi) {
throw new Exception(pht('%s failed!', 'curl_multi_init()'));
}
}
if (!empty(self::$pool[$domain])) {
$curl = array_pop(self::$pool[$domain]);
} else {
$curl = curl_init();
if (!$curl) {
throw new Exception(pht('%s failed!', 'curl_init()'));
}
}
$this->handle = $curl;
curl_multi_add_handle(self::$multi, $curl);
curl_setopt($curl, CURLOPT_URL, $uri);
if (defined('CURLOPT_PROTOCOLS')) {
// cURL supports a lot of protocols, and by default it will honor
// redirects across protocols (for instance, from HTTP to POP3). Beyond
// being very silly, this also has security implications:
//
// http://blog.volema.com/curl-rce.html
//
// Disable all protocols other than HTTP and HTTPS.
$allowed_protocols = CURLPROTO_HTTPS | CURLPROTO_HTTP;
curl_setopt($curl, CURLOPT_PROTOCOLS, $allowed_protocols);
curl_setopt($curl, CURLOPT_REDIR_PROTOCOLS, $allowed_protocols);
}
if (strlen($this->rawBody)) {
if ($this->getData()) {
throw new Exception(pht('You can not execute an HTTP future with both a raw request ' . 'body and structured request data.'));
}
// We aren't actually going to use this file handle, since we are
// just pushing data through the callback, but cURL gets upset if
// we don't hand it a real file handle.
$tmp = new TempFile();
$this->fileHandle = fopen($tmp, 'r');
// NOTE: We must set CURLOPT_PUT here to make cURL use CURLOPT_INFILE.
// We'll possibly overwrite the method later on, unless this is really
// a PUT request.
curl_setopt($curl, CURLOPT_PUT, true);
curl_setopt($curl, CURLOPT_INFILE, $this->fileHandle);
curl_setopt($curl, CURLOPT_INFILESIZE, strlen($this->rawBody));
curl_setopt($curl, CURLOPT_READFUNCTION, array($this, 'willWriteBody'));
} else {
$data = $this->formatRequestDataForCURL();
curl_setopt($curl, CURLOPT_POSTFIELDS, $data);
}
$headers = $this->getHeaders();
$saw_expect = false;
for ($ii = 0; $ii < count($headers); $ii++) {
list($name, $value) = $headers[$ii];
$headers[$ii] = $name . ': ' . $value;
if (!strncasecmp($name, 'Expect', strlen('Expect'))) {
$saw_expect = true;
}
}
if (!$saw_expect) {
// cURL sends an "Expect" header by default for certain requests. While
// there is some reasoning behind this, it causes a practical problem
// in that lighttpd servers reject these requests with a 417. Both sides
// are locked in an eternal struggle (lighttpd has introduced a
// 'server.reject-expect-100-with-417' option to deal with this case).
//
// The ostensibly correct way to suppress this behavior on the cURL side
// is to add an empty "Expect:" header. If we haven't seen some other
// explicit "Expect:" header, do so.
//
// See here, for example, although this issue is fairly widespread:
// http://curl.haxx.se/mail/archive-2009-07/0008.html
$headers[] = 'Expect:';
}
curl_setopt($curl, CURLOPT_HTTPHEADER, $headers);
// Set the requested HTTP method, e.g. GET / POST / PUT.
curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $this->getMethod());
// Make sure we get the headers and data back.
curl_setopt($curl, CURLOPT_HEADER, true);
curl_setopt($curl, CURLOPT_WRITEFUNCTION, array($this, 'didReceiveDataCallback'));
if ($this->followLocation) {
curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($curl, CURLOPT_MAXREDIRS, 20);
}
if (defined('CURLOPT_TIMEOUT_MS')) {
// If CURLOPT_TIMEOUT_MS is available, use the higher-precision timeout.
$timeout = max(1, ceil(1000 * $this->getTimeout()));
curl_setopt($curl, CURLOPT_TIMEOUT_MS, $timeout);
} else {
// Otherwise, fall back to the lower-precision timeout.
$timeout = max(1, ceil($this->getTimeout()));
curl_setopt($curl, CURLOPT_TIMEOUT, $timeout);
}
// We're going to try to set CAINFO below. This doesn't work at all on
// OSX around Yosemite (see T5913). On these systems, we'll use the
// system CA and then try to tell the user that their settings were
// ignored and how to fix things if we encounter a CA-related error.
// Assume we have custom CA settings to start with; we'll clear this
// flag if we read the default CA info below.
// Try some decent fallbacks here:
// - First, check if a bundle is set explicitly for this request, via
// `setCABundle()` or similar.
// - Then, check if a global bundle is set explicitly for all requests,
// via `setGlobalCABundle()` or similar.
// - Then, if a local custom.pem exists, use that, because it probably
// means that the user wants to override everything (also because the
// user might not have access to change the box's php.ini to add
// curl.cainfo).
// - Otherwise, try using curl.cainfo. If it's set explicitly, it's
// probably reasonable to try using it before we fall back to what
// libphutil ships with.
// - Lastly, try the default that libphutil ships with. If it doesn't
// work, give up and yell at the user.
if (!$this->getCABundle()) {
$caroot = dirname(phutil_get_library_root('phutil')) . '/resources/ssl/';
$ini_val = ini_get('curl.cainfo');
if (self::getGlobalCABundle()) {
$this->setCABundleFromPath(self::getGlobalCABundle());
} else {
if (Filesystem::pathExists($caroot . 'custom.pem')) {
$this->setCABundleFromPath($caroot . 'custom.pem');
} else {
if ($ini_val) {
// TODO: We can probably do a pathExists() here, even.
$this->setCABundleFromPath($ini_val);
} else {
$this->setCABundleFromPath($caroot . 'default.pem');
}
}
}
}
if ($this->canSetCAInfo()) {
curl_setopt($curl, CURLOPT_CAINFO, $this->getCABundle());
}
$verify_peer = 1;
$verify_host = 2;
$extensions = PhutilHTTPEngineExtension::getAllExtensions();
foreach ($extensions as $extension) {
if ($extension->shouldTrustAnySSLAuthorityForURI($uri_object)) {
$verify_peer = 0;
}
if ($extension->shouldTrustAnySSLHostnameForURI($uri_object)) {
$verify_host = 0;
}
}
curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, $verify_peer);
curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, $verify_host);
curl_setopt($curl, CURLOPT_SSLVERSION, 0);
if ($proxy) {
curl_setopt($curl, CURLOPT_PROXY, (string) $proxy);
}
} else {
$curl = $this->handle;
if (!self::$results) {
// NOTE: In curl_multi_select(), PHP calls curl_multi_fdset() but does
// not check the return value of &maxfd for -1 until recent versions
// of PHP (5.4.8 and newer). cURL may return -1 as maxfd in some unusual
// situations; if it does, PHP enters select() with nfds=0, which blocks
// until the timeout is reached.
//
// We could try to guess whether this will happen or not by examining
// the version identifier, but we can also just sleep for only a short
// period of time.
curl_multi_select(self::$multi, 0.01);
}
}
do {
$active = null;
$result = curl_multi_exec(self::$multi, $active);
} while ($result == CURLM_CALL_MULTI_PERFORM);
while ($info = curl_multi_info_read(self::$multi)) {
if ($info['msg'] == CURLMSG_DONE) {
self::$results[(int) $info['handle']] = $info;
}
}
if (!array_key_exists((int) $curl, self::$results)) {
return false;
}
// The request is complete, so release any temporary files we wrote
// earlier.
$this->temporaryFiles = array();
$info = self::$results[(int) $curl];
$result = $this->responseBuffer;
$err_code = $info['result'];
if ($err_code) {
if ($err_code == CURLE_SSL_CACERT && !$this->canSetCAInfo()) {
$status = new HTTPFutureCertificateResponseStatus(HTTPFutureCertificateResponseStatus::ERROR_IMMUTABLE_CERTIFICATES, $uri);
} else {
$status = new HTTPFutureCURLResponseStatus($err_code, $uri);
}
$body = null;
$headers = array();
$this->result = array($status, $body, $headers);
} else {
// cURL returns headers of all redirects, we strip all but the final one.
$redirects = curl_getinfo($curl, CURLINFO_REDIRECT_COUNT);
$result = preg_replace('/^(.*\\r\\n\\r\\n){' . $redirects . '}/sU', '', $result);
$this->result = $this->parseRawHTTPResponse($result);
}
curl_multi_remove_handle(self::$multi, $curl);
unset(self::$results[(int) $curl]);
// NOTE: We want to use keepalive if possible. Return the handle to a
// pool for the domain; don't close it.
if ($this->shouldReuseHandles()) {
self::$pool[$domain][] = $curl;
}
$profiler = PhutilServiceProfiler::getInstance();
$profiler->endServiceCall($this->profilerCallID, array());
return true;
}
private function bindLDAP($conn, $user, PhutilOpaqueEnvelope $pass)
{
$profiler = PhutilServiceProfiler::getInstance();
$call_id = $profiler->beginServiceCall(array('type' => 'ldap', 'call' => 'bind', 'user' => $user));
// NOTE: ldap_bind() dumps cleartext passwords into logs by default. Keep
// it quiet.
if (strlen($user)) {
$ok = @ldap_bind($conn, $user, $pass->openEnvelope());
} else {
$ok = @ldap_bind($conn);
}
$profiler->endServiceCall($call_id, array());
if (!$ok) {
if (strlen($user)) {
$this->raiseConnectionException($conn, pht('Failed to bind to LDAP server (as user "%s").', $user));
} else {
$this->raiseConnectionException($conn, pht('Failed to bind to LDAP server (without username).'));
}
}
}
public function callMethod($method, array $params)
{
$meta = array();
if ($this->sessionKey) {
$meta['sessionKey'] = $this->sessionKey;
}
if ($this->connectionID) {
$meta['connectionID'] = $this->connectionID;
}
if ($method == 'conduit.connect') {
$certificate = idx($params, 'certificate');
if ($certificate) {
$token = time();
$params['authToken'] = $token;
$params['authSignature'] = sha1($token . $certificate);
}
unset($params['certificate']);
}
if ($meta) {
$params['__conduit__'] = $meta;
}
$port = null;
if ($this->port) {
$port = ':' . $this->port;
}
$uri = $this->protocol . '://' . $this->host . $port . '/' . $this->path . $method;
$data = array('params' => json_encode($params), 'output' => 'json');
if ($this->protocol == 'https') {
$core_future = new HTTPSFuture($uri, $data);
} else {
$core_future = new HTTPFuture($uri, $data);
}
$core_future->setMethod('POST');
$core_future->setTimeout($this->timeout);
$profiler = PhutilServiceProfiler::getInstance();
$this->profilerCallID = $profiler->beginServiceCall(array('type' => 'conduit', 'method' => $method));
$conduit_future = new ConduitFuture($core_future);
$conduit_future->setClient($this, $method);
$conduit_future->isReady();
return $conduit_future;
}
private function endLintServiceCall($call_id)
{
$profiler = PhutilServiceProfiler::getInstance();
$profiler->endServiceCall($call_id, array());
}
public function executeRawQuery($raw_query)
{
$this->lastResult = null;
$retries = max(1, PhabricatorEnv::getEnvConfig('mysql.connection-retries'));
while ($retries--) {
try {
$this->requireConnection();
// TODO: Do we need to include transactional statements here?
$is_write = !preg_match('/^(SELECT|SHOW|EXPLAIN)\\s/', $raw_query);
if ($is_write) {
AphrontWriteGuard::willWrite();
}
$start = microtime(true);
$profiler = PhutilServiceProfiler::getInstance();
$call_id = $profiler->beginServiceCall(array('type' => 'query', 'config' => $this->configuration, 'query' => $raw_query, 'write' => $is_write));
$result = $this->rawQuery($raw_query);
$profiler->endServiceCall($call_id, array());
if ($this->nextError) {
$result = null;
}
if ($result) {
$this->lastResult = $result;
break;
}
$this->throwQueryException($this->connection);
} catch (AphrontQueryConnectionLostException $ex) {
if ($this->isInsideTransaction()) {
// Zero out the transaction state to prevent a second exception
// ("program exited with open transaction") from being thrown, since
// we're about to throw a more relevant/useful one instead.
$state = $this->getTransactionState();
while ($state->getDepth()) {
$state->decreaseDepth();
}
// We can't close the connection before this because
// isInsideTransaction() and getTransactionState() depend on the
// connection.
$this->closeConnection();
throw $ex;
}
$this->closeConnection();
if (!$retries) {
throw $ex;
}
$class = get_class($ex);
$message = $ex->getMessage();
phlog("Retrying ({$retries}) after {$class}: {$message}");
}
}
}
public static function installEchoListener()
{
$instance = PhutilServiceProfiler::getInstance();
$instance->addListener(array('PhutilServiceProfiler', 'echoListener'));
}
private function checkSocket()
{
$timeout = false;
$now = microtime(true);
if ($now - $this->stateStartTime > $this->getTimeout()) {
$timeout = true;
}
if (!feof($this->socket) && !$timeout) {
return false;
}
$this->stateReady = true;
if ($timeout) {
$this->result = $this->buildErrorResult(HTTPFutureResponseStatusTransport::ERROR_TIMEOUT);
} else {
if (!$this->stateConnected) {
$this->result = $this->buildErrorResult(HTTPFutureResponseStatusTransport::ERROR_CONNECTION_REFUSED);
} else {
if (!$this->stateWriteComplete) {
$this->result = $this->buildErrorResult(HTTPFutureResponseStatusTransport::ERROR_CONNECTION_FAILED);
} else {
$this->result = $this->parseRawHTTPResponse($this->response);
}
}
}
$profiler = PhutilServiceProfiler::getInstance();
$profiler->endServiceCall($this->profilerCallID, array());
return true;
}
protected final function didRunLinters(array $linters)
{
assert_instances_of($linters, 'ArcanistLinter');
$exceptions = array();
$profiler = PhutilServiceProfiler::getInstance();
foreach ($linters as $linter_name => $linter) {
if (!is_string($linter_name)) {
$linter_name = get_class($linter);
}
$call_id = $profiler->beginServiceCall(array('type' => 'lint', 'linter' => $linter_name));
try {
$linter->didRunLinters();
} catch (Exception $ex) {
$exceptions[$linter_name] = $ex;
}
$profiler->endServiceCall($call_id, array());
}
return $exceptions;
}
private static function addProfilerToCaches(array $caches)
{
foreach ($caches as $key => $cache) {
$pcache = new PhutilKeyValueCacheProfiler($cache);
$pcache->setProfiler(PhutilServiceProfiler::getInstance());
$caches[$key] = $pcache;
}
return $caches;
}
public function executeRawQueries(array $raw_queries)
{
if (!$raw_queries) {
return array();
}
$is_write = false;
foreach ($raw_queries as $key => $raw_query) {
$is_write = $is_write || $this->checkWrite($raw_query);
$raw_queries[$key] = rtrim($raw_query, "\r\n\t ;");
}
$profiler = PhutilServiceProfiler::getInstance();
$call_id = $profiler->beginServiceCall(array('type' => 'multi-query', 'config' => $this->configuration, 'queries' => $raw_queries, 'write' => $is_write));
$results = $this->rawQueries($raw_queries);
$profiler->endServiceCall($call_id, array());
return $results;
}
public function isReady()
{
if (isset($this->result)) {
return true;
}
$uri = $this->getURI();
$domain = id(new PhutilURI($uri))->getDomain();
if (!$this->handle) {
$profiler = PhutilServiceProfiler::getInstance();
$this->profilerCallID = $profiler->beginServiceCall(array('type' => 'http', 'uri' => $uri));
if (!self::$multi) {
self::$multi = curl_multi_init();
if (!self::$multi) {
throw new Exception('curl_multi_init() failed!');
}
}
if (!empty(self::$pool[$domain])) {
$curl = array_pop(self::$pool[$domain]);
} else {
$curl = curl_init();
if (!$curl) {
throw new Exception('curl_init() failed!');
}
}
$this->handle = $curl;
curl_multi_add_handle(self::$multi, $curl);
curl_setopt($curl, CURLOPT_URL, $uri);
if (defined('CURLOPT_PROTOCOLS')) {
// cURL supports a lot of protocols, and by default it will honor
// redirects across protocols (for instance, from HTTP to POP3). Beyond
// being very silly, this also has security implications:
//
// http://blog.volema.com/curl-rce.html
//
// Disable all protocols other than HTTP and HTTPS.
$allowed_protocols = CURLPROTO_HTTPS | CURLPROTO_HTTP;
curl_setopt($curl, CURLOPT_PROTOCOLS, $allowed_protocols);
curl_setopt($curl, CURLOPT_REDIR_PROTOCOLS, $allowed_protocols);
}
$data = $this->getData();
if ($data) {
// NOTE: PHP's cURL implementation has a piece of magic which treats
// parameters as file paths if they begin with '@'. This means that
// an array like "array('name' => '@/usr/local/secret')" will attempt to
// read that file off disk and send it to the remote server. This
// behavior is pretty surprising, and it can easily become a relatively
// severe security vulnerability which allows an attacker to read any
// file the HTTP process has access to. Since this feature is very
// dangerous and not particularly useful, we prevent its use.
//
// After PHP 5.2.0, it is sufficient to pass a string to avoid this
// "feature" (it is only invoked in the array version). Prior to
// PHP 5.2.0, we block any request which have string data beginning with
// '@' (they would not work anyway).
if (is_array($data)) {
// Explicitly build a query string to prevent "@" security problems.
$data = http_build_query($data, '', '&');
}
if ($data[0] == '@' && version_compare(phpversion(), '5.2.0', '<')) {
throw new Exception("Attempting to make an HTTP request including string data that " . "begins with '@'. Prior to PHP 5.2.0, this reads files off disk, " . "which creates a wide attack window for security vulnerabilities. " . "Upgrade PHP or avoid making cURL requests which begin with '@'.");
}
curl_setopt($curl, CURLOPT_POSTFIELDS, $data);
} else {
curl_setopt($curl, CURLOPT_POSTFIELDS, null);
}
$headers = $this->getHeaders();
$saw_expect = false;
for ($ii = 0; $ii < count($headers); $ii++) {
list($name, $value) = $headers[$ii];
$headers[$ii] = $name . ': ' . $value;
if (!strncasecmp($name, 'Expect', strlen('Expect'))) {
$saw_expect = true;
}
}
if (!$saw_expect) {
// cURL sends an "Expect" header by default for certain requests. While
// there is some reasoning behind this, it causes a practical problem
// in that lighttpd servers reject these requests with a 417. Both sides
// are locked in an eternal struggle (lighttpd has introduced a
// 'server.reject-expect-100-with-417' option to deal with this case).
//
// The ostensibly correct way to suppress this behavior on the cURL side
// is to add an empty "Expect:" header. If we haven't seen some other
// explicit "Expect:" header, do so.
//
// See here, for example, although this issue is fairly widespread:
// http://curl.haxx.se/mail/archive-2009-07/0008.html
$headers[] = 'Expect:';
}
curl_setopt($curl, CURLOPT_HTTPHEADER, $headers);
// Set the requested HTTP method, e.g. GET / POST / PUT.
curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $this->getMethod());
// Make sure we get the headers and data back.
curl_setopt($curl, CURLOPT_HEADER, true);
curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($curl, CURLOPT_MAXREDIRS, 20);
if (defined('CURLOPT_TIMEOUT_MS')) {
// If CURLOPT_TIMEOUT_MS is available, use the higher-precision timeout.
$timeout = max(1, ceil(1000 * $this->getTimeout()));
curl_setopt($curl, CURLOPT_TIMEOUT_MS, $timeout);
} else {
// Otherwise, fall back to the lower-precision timeout.
$timeout = max(1, ceil($this->getTimeout()));
curl_setopt($curl, CURLOPT_TIMEOUT, $timeout);
}
// Try some decent fallbacks here:
// - First, check if a bundle is set explicit for this request, via
// `setCABundle()` or similar.
// - Then, check if a global bundle is set explicitly for all requests,
// via `setGlobalCABundle()` or similar.
// - Then, if a local custom.pem exists, use that, because it probably
// means that the user wants to override everything (also because the
// user might not have access to change the box's php.ini to add
// curl.cainfo).
// - Otherwise, try using curl.cainfo. If it's set explicitly, it's
// probably reasonable to try using it before we fall back to what
// libphutil ships with.
// - Lastly, try the default that libphutil ships with. If it doesn't
// work, give up and yell at the user.
if (!$this->getCABundle()) {
$caroot = dirname(phutil_get_library_root('phutil')) . '/resources/ssl/';
$ini_val = ini_get('curl.cainfo');
if (self::getGlobalCABundle()) {
$this->setCABundleFromPath(self::getGlobalCABundle());
} else {
if (Filesystem::pathExists($caroot . 'custom.pem')) {
$this->setCABundleFromPath($caroot . 'custom.pem');
} else {
if ($ini_val) {
// TODO: We can probably do a pathExists() here, even.
$this->setCABundleFromPath($ini_val);
} else {
$this->setCABundleFromPath($caroot . 'default.pem');
}
}
}
}
curl_setopt($curl, CURLOPT_CAINFO, $this->getCABundle());
curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, true);
curl_setopt($curl, CURLOPT_SSLVERSION, 0);
} else {
$curl = $this->handle;
if (!self::$results) {
// NOTE: In curl_multi_select(), PHP calls curl_multi_fdset() but does
// not check the return value of &maxfd for -1 until recent versions
// of PHP (5.4.8 and newer). cURL may return -1 as maxfd in some unusual
// situations; if it does, PHP enters select() with nfds=0, which blocks
// until the timeout is reached.
//
// We could try to guess whether this will happen or not by examining
// the version identifier, but we can also just sleep for only a short
// period of time.
curl_multi_select(self::$multi, 0.01);
}
}
do {
$active = null;
$result = curl_multi_exec(self::$multi, $active);
} while ($result == CURLM_CALL_MULTI_PERFORM);
while ($info = curl_multi_info_read(self::$multi)) {
if ($info['msg'] == CURLMSG_DONE) {
self::$results[(int) $info['handle']] = $info;
}
}
if (!array_key_exists((int) $curl, self::$results)) {
return false;
}
$info = self::$results[(int) $curl];
$result = curl_multi_getcontent($curl);
$err_code = $info['result'];
if ($err_code) {
$status = new Status_HTTPFutureResponseStatusCURL($err_code, $uri);
$body = null;
$headers = array();
$this->result = array($status, $body, $headers);
} else {
// cURL returns headers of all redirects, we strip all but the final one.
$redirects = curl_getinfo($curl, CURLINFO_REDIRECT_COUNT);
$result = preg_replace('/^(.*\\r\\n\\r\\n){' . $redirects . '}/sU', '', $result);
$this->result = $this->parseRawHTTPResponse($result);
}
curl_multi_remove_handle(self::$multi, $curl);
unset(self::$results[(int) $curl]);
// NOTE: We want to use keepalive if possible. Return the handle to a
// pool for the domain; don't close it.
self::$pool[$domain][] = $curl;
$profiler = PhutilServiceProfiler::getInstance();
$profiler->endServiceCall($this->profilerCallID, array());
return true;
}
