protected function didReceiveResult($result) { if ($this->profilerCallID !== null) { $profiler = PhutilServiceProfiler::getInstance(); $profiler->endServiceCall($this->profilerCallID, array()); } list($status, $body, $headers) = $result; if ($status->isError()) { throw $status; } $raw = $body; $shield = 'for(;;);'; if (!strncmp($raw, $shield, strlen($shield))) { $raw = substr($raw, strlen($shield)); } $data = json_decode($raw, true); if (!is_array($data)) { throw new Exception("Host returned HTTP/200, but invalid JSON data in response to " . "a Conduit method call:\n{$raw}"); } if ($data['error_code']) { throw new ConduitClientException($data['error_code'], $data['error_info']); } $result = $data['result']; $result = $this->client->didReceiveResponse($this->conduitMethod, $result); return $result; }
/** * Execute this command. * * @return int Error code returned by the subprocess. * * @task command */ public function execute() { $command = $this->command; $profiler = PhutilServiceProfiler::getInstance(); $call_id = $profiler->beginServiceCall(array('type' => 'exec', 'subtype' => 'passthru', 'command' => $command)); $spec = array(STDIN, STDOUT, STDERR); $pipes = array(); if ($command instanceof PhutilCommandString) { $unmasked_command = $command->getUnmaskedString(); } else { $unmasked_command = $command; } $env = $this->env; $cwd = $this->cwd; $options = array(); if (phutil_is_windows()) { // Without 'bypass_shell', things like launching vim don't work properly, // and we can't execute commands with spaces in them, and all commands // invoked from git bash fail horridly, and everything is a mess in // general. $options['bypass_shell'] = true; } $trap = new PhutilErrorTrap(); $proc = @proc_open($unmasked_command, $spec, $pipes, $cwd, $env, $options); $errors = $trap->getErrorsAsString(); $trap->destroy(); if (!is_resource($proc)) { throw new Exception(pht('Failed to passthru %s: %s', 'proc_open()', $errors)); } $err = proc_close($proc); $profiler->endServiceCall($call_id, array('err' => $err)); return $err; }
protected function didReceiveResult($result) { if ($this->profilerCallID !== null) { $profiler = PhutilServiceProfiler::getInstance(); $profiler->endServiceCall($this->profilerCallID, array()); } list($status, $body, $headers) = $result; if ($status->isError()) { throw $status; } $raw = $body; $shield = 'for(;;);'; if (!strncmp($raw, $shield, strlen($shield))) { $raw = substr($raw, strlen($shield)); } $data = null; try { $data = phutil_json_decode($raw); } catch (PhutilJSONParserException $ex) { throw new PhutilProxyException(pht('Host returned HTTP/200, but invalid JSON data in response to ' . 'a Conduit method call.'), $ex); } if ($data['error_code']) { throw new ConduitClientException($data['error_code'], $data['error_info']); } $result = $data['result']; $result = $this->client->didReceiveResponse($this->conduitMethod, $result); return $result; }
/** * Execute a command which takes over stdin, stdout and stderr, similar to * passthru(), but which preserves TTY semantics, escapes arguments, and is * traceable. * * @param string sprintf()-style command pattern to execute. * @param ... Arguments to sprintf pattern. * @return int Return code. * @group exec */ function phutil_passthru($cmd) { $args = func_get_args(); $command = call_user_func_array('csprintf', $args); $profiler = PhutilServiceProfiler::getInstance(); $call_id = $profiler->beginServiceCall(array('type' => 'exec', 'subtype' => 'passthru', 'command' => $command)); $spec = array(STDIN, STDOUT, STDERR); $pipes = array(); if (phutil_is_windows()) { // Without 'bypass_shell', things like launching vim don't work properly, // and we can't execute commands with spaces in them, and all commands // invoked from git bash fail horridly, and everything is a mess in general. $options = array('bypass_shell' => true); $proc = @proc_open($command, $spec, $pipes, null, null, $options); } else { $proc = @proc_open($command, $spec, $pipes); } if ($proc === false) { $err = 1; } else { $err = proc_close($proc); } $profiler->endServiceCall($call_id, array('err' => $err)); return $err; }
public static function getInstance() { if (empty(self::$instance)) { self::$instance = new PhutilServiceProfiler(); } return self::$instance; }
public static function dispatchEvent(PhutilEvent $event) { $instance = self::getInstance(); $listeners = idx($instance->listeners, $event->getType(), array()); $global_listeners = idx($instance->listeners, PhutilEventType::TYPE_ALL, array()); // Merge and deduplicate listeners (we want to send the event to each // listener only once, even if it satisfies multiple criteria for the // event). $listeners = array_merge($listeners, $global_listeners); $listeners = mpull($listeners, null, 'getListenerID'); $profiler = PhutilServiceProfiler::getInstance(); $profiler_id = $profiler->beginServiceCall(array('type' => 'event', 'kind' => $event->getType(), 'count' => count($listeners))); $caught = null; try { foreach ($listeners as $listener) { if ($event->isStopped()) { // Do this first so if someone tries to dispatch a stopped event it // doesn't go anywhere. Silly but less surprising. break; } $listener->handleEvent($event); } } catch (Exception $ex) { $profiler->endServiceCall($profiler_id, array()); throw $ex; } $profiler->endServiceCall($profiler_id, array()); }
/** * Delete a blob from Amazon S3. */ public function deleteFile($handle) { AphrontWriteGuard::willWrite(); $s3 = $this->newS3API(); $profiler = PhutilServiceProfiler::getInstance(); $call_id = $profiler->beginServiceCall(array('type' => 's3', 'method' => 'deleteObject')); $s3->deleteObject($this->getBucketName(), $handle); $profiler->endServiceCall($call_id, array()); }
/** * Delete a blob from Amazon S3. */ public function deleteFile($handle) { $s3 = $this->newS3API(); AphrontWriteGuard::willWrite(); $profiler = PhutilServiceProfiler::getInstance(); $call_id = $profiler->beginServiceCall(array('type' => 's3', 'method' => 'deleteObject')); $s3->setParametersForDeleteObject($handle)->resolve(); $profiler->endServiceCall($call_id, array()); }
/** * Execute a command which takes over stdin, stdout and stderr, similar to * passthru(), but which preserves TTY semantics, escapes arguments, and is * traceable. * * @param string sprintf()-style command pattern to execute. * @param ... Arguments to sprintf pattern. * @return int Return code. * @group exec */ function phutil_passthru($cmd) { $args = func_get_args(); $command = call_user_func_array('csprintf', $args); $profiler = PhutilServiceProfiler::getInstance(); $call_id = $profiler->beginServiceCall(array('type' => 'exec', 'subtype' => 'passthru', 'command' => $command)); $pipes = array(); $proc = proc_open($command, array(STDIN, STDOUT, STDERR), $pipes); $err = proc_close($proc); $profiler->endServiceCall($call_id, array('err' => $err)); return $err; }
public function execute() { $profiler = PhutilServiceProfiler::getInstance(); $call_id = $profiler->beginServiceCall(array('type' => 'conduit', 'method' => $this->method)); try { $result = $this->executeMethod(); } catch (Exception $ex) { $profiler->endServiceCall($call_id, array()); throw $ex; } $profiler->endServiceCall($call_id, array()); return $result; }
public static function initializeScriptEnvironment() { self::initializeCommonEnvironment(); // NOTE: This is dangerous in general, but we know we're in a script context // and are not vulnerable to CSRF. AphrontWriteGuard::allowDangerousUnguardedWrites(true); // There are several places where we log information (about errors, events, // service calls, etc.) for analysis via DarkConsole or similar. These are // useful for web requests, but grow unboundedly in long-running scripts and // daemons. Discard data as it arrives in these cases. PhutilServiceProfiler::getInstance()->enableDiscardMode(); DarkConsoleErrorLogPluginAPI::enableDiscardMode(); DarkConsoleEventPluginAPI::enableDiscardMode(); }
private function doCacheTest(PhutilKeyValueCache $cache) { $key1 = 'test:' . mt_rand(); $key2 = 'test:' . mt_rand(); $default = 'cache-miss'; $value1 = 'cache-hit1'; $value2 = 'cache-hit2'; $cache->setProfiler(PhutilServiceProfiler::getInstance()); $test_info = get_class($cache); // Test that we miss correctly on missing values. $this->assertEqual($default, $cache->getKey($key1, $default), $test_info); $this->assertEqual(array(), $cache->getKeys(array($key1, $key2)), $test_info); // Test that we can set individual keys. $cache->setKey($key1, $value1); $this->assertEqual($value1, $cache->getKey($key1, $default), $test_info); $this->assertEqual(array($key1 => $value1), $cache->getKeys(array($key1, $key2)), $test_info); // Test that we can delete individual keys. $cache->deleteKey($key1); $this->assertEqual($default, $cache->getKey($key1, $default), $test_info); $this->assertEqual(array(), $cache->getKeys(array($key1, $key2)), $test_info); // Test that we can set multiple keys. $cache->setKeys(array($key1 => $value1, $key2 => $value2)); $this->assertEqual($value1, $cache->getKey($key1, $default), $test_info); $this->assertEqual(array($key1 => $value1, $key2 => $value2), $cache->getKeys(array($key1, $key2)), $test_info); // Test that we can delete multiple keys. $cache->deleteKeys(array($key1, $key2)); $this->assertEqual($default, $cache->getKey($key1, $default), $test_info); $this->assertEqual(array(), $cache->getKeys(array($key1, $key2)), $test_info); // NOTE: The TTL tests are necessarily slow (we must sleep() through the // TTLs) and do not work with APC (it does not TTL until the next request) // so they're disabled by default. If you're developing the cache stack, // it may be useful to run them. return; // Test that keys expire when they TTL. $cache->setKey($key1, $value1, 1); $cache->setKey($key2, $value2, 5); $this->assertEqual($value1, $cache->getKey($key1, $default)); $this->assertEqual($value2, $cache->getKey($key2, $default)); sleep(2); $this->assertEqual($default, $cache->getKey($key1, $default)); $this->assertEqual($value2, $cache->getKey($key2, $default)); // Test that setting a 0 TTL overwrites a nonzero TTL. $cache->setKey($key1, $value1, 1); $this->assertEqual($value1, $cache->getKey($key1, $default)); $cache->setKey($key1, $value1, 0); $this->assertEqual($value1, $cache->getKey($key1, $default)); sleep(2); $this->assertEqual($value1, $cache->getKey($key1, $default)); }
public function processRequest() { $request = $this->getRequest(); $user = $request->getUser(); $cache = new PhabricatorKeyValueDatabaseCache(); $cache = new PhutilKeyValueCacheProfiler($cache); $cache->setProfiler(PhutilServiceProfiler::getInstance()); $result = $cache->getKey('darkconsole:' . $this->key); if (!$result) { return new Aphront400Response(); } try { $result = phutil_json_decode($result); } catch (PhutilJSONParserException $ex) { return new Aphront400Response(); } if ($result['vers'] != DarkConsoleCore::STORAGE_VERSION) { return new Aphront400Response(); } if ($result['user'] != $user->getPHID()) { return new Aphront400Response(); } $output = array(); $output['tabs'] = $result['tabs']; $output['panel'] = array(); foreach ($result['data'] as $class => $data) { try { $obj = newv($class, array()); $obj->setData($data); $obj->setRequest($request); $panel = $obj->renderPanel(); // Because cookie names can now be prefixed, wipe out any cookie value // with the session cookie name anywhere in its name. $pattern = '(' . preg_quote(PhabricatorCookies::COOKIE_SESSION) . ')'; foreach ($_COOKIE as $cookie_name => $cookie_value) { if (preg_match($pattern, $cookie_name)) { $panel = PhutilSafeHTML::applyFunction('str_replace', $cookie_value, '(session-key)', $panel); } } $output['panel'][$class] = $panel; } catch (Exception $ex) { $output['panel'][$class] = 'error'; } } return id(new AphrontAjaxResponse())->setContent($output); }
public function getKey(AphrontRequest $request) { $plugins = $this->getPlugins(); foreach ($plugins as $plugin) { $plugin->setRequest($request); $plugin->willShutdown(); } foreach ($plugins as $plugin) { $plugin->didShutdown(); } foreach ($plugins as $plugin) { $plugin->setData($plugin->generateData()); } $plugins = msort($plugins, 'getOrderKey'); $key = Filesystem::readRandomCharacters(24); $tabs = array(); $data = array(); foreach ($plugins as $plugin) { $class = get_class($plugin); $tabs[] = array('class' => $class, 'name' => $plugin->getName(), 'color' => $plugin->getColor()); $data[$class] = $this->sanitizeForJSON($plugin->getData()); } $storage = array('vers' => self::STORAGE_VERSION, 'tabs' => $tabs, 'data' => $data, 'user' => $request->getUser() ? $request->getUser()->getPHID() : null); $cache = new PhabricatorKeyValueDatabaseCache(); $cache = new PhutilKeyValueCacheProfiler($cache); $cache->setProfiler(PhutilServiceProfiler::getInstance()); // This encoding may fail if there are, e.g., database queries which // include binary data. It would be a little cleaner to try to strip these, // but just do something non-broken here if we end up with unrepresentable // data. $json = @json_encode($storage); if (!$json) { $json = '{}'; } $cache->setKeys(array('darkconsole:' . $key => $json), $ttl = 60 * 60 * 6); return $key; }
} echo phutil_console_wrap(phutil_console_format('This script will test that you have configured valid credentials for ' . 'access to a repository, so the Phabricator daemons can pull from it. ' . 'You should run this as the **same user you will run the daemons as**, ' . 'from the **same machine they will run from**. Doing this will help ' . 'detect various problems with your configuration, such as SSH issues.')); list($whoami) = execx('whoami'); $whoami = trim($whoami); $ok = phutil_console_confirm("Do you want to continue as '{$whoami}'?"); if (!$ok) { die(1); } $callsign = $argv[1]; echo "Loading '{$callsign}' repository...\n"; $repository = id(new PhabricatorRepository())->loadOneWhere('callsign = %s', $argv[1]); if (!$repository) { throw new Exception("No such repository exists!"); } $vcs = $repository->getVersionControlSystem(); PhutilServiceProfiler::installEchoListener(); echo "Trying to connect to the remote...\n"; switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: $err = $repository->passthruRemoteCommand('--limit 1 log %s', $repository->getRemoteURI()); break; case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: // Do an ls-remote on a nonexistent ref, which we expect to just return // nothing. $err = $repository->passthruRemoteCommand('ls-remote %s %s', $repository->getRemoteURI(), 'just-testing'); break; default: throw new Exception("Unsupported repository type."); } if ($err) { echo phutil_console_format("<bg:red>** FAIL **</bg> Connection failed. The credentials for this " . "repository appear to be incorrectly configured.\n");
public function __construct(array $argv) { PhutilServiceProfiler::getInstance()->enableDiscardMode(); $original_argv = $argv; $args = new PhutilArgumentParser($argv); $args->setTagline('daemon overseer'); $args->setSynopsis(<<<EOHELP **launch_daemon.php** [__options__] __daemon__ Launch and oversee an instance of __daemon__. EOHELP ); $args->parseStandardArguments(); $args->parsePartial(array(array('name' => 'trace-memory', 'help' => 'Enable debug memory tracing.'), array('name' => 'log', 'param' => 'file', 'help' => 'Send output to __file__.'), array('name' => 'daemonize', 'help' => 'Run in the background.'), array('name' => 'phd', 'param' => 'dir', 'help' => 'Write PID information to __dir__.'), array('name' => 'verbose', 'help' => 'Enable verbose activity logging.'), array('name' => 'load-phutil-library', 'param' => 'library', 'repeat' => true, 'help' => 'Load __library__.'))); $argv = array(); $more = $args->getUnconsumedArgumentVector(); $this->daemon = array_shift($more); if (!$this->daemon) { $args->printHelpAndExit(); } if ($args->getArg('trace')) { $this->traceMode = true; $argv[] = '--trace'; } if ($args->getArg('trace-memory')) { $this->traceMode = true; $this->traceMemory = true; $argv[] = '--trace-memory'; } if ($args->getArg('load-phutil-library')) { foreach ($args->getArg('load-phutil-library') as $library) { $argv[] = '--load-phutil-library=' . $library; } } $log = $args->getArg('log'); if ($log) { ini_set('error_log', $log); $argv[] = '--log=' . $log; } $verbose = $args->getArg('verbose'); if ($verbose) { $this->verbose = true; $argv[] = '--verbose'; } $this->daemonize = $args->getArg('daemonize'); $this->phddir = $args->getArg('phd'); $this->argv = $argv; $this->moreArgs = coalesce($more, array()); error_log("Bringing daemon '{$this->daemon}' online..."); if (self::$instance) { throw new Exception('You may not instantiate more than one Overseer per process.'); } self::$instance = $this; if ($this->daemonize) { // We need to get rid of these or the daemon will hang when we TERM it // waiting for something to read the buffers. TODO: Learn how unix works. fclose(STDOUT); fclose(STDERR); ob_start(); $pid = pcntl_fork(); if ($pid === -1) { throw new Exception('Unable to fork!'); } else { if ($pid) { exit(0); } } } if ($this->phddir) { $desc = array('name' => $this->daemon, 'argv' => $this->moreArgs, 'pid' => getmypid(), 'start' => time()); Filesystem::writeFile($this->phddir . '/daemon.' . getmypid(), json_encode($desc)); } $this->daemonID = $this->generateDaemonID(); $this->dispatchEvent(self::EVENT_DID_LAUNCH, array('argv' => array_slice($original_argv, 1), 'explicitArgv' => $this->moreArgs)); declare (ticks=1); pcntl_signal(SIGUSR1, array($this, 'didReceiveKeepaliveSignal')); pcntl_signal(SIGUSR2, array($this, 'didReceiveNotifySignal')); pcntl_signal(SIGINT, array($this, 'didReceiveGracefulSignal')); pcntl_signal(SIGTERM, array($this, 'didReceiveTerminalSignal')); }
public function __construct(array $argv) { PhutilServiceProfiler::getInstance()->enableDiscardMode(); $args = new PhutilArgumentParser($argv); $args->setTagline(pht('daemon overseer')); $args->setSynopsis(<<<EOHELP **launch_daemon.php** [__options__] __daemon__ Launch and oversee an instance of __daemon__. EOHELP ); $args->parseStandardArguments(); $args->parse(array(array('name' => 'trace-memory', 'help' => pht('Enable debug memory tracing.')), array('name' => 'verbose', 'help' => pht('Enable verbose activity logging.')), array('name' => 'label', 'short' => 'l', 'param' => 'label', 'help' => pht('Optional process label. Makes "%s" nicer, no behavioral effects.', 'ps')))); $argv = array(); if ($args->getArg('trace')) { $this->traceMode = true; $argv[] = '--trace'; } if ($args->getArg('trace-memory')) { $this->traceMode = true; $this->traceMemory = true; $argv[] = '--trace-memory'; } $verbose = $args->getArg('verbose'); if ($verbose) { $this->verbose = true; $argv[] = '--verbose'; } $label = $args->getArg('label'); if ($label) { $argv[] = '-l'; $argv[] = $label; } $this->argv = $argv; if (function_exists('posix_isatty') && posix_isatty(STDIN)) { fprintf(STDERR, pht('Reading daemon configuration from stdin...') . "\n"); } $config = @file_get_contents('php://stdin'); $config = id(new PhutilJSONParser())->parse($config); $this->libraries = idx($config, 'load'); $this->log = idx($config, 'log'); $this->daemonize = idx($config, 'daemonize'); $this->piddir = idx($config, 'piddir'); $this->config = $config; if (self::$instance) { throw new Exception(pht('You may not instantiate more than one Overseer per process.')); } self::$instance = $this; $this->startEpoch = time(); // Check this before we daemonize, since if it's an issue the child will // exit immediately. if ($this->piddir) { $dir = $this->piddir; try { Filesystem::assertWritable($dir); } catch (Exception $ex) { throw new Exception(pht("Specified daemon PID directory ('%s') does not exist or is " . "not writable by the daemon user!", $dir)); } } if (!idx($config, 'daemons')) { throw new PhutilArgumentUsageException(pht('You must specify at least one daemon to start!')); } if ($this->log) { // NOTE: Now that we're committed to daemonizing, redirect the error // log if we have a `--log` parameter. Do this at the last moment // so as many setup issues as possible are surfaced. ini_set('error_log', $this->log); } if ($this->daemonize) { // We need to get rid of these or the daemon will hang when we TERM it // waiting for something to read the buffers. TODO: Learn how unix works. fclose(STDOUT); fclose(STDERR); ob_start(); $pid = pcntl_fork(); if ($pid === -1) { throw new Exception(pht('Unable to fork!')); } else { if ($pid) { exit(0); } } } $this->modules = PhutilDaemonOverseerModule::getAllModules(); pcntl_signal(SIGUSR2, array($this, 'didReceiveNotifySignal')); pcntl_signal(SIGHUP, array($this, 'didReceiveReloadSignal')); pcntl_signal(SIGINT, array($this, 'didReceiveGracefulSignal')); pcntl_signal(SIGTERM, array($this, 'didReceiveTerminalSignal')); }
/** * Close and free resources if necessary. * * @return void * @task internal */ private function closeProcess() { foreach ($this->pipes as $pipe) { if (isset($pipe)) { @fclose($pipe); } } $this->pipes = array(null, null, null); if ($this->proc) { @proc_close($this->proc); $this->proc = null; } $this->stdin = null; if ($this->profilerCallID !== null) { $profiler = PhutilServiceProfiler::getInstance(); $profiler->endServiceCall($this->profilerCallID, array('err' => $this->result ? idx($this->result, 0) : null)); $this->profilerCallID = null; } }
public function __construct($cache_directory, $name) { $dir_cache = id(new PhutilDirectoryKeyValueCache())->setCacheDirectory($cache_directory); $profiled_cache = id(new PhutilKeyValueCacheProfiler($dir_cache))->setProfiler(PhutilServiceProfiler::getInstance())->setName($name); $this->cache = $profiled_cache; }
public function isReady() { if (isset($this->result)) { return true; } $uri = $this->getURI(); $domain = id(new PhutilURI($uri))->getDomain(); if (!$this->handle) { $uri_object = new PhutilURI($uri); $proxy = PhutilHTTPEngineExtension::buildHTTPProxyURI($uri_object); $profiler = PhutilServiceProfiler::getInstance(); $this->profilerCallID = $profiler->beginServiceCall(array('type' => 'http', 'uri' => $uri, 'proxy' => (string) $proxy)); if (!self::$multi) { self::$multi = curl_multi_init(); if (!self::$multi) { throw new Exception(pht('%s failed!', 'curl_multi_init()')); } } if (!empty(self::$pool[$domain])) { $curl = array_pop(self::$pool[$domain]); } else { $curl = curl_init(); if (!$curl) { throw new Exception(pht('%s failed!', 'curl_init()')); } } $this->handle = $curl; curl_multi_add_handle(self::$multi, $curl); curl_setopt($curl, CURLOPT_URL, $uri); if (defined('CURLOPT_PROTOCOLS')) { // cURL supports a lot of protocols, and by default it will honor // redirects across protocols (for instance, from HTTP to POP3). Beyond // being very silly, this also has security implications: // // http://blog.volema.com/curl-rce.html // // Disable all protocols other than HTTP and HTTPS. $allowed_protocols = CURLPROTO_HTTPS | CURLPROTO_HTTP; curl_setopt($curl, CURLOPT_PROTOCOLS, $allowed_protocols); curl_setopt($curl, CURLOPT_REDIR_PROTOCOLS, $allowed_protocols); } if (strlen($this->rawBody)) { if ($this->getData()) { throw new Exception(pht('You can not execute an HTTP future with both a raw request ' . 'body and structured request data.')); } // We aren't actually going to use this file handle, since we are // just pushing data through the callback, but cURL gets upset if // we don't hand it a real file handle. $tmp = new TempFile(); $this->fileHandle = fopen($tmp, 'r'); // NOTE: We must set CURLOPT_PUT here to make cURL use CURLOPT_INFILE. // We'll possibly overwrite the method later on, unless this is really // a PUT request. curl_setopt($curl, CURLOPT_PUT, true); curl_setopt($curl, CURLOPT_INFILE, $this->fileHandle); curl_setopt($curl, CURLOPT_INFILESIZE, strlen($this->rawBody)); curl_setopt($curl, CURLOPT_READFUNCTION, array($this, 'willWriteBody')); } else { $data = $this->formatRequestDataForCURL(); curl_setopt($curl, CURLOPT_POSTFIELDS, $data); } $headers = $this->getHeaders(); $saw_expect = false; for ($ii = 0; $ii < count($headers); $ii++) { list($name, $value) = $headers[$ii]; $headers[$ii] = $name . ': ' . $value; if (!strncasecmp($name, 'Expect', strlen('Expect'))) { $saw_expect = true; } } if (!$saw_expect) { // cURL sends an "Expect" header by default for certain requests. While // there is some reasoning behind this, it causes a practical problem // in that lighttpd servers reject these requests with a 417. Both sides // are locked in an eternal struggle (lighttpd has introduced a // 'server.reject-expect-100-with-417' option to deal with this case). // // The ostensibly correct way to suppress this behavior on the cURL side // is to add an empty "Expect:" header. If we haven't seen some other // explicit "Expect:" header, do so. // // See here, for example, although this issue is fairly widespread: // http://curl.haxx.se/mail/archive-2009-07/0008.html $headers[] = 'Expect:'; } curl_setopt($curl, CURLOPT_HTTPHEADER, $headers); // Set the requested HTTP method, e.g. GET / POST / PUT. curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $this->getMethod()); // Make sure we get the headers and data back. curl_setopt($curl, CURLOPT_HEADER, true); curl_setopt($curl, CURLOPT_WRITEFUNCTION, array($this, 'didReceiveDataCallback')); if ($this->followLocation) { curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); curl_setopt($curl, CURLOPT_MAXREDIRS, 20); } if (defined('CURLOPT_TIMEOUT_MS')) { // If CURLOPT_TIMEOUT_MS is available, use the higher-precision timeout. $timeout = max(1, ceil(1000 * $this->getTimeout())); curl_setopt($curl, CURLOPT_TIMEOUT_MS, $timeout); } else { // Otherwise, fall back to the lower-precision timeout. $timeout = max(1, ceil($this->getTimeout())); curl_setopt($curl, CURLOPT_TIMEOUT, $timeout); } // We're going to try to set CAINFO below. This doesn't work at all on // OSX around Yosemite (see T5913). On these systems, we'll use the // system CA and then try to tell the user that their settings were // ignored and how to fix things if we encounter a CA-related error. // Assume we have custom CA settings to start with; we'll clear this // flag if we read the default CA info below. // Try some decent fallbacks here: // - First, check if a bundle is set explicitly for this request, via // `setCABundle()` or similar. // - Then, check if a global bundle is set explicitly for all requests, // via `setGlobalCABundle()` or similar. // - Then, if a local custom.pem exists, use that, because it probably // means that the user wants to override everything (also because the // user might not have access to change the box's php.ini to add // curl.cainfo). // - Otherwise, try using curl.cainfo. If it's set explicitly, it's // probably reasonable to try using it before we fall back to what // libphutil ships with. // - Lastly, try the default that libphutil ships with. If it doesn't // work, give up and yell at the user. if (!$this->getCABundle()) { $caroot = dirname(phutil_get_library_root('phutil')) . '/resources/ssl/'; $ini_val = ini_get('curl.cainfo'); if (self::getGlobalCABundle()) { $this->setCABundleFromPath(self::getGlobalCABundle()); } else { if (Filesystem::pathExists($caroot . 'custom.pem')) { $this->setCABundleFromPath($caroot . 'custom.pem'); } else { if ($ini_val) { // TODO: We can probably do a pathExists() here, even. $this->setCABundleFromPath($ini_val); } else { $this->setCABundleFromPath($caroot . 'default.pem'); } } } } if ($this->canSetCAInfo()) { curl_setopt($curl, CURLOPT_CAINFO, $this->getCABundle()); } $verify_peer = 1; $verify_host = 2; $extensions = PhutilHTTPEngineExtension::getAllExtensions(); foreach ($extensions as $extension) { if ($extension->shouldTrustAnySSLAuthorityForURI($uri_object)) { $verify_peer = 0; } if ($extension->shouldTrustAnySSLHostnameForURI($uri_object)) { $verify_host = 0; } } curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, $verify_peer); curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, $verify_host); curl_setopt($curl, CURLOPT_SSLVERSION, 0); if ($proxy) { curl_setopt($curl, CURLOPT_PROXY, (string) $proxy); } } else { $curl = $this->handle; if (!self::$results) { // NOTE: In curl_multi_select(), PHP calls curl_multi_fdset() but does // not check the return value of &maxfd for -1 until recent versions // of PHP (5.4.8 and newer). cURL may return -1 as maxfd in some unusual // situations; if it does, PHP enters select() with nfds=0, which blocks // until the timeout is reached. // // We could try to guess whether this will happen or not by examining // the version identifier, but we can also just sleep for only a short // period of time. curl_multi_select(self::$multi, 0.01); } } do { $active = null; $result = curl_multi_exec(self::$multi, $active); } while ($result == CURLM_CALL_MULTI_PERFORM); while ($info = curl_multi_info_read(self::$multi)) { if ($info['msg'] == CURLMSG_DONE) { self::$results[(int) $info['handle']] = $info; } } if (!array_key_exists((int) $curl, self::$results)) { return false; } // The request is complete, so release any temporary files we wrote // earlier. $this->temporaryFiles = array(); $info = self::$results[(int) $curl]; $result = $this->responseBuffer; $err_code = $info['result']; if ($err_code) { if ($err_code == CURLE_SSL_CACERT && !$this->canSetCAInfo()) { $status = new HTTPFutureCertificateResponseStatus(HTTPFutureCertificateResponseStatus::ERROR_IMMUTABLE_CERTIFICATES, $uri); } else { $status = new HTTPFutureCURLResponseStatus($err_code, $uri); } $body = null; $headers = array(); $this->result = array($status, $body, $headers); } else { // cURL returns headers of all redirects, we strip all but the final one. $redirects = curl_getinfo($curl, CURLINFO_REDIRECT_COUNT); $result = preg_replace('/^(.*\\r\\n\\r\\n){' . $redirects . '}/sU', '', $result); $this->result = $this->parseRawHTTPResponse($result); } curl_multi_remove_handle(self::$multi, $curl); unset(self::$results[(int) $curl]); // NOTE: We want to use keepalive if possible. Return the handle to a // pool for the domain; don't close it. if ($this->shouldReuseHandles()) { self::$pool[$domain][] = $curl; } $profiler = PhutilServiceProfiler::getInstance(); $profiler->endServiceCall($this->profilerCallID, array()); return true; }
private function bindLDAP($conn, $user, PhutilOpaqueEnvelope $pass) { $profiler = PhutilServiceProfiler::getInstance(); $call_id = $profiler->beginServiceCall(array('type' => 'ldap', 'call' => 'bind', 'user' => $user)); // NOTE: ldap_bind() dumps cleartext passwords into logs by default. Keep // it quiet. if (strlen($user)) { $ok = @ldap_bind($conn, $user, $pass->openEnvelope()); } else { $ok = @ldap_bind($conn); } $profiler->endServiceCall($call_id, array()); if (!$ok) { if (strlen($user)) { $this->raiseConnectionException($conn, pht('Failed to bind to LDAP server (as user "%s").', $user)); } else { $this->raiseConnectionException($conn, pht('Failed to bind to LDAP server (without username).')); } } }
public function callMethod($method, array $params) { $meta = array(); if ($this->sessionKey) { $meta['sessionKey'] = $this->sessionKey; } if ($this->connectionID) { $meta['connectionID'] = $this->connectionID; } if ($method == 'conduit.connect') { $certificate = idx($params, 'certificate'); if ($certificate) { $token = time(); $params['authToken'] = $token; $params['authSignature'] = sha1($token . $certificate); } unset($params['certificate']); } if ($meta) { $params['__conduit__'] = $meta; } $port = null; if ($this->port) { $port = ':' . $this->port; } $uri = $this->protocol . '://' . $this->host . $port . '/' . $this->path . $method; $data = array('params' => json_encode($params), 'output' => 'json'); if ($this->protocol == 'https') { $core_future = new HTTPSFuture($uri, $data); } else { $core_future = new HTTPFuture($uri, $data); } $core_future->setMethod('POST'); $core_future->setTimeout($this->timeout); $profiler = PhutilServiceProfiler::getInstance(); $this->profilerCallID = $profiler->beginServiceCall(array('type' => 'conduit', 'method' => $method)); $conduit_future = new ConduitFuture($core_future); $conduit_future->setClient($this, $method); $conduit_future->isReady(); return $conduit_future; }
private function endLintServiceCall($call_id) { $profiler = PhutilServiceProfiler::getInstance(); $profiler->endServiceCall($call_id, array()); }
public function executeRawQuery($raw_query) { $this->lastResult = null; $retries = max(1, PhabricatorEnv::getEnvConfig('mysql.connection-retries')); while ($retries--) { try { $this->requireConnection(); // TODO: Do we need to include transactional statements here? $is_write = !preg_match('/^(SELECT|SHOW|EXPLAIN)\\s/', $raw_query); if ($is_write) { AphrontWriteGuard::willWrite(); } $start = microtime(true); $profiler = PhutilServiceProfiler::getInstance(); $call_id = $profiler->beginServiceCall(array('type' => 'query', 'config' => $this->configuration, 'query' => $raw_query, 'write' => $is_write)); $result = $this->rawQuery($raw_query); $profiler->endServiceCall($call_id, array()); if ($this->nextError) { $result = null; } if ($result) { $this->lastResult = $result; break; } $this->throwQueryException($this->connection); } catch (AphrontQueryConnectionLostException $ex) { if ($this->isInsideTransaction()) { // Zero out the transaction state to prevent a second exception // ("program exited with open transaction") from being thrown, since // we're about to throw a more relevant/useful one instead. $state = $this->getTransactionState(); while ($state->getDepth()) { $state->decreaseDepth(); } // We can't close the connection before this because // isInsideTransaction() and getTransactionState() depend on the // connection. $this->closeConnection(); throw $ex; } $this->closeConnection(); if (!$retries) { throw $ex; } $class = get_class($ex); $message = $ex->getMessage(); phlog("Retrying ({$retries}) after {$class}: {$message}"); } } }
public static function installEchoListener() { $instance = PhutilServiceProfiler::getInstance(); $instance->addListener(array('PhutilServiceProfiler', 'echoListener')); }
private function checkSocket() { $timeout = false; $now = microtime(true); if ($now - $this->stateStartTime > $this->getTimeout()) { $timeout = true; } if (!feof($this->socket) && !$timeout) { return false; } $this->stateReady = true; if ($timeout) { $this->result = $this->buildErrorResult(HTTPFutureResponseStatusTransport::ERROR_TIMEOUT); } else { if (!$this->stateConnected) { $this->result = $this->buildErrorResult(HTTPFutureResponseStatusTransport::ERROR_CONNECTION_REFUSED); } else { if (!$this->stateWriteComplete) { $this->result = $this->buildErrorResult(HTTPFutureResponseStatusTransport::ERROR_CONNECTION_FAILED); } else { $this->result = $this->parseRawHTTPResponse($this->response); } } } $profiler = PhutilServiceProfiler::getInstance(); $profiler->endServiceCall($this->profilerCallID, array()); return true; }
protected final function didRunLinters(array $linters) { assert_instances_of($linters, 'ArcanistLinter'); $exceptions = array(); $profiler = PhutilServiceProfiler::getInstance(); foreach ($linters as $linter_name => $linter) { if (!is_string($linter_name)) { $linter_name = get_class($linter); } $call_id = $profiler->beginServiceCall(array('type' => 'lint', 'linter' => $linter_name)); try { $linter->didRunLinters(); } catch (Exception $ex) { $exceptions[$linter_name] = $ex; } $profiler->endServiceCall($call_id, array()); } return $exceptions; }
private static function addProfilerToCaches(array $caches) { foreach ($caches as $key => $cache) { $pcache = new PhutilKeyValueCacheProfiler($cache); $pcache->setProfiler(PhutilServiceProfiler::getInstance()); $caches[$key] = $pcache; } return $caches; }
public function executeRawQueries(array $raw_queries) { if (!$raw_queries) { return array(); } $is_write = false; foreach ($raw_queries as $key => $raw_query) { $is_write = $is_write || $this->checkWrite($raw_query); $raw_queries[$key] = rtrim($raw_query, "\r\n\t ;"); } $profiler = PhutilServiceProfiler::getInstance(); $call_id = $profiler->beginServiceCall(array('type' => 'multi-query', 'config' => $this->configuration, 'queries' => $raw_queries, 'write' => $is_write)); $results = $this->rawQueries($raw_queries); $profiler->endServiceCall($call_id, array()); return $results; }
public function isReady() { if (isset($this->result)) { return true; } $uri = $this->getURI(); $domain = id(new PhutilURI($uri))->getDomain(); if (!$this->handle) { $profiler = PhutilServiceProfiler::getInstance(); $this->profilerCallID = $profiler->beginServiceCall(array('type' => 'http', 'uri' => $uri)); if (!self::$multi) { self::$multi = curl_multi_init(); if (!self::$multi) { throw new Exception('curl_multi_init() failed!'); } } if (!empty(self::$pool[$domain])) { $curl = array_pop(self::$pool[$domain]); } else { $curl = curl_init(); if (!$curl) { throw new Exception('curl_init() failed!'); } } $this->handle = $curl; curl_multi_add_handle(self::$multi, $curl); curl_setopt($curl, CURLOPT_URL, $uri); if (defined('CURLOPT_PROTOCOLS')) { // cURL supports a lot of protocols, and by default it will honor // redirects across protocols (for instance, from HTTP to POP3). Beyond // being very silly, this also has security implications: // // http://blog.volema.com/curl-rce.html // // Disable all protocols other than HTTP and HTTPS. $allowed_protocols = CURLPROTO_HTTPS | CURLPROTO_HTTP; curl_setopt($curl, CURLOPT_PROTOCOLS, $allowed_protocols); curl_setopt($curl, CURLOPT_REDIR_PROTOCOLS, $allowed_protocols); } $data = $this->getData(); if ($data) { // NOTE: PHP's cURL implementation has a piece of magic which treats // parameters as file paths if they begin with '@'. This means that // an array like "array('name' => '@/usr/local/secret')" will attempt to // read that file off disk and send it to the remote server. This // behavior is pretty surprising, and it can easily become a relatively // severe security vulnerability which allows an attacker to read any // file the HTTP process has access to. Since this feature is very // dangerous and not particularly useful, we prevent its use. // // After PHP 5.2.0, it is sufficient to pass a string to avoid this // "feature" (it is only invoked in the array version). Prior to // PHP 5.2.0, we block any request which have string data beginning with // '@' (they would not work anyway). if (is_array($data)) { // Explicitly build a query string to prevent "@" security problems. $data = http_build_query($data, '', '&'); } if ($data[0] == '@' && version_compare(phpversion(), '5.2.0', '<')) { throw new Exception("Attempting to make an HTTP request including string data that " . "begins with '@'. Prior to PHP 5.2.0, this reads files off disk, " . "which creates a wide attack window for security vulnerabilities. " . "Upgrade PHP or avoid making cURL requests which begin with '@'."); } curl_setopt($curl, CURLOPT_POSTFIELDS, $data); } else { curl_setopt($curl, CURLOPT_POSTFIELDS, null); } $headers = $this->getHeaders(); $saw_expect = false; for ($ii = 0; $ii < count($headers); $ii++) { list($name, $value) = $headers[$ii]; $headers[$ii] = $name . ': ' . $value; if (!strncasecmp($name, 'Expect', strlen('Expect'))) { $saw_expect = true; } } if (!$saw_expect) { // cURL sends an "Expect" header by default for certain requests. While // there is some reasoning behind this, it causes a practical problem // in that lighttpd servers reject these requests with a 417. Both sides // are locked in an eternal struggle (lighttpd has introduced a // 'server.reject-expect-100-with-417' option to deal with this case). // // The ostensibly correct way to suppress this behavior on the cURL side // is to add an empty "Expect:" header. If we haven't seen some other // explicit "Expect:" header, do so. // // See here, for example, although this issue is fairly widespread: // http://curl.haxx.se/mail/archive-2009-07/0008.html $headers[] = 'Expect:'; } curl_setopt($curl, CURLOPT_HTTPHEADER, $headers); // Set the requested HTTP method, e.g. GET / POST / PUT. curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $this->getMethod()); // Make sure we get the headers and data back. curl_setopt($curl, CURLOPT_HEADER, true); curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); curl_setopt($curl, CURLOPT_MAXREDIRS, 20); if (defined('CURLOPT_TIMEOUT_MS')) { // If CURLOPT_TIMEOUT_MS is available, use the higher-precision timeout. $timeout = max(1, ceil(1000 * $this->getTimeout())); curl_setopt($curl, CURLOPT_TIMEOUT_MS, $timeout); } else { // Otherwise, fall back to the lower-precision timeout. $timeout = max(1, ceil($this->getTimeout())); curl_setopt($curl, CURLOPT_TIMEOUT, $timeout); } // Try some decent fallbacks here: // - First, check if a bundle is set explicit for this request, via // `setCABundle()` or similar. // - Then, check if a global bundle is set explicitly for all requests, // via `setGlobalCABundle()` or similar. // - Then, if a local custom.pem exists, use that, because it probably // means that the user wants to override everything (also because the // user might not have access to change the box's php.ini to add // curl.cainfo). // - Otherwise, try using curl.cainfo. If it's set explicitly, it's // probably reasonable to try using it before we fall back to what // libphutil ships with. // - Lastly, try the default that libphutil ships with. If it doesn't // work, give up and yell at the user. if (!$this->getCABundle()) { $caroot = dirname(phutil_get_library_root('phutil')) . '/resources/ssl/'; $ini_val = ini_get('curl.cainfo'); if (self::getGlobalCABundle()) { $this->setCABundleFromPath(self::getGlobalCABundle()); } else { if (Filesystem::pathExists($caroot . 'custom.pem')) { $this->setCABundleFromPath($caroot . 'custom.pem'); } else { if ($ini_val) { // TODO: We can probably do a pathExists() here, even. $this->setCABundleFromPath($ini_val); } else { $this->setCABundleFromPath($caroot . 'default.pem'); } } } } curl_setopt($curl, CURLOPT_CAINFO, $this->getCABundle()); curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, true); curl_setopt($curl, CURLOPT_SSLVERSION, 0); } else { $curl = $this->handle; if (!self::$results) { // NOTE: In curl_multi_select(), PHP calls curl_multi_fdset() but does // not check the return value of &maxfd for -1 until recent versions // of PHP (5.4.8 and newer). cURL may return -1 as maxfd in some unusual // situations; if it does, PHP enters select() with nfds=0, which blocks // until the timeout is reached. // // We could try to guess whether this will happen or not by examining // the version identifier, but we can also just sleep for only a short // period of time. curl_multi_select(self::$multi, 0.01); } } do { $active = null; $result = curl_multi_exec(self::$multi, $active); } while ($result == CURLM_CALL_MULTI_PERFORM); while ($info = curl_multi_info_read(self::$multi)) { if ($info['msg'] == CURLMSG_DONE) { self::$results[(int) $info['handle']] = $info; } } if (!array_key_exists((int) $curl, self::$results)) { return false; } $info = self::$results[(int) $curl]; $result = curl_multi_getcontent($curl); $err_code = $info['result']; if ($err_code) { $status = new Status_HTTPFutureResponseStatusCURL($err_code, $uri); $body = null; $headers = array(); $this->result = array($status, $body, $headers); } else { // cURL returns headers of all redirects, we strip all but the final one. $redirects = curl_getinfo($curl, CURLINFO_REDIRECT_COUNT); $result = preg_replace('/^(.*\\r\\n\\r\\n){' . $redirects . '}/sU', '', $result); $this->result = $this->parseRawHTTPResponse($result); } curl_multi_remove_handle(self::$multi, $curl); unset(self::$results[(int) $curl]); // NOTE: We want to use keepalive if possible. Return the handle to a // pool for the domain; don't close it. self::$pool[$domain][] = $curl; $profiler = PhutilServiceProfiler::getInstance(); $profiler->endServiceCall($this->profilerCallID, array()); return true; }