/** * Sends an HTTP request using the specified transport method. * * @param string $method * @param string $aUrl * @param int $timeout * @param string $userAgent * @param string $destinationPath * @param resource $file * @param int $followDepth * @param bool|string $acceptLanguage Accept-language header * @param bool $acceptInvalidSslCertificate Only used with $method == 'curl'. If set to true (NOT recommended!) the SSL certificate will not be checked * @param array|bool $byteRange For Range: header. Should be two element array of bytes, eg, array(0, 1024) * Doesn't work w/ fopen method. * @param bool $getExtendedInfo True to return status code, headers & response, false if just response. * @param string $httpMethod The HTTP method to use. Defaults to `'GET'`. * * @throws Exception * @return bool true (or string/array) on success; false on HTTP response error code (1xx or 4xx) */ public static function sendHttpRequestBy($method = 'socket', $aUrl, $timeout, $userAgent = null, $destinationPath = null, $file = null, $followDepth = 0, $acceptLanguage = false, $acceptInvalidSslCertificate = false, $byteRange = false, $getExtendedInfo = false, $httpMethod = 'GET') { if ($followDepth > 5) { throw new Exception('Too many redirects (' . $followDepth . ')'); } $contentLength = 0; $fileLength = 0; // Piwik services behave like a proxy, so we should act like one. $xff = 'X-Forwarded-For: ' . (isset($_SERVER['HTTP_X_FORWARDED_FOR']) && !empty($_SERVER['HTTP_X_FORWARDED_FOR']) ? $_SERVER['HTTP_X_FORWARDED_FOR'] . ',' : '') . IP::getIpFromHeader(); if (empty($userAgent)) { $userAgent = self::getUserAgent(); } $via = 'Via: ' . (isset($_SERVER['HTTP_VIA']) && !empty($_SERVER['HTTP_VIA']) ? $_SERVER['HTTP_VIA'] . ', ' : '') . Version::VERSION . ' ' . ($userAgent ? " ({$userAgent})" : ''); // range header $rangeHeader = ''; if (!empty($byteRange)) { $rangeHeader = 'Range: bytes=' . $byteRange[0] . '-' . $byteRange[1] . "\r\n"; } // proxy configuration $proxyHost = Config::getInstance()->proxy['host']; $proxyPort = Config::getInstance()->proxy['port']; $proxyUser = Config::getInstance()->proxy['username']; $proxyPassword = Config::getInstance()->proxy['password']; $aUrl = trim($aUrl); // other result data $status = null; $headers = array(); if ($method == 'socket') { if (!self::isSocketEnabled()) { // can be triggered in tests throw new Exception("HTTP socket support is not enabled (php function fsockopen is not available) "); } // initialization $url = @parse_url($aUrl); if ($url === false || !isset($url['scheme'])) { throw new Exception('Malformed URL: ' . $aUrl); } if ($url['scheme'] != 'http') { throw new Exception('Invalid protocol/scheme: ' . $url['scheme']); } $host = $url['host']; $port = isset($url['port)']) ? $url['port'] : 80; $path = isset($url['path']) ? $url['path'] : '/'; if (isset($url['query'])) { $path .= '?' . $url['query']; } $errno = null; $errstr = null; if (!empty($proxyHost) && !empty($proxyPort) || !empty($byteRange)) { $httpVer = '1.1'; } else { $httpVer = '1.0'; } $proxyAuth = null; if (!empty($proxyHost) && !empty($proxyPort)) { $connectHost = $proxyHost; $connectPort = $proxyPort; if (!empty($proxyUser) && !empty($proxyPassword)) { $proxyAuth = 'Proxy-Authorization: Basic ' . base64_encode("{$proxyUser}:{$proxyPassword}") . "\r\n"; } $requestHeader = "{$httpMethod} {$aUrl} HTTP/{$httpVer}\r\n"; } else { $connectHost = $host; $connectPort = $port; $requestHeader = "{$httpMethod} {$path} HTTP/{$httpVer}\r\n"; } // connection attempt if (($fsock = @fsockopen($connectHost, $connectPort, $errno, $errstr, $timeout)) === false || !is_resource($fsock)) { if (is_resource($file)) { @fclose($file); } throw new Exception("Error while connecting to: {$host}. Please try again later. {$errstr}"); } // send HTTP request header $requestHeader .= "Host: {$host}" . ($port != 80 ? ':' . $port : '') . "\r\n" . ($proxyAuth ? $proxyAuth : '') . 'User-Agent: ' . $userAgent . "\r\n" . ($acceptLanguage ? $acceptLanguage . "\r\n" : '') . $xff . "\r\n" . $via . "\r\n" . $rangeHeader . "Connection: close\r\n" . "\r\n"; fwrite($fsock, $requestHeader); $streamMetaData = array('timed_out' => false); @stream_set_blocking($fsock, true); if (function_exists('stream_set_timeout')) { @stream_set_timeout($fsock, $timeout); } elseif (function_exists('socket_set_timeout')) { @socket_set_timeout($fsock, $timeout); } // process header $status = null; while (!feof($fsock)) { $line = fgets($fsock, 4096); $streamMetaData = @stream_get_meta_data($fsock); if ($streamMetaData['timed_out']) { if (is_resource($file)) { @fclose($file); } @fclose($fsock); throw new Exception('Timed out waiting for server response'); } // a blank line marks the end of the server response header if (rtrim($line, "\r\n") == '') { break; } // parse first line of server response header if (!$status) { // expect first line to be HTTP response status line, e.g., HTTP/1.1 200 OK if (!preg_match('~^HTTP/(\\d\\.\\d)\\s+(\\d+)(\\s*.*)?~', $line, $m)) { if (is_resource($file)) { @fclose($file); } @fclose($fsock); throw new Exception('Expected server response code. Got ' . rtrim($line, "\r\n")); } $status = (int) $m[2]; // Informational 1xx or Client Error 4xx if ($status < 200 || $status >= 400) { if (is_resource($file)) { @fclose($file); } @fclose($fsock); if (!$getExtendedInfo) { return false; } else { return array('status' => $status); } } continue; } // handle redirect if (preg_match('/^Location:\\s*(.+)/', rtrim($line, "\r\n"), $m)) { if (is_resource($file)) { @fclose($file); } @fclose($fsock); // Successful 2xx vs Redirect 3xx if ($status < 300) { throw new Exception('Unexpected redirect to Location: ' . rtrim($line) . ' for status code ' . $status); } return self::sendHttpRequestBy($method, trim($m[1]), $timeout, $userAgent, $destinationPath, $file, $followDepth + 1, $acceptLanguage, $acceptInvalidSslCertificate = false, $byteRange, $getExtendedInfo, $httpMethod); } // save expected content length for later verification if (preg_match('/^Content-Length:\\s*(\\d+)/', $line, $m)) { $contentLength = (int) $m[1]; } self::parseHeaderLine($headers, $line); } if (feof($fsock) && $httpMethod != 'HEAD') { throw new Exception('Unexpected end of transmission'); } // process content/body $response = ''; while (!feof($fsock)) { $line = fread($fsock, 8192); $streamMetaData = @stream_get_meta_data($fsock); if ($streamMetaData['timed_out']) { if (is_resource($file)) { @fclose($file); } @fclose($fsock); throw new Exception('Timed out waiting for server response'); } $fileLength += strlen($line); if (is_resource($file)) { // save to file fwrite($file, $line); } else { // concatenate to response string $response .= $line; } } // determine success or failure @fclose(@$fsock); } else { if ($method == 'fopen') { $response = false; // we make sure the request takes less than a few seconds to fail // we create a stream_context (works in php >= 5.2.1) // we also set the socket_timeout (for php < 5.2.1) $default_socket_timeout = @ini_get('default_socket_timeout'); @ini_set('default_socket_timeout', $timeout); $ctx = null; if (function_exists('stream_context_create')) { $stream_options = array('http' => array('header' => 'User-Agent: ' . $userAgent . "\r\n" . ($acceptLanguage ? $acceptLanguage . "\r\n" : '') . $xff . "\r\n" . $via . "\r\n" . $rangeHeader, 'max_redirects' => 5, 'timeout' => $timeout)); if (!empty($proxyHost) && !empty($proxyPort)) { $stream_options['http']['proxy'] = 'tcp://' . $proxyHost . ':' . $proxyPort; $stream_options['http']['request_fulluri'] = true; // required by squid proxy if (!empty($proxyUser) && !empty($proxyPassword)) { $stream_options['http']['header'] .= 'Proxy-Authorization: Basic ' . base64_encode("{$proxyUser}:{$proxyPassword}") . "\r\n"; } } $ctx = stream_context_create($stream_options); } // save to file if (is_resource($file)) { $handle = fopen($aUrl, 'rb', false, $ctx); while (!feof($handle)) { $response = fread($handle, 8192); $fileLength += strlen($response); fwrite($file, $response); } fclose($handle); } else { $response = file_get_contents($aUrl, 0, $ctx); $fileLength = strlen($response); } // restore the socket_timeout value if (!empty($default_socket_timeout)) { @ini_set('default_socket_timeout', $default_socket_timeout); } } else { if ($method == 'curl') { if (!self::isCurlEnabled()) { // can be triggered in tests throw new Exception("CURL is not enabled in php.ini, but is being used."); } $ch = @curl_init(); if (!empty($proxyHost) && !empty($proxyPort)) { @curl_setopt($ch, CURLOPT_PROXY, $proxyHost . ':' . $proxyPort); if (!empty($proxyUser) && !empty($proxyPassword)) { // PROXYAUTH defaults to BASIC @curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyUser . ':' . $proxyPassword); } } $curl_options = array(CURLOPT_BINARYTRANSFER => is_resource($file), CURLOPT_URL => $aUrl, CURLOPT_USERAGENT => $userAgent, CURLOPT_HTTPHEADER => array($xff, $via, $rangeHeader, $acceptLanguage), CURLOPT_HEADER => is_resource($file) ? false : true, CURLOPT_CONNECTTIMEOUT => $timeout); // Case core:archive command is triggering archiving on https:// and the certificate is not valid if ($acceptInvalidSslCertificate) { $curl_options += array(CURLOPT_SSL_VERIFYHOST => false, CURLOPT_SSL_VERIFYPEER => false); } @curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $httpMethod); if ($httpMethod == 'HEAD') { @curl_setopt($ch, CURLOPT_NOBODY, true); } @curl_setopt_array($ch, $curl_options); self::configCurlCertificate($ch); /* * as of php 5.2.0, CURLOPT_FOLLOWLOCATION can't be set if * in safe_mode or open_basedir is set */ if ((string) ini_get('safe_mode') == '' && ini_get('open_basedir') == '') { $curl_options = array(CURLOPT_FOLLOWLOCATION => true, CURLOPT_MAXREDIRS => 5); @curl_setopt_array($ch, $curl_options); } if (is_resource($file)) { // write output directly to file @curl_setopt($ch, CURLOPT_FILE, $file); } else { // internal to ext/curl @curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); } ob_start(); $response = @curl_exec($ch); ob_end_clean(); if ($response === true) { $response = ''; } else { if ($response === false) { $errstr = curl_error($ch); if ($errstr != '') { throw new Exception('curl_exec: ' . $errstr . '. Hostname requested was: ' . UrlHelper::getHostFromUrl($aUrl)); } $response = ''; } else { $header = ''; // redirects are included in the output html, so we look for the last line that starts w/ HTTP/... // to split the response while (substr($response, 0, 5) == "HTTP/") { list($header, $response) = explode("\r\n\r\n", $response, 2); } foreach (explode("\r\n", $header) as $line) { self::parseHeaderLine($headers, $line); } } } $contentLength = @curl_getinfo($ch, CURLINFO_CONTENT_LENGTH_DOWNLOAD); $fileLength = is_resource($file) ? @curl_getinfo($ch, CURLINFO_SIZE_DOWNLOAD) : strlen($response); $status = @curl_getinfo($ch, CURLINFO_HTTP_CODE); @curl_close($ch); unset($ch); } else { throw new Exception('Invalid request method: ' . $method); } } } if (is_resource($file)) { fflush($file); @fclose($file); $fileSize = filesize($destinationPath); if ($contentLength > 0 && $fileLength != $contentLength || $fileSize != $fileLength) { throw new Exception('File size error: ' . $destinationPath . '; expected ' . $contentLength . ' bytes; received ' . $fileLength . ' bytes; saved ' . $fileSize . ' bytes to file'); } return true; } if (!$getExtendedInfo) { return trim($response); } else { return array('status' => $status, 'headers' => $headers, 'data' => $response); } }
protected function initPiwikHost(InputInterface $input) { $piwikHostname = $input->getParameterOption('--piwik-domain'); if (empty($piwikHostname)) { $piwikHostname = $input->getParameterOption('--url'); } $piwikHostname = UrlHelper::getHostFromUrl($piwikHostname); Url::setHost($piwikHostname); }
private function executeAsyncCli($url, Output $output, $cmdId) { $this->processes[] = new Process($cmdId); $url = $this->appendTestmodeParamToUrlIfNeeded($url); $query = UrlHelper::getQueryFromUrl($url, array('pid' => $cmdId)); $hostname = UrlHelper::getHostFromUrl($url); $command = $this->buildCommand($hostname, $query, $output->getPathToFile()); Log::debug($command); shell_exec($command); }
/** * Returns Proxy to use for connecting via HTTP to given URL * * @param string $url * @return array */ private static function getProxyConfiguration($url) { $hostname = UrlHelper::getHostFromUrl($url); $localHostnames = Url::getLocalHostnames(); if (in_array($hostname, $localHostnames)) { return array(null, null, null, null); } // proxy configuration $proxyHost = Config::getInstance()->proxy['host']; $proxyPort = Config::getInstance()->proxy['port']; $proxyUser = Config::getInstance()->proxy['username']; $proxyPassword = Config::getInstance()->proxy['password']; return array($proxyHost, $proxyPort, $proxyUser, $proxyPassword); }
/** * @group Core */ public function test_getHostFromUrl() { $this->assertEquals('', UrlHelper::getHostFromUrl('')); $this->assertEquals('', UrlHelper::getHostFromUrl(null)); $this->assertEquals('localhost', UrlHelper::getHostFromUrl('http://localhost')); $this->assertEquals('localhost', UrlHelper::getHostFromUrl('http://localhost/path')); $this->assertEquals('localhost', UrlHelper::getHostFromUrl('localhost/path')); $this->assertEquals('sub.localhost', UrlHelper::getHostFromUrl('sub.localhost/path')); $this->assertEquals('sub.localhost', UrlHelper::getHostFromUrl('http://sub.localhost/path/?query=test')); if (SystemTestCase::isPhpVersion53()) { //parse_url was fixed in 5,4,7 // Fixed host recognition when scheme is omitted and a leading component separator is present. // http://php.net/parse_url return; } $this->assertEquals('localhost', UrlHelper::getHostFromUrl('//localhost/path')); $this->assertEquals('localhost', UrlHelper::getHostFromUrl('//localhost/path?test=test2')); $this->assertEquals('example.org', UrlHelper::getHostFromUrl('//example.org/path')); $this->assertEquals('example.org', UrlHelper::getHostFromUrl('//example.org/path?test=test2')); }
/** * @group Core */ public function test_getHostFromUrl() { $this->assertEquals('', UrlHelper::getHostFromUrl('')); $this->assertEquals('', UrlHelper::getHostFromUrl(null)); $this->assertEquals('localhost', UrlHelper::getHostFromUrl('http://localhost')); $this->assertEquals('localhost', UrlHelper::getHostFromUrl('http://localhost/path')); $this->assertEquals('localhost', UrlHelper::getHostFromUrl('localhost/path')); $this->assertEquals('sub.localhost', UrlHelper::getHostFromUrl('sub.localhost/path')); $this->assertEquals('sub.localhost', UrlHelper::getHostFromUrl('http://sub.localhost/path/?query=test')); }