Sends an HTTP request using best available transport method.
| public static sendHttpRequest ( string $aUrl, integer $timeout, string | null $userAgent = null, string | null $destinationPath = null, integer | null $followDepth, boolean $acceptLanguage = false, array | boolean $byteRange = false, boolean $getExtendedInfo = false, string $httpMethod = 'GET', string $httpUsername = null, string $httpPassword = null ) : boolean | string | ||
| $aUrl | string | The target URL. |
| $timeout | integer | The number of seconds to wait before aborting the HTTP request. |
| $userAgent | string | null | The user agent to use. |
| $destinationPath | string | null | If supplied, the HTTP response will be saved to the file specified by this path. |
| $followDepth | integer | null | Internal redirect count. Should always pass `null` for this parameter. |
| $acceptLanguage | boolean | The value to use for the `'Accept-Language'` HTTP request header. |
| $byteRange | array | boolean | For `Range:` header. Should be two element array of bytes, eg, `array(0, 1024)` Doesn't work w/ `fopen` transport method. |
| $getExtendedInfo | boolean | If true returns the status code, headers & response, if false just the response. |
| $httpMethod | string | The HTTP method to use. Defaults to `'GET'`. |
| $httpUsername | string | HTTP Auth username |
| $httpPassword | string | HTTP Auth password |
| return | boolean | string | If `$destinationPath` is not specified the HTTP response is returned on success. `false` is returned on failure. If `$getExtendedInfo` is `true` and `$destinationPath` is not specified an array with the following information is returned on success: - **status**: the HTTP status code - **headers**: the HTTP headers - **data**: the HTTP response data `false` is still returned on failure. |
public function testWebArchiving()
{
if (self::isMysqli() && self::isTravisCI()) {
$this->markTestSkipped('Skipping on Mysqli as it randomly fails.');
}
$host = Fixture::getRootUrl();
$token = Fixture::getTokenAuth();
$urlTmp = Option::get('piwikUrl');
Option::set('piwikUrl', $host . 'tests/PHPUnit/proxy/index.php');
$url = $host . 'tests/PHPUnit/proxy/archive.php?token_auth=' . $token;
$output = Http::sendHttpRequest($url, 600);
// ignore random build issues
if (empty($output) || strpos($output, \Piwik\CronArchive::NO_ERROR) === false) {
$message = "This test has failed. Because it sometimes randomly fails, we skip the test, and ignore this failure.\n";
$message .= "If you see this message often, or in every build, please investigate as this should only be a random and rare occurence!\n";
$message .= "\n\narchive web failed: " . $output . "\n\nurl used: {$url}";
$this->markTestSkipped($message);
}
if (!empty($urlTmp)) {
Option::set('piwikUrl', $urlTmp);
} else {
Option::delete('piwikUrl');
}
$this->assertContains('Starting Piwik reports archiving...', $output);
$this->assertContains('Archived website id = 1', $output);
$this->assertContains('Done archiving!', $output);
$this->compareArchivePhpOutputAgainstExpected($output);
}
public function triggerWebtreesAdminTasks()
{
$settings = new Settings();
$this->logger = \Piwik\Container\StaticContainer::get('Psr\\Log\\LoggerInterface');
$this->logger->info('Webtrees Admin Task triggered');
$rooturl = $settings->getSetting('webtreesRootUrl');
if (!$rooturl || strlen($rooturl->getValue()) === 0) {
return;
}
$token = $settings->getSetting('webtreesToken');
if (!$token || strlen($token->getValue()) === 0) {
return;
}
$taskname = $settings->getSetting('webtreesTaskName');
if (!$taskname || strlen($taskname->getValue()) === 0) {
return;
}
$url = sprintf('%1$s/module.php?mod=perso_admintasks&mod_action=trigger&force=%2$s&task=%3$s', $rooturl->getValue(), $token->getValue(), $taskname->getValue());
$this->logger->info('webtrees url : {url}', array('url' => $url));
try {
\Piwik\Http::sendHttpRequest($url, Webtrees::SOCKET_TIMEOUT);
} catch (Exception $e) {
$this->logger->warning('an error occured', array('exception' => $e));
}
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$buildNumber = $input->getArgument('buildnumber');
$screenshotsRegex = $input->getArgument('screenshotsRegex');
if (empty($buildNumber)) {
throw new \InvalidArgumentException('Missing build number.');
}
$urlBase = sprintf('http://builds-artifacts.piwik.org/ui-tests.master/%s', $buildNumber);
$diffviewer = Http::sendHttpRequest($urlBase . "/screenshot-diffs/diffviewer.html", $timeout = 60);
$diffviewer = str_replace('&', '&', $diffviewer);
$dom = new \DOMDocument();
$dom->loadHTML($diffviewer);
foreach ($dom->getElementsByTagName("tr") as $row) {
$columns = $row->getElementsByTagName("td");
$nameColumn = $columns->item(0);
$processedColumn = $columns->item(3);
$testPlugin = null;
if ($nameColumn && preg_match("/\\(for ([a-zA-Z_]+) plugin\\)/", $dom->saveXml($nameColumn), $matches)) {
$testPlugin = $matches[1];
}
$file = null;
if ($processedColumn && preg_match("/href=\".*\\/(.*)\"/", $dom->saveXml($processedColumn), $matches)) {
$file = $matches[1];
}
if ($file !== null && preg_match("/" . $screenshotsRegex . "/", $file)) {
if ($testPlugin == null) {
$downloadTo = "tests/PHPUnit/UI/expected-ui-screenshots/{$file}";
} else {
$downloadTo = "plugins/{$testPlugin}/tests/UI/expected-ui-screenshots/{$file}";
}
$output->write("<info>Downloading {$file} to .{$downloadTo}...</info>\n");
Http::sendHttpRequest("{$urlBase}/processed-ui-screenshots/{$file}", $timeout = 60, $userAgent = null, PIWIK_DOCUMENT_ROOT . "/" . $downloadTo);
}
}
}
/**
* Check for a newer version
*
* @param bool $force Force check
* @param int $interval Interval used for update checks
*/
public static function check($force = false, $interval = null)
{
if (!self::isAutoUpdateEnabled()) {
return;
}
if ($interval === null) {
$interval = self::CHECK_INTERVAL;
}
$lastTimeChecked = Option::get(self::LAST_TIME_CHECKED);
if ($force || $lastTimeChecked === false || time() - $interval > $lastTimeChecked) {
// set the time checked first, so that parallel Piwik requests don't all trigger the http requests
Option::set(self::LAST_TIME_CHECKED, time(), $autoLoad = 1);
$parameters = array('piwik_version' => Version::VERSION, 'php_version' => PHP_VERSION, 'url' => Url::getCurrentUrlWithoutQueryString(), 'trigger' => Common::getRequestVar('module', '', 'string'), 'timezone' => API::getInstance()->getDefaultTimezone());
$url = Config::getInstance()->General['api_service_url'] . '/1.0/getLatestVersion/' . '?' . http_build_query($parameters, '', '&');
$timeout = self::SOCKET_TIMEOUT;
if (@Config::getInstance()->Debug['allow_upgrades_to_beta']) {
$url = 'http://builds.piwik.org/LATEST_BETA';
}
try {
$latestVersion = Http::sendHttpRequest($url, $timeout);
if (!preg_match('~^[0-9][0-9a-zA-Z_.-]*$~D', $latestVersion)) {
$latestVersion = '';
}
} catch (Exception $e) {
// e.g., disable_functions = fsockopen; allow_url_open = Off
$latestVersion = '';
}
Option::set(self::LATEST_VERSION, $latestVersion);
}
}
/**
* Web service proxy that retrieves the content at the specified URL
*
* @param string $url
* @return string
*/
private function getPage($url)
{
try {
return str_replace(' ', ' ', Http::sendHttpRequest($url, $timeout = 10, @$_SERVER['HTTP_USER_AGENT']));
} catch (Exception $e) {
return '';
}
}
/**
* Update the social definitions
*
* @see https://github.com/piwik/searchengine-and-social-list
*/
public function updateSocials()
{
$url = 'https://raw.githubusercontent.com/piwik/searchengine-and-social-list/master/Socials.yml';
$list = Http::sendHttpRequest($url, 30);
$socials = Social::getInstance()->loadYmlData($list);
if (count($socials) < 50) {
return;
}
Option::set(Social::OPTION_STORAGE_NAME, base64_encode(serialize($socials)));
}
/**
* Get the latest available version number for the currently active release channel. Eg '2.15.0-b4' or '2.15.0'.
* Should return a semantic version number in format MAJOR.MINOR.PATCH (http://semver.org/).
* Returns an empty string in case one cannot connect to the remote server.
* @return string
*/
private static function getLatestAvailableVersionNumber()
{
$channel = StaticContainer::get('\\Piwik\\Plugin\\ReleaseChannels')->getActiveReleaseChannel();
$url = $channel->getUrlToCheckForLatestAvailableVersion();
try {
$latestVersion = Http::sendHttpRequest($url, self::SOCKET_TIMEOUT);
} catch (\Exception $e) {
// e.g., disable_functions = fsockopen; allow_url_open = Off
$latestVersion = '';
}
return $latestVersion;
}
private function isPageSpeedEnabled()
{
$url = Url::getCurrentUrlWithoutQueryString() . '?module=Installation&action=getEmptyPageForSystemCheck';
try {
$page = Http::sendHttpRequest($url, $timeout = 1, $userAgent = null, $destinationPath = null, $followDepth = 0, $acceptLanguage = false, $byteRange = false, $getExtendedInfo = true);
} catch (\Exception $e) {
$this->logger->info('Unable to test if mod_pagespeed is enabled: the request to {url} failed', array('url' => $url));
// If the test failed, we assume Page speed is not enabled
return false;
}
$headers = $page['headers'];
return isset($headers['X-Mod-Pagespeed']) || isset($headers['X-Page-Speed']);
}
private function getAllTlds()
{
/** @var array $response */
$response = \Piwik\Http::sendHttpRequest("http://data.iana.org/TLD/tlds-alpha-by-domain.txt", 30, null, null, null, null, null, true);
$this->assertEquals("200", $response['status']);
$tlds = explode("\n", $response['data']);
foreach ($tlds as $key => $tld) {
if (strpos($tld, '#') !== false || $tld == "") {
unset($tlds[$key]);
}
}
return $tlds;
}
public function fetchPageRank($domain)
{
$chwrite = $this->checkHash($this->hashURL($domain));
$url = "http://toolbarqueries.google.com/tbr?client=navclient-auto&ch=" . $chwrite . "&features=Rank&q=info:" . $domain . "&num=100&filter=0";
try {
$response = Http::sendHttpRequest($url, $timeout = 10, @$_SERVER['HTTP_USER_AGENT']);
preg_match('#Rank_[0-9]:[0-9]:([0-9]+){1,}#si', $response, $p);
return isset($p[1]) ? $p[1] : null;
} catch (\Exception $e) {
$this->logger->warning('Error while getting Google PageRank for SEO stats: {message}', array('message' => $e->getMessage()));
return null;
}
}
public function getMetrics($domain)
{
try {
$response = Http::sendHttpRequest(self::URL . urlencode($domain), $timeout = 10, @$_SERVER['HTTP_USER_AGENT']);
$xml = @simplexml_load_string($response);
$value = $xml ? (string) $xml->SD->POPULARITY['TEXT'] : null;
} catch (\Exception $e) {
$this->logger->warning('Error while getting Alexa SEO stats: {message}', array('message' => $e->getMessage()));
$value = null;
}
$logo = \Piwik\Plugins\Referrers\getSearchEngineLogoFromUrl('http://alexa.com');
$link = self::LINK . urlencode($domain);
return array(new Metric('alexa', 'SEO_AlexaRank', $value, $logo, $link));
}
public function fetchIndexedPagesCount($domain)
{
$url = self::SEARCH_URL . urlencode($domain);
try {
$response = str_replace(' ', ' ', Http::sendHttpRequest($url, $timeout = 10, @$_SERVER['HTTP_USER_AGENT']));
if (preg_match('#([0-9,\\.]+) results#i', $response, $p)) {
return NumberFormatter::getInstance()->formatNumber((int) str_replace(array(',', '.'), '', $p[1]));
} else {
return 0;
}
} catch (\Exception $e) {
$this->logger->warning('Error while getting Google search SEO stats: {message}', array('message' => $e->getMessage()));
return null;
}
}
private function getAllTlds()
{
/** @var array $response */
$response = \Piwik\Http::sendHttpRequest("http://data.iana.org/TLD/tlds-alpha-by-domain.txt", 30, null, null, null, null, null, true);
$this->assertEquals("200", $response['status']);
$tlds = explode("\n", $response['data']);
foreach ($tlds as $key => $tld) {
if (strpos($tld, '#') !== false || $tld == "") {
unset($tlds[$key]);
}
}
$minimumTlds = 1200;
$this->assertGreaterThan($minimumTlds, count($tlds), "expected to download at least {$minimumTlds} domain names");
return $tlds;
}
/**
* Returns backlink statistics including the count of backlinks and count of
* referrer domains (domains with backlinks).
*
* This method issues an HTTP request and waits for it to return.
*
* @param string $siteDomain The domain of the website to get stats for.
* @param int $timeout The number of seconds to wait before aborting
* the HTTP request.
* @return array An array containing the backlink count and referrer
* domain count:
* array(
* 'backlink_count' => X,
* 'referrer_domains_count' => Y
* )
* If either stat is false, either the API returned an
* error, or the IP was blocked for this request.
*/
public function getBacklinkStats($siteDomain, $timeout = 300)
{
$apiUrl = $this->getApiUrl($method = 'GetBacklinkStats', $args = array('items' => '1', 'item0' => $siteDomain));
$apiResponse = Http::sendHttpRequest($apiUrl, $timeout);
$result = array('backlink_count' => false, 'referrer_domains_count' => false);
$apiResponse = Common::json_decode($apiResponse, $assoc = true);
if (!empty($apiResponse) && !empty($apiResponse['Data'])) {
$siteSeoStats = reset($apiResponse['Data']);
if (isset($siteSeoStats['ExtBackLinks']) && $siteSeoStats['ExtBackLinks'] !== -1) {
$result['backlink_count'] = $siteSeoStats['ExtBackLinks'];
}
if (isset($siteSeoStats['RefDomains']) && $siteSeoStats['RefDomains'] !== -1) {
$result['referrer_domains_count'] = $siteSeoStats['RefDomains'];
}
}
return $result;
}
public function getMetrics($domain)
{
$url = self::URL . urlencode($domain);
try {
$response = str_replace(' ', ' ', Http::sendHttpRequest($url, $timeout = 10, @$_SERVER['HTTP_USER_AGENT']));
if (preg_match('#([0-9,\\.]+) results#i', $response, $p)) {
$pageCount = NumberFormatter::getInstance()->formatNumber((int) str_replace(array(',', '.'), '', $p[1]));
} else {
$pageCount = 0;
}
} catch (\Exception $e) {
$this->logger->warning('Error while getting Bing SEO stats: {message}', array('message' => $e->getMessage()));
$pageCount = null;
}
$logo = SearchEngine::getInstance()->getLogoFromUrl('http://bing.com');
return array(new Metric('bing-index', 'SEO_Bing_IndexedPages', $pageCount, $logo, null, null, 'General_Pages'));
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$branch = $input->getOption('branch');
$buildNumber = $input->getArgument('buildnumber');
$targetDir = PIWIK_INCLUDE_PATH . '/' . dirname($this->targetDir);
$tmpDir = StaticContainer::get('path.tmp');
$this->validate($buildNumber, $targetDir, $tmpDir);
$filename = sprintf('processed.%s.tar.bz2', $buildNumber);
$urlBase = sprintf('http://builds-artifacts.piwik.org/%s/%s/%s', $branch, $buildNumber, $filename);
$tests = Http::sendHttpRequest($urlBase, $timeout = 120);
$tarFile = $tmpDir . $filename;
file_put_contents($tarFile, $tests);
$tar = new Tar($tarFile, 'bz2');
$tar->extract($targetDir);
$this->writeSuccessMessage($output, array('All processed system test results were copied to <comment>' . $this->targetDir . '</comment>', 'Compare them with the expected test results and commit them if needed.'));
unlink($tarFile);
}
public function getMetrics($domain)
{
try {
$response = Http::sendHttpRequest(self::URL . urlencode($domain), $timeout = 10, @$_SERVER['HTTP_USER_AGENT']);
preg_match('#DMOZ Sites[^\\(]+\\([0-9]-[0-9]+ of ([0-9]+)\\)#', $response, $p);
if (!empty($p[1])) {
$value = NumberFormatter::getInstance()->formatNumber((int) $p[1]);
} else {
$value = 0;
}
// Add DMOZ only if > 0 entries found
if ($value == 0) {
return array();
}
} catch (\Exception $e) {
$this->logger->warning('Error while getting Dmoz SEO stats: {message}', array('message' => $e->getMessage()));
$value = null;
}
$logo = SearchEngine::getInstance()->getLogoFromUrl('http://dmoz.org');
return array(new Metric('dmoz', 'SEO_Dmoz', $value, $logo));
}
public function test_WebArchiving()
{
if (self::isMysqli() && self::isTravisCI()) {
$this->markTestSkipped('Skipping on Mysqli as it randomly fails.');
}
$host = Fixture::getRootUrl();
$token = Fixture::getTokenAuth();
$urlTmp = Option::get('piwikUrl');
Option::set('piwikUrl', $host . 'tests/PHPUnit/proxy/index.php');
$url = $host . 'tests/PHPUnit/proxy/archive.php?token_auth=' . $token;
$output = Http::sendHttpRequest($url, 600);
// ignore random build issues
if (empty($output) || strpos($output, \Piwik\CronArchive::NO_ERROR) === false) {
$this->fail("archive web failed: " . $output . "\n\nurl used: {$url}");
}
if (!empty($urlTmp)) {
Option::set('piwikUrl', $urlTmp);
} else {
Option::delete('piwikUrl');
}
$this->assertWebArchivingDone($output);
}
private function downloadScreenshot($url, $repository, $screenshot, $httpUser, $httpPassword)
{
$downloadTo = $this->getDownloadToPath($repository) . $screenshot;
$url = 'http://builds-artifacts.piwik.org' . $url;
$this->logger->debug("Downloading {url} to {destination}", array('url' => $url, 'destination' => $downloadTo));
Http::sendHttpRequest($url, $timeout = 60, $userAgent = null, $downloadTo, $followDepth = 0, $acceptLanguage = false, $byteRange = false, $getExtendedInfo = true, $httpMethod = 'GET', $httpUser, $httpPassword);
}
/**
* Installation Step 6: General Set-up (superuser login/password/email and subscriptions)
*/
function generalSetup()
{
$this->checkPreviousStepIsValid(__FUNCTION__);
$view = new View('@Installation/generalSetup', $this->getInstallationSteps(), __FUNCTION__);
$this->skipThisStep(__FUNCTION__);
$form = new FormGeneralSetup();
if ($form->validate()) {
$superUserInfos = array('login' => $form->getSubmitValue('login'), 'password' => md5($form->getSubmitValue('password')), 'email' => $form->getSubmitValue('email'), 'salt' => Common::generateUniqId());
$this->session->superuser_infos = $superUserInfos;
$url = Config::getInstance()->General['api_service_url'];
$url .= '/1.0/subscribeNewsletter/';
$params = array('email' => $form->getSubmitValue('email'), 'security' => $form->getSubmitValue('subscribe_newsletter_security'), 'community' => $form->getSubmitValue('subscribe_newsletter_community'), 'url' => Url::getCurrentUrlWithoutQueryString());
if ($params['security'] == '1' || $params['community'] == '1') {
if (!isset($params['security'])) {
$params['security'] = '0';
}
if (!isset($params['community'])) {
$params['community'] = '0';
}
$url .= '?' . http_build_query($params, '', '&');
try {
Http::sendHttpRequest($url, $timeout = 2);
} catch (Exception $e) {
// e.g., disable_functions = fsockopen; allow_url_open = Off
}
}
$this->redirectToNextStep(__FUNCTION__);
}
$view->addForm($form);
return $view->render();
}
/**
* @param $email
* @param $newsletterPiwikORG
* @param $newsletterPiwikPRO
*/
protected function registerNewsletter($email, $newsletterPiwikORG, $newsletterPiwikPRO)
{
$url = Config::getInstance()->General['api_service_url'];
$url .= '/1.0/subscribeNewsletter/';
$params = array('email' => $email, 'piwikorg' => $newsletterPiwikORG, 'piwikpro' => $newsletterPiwikPRO, 'url' => Url::getCurrentUrlWithoutQueryString());
if ($params['piwikorg'] == '1' || $params['piwikpro'] == '1') {
if (!isset($params['piwikorg'])) {
$params['piwikorg'] = '0';
}
if (!isset($params['piwikpro'])) {
$params['piwikpro'] = '0';
}
$url .= '?' . http_build_query($params, '', '&');
try {
Http::sendHttpRequest($url, $timeout = 2);
} catch (Exception $e) {
// e.g., disable_functions = fsockopen; allow_url_open = Off
}
}
}
private function fetch($action, $params)
{
ksort($params);
$query = http_build_query($params);
$cacheId = $this->getCacheKey($action, $query);
$cache = $this->buildCache();
$result = $cache->fetch($cacheId);
if (false === $result) {
$endpoint = $this->domain . '/api/1.0/';
$url = sprintf('%s%s?%s', $endpoint, $action, $query);
$response = Http::sendHttpRequest($url, static::HTTP_REQUEST_TIMEOUT);
$result = json_decode($response, true);
if (is_null($result)) {
$message = sprintf('There was an error reading the response from the Marketplace: %s. Please try again later.', substr($response, 0, 50));
throw new MarketplaceApiException($message);
}
if (!empty($result['error'])) {
throw new MarketplaceApiException($result['error']);
}
$cache->save($cacheId, $result, self::CACHE_TIMEOUT_IN_SECONDS);
}
return $result;
}
private function downloadProcessedScreenshot(OutputInterface $output, $urlBase, $file, $plugin, $httpUser, $httpPassword)
{
$downloadTo = $this->getDownloadToPath($plugin) . $file;
$output->write("<info>Downloading {$file} to {$downloadTo}...</info>\n");
$screenshotUrl = $urlBase . "/processed-ui-screenshots/{$file}";
$this->logger->debug('Downloading {url}', array('url' => $screenshotUrl));
Http::sendHttpRequest($screenshotUrl, $timeout = 60, $userAgent = null, $downloadTo, $followDepth = 0, $acceptLanguage = false, $byteRange = false, $getExtendedInfo = true, $httpMethod = 'GET', $httpUser, $httpPassword);
}
/**
* Downloads the next chunk of a specific file. The next chunk's byte range
* is determined by the existing file's size and the expected file size, which
* is stored in the piwik_option table before starting a download. The expected
* file size is obtained through a `HEAD` HTTP request.
*
* _Note: this function uses the **Range** HTTP header to accomplish downloading in
* parts. Not every server supports this header._
*
* The proper use of this function is to call it once per request. The browser
* should continue to send requests to Piwik which will in turn call this method
* until the file has completely downloaded. In this way, the user can be informed
* of a download's progress.
*
* **Example Usage**
*
* ```
* // browser JavaScript
* var downloadFile = function (isStart) {
* var ajax = new ajaxHelper();
* ajax.addParams({
* module: 'MyPlugin',
* action: 'myAction',
* isStart: isStart ? 1 : 0
* }, 'post');
* ajax.setCallback(function (response) {
* var progress = response.progress
* // ...update progress...
*
* downloadFile(false);
* });
* ajax.send();
* }
*
* downloadFile(true);
* ```
*
* ```
* // PHP controller action
* public function myAction()
* {
* $outputPath = PIWIK_INCLUDE_PATH . '/tmp/averybigfile.zip';
* $isStart = Common::getRequestVar('isStart', 1, 'int');
* Http::downloadChunk("http://bigfiles.com/averybigfile.zip", $outputPath, $isStart == 1);
* }
* ```
*
* @param string $url The url to download from.
* @param string $outputPath The path to the file to save/append to.
* @param bool $isContinuation `true` if this is the continuation of a download,
* or if we're starting a fresh one.
* @throws Exception if the file already exists and we're starting a new download,
* if we're trying to continue a download that never started
* @return array
* @api
*/
public static function downloadChunk($url, $outputPath, $isContinuation)
{
// make sure file doesn't already exist if we're starting a new download
if (!$isContinuation && file_exists($outputPath)) {
throw new Exception(Piwik::translate('General_DownloadFail_FileExists', "'" . $outputPath . "'") . ' ' . Piwik::translate('General_DownloadPleaseRemoveExisting'));
}
// if we're starting a download, get the expected file size & save as an option
$downloadOption = $outputPath . '_expectedDownloadSize';
if (!$isContinuation) {
$expectedFileSizeResult = Http::sendHttpRequest($url, $timeout = 300, $userAgent = null, $destinationPath = null, $followDepth = 0, $acceptLanguage = false, $byteRange = false, $getExtendedInfo = true, $httpMethod = 'HEAD');
$expectedFileSize = 0;
if (isset($expectedFileSizeResult['headers']['Content-Length'])) {
$expectedFileSize = (int) $expectedFileSizeResult['headers']['Content-Length'];
}
if ($expectedFileSize == 0) {
Log::info("HEAD request for '%s' failed, got following: %s", $url, print_r($expectedFileSizeResult, true));
throw new Exception(Piwik::translate('General_DownloadFail_HttpRequestFail'));
}
Option::set($downloadOption, $expectedFileSize);
} else {
$expectedFileSize = (int) Option::get($downloadOption);
if ($expectedFileSize === false) {
// sanity check
throw new Exception("Trying to continue a download that never started?! That's not supposed to happen...");
}
}
// if existing file is already big enough, then fail so we don't accidentally overwrite
// existing DB
$existingSize = file_exists($outputPath) ? filesize($outputPath) : 0;
if ($existingSize >= $expectedFileSize) {
throw new Exception(Piwik::translate('General_DownloadFail_FileExistsContinue', "'" . $outputPath . "'") . ' ' . Piwik::translate('General_DownloadPleaseRemoveExisting'));
}
// download a chunk of the file
$result = Http::sendHttpRequest($url, $timeout = 300, $userAgent = null, $destinationPath = null, $followDepth = 0, $acceptLanguage = false, $byteRange = array($existingSize, min($existingSize + 1024 * 1024 - 1, $expectedFileSize)), $getExtendedInfo = true);
if ($result === false || $result['status'] < 200 || $result['status'] > 299) {
$result['data'] = self::truncateStr($result['data'], 1024);
Log::info("Failed to download range '%s-%s' of file from url '%s'. Got result: %s", $byteRange[0], $byteRange[1], $url, print_r($result, true));
throw new Exception(Piwik::translate('General_DownloadFail_HttpRequestFail'));
}
// write chunk to file
$f = fopen($outputPath, 'ab');
fwrite($f, $result['data']);
fclose($f);
clearstatcache($clear_realpath_cache = true, $outputPath);
return array('current_size' => filesize($outputPath), 'expected_file_size' => $expectedFileSize);
}
/**
* Update the referrer spam blacklist
*
* @see https://github.com/piwik/referrer-spam-blacklist
*/
public function updateSpammerBlacklist()
{
$url = 'https://raw.githubusercontent.com/piwik/referrer-spam-blacklist/master/spammers.txt';
$list = Http::sendHttpRequest($url, 30);
$list = preg_split("/\r\n|\n|\r/", $list);
if (count($list) < 10) {
throw new \Exception(sprintf('The spammers list downloaded from %s contains less than 10 entries, considering it a fail', $url));
}
Option::set(ReferrerSpamFilter::OPTION_STORAGE_NAME, serialize($list));
}
/**
* Downloads a GeoIP database archive, extracts the .dat file and overwrites the existing
* old database.
*
* If something happens that causes the download to fail, no exception is thrown, but
* an error is logged.
*
* @param string $dbType
* @param string $url URL to the database to download. The type of database is determined
* from this URL.
* @throws Exception
*/
protected function downloadFile($dbType, $url)
{
$ext = GeoIPAutoUpdater::getGeoIPUrlExtension($url);
// NOTE: using the first item in $dbNames[$dbType] makes sure GeoLiteCity will be renamed to GeoIPCity
$zippedFilename = GeoIp::$dbNames[$dbType][0] . '.' . $ext;
$zippedOutputPath = GeoIp::getPathForGeoIpDatabase($zippedFilename);
$url = self::removeDateFromUrl($url);
// download zipped file to misc dir
try {
$success = Http::sendHttpRequest($url, $timeout = 3600, $userAgent = null, $zippedOutputPath);
} catch (Exception $ex) {
throw new Exception("GeoIPAutoUpdater: failed to download '{$url}' to " . "'{$zippedOutputPath}': " . $ex->getMessage());
}
if ($success !== true) {
throw new Exception("GeoIPAutoUpdater: failed to download '{$url}' to " . "'{$zippedOutputPath}'! (Unknown error)");
}
Log::info("GeoIPAutoUpdater: successfully downloaded '%s'", $url);
try {
self::unzipDownloadedFile($zippedOutputPath, $unlink = true);
} catch (Exception $ex) {
throw new Exception("GeoIPAutoUpdater: failed to unzip '{$zippedOutputPath}' after " . "downloading " . "'{$url}': " . $ex->getMessage());
}
Log::info("GeoIPAutoUpdater: successfully updated GeoIP database '%s'", $url);
}
/**
* Returns response for API request with given path
*
* @param $apiPath
* @param bool $raw
* @return mixed
* @throws AuthenticationFailedException
* @throws Exception
*/
protected function getApiResults($apiPath, $raw = false)
{
$apiUrl = $this->apiUrl . $apiPath;
$response = Http::sendHttpRequest($apiUrl, 1000, null, null, 5, false, false, true, 'GET', $this->username, $this->password);
$httpStatus = $response['status'];
$response = $response['data'];
if ($httpStatus == 401) {
throw new AuthenticationFailedException();
} else {
if ($httpStatus != 200) {
throw new Exception('Error while getting API results', $httpStatus);
}
}
return $raw ? $response : json_decode($response);
}
private function getUrl($url)
{
try {
return str_replace(' ', ' ', Http::sendHttpRequest($url, $timeout = 10, @$_SERVER['HTTP_USER_AGENT']));
} catch (\Exception $e) {
$this->logger->warning('Error while getting SEO stats (domain age): {message}', array('message' => $e->getMessage()));
return '';
}
}
/**
* @param $requestUrl
* @return string
* @throws Exception
*/
protected function getResponseFromHttpAPI($requestUrl)
{
$queryString = Url::getQueryStringFromParameters($requestUrl);
$hostAndPath = Fixture::getTestRootUrl();
$url = $hostAndPath . '?' . $queryString;
$response = Http::sendHttpRequest($url, $timeout = 300);
return $response;
}
