sendHttpRequest() public static method

Sends an HTTP request using best available transport method.
public static sendHttpRequest ( string $aUrl, integer $timeout, string | null $userAgent = null, string | null $destinationPath = null, integer | null $followDepth, boolean $acceptLanguage = false, array | boolean $byteRange = false, boolean $getExtendedInfo = false, string $httpMethod = 'GET', string $httpUsername = null, string $httpPassword = null ) : boolean | string
$aUrl string The target URL.
$timeout integer The number of seconds to wait before aborting the HTTP request.
$userAgent string | null The user agent to use.
$destinationPath string | null If supplied, the HTTP response will be saved to the file specified by this path.
$followDepth integer | null Internal redirect count. Should always pass `null` for this parameter.
$acceptLanguage boolean The value to use for the `'Accept-Language'` HTTP request header.
$byteRange array | boolean For `Range:` header. Should be two element array of bytes, eg, `array(0, 1024)` Doesn't work w/ `fopen` transport method.
$getExtendedInfo boolean If true returns the status code, headers & response, if false just the response.
$httpMethod string The HTTP method to use. Defaults to `'GET'`.
$httpUsername string HTTP Auth username
$httpPassword string HTTP Auth password
return boolean | string If `$destinationPath` is not specified the HTTP response is returned on success. `false` is returned on failure. If `$getExtendedInfo` is `true` and `$destinationPath` is not specified an array with the following information is returned on success: - **status**: the HTTP status code - **headers**: the HTTP headers - **data**: the HTTP response data `false` is still returned on failure.
Esempio n. 1
0
 public function testWebArchiving()
 {
     if (self::isMysqli() && self::isTravisCI()) {
         $this->markTestSkipped('Skipping on Mysqli as it randomly fails.');
     }
     $host = Fixture::getRootUrl();
     $token = Fixture::getTokenAuth();
     $urlTmp = Option::get('piwikUrl');
     Option::set('piwikUrl', $host . 'tests/PHPUnit/proxy/index.php');
     $url = $host . 'tests/PHPUnit/proxy/archive.php?token_auth=' . $token;
     $output = Http::sendHttpRequest($url, 600);
     // ignore random build issues
     if (empty($output) || strpos($output, \Piwik\CronArchive::NO_ERROR) === false) {
         $message = "This test has failed. Because it sometimes randomly fails, we skip the test, and ignore this failure.\n";
         $message .= "If you see this message often, or in every build, please investigate as this should only be a random and rare occurence!\n";
         $message .= "\n\narchive web failed: " . $output . "\n\nurl used: {$url}";
         $this->markTestSkipped($message);
     }
     if (!empty($urlTmp)) {
         Option::set('piwikUrl', $urlTmp);
     } else {
         Option::delete('piwikUrl');
     }
     $this->assertContains('Starting Piwik reports archiving...', $output);
     $this->assertContains('Archived website id = 1', $output);
     $this->assertContains('Done archiving!', $output);
     $this->compareArchivePhpOutputAgainstExpected($output);
 }
Esempio n. 2
0
 public function triggerWebtreesAdminTasks()
 {
     $settings = new Settings();
     $this->logger = \Piwik\Container\StaticContainer::get('Psr\\Log\\LoggerInterface');
     $this->logger->info('Webtrees Admin Task triggered');
     $rooturl = $settings->getSetting('webtreesRootUrl');
     if (!$rooturl || strlen($rooturl->getValue()) === 0) {
         return;
     }
     $token = $settings->getSetting('webtreesToken');
     if (!$token || strlen($token->getValue()) === 0) {
         return;
     }
     $taskname = $settings->getSetting('webtreesTaskName');
     if (!$taskname || strlen($taskname->getValue()) === 0) {
         return;
     }
     $url = sprintf('%1$s/module.php?mod=perso_admintasks&mod_action=trigger&force=%2$s&task=%3$s', $rooturl->getValue(), $token->getValue(), $taskname->getValue());
     $this->logger->info('webtrees url : {url}', array('url' => $url));
     try {
         \Piwik\Http::sendHttpRequest($url, Webtrees::SOCKET_TIMEOUT);
     } catch (Exception $e) {
         $this->logger->warning('an error occured', array('exception' => $e));
     }
 }
 protected function execute(InputInterface $input, OutputInterface $output)
 {
     $buildNumber = $input->getArgument('buildnumber');
     $screenshotsRegex = $input->getArgument('screenshotsRegex');
     if (empty($buildNumber)) {
         throw new \InvalidArgumentException('Missing build number.');
     }
     $urlBase = sprintf('http://builds-artifacts.piwik.org/ui-tests.master/%s', $buildNumber);
     $diffviewer = Http::sendHttpRequest($urlBase . "/screenshot-diffs/diffviewer.html", $timeout = 60);
     $diffviewer = str_replace('&', '&', $diffviewer);
     $dom = new \DOMDocument();
     $dom->loadHTML($diffviewer);
     foreach ($dom->getElementsByTagName("tr") as $row) {
         $columns = $row->getElementsByTagName("td");
         $nameColumn = $columns->item(0);
         $processedColumn = $columns->item(3);
         $testPlugin = null;
         if ($nameColumn && preg_match("/\\(for ([a-zA-Z_]+) plugin\\)/", $dom->saveXml($nameColumn), $matches)) {
             $testPlugin = $matches[1];
         }
         $file = null;
         if ($processedColumn && preg_match("/href=\".*\\/(.*)\"/", $dom->saveXml($processedColumn), $matches)) {
             $file = $matches[1];
         }
         if ($file !== null && preg_match("/" . $screenshotsRegex . "/", $file)) {
             if ($testPlugin == null) {
                 $downloadTo = "tests/PHPUnit/UI/expected-ui-screenshots/{$file}";
             } else {
                 $downloadTo = "plugins/{$testPlugin}/tests/UI/expected-ui-screenshots/{$file}";
             }
             $output->write("<info>Downloading {$file} to .{$downloadTo}...</info>\n");
             Http::sendHttpRequest("{$urlBase}/processed-ui-screenshots/{$file}", $timeout = 60, $userAgent = null, PIWIK_DOCUMENT_ROOT . "/" . $downloadTo);
         }
     }
 }
Esempio n. 4
0
 /**
  * Check for a newer version
  *
  * @param bool $force Force check
  * @param int $interval Interval used for update checks
  */
 public static function check($force = false, $interval = null)
 {
     if (!self::isAutoUpdateEnabled()) {
         return;
     }
     if ($interval === null) {
         $interval = self::CHECK_INTERVAL;
     }
     $lastTimeChecked = Option::get(self::LAST_TIME_CHECKED);
     if ($force || $lastTimeChecked === false || time() - $interval > $lastTimeChecked) {
         // set the time checked first, so that parallel Piwik requests don't all trigger the http requests
         Option::set(self::LAST_TIME_CHECKED, time(), $autoLoad = 1);
         $parameters = array('piwik_version' => Version::VERSION, 'php_version' => PHP_VERSION, 'url' => Url::getCurrentUrlWithoutQueryString(), 'trigger' => Common::getRequestVar('module', '', 'string'), 'timezone' => API::getInstance()->getDefaultTimezone());
         $url = Config::getInstance()->General['api_service_url'] . '/1.0/getLatestVersion/' . '?' . http_build_query($parameters, '', '&');
         $timeout = self::SOCKET_TIMEOUT;
         if (@Config::getInstance()->Debug['allow_upgrades_to_beta']) {
             $url = 'http://builds.piwik.org/LATEST_BETA';
         }
         try {
             $latestVersion = Http::sendHttpRequest($url, $timeout);
             if (!preg_match('~^[0-9][0-9a-zA-Z_.-]*$~D', $latestVersion)) {
                 $latestVersion = '';
             }
         } catch (Exception $e) {
             // e.g., disable_functions = fsockopen; allow_url_open = Off
             $latestVersion = '';
         }
         Option::set(self::LATEST_VERSION, $latestVersion);
     }
 }
Esempio n. 5
0
 /**
  * Web service proxy that retrieves the content at the specified URL
  *
  * @param string $url
  * @return string
  */
 private function getPage($url)
 {
     try {
         return str_replace('&nbsp;', ' ', Http::sendHttpRequest($url, $timeout = 10, @$_SERVER['HTTP_USER_AGENT']));
     } catch (Exception $e) {
         return '';
     }
 }
Esempio n. 6
0
 /**
  * Update the social definitions
  *
  * @see https://github.com/piwik/searchengine-and-social-list
  */
 public function updateSocials()
 {
     $url = 'https://raw.githubusercontent.com/piwik/searchengine-and-social-list/master/Socials.yml';
     $list = Http::sendHttpRequest($url, 30);
     $socials = Social::getInstance()->loadYmlData($list);
     if (count($socials) < 50) {
         return;
     }
     Option::set(Social::OPTION_STORAGE_NAME, base64_encode(serialize($socials)));
 }
Esempio n. 7
0
 /**
  * Get the latest available version number for the currently active release channel. Eg '2.15.0-b4' or '2.15.0'.
  * Should return a semantic version number in format MAJOR.MINOR.PATCH (http://semver.org/).
  * Returns an empty string in case one cannot connect to the remote server.
  * @return string
  */
 private static function getLatestAvailableVersionNumber()
 {
     $channel = StaticContainer::get('\\Piwik\\Plugin\\ReleaseChannels')->getActiveReleaseChannel();
     $url = $channel->getUrlToCheckForLatestAvailableVersion();
     try {
         $latestVersion = Http::sendHttpRequest($url, self::SOCKET_TIMEOUT);
     } catch (\Exception $e) {
         // e.g., disable_functions = fsockopen; allow_url_open = Off
         $latestVersion = '';
     }
     return $latestVersion;
 }
Esempio n. 8
0
 private function isPageSpeedEnabled()
 {
     $url = Url::getCurrentUrlWithoutQueryString() . '?module=Installation&action=getEmptyPageForSystemCheck';
     try {
         $page = Http::sendHttpRequest($url, $timeout = 1, $userAgent = null, $destinationPath = null, $followDepth = 0, $acceptLanguage = false, $byteRange = false, $getExtendedInfo = true);
     } catch (\Exception $e) {
         $this->logger->info('Unable to test if mod_pagespeed is enabled: the request to {url} failed', array('url' => $url));
         // If the test failed, we assume Page speed is not enabled
         return false;
     }
     $headers = $page['headers'];
     return isset($headers['X-Mod-Pagespeed']) || isset($headers['X-Page-Speed']);
 }
Esempio n. 9
0
 private function getAllTlds()
 {
     /** @var array $response */
     $response = \Piwik\Http::sendHttpRequest("http://data.iana.org/TLD/tlds-alpha-by-domain.txt", 30, null, null, null, null, null, true);
     $this->assertEquals("200", $response['status']);
     $tlds = explode("\n", $response['data']);
     foreach ($tlds as $key => $tld) {
         if (strpos($tld, '#') !== false || $tld == "") {
             unset($tlds[$key]);
         }
     }
     return $tlds;
 }
Esempio n. 10
0
 public function fetchPageRank($domain)
 {
     $chwrite = $this->checkHash($this->hashURL($domain));
     $url = "http://toolbarqueries.google.com/tbr?client=navclient-auto&ch=" . $chwrite . "&features=Rank&q=info:" . $domain . "&num=100&filter=0";
     try {
         $response = Http::sendHttpRequest($url, $timeout = 10, @$_SERVER['HTTP_USER_AGENT']);
         preg_match('#Rank_[0-9]:[0-9]:([0-9]+){1,}#si', $response, $p);
         return isset($p[1]) ? $p[1] : null;
     } catch (\Exception $e) {
         $this->logger->warning('Error while getting Google PageRank for SEO stats: {message}', array('message' => $e->getMessage()));
         return null;
     }
 }
Esempio n. 11
0
 public function getMetrics($domain)
 {
     try {
         $response = Http::sendHttpRequest(self::URL . urlencode($domain), $timeout = 10, @$_SERVER['HTTP_USER_AGENT']);
         $xml = @simplexml_load_string($response);
         $value = $xml ? (string) $xml->SD->POPULARITY['TEXT'] : null;
     } catch (\Exception $e) {
         $this->logger->warning('Error while getting Alexa SEO stats: {message}', array('message' => $e->getMessage()));
         $value = null;
     }
     $logo = \Piwik\Plugins\Referrers\getSearchEngineLogoFromUrl('http://alexa.com');
     $link = self::LINK . urlencode($domain);
     return array(new Metric('alexa', 'SEO_AlexaRank', $value, $logo, $link));
 }
Esempio n. 12
0
 public function fetchIndexedPagesCount($domain)
 {
     $url = self::SEARCH_URL . urlencode($domain);
     try {
         $response = str_replace('&nbsp;', ' ', Http::sendHttpRequest($url, $timeout = 10, @$_SERVER['HTTP_USER_AGENT']));
         if (preg_match('#([0-9,\\.]+) results#i', $response, $p)) {
             return NumberFormatter::getInstance()->formatNumber((int) str_replace(array(',', '.'), '', $p[1]));
         } else {
             return 0;
         }
     } catch (\Exception $e) {
         $this->logger->warning('Error while getting Google search SEO stats: {message}', array('message' => $e->getMessage()));
         return null;
     }
 }
Esempio n. 13
0
 private function getAllTlds()
 {
     /** @var array $response */
     $response = \Piwik\Http::sendHttpRequest("http://data.iana.org/TLD/tlds-alpha-by-domain.txt", 30, null, null, null, null, null, true);
     $this->assertEquals("200", $response['status']);
     $tlds = explode("\n", $response['data']);
     foreach ($tlds as $key => $tld) {
         if (strpos($tld, '#') !== false || $tld == "") {
             unset($tlds[$key]);
         }
     }
     $minimumTlds = 1200;
     $this->assertGreaterThan($minimumTlds, count($tlds), "expected to download at least {$minimumTlds} domain names");
     return $tlds;
 }
Esempio n. 14
0
 /**
  * Returns backlink statistics including the count of backlinks and count of
  * referrer domains (domains with backlinks).
  *
  * This method issues an HTTP request and waits for it to return.
  *
  * @param string $siteDomain The domain of the website to get stats for.
  * @param int $timeout The number of seconds to wait before aborting
  *                     the HTTP request.
  * @return array An array containing the backlink count and referrer
  *               domain count:
  *               array(
  *                   'backlink_count' => X,
  *                   'referrer_domains_count' => Y
  *               )
  *               If either stat is false, either the API returned an
  *               error, or the IP was blocked for this request.
  */
 public function getBacklinkStats($siteDomain, $timeout = 300)
 {
     $apiUrl = $this->getApiUrl($method = 'GetBacklinkStats', $args = array('items' => '1', 'item0' => $siteDomain));
     $apiResponse = Http::sendHttpRequest($apiUrl, $timeout);
     $result = array('backlink_count' => false, 'referrer_domains_count' => false);
     $apiResponse = Common::json_decode($apiResponse, $assoc = true);
     if (!empty($apiResponse) && !empty($apiResponse['Data'])) {
         $siteSeoStats = reset($apiResponse['Data']);
         if (isset($siteSeoStats['ExtBackLinks']) && $siteSeoStats['ExtBackLinks'] !== -1) {
             $result['backlink_count'] = $siteSeoStats['ExtBackLinks'];
         }
         if (isset($siteSeoStats['RefDomains']) && $siteSeoStats['RefDomains'] !== -1) {
             $result['referrer_domains_count'] = $siteSeoStats['RefDomains'];
         }
     }
     return $result;
 }
Esempio n. 15
0
File: Bing.php Progetto: piwik/piwik
 public function getMetrics($domain)
 {
     $url = self::URL . urlencode($domain);
     try {
         $response = str_replace('&nbsp;', ' ', Http::sendHttpRequest($url, $timeout = 10, @$_SERVER['HTTP_USER_AGENT']));
         if (preg_match('#([0-9,\\.]+) results#i', $response, $p)) {
             $pageCount = NumberFormatter::getInstance()->formatNumber((int) str_replace(array(',', '.'), '', $p[1]));
         } else {
             $pageCount = 0;
         }
     } catch (\Exception $e) {
         $this->logger->warning('Error while getting Bing SEO stats: {message}', array('message' => $e->getMessage()));
         $pageCount = null;
     }
     $logo = SearchEngine::getInstance()->getLogoFromUrl('http://bing.com');
     return array(new Metric('bing-index', 'SEO_Bing_IndexedPages', $pageCount, $logo, null, null, 'General_Pages'));
 }
 protected function execute(InputInterface $input, OutputInterface $output)
 {
     $branch = $input->getOption('branch');
     $buildNumber = $input->getArgument('buildnumber');
     $targetDir = PIWIK_INCLUDE_PATH . '/' . dirname($this->targetDir);
     $tmpDir = StaticContainer::get('path.tmp');
     $this->validate($buildNumber, $targetDir, $tmpDir);
     $filename = sprintf('processed.%s.tar.bz2', $buildNumber);
     $urlBase = sprintf('http://builds-artifacts.piwik.org/%s/%s/%s', $branch, $buildNumber, $filename);
     $tests = Http::sendHttpRequest($urlBase, $timeout = 120);
     $tarFile = $tmpDir . $filename;
     file_put_contents($tarFile, $tests);
     $tar = new Tar($tarFile, 'bz2');
     $tar->extract($targetDir);
     $this->writeSuccessMessage($output, array('All processed system test results were copied to <comment>' . $this->targetDir . '</comment>', 'Compare them with the expected test results and commit them if needed.'));
     unlink($tarFile);
 }
Esempio n. 17
0
 public function getMetrics($domain)
 {
     try {
         $response = Http::sendHttpRequest(self::URL . urlencode($domain), $timeout = 10, @$_SERVER['HTTP_USER_AGENT']);
         preg_match('#DMOZ Sites[^\\(]+\\([0-9]-[0-9]+ of ([0-9]+)\\)#', $response, $p);
         if (!empty($p[1])) {
             $value = NumberFormatter::getInstance()->formatNumber((int) $p[1]);
         } else {
             $value = 0;
         }
         // Add DMOZ only if > 0 entries found
         if ($value == 0) {
             return array();
         }
     } catch (\Exception $e) {
         $this->logger->warning('Error while getting Dmoz SEO stats: {message}', array('message' => $e->getMessage()));
         $value = null;
     }
     $logo = SearchEngine::getInstance()->getLogoFromUrl('http://dmoz.org');
     return array(new Metric('dmoz', 'SEO_Dmoz', $value, $logo));
 }
Esempio n. 18
0
 public function test_WebArchiving()
 {
     if (self::isMysqli() && self::isTravisCI()) {
         $this->markTestSkipped('Skipping on Mysqli as it randomly fails.');
     }
     $host = Fixture::getRootUrl();
     $token = Fixture::getTokenAuth();
     $urlTmp = Option::get('piwikUrl');
     Option::set('piwikUrl', $host . 'tests/PHPUnit/proxy/index.php');
     $url = $host . 'tests/PHPUnit/proxy/archive.php?token_auth=' . $token;
     $output = Http::sendHttpRequest($url, 600);
     // ignore random build issues
     if (empty($output) || strpos($output, \Piwik\CronArchive::NO_ERROR) === false) {
         $this->fail("archive web failed: " . $output . "\n\nurl used: {$url}");
     }
     if (!empty($urlTmp)) {
         Option::set('piwikUrl', $urlTmp);
     } else {
         Option::delete('piwikUrl');
     }
     $this->assertWebArchivingDone($output);
 }
Esempio n. 19
0
 private function downloadScreenshot($url, $repository, $screenshot, $httpUser, $httpPassword)
 {
     $downloadTo = $this->getDownloadToPath($repository) . $screenshot;
     $url = 'http://builds-artifacts.piwik.org' . $url;
     $this->logger->debug("Downloading {url} to {destination}", array('url' => $url, 'destination' => $downloadTo));
     Http::sendHttpRequest($url, $timeout = 60, $userAgent = null, $downloadTo, $followDepth = 0, $acceptLanguage = false, $byteRange = false, $getExtendedInfo = true, $httpMethod = 'GET', $httpUser, $httpPassword);
 }
Esempio n. 20
0
 /**
  * Installation Step 6: General Set-up (superuser login/password/email and subscriptions)
  */
 function generalSetup()
 {
     $this->checkPreviousStepIsValid(__FUNCTION__);
     $view = new View('@Installation/generalSetup', $this->getInstallationSteps(), __FUNCTION__);
     $this->skipThisStep(__FUNCTION__);
     $form = new FormGeneralSetup();
     if ($form->validate()) {
         $superUserInfos = array('login' => $form->getSubmitValue('login'), 'password' => md5($form->getSubmitValue('password')), 'email' => $form->getSubmitValue('email'), 'salt' => Common::generateUniqId());
         $this->session->superuser_infos = $superUserInfos;
         $url = Config::getInstance()->General['api_service_url'];
         $url .= '/1.0/subscribeNewsletter/';
         $params = array('email' => $form->getSubmitValue('email'), 'security' => $form->getSubmitValue('subscribe_newsletter_security'), 'community' => $form->getSubmitValue('subscribe_newsletter_community'), 'url' => Url::getCurrentUrlWithoutQueryString());
         if ($params['security'] == '1' || $params['community'] == '1') {
             if (!isset($params['security'])) {
                 $params['security'] = '0';
             }
             if (!isset($params['community'])) {
                 $params['community'] = '0';
             }
             $url .= '?' . http_build_query($params, '', '&');
             try {
                 Http::sendHttpRequest($url, $timeout = 2);
             } catch (Exception $e) {
                 // e.g., disable_functions = fsockopen; allow_url_open = Off
             }
         }
         $this->redirectToNextStep(__FUNCTION__);
     }
     $view->addForm($form);
     return $view->render();
 }
Esempio n. 21
0
 /**
  * @param $email
  * @param $newsletterPiwikORG
  * @param $newsletterPiwikPRO
  */
 protected function registerNewsletter($email, $newsletterPiwikORG, $newsletterPiwikPRO)
 {
     $url = Config::getInstance()->General['api_service_url'];
     $url .= '/1.0/subscribeNewsletter/';
     $params = array('email' => $email, 'piwikorg' => $newsletterPiwikORG, 'piwikpro' => $newsletterPiwikPRO, 'url' => Url::getCurrentUrlWithoutQueryString());
     if ($params['piwikorg'] == '1' || $params['piwikpro'] == '1') {
         if (!isset($params['piwikorg'])) {
             $params['piwikorg'] = '0';
         }
         if (!isset($params['piwikpro'])) {
             $params['piwikpro'] = '0';
         }
         $url .= '?' . http_build_query($params, '', '&');
         try {
             Http::sendHttpRequest($url, $timeout = 2);
         } catch (Exception $e) {
             // e.g., disable_functions = fsockopen; allow_url_open = Off
         }
     }
 }
 private function fetch($action, $params)
 {
     ksort($params);
     $query = http_build_query($params);
     $cacheId = $this->getCacheKey($action, $query);
     $cache = $this->buildCache();
     $result = $cache->fetch($cacheId);
     if (false === $result) {
         $endpoint = $this->domain . '/api/1.0/';
         $url = sprintf('%s%s?%s', $endpoint, $action, $query);
         $response = Http::sendHttpRequest($url, static::HTTP_REQUEST_TIMEOUT);
         $result = json_decode($response, true);
         if (is_null($result)) {
             $message = sprintf('There was an error reading the response from the Marketplace: %s. Please try again later.', substr($response, 0, 50));
             throw new MarketplaceApiException($message);
         }
         if (!empty($result['error'])) {
             throw new MarketplaceApiException($result['error']);
         }
         $cache->save($cacheId, $result, self::CACHE_TIMEOUT_IN_SECONDS);
     }
     return $result;
 }
 private function downloadProcessedScreenshot(OutputInterface $output, $urlBase, $file, $plugin, $httpUser, $httpPassword)
 {
     $downloadTo = $this->getDownloadToPath($plugin) . $file;
     $output->write("<info>Downloading {$file} to  {$downloadTo}...</info>\n");
     $screenshotUrl = $urlBase . "/processed-ui-screenshots/{$file}";
     $this->logger->debug('Downloading {url}', array('url' => $screenshotUrl));
     Http::sendHttpRequest($screenshotUrl, $timeout = 60, $userAgent = null, $downloadTo, $followDepth = 0, $acceptLanguage = false, $byteRange = false, $getExtendedInfo = true, $httpMethod = 'GET', $httpUser, $httpPassword);
 }
Esempio n. 24
0
 /**
  * Downloads the next chunk of a specific file. The next chunk's byte range
  * is determined by the existing file's size and the expected file size, which
  * is stored in the piwik_option table before starting a download. The expected
  * file size is obtained through a `HEAD` HTTP request.
  *
  * _Note: this function uses the **Range** HTTP header to accomplish downloading in
  * parts. Not every server supports this header._
  *
  * The proper use of this function is to call it once per request. The browser
  * should continue to send requests to Piwik which will in turn call this method
  * until the file has completely downloaded. In this way, the user can be informed
  * of a download's progress.
  *
  * **Example Usage**
  *
  * ```
  * // browser JavaScript
  * var downloadFile = function (isStart) {
  *     var ajax = new ajaxHelper();
  *     ajax.addParams({
  *         module: 'MyPlugin',
  *         action: 'myAction',
  *         isStart: isStart ? 1 : 0
  *     }, 'post');
  *     ajax.setCallback(function (response) {
  *         var progress = response.progress
  *         // ...update progress...
  *
  *         downloadFile(false);
  *     });
  *     ajax.send();
  * }
  *
  * downloadFile(true);
  * ```
  *
  * ```
  * // PHP controller action
  * public function myAction()
  * {
  *     $outputPath = PIWIK_INCLUDE_PATH . '/tmp/averybigfile.zip';
  *     $isStart = Common::getRequestVar('isStart', 1, 'int');
  *     Http::downloadChunk("http://bigfiles.com/averybigfile.zip", $outputPath, $isStart == 1);
  * }
  * ```
  *
  * @param string $url The url to download from.
  * @param string $outputPath The path to the file to save/append to.
  * @param bool $isContinuation `true` if this is the continuation of a download,
  *                             or if we're starting a fresh one.
  * @throws Exception if the file already exists and we're starting a new download,
  *                   if we're trying to continue a download that never started
  * @return array
  * @api
  */
 public static function downloadChunk($url, $outputPath, $isContinuation)
 {
     // make sure file doesn't already exist if we're starting a new download
     if (!$isContinuation && file_exists($outputPath)) {
         throw new Exception(Piwik::translate('General_DownloadFail_FileExists', "'" . $outputPath . "'") . ' ' . Piwik::translate('General_DownloadPleaseRemoveExisting'));
     }
     // if we're starting a download, get the expected file size & save as an option
     $downloadOption = $outputPath . '_expectedDownloadSize';
     if (!$isContinuation) {
         $expectedFileSizeResult = Http::sendHttpRequest($url, $timeout = 300, $userAgent = null, $destinationPath = null, $followDepth = 0, $acceptLanguage = false, $byteRange = false, $getExtendedInfo = true, $httpMethod = 'HEAD');
         $expectedFileSize = 0;
         if (isset($expectedFileSizeResult['headers']['Content-Length'])) {
             $expectedFileSize = (int) $expectedFileSizeResult['headers']['Content-Length'];
         }
         if ($expectedFileSize == 0) {
             Log::info("HEAD request for '%s' failed, got following: %s", $url, print_r($expectedFileSizeResult, true));
             throw new Exception(Piwik::translate('General_DownloadFail_HttpRequestFail'));
         }
         Option::set($downloadOption, $expectedFileSize);
     } else {
         $expectedFileSize = (int) Option::get($downloadOption);
         if ($expectedFileSize === false) {
             // sanity check
             throw new Exception("Trying to continue a download that never started?! That's not supposed to happen...");
         }
     }
     // if existing file is already big enough, then fail so we don't accidentally overwrite
     // existing DB
     $existingSize = file_exists($outputPath) ? filesize($outputPath) : 0;
     if ($existingSize >= $expectedFileSize) {
         throw new Exception(Piwik::translate('General_DownloadFail_FileExistsContinue', "'" . $outputPath . "'") . ' ' . Piwik::translate('General_DownloadPleaseRemoveExisting'));
     }
     // download a chunk of the file
     $result = Http::sendHttpRequest($url, $timeout = 300, $userAgent = null, $destinationPath = null, $followDepth = 0, $acceptLanguage = false, $byteRange = array($existingSize, min($existingSize + 1024 * 1024 - 1, $expectedFileSize)), $getExtendedInfo = true);
     if ($result === false || $result['status'] < 200 || $result['status'] > 299) {
         $result['data'] = self::truncateStr($result['data'], 1024);
         Log::info("Failed to download range '%s-%s' of file from url '%s'. Got result: %s", $byteRange[0], $byteRange[1], $url, print_r($result, true));
         throw new Exception(Piwik::translate('General_DownloadFail_HttpRequestFail'));
     }
     // write chunk to file
     $f = fopen($outputPath, 'ab');
     fwrite($f, $result['data']);
     fclose($f);
     clearstatcache($clear_realpath_cache = true, $outputPath);
     return array('current_size' => filesize($outputPath), 'expected_file_size' => $expectedFileSize);
 }
Esempio n. 25
0
 /**
  * Update the referrer spam blacklist
  *
  * @see https://github.com/piwik/referrer-spam-blacklist
  */
 public function updateSpammerBlacklist()
 {
     $url = 'https://raw.githubusercontent.com/piwik/referrer-spam-blacklist/master/spammers.txt';
     $list = Http::sendHttpRequest($url, 30);
     $list = preg_split("/\r\n|\n|\r/", $list);
     if (count($list) < 10) {
         throw new \Exception(sprintf('The spammers list downloaded from %s contains less than 10 entries, considering it a fail', $url));
     }
     Option::set(ReferrerSpamFilter::OPTION_STORAGE_NAME, serialize($list));
 }
 /**
  * Downloads a GeoIP database archive, extracts the .dat file and overwrites the existing
  * old database.
  *
  * If something happens that causes the download to fail, no exception is thrown, but
  * an error is logged.
  *
  * @param string $dbType
  * @param string $url URL to the database to download. The type of database is determined
  *                    from this URL.
  * @throws Exception
  */
 protected function downloadFile($dbType, $url)
 {
     $ext = GeoIPAutoUpdater::getGeoIPUrlExtension($url);
     // NOTE: using the first item in $dbNames[$dbType] makes sure GeoLiteCity will be renamed to GeoIPCity
     $zippedFilename = GeoIp::$dbNames[$dbType][0] . '.' . $ext;
     $zippedOutputPath = GeoIp::getPathForGeoIpDatabase($zippedFilename);
     $url = self::removeDateFromUrl($url);
     // download zipped file to misc dir
     try {
         $success = Http::sendHttpRequest($url, $timeout = 3600, $userAgent = null, $zippedOutputPath);
     } catch (Exception $ex) {
         throw new Exception("GeoIPAutoUpdater: failed to download '{$url}' to " . "'{$zippedOutputPath}': " . $ex->getMessage());
     }
     if ($success !== true) {
         throw new Exception("GeoIPAutoUpdater: failed to download '{$url}' to " . "'{$zippedOutputPath}'! (Unknown error)");
     }
     Log::info("GeoIPAutoUpdater: successfully downloaded '%s'", $url);
     try {
         self::unzipDownloadedFile($zippedOutputPath, $unlink = true);
     } catch (Exception $ex) {
         throw new Exception("GeoIPAutoUpdater: failed to unzip '{$zippedOutputPath}' after " . "downloading " . "'{$url}': " . $ex->getMessage());
     }
     Log::info("GeoIPAutoUpdater: successfully updated GeoIP database '%s'", $url);
 }
Esempio n. 27
0
 /**
  * Returns response for API request with given path
  *
  * @param $apiPath
  * @param bool $raw
  * @return mixed
  * @throws AuthenticationFailedException
  * @throws Exception
  */
 protected function getApiResults($apiPath, $raw = false)
 {
     $apiUrl = $this->apiUrl . $apiPath;
     $response = Http::sendHttpRequest($apiUrl, 1000, null, null, 5, false, false, true, 'GET', $this->username, $this->password);
     $httpStatus = $response['status'];
     $response = $response['data'];
     if ($httpStatus == 401) {
         throw new AuthenticationFailedException();
     } else {
         if ($httpStatus != 200) {
             throw new Exception('Error while getting API results', $httpStatus);
         }
     }
     return $raw ? $response : json_decode($response);
 }
Esempio n. 28
0
 private function getUrl($url)
 {
     try {
         return str_replace('&nbsp;', ' ', Http::sendHttpRequest($url, $timeout = 10, @$_SERVER['HTTP_USER_AGENT']));
     } catch (\Exception $e) {
         $this->logger->warning('Error while getting SEO stats (domain age): {message}', array('message' => $e->getMessage()));
         return '';
     }
 }
Esempio n. 29
0
 /**
  * @param $requestUrl
  * @return string
  * @throws Exception
  */
 protected function getResponseFromHttpAPI($requestUrl)
 {
     $queryString = Url::getQueryStringFromParameters($requestUrl);
     $hostAndPath = Fixture::getTestRootUrl();
     $url = $hostAndPath . '?' . $queryString;
     $response = Http::sendHttpRequest($url, $timeout = 300);
     return $response;
 }