/**
* Attempts to authenticate with the information set on this instance.
*
* @return AuthResult
*/
public function authenticate()
{
try {
$webServerAuthUser = $this->getAlreadyAuthenticatedLogin();
if (empty($webServerAuthUser)) {
Log::debug("using web server authentication, but REMOTE_USER server variable not found.");
return $this->tryFallbackAuth($onlySuperUsers = false, $this->fallbackAuth);
} else {
$this->login = preg_replace('/@.*/', '', $webServerAuthUser);
$this->password = '';
Log::info("User '%s' authenticated by webserver.", $this->login);
if ($this->synchronizeUsersAfterSuccessfulLogin) {
$this->synchronizeLoggedInUser();
} else {
Log::debug("WebServerAuth::%s: not synchronizing user '%s'.", __FUNCTION__, $this->login);
}
return $this->makeSuccessLogin($this->getUserForLogin());
}
} catch (ConnectionException $ex) {
throw $ex;
} catch (Exception $ex) {
Log::debug($ex);
}
return $this->makeAuthFailure();
}
/**
* The database logs requests at DEBUG level, so we check that there is no recursive
* loop (logger insert in databases, which logs the query, ...)
* @link https://github.com/piwik/piwik/issues/7017
*/
public function testNoInfiniteLoopWhenLoggingToDatabase()
{
$this->recreateLogSingleton('database');
Log::info(self::TESTMESSAGE);
$this->checkBackend('database', self::TESTMESSAGE, $formatMessage = true, $tag = 'Monolog');
}
public static function downloadAndUnzip($url, $outputDir, $filename)
{
$bufferSize = 1024 * 1024;
if (!is_dir($outputDir)) {
mkdir($outputDir);
}
$deflatedOut = $outputDir . '/' . $filename;
$outfileName = $deflatedOut . '.gz';
if (file_exists($deflatedOut)) {
return;
}
$dump = fopen($url, 'rb');
$outfile = fopen($outfileName, 'wb');
while (!feof($dump)) {
fwrite($outfile, fread($dump, $bufferSize), $bufferSize);
}
fclose($dump);
fclose($outfile);
// unzip the dump
exec("gunzip -c \"" . $outfileName . "\" > \"{$deflatedOut}\"", $output, $return);
if ($return !== 0) {
Log::info("gunzip failed with file that has following contents:");
Log::info(file_get_contents($outfile));
throw new Exception("gunzip failed({$return}): " . implode("\n", $output));
}
}
public static function downloadAndUnzip($url, $outputDir, $filename)
{
$bufferSize = 1024 * 1024;
if (!is_dir($outputDir)) {
mkdir($outputDir);
}
$deflatedOut = $outputDir . '/' . $filename;
$outfileName = $deflatedOut . '.gz';
if (file_exists($deflatedOut)) {
$filesize = filesize($deflatedOut);
if ($filesize == 0) {
throw new Exception("The file {$deflatedOut} is empty. Suggestion: delete it and try again.");
}
self::copyDownloadedGeoIp($deflatedOut, $filename);
// Valid geoip db found
return;
}
echo "Geoip database {$outfileName} is not found. Downloading from {$url}...\n";
$dump = fopen($url, 'rb');
if ($dump === false) {
throw new Exception('Could not download Geoip database from ' . $url);
}
$outfile = fopen($outfileName, 'wb');
if (!$outfile) {
throw new Exception("Failed to create file {$outfileName} - please check permissions");
}
while (!feof($dump)) {
fwrite($outfile, fread($dump, $bufferSize), $bufferSize);
}
fclose($dump);
fclose($outfile);
// unzip the dump
exec("gunzip -c \"" . $outfileName . "\" > \"{$deflatedOut}\"", $output, $return);
if ($return !== 0) {
Log::info("gunzip failed with file that has following contents:");
Log::info(file_get_contents($outfile));
throw new Exception("gunzip failed({$return}): " . implode("\n", $output));
}
self::copyDownloadedGeoIp($deflatedOut, $filename);
}
/**
* Downloads the next chunk of a specific file. The next chunk's byte range
* is determined by the existing file's size and the expected file size, which
* is stored in the piwik_option table before starting a download. The expected
* file size is obtained through a `HEAD` HTTP request.
*
* _Note: this function uses the **Range** HTTP header to accomplish downloading in
* parts. Not every server supports this header._
*
* The proper use of this function is to call it once per request. The browser
* should continue to send requests to Piwik which will in turn call this method
* until the file has completely downloaded. In this way, the user can be informed
* of a download's progress.
*
* **Example Usage**
*
* ```
* // browser JavaScript
* var downloadFile = function (isStart) {
* var ajax = new ajaxHelper();
* ajax.addParams({
* module: 'MyPlugin',
* action: 'myAction',
* isStart: isStart ? 1 : 0
* }, 'post');
* ajax.setCallback(function (response) {
* var progress = response.progress
* // ...update progress...
*
* downloadFile(false);
* });
* ajax.send();
* }
*
* downloadFile(true);
* ```
*
* ```
* // PHP controller action
* public function myAction()
* {
* $outputPath = PIWIK_INCLUDE_PATH . '/tmp/averybigfile.zip';
* $isStart = Common::getRequestVar('isStart', 1, 'int');
* Http::downloadChunk("http://bigfiles.com/averybigfile.zip", $outputPath, $isStart == 1);
* }
* ```
*
* @param string $url The url to download from.
* @param string $outputPath The path to the file to save/append to.
* @param bool $isContinuation `true` if this is the continuation of a download,
* or if we're starting a fresh one.
* @throws Exception if the file already exists and we're starting a new download,
* if we're trying to continue a download that never started
* @return array
* @api
*/
public static function downloadChunk($url, $outputPath, $isContinuation)
{
// make sure file doesn't already exist if we're starting a new download
if (!$isContinuation && file_exists($outputPath)) {
throw new Exception(Piwik::translate('General_DownloadFail_FileExists', "'" . $outputPath . "'") . ' ' . Piwik::translate('General_DownloadPleaseRemoveExisting'));
}
// if we're starting a download, get the expected file size & save as an option
$downloadOption = $outputPath . '_expectedDownloadSize';
if (!$isContinuation) {
$expectedFileSizeResult = Http::sendHttpRequest($url, $timeout = 300, $userAgent = null, $destinationPath = null, $followDepth = 0, $acceptLanguage = false, $byteRange = false, $getExtendedInfo = true, $httpMethod = 'HEAD');
$expectedFileSize = 0;
if (isset($expectedFileSizeResult['headers']['Content-Length'])) {
$expectedFileSize = (int) $expectedFileSizeResult['headers']['Content-Length'];
}
if ($expectedFileSize == 0) {
Log::info("HEAD request for '%s' failed, got following: %s", $url, print_r($expectedFileSizeResult, true));
throw new Exception(Piwik::translate('General_DownloadFail_HttpRequestFail'));
}
Option::set($downloadOption, $expectedFileSize);
} else {
$expectedFileSize = (int) Option::get($downloadOption);
if ($expectedFileSize === false) {
// sanity check
throw new Exception("Trying to continue a download that never started?! That's not supposed to happen...");
}
}
// if existing file is already big enough, then fail so we don't accidentally overwrite
// existing DB
$existingSize = file_exists($outputPath) ? filesize($outputPath) : 0;
if ($existingSize >= $expectedFileSize) {
throw new Exception(Piwik::translate('General_DownloadFail_FileExistsContinue', "'" . $outputPath . "'") . ' ' . Piwik::translate('General_DownloadPleaseRemoveExisting'));
}
// download a chunk of the file
$result = Http::sendHttpRequest($url, $timeout = 300, $userAgent = null, $destinationPath = null, $followDepth = 0, $acceptLanguage = false, $byteRange = array($existingSize, min($existingSize + 1024 * 1024 - 1, $expectedFileSize)), $getExtendedInfo = true);
if ($result === false || $result['status'] < 200 || $result['status'] > 299) {
$result['data'] = self::truncateStr($result['data'], 1024);
Log::info("Failed to download range '%s-%s' of file from url '%s'. Got result: %s", $byteRange[0], $byteRange[1], $url, print_r($result, true));
throw new Exception(Piwik::translate('General_DownloadFail_HttpRequestFail'));
}
// write chunk to file
$f = fopen($outputPath, 'ab');
fwrite($f, $result['data']);
fclose($f);
clearstatcache($clear_realpath_cache = true, $outputPath);
return array('current_size' => filesize($outputPath), 'expected_file_size' => $expectedFileSize);
}
/**
* Unzips a downloaded GeoIP database. Only unzips .gz & .tar.gz files.
*
* @param string $path Path to zipped file.
* @param bool $unlink Whether to unlink archive or not.
* @throws Exception
*/
public static function unzipDownloadedFile($path, $unlink = false)
{
$parts = explode('.', basename($path));
$filenameStart = $parts[0];
$dbFilename = $filenameStart . '.dat';
$tempFilename = $filenameStart . '.dat.new';
$outputPath = GeoIp::getPathForGeoIpDatabase($tempFilename);
// extract file
if (substr($path, -7, 7) == '.tar.gz') {
// find the .dat file in the tar archive
$unzip = Unzip::factory('tar.gz', $path);
$content = $unzip->listContent();
if (empty($content)) {
throw new Exception(Piwik::translate('UserCountry_CannotListContent', array("'{$path}'", $unzip->errorInfo())));
}
$datFile = null;
foreach ($content as $info) {
$archivedPath = $info['filename'];
if (basename($archivedPath) === $dbFilename) {
$datFile = $archivedPath;
}
}
if ($datFile === null) {
throw new Exception(Piwik::translate('UserCountry_CannotFindGeoIPDatabaseInArchive', array($dbFilename, "'{$path}'")));
}
// extract JUST the .dat file
$unzipped = $unzip->extractInString($datFile);
if (empty($unzipped)) {
throw new Exception(Piwik::translate('UserCountry_CannotUnzipDatFile', array("'{$path}'", $unzip->errorInfo())));
}
// write unzipped to file
$fd = fopen($outputPath, 'wb');
fwrite($fd, $unzipped);
fclose($fd);
} else {
if (substr($path, -3, 3) == '.gz') {
$unzip = Unzip::factory('gz', $path);
$success = $unzip->extract($outputPath);
if ($success !== true) {
throw new Exception(Piwik::translate('UserCountry_CannotUnzipDatFile', array("'{$path}'", $unzip->errorInfo())));
}
} else {
$ext = end(explode(basename($path), '.', 2));
throw new Exception(Piwik::translate('UserCountry_UnsupportedArchiveType', "'{$ext}'"));
}
}
try {
// test that the new archive is a valid GeoIP database
$dbType = GeoIp::getGeoIPDatabaseTypeFromFilename($dbFilename);
if ($dbType === false) {
throw new Exception("Unexpected GeoIP archive file name '{$path}'.");
}
$customDbNames = array('loc' => array(), 'isp' => array(), 'org' => array());
$customDbNames[$dbType] = array($tempFilename);
$phpProvider = new Php($customDbNames);
$location = self::getTestLocationCatchPhpErrors($phpProvider);
if (empty($location) || self::$unzipPhpError !== null) {
if (self::$unzipPhpError !== null) {
list($errno, $errstr, $errfile, $errline) = self::$unzipPhpError;
Log::info("GeoIPAutoUpdater: Encountered PHP error when testing newly downloaded" . " GeoIP database: %s: %s on line %s of %s.", $errno, $errstr, $errline, $errfile);
}
throw new Exception(Piwik::translate('UserCountry_ThisUrlIsNotAValidGeoIPDB'));
}
// delete the existing GeoIP database (if any) and rename the downloaded file
$oldDbFile = GeoIp::getPathForGeoIpDatabase($dbFilename);
if (file_exists($oldDbFile)) {
unlink($oldDbFile);
}
$tempFile = GeoIp::getPathForGeoIpDatabase($tempFilename);
rename($existing = $tempFile, $newName = $oldDbFile);
// delete original archive
if ($unlink) {
unlink($path);
}
} catch (Exception $ex) {
// remove downloaded files
if (file_exists($outputPath)) {
unlink($outputPath);
}
unlink($path);
throw $ex;
}
}
/**
* Given a monthly archive table, will delete all reports that are now outdated,
* or reports that ended with an error
*
* @param \Piwik\Date $date
* @return int|bool False, or timestamp indicating which archives to delete
*/
public static function shouldPurgeOutdatedArchives(Date $date)
{
if (self::$purgeOutdatedArchivesIsDisabled) {
return false;
}
$key = self::FLAG_TABLE_PURGED . "blob_" . $date->toString('Y_m');
$timestamp = Option::get($key);
// we shall purge temporary archives after their timeout is finished, plus an extra 6 hours
// in case archiving is disabled or run once a day, we give it this extra time to run
// and re-process more recent records...
$temporaryArchivingTimeout = self::getTodayArchiveTimeToLive();
$hoursBetweenPurge = 6;
$purgeEveryNSeconds = max($temporaryArchivingTimeout, $hoursBetweenPurge * 3600);
// we only delete archives if we are able to process them, otherwise, the browser might process reports
// when &segment= is specified (or custom date range) and would below, delete temporary archives that the
// browser is not able to process until next cron run (which could be more than 1 hour away)
if (self::isRequestAuthorizedToArchive() && (!$timestamp || $timestamp < time() - $purgeEveryNSeconds)) {
Option::set($key, time());
if (self::isBrowserTriggerEnabled()) {
// If Browser Archiving is enabled, it is likely there are many more temporary archives
// We delete more often which is safe, since reports are re-processed on demand
$purgeArchivesOlderThan = Date::factory(time() - 2 * $temporaryArchivingTimeout)->getDateTime();
} else {
// If cron core:archive command is building the reports, we should keep all temporary reports from today
$purgeArchivesOlderThan = Date::factory('today')->getDateTime();
}
return $purgeArchivesOlderThan;
}
Log::info("Purging temporary archives: skipped.");
return false;
}
/**
* Creates a UserAccessAttributeParser instance using INI configuration.
*
* @return UserAccessAttributeParser
*/
public static function makeConfigured()
{
$result = new UserAccessAttributeParser();
$serverSpecificationDelimiter = Config::getUserAccessAttributeServerSpecificationDelimiter();
if (!empty($serverSpecificationDelimiter)) {
$result->setServerSpecificationDelimiter($serverSpecificationDelimiter);
}
$serverListSeparator = Config::getUserAccessAttributeServerSiteListSeparator();
if (!empty($serverListSeparator)) {
$result->setServerIdsSeparator($serverListSeparator);
}
$thisPiwikInstanceName = Config::getDesignatedPiwikInstanceName();
if (!empty($thisPiwikInstanceName)) {
$result->setThisPiwikInstanceName($thisPiwikInstanceName);
} else {
if ($result->getServerIdsSeparator() == ':') {
Log::info("UserAttributesParser::%s: Configured with no instance name so matching by URL, but server/site IDs" . " separator set to special ':' character. This character may show up in URLs in LDAP, which will " . "cause problems. We recommend you use a character not often found in URLs, such as '|'.", __FUNCTION__);
}
}
Log::debug("UserAccessAttributeParser::%s: configuring with serverSpecificationDelimiter = %s, serverSiteIdListSeparator = %s, " . "thisPiwikInstanceName = %s", __FUNCTION__, $serverSpecificationDelimiter, $serverListSeparator, $thisPiwikInstanceName);
return $result;
}
function log($m)
{
Log::info($m);
}
public function log($m)
{
$this->output .= $m . "\n";
try {
Log::info($m);
} catch (Exception $e) {
print $m . "\n";
}
}
/**
* The database logs requests at DEBUG level, so we check that there is no recursive
* loop (logger insert in databases, which logs the query, ...)
* @link https://github.com/piwik/piwik/issues/7017
*/
public function testNoInfiniteLoopWhenLoggingToDatabase()
{
Config::getInstance()->log['log_writers'] = array('database');
Config::getInstance()->log['log_level'] = 'DEBUG';
Log::info(self::TESTMESSAGE);
$this->checkBackend('database', self::TESTMESSAGE, $formatMessage = true, $tag = 'Monolog');
}
private function migrateArchives()
{
Log::info('Migrating archive data');
$this->archiveMigrator->migrate($this->settings->idSite, $this->settings->dateFrom, $this->settings->dateTo);
}
private function makeLdapClient()
{
if (empty($this->ldapServers)) {
// sanity check
throw new Exception("No LDAP servers configured in LdapUsers instance.");
}
$ldapClientClass = $this->ldapClientClass;
$this->ldapClient = is_string($ldapClientClass) ? new $ldapClientClass() : $ldapClientClass;
foreach ($this->ldapServers as $server) {
try {
$this->ldapClient->connect($server->getServerHostname(), $server->getServerPort(), $this->getLdapNetworkTimeout());
$this->currentServerInfo = $server;
Log::info("LdapUsers::%s: Using LDAP server %s:%s", __FUNCTION__, $server->getServerHostname(), $server->getServerPort());
return;
} catch (Exception $ex) {
Log::debug($ex);
Log::info("Model\\LdapUsers::%s: Could not connect to LDAP server %s:%s: %s", __FUNCTION__, $server->getServerHostname(), $server->getServerPort(), $ex->getMessage());
}
}
$this->throwCouldNotConnectException();
}
/**
* @group Core
*
* @dataProvider getBackendsToTest
*/
public function testLogMessagesIgnoredWhenNotWithinLevel($backend)
{
Config::getInstance()->log['log_writers'] = array($backend);
Config::getInstance()->log['log_level'] = 'ERROR';
ob_start();
Log::info(self::TESTMESSAGE);
$this->screenOutput = ob_get_contents();
ob_end_clean();
$this->checkNoMessagesLogged($backend);
}
public function tearDown()
{
Log::info("Tearing down " . get_class($this));
Log::unsetInstance();
parent::tearDown();
}
/**
* Batch insert into table from CSV (or other delimited) file.
*
* @param string $tableName Name of table
* @param array $fields Field names
* @param string $filePath Path name of a file.
* @param array $fileSpec File specifications (delimiter, line terminator, etc)
*
* @throws Exception
* @return bool True if successful; false otherwise
*/
public static function createTableFromCSVFile($tableName, $fields, $filePath, $fileSpec)
{
// Chroot environment: prefix the path with the absolute chroot path
$chrootPath = Config::getInstance()->General['absolute_chroot_path'];
if (!empty($chrootPath)) {
$filePath = $chrootPath . $filePath;
}
// On Windows, MySQL expects forward slashes as directory separators
if (SettingsServer::isWindows()) {
$filePath = str_replace('\\', '/', $filePath);
}
$query = "\n\t\t\t\t'{$filePath}'\n\t\t\tREPLACE\n\t\t\tINTO TABLE\n\t\t\t\t`" . $tableName . "`";
if (isset($fileSpec['charset'])) {
$query .= ' CHARACTER SET ' . $fileSpec['charset'];
}
$fieldList = '(' . join(',', $fields) . ')';
$query .= "\n\t\t\tFIELDS TERMINATED BY\n\t\t\t\t'" . $fileSpec['delim'] . "'\n\t\t\tENCLOSED BY\n\t\t\t\t'" . $fileSpec['quote'] . "'\n\t\t";
if (isset($fileSpec['escape'])) {
$query .= " ESCAPED BY '" . $fileSpec['escape'] . "'";
}
$query .= "\n\t\t\tLINES TERMINATED BY\n\t\t\t\t'" . $fileSpec['eol'] . "'\n\t\t\t{$fieldList}\n\t\t";
/*
* First attempt: assume web server and MySQL server are on the same machine;
* this requires that the db user have the FILE privilege; however, since this is
* a global privilege, it may not be granted due to security concerns
*/
$keywords = array('');
/*
* Second attempt: using the LOCAL keyword means the client reads the file and sends it to the server;
* the LOCAL keyword may trigger a known PHP PDO\MYSQL bug when MySQL not built with --enable-local-infile
* @see http://bugs.php.net/bug.php?id=54158
*/
$openBaseDir = ini_get('open_basedir');
$safeMode = ini_get('safe_mode');
if (empty($openBaseDir) && empty($safeMode)) {
// php 5.x - LOAD DATA LOCAL INFILE is disabled if open_basedir restrictions or safe_mode enabled
$keywords[] = 'LOCAL ';
}
$exceptions = array();
foreach ($keywords as $keyword) {
$queryStart = 'LOAD DATA ' . $keyword . 'INFILE ';
$sql = $queryStart . $query;
try {
$result = @Db::exec($sql);
if (empty($result) || $result < 0) {
continue;
}
return true;
} catch (Exception $e) {
// echo $sql . ' ---- ' . $e->getMessage();
$code = $e->getCode();
$message = $e->getMessage() . ($code ? "[{$code}]" : '');
if (!Db::get()->isErrNo($e, '1148')) {
Log::info("LOAD DATA INFILE failed... Error was: %s", $message);
}
$exceptions[] = "\n Try #" . (count($exceptions) + 1) . ': ' . $queryStart . ": " . $message;
}
}
if (count($exceptions)) {
throw new Exception(implode(",", $exceptions));
}
return false;
}
/**
* Returns the number of referrer domains that link to the current site.
*
* @return int
*/
public function getReferrerDomainCount()
{
try {
$majesticInfo = $this->getMajesticInfo();
return $majesticInfo['referrer_domains_count'];
} catch (Exception $e) {
Log::info($e);
return 0;
}
}
if (!empty($location[LocationProvider::COUNTRY_CODE_KEY])) {
$location[LocationProvider::COUNTRY_CODE_KEY] = strtolower($location[LocationProvider::COUNTRY_CODE_KEY]);
}
$row['location_country'] = strtolower($row['location_country']);
$columnsToSet = array();
$bind = array();
foreach ($logVisitFieldsToUpdate as $column => $locationKey) {
if (!empty($location[$locationKey]) && $location[$locationKey] != $row[$column]) {
$columnsToSet[] = $column . ' = ?';
$bind[] = $location[$locationKey];
}
}
if (empty($columnsToSet)) {
continue;
}
$bind[] = $row['idvisit'];
// update log_visit
$sql = "UPDATE " . Common::prefixTable('log_visit') . "\n\t\t\t\t SET " . implode(', ', $columnsToSet) . "\n\t\t\t\t WHERE idvisit = ?";
Db::query($sql, $bind);
// update log_conversion
$sql = "UPDATE " . Common::prefixTable('log_conversion') . "\n\t\t\t\t SET " . implode(', ', $columnsToSet) . "\n\t\t\t\t WHERE idvisit = ?";
Db::query($sql, $bind);
}
Log::info(round($start * 100 / $count) . "% done...");
flush();
}
if ($start >= $count) {
Log::info("100% done!");
Log::info("");
Log::info("[note] Now that you've geolocated your old visits, you need to force your reports to be re-processed. See this FAQ entry: http://piwik.org/faq/how-to/#faq_59");
}
