/** * Attempts to authenticate with the information set on this instance. * * @return AuthResult */ public function authenticate() { try { $webServerAuthUser = $this->getAlreadyAuthenticatedLogin(); if (empty($webServerAuthUser)) { Log::debug("using web server authentication, but REMOTE_USER server variable not found."); return $this->tryFallbackAuth($onlySuperUsers = false, $this->fallbackAuth); } else { $this->login = preg_replace('/@.*/', '', $webServerAuthUser); $this->password = ''; Log::info("User '%s' authenticated by webserver.", $this->login); if ($this->synchronizeUsersAfterSuccessfulLogin) { $this->synchronizeLoggedInUser(); } else { Log::debug("WebServerAuth::%s: not synchronizing user '%s'.", __FUNCTION__, $this->login); } return $this->makeSuccessLogin($this->getUserForLogin()); } } catch (ConnectionException $ex) { throw $ex; } catch (Exception $ex) { Log::debug($ex); } return $this->makeAuthFailure(); }
/** * The database logs requests at DEBUG level, so we check that there is no recursive * loop (logger insert in databases, which logs the query, ...) * @link https://github.com/piwik/piwik/issues/7017 */ public function testNoInfiniteLoopWhenLoggingToDatabase() { $this->recreateLogSingleton('database'); Log::info(self::TESTMESSAGE); $this->checkBackend('database', self::TESTMESSAGE, $formatMessage = true, $tag = 'Monolog'); }
public static function downloadAndUnzip($url, $outputDir, $filename) { $bufferSize = 1024 * 1024; if (!is_dir($outputDir)) { mkdir($outputDir); } $deflatedOut = $outputDir . '/' . $filename; $outfileName = $deflatedOut . '.gz'; if (file_exists($deflatedOut)) { return; } $dump = fopen($url, 'rb'); $outfile = fopen($outfileName, 'wb'); while (!feof($dump)) { fwrite($outfile, fread($dump, $bufferSize), $bufferSize); } fclose($dump); fclose($outfile); // unzip the dump exec("gunzip -c \"" . $outfileName . "\" > \"{$deflatedOut}\"", $output, $return); if ($return !== 0) { Log::info("gunzip failed with file that has following contents:"); Log::info(file_get_contents($outfile)); throw new Exception("gunzip failed({$return}): " . implode("\n", $output)); } }
public static function downloadAndUnzip($url, $outputDir, $filename) { $bufferSize = 1024 * 1024; if (!is_dir($outputDir)) { mkdir($outputDir); } $deflatedOut = $outputDir . '/' . $filename; $outfileName = $deflatedOut . '.gz'; if (file_exists($deflatedOut)) { $filesize = filesize($deflatedOut); if ($filesize == 0) { throw new Exception("The file {$deflatedOut} is empty. Suggestion: delete it and try again."); } self::copyDownloadedGeoIp($deflatedOut, $filename); // Valid geoip db found return; } echo "Geoip database {$outfileName} is not found. Downloading from {$url}...\n"; $dump = fopen($url, 'rb'); if ($dump === false) { throw new Exception('Could not download Geoip database from ' . $url); } $outfile = fopen($outfileName, 'wb'); if (!$outfile) { throw new Exception("Failed to create file {$outfileName} - please check permissions"); } while (!feof($dump)) { fwrite($outfile, fread($dump, $bufferSize), $bufferSize); } fclose($dump); fclose($outfile); // unzip the dump exec("gunzip -c \"" . $outfileName . "\" > \"{$deflatedOut}\"", $output, $return); if ($return !== 0) { Log::info("gunzip failed with file that has following contents:"); Log::info(file_get_contents($outfile)); throw new Exception("gunzip failed({$return}): " . implode("\n", $output)); } self::copyDownloadedGeoIp($deflatedOut, $filename); }
/** * Downloads the next chunk of a specific file. The next chunk's byte range * is determined by the existing file's size and the expected file size, which * is stored in the piwik_option table before starting a download. The expected * file size is obtained through a `HEAD` HTTP request. * * _Note: this function uses the **Range** HTTP header to accomplish downloading in * parts. Not every server supports this header._ * * The proper use of this function is to call it once per request. The browser * should continue to send requests to Piwik which will in turn call this method * until the file has completely downloaded. In this way, the user can be informed * of a download's progress. * * **Example Usage** * * ``` * // browser JavaScript * var downloadFile = function (isStart) { * var ajax = new ajaxHelper(); * ajax.addParams({ * module: 'MyPlugin', * action: 'myAction', * isStart: isStart ? 1 : 0 * }, 'post'); * ajax.setCallback(function (response) { * var progress = response.progress * // ...update progress... * * downloadFile(false); * }); * ajax.send(); * } * * downloadFile(true); * ``` * * ``` * // PHP controller action * public function myAction() * { * $outputPath = PIWIK_INCLUDE_PATH . '/tmp/averybigfile.zip'; * $isStart = Common::getRequestVar('isStart', 1, 'int'); * Http::downloadChunk("http://bigfiles.com/averybigfile.zip", $outputPath, $isStart == 1); * } * ``` * * @param string $url The url to download from. * @param string $outputPath The path to the file to save/append to. * @param bool $isContinuation `true` if this is the continuation of a download, * or if we're starting a fresh one. * @throws Exception if the file already exists and we're starting a new download, * if we're trying to continue a download that never started * @return array * @api */ public static function downloadChunk($url, $outputPath, $isContinuation) { // make sure file doesn't already exist if we're starting a new download if (!$isContinuation && file_exists($outputPath)) { throw new Exception(Piwik::translate('General_DownloadFail_FileExists', "'" . $outputPath . "'") . ' ' . Piwik::translate('General_DownloadPleaseRemoveExisting')); } // if we're starting a download, get the expected file size & save as an option $downloadOption = $outputPath . '_expectedDownloadSize'; if (!$isContinuation) { $expectedFileSizeResult = Http::sendHttpRequest($url, $timeout = 300, $userAgent = null, $destinationPath = null, $followDepth = 0, $acceptLanguage = false, $byteRange = false, $getExtendedInfo = true, $httpMethod = 'HEAD'); $expectedFileSize = 0; if (isset($expectedFileSizeResult['headers']['Content-Length'])) { $expectedFileSize = (int) $expectedFileSizeResult['headers']['Content-Length']; } if ($expectedFileSize == 0) { Log::info("HEAD request for '%s' failed, got following: %s", $url, print_r($expectedFileSizeResult, true)); throw new Exception(Piwik::translate('General_DownloadFail_HttpRequestFail')); } Option::set($downloadOption, $expectedFileSize); } else { $expectedFileSize = (int) Option::get($downloadOption); if ($expectedFileSize === false) { // sanity check throw new Exception("Trying to continue a download that never started?! That's not supposed to happen..."); } } // if existing file is already big enough, then fail so we don't accidentally overwrite // existing DB $existingSize = file_exists($outputPath) ? filesize($outputPath) : 0; if ($existingSize >= $expectedFileSize) { throw new Exception(Piwik::translate('General_DownloadFail_FileExistsContinue', "'" . $outputPath . "'") . ' ' . Piwik::translate('General_DownloadPleaseRemoveExisting')); } // download a chunk of the file $result = Http::sendHttpRequest($url, $timeout = 300, $userAgent = null, $destinationPath = null, $followDepth = 0, $acceptLanguage = false, $byteRange = array($existingSize, min($existingSize + 1024 * 1024 - 1, $expectedFileSize)), $getExtendedInfo = true); if ($result === false || $result['status'] < 200 || $result['status'] > 299) { $result['data'] = self::truncateStr($result['data'], 1024); Log::info("Failed to download range '%s-%s' of file from url '%s'. Got result: %s", $byteRange[0], $byteRange[1], $url, print_r($result, true)); throw new Exception(Piwik::translate('General_DownloadFail_HttpRequestFail')); } // write chunk to file $f = fopen($outputPath, 'ab'); fwrite($f, $result['data']); fclose($f); clearstatcache($clear_realpath_cache = true, $outputPath); return array('current_size' => filesize($outputPath), 'expected_file_size' => $expectedFileSize); }
/** * Unzips a downloaded GeoIP database. Only unzips .gz & .tar.gz files. * * @param string $path Path to zipped file. * @param bool $unlink Whether to unlink archive or not. * @throws Exception */ public static function unzipDownloadedFile($path, $unlink = false) { $parts = explode('.', basename($path)); $filenameStart = $parts[0]; $dbFilename = $filenameStart . '.dat'; $tempFilename = $filenameStart . '.dat.new'; $outputPath = GeoIp::getPathForGeoIpDatabase($tempFilename); // extract file if (substr($path, -7, 7) == '.tar.gz') { // find the .dat file in the tar archive $unzip = Unzip::factory('tar.gz', $path); $content = $unzip->listContent(); if (empty($content)) { throw new Exception(Piwik::translate('UserCountry_CannotListContent', array("'{$path}'", $unzip->errorInfo()))); } $datFile = null; foreach ($content as $info) { $archivedPath = $info['filename']; if (basename($archivedPath) === $dbFilename) { $datFile = $archivedPath; } } if ($datFile === null) { throw new Exception(Piwik::translate('UserCountry_CannotFindGeoIPDatabaseInArchive', array($dbFilename, "'{$path}'"))); } // extract JUST the .dat file $unzipped = $unzip->extractInString($datFile); if (empty($unzipped)) { throw new Exception(Piwik::translate('UserCountry_CannotUnzipDatFile', array("'{$path}'", $unzip->errorInfo()))); } // write unzipped to file $fd = fopen($outputPath, 'wb'); fwrite($fd, $unzipped); fclose($fd); } else { if (substr($path, -3, 3) == '.gz') { $unzip = Unzip::factory('gz', $path); $success = $unzip->extract($outputPath); if ($success !== true) { throw new Exception(Piwik::translate('UserCountry_CannotUnzipDatFile', array("'{$path}'", $unzip->errorInfo()))); } } else { $ext = end(explode(basename($path), '.', 2)); throw new Exception(Piwik::translate('UserCountry_UnsupportedArchiveType', "'{$ext}'")); } } try { // test that the new archive is a valid GeoIP database $dbType = GeoIp::getGeoIPDatabaseTypeFromFilename($dbFilename); if ($dbType === false) { throw new Exception("Unexpected GeoIP archive file name '{$path}'."); } $customDbNames = array('loc' => array(), 'isp' => array(), 'org' => array()); $customDbNames[$dbType] = array($tempFilename); $phpProvider = new Php($customDbNames); $location = self::getTestLocationCatchPhpErrors($phpProvider); if (empty($location) || self::$unzipPhpError !== null) { if (self::$unzipPhpError !== null) { list($errno, $errstr, $errfile, $errline) = self::$unzipPhpError; Log::info("GeoIPAutoUpdater: Encountered PHP error when testing newly downloaded" . " GeoIP database: %s: %s on line %s of %s.", $errno, $errstr, $errline, $errfile); } throw new Exception(Piwik::translate('UserCountry_ThisUrlIsNotAValidGeoIPDB')); } // delete the existing GeoIP database (if any) and rename the downloaded file $oldDbFile = GeoIp::getPathForGeoIpDatabase($dbFilename); if (file_exists($oldDbFile)) { unlink($oldDbFile); } $tempFile = GeoIp::getPathForGeoIpDatabase($tempFilename); rename($existing = $tempFile, $newName = $oldDbFile); // delete original archive if ($unlink) { unlink($path); } } catch (Exception $ex) { // remove downloaded files if (file_exists($outputPath)) { unlink($outputPath); } unlink($path); throw $ex; } }
/** * Given a monthly archive table, will delete all reports that are now outdated, * or reports that ended with an error * * @param \Piwik\Date $date * @return int|bool False, or timestamp indicating which archives to delete */ public static function shouldPurgeOutdatedArchives(Date $date) { if (self::$purgeOutdatedArchivesIsDisabled) { return false; } $key = self::FLAG_TABLE_PURGED . "blob_" . $date->toString('Y_m'); $timestamp = Option::get($key); // we shall purge temporary archives after their timeout is finished, plus an extra 6 hours // in case archiving is disabled or run once a day, we give it this extra time to run // and re-process more recent records... $temporaryArchivingTimeout = self::getTodayArchiveTimeToLive(); $hoursBetweenPurge = 6; $purgeEveryNSeconds = max($temporaryArchivingTimeout, $hoursBetweenPurge * 3600); // we only delete archives if we are able to process them, otherwise, the browser might process reports // when &segment= is specified (or custom date range) and would below, delete temporary archives that the // browser is not able to process until next cron run (which could be more than 1 hour away) if (self::isRequestAuthorizedToArchive() && (!$timestamp || $timestamp < time() - $purgeEveryNSeconds)) { Option::set($key, time()); if (self::isBrowserTriggerEnabled()) { // If Browser Archiving is enabled, it is likely there are many more temporary archives // We delete more often which is safe, since reports are re-processed on demand $purgeArchivesOlderThan = Date::factory(time() - 2 * $temporaryArchivingTimeout)->getDateTime(); } else { // If cron core:archive command is building the reports, we should keep all temporary reports from today $purgeArchivesOlderThan = Date::factory('today')->getDateTime(); } return $purgeArchivesOlderThan; } Log::info("Purging temporary archives: skipped."); return false; }
/** * Creates a UserAccessAttributeParser instance using INI configuration. * * @return UserAccessAttributeParser */ public static function makeConfigured() { $result = new UserAccessAttributeParser(); $serverSpecificationDelimiter = Config::getUserAccessAttributeServerSpecificationDelimiter(); if (!empty($serverSpecificationDelimiter)) { $result->setServerSpecificationDelimiter($serverSpecificationDelimiter); } $serverListSeparator = Config::getUserAccessAttributeServerSiteListSeparator(); if (!empty($serverListSeparator)) { $result->setServerIdsSeparator($serverListSeparator); } $thisPiwikInstanceName = Config::getDesignatedPiwikInstanceName(); if (!empty($thisPiwikInstanceName)) { $result->setThisPiwikInstanceName($thisPiwikInstanceName); } else { if ($result->getServerIdsSeparator() == ':') { Log::info("UserAttributesParser::%s: Configured with no instance name so matching by URL, but server/site IDs" . " separator set to special ':' character. This character may show up in URLs in LDAP, which will " . "cause problems. We recommend you use a character not often found in URLs, such as '|'.", __FUNCTION__); } } Log::debug("UserAccessAttributeParser::%s: configuring with serverSpecificationDelimiter = %s, serverSiteIdListSeparator = %s, " . "thisPiwikInstanceName = %s", __FUNCTION__, $serverSpecificationDelimiter, $serverListSeparator, $thisPiwikInstanceName); return $result; }
function log($m) { Log::info($m); }
public function log($m) { $this->output .= $m . "\n"; try { Log::info($m); } catch (Exception $e) { print $m . "\n"; } }
/** * The database logs requests at DEBUG level, so we check that there is no recursive * loop (logger insert in databases, which logs the query, ...) * @link https://github.com/piwik/piwik/issues/7017 */ public function testNoInfiniteLoopWhenLoggingToDatabase() { Config::getInstance()->log['log_writers'] = array('database'); Config::getInstance()->log['log_level'] = 'DEBUG'; Log::info(self::TESTMESSAGE); $this->checkBackend('database', self::TESTMESSAGE, $formatMessage = true, $tag = 'Monolog'); }
private function migrateArchives() { Log::info('Migrating archive data'); $this->archiveMigrator->migrate($this->settings->idSite, $this->settings->dateFrom, $this->settings->dateTo); }
private function makeLdapClient() { if (empty($this->ldapServers)) { // sanity check throw new Exception("No LDAP servers configured in LdapUsers instance."); } $ldapClientClass = $this->ldapClientClass; $this->ldapClient = is_string($ldapClientClass) ? new $ldapClientClass() : $ldapClientClass; foreach ($this->ldapServers as $server) { try { $this->ldapClient->connect($server->getServerHostname(), $server->getServerPort(), $this->getLdapNetworkTimeout()); $this->currentServerInfo = $server; Log::info("LdapUsers::%s: Using LDAP server %s:%s", __FUNCTION__, $server->getServerHostname(), $server->getServerPort()); return; } catch (Exception $ex) { Log::debug($ex); Log::info("Model\\LdapUsers::%s: Could not connect to LDAP server %s:%s: %s", __FUNCTION__, $server->getServerHostname(), $server->getServerPort(), $ex->getMessage()); } } $this->throwCouldNotConnectException(); }
/** * @group Core * * @dataProvider getBackendsToTest */ public function testLogMessagesIgnoredWhenNotWithinLevel($backend) { Config::getInstance()->log['log_writers'] = array($backend); Config::getInstance()->log['log_level'] = 'ERROR'; ob_start(); Log::info(self::TESTMESSAGE); $this->screenOutput = ob_get_contents(); ob_end_clean(); $this->checkNoMessagesLogged($backend); }
public function tearDown() { Log::info("Tearing down " . get_class($this)); Log::unsetInstance(); parent::tearDown(); }
/** * Batch insert into table from CSV (or other delimited) file. * * @param string $tableName Name of table * @param array $fields Field names * @param string $filePath Path name of a file. * @param array $fileSpec File specifications (delimiter, line terminator, etc) * * @throws Exception * @return bool True if successful; false otherwise */ public static function createTableFromCSVFile($tableName, $fields, $filePath, $fileSpec) { // Chroot environment: prefix the path with the absolute chroot path $chrootPath = Config::getInstance()->General['absolute_chroot_path']; if (!empty($chrootPath)) { $filePath = $chrootPath . $filePath; } // On Windows, MySQL expects forward slashes as directory separators if (SettingsServer::isWindows()) { $filePath = str_replace('\\', '/', $filePath); } $query = "\n\t\t\t\t'{$filePath}'\n\t\t\tREPLACE\n\t\t\tINTO TABLE\n\t\t\t\t`" . $tableName . "`"; if (isset($fileSpec['charset'])) { $query .= ' CHARACTER SET ' . $fileSpec['charset']; } $fieldList = '(' . join(',', $fields) . ')'; $query .= "\n\t\t\tFIELDS TERMINATED BY\n\t\t\t\t'" . $fileSpec['delim'] . "'\n\t\t\tENCLOSED BY\n\t\t\t\t'" . $fileSpec['quote'] . "'\n\t\t"; if (isset($fileSpec['escape'])) { $query .= " ESCAPED BY '" . $fileSpec['escape'] . "'"; } $query .= "\n\t\t\tLINES TERMINATED BY\n\t\t\t\t'" . $fileSpec['eol'] . "'\n\t\t\t{$fieldList}\n\t\t"; /* * First attempt: assume web server and MySQL server are on the same machine; * this requires that the db user have the FILE privilege; however, since this is * a global privilege, it may not be granted due to security concerns */ $keywords = array(''); /* * Second attempt: using the LOCAL keyword means the client reads the file and sends it to the server; * the LOCAL keyword may trigger a known PHP PDO\MYSQL bug when MySQL not built with --enable-local-infile * @see http://bugs.php.net/bug.php?id=54158 */ $openBaseDir = ini_get('open_basedir'); $safeMode = ini_get('safe_mode'); if (empty($openBaseDir) && empty($safeMode)) { // php 5.x - LOAD DATA LOCAL INFILE is disabled if open_basedir restrictions or safe_mode enabled $keywords[] = 'LOCAL '; } $exceptions = array(); foreach ($keywords as $keyword) { $queryStart = 'LOAD DATA ' . $keyword . 'INFILE '; $sql = $queryStart . $query; try { $result = @Db::exec($sql); if (empty($result) || $result < 0) { continue; } return true; } catch (Exception $e) { // echo $sql . ' ---- ' . $e->getMessage(); $code = $e->getCode(); $message = $e->getMessage() . ($code ? "[{$code}]" : ''); if (!Db::get()->isErrNo($e, '1148')) { Log::info("LOAD DATA INFILE failed... Error was: %s", $message); } $exceptions[] = "\n Try #" . (count($exceptions) + 1) . ': ' . $queryStart . ": " . $message; } } if (count($exceptions)) { throw new Exception(implode(",", $exceptions)); } return false; }
/** * Returns the number of referrer domains that link to the current site. * * @return int */ public function getReferrerDomainCount() { try { $majesticInfo = $this->getMajesticInfo(); return $majesticInfo['referrer_domains_count']; } catch (Exception $e) { Log::info($e); return 0; } }
if (!empty($location[LocationProvider::COUNTRY_CODE_KEY])) { $location[LocationProvider::COUNTRY_CODE_KEY] = strtolower($location[LocationProvider::COUNTRY_CODE_KEY]); } $row['location_country'] = strtolower($row['location_country']); $columnsToSet = array(); $bind = array(); foreach ($logVisitFieldsToUpdate as $column => $locationKey) { if (!empty($location[$locationKey]) && $location[$locationKey] != $row[$column]) { $columnsToSet[] = $column . ' = ?'; $bind[] = $location[$locationKey]; } } if (empty($columnsToSet)) { continue; } $bind[] = $row['idvisit']; // update log_visit $sql = "UPDATE " . Common::prefixTable('log_visit') . "\n\t\t\t\t SET " . implode(', ', $columnsToSet) . "\n\t\t\t\t WHERE idvisit = ?"; Db::query($sql, $bind); // update log_conversion $sql = "UPDATE " . Common::prefixTable('log_conversion') . "\n\t\t\t\t SET " . implode(', ', $columnsToSet) . "\n\t\t\t\t WHERE idvisit = ?"; Db::query($sql, $bind); } Log::info(round($start * 100 / $count) . "% done..."); flush(); } if ($start >= $count) { Log::info("100% done!"); Log::info(""); Log::info("[note] Now that you've geolocated your old visits, you need to force your reports to be re-processed. See this FAQ entry: http://piwik.org/faq/how-to/#faq_59"); }