/**
* Purges old data from the following tables:
* - log_visit
* - log_link_visit_action
* - log_conversion
* - log_conversion_item
* - log_action
*/
public function purgeData()
{
$maxIdVisit = $this->getDeleteIdVisitOffset();
// break if no ID was found (nothing to delete for given period)
if (empty($maxIdVisit)) {
return;
}
$logTables = self::getDeleteTableLogTables();
// delete data from log tables
$where = "WHERE idvisit <= ?";
foreach ($logTables as $logTable) {
// deleting from log_action must be handled differently, so we do it later
if ($logTable != Piwik_Common::prefixTable('log_action')) {
Piwik_DeleteAllRows($logTable, $where, $this->maxRowsToDeletePerQuery, array($maxIdVisit));
}
}
// delete unused actions from the log_action table (but only if we can lock tables)
if (Piwik::isLockPrivilegeGranted()) {
$this->purgeUnusedLogActions();
} else {
$logMessage = get_class($this) . ": LOCK TABLES privilege not granted; skipping unused actions purge";
Piwik::log($logMessage);
}
// optimize table overhead after deletion
Piwik_OptimizeTables($logTables);
}
/**
* @see Piwik_ViewDataTable::main()
* @throws Exception|Piwik_Access_NoAccessException
* @return null
*/
public function main()
{
if ($this->mainAlreadyExecuted) {
return;
}
$this->mainAlreadyExecuted = true;
$this->isDataAvailable = true;
try {
$this->loadDataTableFromAPI();
} catch (Piwik_Access_NoAccessException $e) {
throw $e;
} catch (Exception $e) {
Piwik::log("Failed to get data from API: " . $e->getMessage());
$this->isDataAvailable = false;
}
$this->postDataTableLoadedFromAPI();
$this->view = $this->buildView();
}
public static function printMemoryUsage($prefixString = null)
{
$memory = false;
if (function_exists('xdebug_memory_usage')) {
$memory = xdebug_memory_usage();
} elseif (function_exists('memory_get_usage')) {
$memory = memory_get_usage();
}
if ($memory !== false) {
$usage = round($memory / 1024 / 1024, 2);
if (!is_null($prefixString)) {
Piwik::log($prefixString);
}
Piwik::log("Memory usage = {$usage} Mb");
} else {
Piwik::log("Memory usage function not found.");
}
}
/**
* Must be called before dispatch()
* - checks that directories are writable,
* - loads the configuration file,
* - loads the plugin,
* - inits the DB connection,
* - etc.
*/
function init()
{
static $initialized = false;
if($initialized)
{
return;
}
$initialized = true;
try {
Zend_Registry::set('timer', new Piwik_Timer);
$directoriesToCheck = array(
'/tmp/',
'/tmp/templates_c/',
'/tmp/cache/',
'/tmp/assets/'
);
Piwik::checkDirectoriesWritableOrDie($directoriesToCheck);
Piwik_Common::assignCliParametersToRequest();
Piwik_Translate::getInstance()->loadEnglishTranslation();
$exceptionToThrow = false;
try {
Piwik::createConfigObject();
} catch(Exception $e) {
Piwik_PostEvent('FrontController.NoConfigurationFile', $e, $info = array(), $pending = true);
$exceptionToThrow = $e;
}
if(Zend_Registry::get('config')->General->maintenance_mode == 1
&& !Piwik_Common::isPhpCliMode())
{
throw new Exception("Piwik is in scheduled maintenance. Please come back later.");
}
$pluginsManager = Piwik_PluginsManager::getInstance();
$pluginsManager->loadPlugins( Zend_Registry::get('config')->Plugins->Plugins->toArray() );
if($exceptionToThrow)
{
throw $exceptionToThrow;
}
try {
Piwik::createDatabaseObject();
} catch(Exception $e) {
Piwik_PostEvent('FrontController.badConfigurationFile', $e, $info = array(), $pending = true);
throw $e;
}
Piwik::createLogObject();
// creating the access object, so that core/Updates/* can enforce Super User and use some APIs
Piwik::createAccessObject();
Piwik_PostEvent('FrontController.dispatchCoreAndPluginUpdatesScreen');
Piwik_PluginsManager::getInstance()->installLoadedPlugins();
Piwik::install();
// ensure the current Piwik URL is known for later use
if(method_exists('Piwik', 'getPiwikUrl'))
{
$host = Piwik::getPiwikUrl();
}
Piwik_PostEvent('FrontController.initAuthenticationObject');
try {
$authAdapter = Zend_Registry::get('auth');
} catch(Exception $e){
throw new Exception("Authentication object cannot be found in the Registry. Maybe the Login plugin is not activated?
<br />You can activate the plugin by adding:<br />
<code>Plugins[] = Login</code><br />
under the <code>[Plugins]</code> section in your config/config.inc.php");
}
Zend_Registry::get('access')->reloadAccess($authAdapter);
Piwik_Translate::getInstance()->reloadLanguage();
Piwik::raiseMemoryLimitIfNecessary();
$pluginsManager->postLoadPlugins();
Piwik_PostEvent('FrontController.checkForUpdates');
} catch(Exception $e) {
Piwik_ExitWithMessage($e->getMessage(), false, true);
}
Piwik::log('End FrontController->init() - Request: '. var_export($_REQUEST, true));
}
/**
* Activate the specified plugin and install (if needed)
*
* @param string $pluginName Name of plugin
* @throws Exception
*/
public function activatePlugin($pluginName)
{
$plugins = Piwik_Config::getInstance()->Plugins['Plugins'];
if (in_array($pluginName, $plugins)) {
throw new Exception("Plugin '{$pluginName}' already activated.");
}
$existingPlugins = $this->readPluginsDirectory();
if (array_search($pluginName, $existingPlugins) === false) {
Piwik::log("Unable to find the plugin '{$pluginName}' in activatePlugin.");
return;
}
$plugin = $this->loadPlugin($pluginName);
if ($plugin === null) {
return;
}
$this->installPluginIfNecessary($plugin);
$plugin->activate();
// we add the plugin to the list of activated plugins
if (!in_array($pluginName, $plugins)) {
$plugins[] = $pluginName;
} else {
// clean up if we find a dupe
$plugins = array_unique($plugins);
}
// the config file will automatically be saved with the new plugin
$this->updatePluginsConfig($plugins);
Piwik_Config::getInstance()->forceSave();
// Delete merged js/css files to force regenerations to include the activated plugin
Piwik::deleteAllCacheOnUpdate();
}
/**
* Performs a batch insert into a specific table using either LOAD DATA INFILE or plain INSERTs,
* as a fallback. On MySQL, LOAD DATA INFILE is 20x faster than a series of plain INSERTs.
*
* @param string $tableName PREFIXED table name! you must call Piwik_Common::prefixTable() before passing the table name
* @param array $fields array of unquoted field names
* @param array $values array of data to be inserted
* @return bool True if the bulk LOAD was used, false if we fallback to plain INSERTs
*/
public static function tableInsertBatch($tableName, $fields, $values)
{
$filePath = PIWIK_USER_PATH . '/' . Piwik_AssetManager::MERGED_FILE_DIR . $tableName . '-' . Piwik_Common::generateUniqId() . '.csv';
if (Zend_Registry::get('db')->hasBulkLoader()) {
try {
// throw new Exception('');
$fileSpec = array('delim' => "\t", 'quote' => '"', 'escape' => '\\\\', 'escapespecial_cb' => create_function('$str', 'return str_replace(array(chr(92), chr(34)), array(chr(92).chr(92), chr(92).chr(34)), $str);'), 'eol' => "\r\n", 'null' => 'NULL');
// hack for charset mismatch
if (!self::isDatabaseConnectionUTF8() && !isset(Zend_Registry::get('config')->database->charset)) {
$fileSpec['charset'] = 'latin1';
}
self::createCSVFile($filePath, $fileSpec, $values);
$rc = self::createTableFromCSVFile($tableName, $fields, $filePath, $fileSpec);
if ($rc) {
unlink($filePath);
return true;
}
throw new Exception('unknown cause');
} catch (Exception $e) {
Piwik::log("LOAD DATA INFILE failed or not supported, falling back to normal INSERTs... Error was:" . $e->getMessage());
}
}
// if all else fails, fallback to a series of INSERTs
@unlink($filePath);
self::tableInsertBatchIterate($tableName, $fields, $values);
return false;
}
public static function start($options = false)
{
if(Piwik_Common::isPhpCliMode() || version_compare(Piwik_GetOption('version_core'), '1.5-b5') < 0)
{
return;
}
// use cookies to store session id on the client side
@ini_set('session.use_cookies', '1');
// prevent attacks involving session ids passed in URLs
@ini_set('session.use_only_cookies', '1');
// advise browser that session cookie should only be sent over secure connection
if(Piwik_Url::getCurrentScheme() === 'https')
{
@ini_set('session.cookie_secure', '1');
}
// advise browser that session cookie should only be accessible through the HTTP protocol (i.e., not JavaScript)
@ini_set('session.cookie_httponly', '1');
// don't use the default: PHPSESSID
$sessionName = defined('PIWIK_SESSION_NAME') ? PIWIK_SESSION_NAME : 'PIWIK_SESSID';
@ini_set('session.name', $sessionName);
// proxies may cause the referer check to fail and
// incorrectly invalidate the session
@ini_set('session.referer_check', '');
// we consider these to be misconfigurations, in that
// - user - we can't verify that user-defined session handler functions have been set via session_set_save_handler()
// - mm - this handler is not recommended, unsupported, not available for Windows, and has a potential concurrency issue
// - files - this handler doesn't work well in load-balanced environments and may have a concurrency issue with locked session files
$currentSaveHandler = ini_get('session.save_handler');
if(in_array($currentSaveHandler, array('user', 'mm', 'files')))
{
$db = Zend_Registry::get('db');
$config = array(
'name' => Piwik_Common::prefixTable('session'),
'primary' => 'id',
'modifiedColumn' => 'modified',
'dataColumn' => 'data',
'lifetimeColumn' => 'lifetime',
'db' => $db,
);
$saveHandler = new Piwik_Session_SaveHandler_DbTable($config);
if($saveHandler)
{
self::setSaveHandler($saveHandler);
}
}
// garbage collection may disabled by default (e.g., Debian)
if(ini_get('session.gc_probability') == 0)
{
@ini_set('session.gc_probability', 1);
}
try {
Zend_Session::start();
register_shutdown_function(array('Zend_Session', 'writeClose'), true);
} catch(Exception $e) {
Piwik::log('Unable to start session: ' . $e->getMessage());
Piwik_ExitWithMessage(Piwik_Translate('General_ExceptionUnableToStartSession'));
}
}
$location = $provider->getLocation(array('ip' => $ip));
if (!empty($location[Piwik_UserCountry_LocationProvider::COUNTRY_CODE_KEY])) {
$location[Piwik_UserCountry_LocationProvider::COUNTRY_CODE_KEY] = strtolower($location[Piwik_UserCountry_LocationProvider::COUNTRY_CODE_KEY]);
}
$row['location_country'] = strtolower($row['location_country']);
$columnsToSet = array();
$bind = array();
foreach ($logVisitFieldsToUpdate as $column => $locationKey) {
if (!empty($location[$locationKey]) && $location[$locationKey] != $row[$column]) {
$columnsToSet[] = $column . ' = ?';
$bind[] = $location[$locationKey];
}
}
if (empty($columnsToSet)) {
continue;
}
$bind[] = $row['idvisit'];
// update log_visit
$sql = "UPDATE " . Piwik_Common::prefixTable('log_visit') . "\n\t\t\t\t SET " . implode(', ', $columnsToSet) . "\n\t\t\t\t WHERE idvisit = ?";
Piwik_Query($sql, $bind);
// update log_conversion
$sql = "UPDATE " . Piwik_Common::prefixTable('log_conversion') . "\n\t\t\t\t SET " . implode(', ', $columnsToSet) . "\n\t\t\t\t WHERE idvisit = ?";
Piwik_Query($sql, $bind);
}
Piwik::log(round($start * 100 / $count) . "% done...");
flush();
}
Piwik::log("100% done!");
Piwik::log("");
Piwik::log("[note] Now that you've geolocated your old visits, you need to force your reports to be re-processed. See this FAQ entry: http://piwik.org/faq/how-to/#faq_59");
/**
* Called at the end of the archiving process.
* Does some cleaning job in the database.
*/
protected function postCompute()
{
parent::postCompute();
$blobTable = $this->tableArchiveBlob->getTableName();
$numericTable = $this->tableArchiveNumeric->getTableName();
$key = 'lastPurge_' . $blobTable;
$timestamp = Piwik_GetOption($key);
if (!$timestamp || $timestamp < time() - 86400) {
Piwik_SetOption($key, time());
// we delete out of date daily archives from table, maximum once per day
// we only delete archives processed that are older than 1 day, to not delete archives we just processed
$yesterday = Piwik_Date::factory('yesterday')->getDateTime();
$result = Piwik_FetchAll("\n\t\t\t\t\t\t\tSELECT idarchive\n\t\t\t\t\t\t\tFROM {$numericTable}\n\t\t\t\t\t\t\tWHERE name LIKE 'done%'\n\t\t\t\t\t\t\t\tAND value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . "\n\t\t\t\t\t\t\t\tAND ts_archived < ?", array($yesterday));
$idArchivesToDelete = array();
if (!empty($result)) {
foreach ($result as $row) {
$idArchivesToDelete[] = $row['idarchive'];
}
$query = "DELETE \n \t\t\t\t\t\tFROM %s\n \t\t\t\t\t\tWHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ")\n \t\t\t\t\t\t";
Piwik_Query(sprintf($query, $blobTable));
Piwik_Query(sprintf($query, $numericTable));
}
Piwik::log("Purging temporary archives: done [ purged archives older than {$yesterday} from {$blobTable} and {$numericTable} ] [Deleted IDs: " . implode(',', $idArchivesToDelete) . "]");
// Deleting "Custom Date Range" reports after 1 day, since they can be re-processed
// and would take up unecessary space
$query = "DELETE \n \t\t\t\t\tFROM %s\n \t\t\t\t\tWHERE period = ?\n \t\t\t\t\t\tAND ts_archived < ?";
$bind = array(Piwik::$idPeriods['range'], $yesterday);
Piwik_Query(sprintf($query, $blobTable), $bind);
Piwik_Query(sprintf($query, $numericTable), $bind);
} else {
Piwik::log("Purging temporary archives: skipped.");
}
if (!isset($this->archives)) {
return;
}
foreach ($this->archives as $archive) {
destroy($archive);
}
$this->archives = array();
}
private function log($m)
{
$this->output .= $m . "\n";
Piwik::log($m);
}
/**
* Init the object before launching the real archive processing
*/
protected function initCompute()
{
$this->loadNextIdarchive();
$done = $this->getDoneStringFlag();
$this->insertNumericRecord($done, Piwik_ArchiveProcessing::DONE_ERROR);
// Can be removed when GeoIp is in core
$this->logTable = Piwik_Common::prefixTable('log_visit');
$temporary = 'definitive archive';
if($this->isArchiveTemporary())
{
$temporary = 'temporary archive';
}
Piwik::log("'" . $this->period->getLabel() . "'"
.", idSite = ". $this->idsite." ($temporary)"
.", segment = '". $this->getSegment()->getString()."'"
.", report = '". $this->getRequestedReport()."'"
.", UTC datetime [".$this->startDatetimeUTC." -> ".$this->endDatetimeUTC." ]...");
}
/**
* Uses a GeoIP database to get a visitor's location based on their IP address.
*
* This function will return different results based on the data used. If a city
* database is used, it may return the country code, region code, city name, area
* code, latitude, longitude and postal code of the visitor.
*
* Alternatively, if used with a country database, only the country code will be
* returned.
*
* @param array $info Must have an 'ip' field.
* @return array
*/
public function getLocation($info)
{
$ip = $this->getIpFromInfo($info);
$result = array();
$locationGeoIp = $this->getGeoIpInstance($key = 'loc');
if ($locationGeoIp) {
switch ($locationGeoIp->databaseType) {
case GEOIP_CITY_EDITION_REV0:
// city database type
// city database type
case GEOIP_CITY_EDITION_REV1:
case GEOIP_CITYCOMBINED_EDITION:
$location = geoip_record_by_addr($locationGeoIp, $ip);
if (!empty($location)) {
$result[self::COUNTRY_CODE_KEY] = $location->country_code;
$result[self::REGION_CODE_KEY] = $location->region;
$result[self::CITY_NAME_KEY] = utf8_encode($location->city);
$result[self::AREA_CODE_KEY] = $location->area_code;
$result[self::LATITUDE_KEY] = $location->latitude;
$result[self::LONGITUDE_KEY] = $location->longitude;
$result[self::POSTAL_CODE_KEY] = $location->postal_code;
}
break;
case GEOIP_REGION_EDITION_REV0:
// region database type
// region database type
case GEOIP_REGION_EDITION_REV1:
$location = geoip_region_by_addr($locationGeoIp, $ip);
if (!empty($location)) {
$result[self::COUNTRY_CODE_KEY] = $location[0];
$result[self::REGION_CODE_KEY] = $location[1];
}
break;
case GEOIP_COUNTRY_EDITION:
// country database type
$result[self::COUNTRY_CODE_KEY] = geoip_country_code_by_addr($locationGeoIp, $ip);
break;
default:
// unknown database type, log warning and fallback to country edition
Piwik::log("Found unrecognized database type: " . $locationGeoIp->databaseType);
$result[self::COUNTRY_CODE_KEY] = geoip_country_code_by_addr($locationGeoIp, $ip);
break;
}
}
// NOTE: ISP & ORG require commercial dbs to test. this code has been tested manually,
// but not by integration tests.
$ispGeoIp = $this->getGeoIpInstance($key = 'isp');
if ($ispGeoIp) {
$isp = geoip_org_by_addr($ispGeoIp, $ip);
if (!empty($isp)) {
$result[self::ISP_KEY] = utf8_encode($isp);
}
}
$orgGeoIp = $this->getGeoIpInstance($key = 'org');
if ($orgGeoIp) {
$org = geoip_org_by_addr($orgGeoIp, $ip);
if (!empty($org)) {
$result[self::ORG_KEY] = utf8_encode($org);
}
}
if (empty($result)) {
return false;
}
$this->completeLocationResult($result);
return $result;
}
function log($m)
{
Piwik::log($m);
}
/**
* Called at the end of the archiving process.
* Does some cleaning job in the database.
*/
protected function postCompute()
{
parent::postCompute();
$blobTable = $this->tableArchiveBlob->getTableName();
$numericTable = $this->tableArchiveNumeric->getTableName();
$key = 'lastPurge_' . $blobTable;
$timestamp = Piwik_GetOption($key);
// we shall purge temporary archives after their timeout is finished, plus an extra 2 hours
// in case archiving is disabled and is late to run, we give it this extra time to run and re-process more recent records
$temporaryArchivingTimeout = self::getTodayArchiveTimeToLive();
$purgeEveryNSeconds = $temporaryArchivingTimeout + 2 * 3600;
// we only delete archives if we are able to process them, otherwise, the browser might process reports
// when &segment= is specified (or custom date range) and would below, delete temporary archives that the
// browser is not able to process until next cron run (which could be more than 1 hour away)
if ($this->isRequestAuthorizedToArchive() && (!$timestamp || $timestamp < time() - $purgeEveryNSeconds)) {
Piwik_SetOption($key, time());
$purgeArchivesOlderThan = Piwik_Date::factory(time() - $purgeEveryNSeconds)->getDateTime();
$result = Piwik_FetchAll("\n\t\t\t\t\t\t\tSELECT idarchive\n\t\t\t\t\t\t\tFROM {$numericTable}\n\t\t\t\t\t\t\tWHERE name LIKE 'done%'\n\t\t\t\t\t\t\t\tAND value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . "\n\t\t\t\t\t\t\t\tAND ts_archived < ?", array($purgeArchivesOlderThan));
$idArchivesToDelete = array();
if (!empty($result)) {
foreach ($result as $row) {
$idArchivesToDelete[] = $row['idarchive'];
}
$query = "DELETE \n \t\t\t\t\t\tFROM %s\n \t\t\t\t\t\tWHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ")\n \t\t\t\t\t\t";
Piwik_Query(sprintf($query, $blobTable));
Piwik_Query(sprintf($query, $numericTable));
}
Piwik::log("Purging temporary archives: done [ purged archives older than {$purgeArchivesOlderThan} from {$blobTable} and {$numericTable} ] [Deleted IDs: " . implode(',', $idArchivesToDelete) . "]");
// Deleting "Custom Date Range" reports after 1 day, since they can be re-processed
// and would take up unecessary space
$yesterday = Piwik_Date::factory('yesterday')->getDateTime();
$query = "DELETE \n \t\t\t\t\tFROM %s\n \t\t\t\t\tWHERE period = ?\n \t\t\t\t\t\tAND ts_archived < ?";
$bind = array(Piwik::$idPeriods['range'], $yesterday);
Piwik::log("Purging Custom Range archives: done [ purged archives older than {$yesterday} from {$blobTable} and {$numericTable} ]");
Piwik_Query(sprintf($query, $blobTable), $bind);
Piwik_Query(sprintf($query, $numericTable), $bind);
// these tables will be OPTIMIZEd daily in a scheduled task, to claim lost space
} else {
Piwik::log("Purging temporary archives: skipped.");
}
if (!isset($this->archives)) {
return;
}
foreach ($this->archives as $archive) {
destroy($archive);
}
$this->archives = array();
}
/**
* Start the session
*
* @param array|bool $options An array of configuration options; the auto-start (bool) setting is ignored
* @return void
*/
public static function start($options = false)
{
if (Piwik_Common::isPhpCliMode() || self::$sessionStarted || defined('PIWIK_ENABLE_SESSION_START') && !PIWIK_ENABLE_SESSION_START) {
return;
}
self::$sessionStarted = true;
// use cookies to store session id on the client side
@ini_set('session.use_cookies', '1');
// prevent attacks involving session ids passed in URLs
@ini_set('session.use_only_cookies', '1');
// advise browser that session cookie should only be sent over secure connection
if (Piwik::isHttps()) {
@ini_set('session.cookie_secure', '1');
}
// advise browser that session cookie should only be accessible through the HTTP protocol (i.e., not JavaScript)
@ini_set('session.cookie_httponly', '1');
// don't use the default: PHPSESSID
$sessionName = defined('PIWIK_SESSION_NAME') ? PIWIK_SESSION_NAME : 'PIWIK_SESSID';
@ini_set('session.name', $sessionName);
// proxies may cause the referer check to fail and
// incorrectly invalidate the session
@ini_set('session.referer_check', '');
$currentSaveHandler = ini_get('session.save_handler');
$config = Piwik_Config::getInstance();
if (self::isFileBasedSessions()) {
// Note: this handler doesn't work well in load-balanced environments and may have a concurrency issue with locked session files
// for "files", use our own folder to prevent local session file hijacking
$sessionPath = PIWIK_USER_PATH . '/tmp/sessions';
// We always call mkdir since it also chmods the directory which might help when permissions were reverted for some reasons
Piwik_Common::mkdir($sessionPath);
@ini_set('session.save_handler', 'files');
@ini_set('session.save_path', $sessionPath);
} else {
if ($config->General['session_save_handler'] === 'dbtable' || in_array($currentSaveHandler, array('user', 'mm'))) {
// We consider these to be misconfigurations, in that:
// - user - we can't verify that user-defined session handler functions have already been set via session_set_save_handler()
// - mm - this handler is not recommended, unsupported, not available for Windows, and has a potential concurrency issue
$db = Zend_Registry::get('db');
$config = array('name' => Piwik_Common::prefixTable('session'), 'primary' => 'id', 'modifiedColumn' => 'modified', 'dataColumn' => 'data', 'lifetimeColumn' => 'lifetime', 'db' => $db);
$saveHandler = new Piwik_Session_SaveHandler_DbTable($config);
if ($saveHandler) {
self::setSaveHandler($saveHandler);
}
}
}
// garbage collection may disabled by default (e.g., Debian)
if (ini_get('session.gc_probability') == 0) {
@ini_set('session.gc_probability', 1);
}
try {
Zend_Session::start();
register_shutdown_function(array('Zend_Session', 'writeClose'), true);
} catch (Exception $e) {
Piwik::log('Unable to start session: ' . $e->getMessage());
$enableDbSessions = '';
if (Piwik::isInstalled()) {
$enableDbSessions = "<br/>If you still experience issues after trying these changes, \n\t\t\t \t\t\twe recommend that you <a href='http://piwik.org/faq/how-to-install/#faq_133' target='_blank'>enable database session storage</a>.";
}
$message = 'Error: ' . Piwik_Translate('General_ExceptionUnableToStartSession') . ' ' . Piwik::getErrorMessageMissingPermissions(Piwik_Common::getPathToPiwikRoot() . '/tmp/sessions/') . $enableDbSessions . "\n<pre>Debug: the original error was \n" . $e->getMessage() . "</pre>";
Piwik_ExitWithMessage($message);
}
}
/**
* Prepares the archive. Gets the idarchive from the ArchiveProcessing.
*
* This will possibly launch the archiving process if the archive was not available.
*/
public function prepareArchive()
{
$archiveJustProcessed = false;
$periodString = $this->period->getLabel();
$plugin = Piwik_ArchiveProcessing::getPluginBeingProcessed($this->getRequestedReport());
$cacheKey = 'all';
if ($periodString == 'range') {
$cacheKey = $plugin;
}
if (!isset($this->alreadyChecked[$cacheKey])) {
$this->isThereSomeVisits = false;
$this->alreadyChecked[$cacheKey] = true;
$dayString = $this->period->getPrettyString();
$logMessage = "Preparing archive: " . $periodString . "(" . $dayString . "), plugin {$plugin} ";
// if the END of the period is BEFORE the website creation date
// we already know there are no stats for this period
// we add one day to make sure we don't miss the day of the website creation
if ($this->period->getDateEnd()->addDay(2)->isEarlier($this->site->getCreationDate())) {
Piwik::log("{$logMessage} skipped, archive is before the website was created.");
return;
}
// if the starting date is in the future we know there is no visit
if ($this->period->getDateStart()->subDay(2)->isLater(Piwik_Date::today())) {
Piwik::log("{$logMessage} skipped, archive is after today.");
return;
}
// we make sure the archive is available for the given date
$periodLabel = $this->period->getLabel();
$this->archiveProcessing = Piwik_ArchiveProcessing::factory($periodLabel);
$this->archiveProcessing->setSite($this->site);
$this->archiveProcessing->setPeriod($this->period);
$this->archiveProcessing->setSegment($this->segment);
$this->archiveProcessing->init();
$this->archiveProcessing->setRequestedReport($this->getRequestedReport());
$archivingDisabledArchiveNotProcessed = false;
$idArchive = $this->archiveProcessing->loadArchive();
if (empty($idArchive)) {
if ($this->archiveProcessing->isArchivingDisabled()) {
$archivingDisabledArchiveNotProcessed = true;
$logMessage = "* ARCHIVING DISABLED, for {$logMessage}";
} else {
Piwik::log("* PROCESSING {$logMessage}, not archived yet...");
$archiveJustProcessed = true;
// Process the reports
$this->archiveProcessing->launchArchiving();
$idArchive = $this->archiveProcessing->getIdArchive();
$logMessage = "PROCESSED: idArchive = " . $idArchive . ", for {$logMessage}";
}
} else {
$logMessage = "* ALREADY PROCESSED, Fetching idArchive = {$idArchive} (idSite=" . $this->site->getId() . "), for {$logMessage}";
}
Piwik::log("{$logMessage}, Visits = " . $this->archiveProcessing->getNumberOfVisits());
$this->isThereSomeVisits = !$archivingDisabledArchiveNotProcessed && $this->archiveProcessing->isThereSomeVisits();
$this->idArchive = $idArchive;
}
return $archiveJustProcessed;
}
/**
* Prepares the archive. Gets the idarchive from the ArchiveProcessing.
*
* This will possibly launch the archiving process if the archive was not available.
* @return bool
*/
public function prepareArchive()
{
$archiveJustProcessed = false;
$periodString = $this->period->getLabel();
$plugin = Piwik_ArchiveProcessing::getPluginBeingProcessed($this->getRequestedReport());
$cacheKey = 'all';
if ($periodString == 'range') {
$cacheKey = $plugin;
}
if (!isset($this->alreadyChecked[$cacheKey])) {
$this->isThereSomeVisits = false;
$this->alreadyChecked[$cacheKey] = true;
$dayString = $this->period->getPrettyString();
$logMessage = sprintf("%s (%s), plugin %s", $periodString, $dayString, $plugin);
// if the END of the period is BEFORE the website creation date
// we already know there are no stats for this period
// we add one day to make sure we don't miss the day of the website creation
if ($this->period->getDateEnd()->addDay(2)->isEarlier($this->site->getCreationDate())) {
Piwik::log(sprintf("Archive %s skipped, archive is before the website was created.", $logMessage));
return;
}
// if the starting date is in the future we know there is no visit
if ($this->period->getDateStart()->subDay(2)->isLater(Piwik_Date::today())) {
Piwik::log(sprintf("Archive %s skipped, archive is after today.", $logMessage));
return;
}
// we make sure the archive is available for the given date
$periodLabel = $this->period->getLabel();
$this->archiveProcessing = Piwik_ArchiveProcessing::factory($periodLabel);
$this->archiveProcessing->setSite($this->site);
$this->archiveProcessing->setPeriod($this->period);
$this->archiveProcessing->setSegment($this->segment);
$this->archiveProcessing->init();
$this->archiveProcessing->setRequestedReport($this->getRequestedReport());
$archivingDisabledArchiveNotProcessed = false;
$idArchive = $this->archiveProcessing->loadArchive();
if (empty($idArchive)) {
if ($this->archiveProcessing->isArchivingDisabled()) {
$archivingDisabledArchiveNotProcessed = true;
$logMessage = sprintf("Archiving disabled, for %s", $logMessage);
} else {
Piwik::log(sprintf("Processing %s, not archived yet...", $logMessage));
$archiveJustProcessed = true;
// Process the reports
$this->archiveProcessing->launchArchiving();
$idArchive = $this->archiveProcessing->getIdArchive();
$logMessage = sprintf("Processed %d, for %s", $idArchive, $logMessage);
}
} else {
$logMessage = sprintf("Already processed, fetching idArchive = %d (idSite=%d), for %s", $idArchive, $this->site->getId(), $logMessage);
}
Piwik::log(sprintf("%s, Visits = %d", $logMessage, $this->archiveProcessing->getNumberOfVisits()));
$this->isThereSomeVisits = !$archivingDisabledArchiveNotProcessed && $this->archiveProcessing->isThereSomeVisits();
$this->idArchive = $idArchive;
}
return $archiveJustProcessed;
}
public static function isArchivingDisabledFor($segment, $period)
{
$processOneReportOnly = !self::shouldProcessReportsAllPluginsFor($segment, $period);
if ($processOneReportOnly) {
// When there is a segment, archiving is not necessary allowed
// If browser archiving is allowed, then archiving is enabled
// if browser archiving is not allowed, then archiving is disabled
if (!$segment->isEmpty() && !self::isRequestAuthorizedToArchive() && Piwik_Config::getInstance()->General['browser_archiving_disabled_enforce']) {
Piwik::log("Archiving is disabled because of config setting browser_archiving_disabled_enforce=1");
return true;
}
return false;
}
$isDisabled = !self::isRequestAuthorizedToArchive();
return $isDisabled;
}
/**
* Given a monthly archive table, will delete all reports that are now outdated,
* or reports that ended with an error
*/
public static function doPurgeOutdatedArchives($numericTable)
{
$blobTable = str_replace("numeric", "blob", $numericTable);
$key = self::FLAG_TABLE_PURGED . $blobTable;
$timestamp = Piwik_GetOption($key);
// we shall purge temporary archives after their timeout is finished, plus an extra 6 hours
// in case archiving is disabled or run once a day, we give it this extra time to run
// and re-process more recent records...
// TODO: Instead of hardcoding 6 we should put the actual number of hours between 2 archiving runs
$temporaryArchivingTimeout = self::getTodayArchiveTimeToLive();
$purgeEveryNSeconds = max($temporaryArchivingTimeout, 6 * 3600);
// we only delete archives if we are able to process them, otherwise, the browser might process reports
// when &segment= is specified (or custom date range) and would below, delete temporary archives that the
// browser is not able to process until next cron run (which could be more than 1 hour away)
if (self::isRequestAuthorizedToArchive() && (!$timestamp || $timestamp < time() - $purgeEveryNSeconds)) {
Piwik_SetOption($key, time());
// If Browser Archiving is enabled, it is likely there are many more temporary archives
// We delete more often which is safe, since reports are re-processed on demand
if (self::isBrowserTriggerArchivingEnabled()) {
$purgeArchivesOlderThan = Piwik_Date::factory(time() - 2 * $temporaryArchivingTimeout)->getDateTime();
} else {
$purgeArchivesOlderThan = Piwik_Date::factory('today')->getDateTime();
}
$result = Piwik_FetchAll("\n\t\t\t\tSELECT idarchive\n\t\t\t\tFROM {$numericTable}\n\t\t\t\tWHERE name LIKE 'done%'\n\t\t\t\t\tAND (( value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . "\n\t\t\t\t\t\t AND ts_archived < ?)\n\t\t\t\t\t\t OR value = " . Piwik_ArchiveProcessing::DONE_ERROR . ")", array($purgeArchivesOlderThan));
$idArchivesToDelete = array();
if (!empty($result)) {
foreach ($result as $row) {
$idArchivesToDelete[] = $row['idarchive'];
}
$query = "DELETE \n \t\t\t\t\t\tFROM %s\n \t\t\t\t\t\tWHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ")\n \t\t\t\t\t\t";
Piwik_Query(sprintf($query, $numericTable));
// Individual blob tables could be missing
try {
Piwik_Query(sprintf($query, $blobTable));
} catch (Exception $e) {
}
}
Piwik::log("Purging temporary archives: done [ purged archives older than {$purgeArchivesOlderThan} from {$blobTable} and {$numericTable} ] [Deleted IDs: " . implode(',', $idArchivesToDelete) . "]");
// Deleting "Custom Date Range" reports after 1 day, since they can be re-processed
// and would take up unecessary space
$yesterday = Piwik_Date::factory('yesterday')->getDateTime();
$query = "DELETE \n \t\t\t\t\tFROM %s\n \t\t\t\t\tWHERE period = ?\n \t\t\t\t\t\tAND ts_archived < ?";
$bind = array(Piwik::$idPeriods['range'], $yesterday);
Piwik::log("Purging Custom Range archives: done [ purged archives older than {$yesterday} from {$blobTable} and {$numericTable} ]");
Piwik_Query(sprintf($query, $numericTable), $bind);
// Individual blob tables could be missing
try {
Piwik_Query(sprintf($query, $blobTable), $bind);
} catch (Exception $e) {
}
// these tables will be OPTIMIZEd daily in a scheduled task, to claim lost space
} else {
Piwik::log("Purging temporary archives: skipped.");
}
}
/**
* Performs a batch insert into a specific table using either LOAD DATA INFILE or plain INSERTs,
* as a fallback. On MySQL, LOAD DATA INFILE is 20x faster than a series of plain INSERTs.
*
* @param string $tableName PREFIXED table name! you must call Piwik_Common::prefixTable() before passing the table name
* @param array $fields array of unquoted field names
* @param array $values array of data to be inserted
* @return bool True if the bulk LOAD was used, false if we fallback to plain INSERTs
*/
public static function tableInsertBatch($tableName, $fields, $values)
{
$fieldList = '(' . join(',', $fields) . ')';
try {
// throw new Exception('');
$filePath = PIWIK_USER_PATH . '/' . Piwik_AssetManager::MERGED_FILE_DIR . $tableName . '-' . Piwik_Common::generateUniqId() . '.csv';
if (Piwik_Common::isWindows()) {
// On windows, MySQL expects slashes as directory separators
$filePath = str_replace('\\', '/', $filePath);
}
// Set up CSV delimiters, quotes, etc
$delim = "\t";
$quote = '"';
$eol = "\r\n";
$null = 'NULL';
$escape = '\\\\';
$fp = fopen($filePath, 'wb');
if (!$fp) {
throw new Exception('Error creating the tmp file ' . $filePath . ', please check that the webserver has write permission to write this file.');
}
@chmod($filePath, 0777);
foreach ($values as $row) {
$output = '';
foreach ($row as $value) {
if (!isset($value) || is_null($value) || $value === false) {
$output .= $null . $delim;
} else {
$output .= $quote . self::escapeString($value) . $quote . $delim;
}
}
// Replace delim with eol
unset($row[strlen($output) - strlen($delim)]);
$output .= $eol;
$ret = fwrite($fp, $output);
if (!$ret) {
fclose($fp);
unlink($filePath);
throw new Exception('Error writing to the tmp file ' . $filePath . ' containing the batch INSERTs.');
}
}
fclose($fp);
$query = "\n\t\t\t\t\t'{$filePath}'\n\t\t\t\tREPLACE\n\t\t\t\tINTO TABLE\n\t\t\t\t\t" . $tableName;
// hack for charset mismatch
if (!self::isDatabaseConnectionUTF8() && !isset(Zend_Registry::get('config')->database->charset)) {
$query .= ' CHARACTER SET latin1';
}
$query .= "\n\t\t\t\tFIELDS TERMINATED BY\n\t\t\t\t\t'" . $delim . "'\n\t\t\t\tENCLOSED BY\n\t\t\t\t\t'" . $quote . "'\n\t\t\t\tESCAPED BY\n\t\t\t\t\t'" . $escape . "'\n\t\t\t\tLINES TERMINATED BY\n\t\t\t\t\t\"" . $eol . "\"\n\t\t\t\t{$fieldList}\n\t\t\t";
// initial attempt with LOCAL keyword
// note: may trigger a known PHP PDO_MYSQL bug when MySQL not built with --enable-local-infile
// @see http://bugs.php.net/bug.php?id=54158
try {
$result = @Piwik_Exec('LOAD DATA LOCAL INFILE' . $query);
if (empty($result)) {
throw new Exception("LOAD DATA LOCAL INFILE failed!");
}
unlink($filePath);
return true;
} catch (Exception $e) {
}
// second attempt without LOCAL keyword if MySQL server appears to be on the same box
// note: requires that the db user have the FILE privilege; however, since this is
// a global privilege, it may not be granted due to security concerns
$dbHost = Zend_Registry::get('config')->database->host;
$localHosts = array('127.0.0.1', 'localhost', 'localhost.local', 'localhost.localdomain', 'localhost.localhost');
$hostName = @php_uname('n');
if (!empty($hostName)) {
$localHosts = array_merge($localHosts, array($hostName, $hostName . '.local', $hostName . '.localdomain', $hostName . '.localhost'));
}
if (!empty($dbHost) && !in_array($dbHost, $localHosts)) {
throw new Exception("MYSQL appears to be on a remote server");
}
$result = @Piwik_Exec('LOAD DATA INFILE' . $query);
if (empty($result)) {
throw new Exception("LOAD DATA INFILE failed!");
}
unlink($filePath);
return true;
} catch (Exception $e) {
Piwik::log("LOAD DATA INFILE failed or not supported, falling back to normal INSERTs... Error was:" . $e->getMessage(), Piwik_Log::WARN);
// if all else fails, fallback to a series of INSERTs
unlink($filePath);
self::tableInsertBatchIterate($tableName, $fields, $values);
}
return false;
}
