/** * Purges old data from the following tables: * - log_visit * - log_link_visit_action * - log_conversion * - log_conversion_item * - log_action */ public function purgeData() { $maxIdVisit = $this->getDeleteIdVisitOffset(); // break if no ID was found (nothing to delete for given period) if (empty($maxIdVisit)) { return; } $logTables = self::getDeleteTableLogTables(); // delete data from log tables $where = "WHERE idvisit <= ?"; foreach ($logTables as $logTable) { // deleting from log_action must be handled differently, so we do it later if ($logTable != Piwik_Common::prefixTable('log_action')) { Piwik_DeleteAllRows($logTable, $where, $this->maxRowsToDeletePerQuery, array($maxIdVisit)); } } // delete unused actions from the log_action table (but only if we can lock tables) if (Piwik::isLockPrivilegeGranted()) { $this->purgeUnusedLogActions(); } else { $logMessage = get_class($this) . ": LOCK TABLES privilege not granted; skipping unused actions purge"; Piwik::log($logMessage); } // optimize table overhead after deletion Piwik_OptimizeTables($logTables); }
/** * @see Piwik_ViewDataTable::main() * @throws Exception|Piwik_Access_NoAccessException * @return null */ public function main() { if ($this->mainAlreadyExecuted) { return; } $this->mainAlreadyExecuted = true; $this->isDataAvailable = true; try { $this->loadDataTableFromAPI(); } catch (Piwik_Access_NoAccessException $e) { throw $e; } catch (Exception $e) { Piwik::log("Failed to get data from API: " . $e->getMessage()); $this->isDataAvailable = false; } $this->postDataTableLoadedFromAPI(); $this->view = $this->buildView(); }
public static function printMemoryUsage($prefixString = null) { $memory = false; if (function_exists('xdebug_memory_usage')) { $memory = xdebug_memory_usage(); } elseif (function_exists('memory_get_usage')) { $memory = memory_get_usage(); } if ($memory !== false) { $usage = round($memory / 1024 / 1024, 2); if (!is_null($prefixString)) { Piwik::log($prefixString); } Piwik::log("Memory usage = {$usage} Mb"); } else { Piwik::log("Memory usage function not found."); } }
/** * Must be called before dispatch() * - checks that directories are writable, * - loads the configuration file, * - loads the plugin, * - inits the DB connection, * - etc. */ function init() { static $initialized = false; if($initialized) { return; } $initialized = true; try { Zend_Registry::set('timer', new Piwik_Timer); $directoriesToCheck = array( '/tmp/', '/tmp/templates_c/', '/tmp/cache/', '/tmp/assets/' ); Piwik::checkDirectoriesWritableOrDie($directoriesToCheck); Piwik_Common::assignCliParametersToRequest(); Piwik_Translate::getInstance()->loadEnglishTranslation(); $exceptionToThrow = false; try { Piwik::createConfigObject(); } catch(Exception $e) { Piwik_PostEvent('FrontController.NoConfigurationFile', $e, $info = array(), $pending = true); $exceptionToThrow = $e; } if(Zend_Registry::get('config')->General->maintenance_mode == 1 && !Piwik_Common::isPhpCliMode()) { throw new Exception("Piwik is in scheduled maintenance. Please come back later."); } $pluginsManager = Piwik_PluginsManager::getInstance(); $pluginsManager->loadPlugins( Zend_Registry::get('config')->Plugins->Plugins->toArray() ); if($exceptionToThrow) { throw $exceptionToThrow; } try { Piwik::createDatabaseObject(); } catch(Exception $e) { Piwik_PostEvent('FrontController.badConfigurationFile', $e, $info = array(), $pending = true); throw $e; } Piwik::createLogObject(); // creating the access object, so that core/Updates/* can enforce Super User and use some APIs Piwik::createAccessObject(); Piwik_PostEvent('FrontController.dispatchCoreAndPluginUpdatesScreen'); Piwik_PluginsManager::getInstance()->installLoadedPlugins(); Piwik::install(); // ensure the current Piwik URL is known for later use if(method_exists('Piwik', 'getPiwikUrl')) { $host = Piwik::getPiwikUrl(); } Piwik_PostEvent('FrontController.initAuthenticationObject'); try { $authAdapter = Zend_Registry::get('auth'); } catch(Exception $e){ throw new Exception("Authentication object cannot be found in the Registry. Maybe the Login plugin is not activated? <br />You can activate the plugin by adding:<br /> <code>Plugins[] = Login</code><br /> under the <code>[Plugins]</code> section in your config/config.inc.php"); } Zend_Registry::get('access')->reloadAccess($authAdapter); Piwik_Translate::getInstance()->reloadLanguage(); Piwik::raiseMemoryLimitIfNecessary(); $pluginsManager->postLoadPlugins(); Piwik_PostEvent('FrontController.checkForUpdates'); } catch(Exception $e) { Piwik_ExitWithMessage($e->getMessage(), false, true); } Piwik::log('End FrontController->init() - Request: '. var_export($_REQUEST, true)); }
/** * Activate the specified plugin and install (if needed) * * @param string $pluginName Name of plugin * @throws Exception */ public function activatePlugin($pluginName) { $plugins = Piwik_Config::getInstance()->Plugins['Plugins']; if (in_array($pluginName, $plugins)) { throw new Exception("Plugin '{$pluginName}' already activated."); } $existingPlugins = $this->readPluginsDirectory(); if (array_search($pluginName, $existingPlugins) === false) { Piwik::log("Unable to find the plugin '{$pluginName}' in activatePlugin."); return; } $plugin = $this->loadPlugin($pluginName); if ($plugin === null) { return; } $this->installPluginIfNecessary($plugin); $plugin->activate(); // we add the plugin to the list of activated plugins if (!in_array($pluginName, $plugins)) { $plugins[] = $pluginName; } else { // clean up if we find a dupe $plugins = array_unique($plugins); } // the config file will automatically be saved with the new plugin $this->updatePluginsConfig($plugins); Piwik_Config::getInstance()->forceSave(); // Delete merged js/css files to force regenerations to include the activated plugin Piwik::deleteAllCacheOnUpdate(); }
/** * Performs a batch insert into a specific table using either LOAD DATA INFILE or plain INSERTs, * as a fallback. On MySQL, LOAD DATA INFILE is 20x faster than a series of plain INSERTs. * * @param string $tableName PREFIXED table name! you must call Piwik_Common::prefixTable() before passing the table name * @param array $fields array of unquoted field names * @param array $values array of data to be inserted * @return bool True if the bulk LOAD was used, false if we fallback to plain INSERTs */ public static function tableInsertBatch($tableName, $fields, $values) { $filePath = PIWIK_USER_PATH . '/' . Piwik_AssetManager::MERGED_FILE_DIR . $tableName . '-' . Piwik_Common::generateUniqId() . '.csv'; if (Zend_Registry::get('db')->hasBulkLoader()) { try { // throw new Exception(''); $fileSpec = array('delim' => "\t", 'quote' => '"', 'escape' => '\\\\', 'escapespecial_cb' => create_function('$str', 'return str_replace(array(chr(92), chr(34)), array(chr(92).chr(92), chr(92).chr(34)), $str);'), 'eol' => "\r\n", 'null' => 'NULL'); // hack for charset mismatch if (!self::isDatabaseConnectionUTF8() && !isset(Zend_Registry::get('config')->database->charset)) { $fileSpec['charset'] = 'latin1'; } self::createCSVFile($filePath, $fileSpec, $values); $rc = self::createTableFromCSVFile($tableName, $fields, $filePath, $fileSpec); if ($rc) { unlink($filePath); return true; } throw new Exception('unknown cause'); } catch (Exception $e) { Piwik::log("LOAD DATA INFILE failed or not supported, falling back to normal INSERTs... Error was:" . $e->getMessage()); } } // if all else fails, fallback to a series of INSERTs @unlink($filePath); self::tableInsertBatchIterate($tableName, $fields, $values); return false; }
public static function start($options = false) { if(Piwik_Common::isPhpCliMode() || version_compare(Piwik_GetOption('version_core'), '1.5-b5') < 0) { return; } // use cookies to store session id on the client side @ini_set('session.use_cookies', '1'); // prevent attacks involving session ids passed in URLs @ini_set('session.use_only_cookies', '1'); // advise browser that session cookie should only be sent over secure connection if(Piwik_Url::getCurrentScheme() === 'https') { @ini_set('session.cookie_secure', '1'); } // advise browser that session cookie should only be accessible through the HTTP protocol (i.e., not JavaScript) @ini_set('session.cookie_httponly', '1'); // don't use the default: PHPSESSID $sessionName = defined('PIWIK_SESSION_NAME') ? PIWIK_SESSION_NAME : 'PIWIK_SESSID'; @ini_set('session.name', $sessionName); // proxies may cause the referer check to fail and // incorrectly invalidate the session @ini_set('session.referer_check', ''); // we consider these to be misconfigurations, in that // - user - we can't verify that user-defined session handler functions have been set via session_set_save_handler() // - mm - this handler is not recommended, unsupported, not available for Windows, and has a potential concurrency issue // - files - this handler doesn't work well in load-balanced environments and may have a concurrency issue with locked session files $currentSaveHandler = ini_get('session.save_handler'); if(in_array($currentSaveHandler, array('user', 'mm', 'files'))) { $db = Zend_Registry::get('db'); $config = array( 'name' => Piwik_Common::prefixTable('session'), 'primary' => 'id', 'modifiedColumn' => 'modified', 'dataColumn' => 'data', 'lifetimeColumn' => 'lifetime', 'db' => $db, ); $saveHandler = new Piwik_Session_SaveHandler_DbTable($config); if($saveHandler) { self::setSaveHandler($saveHandler); } } // garbage collection may disabled by default (e.g., Debian) if(ini_get('session.gc_probability') == 0) { @ini_set('session.gc_probability', 1); } try { Zend_Session::start(); register_shutdown_function(array('Zend_Session', 'writeClose'), true); } catch(Exception $e) { Piwik::log('Unable to start session: ' . $e->getMessage()); Piwik_ExitWithMessage(Piwik_Translate('General_ExceptionUnableToStartSession')); } }
$location = $provider->getLocation(array('ip' => $ip)); if (!empty($location[Piwik_UserCountry_LocationProvider::COUNTRY_CODE_KEY])) { $location[Piwik_UserCountry_LocationProvider::COUNTRY_CODE_KEY] = strtolower($location[Piwik_UserCountry_LocationProvider::COUNTRY_CODE_KEY]); } $row['location_country'] = strtolower($row['location_country']); $columnsToSet = array(); $bind = array(); foreach ($logVisitFieldsToUpdate as $column => $locationKey) { if (!empty($location[$locationKey]) && $location[$locationKey] != $row[$column]) { $columnsToSet[] = $column . ' = ?'; $bind[] = $location[$locationKey]; } } if (empty($columnsToSet)) { continue; } $bind[] = $row['idvisit']; // update log_visit $sql = "UPDATE " . Piwik_Common::prefixTable('log_visit') . "\n\t\t\t\t SET " . implode(', ', $columnsToSet) . "\n\t\t\t\t WHERE idvisit = ?"; Piwik_Query($sql, $bind); // update log_conversion $sql = "UPDATE " . Piwik_Common::prefixTable('log_conversion') . "\n\t\t\t\t SET " . implode(', ', $columnsToSet) . "\n\t\t\t\t WHERE idvisit = ?"; Piwik_Query($sql, $bind); } Piwik::log(round($start * 100 / $count) . "% done..."); flush(); } Piwik::log("100% done!"); Piwik::log(""); Piwik::log("[note] Now that you've geolocated your old visits, you need to force your reports to be re-processed. See this FAQ entry: http://piwik.org/faq/how-to/#faq_59");
/** * Called at the end of the archiving process. * Does some cleaning job in the database. */ protected function postCompute() { parent::postCompute(); $blobTable = $this->tableArchiveBlob->getTableName(); $numericTable = $this->tableArchiveNumeric->getTableName(); $key = 'lastPurge_' . $blobTable; $timestamp = Piwik_GetOption($key); if (!$timestamp || $timestamp < time() - 86400) { Piwik_SetOption($key, time()); // we delete out of date daily archives from table, maximum once per day // we only delete archives processed that are older than 1 day, to not delete archives we just processed $yesterday = Piwik_Date::factory('yesterday')->getDateTime(); $result = Piwik_FetchAll("\n\t\t\t\t\t\t\tSELECT idarchive\n\t\t\t\t\t\t\tFROM {$numericTable}\n\t\t\t\t\t\t\tWHERE name LIKE 'done%'\n\t\t\t\t\t\t\t\tAND value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . "\n\t\t\t\t\t\t\t\tAND ts_archived < ?", array($yesterday)); $idArchivesToDelete = array(); if (!empty($result)) { foreach ($result as $row) { $idArchivesToDelete[] = $row['idarchive']; } $query = "DELETE \n \t\t\t\t\t\tFROM %s\n \t\t\t\t\t\tWHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ")\n \t\t\t\t\t\t"; Piwik_Query(sprintf($query, $blobTable)); Piwik_Query(sprintf($query, $numericTable)); } Piwik::log("Purging temporary archives: done [ purged archives older than {$yesterday} from {$blobTable} and {$numericTable} ] [Deleted IDs: " . implode(',', $idArchivesToDelete) . "]"); // Deleting "Custom Date Range" reports after 1 day, since they can be re-processed // and would take up unecessary space $query = "DELETE \n \t\t\t\t\tFROM %s\n \t\t\t\t\tWHERE period = ?\n \t\t\t\t\t\tAND ts_archived < ?"; $bind = array(Piwik::$idPeriods['range'], $yesterday); Piwik_Query(sprintf($query, $blobTable), $bind); Piwik_Query(sprintf($query, $numericTable), $bind); } else { Piwik::log("Purging temporary archives: skipped."); } if (!isset($this->archives)) { return; } foreach ($this->archives as $archive) { destroy($archive); } $this->archives = array(); }
private function log($m) { $this->output .= $m . "\n"; Piwik::log($m); }
/** * Init the object before launching the real archive processing */ protected function initCompute() { $this->loadNextIdarchive(); $done = $this->getDoneStringFlag(); $this->insertNumericRecord($done, Piwik_ArchiveProcessing::DONE_ERROR); // Can be removed when GeoIp is in core $this->logTable = Piwik_Common::prefixTable('log_visit'); $temporary = 'definitive archive'; if($this->isArchiveTemporary()) { $temporary = 'temporary archive'; } Piwik::log("'" . $this->period->getLabel() . "'" .", idSite = ". $this->idsite." ($temporary)" .", segment = '". $this->getSegment()->getString()."'" .", report = '". $this->getRequestedReport()."'" .", UTC datetime [".$this->startDatetimeUTC." -> ".$this->endDatetimeUTC." ]..."); }
/** * Uses a GeoIP database to get a visitor's location based on their IP address. * * This function will return different results based on the data used. If a city * database is used, it may return the country code, region code, city name, area * code, latitude, longitude and postal code of the visitor. * * Alternatively, if used with a country database, only the country code will be * returned. * * @param array $info Must have an 'ip' field. * @return array */ public function getLocation($info) { $ip = $this->getIpFromInfo($info); $result = array(); $locationGeoIp = $this->getGeoIpInstance($key = 'loc'); if ($locationGeoIp) { switch ($locationGeoIp->databaseType) { case GEOIP_CITY_EDITION_REV0: // city database type // city database type case GEOIP_CITY_EDITION_REV1: case GEOIP_CITYCOMBINED_EDITION: $location = geoip_record_by_addr($locationGeoIp, $ip); if (!empty($location)) { $result[self::COUNTRY_CODE_KEY] = $location->country_code; $result[self::REGION_CODE_KEY] = $location->region; $result[self::CITY_NAME_KEY] = utf8_encode($location->city); $result[self::AREA_CODE_KEY] = $location->area_code; $result[self::LATITUDE_KEY] = $location->latitude; $result[self::LONGITUDE_KEY] = $location->longitude; $result[self::POSTAL_CODE_KEY] = $location->postal_code; } break; case GEOIP_REGION_EDITION_REV0: // region database type // region database type case GEOIP_REGION_EDITION_REV1: $location = geoip_region_by_addr($locationGeoIp, $ip); if (!empty($location)) { $result[self::COUNTRY_CODE_KEY] = $location[0]; $result[self::REGION_CODE_KEY] = $location[1]; } break; case GEOIP_COUNTRY_EDITION: // country database type $result[self::COUNTRY_CODE_KEY] = geoip_country_code_by_addr($locationGeoIp, $ip); break; default: // unknown database type, log warning and fallback to country edition Piwik::log("Found unrecognized database type: " . $locationGeoIp->databaseType); $result[self::COUNTRY_CODE_KEY] = geoip_country_code_by_addr($locationGeoIp, $ip); break; } } // NOTE: ISP & ORG require commercial dbs to test. this code has been tested manually, // but not by integration tests. $ispGeoIp = $this->getGeoIpInstance($key = 'isp'); if ($ispGeoIp) { $isp = geoip_org_by_addr($ispGeoIp, $ip); if (!empty($isp)) { $result[self::ISP_KEY] = utf8_encode($isp); } } $orgGeoIp = $this->getGeoIpInstance($key = 'org'); if ($orgGeoIp) { $org = geoip_org_by_addr($orgGeoIp, $ip); if (!empty($org)) { $result[self::ORG_KEY] = utf8_encode($org); } } if (empty($result)) { return false; } $this->completeLocationResult($result); return $result; }
function log($m) { Piwik::log($m); }
/** * Called at the end of the archiving process. * Does some cleaning job in the database. */ protected function postCompute() { parent::postCompute(); $blobTable = $this->tableArchiveBlob->getTableName(); $numericTable = $this->tableArchiveNumeric->getTableName(); $key = 'lastPurge_' . $blobTable; $timestamp = Piwik_GetOption($key); // we shall purge temporary archives after their timeout is finished, plus an extra 2 hours // in case archiving is disabled and is late to run, we give it this extra time to run and re-process more recent records $temporaryArchivingTimeout = self::getTodayArchiveTimeToLive(); $purgeEveryNSeconds = $temporaryArchivingTimeout + 2 * 3600; // we only delete archives if we are able to process them, otherwise, the browser might process reports // when &segment= is specified (or custom date range) and would below, delete temporary archives that the // browser is not able to process until next cron run (which could be more than 1 hour away) if ($this->isRequestAuthorizedToArchive() && (!$timestamp || $timestamp < time() - $purgeEveryNSeconds)) { Piwik_SetOption($key, time()); $purgeArchivesOlderThan = Piwik_Date::factory(time() - $purgeEveryNSeconds)->getDateTime(); $result = Piwik_FetchAll("\n\t\t\t\t\t\t\tSELECT idarchive\n\t\t\t\t\t\t\tFROM {$numericTable}\n\t\t\t\t\t\t\tWHERE name LIKE 'done%'\n\t\t\t\t\t\t\t\tAND value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . "\n\t\t\t\t\t\t\t\tAND ts_archived < ?", array($purgeArchivesOlderThan)); $idArchivesToDelete = array(); if (!empty($result)) { foreach ($result as $row) { $idArchivesToDelete[] = $row['idarchive']; } $query = "DELETE \n \t\t\t\t\t\tFROM %s\n \t\t\t\t\t\tWHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ")\n \t\t\t\t\t\t"; Piwik_Query(sprintf($query, $blobTable)); Piwik_Query(sprintf($query, $numericTable)); } Piwik::log("Purging temporary archives: done [ purged archives older than {$purgeArchivesOlderThan} from {$blobTable} and {$numericTable} ] [Deleted IDs: " . implode(',', $idArchivesToDelete) . "]"); // Deleting "Custom Date Range" reports after 1 day, since they can be re-processed // and would take up unecessary space $yesterday = Piwik_Date::factory('yesterday')->getDateTime(); $query = "DELETE \n \t\t\t\t\tFROM %s\n \t\t\t\t\tWHERE period = ?\n \t\t\t\t\t\tAND ts_archived < ?"; $bind = array(Piwik::$idPeriods['range'], $yesterday); Piwik::log("Purging Custom Range archives: done [ purged archives older than {$yesterday} from {$blobTable} and {$numericTable} ]"); Piwik_Query(sprintf($query, $blobTable), $bind); Piwik_Query(sprintf($query, $numericTable), $bind); // these tables will be OPTIMIZEd daily in a scheduled task, to claim lost space } else { Piwik::log("Purging temporary archives: skipped."); } if (!isset($this->archives)) { return; } foreach ($this->archives as $archive) { destroy($archive); } $this->archives = array(); }
/** * Start the session * * @param array|bool $options An array of configuration options; the auto-start (bool) setting is ignored * @return void */ public static function start($options = false) { if (Piwik_Common::isPhpCliMode() || self::$sessionStarted || defined('PIWIK_ENABLE_SESSION_START') && !PIWIK_ENABLE_SESSION_START) { return; } self::$sessionStarted = true; // use cookies to store session id on the client side @ini_set('session.use_cookies', '1'); // prevent attacks involving session ids passed in URLs @ini_set('session.use_only_cookies', '1'); // advise browser that session cookie should only be sent over secure connection if (Piwik::isHttps()) { @ini_set('session.cookie_secure', '1'); } // advise browser that session cookie should only be accessible through the HTTP protocol (i.e., not JavaScript) @ini_set('session.cookie_httponly', '1'); // don't use the default: PHPSESSID $sessionName = defined('PIWIK_SESSION_NAME') ? PIWIK_SESSION_NAME : 'PIWIK_SESSID'; @ini_set('session.name', $sessionName); // proxies may cause the referer check to fail and // incorrectly invalidate the session @ini_set('session.referer_check', ''); $currentSaveHandler = ini_get('session.save_handler'); $config = Piwik_Config::getInstance(); if (self::isFileBasedSessions()) { // Note: this handler doesn't work well in load-balanced environments and may have a concurrency issue with locked session files // for "files", use our own folder to prevent local session file hijacking $sessionPath = PIWIK_USER_PATH . '/tmp/sessions'; // We always call mkdir since it also chmods the directory which might help when permissions were reverted for some reasons Piwik_Common::mkdir($sessionPath); @ini_set('session.save_handler', 'files'); @ini_set('session.save_path', $sessionPath); } else { if ($config->General['session_save_handler'] === 'dbtable' || in_array($currentSaveHandler, array('user', 'mm'))) { // We consider these to be misconfigurations, in that: // - user - we can't verify that user-defined session handler functions have already been set via session_set_save_handler() // - mm - this handler is not recommended, unsupported, not available for Windows, and has a potential concurrency issue $db = Zend_Registry::get('db'); $config = array('name' => Piwik_Common::prefixTable('session'), 'primary' => 'id', 'modifiedColumn' => 'modified', 'dataColumn' => 'data', 'lifetimeColumn' => 'lifetime', 'db' => $db); $saveHandler = new Piwik_Session_SaveHandler_DbTable($config); if ($saveHandler) { self::setSaveHandler($saveHandler); } } } // garbage collection may disabled by default (e.g., Debian) if (ini_get('session.gc_probability') == 0) { @ini_set('session.gc_probability', 1); } try { Zend_Session::start(); register_shutdown_function(array('Zend_Session', 'writeClose'), true); } catch (Exception $e) { Piwik::log('Unable to start session: ' . $e->getMessage()); $enableDbSessions = ''; if (Piwik::isInstalled()) { $enableDbSessions = "<br/>If you still experience issues after trying these changes, \n\t\t\t \t\t\twe recommend that you <a href='http://piwik.org/faq/how-to-install/#faq_133' target='_blank'>enable database session storage</a>."; } $message = 'Error: ' . Piwik_Translate('General_ExceptionUnableToStartSession') . ' ' . Piwik::getErrorMessageMissingPermissions(Piwik_Common::getPathToPiwikRoot() . '/tmp/sessions/') . $enableDbSessions . "\n<pre>Debug: the original error was \n" . $e->getMessage() . "</pre>"; Piwik_ExitWithMessage($message); } }
/** * Prepares the archive. Gets the idarchive from the ArchiveProcessing. * * This will possibly launch the archiving process if the archive was not available. */ public function prepareArchive() { $archiveJustProcessed = false; $periodString = $this->period->getLabel(); $plugin = Piwik_ArchiveProcessing::getPluginBeingProcessed($this->getRequestedReport()); $cacheKey = 'all'; if ($periodString == 'range') { $cacheKey = $plugin; } if (!isset($this->alreadyChecked[$cacheKey])) { $this->isThereSomeVisits = false; $this->alreadyChecked[$cacheKey] = true; $dayString = $this->period->getPrettyString(); $logMessage = "Preparing archive: " . $periodString . "(" . $dayString . "), plugin {$plugin} "; // if the END of the period is BEFORE the website creation date // we already know there are no stats for this period // we add one day to make sure we don't miss the day of the website creation if ($this->period->getDateEnd()->addDay(2)->isEarlier($this->site->getCreationDate())) { Piwik::log("{$logMessage} skipped, archive is before the website was created."); return; } // if the starting date is in the future we know there is no visit if ($this->period->getDateStart()->subDay(2)->isLater(Piwik_Date::today())) { Piwik::log("{$logMessage} skipped, archive is after today."); return; } // we make sure the archive is available for the given date $periodLabel = $this->period->getLabel(); $this->archiveProcessing = Piwik_ArchiveProcessing::factory($periodLabel); $this->archiveProcessing->setSite($this->site); $this->archiveProcessing->setPeriod($this->period); $this->archiveProcessing->setSegment($this->segment); $this->archiveProcessing->init(); $this->archiveProcessing->setRequestedReport($this->getRequestedReport()); $archivingDisabledArchiveNotProcessed = false; $idArchive = $this->archiveProcessing->loadArchive(); if (empty($idArchive)) { if ($this->archiveProcessing->isArchivingDisabled()) { $archivingDisabledArchiveNotProcessed = true; $logMessage = "* ARCHIVING DISABLED, for {$logMessage}"; } else { Piwik::log("* PROCESSING {$logMessage}, not archived yet..."); $archiveJustProcessed = true; // Process the reports $this->archiveProcessing->launchArchiving(); $idArchive = $this->archiveProcessing->getIdArchive(); $logMessage = "PROCESSED: idArchive = " . $idArchive . ", for {$logMessage}"; } } else { $logMessage = "* ALREADY PROCESSED, Fetching idArchive = {$idArchive} (idSite=" . $this->site->getId() . "), for {$logMessage}"; } Piwik::log("{$logMessage}, Visits = " . $this->archiveProcessing->getNumberOfVisits()); $this->isThereSomeVisits = !$archivingDisabledArchiveNotProcessed && $this->archiveProcessing->isThereSomeVisits(); $this->idArchive = $idArchive; } return $archiveJustProcessed; }
/** * Prepares the archive. Gets the idarchive from the ArchiveProcessing. * * This will possibly launch the archiving process if the archive was not available. * @return bool */ public function prepareArchive() { $archiveJustProcessed = false; $periodString = $this->period->getLabel(); $plugin = Piwik_ArchiveProcessing::getPluginBeingProcessed($this->getRequestedReport()); $cacheKey = 'all'; if ($periodString == 'range') { $cacheKey = $plugin; } if (!isset($this->alreadyChecked[$cacheKey])) { $this->isThereSomeVisits = false; $this->alreadyChecked[$cacheKey] = true; $dayString = $this->period->getPrettyString(); $logMessage = sprintf("%s (%s), plugin %s", $periodString, $dayString, $plugin); // if the END of the period is BEFORE the website creation date // we already know there are no stats for this period // we add one day to make sure we don't miss the day of the website creation if ($this->period->getDateEnd()->addDay(2)->isEarlier($this->site->getCreationDate())) { Piwik::log(sprintf("Archive %s skipped, archive is before the website was created.", $logMessage)); return; } // if the starting date is in the future we know there is no visit if ($this->period->getDateStart()->subDay(2)->isLater(Piwik_Date::today())) { Piwik::log(sprintf("Archive %s skipped, archive is after today.", $logMessage)); return; } // we make sure the archive is available for the given date $periodLabel = $this->period->getLabel(); $this->archiveProcessing = Piwik_ArchiveProcessing::factory($periodLabel); $this->archiveProcessing->setSite($this->site); $this->archiveProcessing->setPeriod($this->period); $this->archiveProcessing->setSegment($this->segment); $this->archiveProcessing->init(); $this->archiveProcessing->setRequestedReport($this->getRequestedReport()); $archivingDisabledArchiveNotProcessed = false; $idArchive = $this->archiveProcessing->loadArchive(); if (empty($idArchive)) { if ($this->archiveProcessing->isArchivingDisabled()) { $archivingDisabledArchiveNotProcessed = true; $logMessage = sprintf("Archiving disabled, for %s", $logMessage); } else { Piwik::log(sprintf("Processing %s, not archived yet...", $logMessage)); $archiveJustProcessed = true; // Process the reports $this->archiveProcessing->launchArchiving(); $idArchive = $this->archiveProcessing->getIdArchive(); $logMessage = sprintf("Processed %d, for %s", $idArchive, $logMessage); } } else { $logMessage = sprintf("Already processed, fetching idArchive = %d (idSite=%d), for %s", $idArchive, $this->site->getId(), $logMessage); } Piwik::log(sprintf("%s, Visits = %d", $logMessage, $this->archiveProcessing->getNumberOfVisits())); $this->isThereSomeVisits = !$archivingDisabledArchiveNotProcessed && $this->archiveProcessing->isThereSomeVisits(); $this->idArchive = $idArchive; } return $archiveJustProcessed; }
public static function isArchivingDisabledFor($segment, $period) { $processOneReportOnly = !self::shouldProcessReportsAllPluginsFor($segment, $period); if ($processOneReportOnly) { // When there is a segment, archiving is not necessary allowed // If browser archiving is allowed, then archiving is enabled // if browser archiving is not allowed, then archiving is disabled if (!$segment->isEmpty() && !self::isRequestAuthorizedToArchive() && Piwik_Config::getInstance()->General['browser_archiving_disabled_enforce']) { Piwik::log("Archiving is disabled because of config setting browser_archiving_disabled_enforce=1"); return true; } return false; } $isDisabled = !self::isRequestAuthorizedToArchive(); return $isDisabled; }
/** * Given a monthly archive table, will delete all reports that are now outdated, * or reports that ended with an error */ public static function doPurgeOutdatedArchives($numericTable) { $blobTable = str_replace("numeric", "blob", $numericTable); $key = self::FLAG_TABLE_PURGED . $blobTable; $timestamp = Piwik_GetOption($key); // we shall purge temporary archives after their timeout is finished, plus an extra 6 hours // in case archiving is disabled or run once a day, we give it this extra time to run // and re-process more recent records... // TODO: Instead of hardcoding 6 we should put the actual number of hours between 2 archiving runs $temporaryArchivingTimeout = self::getTodayArchiveTimeToLive(); $purgeEveryNSeconds = max($temporaryArchivingTimeout, 6 * 3600); // we only delete archives if we are able to process them, otherwise, the browser might process reports // when &segment= is specified (or custom date range) and would below, delete temporary archives that the // browser is not able to process until next cron run (which could be more than 1 hour away) if (self::isRequestAuthorizedToArchive() && (!$timestamp || $timestamp < time() - $purgeEveryNSeconds)) { Piwik_SetOption($key, time()); // If Browser Archiving is enabled, it is likely there are many more temporary archives // We delete more often which is safe, since reports are re-processed on demand if (self::isBrowserTriggerArchivingEnabled()) { $purgeArchivesOlderThan = Piwik_Date::factory(time() - 2 * $temporaryArchivingTimeout)->getDateTime(); } else { $purgeArchivesOlderThan = Piwik_Date::factory('today')->getDateTime(); } $result = Piwik_FetchAll("\n\t\t\t\tSELECT idarchive\n\t\t\t\tFROM {$numericTable}\n\t\t\t\tWHERE name LIKE 'done%'\n\t\t\t\t\tAND (( value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . "\n\t\t\t\t\t\t AND ts_archived < ?)\n\t\t\t\t\t\t OR value = " . Piwik_ArchiveProcessing::DONE_ERROR . ")", array($purgeArchivesOlderThan)); $idArchivesToDelete = array(); if (!empty($result)) { foreach ($result as $row) { $idArchivesToDelete[] = $row['idarchive']; } $query = "DELETE \n \t\t\t\t\t\tFROM %s\n \t\t\t\t\t\tWHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ")\n \t\t\t\t\t\t"; Piwik_Query(sprintf($query, $numericTable)); // Individual blob tables could be missing try { Piwik_Query(sprintf($query, $blobTable)); } catch (Exception $e) { } } Piwik::log("Purging temporary archives: done [ purged archives older than {$purgeArchivesOlderThan} from {$blobTable} and {$numericTable} ] [Deleted IDs: " . implode(',', $idArchivesToDelete) . "]"); // Deleting "Custom Date Range" reports after 1 day, since they can be re-processed // and would take up unecessary space $yesterday = Piwik_Date::factory('yesterday')->getDateTime(); $query = "DELETE \n \t\t\t\t\tFROM %s\n \t\t\t\t\tWHERE period = ?\n \t\t\t\t\t\tAND ts_archived < ?"; $bind = array(Piwik::$idPeriods['range'], $yesterday); Piwik::log("Purging Custom Range archives: done [ purged archives older than {$yesterday} from {$blobTable} and {$numericTable} ]"); Piwik_Query(sprintf($query, $numericTable), $bind); // Individual blob tables could be missing try { Piwik_Query(sprintf($query, $blobTable), $bind); } catch (Exception $e) { } // these tables will be OPTIMIZEd daily in a scheduled task, to claim lost space } else { Piwik::log("Purging temporary archives: skipped."); } }
/** * Performs a batch insert into a specific table using either LOAD DATA INFILE or plain INSERTs, * as a fallback. On MySQL, LOAD DATA INFILE is 20x faster than a series of plain INSERTs. * * @param string $tableName PREFIXED table name! you must call Piwik_Common::prefixTable() before passing the table name * @param array $fields array of unquoted field names * @param array $values array of data to be inserted * @return bool True if the bulk LOAD was used, false if we fallback to plain INSERTs */ public static function tableInsertBatch($tableName, $fields, $values) { $fieldList = '(' . join(',', $fields) . ')'; try { // throw new Exception(''); $filePath = PIWIK_USER_PATH . '/' . Piwik_AssetManager::MERGED_FILE_DIR . $tableName . '-' . Piwik_Common::generateUniqId() . '.csv'; if (Piwik_Common::isWindows()) { // On windows, MySQL expects slashes as directory separators $filePath = str_replace('\\', '/', $filePath); } // Set up CSV delimiters, quotes, etc $delim = "\t"; $quote = '"'; $eol = "\r\n"; $null = 'NULL'; $escape = '\\\\'; $fp = fopen($filePath, 'wb'); if (!$fp) { throw new Exception('Error creating the tmp file ' . $filePath . ', please check that the webserver has write permission to write this file.'); } @chmod($filePath, 0777); foreach ($values as $row) { $output = ''; foreach ($row as $value) { if (!isset($value) || is_null($value) || $value === false) { $output .= $null . $delim; } else { $output .= $quote . self::escapeString($value) . $quote . $delim; } } // Replace delim with eol unset($row[strlen($output) - strlen($delim)]); $output .= $eol; $ret = fwrite($fp, $output); if (!$ret) { fclose($fp); unlink($filePath); throw new Exception('Error writing to the tmp file ' . $filePath . ' containing the batch INSERTs.'); } } fclose($fp); $query = "\n\t\t\t\t\t'{$filePath}'\n\t\t\t\tREPLACE\n\t\t\t\tINTO TABLE\n\t\t\t\t\t" . $tableName; // hack for charset mismatch if (!self::isDatabaseConnectionUTF8() && !isset(Zend_Registry::get('config')->database->charset)) { $query .= ' CHARACTER SET latin1'; } $query .= "\n\t\t\t\tFIELDS TERMINATED BY\n\t\t\t\t\t'" . $delim . "'\n\t\t\t\tENCLOSED BY\n\t\t\t\t\t'" . $quote . "'\n\t\t\t\tESCAPED BY\n\t\t\t\t\t'" . $escape . "'\n\t\t\t\tLINES TERMINATED BY\n\t\t\t\t\t\"" . $eol . "\"\n\t\t\t\t{$fieldList}\n\t\t\t"; // initial attempt with LOCAL keyword // note: may trigger a known PHP PDO_MYSQL bug when MySQL not built with --enable-local-infile // @see http://bugs.php.net/bug.php?id=54158 try { $result = @Piwik_Exec('LOAD DATA LOCAL INFILE' . $query); if (empty($result)) { throw new Exception("LOAD DATA LOCAL INFILE failed!"); } unlink($filePath); return true; } catch (Exception $e) { } // second attempt without LOCAL keyword if MySQL server appears to be on the same box // note: requires that the db user have the FILE privilege; however, since this is // a global privilege, it may not be granted due to security concerns $dbHost = Zend_Registry::get('config')->database->host; $localHosts = array('127.0.0.1', 'localhost', 'localhost.local', 'localhost.localdomain', 'localhost.localhost'); $hostName = @php_uname('n'); if (!empty($hostName)) { $localHosts = array_merge($localHosts, array($hostName, $hostName . '.local', $hostName . '.localdomain', $hostName . '.localhost')); } if (!empty($dbHost) && !in_array($dbHost, $localHosts)) { throw new Exception("MYSQL appears to be on a remote server"); } $result = @Piwik_Exec('LOAD DATA INFILE' . $query); if (empty($result)) { throw new Exception("LOAD DATA INFILE failed!"); } unlink($filePath); return true; } catch (Exception $e) { Piwik::log("LOAD DATA INFILE failed or not supported, falling back to normal INSERTs... Error was:" . $e->getMessage(), Piwik_Log::WARN); // if all else fails, fallback to a series of INSERTs unlink($filePath); self::tableInsertBatchIterate($tableName, $fields, $values); } return false; }