/**
* Check for a newer version
*/
public static function check()
{
$lastTimeChecked = Piwik_GetOption(self::LAST_TIME_CHECKED);
if($lastTimeChecked === false
|| time() - self::CHECK_INTERVAL > $lastTimeChecked )
{
$parameters = array(
'piwik_version' => Piwik_Version::VERSION,
'php_version' => phpversion(),
'url' => Piwik_Url::getCurrentUrlWithoutQueryString(),
'trigger' => Piwik_Common::getRequestVar('module','','string'),
);
$url = self::PIWIK_HOST . "?" . http_build_query($parameters, '', '&');
$timeout = self::SOCKET_TIMEOUT;
try {
$latestVersion = Piwik::sendHttpRequest($url, $timeout);
Piwik_SetOption(self::LATEST_VERSION, $latestVersion);
} catch(Exception $e) {
// e.g., disable_functions = fsockopen; allow_url_open = Off
Piwik_SetOption(self::LATEST_VERSION, '');
}
Piwik_SetOption(self::LAST_TIME_CHECKED, time(), $autoload = 1);
}
}
/**
* Check for a newer version
*
* @param bool $force Force check
*/
public static function check($force = false)
{
$lastTimeChecked = Piwik_GetOption(self::LAST_TIME_CHECKED);
if($force || $lastTimeChecked === false
|| time() - self::CHECK_INTERVAL > $lastTimeChecked )
{
// set the time checked first, so that parallel Piwik requests don't all trigger the http requests
Piwik_SetOption(self::LAST_TIME_CHECKED, time(), $autoload = 1);
$parameters = array(
'piwik_version' => Piwik_Version::VERSION,
'php_version' => PHP_VERSION,
'url' => Piwik_Url::getCurrentUrlWithoutQueryString(),
'trigger' => Piwik_Common::getRequestVar('module','','string'),
'timezone' => Piwik_SitesManager_API::getInstance()->getDefaultTimezone(),
);
$url = Zend_Registry::get('config')->General->api_service_url;
$url .= '/1.0/getLatestVersion/';
$url .= '?' . http_build_query($parameters, '', '&');
$timeout = self::SOCKET_TIMEOUT;
try {
$latestVersion = Piwik_Http::sendHttpRequest($url, $timeout);
Piwik_SetOption(self::LATEST_VERSION, $latestVersion);
} catch(Exception $e) {
// e.g., disable_functions = fsockopen; allow_url_open = Off
Piwik_SetOption(self::LATEST_VERSION, '');
}
}
}
/**
* Caches the intermediate DataTables used in the getIndividualReportsSummary and
* getIndividualMetricsSummary reports in the option table.
*/
public function cacheDataByArchiveNameReports()
{
$api = Piwik_DBStats_API::getInstance();
$api->getIndividualReportsSummary(true);
$api->getIndividualMetricsSummary(true);
$now = Piwik_Date::now()->getLocalized("%longYear%, %shortMonth% %day%");
Piwik_SetOption(self::TIME_OF_LAST_TASK_RUN_OPTION, $now);
}
/**
* Record version of successfully completed component update
*
* @param string $name
* @param string $version
*/
public function recordComponentSuccessfullyUpdated($name, $version)
{
try {
Piwik_SetOption('version_' . $name, $version, $autoload = 1);
} catch (Exception $e) {
// case when the option table is not yet created (before 0.2.10)
}
}
static public function runTasks()
{
// Gets the array where rescheduled timetables are stored
$option = Piwik_GetOption(self::TIMETABLE_OPTION_STRING);
$timetable = self::getTimetableFromOption($option);
if($timetable === false) {
return;
}
if(isset($GLOBALS['PIWIK_TRACKER_DEBUG_FORCE_SCHEDULED_TASKS']) && $GLOBALS['PIWIK_TRACKER_DEBUG_FORCE_SCHEDULED_TASKS'])
{
$timetable = array();
}
// Collects tasks
Piwik_PostEvent(self::GET_TASKS_EVENT, $tasks);
$return = array();
// Loop through each task
foreach ($tasks as $task)
{
$scheduledTime = $task->getScheduledTime();
$className = $task->getClassName();
$methodName = $task->getMethodName();
$fullyQualifiedMethodName = get_class($className) . '.' . $methodName;
/*
* Task has to be executed if :
* - it is the first time, ie. rescheduledTime is not set
* - that task has already been executed and the current system time is greater than the
* rescheduled time.
*/
if ( !isset($timetable[$fullyQualifiedMethodName])
|| (isset($timetable[$fullyQualifiedMethodName])
&& time() >= $timetable[$fullyQualifiedMethodName]) )
{
// Updates the rescheduled time
$timetable[$fullyQualifiedMethodName] = $scheduledTime->getRescheduledTime();
Piwik_SetOption(self::TIMETABLE_OPTION_STRING, serialize($timetable));
// Run the task
try {
$timer = new Piwik_Timer;
call_user_func ( array($className,$methodName) );
$message = $timer->__toString();
} catch(Exception $e) {
$message = 'ERROR: '.$e->getMessage();
}
$return[] = array('task' => $fullyQualifiedMethodName, 'output' => $message);
}
}
return $return;
}
/**
* runTasks collects tasks defined within piwik plugins, runs them if they are scheduled and reschedules
* the tasks that have been executed.
*
* @return array
*/
public static function runTasks()
{
// Gets the array where rescheduled timetables are stored
$option = Piwik_GetOption(self::TIMETABLE_OPTION_STRING);
$timetable = self::getTimetableFromOption($option);
if ($timetable === false) {
return;
}
$forceScheduledTasks = false;
if (isset($GLOBALS['PIWIK_TRACKER_DEBUG_FORCE_SCHEDULED_TASKS']) && $GLOBALS['PIWIK_TRACKER_DEBUG_FORCE_SCHEDULED_TASKS'] || DEBUG_FORCE_SCHEDULED_TASKS) {
$forceScheduledTasks = true;
$timetable = array();
}
// Collects tasks
Piwik_PostEvent(self::GET_TASKS_EVENT, $tasks);
$return = array();
// for every priority level, starting with the highest and concluding with the lowest
for ($priority = Piwik_ScheduledTask::HIGHEST_PRIORITY; $priority <= Piwik_ScheduledTask::LOWEST_PRIORITY; ++$priority) {
// Loop through each task
foreach ($tasks as $task) {
// if the task does not have the current priority level, don't execute it yet
if ($task->getPriority() != $priority) {
continue;
}
$scheduledTime = $task->getScheduledTime();
$className = $task->getClassName();
$methodName = $task->getMethodName();
$fullyQualifiedMethodName = get_class($className) . '.' . $methodName;
/*
* Task has to be executed if :
* - it is the first time, ie. rescheduledTime is not set
* - that task has already been executed and the current system time is greater than the
* rescheduled time.
*/
if (!isset($timetable[$fullyQualifiedMethodName]) || isset($timetable[$fullyQualifiedMethodName]) && time() >= $timetable[$fullyQualifiedMethodName] || $forceScheduledTasks) {
// Updates the rescheduled time
$timetable[$fullyQualifiedMethodName] = $scheduledTime->getRescheduledTime();
Piwik_SetOption(self::TIMETABLE_OPTION_STRING, serialize($timetable));
self::$running = true;
// Run the task
try {
$timer = new Piwik_Timer();
call_user_func(array($className, $methodName));
$message = $timer->__toString();
} catch (Exception $e) {
$message = 'ERROR: ' . $e->getMessage();
}
self::$running = false;
$return[] = array('task' => $fullyQualifiedMethodName, 'output' => $message);
}
}
}
return $return;
}
function deleteLogTables()
{
$deleteSettings = Piwik_Config::getInstance()->Deletelogs;
//Make sure, log deletion is enabled
if ($deleteSettings['delete_logs_enable'] == 0) {
return;
}
//Log deletion may not run until it is once rescheduled (initial run). This is the only way to guarantee the calculated next scheduled deletion time.
$initialDelete = Piwik_GetOption(self::OPTION_LAST_DELETE_PIWIK_LOGS_INITIAL);
if (empty($initialDelete)) {
Piwik_SetOption(self::OPTION_LAST_DELETE_PIWIK_LOGS_INITIAL, 1);
return;
}
//Make sure, log purging is allowed to run now
$lastDelete = Piwik_GetOption(self::OPTION_LAST_DELETE_PIWIK_LOGS);
$deleteIntervalSeconds = $this->getDeleteIntervalInSeconds($deleteSettings['delete_logs_schedule_lowest_interval']);
if ($lastDelete === false || $lastDelete !== false && (int) $lastDelete + $deleteIntervalSeconds <= time()) {
$maxIdVisit = $this->getDeleteIdVisitOffset($deleteSettings['delete_logs_older_than']);
$logTables = $this->getDeleteTableLogTables();
//set lastDelete time to today
$date = Piwik_Date::factory("today");
$lastDeleteDate = $date->getTimestamp();
/*
* Tell the DB that log deletion has run BEFORE deletion is executed;
* If deletion / table optimization exceeds execution time, other tasks maybe prevented of being executed every time,
* when the schedule is triggered.
*/
Piwik_SetOption(self::OPTION_LAST_DELETE_PIWIK_LOGS, $lastDeleteDate);
//Break if no ID was found (nothing to delete for given period)
if (empty($maxIdVisit)) {
return;
}
foreach ($logTables as $logTable) {
$this->deleteRowsFromTable($logTable, $maxIdVisit, $deleteSettings['delete_max_rows_per_run'] * self::DELETE_MAX_ROWS_MULTIPLICATOR);
}
//optimize table overhead after deletion
$query = "OPTIMIZE TABLE " . implode(",", $logTables);
Piwik_Query($query);
}
}
public function test_RunAllTests()
{
parent::test_RunAllTests();
if (Test_Integration::$apiTestingLevel != Test_Integration::NO_API_TESTING) {
$this->_checkExpectedRowsInArchiveTable();
// Force Purge to happen again this table (since it was called at end of archiving)
Piwik_SetOption(Piwik_ArchiveProcessing_Period::FLAG_TABLE_PURGED . Piwik_Common::prefixTable('archive_blob_2010_12'), '2010-01-01');
Piwik_SetOption(Piwik_ArchiveProcessing_Period::FLAG_TABLE_PURGED . Piwik_Common::prefixTable('archive_blob_2011_01'), '2010-01-01');
foreach (array('archive_numeric_2010_12', 'archive_blob_2010_12', 'archive_numeric_2011_01', 'archive_blob_2011_01') as $table) {
// INSERT some ERROR records.
// We use period=range since the test below is restricted to these
Piwik_Query(" INSERT INTO `" . Piwik_Common::prefixTable($table) . "` (`idarchive` ,`name` ,`idsite` ,`date1` ,`date2` ,`period` ,`ts_archived` ,`value`)\n\t\t\t\t\tVALUES (10000, 'done', '1', '2010-12-06', '2010-12-06', '" . Piwik::$idPeriods['range'] . "', '2010-12-06' , '" . Piwik_ArchiveProcessing::DONE_ERROR . "'), \n\t\t\t\t\t\t\t(20000, 'doneX', '2', '2012-07-06', '2012-07-06', '" . Piwik::$idPeriods['range'] . "', '' , '" . Piwik_ArchiveProcessing::DONE_ERROR . "')");
}
// We've added error rows which should fail the following test
$this->_checkExpectedRowsInArchiveTable($expectPass = false);
// Test Scheduled tasks Table Optimize, and Delete old reports
Piwik_CoreAdminHome::purgeOutdatedArchives();
Piwik_CoreAdminHome::optimizeArchiveTable();
// Check that only the good reports were not deleted:
$this->_checkExpectedRowsInArchiveTable();
}
}
/**
* Check for a newer version
*
* @param bool $force Force check
*/
public static function check($force = false)
{
$lastTimeChecked = Piwik_GetOption(self::LAST_TIME_CHECKED);
if ($force || $lastTimeChecked === false || time() - self::CHECK_INTERVAL > $lastTimeChecked) {
// set the time checked first, so that parallel Piwik requests don't all trigger the http requests
Piwik_SetOption(self::LAST_TIME_CHECKED, time(), $autoload = 1);
$parameters = array('piwik_version' => Piwik_Version::VERSION, 'php_version' => PHP_VERSION, 'url' => Piwik_Url::getCurrentUrlWithoutQueryString(), 'trigger' => Piwik_Common::getRequestVar('module', '', 'string'), 'timezone' => Piwik_SitesManager_API::getInstance()->getDefaultTimezone());
$url = Piwik_Config::getInstance()->General['api_service_url'] . '/1.0/getLatestVersion/' . '?' . http_build_query($parameters, '', '&');
$timeout = self::SOCKET_TIMEOUT;
if (@Piwik_Config::getInstance()->Debug['allow_upgrades_to_beta']) {
$url = 'http://builds.piwik.org/LATEST_BETA';
}
try {
$latestVersion = Piwik_Http::sendHttpRequest($url, $timeout);
if (!preg_match('~^[0-9][0-9a-zA-Z_.-]*$~D', $latestVersion)) {
$latestVersion = '';
}
} catch (Exception $e) {
// e.g., disable_functions = fsockopen; allow_url_open = Off
$latestVersion = '';
}
Piwik_SetOption(self::LATEST_VERSION, $latestVersion);
}
}
public static function setUpBeforeClass()
{
$dbName = false;
if (!empty($GLOBALS['PIWIK_BENCHMARK_DATABASE'])) {
$dbName = $GLOBALS['PIWIK_BENCHMARK_DATABASE'];
}
// connect to database
self::createTestConfig();
self::connectWithoutDatabase();
// create specified fixture (global var not set, use default no-data fixture (see end of this file))
if (empty($GLOBALS['PIWIK_BENCHMARK_FIXTURE'])) {
$fixtureName = 'Piwik_Test_Fixture_EmptyOneSite';
} else {
$fixtureName = 'Piwik_Test_Fixture_' . $GLOBALS['PIWIK_BENCHMARK_FIXTURE'];
}
self::$fixture = new $fixtureName();
// figure out if the desired fixture has already been setup, and if not empty the database
$installedFixture = false;
try {
if (isset(self::$fixture->tablesPrefix)) {
Piwik_Config::getInstance()->database['tables_prefix'] = self::$fixture->tablesPrefix;
Piwik_Common::$cachedTablePrefix = null;
}
Piwik_Query("USE " . $dbName);
$installedFixture = Piwik_GetOption('benchmark_fixture_name');
} catch (Exception $ex) {
// ignore
}
$createEmptyDatabase = $fixtureName != $installedFixture;
parent::setUpBeforeClass($dbName, $createEmptyDatabase, $createConfig = false);
// if we created an empty database, setup the fixture
if ($createEmptyDatabase) {
self::$fixture->setUp();
Piwik_SetOption('benchmark_fixture_name', $fixtureName);
}
}
public function test_deleteLike()
{
// empty table, expect false (i.e., not found)
$this->assertTrue(Piwik_GetOption('anonymous_defaultReport') === false);
$this->assertTrue(Piwik_GetOption('admin_defaultReport') === false);
$this->assertTrue(Piwik_GetOption('visitor_defaultReport') === false);
// insert guard - to test unescaped underscore
Piwik_SetOption('adefaultReport', '0', true);
$this->assertTrue(Piwik_GetOption('adefaultReport') === '0');
// populate table, expect '1'
Piwik_SetOption('anonymous_defaultReport', '1', true);
Piwik_Option::getInstance()->deleteLike('\\_defaultReport');
$this->assertTrue(Piwik_Option::getInstance()->get('anonymous_defaultReport') === '1');
$this->assertTrue(Piwik_GetOption('adefaultReport') === '0');
// populate table, expect '2'
Piwik_SetOption('admin_defaultReport', '2', false);
Piwik_Option::getInstance()->deleteLike('\\_defaultReport');
$this->assertTrue(Piwik_Option::getInstance()->get('admin_defaultReport') === '2');
$this->assertTrue(Piwik_GetOption('adefaultReport') === '0');
// populate table, expect '3'
Piwik_SetOption('visitor_defaultReport', '3', false);
Piwik_Option::getInstance()->deleteLike('\\_defaultReport');
$this->assertTrue(Piwik_Option::getInstance()->get('visitor_defaultReport') === '3');
$this->assertTrue(Piwik_GetOption('adefaultReport') === '0');
// delete with non-matching value, expect '1'
Piwik_Option::getInstance()->deleteLike('%\\_defaultReport', '4');
$this->assertTrue(Piwik_Option::getInstance()->get('anonymous_defaultReport') === '1');
$this->assertTrue(Piwik_GetOption('adefaultReport') === '0');
// delete with matching pattern, expect false
Piwik_Option::getInstance()->deleteLike('%\\_defaultReport', '1');
$this->assertTrue(Piwik_Option::getInstance()->get('anonymous_defaultReport') === false);
$this->assertTrue(Piwik_GetOption('adefaultReport') === '0');
// this shouldn't have been deleted, expect '2' and '3'
$this->assertTrue(Piwik_Option::getInstance()->get('admin_defaultReport') === '2');
$this->assertTrue(Piwik_Option::getInstance()->get('visitor_defaultReport') === '3');
$this->assertTrue(Piwik_GetOption('adefaultReport') === '0');
// deleted, expect false (except for the guard)
Piwik_Option::getInstance()->deleteLike('%\\_defaultReport');
$this->assertTrue(Piwik_Option::getInstance()->get('admin_defaultReport') === false);
$this->assertTrue(Piwik_Option::getInstance()->get('visitor_defaultReport') === false);
// unescaped backslash (single quotes)
Piwik_Option::getInstance()->deleteLike('%\\_defaultReport');
$this->assertTrue(Piwik_GetOption('adefaultReport') === '0');
// escaped backslash (single quotes)
Piwik_Option::getInstance()->deleteLike('%\\_defaultReport');
$this->assertTrue(Piwik_GetOption('adefaultReport') === '0');
// unescaped backslash (double quotes)
Piwik_Option::getInstance()->deleteLike("%\\_defaultReport");
$this->assertTrue(Piwik_GetOption('adefaultReport') === '0');
// escaped backslash (double quotes)
Piwik_Option::getInstance()->deleteLike("%\\_defaultReport");
$this->assertTrue(Piwik_GetOption('adefaultReport') === '0');
}
/**
* Utility function. Gets row count of a set of tables grouped by the 'name' column.
* This is the implementation of the getRowCountsAndSizeBy... functions.
*/
private function getRowCountsByArchiveName($statuses, $getRowSizeMethod, $forceCache = false, $otherSelects = array(), $otherDataTableColumns = array())
{
$extraCols = '';
if (!empty($otherSelects)) {
$extraCols = ', ' . implode(', ', $otherSelects);
}
$cols = array_merge(array('row_count'), $otherDataTableColumns);
$dataTable = new Piwik_DataTable();
foreach ($statuses as $status) {
$dataTableOptionName = $this->getCachedOptionName($status['Name'], 'byArchiveName');
// if option exists && !$forceCache, use the cached data, otherwise create the
$cachedData = Piwik_GetOption($dataTableOptionName);
if ($cachedData !== false && !$forceCache) {
$table = new Piwik_DataTable();
$table->addRowsFromSerializedArray($cachedData);
} else {
// otherwise, create data table & cache it
$sql = "SELECT name as 'label', COUNT(*) as 'row_count'{$extraCols} FROM {$status['Name']} GROUP BY name";
$table = new Piwik_DataTable();
$table->addRowsFromSimpleArray(Piwik_FetchAll($sql));
$reduceArchiveRowName = array($this, 'reduceArchiveRowName');
$table->filter('GroupBy', array('label', $reduceArchiveRowName));
$serializedTables = $table->getSerialized();
$serializedTable = reset($serializedTables);
Piwik_SetOption($dataTableOptionName, $serializedTable);
}
// add estimated_size column
$getEstimatedSize = array($this, $getRowSizeMethod);
$table->filter('ColumnCallbackAddColumn', array($cols, 'estimated_size', $getEstimatedSize, array($status)));
$dataTable->addDataTable($table);
destroy($table);
}
return $dataTable;
}
/**
* Sets the provider to use when tracking.
*
* @param string $providerId The ID of the provider to use.
* @return Piwik_UserCountry_LocationProvider The new current provider.
* @throws Exception If the provider ID is invalid.
*/
public static function setCurrentProvider($providerId)
{
$provider = self::getProviderById($providerId);
if ($provider === false) {
throw new Exception("Invalid provider ID '{$providerId}'. The provider either does not exist or is not available");
}
Piwik_SetOption(self::CURRENT_PROVIDER_OPTION_NAME, $providerId);
return $provider;
}
/**
* Sets the default timezone that will be used when creating websites
*
* @param string $defaultTimezone Timezone string eg. Europe/Paris or UTC+8
* @return bool
*/
public function setDefaultTimezone($defaultTimezone)
{
Piwik::checkUserIsSuperUser();
$this->checkValidTimezone($defaultTimezone);
Piwik_SetOption(self::OPTION_DEFAULT_TIMEZONE, $defaultTimezone);
return true;
}
public static function setBrowserTriggerArchiving($enabled)
{
if (!is_bool($enabled)) {
throw new Exception('Browser trigger archiving must be set to true or false.');
}
Piwik_SetOption(self::OPTION_BROWSER_TRIGGER_ARCHIVING, (int) $enabled, $autoload = true);
Piwik_Common::clearCacheGeneral();
}
/**
* Returns true if one of the purge data tasks should run now, false if it shouldn't.
*/
private function shouldPurgeData($settings, $lastRanOption)
{
// Log deletion may not run until it is once rescheduled (initial run). This is the
// only way to guarantee the calculated next scheduled deletion time.
$initialDelete = Piwik_GetOption(self::OPTION_LAST_DELETE_PIWIK_LOGS_INITIAL);
if (empty($initialDelete)) {
Piwik_SetOption(self::OPTION_LAST_DELETE_PIWIK_LOGS_INITIAL, 1);
return false;
}
// Make sure, log purging is allowed to run now
$lastDelete = Piwik_GetOption($lastRanOption);
$deleteIntervalDays = $settings['delete_logs_schedule_lowest_interval'];
$deleteIntervalSeconds = $this->getDeleteIntervalInSeconds($deleteIntervalDays);
if ($lastDelete === false || $lastDelete !== false && (int) $lastDelete + $deleteIntervalSeconds <= time()) {
return true;
} else {
return false;
}
}
/**
* End of the script
*/
public function end()
{
// How to test the error handling code?
// - Generate some hits since last archive.php run
// - Start the script, in the middle, shutdown apache, then restore
// Some errors should be logged and script should successfully finish and then report the errors and trigger a PHP error
if (!empty($this->errors)) {
$this->logSection("SUMMARY OF ERRORS");
foreach ($this->errors as $error) {
$this->log("Error: " . $error);
}
$summary = count($this->errors) . " total errors during this script execution, please investigate and try and fix these errors";
$this->log($summary);
$summary .= '. First error was: ' . reset($this->errors);
$this->logFatalError($summary);
} else {
// No error -> Logs the successful script execution until completion
Piwik_SetOption(self::OPTION_ARCHIVING_FINISHED_TS, time());
}
}
/**
* Sets a user preference
* @param string $userLogin
* @param string $preferenceName
* @param string $preferenceValue
* @return void
*/
public function setUserPreference($userLogin, $preferenceName, $preferenceValue)
{
Piwik::checkUserIsSuperUserOrTheUser($userLogin);
Piwik_SetOption($this->getPreferenceId($userLogin, $preferenceName), $preferenceValue);
}
/**
* Called at the end of the archiving process.
* Does some cleaning job in the database.
*/
protected function postCompute()
{
parent::postCompute();
$blobTable = $this->tableArchiveBlob->getTableName();
$numericTable = $this->tableArchiveNumeric->getTableName();
$key = 'lastPurge_' . $blobTable;
$timestamp = Piwik_GetOption($key);
if (!$timestamp || $timestamp < time() - 86400) {
Piwik_SetOption($key, time());
// we delete out of date daily archives from table, maximum once per day
// we only delete archives processed that are older than 1 day, to not delete archives we just processed
$yesterday = Piwik_Date::factory('yesterday')->getDateTime();
$result = Piwik_FetchAll("\n\t\t\t\t\t\t\tSELECT idarchive\n\t\t\t\t\t\t\tFROM {$numericTable}\n\t\t\t\t\t\t\tWHERE name LIKE 'done%'\n\t\t\t\t\t\t\t\tAND value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . "\n\t\t\t\t\t\t\t\tAND ts_archived < ?", array($yesterday));
$idArchivesToDelete = array();
if (!empty($result)) {
foreach ($result as $row) {
$idArchivesToDelete[] = $row['idarchive'];
}
$query = "DELETE \n \t\t\t\t\t\tFROM %s\n \t\t\t\t\t\tWHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ")\n \t\t\t\t\t\t";
Piwik_Query(sprintf($query, $blobTable));
Piwik_Query(sprintf($query, $numericTable));
}
Piwik::log("Purging temporary archives: done [ purged archives older than {$yesterday} from {$blobTable} and {$numericTable} ] [Deleted IDs: " . implode(',', $idArchivesToDelete) . "]");
// Deleting "Custom Date Range" reports after 1 day, since they can be re-processed
// and would take up unecessary space
$query = "DELETE \n \t\t\t\t\tFROM %s\n \t\t\t\t\tWHERE period = ?\n \t\t\t\t\t\tAND ts_archived < ?";
$bind = array(Piwik::$idPeriods['range'], $yesterday);
Piwik_Query(sprintf($query, $blobTable), $bind);
Piwik_Query(sprintf($query, $numericTable), $bind);
} else {
Piwik::log("Purging temporary archives: skipped.");
}
if (!isset($this->archives)) {
return;
}
foreach ($this->archives as $archive) {
destroy($archive);
}
$this->archives = array();
}
/**
* Returns the cached the Piwik URL, eg. http://demo.piwik.org/ or http://example.org/piwik/
* If not found, then tries to cache it and returns the value.
*
* If the Piwik URL changes (eg. Piwik moved to new server), the value will automatically be refreshed in the cache.
*
* @return string
*/
static public function getPiwikUrl()
{
$key = 'piwikUrl';
$url = Piwik_GetOption($key);
if(Piwik_Common::isPhpCliMode())
{
return $url;
}
$currentUrl = Piwik_Common::sanitizeInputValue(Piwik_Url::getCurrentUrlWithoutFileName());
if(self::isHttps()
&& strpos($currentUrl, 'http://') === 0)
{
$currentUrl = str_replace('http://', 'https://', $currentUrl);
}
if(empty($url)
// if URL changes, always update the cache
|| $currentUrl != $url)
{
if(strlen($currentUrl) >= strlen('http://a/'))
{
Piwik_SetOption($key, $currentUrl, $autoload = true);
}
$url = $currentUrl;
}
return $url;
}
/**
* Returns the cached the Piwik URL, eg. http://demo.piwik.org/ or http://example.org/piwik/
* If not found, then tries to cache it and returns the value.
*
* If the Piwik URL changes (eg. Piwik moved to new server), the value will automatically be refreshed in the cache.
* @return string
*/
public static function getPiwikUrl()
{
$key = 'piwikUrl';
$url = Piwik_GetOption($key);
if (Piwik_Common::isPhpCliMode() || Piwik_Common::isArchivePhpTriggered() || defined('PIWIK_MODE_ARCHIVE')) {
return $url;
}
$currentUrl = Piwik_Common::sanitizeInputValue(Piwik_Url::getCurrentUrlWithoutFileName());
if (empty($url) || $currentUrl != $url) {
if (strlen($currentUrl) >= strlen('http://a/')) {
Piwik_SetOption($key, $currentUrl, $autoload = true);
}
$url = $currentUrl;
}
return $url;
}
/**
* When tracking data in the past (using Tracking API), this function
* can be used to invalidate reports for the idSites and dates where new data
* was added.
* DEV: If you call this API, the UI should display the data correctly, but will process
* in real time, which could be very slow after large data imports.
* After calling this function via REST, you can manually force all data
* to be reprocessed by visiting the script as the Super User:
* http://example.net/piwik/misc/cron/archive.php?token_auth=$SUPER_USER_TOKEN_AUTH_HERE
* REQUIREMENTS: On large piwik setups, you will need in PHP configuration: max_execution_time = 0
* We recommend to use an hourly schedule of the script at misc/cron/archive.php
* More information: http://piwik.org/setup-auto-archiving/
*
* @param string $idSites Comma separated list of idSite that have had data imported for the specified dates
* @param string $dates Comma separated list of dates to invalidate for all these websites
* @return array
*/
public function invalidateArchivedReports($idSites, $dates)
{
$idSites = Piwik_Site::getIdSitesFromIdSitesString($idSites);
if (empty($idSites)) {
throw new Exception("Specify a value for &idSites= as a comma separated list of website IDs, for which your token_auth has 'admin' permission");
}
Piwik::checkUserHasAdminAccess($idSites);
// Ensure the specified dates are valid
$toInvalidate = $invalidDates = array();
$dates = explode(',', $dates);
$dates = array_unique($dates);
foreach ($dates as $theDate) {
try {
$date = Piwik_Date::factory($theDate);
} catch (Exception $e) {
$invalidDates[] = $theDate;
continue;
}
if ($date->toString() == $theDate) {
$toInvalidate[] = $date;
} else {
$invalidDates[] = $theDate;
}
}
// Lookup archive tables
$tables = Piwik::getTablesInstalled();
$archiveTables = Piwik::getTablesArchivesInstalled();
// If using the feature "Delete logs older than N days"...
$logsAreDeletedBeforeThisDate = Piwik_Config::getInstance()->Deletelogs['delete_logs_schedule_lowest_interval'];
$logsDeleteEnabled = Piwik_Config::getInstance()->Deletelogs['delete_logs_enable'];
$minimumDateWithLogs = false;
if ($logsDeleteEnabled && $logsAreDeletedBeforeThisDate) {
$minimumDateWithLogs = Piwik_Date::factory('today')->subDay($logsAreDeletedBeforeThisDate);
}
// Given the list of dates, process which tables they should be deleted from
$minDate = false;
$warningDates = $processedDates = array();
/* @var $date Piwik_Date */
foreach ($toInvalidate as $date) {
// we should only delete reports for dates that are more recent than N days
if ($minimumDateWithLogs && $date->isEarlier($minimumDateWithLogs)) {
$warningDates[] = $date->toString();
} else {
$processedDates[] = $date->toString();
}
$month = $date->toString('Y_m');
// For a given date, we must invalidate in the monthly archive table
$datesByMonth[$month][] = $date->toString();
// But also the year stored in January
$year = $date->toString('Y_01');
$datesByMonth[$year][] = $date->toString();
// but also weeks overlapping several months stored in the month where the week is starting
/* @var $week Piwik_Period_Week */
$week = Piwik_Period::factory('week', $date);
$week = $week->getDateStart()->toString('Y_m');
$datesByMonth[$week][] = $date->toString();
// Keep track of the minimum date for each website
if ($minDate === false || $date->isEarlier($minDate)) {
$minDate = $date;
}
}
// In each table, invalidate day/week/month/year containing this date
$sqlIdSites = implode(",", $idSites);
foreach ($archiveTables as $table) {
// Extract Y_m from table name
$suffix = str_replace(array('archive_numeric_', 'archive_blob_'), '', Piwik_Common::unprefixTable($table));
if (!isset($datesByMonth[$suffix])) {
continue;
}
// Dates which are to be deleted from this table
$datesToDeleteInTable = $datesByMonth[$suffix];
// Build one statement to delete all dates from the given table
$sql = $bind = array();
$datesToDeleteInTable = array_unique($datesToDeleteInTable);
foreach ($datesToDeleteInTable as $dateToDelete) {
$sql[] = '(date1 <= ? AND ? <= date2)';
$bind[] = $dateToDelete;
$bind[] = $dateToDelete;
}
$sql = implode(" OR ", $sql);
$query = "DELETE FROM {$table} " . " WHERE ( {$sql} ) " . " AND idsite IN (" . $sqlIdSites . ")";
Piwik_Query($query, $bind);
// var_dump($query);var_dump($bind);
}
// Update piwik_site.ts_created
$query = "UPDATE " . Piwik_Common::prefixTable("site") . " SET ts_created = ?" . " WHERE idsite IN ( {$sqlIdSites} )\n\t\t\t\t\tAND ts_created > ?";
$minDateSql = $minDate->subDay(1)->getDatetime();
$bind = array($minDateSql, $minDateSql);
Piwik_Query($query, $bind);
// var_dump($query);var_dump($bind);
// Force to re-process data for these websites in the next archive.php cron run
$invalidatedIdSites = Piwik_CoreAdminHome_API::getWebsiteIdsToInvalidate();
$invalidatedIdSites = array_merge($invalidatedIdSites, $idSites);
$invalidatedIdSites = array_unique($invalidatedIdSites);
$invalidatedIdSites = array_values($invalidatedIdSites);
Piwik_SetOption(self::OPTION_INVALIDATED_IDSITES, serialize($invalidatedIdSites));
Piwik_Site::clearCache();
$output = array();
// output logs
if ($warningDates) {
$output[] = 'Warning: the following Dates have not been invalidated, because they are earlier than your Log Deletion limit: ' . implode(", ", $warningDates) . "\n The last day with logs is " . $minimumDateWithLogs . ". " . "\n Please disable 'Delete old Logs' or set it to a higher deletion threshold (eg. 180 days or 365 years).'.";
}
$output[] = "Success. The following dates were invalidated successfully: " . implode(", ", $processedDates);
return $output;
}
/**
* Stores password reset info for a specific login.
*
* @param string $login The user login for whom a password change was requested.
* @param string $password The new password to set.
*/
public static function savePasswordResetInfo($login, $password)
{
$optionName = self::getPasswordResetInfoOptionName($login);
$optionData = Piwik_UsersManager::getPasswordHash($password);
Piwik_SetOption($optionName, $optionData);
}
/**
* Called at the end of the archiving process.
* Does some cleaning job in the database.
*/
protected function postCompute()
{
parent::postCompute();
$blobTable = $this->tableArchiveBlob->getTableName();
$numericTable = $this->tableArchiveNumeric->getTableName();
$key = 'lastPurge_' . $blobTable;
$timestamp = Piwik_GetOption($key);
// we shall purge temporary archives after their timeout is finished, plus an extra 2 hours
// in case archiving is disabled and is late to run, we give it this extra time to run and re-process more recent records
$temporaryArchivingTimeout = self::getTodayArchiveTimeToLive();
$purgeEveryNSeconds = $temporaryArchivingTimeout + 2 * 3600;
// we only delete archives if we are able to process them, otherwise, the browser might process reports
// when &segment= is specified (or custom date range) and would below, delete temporary archives that the
// browser is not able to process until next cron run (which could be more than 1 hour away)
if ($this->isRequestAuthorizedToArchive() && (!$timestamp || $timestamp < time() - $purgeEveryNSeconds)) {
Piwik_SetOption($key, time());
$purgeArchivesOlderThan = Piwik_Date::factory(time() - $purgeEveryNSeconds)->getDateTime();
$result = Piwik_FetchAll("\n\t\t\t\t\t\t\tSELECT idarchive\n\t\t\t\t\t\t\tFROM {$numericTable}\n\t\t\t\t\t\t\tWHERE name LIKE 'done%'\n\t\t\t\t\t\t\t\tAND value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . "\n\t\t\t\t\t\t\t\tAND ts_archived < ?", array($purgeArchivesOlderThan));
$idArchivesToDelete = array();
if (!empty($result)) {
foreach ($result as $row) {
$idArchivesToDelete[] = $row['idarchive'];
}
$query = "DELETE \n \t\t\t\t\t\tFROM %s\n \t\t\t\t\t\tWHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ")\n \t\t\t\t\t\t";
Piwik_Query(sprintf($query, $blobTable));
Piwik_Query(sprintf($query, $numericTable));
}
Piwik::log("Purging temporary archives: done [ purged archives older than {$purgeArchivesOlderThan} from {$blobTable} and {$numericTable} ] [Deleted IDs: " . implode(',', $idArchivesToDelete) . "]");
// Deleting "Custom Date Range" reports after 1 day, since they can be re-processed
// and would take up unecessary space
$yesterday = Piwik_Date::factory('yesterday')->getDateTime();
$query = "DELETE \n \t\t\t\t\tFROM %s\n \t\t\t\t\tWHERE period = ?\n \t\t\t\t\t\tAND ts_archived < ?";
$bind = array(Piwik::$idPeriods['range'], $yesterday);
Piwik::log("Purging Custom Range archives: done [ purged archives older than {$yesterday} from {$blobTable} and {$numericTable} ]");
Piwik_Query(sprintf($query, $blobTable), $bind);
Piwik_Query(sprintf($query, $numericTable), $bind);
// these tables will be OPTIMIZEd daily in a scheduled task, to claim lost space
} else {
Piwik::log("Purging temporary archives: skipped.");
}
if (!isset($this->archives)) {
return;
}
foreach ($this->archives as $archive) {
destroy($archive);
}
$this->archives = array();
}
/**
* Tracker requests will automatically trigger the Scheduled tasks.
* This is useful for users who don't setup the cron,
* but still want daily/weekly/monthly PDF reports emailed automatically.
*
* This is similar to calling the API CoreAdminHome.runScheduledTasks (see misc/cron/archive.sh)
*
* @param int $now Current timestamp
*/
public static function runScheduledTasks($now)
{
// Currently, there is no hourly tasks. When there are some,
// this could be too agressive minimum interval (some hours would be skipped in case of low traffic)
$minimumInterval = Piwik_Config::getInstance()->Tracker['scheduled_tasks_min_interval'];
// If the user disabled browser archiving, he has already setup a cron
// To avoid parallel requests triggering the Scheduled Tasks,
// Get last time tasks started executing
$cache = Piwik_Common::getCacheGeneral();
if ($minimumInterval <= 0 || empty($cache['isBrowserTriggerArchivingEnabled'])) {
printDebug("-> Scheduled tasks not running in Tracker: Browser archiving is disabled.");
return;
}
$nextRunTime = $cache['lastTrackerCronRun'] + $minimumInterval;
if (isset($GLOBALS['PIWIK_TRACKER_DEBUG_FORCE_SCHEDULED_TASKS']) && $GLOBALS['PIWIK_TRACKER_DEBUG_FORCE_SCHEDULED_TASKS'] || $cache['lastTrackerCronRun'] === false || $nextRunTime < $now) {
$cache['lastTrackerCronRun'] = $now;
Piwik_Common::setCacheGeneral($cache);
Piwik_Common::initCorePiwikInTrackerMode();
Piwik_SetOption('lastTrackerCronRun', $cache['lastTrackerCronRun']);
printDebug('-> Scheduled Tasks: Starting...');
// save current user privilege and temporarily assume super user privilege
$isSuperUser = Piwik::isUserIsSuperUser();
// Scheduled tasks assume Super User is running
Piwik::setUserIsSuperUser();
// While each plugins should ensure that necessary languages are loaded,
// we ensure English translations at least are loaded
Piwik_Translate::getInstance()->loadEnglishTranslation();
$resultTasks = Piwik_TaskScheduler::runTasks();
// restore original user privilege
Piwik::setUserIsSuperUser($isSuperUser);
printDebug($resultTasks);
printDebug('Finished Scheduled Tasks.');
} else {
printDebug("-> Scheduled tasks not triggered.");
}
printDebug("Next run will be from: " . date('Y-m-d H:i:s', $nextRunTime) . ' UTC');
}
/**
* Called at the end of the archiving process.
* Does some cleaning job in the database.
*
* @return void
*/
protected function postCompute()
{
parent::postCompute();
$blobTable = $this->tableArchiveBlob->getTableName();
$numericTable = $this->tableArchiveNumeric->getTableName();
// delete out of date records maximum once per day (DELETE request is costly)
$key = 'lastPurge_' . $blobTable;
$timestamp = Piwik_GetOption($key);
if (!$timestamp || $timestamp < time() - 86400) {
// we delete out of date daily archives from table, maximum once per day
// those for day N that were processed on day N (means the archives are only partial as the day wasn't finished)
$query = "/* SHARDING_ID_SITE = " . $this->idsite . " */ \tDELETE \n\t\t\t\t\t\tFROM %s\n\t\t\t\t\t\tWHERE period = ? \n\t\t\t\t\t\t\tAND date1 = DATE(ts_archived)\n\t\t\t\t\t\t\tAND DATE(ts_archived) <> CURRENT_DATE()\n\t\t\t\t\t\t";
Piwik_Query(sprintf($query, $blobTable), Piwik::$idPeriods['day']);
Piwik_Query(sprintf($query, $numericTable), Piwik::$idPeriods['day']);
// we delete out of date Period records (week/month/etc)
// we delete archives that were archived before the end of the period
// and only if they are at least 1 day old (so we don't delete archives computed today that may be stil valid)
$query = "\tDELETE \n\t\t\t\t\t\tFROM %s\n\t\t\t\t\t\tWHERE period > ? \n\t\t\t\t\t\t\tAND DATE(ts_archived) <= date2\n\t\t\t\t\t\t\tAND date(ts_archived) < date_sub(CURRENT_DATE(), INTERVAL 1 DAY)\n\t\t\t\t\t\t";
Piwik_Query(sprintf($query, $blobTable), Piwik::$idPeriods['day']);
Piwik_Query(sprintf($query, $numericTable), Piwik::$idPeriods['day']);
Piwik_SetOption($key, time());
}
foreach ($this->archives as $archive) {
destroy($archive);
}
$this->archives = array();
}
/**
* Given a monthly archive table, will delete all reports that are now outdated,
* or reports that ended with an error
*/
public static function doPurgeOutdatedArchives($numericTable)
{
$blobTable = str_replace("numeric", "blob", $numericTable);
$key = self::FLAG_TABLE_PURGED . $blobTable;
$timestamp = Piwik_GetOption($key);
// we shall purge temporary archives after their timeout is finished, plus an extra 6 hours
// in case archiving is disabled or run once a day, we give it this extra time to run
// and re-process more recent records...
// TODO: Instead of hardcoding 6 we should put the actual number of hours between 2 archiving runs
$temporaryArchivingTimeout = self::getTodayArchiveTimeToLive();
$purgeEveryNSeconds = max($temporaryArchivingTimeout, 6 * 3600);
// we only delete archives if we are able to process them, otherwise, the browser might process reports
// when &segment= is specified (or custom date range) and would below, delete temporary archives that the
// browser is not able to process until next cron run (which could be more than 1 hour away)
if (self::isRequestAuthorizedToArchive() && (!$timestamp || $timestamp < time() - $purgeEveryNSeconds)) {
Piwik_SetOption($key, time());
// If Browser Archiving is enabled, it is likely there are many more temporary archives
// We delete more often which is safe, since reports are re-processed on demand
if (self::isBrowserTriggerArchivingEnabled()) {
$purgeArchivesOlderThan = Piwik_Date::factory(time() - 2 * $temporaryArchivingTimeout)->getDateTime();
} else {
$purgeArchivesOlderThan = Piwik_Date::factory('today')->getDateTime();
}
$result = Piwik_FetchAll("\n\t\t\t\tSELECT idarchive\n\t\t\t\tFROM {$numericTable}\n\t\t\t\tWHERE name LIKE 'done%'\n\t\t\t\t\tAND (( value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . "\n\t\t\t\t\t\t AND ts_archived < ?)\n\t\t\t\t\t\t OR value = " . Piwik_ArchiveProcessing::DONE_ERROR . ")", array($purgeArchivesOlderThan));
$idArchivesToDelete = array();
if (!empty($result)) {
foreach ($result as $row) {
$idArchivesToDelete[] = $row['idarchive'];
}
$query = "DELETE \n \t\t\t\t\t\tFROM %s\n \t\t\t\t\t\tWHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ")\n \t\t\t\t\t\t";
Piwik_Query(sprintf($query, $numericTable));
// Individual blob tables could be missing
try {
Piwik_Query(sprintf($query, $blobTable));
} catch (Exception $e) {
}
}
Piwik::log("Purging temporary archives: done [ purged archives older than {$purgeArchivesOlderThan} from {$blobTable} and {$numericTable} ] [Deleted IDs: " . implode(',', $idArchivesToDelete) . "]");
// Deleting "Custom Date Range" reports after 1 day, since they can be re-processed
// and would take up unecessary space
$yesterday = Piwik_Date::factory('yesterday')->getDateTime();
$query = "DELETE \n \t\t\t\t\tFROM %s\n \t\t\t\t\tWHERE period = ?\n \t\t\t\t\t\tAND ts_archived < ?";
$bind = array(Piwik::$idPeriods['range'], $yesterday);
Piwik::log("Purging Custom Range archives: done [ purged archives older than {$yesterday} from {$blobTable} and {$numericTable} ]");
Piwik_Query(sprintf($query, $numericTable), $bind);
// Individual blob tables could be missing
try {
Piwik_Query(sprintf($query, $blobTable), $bind);
} catch (Exception $e) {
}
// these tables will be OPTIMIZEd daily in a scheduled task, to claim lost space
} else {
Piwik::log("Purging temporary archives: skipped.");
}
}
/**
* Specify if normal users can manage their own SMS API credential
*
* @param bool $delegatedManagement false if SMS API credential only manageable by super admin, true otherwise
*/
public function setDelegatedManagement($delegatedManagement)
{
Piwik::checkUserIsSuperUser();
Piwik_SetOption(Piwik_MobileMessaging::DELEGATED_MANAGEMENT_OPTION, $delegatedManagement);
}
private function setTimeToRun()
{
$lastDateSecs = Piwik_Date::factory('today')->subDay(8)->getTimestamp();
Piwik_SetOption(Piwik_PrivacyManager::OPTION_LAST_DELETE_PIWIK_LOGS_INITIAL, 1);
Piwik_SetOption(Piwik_PrivacyManager::OPTION_LAST_DELETE_PIWIK_LOGS, $lastDateSecs);
Piwik_SetOption(Piwik_PrivacyManager::OPTION_LAST_DELETE_PIWIK_REPORTS, $lastDateSecs);
}
function install()
{
$delegatedManagement = Piwik_GetOption(self::DELEGATED_MANAGEMENT_OPTION);
if (empty($delegatedManagement)) {
Piwik_SetOption(self::DELEGATED_MANAGEMENT_OPTION, self::DELEGATED_MANAGEMENT_OPTION_DEFAULT);
}
}
