/**
* Will run all scheduled tasks due to run at this time.
*
* @return array
*/
public function runScheduledTasks()
{
Piwik::checkUserHasSuperUserAccess();
return TaskScheduler::runTasks();
}
/**
* Sets the options used by this class based on the elements in $options.
*
* The following elements of $options are used:
* 'loc' - URL for location database.
* 'isp' - URL for ISP database.
* 'org' - URL for Organization database.
* 'period' - 'weekly' or 'monthly'. When to run the updates.
*
* @param array $options
* @throws Exception
*/
public static function setUpdaterOptions($options)
{
// set url options
foreach (self::$urlOptions as $optionKey => $optionName) {
if (!isset($options[$optionKey])) {
continue;
}
$url = $options[$optionKey];
$url = self::removeDateFromUrl($url);
Option::set($optionName, $url);
}
// set period option
if (!empty($options['period'])) {
$period = $options['period'];
if ($period != self::SCHEDULE_PERIOD_MONTHLY && $period != self::SCHEDULE_PERIOD_WEEKLY) {
throw new Exception(Piwik::translate('UserCountry_InvalidGeoIPUpdatePeriod', array("'{$period}'", "'" . self::SCHEDULE_PERIOD_MONTHLY . "', '" . self::SCHEDULE_PERIOD_WEEKLY . "'")));
}
Option::set(self::SCHEDULE_PERIOD_OPTION_NAME, $period);
TaskScheduler::rescheduleTask(new GeoIPAutoUpdater());
}
}
/**
* Returns the list of websites ID with the 'view' or 'admin' access for the current user.
* For the superUser it returns all the websites in the database.
*
* @param bool $_restrictSitesToLogin
* @return array list of websites ID
*/
public function getSitesIdWithAtLeastViewAccess($_restrictSitesToLogin = false)
{
if (Piwik::hasUserSuperUserAccess() && !TaskScheduler::isTaskBeingExecuted()) {
return Access::getInstance()->getSitesIdWithAtLeastViewAccess();
}
if (!empty($_restrictSitesToLogin) && (Piwik::hasUserSuperUserAccessOrIsTheUser($_restrictSitesToLogin) || TaskScheduler::isTaskBeingExecuted())) {
if (Piwik::hasTheUserSuperUserAccess($_restrictSitesToLogin)) {
return Access::getInstance()->getSitesIdWithAtLeastViewAccess();
}
$accessRaw = Access::getInstance()->getRawSitesWithSomeViewAccess($_restrictSitesToLogin);
$sitesId = array();
foreach ($accessRaw as $access) {
$sitesId[] = $access['idsite'];
}
return $sitesId;
} else {
return Access::getInstance()->getSitesIdWithAtLeastViewAccess();
}
}
protected function getDeleteDataInfo()
{
Piwik::checkUserHasSuperUserAccess();
$deleteDataInfos = array();
$taskScheduler = new TaskScheduler();
$deleteDataInfos["config"] = PrivacyManager::getPurgeDataSettings();
$deleteDataInfos["deleteTables"] = "<br/>" . implode(", ", LogDataPurger::getDeleteTableLogTables());
$scheduleTimetable = $taskScheduler->getScheduledTimeForMethod("PrivacyManager", "deleteLogTables");
$optionTable = Option::get(self::OPTION_LAST_DELETE_PIWIK_LOGS);
//If task was already rescheduled, read time from taskTimetable. Else, calculate next possible runtime.
if (!empty($scheduleTimetable) && $scheduleTimetable - time() > 0) {
$nextPossibleSchedule = (int) $scheduleTimetable;
} else {
$date = Date::factory("today");
$nextPossibleSchedule = $date->addDay(1)->getTimestamp();
}
//deletion schedule did not run before
if (empty($optionTable)) {
$deleteDataInfos["lastRun"] = false;
//next run ASAP (with next schedule run)
$date = Date::factory("today");
$deleteDataInfos["nextScheduleTime"] = $nextPossibleSchedule;
} else {
$deleteDataInfos["lastRun"] = $optionTable;
$deleteDataInfos["lastRunPretty"] = Date::factory((int) $optionTable)->getLocalized('%day% %shortMonth% %longYear%');
//Calculate next run based on last run + interval
$nextScheduleRun = (int) ($deleteDataInfos["lastRun"] + $deleteDataInfos["config"]["delete_logs_schedule_lowest_interval"] * 24 * 60 * 60);
//is the calculated next run in the past? (e.g. plugin was disabled in the meantime or something) -> run ASAP
if ($nextScheduleRun - time() <= 0) {
$deleteDataInfos["nextScheduleTime"] = $nextPossibleSchedule;
} else {
$deleteDataInfos["nextScheduleTime"] = $nextScheduleRun;
}
}
$deleteDataInfos["nextRunPretty"] = MetricsFormatter::getPrettyTimeFromSeconds($deleteDataInfos["nextScheduleTime"] - time());
return $deleteDataInfos;
}
/**
* @param array $reports
* @param array $info
* @return mixed
*/
public function getReportMetadata(&$reports, $info)
{
$idSites = $info['idSites'];
// If only one website is selected, we add the Graph URL
if (count($idSites) != 1) {
return;
}
$idSite = reset($idSites);
// in case API.getReportMetadata was not called with date/period we use sane defaults
if (empty($info['period'])) {
$info['period'] = 'day';
}
if (empty($info['date'])) {
$info['date'] = 'today';
}
// need two sets of period & date, one for single period graphs, one for multiple periods graphs
if (Period::isMultiplePeriod($info['date'], $info['period'])) {
$periodForMultiplePeriodGraph = $info['period'];
$dateForMultiplePeriodGraph = $info['date'];
$periodForSinglePeriodGraph = 'range';
$dateForSinglePeriodGraph = $info['date'];
} else {
$periodForSinglePeriodGraph = $info['period'];
$dateForSinglePeriodGraph = $info['date'];
$piwikSite = new Site($idSite);
if ($periodForSinglePeriodGraph == 'range') {
$periodForMultiplePeriodGraph = Config::getInstance()->General['graphs_default_period_to_plot_when_period_range'];
$dateForMultiplePeriodGraph = $dateForSinglePeriodGraph;
} else {
$periodForMultiplePeriodGraph = $periodForSinglePeriodGraph;
$dateForMultiplePeriodGraph = Range::getRelativeToEndDate($periodForSinglePeriodGraph, 'last' . self::GRAPH_EVOLUTION_LAST_PERIODS, $dateForSinglePeriodGraph, $piwikSite);
}
}
$token_auth = Common::getRequestVar('token_auth', false);
$urlPrefix = "index.php?";
foreach ($reports as &$report) {
$reportModule = $report['module'];
$reportAction = $report['action'];
$reportUniqueId = $reportModule . '_' . $reportAction;
$parameters = array();
$parameters['module'] = 'API';
$parameters['method'] = 'ImageGraph.get';
$parameters['idSite'] = $idSite;
$parameters['apiModule'] = $reportModule;
$parameters['apiAction'] = $reportAction;
if (!empty($token_auth)) {
$parameters['token_auth'] = $token_auth;
}
// Forward custom Report parameters to the graph URL
if (!empty($report['parameters'])) {
$parameters = array_merge($parameters, $report['parameters']);
}
if (empty($report['dimension'])) {
$parameters['period'] = $periodForMultiplePeriodGraph;
$parameters['date'] = $dateForMultiplePeriodGraph;
} else {
$parameters['period'] = $periodForSinglePeriodGraph;
$parameters['date'] = $dateForSinglePeriodGraph;
}
// add the idSubtable if it exists
$idSubtable = Common::getRequestVar('idSubtable', false);
if ($idSubtable !== false) {
$parameters['idSubtable'] = $idSubtable;
}
if (!empty($_GET['_restrictSitesToLogin']) && TaskScheduler::isTaskBeingExecuted()) {
$parameters['_restrictSitesToLogin'] = $_GET['_restrictSitesToLogin'];
}
$report['imageGraphUrl'] = $urlPrefix . Url::getQueryStringFromParameters($parameters);
// thanks to API.getRowEvolution, reports with dimensions can now be plotted using an evolution graph
// however, most reports with a fixed set of dimension values are excluded
// this is done so Piwik Mobile and Scheduled Reports do not display them
$reportWithDimensionsSupportsEvolution = empty($report['constantRowsCount']) || in_array($reportUniqueId, self::$CONSTANT_ROW_COUNT_REPORT_EXCEPTIONS);
$reportSupportsEvolution = !in_array($reportUniqueId, self::$REPORTS_DISABLED_EVOLUTION_GRAPH);
if ($reportSupportsEvolution && $reportWithDimensionsSupportsEvolution) {
$parameters['period'] = $periodForMultiplePeriodGraph;
$parameters['date'] = $dateForMultiplePeriodGraph;
$report['imageGraphEvolutionUrl'] = $urlPrefix . Url::getQueryStringFromParameters($parameters);
}
}
}
private function buildDataTable($idSitesOrIdSite, $period, $date, $segment, $_restrictSitesToLogin, $enhanced, $multipleWebsitesRequested)
{
$allWebsitesRequested = $idSitesOrIdSite == 'all';
if ($allWebsitesRequested) {
// First clear cache
Site::clearCache();
// Then, warm the cache with only the data we should have access to
if (Piwik::isUserIsSuperUser() && !TaskScheduler::isTaskBeingExecuted()) {
$sites = APISitesManager::getInstance()->getAllSites();
} else {
$sites = APISitesManager::getInstance()->getSitesWithAtLeastViewAccess($limit = false, $_restrictSitesToLogin);
}
// Both calls above have called Site::setSitesFromArray. We now get these sites:
$sitesToProblablyAdd = Site::getSites();
} else {
$sitesToProblablyAdd = array(APISitesManager::getInstance()->getSiteFromId($idSitesOrIdSite));
}
// build the archive type used to query archive data
$archive = Archive::build($idSitesOrIdSite, $period, $date, $segment, $_restrictSitesToLogin);
// determine what data will be displayed
$fieldsToGet = array();
$columnNameRewrites = array();
$apiECommerceMetrics = array();
$apiMetrics = API::getApiMetrics($enhanced);
foreach ($apiMetrics as $metricName => $metricSettings) {
$fieldsToGet[] = $metricSettings[self::METRIC_RECORD_NAME_KEY];
$columnNameRewrites[$metricSettings[self::METRIC_RECORD_NAME_KEY]] = $metricName;
if ($metricSettings[self::METRIC_IS_ECOMMERCE_KEY]) {
$apiECommerceMetrics[$metricName] = $metricSettings;
}
}
// get the data
// $dataTable instanceOf Set
$dataTable = $archive->getDataTableFromNumeric($fieldsToGet);
$dataTable = $this->mergeDataTableMapAndPopulateLabel($idSitesOrIdSite, $multipleWebsitesRequested, $dataTable);
if ($dataTable instanceof DataTable\Map) {
foreach ($dataTable->getDataTables() as $table) {
$this->addMissingWebsites($table, $fieldsToGet, $sitesToProblablyAdd);
}
} else {
$this->addMissingWebsites($dataTable, $fieldsToGet, $sitesToProblablyAdd);
}
// calculate total visits/actions/revenue
$this->setMetricsTotalsMetadata($dataTable, $apiMetrics);
// if the period isn't a range & a lastN/previousN date isn't used, we get the same
// data for the last period to show the evolution of visits/actions/revenue
list($strLastDate, $lastPeriod) = Range::getLastDate($date, $period);
if ($strLastDate !== false) {
if ($lastPeriod !== false) {
// NOTE: no easy way to set last period date metadata when range of dates is requested.
// will be easier if DataTable\Map::metadata is removed, and metadata that is
// put there is put directly in DataTable::metadata.
$dataTable->setMetadata(self::getLastPeriodMetadataName('date'), $lastPeriod);
}
$pastArchive = Archive::build($idSitesOrIdSite, $period, $strLastDate, $segment, $_restrictSitesToLogin);
$pastData = $pastArchive->getDataTableFromNumeric($fieldsToGet);
$pastData = $this->mergeDataTableMapAndPopulateLabel($idSitesOrIdSite, $multipleWebsitesRequested, $pastData);
// use past data to calculate evolution percentages
$this->calculateEvolutionPercentages($dataTable, $pastData, $apiMetrics);
$this->setPastDataMetadata($dataTable, $pastData, $apiMetrics);
}
// remove eCommerce related metrics on non eCommerce Piwik sites
// note: this is not optimal in terms of performance: those metrics should not be retrieved in the first place
if ($enhanced) {
if ($dataTable instanceof DataTable\Map) {
foreach ($dataTable->getDataTables() as $table) {
$this->removeEcommerceRelatedMetricsOnNonEcommercePiwikSites($table, $apiECommerceMetrics);
}
} else {
$this->removeEcommerceRelatedMetricsOnNonEcommercePiwikSites($dataTable, $apiECommerceMetrics);
}
}
// move the site id to a metadata column
$dataTable->filter('ColumnCallbackAddMetadata', array('label', 'idsite'));
// set the label of each row to the site name
if ($multipleWebsitesRequested) {
$dataTable->filter('ColumnCallbackReplace', array('label', '\\Piwik\\Site::getNameFor'));
} else {
$dataTable->filter('ColumnDelete', array('label'));
}
// replace record names with user friendly metric names
$dataTable->filter('ReplaceColumnNames', array($columnNameRewrites));
// Ensures data set sorted, for Metadata output
$dataTable->filter('Sort', array(self::NB_VISITS_METRIC, 'desc', $naturalSort = false));
// filter rows without visits
// note: if only one website is queried and there are no visits, we can not remove the row otherwise
// ResponseBuilder throws 'Call to a member function getColumns() on a non-object'
if ($multipleWebsitesRequested && !$enhanced) {
$dataTable->filter('ColumnCallbackDeleteRow', array(self::NB_VISITS_METRIC, function ($value) {
return $value == 0;
}));
}
return $dataTable;
}
/**
* @group Core
*
* @dataProvider testRunTasksTestCases
*/
public function testRunTasks($expectedTimetable, $expectedExecutedTasks, $timetableBeforeTaskExecution, $configuredTasks)
{
// temporarily unload plugins
$plugins = \Piwik\Plugin\Manager::getInstance()->getLoadedPlugins();
$plugins = array_map(function ($p) {
return $p->getPluginName();
}, $plugins);
\Piwik\Plugin\Manager::getInstance()->unloadPlugins();
// make sure the get tasks event returns our configured tasks
\Piwik\Piwik::addAction(TaskScheduler::GET_TASKS_EVENT, function (&$tasks) use($configuredTasks) {
$tasks = $configuredTasks;
});
// stub the piwik option object to control the returned option value
self::stubPiwikOption(serialize($timetableBeforeTaskExecution));
TaskScheduler::unsetInstance();
// execute tasks
$executionResults = TaskScheduler::runTasks();
// assert methods are executed
$executedTasks = array();
foreach ($executionResults as $executionResult) {
$executedTasks[] = $executionResult['task'];
$this->assertNotEmpty($executionResult['output']);
}
$this->assertEquals($expectedExecutedTasks, $executedTasks);
// assert the timetable is correctly updated
$timetable = new ScheduledTaskTimetable();
$this->assertEquals($expectedTimetable, $timetable->getTimetable());
// restore loaded plugins & piwik options
EventDispatcher::getInstance()->clearObservers(TaskScheduler::GET_TASKS_EVENT);
\Piwik\Plugin\Manager::getInstance()->loadPlugins($plugins);
self::resetPiwikOption();
}
/**
* Tracker requests will automatically trigger the Scheduled tasks.
* This is useful for users who don't setup the cron,
* but still want daily/weekly/monthly PDF reports emailed automatically.
*
* This is similar to calling the API CoreAdminHome.runScheduledTasks (see misc/cron/archive.php)
*/
protected static function runScheduledTasks()
{
$now = time();
// Currently, there are no hourly tasks. When there are some,
// this could be too aggressive minimum interval (some hours would be skipped in case of low traffic)
$minimumInterval = Config::getInstance()->Tracker['scheduled_tasks_min_interval'];
// If the user disabled browser archiving, he has already setup a cron
// To avoid parallel requests triggering the Scheduled Tasks,
// Get last time tasks started executing
$cache = Cache::getCacheGeneral();
if ($minimumInterval <= 0 || empty($cache['isBrowserTriggerArchivingEnabled'])) {
Common::printDebug("-> Scheduled tasks not running in Tracker: Browser archiving is disabled.");
return;
}
$nextRunTime = $cache['lastTrackerCronRun'] + $minimumInterval;
if (isset($GLOBALS['PIWIK_TRACKER_DEBUG_FORCE_SCHEDULED_TASKS']) && $GLOBALS['PIWIK_TRACKER_DEBUG_FORCE_SCHEDULED_TASKS'] || $cache['lastTrackerCronRun'] === false || $nextRunTime < $now) {
$cache['lastTrackerCronRun'] = $now;
Cache::setCacheGeneral($cache);
self::initCorePiwikInTrackerMode();
Option::set('lastTrackerCronRun', $cache['lastTrackerCronRun']);
Common::printDebug('-> Scheduled Tasks: Starting...');
// save current user privilege and temporarily assume super user privilege
$isSuperUser = Piwik::isUserIsSuperUser();
// Scheduled tasks assume Super User is running
Piwik::setUserIsSuperUser();
// While each plugins should ensure that necessary languages are loaded,
// we ensure English translations at least are loaded
Translate::loadEnglishTranslation();
$resultTasks = TaskScheduler::runTasks();
// restore original user privilege
Piwik::setUserIsSuperUser($isSuperUser);
Common::printDebug($resultTasks);
Common::printDebug('Finished Scheduled Tasks.');
} else {
Common::printDebug("-> Scheduled tasks not triggered.");
}
Common::printDebug("Next run will be from: " . date('Y-m-d H:i:s', $nextRunTime) . ' UTC');
}
