/** * @return Piwik_CoreAdminHome_API */ public static function getInstance() { if (self::$instance == null) { self::$instance = new self(); } return self::$instance; }
public function init() { $this->initCore(); $this->initTokenAuth(); $this->initCheckCli(); $this->initLog(); $this->displayHelp(); $this->initPiwikHost(); $this->initStateFromParameters(); Piwik::setUserIsSuperUser(true); $this->logSection("INIT"); $this->log("Querying Piwik API at: {$this->piwikUrl}"); $this->log("Running as Super User: "******"accept-invalid-ssl-certificate"); // Test the specified piwik URL is valid $response = $this->request("?module=API&method=API.getDefaultMetricTranslations&format=php"); $responseUnserialized = @unserialize($response); if ($response === false || !is_array($responseUnserialized)) { $this->logFatalError("The Piwik URL {$this->piwikUrl} does not seem to be pointing to a Piwik server. Response was '{$response}'."); } $this->log("Notes"); // Information about timeout $this->todayArchiveTimeToLive = Piwik_ArchiveProcessing::getTodayArchiveTimeToLive(); $this->log("- Reports for today will be processed at most every " . Piwik_ArchiveProcessing::getTodayArchiveTimeToLive() . " seconds. You can change this value in Piwik UI > Settings > General Settings."); $this->log("- Reports for the current week/month/year will be refreshed at most every " . $this->processPeriodsMaximumEverySeconds . " seconds."); // Fetching segments to process $this->segments = Piwik_CoreAdminHome_API::getInstance()->getKnownSegmentsToArchive(); if (empty($this->segments)) { $this->segments = array(); } if (!empty($this->segments)) { $this->log("- Segments to pre-process for each website and each period: " . implode(", ", $this->segments)); } // Try and not request older data we know is already archived if ($this->timeLastCompleted !== false) { $dateLast = time() - $this->timeLastCompleted; $this->log("- Archiving was last executed without error " . Piwik::getPrettyTimeFromSeconds($dateLast, true, $isHtml = false) . " ago"); } $this->initWebsitesToProcess(); flush(); }
protected function initWebsitesToProcess() { $this->allWebsites = Piwik_SitesManager_API::getInstance()->getAllSitesId(); if ($this->shouldArchiveAllWebsites) { $this->websites = $this->allWebsites; $this->log("Will process " . count($this->websites) . " websites"); } else { // 1) All websites with visits since the last archive.php execution $timestampActiveTraffic = $this->timeLastCompleted; if (empty($timestampActiveTraffic)) { $timestampActiveTraffic = time() - $this->firstRunActiveWebsitesWithTraffic; $this->log("--force-all-periods was detected: we will process websites with visits in the last " . Piwik::getPrettyTimeFromSeconds($this->firstRunActiveWebsitesWithTraffic, true, false)); } $this->websites = Piwik_SitesManager_API::getInstance()->getSitesIdWithVisits($timestampActiveTraffic); $websiteIds = !empty($this->websites) ? ", IDs: " . implode(", ", $this->websites) : ""; $prettySeconds = Piwik::getPrettyTimeFromSeconds(empty($this->timeLastCompleted) ? $this->firstRunActiveWebsitesWithTraffic : time() - $this->timeLastCompleted, true, false); $this->log("Will process " . count($this->websites) . " websites with new visits since " . $prettySeconds . " " . $websiteIds); // 2) All websites that had reports in the past invalidated recently // eg. when using Python log import script $this->idSitesInvalidatedOldReports = Piwik_CoreAdminHome_API::getWebsiteIdsToInvalidate(); $this->idSitesInvalidatedOldReports = array_intersect($this->idSitesInvalidatedOldReports, $this->allWebsites); if (count($this->idSitesInvalidatedOldReports) > 0) { $websiteIds = ", IDs: " . implode(", ", $this->idSitesInvalidatedOldReports); $this->log("Will process " . count($this->idSitesInvalidatedOldReports) . " other websites because some old data reports have been invalidated (eg. using the Log Import script) " . $websiteIds); $this->websites = array_merge($this->websites, $this->idSitesInvalidatedOldReports); } // 3) Also process all other websites which days have finished since the last run. // This ensures we process the previous day/week/month/year that just finished, even if there was no new visit $uniqueTimezones = Piwik_SitesManager_API::getInstance()->getUniqueSiteTimezones(); $timezoneToProcess = array(); foreach ($uniqueTimezones as &$timezone) { $processedDateInTz = Piwik_Date::factory((int) $timestampActiveTraffic, $timezone); $currentDateInTz = Piwik_Date::factory('now', $timezone); if ($processedDateInTz->toString() != $currentDateInTz->toString()) { $timezoneToProcess[] = $timezone; } } $websiteDayHasFinishedSinceLastRun = Piwik_SitesManager_API::getInstance()->getSitesIdFromTimezones($timezoneToProcess); $websiteDayHasFinishedSinceLastRun = array_diff($websiteDayHasFinishedSinceLastRun, $this->websites); $this->websiteDayHasFinishedSinceLastRun = $websiteDayHasFinishedSinceLastRun; if (count($websiteDayHasFinishedSinceLastRun) > 0) { $websiteIds = !empty($websiteDayHasFinishedSinceLastRun) ? ", IDs: " . implode(", ", $websiteDayHasFinishedSinceLastRun) : ""; $this->log("Will process " . count($websiteDayHasFinishedSinceLastRun) . " other websites because the last time they were archived was on a different day (in the website's timezone) " . $websiteIds); $this->websites = array_merge($this->websites, $websiteDayHasFinishedSinceLastRun); } } }
/** * When tracking data in the past (using Tracking API), this function * can be used to invalidate reports for the idSites and dates where new data * was added. * DEV: If you call this API, the UI should display the data correctly, but will process * in real time, which could be very slow after large data imports. * After calling this function via REST, you can manually force all data * to be reprocessed by visiting the script as the Super User: * http://example.net/piwik/misc/cron/archive.php?token_auth=$SUPER_USER_TOKEN_AUTH_HERE * REQUIREMENTS: On large piwik setups, you will need in PHP configuration: max_execution_time = 0 * We recommend to use an hourly schedule of the script at misc/cron/archive.php * More information: http://piwik.org/setup-auto-archiving/ * * @param string $idSites Comma separated list of idSite that have had data imported for the specified dates * @param string $dates Comma separated list of dates to invalidate for all these websites * @return array */ public function invalidateArchivedReports($idSites, $dates) { $idSites = Piwik_Site::getIdSitesFromIdSitesString($idSites); if (empty($idSites)) { throw new Exception("Specify a value for &idSites= as a comma separated list of website IDs, for which your token_auth has 'admin' permission"); } Piwik::checkUserHasAdminAccess($idSites); // Ensure the specified dates are valid $toInvalidate = $invalidDates = array(); $dates = explode(',', $dates); $dates = array_unique($dates); foreach ($dates as $theDate) { try { $date = Piwik_Date::factory($theDate); } catch (Exception $e) { $invalidDates[] = $theDate; continue; } if ($date->toString() == $theDate) { $toInvalidate[] = $date; } else { $invalidDates[] = $theDate; } } // Lookup archive tables $tables = Piwik::getTablesInstalled(); $archiveTables = Piwik::getTablesArchivesInstalled(); // If using the feature "Delete logs older than N days"... $logsAreDeletedBeforeThisDate = Piwik_Config::getInstance()->Deletelogs['delete_logs_schedule_lowest_interval']; $logsDeleteEnabled = Piwik_Config::getInstance()->Deletelogs['delete_logs_enable']; $minimumDateWithLogs = false; if ($logsDeleteEnabled && $logsAreDeletedBeforeThisDate) { $minimumDateWithLogs = Piwik_Date::factory('today')->subDay($logsAreDeletedBeforeThisDate); } // Given the list of dates, process which tables they should be deleted from $minDate = false; $warningDates = $processedDates = array(); /* @var $date Piwik_Date */ foreach ($toInvalidate as $date) { // we should only delete reports for dates that are more recent than N days if ($minimumDateWithLogs && $date->isEarlier($minimumDateWithLogs)) { $warningDates[] = $date->toString(); } else { $processedDates[] = $date->toString(); } $month = $date->toString('Y_m'); // For a given date, we must invalidate in the monthly archive table $datesByMonth[$month][] = $date->toString(); // But also the year stored in January $year = $date->toString('Y_01'); $datesByMonth[$year][] = $date->toString(); // but also weeks overlapping several months stored in the month where the week is starting /* @var $week Piwik_Period_Week */ $week = Piwik_Period::factory('week', $date); $week = $week->getDateStart()->toString('Y_m'); $datesByMonth[$week][] = $date->toString(); // Keep track of the minimum date for each website if ($minDate === false || $date->isEarlier($minDate)) { $minDate = $date; } } // In each table, invalidate day/week/month/year containing this date $sqlIdSites = implode(",", $idSites); foreach ($archiveTables as $table) { // Extract Y_m from table name $suffix = str_replace(array('archive_numeric_', 'archive_blob_'), '', Piwik_Common::unprefixTable($table)); if (!isset($datesByMonth[$suffix])) { continue; } // Dates which are to be deleted from this table $datesToDeleteInTable = $datesByMonth[$suffix]; // Build one statement to delete all dates from the given table $sql = $bind = array(); $datesToDeleteInTable = array_unique($datesToDeleteInTable); foreach ($datesToDeleteInTable as $dateToDelete) { $sql[] = '(date1 <= ? AND ? <= date2)'; $bind[] = $dateToDelete; $bind[] = $dateToDelete; } $sql = implode(" OR ", $sql); $query = "DELETE FROM {$table} " . " WHERE ( {$sql} ) " . " AND idsite IN (" . $sqlIdSites . ")"; Piwik_Query($query, $bind); // var_dump($query);var_dump($bind); } // Update piwik_site.ts_created $query = "UPDATE " . Piwik_Common::prefixTable("site") . " SET ts_created = ?" . " WHERE idsite IN ( {$sqlIdSites} )\n\t\t\t\t\tAND ts_created > ?"; $minDateSql = $minDate->subDay(1)->getDatetime(); $bind = array($minDateSql, $minDateSql); Piwik_Query($query, $bind); // var_dump($query);var_dump($bind); // Force to re-process data for these websites in the next archive.php cron run $invalidatedIdSites = Piwik_CoreAdminHome_API::getWebsiteIdsToInvalidate(); $invalidatedIdSites = array_merge($invalidatedIdSites, $idSites); $invalidatedIdSites = array_unique($invalidatedIdSites); $invalidatedIdSites = array_values($invalidatedIdSites); Piwik_SetOption(self::OPTION_INVALIDATED_IDSITES, serialize($invalidatedIdSites)); Piwik_Site::clearCache(); $output = array(); // output logs if ($warningDates) { $output[] = 'Warning: the following Dates have not been invalidated, because they are earlier than your Log Deletion limit: ' . implode(", ", $warningDates) . "\n The last day with logs is " . $minimumDateWithLogs . ". " . "\n Please disable 'Delete old Logs' or set it to a higher deletion threshold (eg. 180 days or 365 years).'."; } $output[] = "Success. The following dates were invalidated successfully: " . implode(", ", $processedDates); return $output; }