Exemple #1
0
 /**
  * @return bool `true` if the purge was executed, `false` if it was skipped.
  * @throws \Exception
  */
 public function purgeOutdatedArchives()
 {
     if ($this->willPurgingCausePotentialProblemInUI()) {
         $this->logger->info("Purging temporary archives: skipped (browser triggered archiving not enabled & not running after core:archive)");
         return false;
     }
     $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled();
     $this->logger->info("Purging archives in {tableCount} archive tables.", array('tableCount' => count($archiveTables)));
     // keep track of dates we purge for, since getTablesArchivesInstalled() will return numeric & blob
     // tables (so dates will appear two times, and we should only purge once per date)
     $datesPurged = array();
     foreach ($archiveTables as $table) {
         $date = ArchiveTableCreator::getDateFromTableName($table);
         list($year, $month) = explode('_', $date);
         // Somehow we may have archive tables created with older dates, prevent exception from being thrown
         if ($year > 1990) {
             if (empty($datesPurged[$date])) {
                 $dateObj = Date::factory("{$year}-{$month}-15");
                 $this->archivePurger->purgeOutdatedArchives($dateObj);
                 $this->archivePurger->purgeArchivesWithPeriodRange($dateObj);
                 $datesPurged[$date] = true;
             } else {
                 $this->logger->debug("Date {date} already purged.", array('date' => $date));
             }
         } else {
             $this->logger->info("Skipping purging of archive tables *_{year}_{month}, year <= 1990.", array('year' => $year, 'month' => $month));
         }
     }
     return true;
 }
 public static function purgeInvalidatedArchives()
 {
     $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled();
     foreach ($archiveTables as $archiveTable) {
         /**
          * Select the archives that have already been invalidated and have been since re-processed.
          * It purges records for each distinct { archive name (includes segment hash) , idsite, date, period } tuple.
          */
         $query = '
             SELECT t1.idarchive FROM `' . $archiveTable . '` t1
             INNER JOIN `' . $archiveTable . '` t2
                 ON t1.name = t2.name AND t1.idsite=t2.idsite
                 AND t1.date1=t2.date1 AND t1.date2=t2.date2 AND t1.period=t2.period
             WHERE t1.value = ' . ArchiveWriter::DONE_INVALIDATED . '
             AND t2.value IN(' . ArchiveWriter::DONE_OK . ', ' . ArchiveWriter::DONE_OK_TEMPORARY . ')
             AND t1.ts_archived < t2.ts_archived AND t1.name LIKE \'done%\'';
         $result = Db::fetchAll($query);
         if (count($result) > 0) {
             $archiveIds = array_map(function ($elm) {
                 return $elm['idarchive'];
             }, $result);
             $date = ArchiveTableCreator::getDateFromTableName($archiveTable);
             $date = Date::factory(str_replace('_', '-', $date) . '-01');
             self::deleteArchiveIds($date, $archiveIds);
         }
     }
 }
Exemple #3
0
 public function purgeOutdatedArchives()
 {
     $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled();
     foreach ($archiveTables as $table) {
         $date = ArchiveTableCreator::getDateFromTableName($table);
         list($year, $month) = explode('_', $date);
         // Somehow we may have archive tables created with older dates, prevent exception from being thrown
         if ($year > 1990) {
             ArchivePurger::purgeOutdatedArchives(Date::factory("{$year}-{$month}-15"));
         }
     }
 }
 public static function purgeInvalidatedArchives()
 {
     $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled();
     foreach ($archiveTables as $archiveTable) {
         /**
          * Select the archives that have already been invalidated and have been since re-processed.
          * It purges records for each distinct { archive name (includes segment hash) , idsite, date, period } tuple.
          */
         $result = self::getModel()->purgeInvalidatedArchiveTable($archiveTable);
         if (count($result) > 0) {
             $archiveIds = array_map(function ($elm) {
                 return $elm['idarchive'];
             }, $result);
             $date = ArchiveTableCreator::getDateFromTableName($archiveTable);
             $date = Date::factory(str_replace('_', '-', $date) . '-01');
             self::deleteArchiveIds($date, $archiveIds);
         }
     }
 }
Exemple #5
0
 /**
  * When tracking data in the past (using Tracking API), this function
  * can be used to invalidate reports for the idSites and dates where new data
  * was added.
  * DEV: If you call this API, the UI should display the data correctly, but will process
  *      in real time, which could be very slow after large data imports.
  *      After calling this function via REST, you can manually force all data
  *      to be reprocessed by visiting the script as the Super User:
  *      http://example.net/piwik/misc/cron/archive.php?token_auth=$SUPER_USER_TOKEN_AUTH_HERE
  * REQUIREMENTS: On large piwik setups, you will need in PHP configuration: max_execution_time = 0
  *    We recommend to use an hourly schedule of the script.
  *    More information: http://piwik.org/setup-auto-archiving/
  *
  * @param string $idSites Comma separated list of idSite that have had data imported for the specified dates
  * @param string $dates Comma separated list of dates to invalidate for all these websites
  * @throws Exception
  * @return array
  */
 public function invalidateArchivedReports($idSites, $dates)
 {
     $idSites = Site::getIdSitesFromIdSitesString($idSites);
     if (empty($idSites)) {
         throw new Exception("Specify a value for &idSites= as a comma separated list of website IDs, for which your token_auth has 'admin' permission");
     }
     Piwik::checkUserHasAdminAccess($idSites);
     // Ensure the specified dates are valid
     $toInvalidate = $invalidDates = array();
     $dates = explode(',', trim($dates));
     $dates = array_unique($dates);
     foreach ($dates as $theDate) {
         $theDate = trim($theDate);
         try {
             $date = Date::factory($theDate);
         } catch (Exception $e) {
             $invalidDates[] = $theDate;
             continue;
         }
         if ($date->toString() == $theDate) {
             $toInvalidate[] = $date;
         } else {
             $invalidDates[] = $theDate;
         }
     }
     // If using the feature "Delete logs older than N days"...
     $purgeDataSettings = PrivacyManager::getPurgeDataSettings();
     $logsAreDeletedBeforeThisDate = $purgeDataSettings['delete_logs_schedule_lowest_interval'];
     $logsDeleteEnabled = $purgeDataSettings['delete_logs_enable'];
     $minimumDateWithLogs = false;
     if ($logsDeleteEnabled && $logsAreDeletedBeforeThisDate) {
         $minimumDateWithLogs = Date::factory('today')->subDay($logsAreDeletedBeforeThisDate);
     }
     // Given the list of dates, process which tables they should be deleted from
     $minDate = false;
     $warningDates = $processedDates = array();
     /* @var $date Date */
     foreach ($toInvalidate as $date) {
         // we should only delete reports for dates that are more recent than N days
         if ($minimumDateWithLogs && $date->isEarlier($minimumDateWithLogs)) {
             $warningDates[] = $date->toString();
         } else {
             $processedDates[] = $date->toString();
         }
         $month = $date->toString('Y_m');
         // For a given date, we must invalidate in the monthly archive table
         $datesByMonth[$month][] = $date->toString();
         // But also the year stored in January
         $year = $date->toString('Y_01');
         $datesByMonth[$year][] = $date->toString();
         // but also weeks overlapping several months stored in the month where the week is starting
         /* @var $week Week */
         $week = Period\Factory::build('week', $date);
         $weekAsString = $week->getDateStart()->toString('Y_m');
         $datesByMonth[$weekAsString][] = $date->toString();
         // Keep track of the minimum date for each website
         if ($minDate === false || $date->isEarlier($minDate)) {
             $minDate = $date;
         }
     }
     if (empty($minDate)) {
         throw new Exception("Check the 'dates' parameter is a valid date.");
     }
     // In each table, invalidate day/week/month/year containing this date
     $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled();
     foreach ($archiveTables as $table) {
         // Extract Y_m from table name
         $suffix = ArchiveTableCreator::getDateFromTableName($table);
         if (!isset($datesByMonth[$suffix])) {
             continue;
         }
         // Dates which are to be deleted from this table
         $datesToDeleteInTable = $datesByMonth[$suffix];
         // Build one statement to delete all dates from the given table
         $sql = $bind = array();
         $datesToDeleteInTable = array_unique($datesToDeleteInTable);
         foreach ($datesToDeleteInTable as $dateToDelete) {
             $sql[] = '(date1 <= ? AND ? <= date2)';
             $bind[] = $dateToDelete;
             $bind[] = $dateToDelete;
         }
         $sql = implode(" OR ", $sql);
         $query = "DELETE FROM {$table} " . " WHERE ( {$sql} ) " . " AND idsite IN (" . implode(",", $idSites) . ")";
         Db::query($query, $bind);
     }
     \Piwik\Plugins\SitesManager\API::getInstance()->updateSiteCreatedTime($idSites, $minDate);
     // Force to re-process data for these websites in the next cron core:archive command run
     $invalidatedIdSites = self::getWebsiteIdsToInvalidate();
     $invalidatedIdSites = array_merge($invalidatedIdSites, $idSites);
     $invalidatedIdSites = array_unique($invalidatedIdSites);
     $invalidatedIdSites = array_values($invalidatedIdSites);
     Option::set(self::OPTION_INVALIDATED_IDSITES, serialize($invalidatedIdSites));
     Site::clearCache();
     $output = array();
     // output logs
     if ($warningDates) {
         $output[] = 'Warning: the following Dates have not been invalidated, because they are earlier than your Log Deletion limit: ' . implode(", ", $warningDates) . "\n The last day with logs is " . $minimumDateWithLogs . ". " . "\n Please disable 'Delete old Logs' or set it to a higher deletion threshold (eg. 180 days or 365 years).'.";
     }
     $output[] = "Success. The following dates were invalidated successfully: " . implode(", ", $processedDates);
     return $output;
 }
 private function getInvalidatedArchives()
 {
     $result = array();
     foreach (ArchiveTableCreator::getTablesArchivesInstalled(ArchiveTableCreator::NUMERIC_TABLE) as $table) {
         $date = ArchiveTableCreator::getDateFromTableName($table);
         $sql = "SELECT CONCAT(idsite, '.', date1, '.', date2, '.', period, '.', name) FROM {$table} WHERE name LIKE 'done%' AND value = ?";
         $archiveSpecs = Db::fetchAll($sql, array(ArchiveWriter::DONE_INVALIDATED));
         $archiveSpecs = array_map('reset', $archiveSpecs);
         $result[$date] = $archiveSpecs;
     }
     return $result;
 }
 /**
  * @param $idSites
  * @param $period string
  * @param $datesByMonth array
  * @throws \Exception
  */
 private function markArchivesInvalidatedFor($idSites, $period, $datesByMonth)
 {
     $invalidateForPeriodId = $this->getPeriodId($period);
     // In each table, invalidate day/week/month/year containing this date
     $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled();
     $archiveNumericTables = array_filter($archiveTables, function ($name) {
         return ArchiveTableCreator::getTypeFromTableName($name) == ArchiveTableCreator::NUMERIC_TABLE;
     });
     foreach ($archiveNumericTables as $table) {
         // Extract Y_m from table name
         $suffix = ArchiveTableCreator::getDateFromTableName($table);
         if (!isset($datesByMonth[$suffix])) {
             continue;
         }
         // Dates which are to be deleted from this table
         $datesToDelete = $datesByMonth[$suffix];
         self::getModel()->updateArchiveAsInvalidated($table, $idSites, $invalidateForPeriodId, $datesToDelete);
     }
 }
 private function getAllArchiveTableMonths()
 {
     $tableMonths = array();
     foreach (ArchiveTableCreator::getTablesArchivesInstalled() as $table) {
         $tableMonths[] = ArchiveTableCreator::getDateFromTableName($table);
     }
     return $tableMonths;
 }
 /**
  * If we're going to keep segmented reports, we need to know which archives are
  * for segments. This info is only in the numeric tables, so we must query them.
  */
 private function findSegmentArchives($numericTables)
 {
     if (!is_null($this->segmentArchiveIds)) {
         return;
     }
     foreach ($numericTables as $table) {
         $tableDate = ArchiveTableCreator::getDateFromTableName($table);
         $maxIdArchive = Db::fetchOne("SELECT MAX(idarchive) FROM {$table}");
         $sql = "SELECT idarchive\n\t\t\t\t\t  FROM {$table}\n\t\t\t\t\t WHERE name != 'done'\n\t\t\t\t\t   AND name LIKE 'done_%.%'\n\t\t\t\t\t   AND idarchive >= ?\n\t\t\t\t\t   AND idarchive < ?";
         $this->segmentArchiveIds[$tableDate] = array();
         foreach (Db::segmentedFetchAll($sql, 0, $maxIdArchive, self::$selectSegmentSize) as $row) {
             $this->segmentArchiveIds[$tableDate][] = $row['idarchive'];
         }
     }
 }
 /**
  * @param int[] $idSites
  * @param string[][][] $dates
  * @throws \Exception
  */
 private function markArchivesInvalidated($idSites, $dates, Segment $segment = null)
 {
     $archiveNumericTables = ArchiveTableCreator::getTablesArchivesInstalled($type = ArchiveTableCreator::NUMERIC_TABLE);
     foreach ($archiveNumericTables as $table) {
         $tableDate = ArchiveTableCreator::getDateFromTableName($table);
         if (empty($dates[$tableDate])) {
             continue;
         }
         $this->model->updateArchiveAsInvalidated($table, $idSites, $dates[$tableDate], $segment);
     }
 }
 /**
  * If we're going to keep segmented reports, we need to know which archives are
  * for segments. This info is only in the numeric tables, so we must query them.
  */
 private function findSegmentArchives($numericTables)
 {
     if (!is_null($this->segmentArchiveIds) || empty($numericTables)) {
         return;
     }
     $Generic = Factory::getGeneric();
     foreach ($numericTables as $table) {
         $tableDate = ArchiveTableCreator::getDateFromTableName($table);
         $maxIdArchive = $Generic->getMax($table, 'idarchive');
         $sql = "SELECT idarchive\n                      FROM {$table}\n                     WHERE name != 'done'\n                       AND name LIKE 'done_%.%'\n                       AND idarchive >= ?\n                       AND idarchive < ?";
         if (is_null($this->segmentArchiveIds)) {
             $this->segmentArchiveIds = array();
         }
         $this->segmentArchiveIds[$tableDate] = array();
         foreach ($Generic->segmentedFetchAll($sql, 0, $maxIdArchive, self::$selectSegmentSize) as $row) {
             $this->segmentArchiveIds[$tableDate][] = $row['idarchive'];
         }
     }
 }
 /**
  * @param InputInterface $input
  * @return Date[]
  */
 private function getDatesToPurgeFor(InputInterface $input)
 {
     $dates = array();
     $dateSpecifier = $input->getArgument('dates');
     if (count($dateSpecifier) === 1 && reset($dateSpecifier) == self::ALL_DATES_STRING) {
         foreach (ArchiveTableCreator::getTablesArchivesInstalled() as $table) {
             $tableDate = ArchiveTableCreator::getDateFromTableName($table);
             list($year, $month) = explode('_', $tableDate);
             $dates[] = Date::factory($year . '-' . $month . '-' . '01');
         }
     } else {
         $includeYearArchives = $input->getOption('include-year-archives');
         foreach ($dateSpecifier as $date) {
             $dateObj = Date::factory($date);
             $yearMonth = $dateObj->toString('Y-m');
             $dates[$yearMonth] = $dateObj;
             // if --include-year-archives is supplied, add a date for the january table for this date's year
             // so year archives will be purged
             if ($includeYearArchives) {
                 $janYearMonth = $dateObj->toString('Y') . '-01';
                 if (empty($dates[$janYearMonth])) {
                     $dates[$janYearMonth] = Date::factory($janYearMonth . '-01');
                 }
             }
         }
         $dates = array_values($dates);
     }
     return $dates;
 }