public function onSiteDeleted($idSite) { // we do not delete logs here on purpose (you can run these queries on the log_ tables to delete all data) Cache::deleteCacheWebsiteAttributes($idSite); $archiveInvalidator = new ArchiveInvalidator(); $archiveInvalidator->forgetRememberedArchivedReportsToInvalidateForSite($idSite); }
/** * When tracking data in the past (using Tracking API), this function * can be used to invalidate reports for the idSites and dates where new data * was added. * DEV: If you call this API, the UI should display the data correctly, but will process * in real time, which could be very slow after large data imports. * After calling this function via REST, you can manually force all data * to be reprocessed by visiting the script as the Super User: * http://example.net/piwik/misc/cron/archive.php?token_auth=$SUPER_USER_TOKEN_AUTH_HERE * REQUIREMENTS: On large piwik setups, you will need in PHP configuration: max_execution_time = 0 * We recommend to use an hourly schedule of the script. * More information: http://piwik.org/setup-auto-archiving/ * * @param string $idSites Comma separated list of idSite that have had data imported for the specified dates * @param string $dates Comma separated list of dates to invalidate for all these websites * @param string $period If specified (one of day, week, month, year, range) it will only invalidates archives for this period. * Note: because week, month, year, range reports aggregate day reports then you need to specifically invalidate day reports to see * other periods reports processed.. * @throws Exception * @return array * @hideExceptForSuperUser */ public function invalidateArchivedReports($idSites, $dates, $period = false) { $idSites = Site::getIdSitesFromIdSitesString($idSites); if (empty($idSites)) { throw new Exception("Specify a value for &idSites= as a comma separated list of website IDs, for which your token_auth has 'admin' permission"); } Piwik::checkUserHasAdminAccess($idSites); $invalidator = new ArchiveInvalidator(); $output = $invalidator->markArchivesAsInvalidated($idSites, $dates, $period); Site::clearCache(); return $output; }
public function test_getColumnNamesFromTable() { Fixture::createWebsite('2014-12-12 00:01:02'); Fixture::createWebsite('2014-12-12 00:01:02'); $ar = new ArchiveInvalidator(); $ar->rememberToInvalidateArchivedReportsLater(1, Date::factory('2014-04-05')); $ar->rememberToInvalidateArchivedReportsLater(2, Date::factory('2014-04-05')); $ar->rememberToInvalidateArchivedReportsLater(2, Date::factory('2014-04-06')); $api = API::getInstance(); $cronarchive = new TestCronArchive(Fixture::getRootUrl() . 'tests/PHPUnit/proxy/index.php'); $cronarchive->setApiToInvalidateArchivedReport($api); $cronarchive->init(); $expectedInvalidations = array(array(array(1, 2), '2014-04-05'), array(array(2), '2014-04-06')); $this->assertEquals($expectedInvalidations, $api->getInvalidatedReports()); }
/** * When tracking data in the past (using Tracking API), this function * can be used to invalidate reports for the idSites and dates where new data * was added. * DEV: If you call this API, the UI should display the data correctly, but will process * in real time, which could be very slow after large data imports. * After calling this function via REST, you can manually force all data * to be reprocessed by visiting the script as the Super User: * http://example.net/piwik/misc/cron/archive.php?token_auth=$SUPER_USER_TOKEN_AUTH_HERE * REQUIREMENTS: On large piwik setups, you will need in PHP configuration: max_execution_time = 0 * We recommend to use an hourly schedule of the script. * More information: http://piwik.org/setup-auto-archiving/ * * @param string $idSites Comma separated list of idSite that have had data imported for the specified dates * @param string $dates Comma separated list of dates to invalidate for all these websites * @param string $period If specified (one of day, week, month, year, range) it will only invalidates archives for this period. * Note: because week, month, year, range reports aggregate day reports then you need to specifically invalidate day reports to see * other periods reports processed.. * @throws Exception * @return array * @hideExceptForSuperUser */ public function invalidateArchivedReports($idSites, $dates, $period = false) { $idSites = Site::getIdSitesFromIdSitesString($idSites); if (empty($idSites)) { throw new Exception("Specify a value for &idSites= as a comma separated list of website IDs, for which your token_auth has 'admin' permission"); } Piwik::checkUserHasAdminAccess($idSites); list($dateObjects, $invalidDates) = $this->getDatesToInvalidateFromString($dates); $invalidator = new ArchiveInvalidator(); $output = $invalidator->markArchivesAsInvalidated($idSites, $dateObjects, $period); if ($invalidDates) { $output[] = 'Warning: some of the Dates to invalidate were invalid: ' . implode(", ", $invalidDates) . ". Piwik simply ignored those and proceeded with the others."; } Site::clearCache(); return $output; }
private function invalidateArchivesUsingActionDuplicates($archivesAffected, OutputInterface $output) { $output->write("Invalidating archives affected by duplicates fixed..."); foreach ($archivesAffected as $archiveInfo) { $this->archiveInvalidator->markArchivesAsInvalidated(array($archiveInfo['idsite']), $archiveInfo['server_time'], $period = false); } $output->writeln("Done."); }
/** * Invalidates report data, forcing it to be recomputed during the next archiving run. * * Note: This is done automatically when tracking or importing visits in the past. * * @param string $idSites Comma separated list of site IDs to invalidate reports for. * @param string $dates Comma separated list of dates of periods to invalidate reports for. * @param string|bool $period The type of period to invalidate: either 'day', 'week', 'month', 'year', 'range'. * The command will automatically cascade up, invalidating reports for parent periods as * well. So invalidating a day will invalidate the week it's in, the month it's in and the * year it's in, since those periods will need to be recomputed too. * @param string|bool $segment Optional. The segment to invalidate reports for. * @param bool $cascadeDown If true, child periods will be invalidated as well. So if it is requested to invalidate * a month, then all the weeks and days within that month will also be invalidated. But only * if this parameter is set. * @throws Exception * @return array * @hideExceptForSuperUser */ public function invalidateArchivedReports($idSites, $dates, $period = false, $segment = false, $cascadeDown = false) { $idSites = Site::getIdSitesFromIdSitesString($idSites); if (empty($idSites)) { throw new Exception("Specify a value for &idSites= as a comma separated list of website IDs, for which your token_auth has 'admin' permission"); } Piwik::checkUserHasAdminAccess($idSites); if (!empty($segment)) { $segment = new Segment($segment, $idSites); } else { $segment = null; } list($dateObjects, $invalidDates) = $this->getDatesToInvalidateFromString($dates); $invalidationResult = $this->invalidator->markArchivesAsInvalidated($idSites, $dateObjects, $period, $segment, (bool) $cascadeDown); $output = $invalidationResult->makeOutputLogs(); if ($invalidDates) { $output[] = 'Warning: some of the Dates to invalidate were invalid: ' . implode(", ", $invalidDates) . ". Piwik simply ignored those and proceeded with the others."; } Site::clearCache(); // TODO: is this needed? it shouldn't be needed... return $invalidationResult->makeOutputLogs(); }
private function markArchivedReportsAsInvalidIfArchiveAlreadyFinished() { $idSite = (int) $this->request->getIdSite(); $time = $this->request->getCurrentTimestamp(); $timezone = $this->getTimezoneForSite($idSite); if (!isset($timezone)) { return; } $date = Date::factory((int) $time, $timezone); if (!$date->isToday()) { // we don't have to handle in case date is in future as it is not allowed by tracker $this->invalidator->rememberToInvalidateArchivedReportsLater($idSite, $date); } }
public function test_onSiteDeleted_shouldRemoveRememberedArchiveReports() { $archive = new ArchiveInvalidator(); $archive->rememberToInvalidateArchivedReportsLater($this->siteId, Date::factory('2014-04-05')); $archive->rememberToInvalidateArchivedReportsLater($this->siteId, Date::factory('2014-04-06')); $archive->rememberToInvalidateArchivedReportsLater(4949, Date::factory('2014-04-05')); $expected = array('2014-04-05' => array($this->siteId, 4949), '2014-04-06' => array($this->siteId)); $this->assertEquals($expected, $archive->getRememberedArchivedReportsThatShouldBeInvalidated()); $this->manager->onSiteDeleted($this->siteId); $expected = array('2014-04-05' => array(4949)); $this->assertEquals($expected, $archive->getRememberedArchivedReportsThatShouldBeInvalidated()); }
public function invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain() { $invalidator = new ArchiveInvalidator(); $sitesPerDays = $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); foreach ($sitesPerDays as $date => $siteIds) { $listSiteIds = implode(',', $siteIds); try { $this->logger->info('Will invalidate archived reports for ' . $date . ' for following websites ids: ' . $listSiteIds); $this->getApiToInvalidateArchivedReport()->invalidateArchivedReports($siteIds, $date); } catch (Exception $e) { $this->logger->info('Failed to invalidate archived reports: ' . $e->getMessage()); } } }
private function rememberReport($idSite, $date) { $date = Date::factory($date); $this->invalidator->rememberToInvalidateArchivedReportsLater($idSite, $date); }
private function invalidatedReportsIfNeeded() { $siteIdsRequested = $this->getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet(); if (empty($siteIdsRequested)) { return; // all requested site ids were already handled } $sitesPerDays = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); foreach ($sitesPerDays as $date => $siteIds) { if (empty($siteIds)) { continue; } $siteIdsToActuallyInvalidate = array_intersect($siteIds, $siteIdsRequested); if (empty($siteIdsToActuallyInvalidate)) { continue; // all site ids that should be handled are already handled } try { $this->invalidator->markArchivesAsInvalidated($siteIdsToActuallyInvalidate, array(Date::factory($date)), false); } catch (\Exception $e) { Site::clearCache(); throw $e; } } Site::clearCache(); }
private function assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $requestDate, $expectedRemeberedArchivedReports) { /** @var Visit $visit */ list($visit) = $this->prepareVisitWithRequest(array('idsite' => $idsite, 'rec' => 1, 'cip' => '156.146.156.146', 'token_auth' => Fixture::getTokenAuth()), $requestDate); $visit->handle(); $archive = new ArchiveInvalidator(); $remembered = $archive->getRememberedArchivedReportsThatShouldBeInvalidated(); $this->assertSame($expectedRemeberedArchivedReports, $remembered); }
public function test_Archive_getNumeric_shouldInvalidateRememberedReportsOncePerRequestIfNeeded() { $this->markTestSkippedOnPhp53(); // Tests that getting a visits summary metric (nb_visits) & a Goal's metric (Goal_revenue) // at the same time works. $dateTimeRange = '2010-01-03,2010-01-06'; $columns = array('nb_visits', 'Goal_nb_conversions', 'nb_actions'); $idSite1 = self::$fixture->idSite1; $archive = Archive::build($idSite1, 'range', $dateTimeRange); $result = $archive->getNumeric($columns); $this->assertEquals(array('nb_visits' => 5, 'Goal_nb_conversions' => 6, 'nb_actions' => 13), $result); $cache = Cache::getTransientCache(); $this->assertEquals(array(self::$fixture->idSite1, self::$fixture->idSite2), $cache->fetch('Archive.SiteIdsOfRememberedReportsInvalidated')); $invalidator = new ArchiveInvalidator(); self::$fixture->trackVisits(); // trackVisits should remember to invalidate archived reports $this->assertNotEmpty($invalidator->getRememberedArchivedReportsThatShouldBeInvalidated()); // although there were new tracked visists it doesn'T change as the report invalidation is cached and was // already invalidated in previous Archive::get(); $archive = Archive::build($idSite1, 'range', $dateTimeRange); $result = $archive->getNumeric($columns); $this->assertEquals(array('nb_visits' => 5, 'Goal_nb_conversions' => 6, 'nb_actions' => 13), $result); // make sure the caching in archive::get() worked and they are still to be invalidated $this->assertCount(10, $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated()); // now we force to actually invalidate archived reports again and then archive will be rebuilt for requsted siteId = 1 $cache->delete('Archive.SiteIdsOfRememberedReportsInvalidated'); $archive = Archive::build($idSite1, 'range', $dateTimeRange); $result = $archive->getNumeric($columns); // archive::get() should have invalidated siteId 1 and siteId 2 should be still to be done $expectedArchiveReportsLeft = array('2010-01-04' => array(2)); $this->assertEquals($expectedArchiveReportsLeft, $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated()); $this->assertEquals(array('nb_visits' => 6, 'Goal_nb_conversions' => 7, 'nb_actions' => 26), $result); }