public function onSiteDeleted($idSite)
{
// we do not delete logs here on purpose (you can run these queries on the log_ tables to delete all data)
Cache::deleteCacheWebsiteAttributes($idSite);
$archiveInvalidator = new ArchiveInvalidator();
$archiveInvalidator->forgetRememberedArchivedReportsToInvalidateForSite($idSite);
}
/**
* When tracking data in the past (using Tracking API), this function
* can be used to invalidate reports for the idSites and dates where new data
* was added.
* DEV: If you call this API, the UI should display the data correctly, but will process
* in real time, which could be very slow after large data imports.
* After calling this function via REST, you can manually force all data
* to be reprocessed by visiting the script as the Super User:
* http://example.net/piwik/misc/cron/archive.php?token_auth=$SUPER_USER_TOKEN_AUTH_HERE
* REQUIREMENTS: On large piwik setups, you will need in PHP configuration: max_execution_time = 0
* We recommend to use an hourly schedule of the script.
* More information: http://piwik.org/setup-auto-archiving/
*
* @param string $idSites Comma separated list of idSite that have had data imported for the specified dates
* @param string $dates Comma separated list of dates to invalidate for all these websites
* @param string $period If specified (one of day, week, month, year, range) it will only invalidates archives for this period.
* Note: because week, month, year, range reports aggregate day reports then you need to specifically invalidate day reports to see
* other periods reports processed..
* @throws Exception
* @return array
* @hideExceptForSuperUser
*/
public function invalidateArchivedReports($idSites, $dates, $period = false)
{
$idSites = Site::getIdSitesFromIdSitesString($idSites);
if (empty($idSites)) {
throw new Exception("Specify a value for &idSites= as a comma separated list of website IDs, for which your token_auth has 'admin' permission");
}
Piwik::checkUserHasAdminAccess($idSites);
$invalidator = new ArchiveInvalidator();
$output = $invalidator->markArchivesAsInvalidated($idSites, $dates, $period);
Site::clearCache();
return $output;
}
public function test_getColumnNamesFromTable()
{
Fixture::createWebsite('2014-12-12 00:01:02');
Fixture::createWebsite('2014-12-12 00:01:02');
$ar = new ArchiveInvalidator();
$ar->rememberToInvalidateArchivedReportsLater(1, Date::factory('2014-04-05'));
$ar->rememberToInvalidateArchivedReportsLater(2, Date::factory('2014-04-05'));
$ar->rememberToInvalidateArchivedReportsLater(2, Date::factory('2014-04-06'));
$api = API::getInstance();
$cronarchive = new TestCronArchive(Fixture::getRootUrl() . 'tests/PHPUnit/proxy/index.php');
$cronarchive->setApiToInvalidateArchivedReport($api);
$cronarchive->init();
$expectedInvalidations = array(array(array(1, 2), '2014-04-05'), array(array(2), '2014-04-06'));
$this->assertEquals($expectedInvalidations, $api->getInvalidatedReports());
}
/**
* When tracking data in the past (using Tracking API), this function
* can be used to invalidate reports for the idSites and dates where new data
* was added.
* DEV: If you call this API, the UI should display the data correctly, but will process
* in real time, which could be very slow after large data imports.
* After calling this function via REST, you can manually force all data
* to be reprocessed by visiting the script as the Super User:
* http://example.net/piwik/misc/cron/archive.php?token_auth=$SUPER_USER_TOKEN_AUTH_HERE
* REQUIREMENTS: On large piwik setups, you will need in PHP configuration: max_execution_time = 0
* We recommend to use an hourly schedule of the script.
* More information: http://piwik.org/setup-auto-archiving/
*
* @param string $idSites Comma separated list of idSite that have had data imported for the specified dates
* @param string $dates Comma separated list of dates to invalidate for all these websites
* @param string $period If specified (one of day, week, month, year, range) it will only invalidates archives for this period.
* Note: because week, month, year, range reports aggregate day reports then you need to specifically invalidate day reports to see
* other periods reports processed..
* @throws Exception
* @return array
* @hideExceptForSuperUser
*/
public function invalidateArchivedReports($idSites, $dates, $period = false)
{
$idSites = Site::getIdSitesFromIdSitesString($idSites);
if (empty($idSites)) {
throw new Exception("Specify a value for &idSites= as a comma separated list of website IDs, for which your token_auth has 'admin' permission");
}
Piwik::checkUserHasAdminAccess($idSites);
list($dateObjects, $invalidDates) = $this->getDatesToInvalidateFromString($dates);
$invalidator = new ArchiveInvalidator();
$output = $invalidator->markArchivesAsInvalidated($idSites, $dateObjects, $period);
if ($invalidDates) {
$output[] = 'Warning: some of the Dates to invalidate were invalid: ' . implode(", ", $invalidDates) . ". Piwik simply ignored those and proceeded with the others.";
}
Site::clearCache();
return $output;
}
private function invalidateArchivesUsingActionDuplicates($archivesAffected, OutputInterface $output)
{
$output->write("Invalidating archives affected by duplicates fixed...");
foreach ($archivesAffected as $archiveInfo) {
$this->archiveInvalidator->markArchivesAsInvalidated(array($archiveInfo['idsite']), $archiveInfo['server_time'], $period = false);
}
$output->writeln("Done.");
}
/**
* Invalidates report data, forcing it to be recomputed during the next archiving run.
*
* Note: This is done automatically when tracking or importing visits in the past.
*
* @param string $idSites Comma separated list of site IDs to invalidate reports for.
* @param string $dates Comma separated list of dates of periods to invalidate reports for.
* @param string|bool $period The type of period to invalidate: either 'day', 'week', 'month', 'year', 'range'.
* The command will automatically cascade up, invalidating reports for parent periods as
* well. So invalidating a day will invalidate the week it's in, the month it's in and the
* year it's in, since those periods will need to be recomputed too.
* @param string|bool $segment Optional. The segment to invalidate reports for.
* @param bool $cascadeDown If true, child periods will be invalidated as well. So if it is requested to invalidate
* a month, then all the weeks and days within that month will also be invalidated. But only
* if this parameter is set.
* @throws Exception
* @return array
* @hideExceptForSuperUser
*/
public function invalidateArchivedReports($idSites, $dates, $period = false, $segment = false, $cascadeDown = false)
{
$idSites = Site::getIdSitesFromIdSitesString($idSites);
if (empty($idSites)) {
throw new Exception("Specify a value for &idSites= as a comma separated list of website IDs, for which your token_auth has 'admin' permission");
}
Piwik::checkUserHasAdminAccess($idSites);
if (!empty($segment)) {
$segment = new Segment($segment, $idSites);
} else {
$segment = null;
}
list($dateObjects, $invalidDates) = $this->getDatesToInvalidateFromString($dates);
$invalidationResult = $this->invalidator->markArchivesAsInvalidated($idSites, $dateObjects, $period, $segment, (bool) $cascadeDown);
$output = $invalidationResult->makeOutputLogs();
if ($invalidDates) {
$output[] = 'Warning: some of the Dates to invalidate were invalid: ' . implode(", ", $invalidDates) . ". Piwik simply ignored those and proceeded with the others.";
}
Site::clearCache();
// TODO: is this needed? it shouldn't be needed...
return $invalidationResult->makeOutputLogs();
}
private function markArchivedReportsAsInvalidIfArchiveAlreadyFinished()
{
$idSite = (int) $this->request->getIdSite();
$time = $this->request->getCurrentTimestamp();
$timezone = $this->getTimezoneForSite($idSite);
if (!isset($timezone)) {
return;
}
$date = Date::factory((int) $time, $timezone);
if (!$date->isToday()) {
// we don't have to handle in case date is in future as it is not allowed by tracker
$this->invalidator->rememberToInvalidateArchivedReportsLater($idSite, $date);
}
}
public function test_onSiteDeleted_shouldRemoveRememberedArchiveReports()
{
$archive = new ArchiveInvalidator();
$archive->rememberToInvalidateArchivedReportsLater($this->siteId, Date::factory('2014-04-05'));
$archive->rememberToInvalidateArchivedReportsLater($this->siteId, Date::factory('2014-04-06'));
$archive->rememberToInvalidateArchivedReportsLater(4949, Date::factory('2014-04-05'));
$expected = array('2014-04-05' => array($this->siteId, 4949), '2014-04-06' => array($this->siteId));
$this->assertEquals($expected, $archive->getRememberedArchivedReportsThatShouldBeInvalidated());
$this->manager->onSiteDeleted($this->siteId);
$expected = array('2014-04-05' => array(4949));
$this->assertEquals($expected, $archive->getRememberedArchivedReportsThatShouldBeInvalidated());
}
public function invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain()
{
$invalidator = new ArchiveInvalidator();
$sitesPerDays = $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated();
foreach ($sitesPerDays as $date => $siteIds) {
$listSiteIds = implode(',', $siteIds);
try {
$this->logger->info('Will invalidate archived reports for ' . $date . ' for following websites ids: ' . $listSiteIds);
$this->getApiToInvalidateArchivedReport()->invalidateArchivedReports($siteIds, $date);
} catch (Exception $e) {
$this->logger->info('Failed to invalidate archived reports: ' . $e->getMessage());
}
}
}
private function rememberReport($idSite, $date)
{
$date = Date::factory($date);
$this->invalidator->rememberToInvalidateArchivedReportsLater($idSite, $date);
}
private function invalidatedReportsIfNeeded()
{
$siteIdsRequested = $this->getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet();
if (empty($siteIdsRequested)) {
return;
// all requested site ids were already handled
}
$sitesPerDays = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated();
foreach ($sitesPerDays as $date => $siteIds) {
if (empty($siteIds)) {
continue;
}
$siteIdsToActuallyInvalidate = array_intersect($siteIds, $siteIdsRequested);
if (empty($siteIdsToActuallyInvalidate)) {
continue;
// all site ids that should be handled are already handled
}
try {
$this->invalidator->markArchivesAsInvalidated($siteIdsToActuallyInvalidate, array(Date::factory($date)), false);
} catch (\Exception $e) {
Site::clearCache();
throw $e;
}
}
Site::clearCache();
}
private function assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $requestDate, $expectedRemeberedArchivedReports)
{
/** @var Visit $visit */
list($visit) = $this->prepareVisitWithRequest(array('idsite' => $idsite, 'rec' => 1, 'cip' => '156.146.156.146', 'token_auth' => Fixture::getTokenAuth()), $requestDate);
$visit->handle();
$archive = new ArchiveInvalidator();
$remembered = $archive->getRememberedArchivedReportsThatShouldBeInvalidated();
$this->assertSame($expectedRemeberedArchivedReports, $remembered);
}
public function test_Archive_getNumeric_shouldInvalidateRememberedReportsOncePerRequestIfNeeded()
{
$this->markTestSkippedOnPhp53();
// Tests that getting a visits summary metric (nb_visits) & a Goal's metric (Goal_revenue)
// at the same time works.
$dateTimeRange = '2010-01-03,2010-01-06';
$columns = array('nb_visits', 'Goal_nb_conversions', 'nb_actions');
$idSite1 = self::$fixture->idSite1;
$archive = Archive::build($idSite1, 'range', $dateTimeRange);
$result = $archive->getNumeric($columns);
$this->assertEquals(array('nb_visits' => 5, 'Goal_nb_conversions' => 6, 'nb_actions' => 13), $result);
$cache = Cache::getTransientCache();
$this->assertEquals(array(self::$fixture->idSite1, self::$fixture->idSite2), $cache->fetch('Archive.SiteIdsOfRememberedReportsInvalidated'));
$invalidator = new ArchiveInvalidator();
self::$fixture->trackVisits();
// trackVisits should remember to invalidate archived reports
$this->assertNotEmpty($invalidator->getRememberedArchivedReportsThatShouldBeInvalidated());
// although there were new tracked visists it doesn'T change as the report invalidation is cached and was
// already invalidated in previous Archive::get();
$archive = Archive::build($idSite1, 'range', $dateTimeRange);
$result = $archive->getNumeric($columns);
$this->assertEquals(array('nb_visits' => 5, 'Goal_nb_conversions' => 6, 'nb_actions' => 13), $result);
// make sure the caching in archive::get() worked and they are still to be invalidated
$this->assertCount(10, $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated());
// now we force to actually invalidate archived reports again and then archive will be rebuilt for requsted siteId = 1
$cache->delete('Archive.SiteIdsOfRememberedReportsInvalidated');
$archive = Archive::build($idSite1, 'range', $dateTimeRange);
$result = $archive->getNumeric($columns);
// archive::get() should have invalidated siteId 1 and siteId 2 should be still to be done
$expectedArchiveReportsLeft = array('2010-01-04' => array(2));
$this->assertEquals($expectedArchiveReportsLeft, $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated());
$this->assertEquals(array('nb_visits' => 6, 'Goal_nb_conversions' => 7, 'nb_actions' => 26), $result);
}
