public function test_usingMultipleSharedSiteIds()
{
$second = new SharedSiteIds(array(7, 9, 11, 6, 1, 2));
// should ignore his queue and help processing the existing queue
$this->assertEquals(4, $second->getNumSites());
$this->assertEquals(4, $this->sharedSiteIds->getNumSites());
$this->assertEquals(array(1, 2, 5, 9), $second->getAllSiteIdsToArchive());
$this->assertEquals(1, $second->getNextSiteId());
$this->assertEquals(1, $second->getNumProcessedWebsites());
$this->assertEquals(array(2, 5, 9), $this->sharedSiteIds->getAllSiteIdsToArchive());
$this->assertEquals(2, $this->sharedSiteIds->getNextSiteId());
$this->assertEquals(2, $this->sharedSiteIds->getNumProcessedWebsites());
$this->assertEquals(array(5, 9), $second->getAllSiteIdsToArchive());
$this->assertEquals(5, $second->getNextSiteId());
$this->assertEquals(3, $second->getNumProcessedWebsites());
$this->assertEquals(array(9), $this->sharedSiteIds->getAllSiteIdsToArchive());
$this->assertEquals(9, $this->sharedSiteIds->getNextSiteId());
$this->assertEquals(4, $this->sharedSiteIds->getNumProcessedWebsites());
$this->assertNull($second->getNextSiteId());
$this->assertEquals(4, $second->getNumProcessedWebsites());
$this->assertEquals(array(), $second->getAllSiteIdsToArchive());
$this->assertNull($this->sharedSiteIds->getNextSiteId());
$this->assertEquals(4, $this->sharedSiteIds->getNumProcessedWebsites());
$this->assertEquals(array(), $this->sharedSiteIds->getAllSiteIdsToArchive());
}
/**
* Main function, runs archiving on all websites with new activity
*/
public function run()
{
$timer = new Timer();
$this->logSection("START");
$this->log("Starting Piwik reports archiving...");
do {
$idSite = $this->websites->getNextSiteId();
if (null === $idSite) {
break;
}
flush();
$requestsBefore = $this->requests;
if ($idSite <= 0) {
continue;
}
$skipWebsiteForced = in_array($idSite, $this->shouldSkipSpecifiedSites);
if ($skipWebsiteForced) {
$this->log("Skipped website id {$idSite}, found in --skip-idsites ");
$this->skipped++;
continue;
}
/**
* This event is triggered before the cron archiving process starts archiving data for a single
* site.
*
* @param int $idSite The ID of the site we're archiving data for.
*/
Piwik::postEvent('CronArchive.archiveSingleSite.start', array($idSite));
$completed = $this->archiveSingleSite($idSite, $requestsBefore);
/**
* This event is triggered immediately after the cron archiving process starts archiving data for a single
* site.
*
* @param int $idSite The ID of the site we're archiving data for.
*/
Piwik::postEvent('CronArchive.archiveSingleSite.finish', array($idSite, $completed));
} while (!empty($idSite));
$this->log("Done archiving!");
$this->logSection("SUMMARY");
$this->log("Total visits for today across archived websites: " . $this->visitsToday);
$totalWebsites = count($this->allWebsites);
$this->skipped = $totalWebsites - $this->websitesWithVisitsSinceLastRun;
$this->log("Archived today's reports for {$this->websitesWithVisitsSinceLastRun} websites");
$this->log("Archived week/month/year for {$this->archivedPeriodsArchivesWebsite} websites");
$this->log("Skipped {$this->skipped} websites: no new visit since the last script execution");
$this->log("Skipped {$this->skippedDayArchivesWebsites} websites day archiving: existing daily reports are less than {$this->todayArchiveTimeToLive} seconds old");
$this->log("Skipped {$this->skippedPeriodsArchivesWebsite} websites week/month/year archiving: existing periods reports are less than {$this->processPeriodsMaximumEverySeconds} seconds old");
$this->log("Total API requests: {$this->requests}");
//DONE: done/total, visits, wtoday, wperiods, reqs, time, errors[count]: first eg.
$percent = $this->websites->getNumSites() == 0 ? "" : " " . round($this->processed * 100 / $this->websites->getNumSites(), 0) . "%";
$this->log("done: " . $this->processed . "/" . $this->websites->getNumSites() . "" . $percent . ", " . $this->visitsToday . " vtoday, {$this->websitesWithVisitsSinceLastRun} wtoday, {$this->archivedPeriodsArchivesWebsite} wperiods, " . $this->requests . " req, " . round($timer->getTimeMs()) . " ms, " . (empty($this->errors) ? self::NO_ERROR : count($this->errors) . " errors."));
$this->log($timer->__toString());
}
/**
* Main function, runs archiving on all websites with new activity
*/
public function run()
{
$timer = new Timer();
$this->logSection("START");
$this->logger->info("Starting Piwik reports archiving...");
do {
$idSite = $this->websites->getNextSiteId();
if (null === $idSite) {
break;
}
flush();
$requestsBefore = $this->requests;
if ($idSite <= 0) {
continue;
}
$skipWebsiteForced = in_array($idSite, $this->shouldSkipSpecifiedSites);
if ($skipWebsiteForced) {
$this->logger->info("Skipped website id {$idSite}, found in --skip-idsites ");
$this->skipped++;
continue;
}
$shouldCheckIfArchivingIsNeeded = !$this->shouldArchiveSpecifiedSites && !$this->shouldArchiveAllSites;
$hasWebsiteDayFinishedSinceLastRun = in_array($idSite, $this->websiteDayHasFinishedSinceLastRun);
$isOldReportInvalidatedForWebsite = $this->isOldReportInvalidatedForWebsite($idSite);
if ($shouldCheckIfArchivingIsNeeded) {
// if not specific sites and not all websites should be archived, we check whether we actually have
// to process the archives for this website (only if there were visits since midnight)
if (!$hasWebsiteDayFinishedSinceLastRun && !$isOldReportInvalidatedForWebsite) {
if ($this->isWebsiteUsingTheTracker($idSite) && !$this->hadWebsiteTrafficSinceMidnightInTimezone($idSite)) {
$this->logger->info("Will skip website {$idSite} as archiving is not needed");
$this->skipped++;
continue;
}
} elseif ($hasWebsiteDayFinishedSinceLastRun) {
$this->logger->info("Day has finished for website {$idSite} since last run");
} elseif ($isOldReportInvalidatedForWebsite) {
$this->logger->info("Old report was invalidated for website {$idSite}");
}
}
/**
* This event is triggered before the cron archiving process starts archiving data for a single
* site.
*
* @param int $idSite The ID of the site we're archiving data for.
*/
Piwik::postEvent('CronArchive.archiveSingleSite.start', array($idSite));
$completed = $this->archiveSingleSite($idSite, $requestsBefore);
/**
* This event is triggered immediately after the cron archiving process starts archiving data for a single
* site.
*
* @param int $idSite The ID of the site we're archiving data for.
*/
Piwik::postEvent('CronArchive.archiveSingleSite.finish', array($idSite, $completed));
} while (!empty($idSite));
$this->logger->info("Done archiving!");
$this->logSection("SUMMARY");
$this->logger->info("Total visits for today across archived websites: " . $this->visitsToday);
$totalWebsites = count($this->allWebsites);
$this->skipped = $totalWebsites - $this->websitesWithVisitsSinceLastRun;
$this->logger->info("Archived today's reports for {$this->websitesWithVisitsSinceLastRun} websites");
$this->logger->info("Archived week/month/year for {$this->archivedPeriodsArchivesWebsite} websites");
$this->logger->info("Skipped {$this->skipped} websites: no new visit since the last script execution");
$this->logger->info("Skipped {$this->skippedDayArchivesWebsites} websites day archiving: existing daily reports are less than {$this->todayArchiveTimeToLive} seconds old");
$this->logger->info("Skipped {$this->skippedPeriodsArchivesWebsite} websites week/month/year archiving: existing periods reports are less than {$this->processPeriodsMaximumEverySeconds} seconds old");
$this->logger->info("Total API requests: {$this->requests}");
//DONE: done/total, visits, wtoday, wperiods, reqs, time, errors[count]: first eg.
$percent = $this->websites->getNumSites() == 0 ? "" : " " . round($this->processed * 100 / $this->websites->getNumSites(), 0) . "%";
$this->logger->info("done: " . $this->processed . "/" . $this->websites->getNumSites() . "" . $percent . ", " . $this->visitsToday . " vtoday, {$this->websitesWithVisitsSinceLastRun} wtoday, {$this->archivedPeriodsArchivesWebsite} wperiods, " . $this->requests . " req, " . round($timer->getTimeMs()) . " ms, " . (empty($this->errors) ? self::NO_ERROR : count($this->errors) . " errors."));
$this->logger->info($timer->__toString());
}