public function test_usingMultipleSharedSiteIds() { $second = new SharedSiteIds(array(7, 9, 11, 6, 1, 2)); // should ignore his queue and help processing the existing queue $this->assertEquals(4, $second->getNumSites()); $this->assertEquals(4, $this->sharedSiteIds->getNumSites()); $this->assertEquals(array(1, 2, 5, 9), $second->getAllSiteIdsToArchive()); $this->assertEquals(1, $second->getNextSiteId()); $this->assertEquals(1, $second->getNumProcessedWebsites()); $this->assertEquals(array(2, 5, 9), $this->sharedSiteIds->getAllSiteIdsToArchive()); $this->assertEquals(2, $this->sharedSiteIds->getNextSiteId()); $this->assertEquals(2, $this->sharedSiteIds->getNumProcessedWebsites()); $this->assertEquals(array(5, 9), $second->getAllSiteIdsToArchive()); $this->assertEquals(5, $second->getNextSiteId()); $this->assertEquals(3, $second->getNumProcessedWebsites()); $this->assertEquals(array(9), $this->sharedSiteIds->getAllSiteIdsToArchive()); $this->assertEquals(9, $this->sharedSiteIds->getNextSiteId()); $this->assertEquals(4, $this->sharedSiteIds->getNumProcessedWebsites()); $this->assertNull($second->getNextSiteId()); $this->assertEquals(4, $second->getNumProcessedWebsites()); $this->assertEquals(array(), $second->getAllSiteIdsToArchive()); $this->assertNull($this->sharedSiteIds->getNextSiteId()); $this->assertEquals(4, $this->sharedSiteIds->getNumProcessedWebsites()); $this->assertEquals(array(), $this->sharedSiteIds->getAllSiteIdsToArchive()); }
/** * Main function, runs archiving on all websites with new activity */ public function run() { $timer = new Timer(); $this->logSection("START"); $this->log("Starting Piwik reports archiving..."); do { $idSite = $this->websites->getNextSiteId(); if (null === $idSite) { break; } flush(); $requestsBefore = $this->requests; if ($idSite <= 0) { continue; } $skipWebsiteForced = in_array($idSite, $this->shouldSkipSpecifiedSites); if ($skipWebsiteForced) { $this->log("Skipped website id {$idSite}, found in --skip-idsites "); $this->skipped++; continue; } /** * This event is triggered before the cron archiving process starts archiving data for a single * site. * * @param int $idSite The ID of the site we're archiving data for. */ Piwik::postEvent('CronArchive.archiveSingleSite.start', array($idSite)); $completed = $this->archiveSingleSite($idSite, $requestsBefore); /** * This event is triggered immediately after the cron archiving process starts archiving data for a single * site. * * @param int $idSite The ID of the site we're archiving data for. */ Piwik::postEvent('CronArchive.archiveSingleSite.finish', array($idSite, $completed)); } while (!empty($idSite)); $this->log("Done archiving!"); $this->logSection("SUMMARY"); $this->log("Total visits for today across archived websites: " . $this->visitsToday); $totalWebsites = count($this->allWebsites); $this->skipped = $totalWebsites - $this->websitesWithVisitsSinceLastRun; $this->log("Archived today's reports for {$this->websitesWithVisitsSinceLastRun} websites"); $this->log("Archived week/month/year for {$this->archivedPeriodsArchivesWebsite} websites"); $this->log("Skipped {$this->skipped} websites: no new visit since the last script execution"); $this->log("Skipped {$this->skippedDayArchivesWebsites} websites day archiving: existing daily reports are less than {$this->todayArchiveTimeToLive} seconds old"); $this->log("Skipped {$this->skippedPeriodsArchivesWebsite} websites week/month/year archiving: existing periods reports are less than {$this->processPeriodsMaximumEverySeconds} seconds old"); $this->log("Total API requests: {$this->requests}"); //DONE: done/total, visits, wtoday, wperiods, reqs, time, errors[count]: first eg. $percent = $this->websites->getNumSites() == 0 ? "" : " " . round($this->processed * 100 / $this->websites->getNumSites(), 0) . "%"; $this->log("done: " . $this->processed . "/" . $this->websites->getNumSites() . "" . $percent . ", " . $this->visitsToday . " vtoday, {$this->websitesWithVisitsSinceLastRun} wtoday, {$this->archivedPeriodsArchivesWebsite} wperiods, " . $this->requests . " req, " . round($timer->getTimeMs()) . " ms, " . (empty($this->errors) ? self::NO_ERROR : count($this->errors) . " errors.")); $this->log($timer->__toString()); }
/** * Main function, runs archiving on all websites with new activity */ public function run() { $timer = new Timer(); $this->logSection("START"); $this->logger->info("Starting Piwik reports archiving..."); do { $idSite = $this->websites->getNextSiteId(); if (null === $idSite) { break; } flush(); $requestsBefore = $this->requests; if ($idSite <= 0) { continue; } $skipWebsiteForced = in_array($idSite, $this->shouldSkipSpecifiedSites); if ($skipWebsiteForced) { $this->logger->info("Skipped website id {$idSite}, found in --skip-idsites "); $this->skipped++; continue; } $shouldCheckIfArchivingIsNeeded = !$this->shouldArchiveSpecifiedSites && !$this->shouldArchiveAllSites; $hasWebsiteDayFinishedSinceLastRun = in_array($idSite, $this->websiteDayHasFinishedSinceLastRun); $isOldReportInvalidatedForWebsite = $this->isOldReportInvalidatedForWebsite($idSite); if ($shouldCheckIfArchivingIsNeeded) { // if not specific sites and not all websites should be archived, we check whether we actually have // to process the archives for this website (only if there were visits since midnight) if (!$hasWebsiteDayFinishedSinceLastRun && !$isOldReportInvalidatedForWebsite) { if ($this->isWebsiteUsingTheTracker($idSite) && !$this->hadWebsiteTrafficSinceMidnightInTimezone($idSite)) { $this->logger->info("Will skip website {$idSite} as archiving is not needed"); $this->skipped++; continue; } } elseif ($hasWebsiteDayFinishedSinceLastRun) { $this->logger->info("Day has finished for website {$idSite} since last run"); } elseif ($isOldReportInvalidatedForWebsite) { $this->logger->info("Old report was invalidated for website {$idSite}"); } } /** * This event is triggered before the cron archiving process starts archiving data for a single * site. * * @param int $idSite The ID of the site we're archiving data for. */ Piwik::postEvent('CronArchive.archiveSingleSite.start', array($idSite)); $completed = $this->archiveSingleSite($idSite, $requestsBefore); /** * This event is triggered immediately after the cron archiving process starts archiving data for a single * site. * * @param int $idSite The ID of the site we're archiving data for. */ Piwik::postEvent('CronArchive.archiveSingleSite.finish', array($idSite, $completed)); } while (!empty($idSite)); $this->logger->info("Done archiving!"); $this->logSection("SUMMARY"); $this->logger->info("Total visits for today across archived websites: " . $this->visitsToday); $totalWebsites = count($this->allWebsites); $this->skipped = $totalWebsites - $this->websitesWithVisitsSinceLastRun; $this->logger->info("Archived today's reports for {$this->websitesWithVisitsSinceLastRun} websites"); $this->logger->info("Archived week/month/year for {$this->archivedPeriodsArchivesWebsite} websites"); $this->logger->info("Skipped {$this->skipped} websites: no new visit since the last script execution"); $this->logger->info("Skipped {$this->skippedDayArchivesWebsites} websites day archiving: existing daily reports are less than {$this->todayArchiveTimeToLive} seconds old"); $this->logger->info("Skipped {$this->skippedPeriodsArchivesWebsite} websites week/month/year archiving: existing periods reports are less than {$this->processPeriodsMaximumEverySeconds} seconds old"); $this->logger->info("Total API requests: {$this->requests}"); //DONE: done/total, visits, wtoday, wperiods, reqs, time, errors[count]: first eg. $percent = $this->websites->getNumSites() == 0 ? "" : " " . round($this->processed * 100 / $this->websites->getNumSites(), 0) . "%"; $this->logger->info("done: " . $this->processed . "/" . $this->websites->getNumSites() . "" . $percent . ", " . $this->visitsToday . " vtoday, {$this->websitesWithVisitsSinceLastRun} wtoday, {$this->archivedPeriodsArchivesWebsite} wperiods, " . $this->requests . " req, " . round($timer->getTimeMs()) . " ms, " . (empty($this->errors) ? self::NO_ERROR : count($this->errors) . " errors.")); $this->logger->info($timer->__toString()); }