Example #1
0
 private function archiveSingleSite($idSite, $requestsBefore)
 {
     $timerWebsite = new Timer();
     $lastTimestampWebsiteProcessedPeriods = $lastTimestampWebsiteProcessedDay = false;
     if ($this->archiveAndRespectTTL) {
         Option::clearCachedOption($this->lastRunKey($idSite, "periods"));
         $lastTimestampWebsiteProcessedPeriods = $this->getPeriodLastProcessedTimestamp($idSite);
         Option::clearCachedOption($this->lastRunKey($idSite, "day"));
         $lastTimestampWebsiteProcessedDay = $this->getDayLastProcessedTimestamp($idSite);
     }
     $this->updateIdSitesInvalidatedOldReports();
     // For period other than days, we only re-process the reports at most
     // 1) every $processPeriodsMaximumEverySeconds
     $secondsSinceLastExecution = time() - $lastTimestampWebsiteProcessedPeriods;
     // if timeout is more than 10 min, we account for a 5 min processing time, and allow trigger 1 min earlier
     if ($this->processPeriodsMaximumEverySeconds > 10 * 60) {
         $secondsSinceLastExecution += 5 * 60;
     }
     $shouldArchivePeriods = $secondsSinceLastExecution > $this->processPeriodsMaximumEverySeconds;
     if (empty($lastTimestampWebsiteProcessedPeriods)) {
         // 2) OR always if script never executed for this website before
         $shouldArchivePeriods = true;
     }
     // (*) If the website is archived because it is a new day in its timezone
     // We make sure all periods are archived, even if there is 0 visit today
     $dayHasEndedMustReprocess = in_array($idSite, $this->websiteDayHasFinishedSinceLastRun);
     if ($dayHasEndedMustReprocess) {
         $shouldArchivePeriods = true;
     }
     // (*) If there was some old reports invalidated for this website
     // we make sure all these old reports are triggered at least once
     $websiteInvalidatedShouldReprocess = $this->isOldReportInvalidatedForWebsite($idSite);
     if ($websiteInvalidatedShouldReprocess) {
         $shouldArchivePeriods = true;
     }
     $websiteIdIsForced = in_array($idSite, $this->shouldArchiveSpecifiedSites);
     if ($websiteIdIsForced) {
         $shouldArchivePeriods = true;
     }
     // Test if we should process this website at all
     $elapsedSinceLastArchiving = time() - $lastTimestampWebsiteProcessedDay;
     // Skip this day archive if last archive was older than TTL
     $existingArchiveIsValid = $elapsedSinceLastArchiving < $this->todayArchiveTimeToLive;
     $skipDayArchive = $existingArchiveIsValid;
     // Invalidate old website forces the archiving for this site
     $skipDayArchive = $skipDayArchive && !$websiteInvalidatedShouldReprocess;
     // Also reprocess when day has ended since last run
     if ($dayHasEndedMustReprocess && !$this->hasBeenProcessedSinceMidnight($idSite, $lastTimestampWebsiteProcessedDay) && !$existingArchiveIsValid) {
         $skipDayArchive = false;
     }
     if ($websiteIdIsForced) {
         $skipDayArchive = false;
     }
     if ($skipDayArchive) {
         $this->logger->info("Skipped website id {$idSite}, already done " . $this->formatter->getPrettyTimeFromSeconds($elapsedSinceLastArchiving, true) . " ago, " . $timerWebsite->__toString());
         $this->skippedDayArchivesWebsites++;
         $this->skipped++;
         return false;
     }
     try {
         $shouldProceed = $this->processArchiveDays($idSite, $lastTimestampWebsiteProcessedDay, $shouldArchivePeriods, $timerWebsite);
     } catch (UnexpectedWebsiteFoundException $e) {
         // this website was deleted in the meantime
         $shouldProceed = false;
         $this->logger->info("Skipped website id {$idSite}, got: UnexpectedWebsiteFoundException, " . $timerWebsite->__toString());
     }
     if (!$shouldProceed) {
         return false;
     }
     if (!$shouldArchivePeriods) {
         $this->logger->info("Skipped website id {$idSite} periods processing, already done " . $this->formatter->getPrettyTimeFromSeconds($elapsedSinceLastArchiving, true) . " ago, " . $timerWebsite->__toString());
         $this->skippedDayArchivesWebsites++;
         $this->skipped++;
         return false;
     }
     $success = $this->processArchiveForPeriods($idSite, $lastTimestampWebsiteProcessedPeriods);
     // Record succesful run of this website's periods archiving
     if ($success) {
         Option::set($this->lastRunKey($idSite, "periods"), time());
     }
     if (!$success) {
         // cancel marking the site as reprocessed
         if ($websiteInvalidatedShouldReprocess) {
             $store = new SitesToReprocessDistributedList();
             $store->add($idSite);
         }
     }
     $this->archivedPeriodsArchivesWebsite++;
     $requestsWebsite = $this->requests - $requestsBefore;
     $this->logger->info("Archived website id = {$idSite}, " . $requestsWebsite . " API requests, " . $timerWebsite->__toString() . " [" . $this->websites->getNumProcessedWebsites() . "/" . $this->websites->getNumSites() . " done]");
     return true;
 }
Example #2
0
 public function test_getNumProcessedWebsites_getNextSiteId()
 {
     $this->assertEquals(0, $this->fixedSiteIds->getNumProcessedWebsites());
     $this->assertEquals(1, $this->fixedSiteIds->getNextSiteId());
     $this->assertEquals(1, $this->fixedSiteIds->getNumProcessedWebsites());
     $this->assertEquals(2, $this->fixedSiteIds->getNextSiteId());
     $this->assertEquals(2, $this->fixedSiteIds->getNumProcessedWebsites());
     $this->assertEquals(5, $this->fixedSiteIds->getNextSiteId());
     $this->assertEquals(3, $this->fixedSiteIds->getNumProcessedWebsites());
     $this->assertEquals(9, $this->fixedSiteIds->getNextSiteId());
     $this->assertEquals(4, $this->fixedSiteIds->getNumProcessedWebsites());
     $this->assertNull($this->fixedSiteIds->getNextSiteId());
     $this->assertEquals(4, $this->fixedSiteIds->getNumProcessedWebsites());
 }
Example #3
0
 private function archiveSingleSite($idSite, $requestsBefore)
 {
     $timerWebsite = new Timer();
     $lastTimestampWebsiteProcessedPeriods = $lastTimestampWebsiteProcessedDay = false;
     if ($this->archiveAndRespectTTL) {
         Option::clearCachedOption($this->lastRunKey($idSite, "periods"));
         $lastTimestampWebsiteProcessedPeriods = Option::get($this->lastRunKey($idSite, "periods"));
         Option::clearCachedOption($this->lastRunKey($idSite, "day"));
         $lastTimestampWebsiteProcessedDay = Option::get($this->lastRunKey($idSite, "day"));
     }
     $this->updateIdSitesInvalidatedOldReports();
     // For period other than days, we only re-process the reports at most
     // 1) every $processPeriodsMaximumEverySeconds
     $secondsSinceLastExecution = time() - $lastTimestampWebsiteProcessedPeriods;
     // if timeout is more than 10 min, we account for a 5 min processing time, and allow trigger 1 min earlier
     if ($this->processPeriodsMaximumEverySeconds > 10 * 60) {
         $secondsSinceLastExecution += 5 * 60;
     }
     $shouldArchivePeriods = $secondsSinceLastExecution > $this->processPeriodsMaximumEverySeconds;
     if (empty($lastTimestampWebsiteProcessedPeriods)) {
         // 2) OR always if script never executed for this website before
         $shouldArchivePeriods = true;
     }
     // (*) If the website is archived because it is a new day in its timezone
     // We make sure all periods are archived, even if there is 0 visit today
     $dayHasEndedMustReprocess = in_array($idSite, $this->websiteDayHasFinishedSinceLastRun);
     if ($dayHasEndedMustReprocess) {
         $shouldArchivePeriods = true;
     }
     // (*) If there was some old reports invalidated for this website
     // we make sure all these old reports are triggered at least once
     $websiteIsOldDataInvalidate = in_array($idSite, $this->idSitesInvalidatedOldReports);
     if ($websiteIsOldDataInvalidate) {
         $shouldArchivePeriods = true;
     }
     $websiteIdIsForced = in_array($idSite, $this->shouldArchiveSpecifiedSites);
     if ($websiteIdIsForced) {
         $shouldArchivePeriods = true;
     }
     // Test if we should process this website at all
     $elapsedSinceLastArchiving = time() - $lastTimestampWebsiteProcessedDay;
     // Skip this day archive if last archive was older than TTL
     $existingArchiveIsValid = $elapsedSinceLastArchiving < $this->todayArchiveTimeToLive;
     $skipDayArchive = $existingArchiveIsValid;
     // Invalidate old website forces the archiving for this site
     $skipDayArchive = $skipDayArchive && !$websiteIsOldDataInvalidate;
     // Also reprocess when day has ended since last run
     if ($dayHasEndedMustReprocess && !$this->hasBeenProcessedSinceMidnight($idSite, $lastTimestampWebsiteProcessedDay) && !$existingArchiveIsValid) {
         $skipDayArchive = false;
     }
     if ($websiteIdIsForced) {
         $skipDayArchive = false;
     }
     if ($skipDayArchive) {
         $this->log("Skipped website id {$idSite}, already done " . \Piwik\MetricsFormatter::getPrettyTimeFromSeconds($elapsedSinceLastArchiving, true, $isHtml = false) . " ago, " . $timerWebsite->__toString());
         $this->skippedDayArchivesWebsites++;
         $this->skipped++;
         return false;
     }
     // Fake that the request is already done, so that other core:archive commands
     // running do not grab the same website from the queue
     Option::set($this->lastRunKey($idSite, "day"), time());
     // Remove this website from the list of websites to be invalidated
     // since it's now just about to being re-processed, makes sure another running cron archiving process
     // does not archive the same idSite
     if ($websiteIsOldDataInvalidate) {
         $this->setSiteIsArchived($idSite);
     }
     // when some data was purged from this website
     // we make sure we query all previous days/weeks/months
     $processDaysSince = $lastTimestampWebsiteProcessedDay;
     if ($websiteIsOldDataInvalidate || $this->shouldArchiveAllSites) {
         $processDaysSince = false;
     }
     $timer = new Timer();
     $dateLast = $this->getApiDateLastParameter($idSite, "day", $processDaysSince);
     $url = $this->getVisitsRequestUrl($idSite, "day", $dateLast);
     $content = $this->request($url);
     $response = @unserialize($content);
     $visitsToday = $this->getVisitsLastPeriodFromApiResponse($response);
     $visitsLastDays = $this->getVisitsFromApiResponse($response);
     if (empty($content) || !is_array($response) || count($response) == 0) {
         // cancel the succesful run flag
         Option::set($this->lastRunKey($idSite, "day"), 0);
         $this->log("WARNING: Empty or invalid response '{$content}' for website id {$idSite}, " . $timerWebsite->__toString() . ", skipping");
         $this->skipped++;
         return false;
     }
     $this->requests++;
     $this->processed++;
     // If there is no visit today and we don't need to process this website, we can skip remaining archives
     if ($visitsToday == 0 && !$shouldArchivePeriods) {
         $this->log("Skipped website id {$idSite}, no visit today, " . $timerWebsite->__toString());
         $this->skipped++;
         return false;
     }
     if ($visitsLastDays == 0 && !$shouldArchivePeriods && $this->shouldArchiveAllSites) {
         $this->log("Skipped website id {$idSite}, no visits in the last " . $dateLast . " days, " . $timerWebsite->__toString());
         $this->skipped++;
         return false;
     }
     $this->visitsToday += $visitsToday;
     $this->websitesWithVisitsSinceLastRun++;
     $this->archiveVisitsAndSegments($idSite, "day", $lastTimestampWebsiteProcessedDay);
     $this->logArchivedWebsite($idSite, "day", $dateLast, $visitsLastDays, $visitsToday, $timer);
     if (!$shouldArchivePeriods) {
         $this->log("Skipped website id {$idSite} periods processing, already done " . \Piwik\MetricsFormatter::getPrettyTimeFromSeconds($elapsedSinceLastArchiving, true, $isHtml = false) . " ago, " . $timerWebsite->__toString());
         $this->skippedDayArchivesWebsites++;
         $this->skipped++;
         return false;
     }
     $success = true;
     foreach (array('week', 'month', 'year') as $period) {
         $success = $this->archiveVisitsAndSegments($idSite, $period, $lastTimestampWebsiteProcessedPeriods) && $success;
     }
     // Record succesful run of this website's periods archiving
     if ($success) {
         Option::set($this->lastRunKey($idSite, "periods"), time());
     }
     $this->archivedPeriodsArchivesWebsite++;
     $requestsWebsite = $this->requests - $requestsBefore;
     Log::info("Archived website id = {$idSite}, " . $requestsWebsite . " API requests, " . $timerWebsite->__toString() . " [" . $this->websites->getNumProcessedWebsites() . "/" . $this->websites->getNumSites() . " done]");
     return true;
 }