/**
  * @param array $idSites
  * @param $datesByMonth
  */
 private function persistInvalidatedArchives(array $idSites, $datesByMonth)
 {
     $yearMonths = array_keys($datesByMonth);
     $yearMonths = array_unique($yearMonths);
     $store = new SitesToReprocessDistributedList();
     $store->add($idSites);
     $archivesToPurge = new ArchivesToPurgeDistributedList();
     $archivesToPurge->add($yearMonths);
 }
Exemple #2
0
 /**
  * @param $idSite
  * @param $lastTimestampWebsiteProcessedDay
  * @param $shouldArchivePeriods
  * @param $timerWebsite
  * @return bool
  */
 protected function processArchiveDays($idSite, $lastTimestampWebsiteProcessedDay, $shouldArchivePeriods, Timer $timerWebsite)
 {
     if (!$this->shouldProcessPeriod("day")) {
         // skip day archiving and proceed to period processing
         return true;
     }
     $timer = new Timer();
     // Fake that the request is already done, so that other core:archive commands
     // running do not grab the same website from the queue
     Option::set($this->lastRunKey($idSite, "day"), time());
     // Remove this website from the list of websites to be invalidated
     // since it's now just about to being re-processed, makes sure another running cron archiving process
     // does not archive the same idSite
     $websiteInvalidatedShouldReprocess = $this->isOldReportInvalidatedForWebsite($idSite);
     if ($websiteInvalidatedShouldReprocess) {
         $store = new SitesToReprocessDistributedList();
         $store->remove($idSite);
     }
     // when some data was purged from this website
     // we make sure we query all previous days/weeks/months
     $processDaysSince = $lastTimestampWebsiteProcessedDay;
     if ($websiteInvalidatedShouldReprocess || $this->shouldArchiveAllSites) {
         $processDaysSince = false;
     }
     $date = $this->getApiDateParameter($idSite, "day", $processDaysSince);
     $url = $this->getVisitsRequestUrl($idSite, "day", $date);
     $this->logArchiveWebsite($idSite, "day", $date);
     $content = $this->request($url);
     $daysResponse = @unserialize($content);
     if (empty($content) || !is_array($daysResponse) || count($daysResponse) == 0) {
         // cancel the succesful run flag
         Option::set($this->lastRunKey($idSite, "day"), 0);
         // cancel marking the site as reprocessed
         if ($websiteInvalidatedShouldReprocess) {
             $store = new SitesToReprocessDistributedList();
             $store->add($idSite);
         }
         $this->logError("Empty or invalid response '{$content}' for website id {$idSite}, " . $timerWebsite->__toString() . ", skipping");
         $this->skipped++;
         return false;
     }
     $visitsToday = $this->getVisitsLastPeriodFromApiResponse($daysResponse);
     $visitsLastDays = $this->getVisitsFromApiResponse($daysResponse);
     $this->requests++;
     $this->processed++;
     // If there is no visit today and we don't need to process this website, we can skip remaining archives
     if (0 == $visitsToday && !$shouldArchivePeriods) {
         $this->logger->info("Skipped website id {$idSite}, no visit today, " . $timerWebsite->__toString());
         $this->skipped++;
         return false;
     }
     if (0 == $visitsLastDays && !$shouldArchivePeriods && $this->shouldArchiveAllSites) {
         $humanReadableDate = $this->formatReadableDateRange($date);
         $this->logger->info("Skipped website id {$idSite}, no visits in the {$humanReadableDate} days, " . $timerWebsite->__toString());
         $this->skipped++;
         return false;
     }
     $this->visitsToday += $visitsToday;
     $this->websitesWithVisitsSinceLastRun++;
     $this->archiveReportsFor($idSite, "day", $this->getApiDateParameter($idSite, "day", $processDaysSince), $archiveSegments = true, $timer);
     return true;
 }
 /**
  * @param array $idSites
  * @param array $yearMonths
  */
 private function markInvalidatedArchivesForReprocessAndPurge(array $idSites, $yearMonths)
 {
     $store = new SitesToReprocessDistributedList();
     $store->add($idSites);
     $archivesToPurge = new ArchivesToPurgeDistributedList();
     $archivesToPurge->add($yearMonths);
 }