/** * @param array $idSites * @param $datesByMonth */ private function persistInvalidatedArchives(array $idSites, $datesByMonth) { $yearMonths = array_keys($datesByMonth); $yearMonths = array_unique($yearMonths); $store = new SitesToReprocessDistributedList(); $store->add($idSites); $archivesToPurge = new ArchivesToPurgeDistributedList(); $archivesToPurge->add($yearMonths); }
/** * @param $idSite * @param $lastTimestampWebsiteProcessedDay * @param $shouldArchivePeriods * @param $timerWebsite * @return bool */ protected function processArchiveDays($idSite, $lastTimestampWebsiteProcessedDay, $shouldArchivePeriods, Timer $timerWebsite) { if (!$this->shouldProcessPeriod("day")) { // skip day archiving and proceed to period processing return true; } $timer = new Timer(); // Fake that the request is already done, so that other core:archive commands // running do not grab the same website from the queue Option::set($this->lastRunKey($idSite, "day"), time()); // Remove this website from the list of websites to be invalidated // since it's now just about to being re-processed, makes sure another running cron archiving process // does not archive the same idSite $websiteInvalidatedShouldReprocess = $this->isOldReportInvalidatedForWebsite($idSite); if ($websiteInvalidatedShouldReprocess) { $store = new SitesToReprocessDistributedList(); $store->remove($idSite); } // when some data was purged from this website // we make sure we query all previous days/weeks/months $processDaysSince = $lastTimestampWebsiteProcessedDay; if ($websiteInvalidatedShouldReprocess || $this->shouldArchiveAllSites) { $processDaysSince = false; } $date = $this->getApiDateParameter($idSite, "day", $processDaysSince); $url = $this->getVisitsRequestUrl($idSite, "day", $date); $this->logArchiveWebsite($idSite, "day", $date); $content = $this->request($url); $daysResponse = @unserialize($content); if (empty($content) || !is_array($daysResponse) || count($daysResponse) == 0) { // cancel the succesful run flag Option::set($this->lastRunKey($idSite, "day"), 0); // cancel marking the site as reprocessed if ($websiteInvalidatedShouldReprocess) { $store = new SitesToReprocessDistributedList(); $store->add($idSite); } $this->logError("Empty or invalid response '{$content}' for website id {$idSite}, " . $timerWebsite->__toString() . ", skipping"); $this->skipped++; return false; } $visitsToday = $this->getVisitsLastPeriodFromApiResponse($daysResponse); $visitsLastDays = $this->getVisitsFromApiResponse($daysResponse); $this->requests++; $this->processed++; // If there is no visit today and we don't need to process this website, we can skip remaining archives if (0 == $visitsToday && !$shouldArchivePeriods) { $this->logger->info("Skipped website id {$idSite}, no visit today, " . $timerWebsite->__toString()); $this->skipped++; return false; } if (0 == $visitsLastDays && !$shouldArchivePeriods && $this->shouldArchiveAllSites) { $humanReadableDate = $this->formatReadableDateRange($date); $this->logger->info("Skipped website id {$idSite}, no visits in the {$humanReadableDate} days, " . $timerWebsite->__toString()); $this->skipped++; return false; } $this->visitsToday += $visitsToday; $this->websitesWithVisitsSinceLastRun++; $this->archiveReportsFor($idSite, "day", $this->getApiDateParameter($idSite, "day", $processDaysSince), $archiveSegments = true, $timer); return true; }
/** * @param array $idSites * @param array $yearMonths */ private function markInvalidatedArchivesForReprocessAndPurge(array $idSites, $yearMonths) { $store = new SitesToReprocessDistributedList(); $store->add($idSites); $archivesToPurge = new ArchivesToPurgeDistributedList(); $archivesToPurge->add($yearMonths); }