/**
* @param array $idSites
* @param $datesByMonth
*/
private function persistInvalidatedArchives(array $idSites, $datesByMonth)
{
$yearMonths = array_keys($datesByMonth);
$yearMonths = array_unique($yearMonths);
$store = new SitesToReprocessDistributedList();
$store->add($idSites);
$archivesToPurge = new ArchivesToPurgeDistributedList();
$archivesToPurge->add($yearMonths);
}
/**
* @param $idSite
* @param $lastTimestampWebsiteProcessedDay
* @param $shouldArchivePeriods
* @param $timerWebsite
* @return bool
*/
protected function processArchiveDays($idSite, $lastTimestampWebsiteProcessedDay, $shouldArchivePeriods, Timer $timerWebsite)
{
if (!$this->shouldProcessPeriod("day")) {
// skip day archiving and proceed to period processing
return true;
}
$timer = new Timer();
// Fake that the request is already done, so that other core:archive commands
// running do not grab the same website from the queue
Option::set($this->lastRunKey($idSite, "day"), time());
// Remove this website from the list of websites to be invalidated
// since it's now just about to being re-processed, makes sure another running cron archiving process
// does not archive the same idSite
$websiteInvalidatedShouldReprocess = $this->isOldReportInvalidatedForWebsite($idSite);
if ($websiteInvalidatedShouldReprocess) {
$store = new SitesToReprocessDistributedList();
$store->remove($idSite);
}
// when some data was purged from this website
// we make sure we query all previous days/weeks/months
$processDaysSince = $lastTimestampWebsiteProcessedDay;
if ($websiteInvalidatedShouldReprocess || $this->shouldArchiveAllSites) {
$processDaysSince = false;
}
$date = $this->getApiDateParameter($idSite, "day", $processDaysSince);
$url = $this->getVisitsRequestUrl($idSite, "day", $date);
$this->logArchiveWebsite($idSite, "day", $date);
$content = $this->request($url);
$daysResponse = @unserialize($content);
if (empty($content) || !is_array($daysResponse) || count($daysResponse) == 0) {
// cancel the succesful run flag
Option::set($this->lastRunKey($idSite, "day"), 0);
// cancel marking the site as reprocessed
if ($websiteInvalidatedShouldReprocess) {
$store = new SitesToReprocessDistributedList();
$store->add($idSite);
}
$this->logError("Empty or invalid response '{$content}' for website id {$idSite}, " . $timerWebsite->__toString() . ", skipping");
$this->skipped++;
return false;
}
$visitsToday = $this->getVisitsLastPeriodFromApiResponse($daysResponse);
$visitsLastDays = $this->getVisitsFromApiResponse($daysResponse);
$this->requests++;
$this->processed++;
// If there is no visit today and we don't need to process this website, we can skip remaining archives
if (0 == $visitsToday && !$shouldArchivePeriods) {
$this->logger->info("Skipped website id {$idSite}, no visit today, " . $timerWebsite->__toString());
$this->skipped++;
return false;
}
if (0 == $visitsLastDays && !$shouldArchivePeriods && $this->shouldArchiveAllSites) {
$humanReadableDate = $this->formatReadableDateRange($date);
$this->logger->info("Skipped website id {$idSite}, no visits in the {$humanReadableDate} days, " . $timerWebsite->__toString());
$this->skipped++;
return false;
}
$this->visitsToday += $visitsToday;
$this->websitesWithVisitsSinceLastRun++;
$this->archiveReportsFor($idSite, "day", $this->getApiDateParameter($idSite, "day", $processDaysSince), $archiveSegments = true, $timer);
return true;
}
/**
* @param array $idSites
* @param array $yearMonths
*/
private function markInvalidatedArchivesForReprocessAndPurge(array $idSites, $yearMonths)
{
$store = new SitesToReprocessDistributedList();
$store->add($idSites);
$archivesToPurge = new ArchivesToPurgeDistributedList();
$archivesToPurge->add($yearMonths);
}