/**
* Record query profile
*
* @param string $query
* @param Timer $timer
*/
protected function recordQueryProfile($query, $timer)
{
if (!isset($this->queriesProfiling[$query])) {
$this->queriesProfiling[$query] = array('sum_time_ms' => 0, 'count' => 0);
}
$time = $timer->getTimeMs(2);
$time += $this->queriesProfiling[$query]['sum_time_ms'];
$count = $this->queriesProfiling[$query]['count'] + 1;
$this->queriesProfiling[$query] = array('sum_time_ms' => $time, 'count' => $count);
}
/**
* Main function, runs archiving on all websites with new activity
*/
public function run()
{
$timer = new Timer();
$this->logSection("START");
$this->logger->info("Starting Piwik reports archiving...");
do {
$idSite = $this->websites->getNextSiteId();
if (null === $idSite) {
break;
}
flush();
$requestsBefore = $this->requests;
if ($idSite <= 0) {
continue;
}
$skipWebsiteForced = in_array($idSite, $this->shouldSkipSpecifiedSites);
if ($skipWebsiteForced) {
$this->logger->info("Skipped website id {$idSite}, found in --skip-idsites ");
$this->skipped++;
continue;
}
$shouldCheckIfArchivingIsNeeded = !$this->shouldArchiveSpecifiedSites && !$this->shouldArchiveAllSites;
$hasWebsiteDayFinishedSinceLastRun = in_array($idSite, $this->websiteDayHasFinishedSinceLastRun);
$isOldReportInvalidatedForWebsite = $this->isOldReportInvalidatedForWebsite($idSite);
if ($shouldCheckIfArchivingIsNeeded) {
// if not specific sites and not all websites should be archived, we check whether we actually have
// to process the archives for this website (only if there were visits since midnight)
if (!$hasWebsiteDayFinishedSinceLastRun && !$isOldReportInvalidatedForWebsite) {
if ($this->isWebsiteUsingTheTracker($idSite) && !$this->hadWebsiteTrafficSinceMidnightInTimezone($idSite)) {
$this->logger->info("Will skip website {$idSite} as archiving is not needed");
$this->skipped++;
continue;
}
} elseif ($hasWebsiteDayFinishedSinceLastRun) {
$this->logger->info("Day has finished for website {$idSite} since last run");
} elseif ($isOldReportInvalidatedForWebsite) {
$this->logger->info("Old report was invalidated for website {$idSite}");
}
}
/**
* This event is triggered before the cron archiving process starts archiving data for a single
* site.
*
* @param int $idSite The ID of the site we're archiving data for.
*/
Piwik::postEvent('CronArchive.archiveSingleSite.start', array($idSite));
$completed = $this->archiveSingleSite($idSite, $requestsBefore);
/**
* This event is triggered immediately after the cron archiving process starts archiving data for a single
* site.
*
* @param int $idSite The ID of the site we're archiving data for.
*/
Piwik::postEvent('CronArchive.archiveSingleSite.finish', array($idSite, $completed));
} while (!empty($idSite));
$this->logger->info("Done archiving!");
$this->logSection("SUMMARY");
$this->logger->info("Total visits for today across archived websites: " . $this->visitsToday);
$totalWebsites = count($this->allWebsites);
$this->skipped = $totalWebsites - $this->websitesWithVisitsSinceLastRun;
$this->logger->info("Archived today's reports for {$this->websitesWithVisitsSinceLastRun} websites");
$this->logger->info("Archived week/month/year for {$this->archivedPeriodsArchivesWebsite} websites");
$this->logger->info("Skipped {$this->skipped} websites: no new visit since the last script execution");
$this->logger->info("Skipped {$this->skippedDayArchivesWebsites} websites day archiving: existing daily reports are less than {$this->todayArchiveTimeToLive} seconds old");
$this->logger->info("Skipped {$this->skippedPeriodsArchivesWebsite} websites week/month/year archiving: existing periods reports are less than {$this->processPeriodsMaximumEverySeconds} seconds old");
$this->logger->info("Total API requests: {$this->requests}");
//DONE: done/total, visits, wtoday, wperiods, reqs, time, errors[count]: first eg.
$percent = $this->websites->getNumSites() == 0 ? "" : " " . round($this->processed * 100 / $this->websites->getNumSites(), 0) . "%";
$this->logger->info("done: " . $this->processed . "/" . $this->websites->getNumSites() . "" . $percent . ", " . $this->visitsToday . " vtoday, {$this->websitesWithVisitsSinceLastRun} wtoday, {$this->archivedPeriodsArchivesWebsite} wperiods, " . $this->requests . " req, " . round($timer->getTimeMs()) . " ms, " . (empty($this->errors) ? self::NO_ERROR : count($this->errors) . " errors."));
$this->logger->info($timer->__toString());
}
/**
* Main function, runs archiving on all websites with new activity
*/
public function run()
{
$timer = new Timer();
$this->logSection("START");
$this->log("Starting Piwik reports archiving...");
do {
$idSite = $this->websites->getNextSiteId();
if (null === $idSite) {
break;
}
flush();
$requestsBefore = $this->requests;
if ($idSite <= 0) {
continue;
}
$skipWebsiteForced = in_array($idSite, $this->shouldSkipSpecifiedSites);
if ($skipWebsiteForced) {
$this->log("Skipped website id {$idSite}, found in --skip-idsites ");
$this->skipped++;
continue;
}
/**
* This event is triggered before the cron archiving process starts archiving data for a single
* site.
*
* @param int $idSite The ID of the site we're archiving data for.
*/
Piwik::postEvent('CronArchive.archiveSingleSite.start', array($idSite));
$completed = $this->archiveSingleSite($idSite, $requestsBefore);
/**
* This event is triggered immediately after the cron archiving process starts archiving data for a single
* site.
*
* @param int $idSite The ID of the site we're archiving data for.
*/
Piwik::postEvent('CronArchive.archiveSingleSite.finish', array($idSite, $completed));
} while (!empty($idSite));
$this->log("Done archiving!");
$this->logSection("SUMMARY");
$this->log("Total visits for today across archived websites: " . $this->visitsToday);
$totalWebsites = count($this->allWebsites);
$this->skipped = $totalWebsites - $this->websitesWithVisitsSinceLastRun;
$this->log("Archived today's reports for {$this->websitesWithVisitsSinceLastRun} websites");
$this->log("Archived week/month/year for {$this->archivedPeriodsArchivesWebsite} websites");
$this->log("Skipped {$this->skipped} websites: no new visit since the last script execution");
$this->log("Skipped {$this->skippedDayArchivesWebsites} websites day archiving: existing daily reports are less than {$this->todayArchiveTimeToLive} seconds old");
$this->log("Skipped {$this->skippedPeriodsArchivesWebsite} websites week/month/year archiving: existing periods reports are less than {$this->processPeriodsMaximumEverySeconds} seconds old");
$this->log("Total API requests: {$this->requests}");
//DONE: done/total, visits, wtoday, wperiods, reqs, time, errors[count]: first eg.
$percent = $this->websites->getNumSites() == 0 ? "" : " " . round($this->processed * 100 / $this->websites->getNumSites(), 0) . "%";
$this->log("done: " . $this->processed . "/" . $this->websites->getNumSites() . "" . $percent . ", " . $this->visitsToday . " vtoday, {$this->websitesWithVisitsSinceLastRun} wtoday, {$this->archivedPeriodsArchivesWebsite} wperiods, " . $this->requests . " req, " . round($timer->getTimeMs()) . " ms, " . (empty($this->errors) ? self::NO_ERROR : count($this->errors) . " errors."));
$this->log($timer->__toString());
}
public function getProcessedReport($idSite, $period, $date, $apiModule, $apiAction, $segment = false, $apiParameters = false, $idGoal = false, $language = false, $showTimer = true, $hideMetricsDoc = false, $idSubtable = false, $showRawMetrics = false, $formatMetrics = null)
{
$timer = new Timer();
if (empty($apiParameters)) {
$apiParameters = array();
}
if (!empty($idGoal) && empty($apiParameters['idGoal'])) {
$apiParameters['idGoal'] = $idGoal;
}
// Is this report found in the Metadata available reports?
$reportMetadata = $this->getMetadata($idSite, $apiModule, $apiAction, $apiParameters, $language, $period, $date, $hideMetricsDoc, $showSubtableReports = true);
if (empty($reportMetadata)) {
throw new Exception("Requested report {$apiModule}.{$apiAction} for Website id={$idSite} not found in the list of available reports. \n");
}
$reportMetadata = reset($reportMetadata);
// Generate Api call URL passing custom parameters
$parameters = array_merge($apiParameters, array('method' => $apiModule . '.' . $apiAction, 'idSite' => $idSite, 'period' => $period, 'date' => $date, 'format' => 'original', 'serialize' => '0', 'language' => $language, 'idSubtable' => $idSubtable));
if (!empty($segment)) {
$parameters['segment'] = $segment;
}
if (!empty($reportMetadata['processedMetrics']) && !empty($reportMetadata['metrics']['nb_visits']) && @$reportMetadata['category'] != Piwik::translate('Goals_Ecommerce') && $apiModule !== 'MultiSites') {
$deleteRowsWithNoVisits = empty($reportMetadata['constantRowsCount']) ? '1' : '0';
$parameters['filter_add_columns_when_show_all_columns'] = $deleteRowsWithNoVisits;
}
$url = Url::getQueryStringFromParameters($parameters);
$request = new Request($url);
try {
/** @var DataTable */
$dataTable = $request->process();
} catch (Exception $e) {
throw new Exception("API returned an error: " . $e->getMessage() . " at " . basename($e->getFile()) . ":" . $e->getLine() . "\n");
}
list($newReport, $columns, $rowsMetadata, $totals) = $this->handleTableReport($idSite, $dataTable, $reportMetadata, $showRawMetrics, $formatMetrics);
foreach ($columns as &$name) {
$name = ucfirst($name);
}
$website = new Site($idSite);
$period = Period\Factory::build($period, $date);
$period = $period->getLocalizedLongString();
$return = array('website' => $website->getName(), 'prettyDate' => $period, 'metadata' => $reportMetadata, 'columns' => $columns, 'reportData' => $newReport, 'reportMetadata' => $rowsMetadata, 'reportTotal' => $totals);
if ($showTimer) {
$return['timerMillis'] = $timer->getTimeMs(0);
}
return $return;
}
/**
* Main function, runs archiving on all websites with new activity
*/
public function run()
{
$websitesWithVisitsSinceLastRun = $skippedPeriodsArchivesWebsite = $skippedDayArchivesWebsites = $skipped = $processed = $archivedPeriodsArchivesWebsite = 0;
$timer = new Timer();
$this->logSection("START");
foreach ($this->websites as $idsite) {
flush();
$requestsBefore = $this->requests;
if ($idsite <= 0) {
continue;
}
$skipWebsiteForced = in_array($idsite, $this->shouldSkipSpecifiedSites);
if ($skipWebsiteForced) {
$this->log("Skipped website id {$idsite}, found in --skip-idsites ");
$skipped++;
continue;
}
$timerWebsite = new Timer();
$lastTimestampWebsiteProcessedPeriods = $lastTimestampWebsiteProcessedDay = false;
if ($this->archiveAndRespectTTL) {
$lastTimestampWebsiteProcessedPeriods = Option::get($this->lastRunKey($idsite, "periods"));
$lastTimestampWebsiteProcessedDay = Option::get($this->lastRunKey($idsite, "day"));
}
// For period other than days, we only re-process the reports at most
// 1) every $processPeriodsMaximumEverySeconds
$secondsSinceLastExecution = time() - $lastTimestampWebsiteProcessedPeriods;
// if timeout is more than 10 min, we account for a 5 min processing time, and allow trigger 1 min earlier
if ($this->processPeriodsMaximumEverySeconds > 10 * 60) {
$secondsSinceLastExecution += 5 * 60;
}
$shouldArchivePeriods = $secondsSinceLastExecution > $this->processPeriodsMaximumEverySeconds;
if (empty($lastTimestampWebsiteProcessedPeriods)) {
// 2) OR always if script never executed for this website before
$shouldArchivePeriods = true;
}
// (*) If the website is archived because it is a new day in its timezone
// We make sure all periods are archived, even if there is 0 visit today
$dayHasEndedMustReprocess = in_array($idsite, $this->websiteDayHasFinishedSinceLastRun);
if ($dayHasEndedMustReprocess) {
$shouldArchivePeriods = true;
}
// (*) If there was some old reports invalidated for this website
// we make sure all these old reports are triggered at least once
$websiteIsOldDataInvalidate = in_array($idsite, $this->idSitesInvalidatedOldReports);
if ($websiteIsOldDataInvalidate) {
$shouldArchivePeriods = true;
}
$websiteIdIsForced = in_array($idsite, $this->shouldArchiveSpecifiedSites);
if ($websiteIdIsForced) {
$shouldArchivePeriods = true;
}
// Test if we should process this website at all
$elapsedSinceLastArchiving = time() - $lastTimestampWebsiteProcessedDay;
// Skip this day archive if last archive was older than TTL
$existingArchiveIsValid = $elapsedSinceLastArchiving < $this->todayArchiveTimeToLive;
$skipDayArchive = $existingArchiveIsValid;
// Invalidate old website forces the archiving for this site
$skipDayArchive = $skipDayArchive && !$websiteIsOldDataInvalidate;
// Also reprocess when day has ended since last run
if ($dayHasEndedMustReprocess && !$existingArchiveIsValid) {
$skipDayArchive = false;
}
if ($websiteIdIsForced) {
$skipDayArchive = false;
}
if ($skipDayArchive) {
$this->log("Skipped website id {$idsite}, already processed today's report in recent run, " . \Piwik\MetricsFormatter::getPrettyTimeFromSeconds($elapsedSinceLastArchiving, true, $isHtml = false) . " ago, " . $timerWebsite->__toString());
$skippedDayArchivesWebsites++;
$skipped++;
continue;
}
// Fake that the request is already done, so that other archive.php
// running do not grab the same website from the queue
Option::set($this->lastRunKey($idsite, "day"), time());
// when some data was purged from this website
// we make sure we query all previous days/weeks/months
$processDaysSince = $lastTimestampWebsiteProcessedDay;
if ($websiteIsOldDataInvalidate || $this->shouldArchiveAllSites) {
$processDaysSince = false;
}
$url = $this->getVisitsRequestUrl($idsite, "day", $processDaysSince);
$content = $this->request($url);
$response = @unserialize($content);
if (empty($content) || !is_array($response) || count($response) == 0) {
// cancel the succesful run flag
Option::set($this->lastRunKey($idsite, "day"), 0);
$this->log("WARNING: Empty or invalid response '{$content}' for website id {$idsite}, " . $timerWebsite->__toString() . ", skipping");
$skipped++;
continue;
}
$visitsToday = end($response);
if (empty($visitsToday)) {
$visitsToday = 0;
}
$this->requests++;
$processed++;
// If there is no visit today and we don't need to process this website, we can skip remaining archives
if ($visitsToday == 0 && !$shouldArchivePeriods) {
$this->log("Skipped website id {$idsite}, no visit today, " . $timerWebsite->__toString());
$skipped++;
continue;
}
$visitsAllDays = array_sum($response);
if ($visitsAllDays == 0 && !$shouldArchivePeriods && $this->shouldArchiveAllSites) {
$this->log("Skipped website id {$idsite}, no visits in the last " . count($response) . " days, " . $timerWebsite->__toString());
$skipped++;
continue;
}
$this->visits += $visitsToday;
$websitesWithVisitsSinceLastRun++;
$this->archiveVisitsAndSegments($idsite, "day", $lastTimestampWebsiteProcessedDay, $timerWebsite);
if (!$shouldArchivePeriods) {
$this->log("Skipped website id {$idsite}, already processed period reports in recent run, " . \Piwik\MetricsFormatter::getPrettyTimeFromSeconds($elapsedSinceLastArchiving, true, $isHtml = false) . " ago, " . $timerWebsite->__toString());
$skippedDayArchivesWebsites++;
$skipped++;
continue;
}
$success = true;
foreach (array('week', 'month', 'year') as $period) {
$success = $this->archiveVisitsAndSegments($idsite, $period, $lastTimestampWebsiteProcessedPeriods) && $success;
}
// Record succesful run of this website's periods archiving
if ($success) {
Option::set($this->lastRunKey($idsite, "periods"), time());
// Remove this website from the list of websites to be invalidated
// since it's now just been re-processing the reports, job is done!
if ($websiteIsOldDataInvalidate) {
$this->setSiteIsArchived($idsite);
}
}
$archivedPeriodsArchivesWebsite++;
$requestsWebsite = $this->requests - $requestsBefore;
$debug = $this->shouldArchiveAllSites ? ", last days = {$visitsAllDays} visits" : "";
Log::info("Archived website id = {$idsite}, today = {$visitsToday} visits" . $debug . ", {$requestsWebsite} API requests, " . $timerWebsite->__toString() . " [" . ($websitesWithVisitsSinceLastRun + $skipped) . "/" . count($this->websites) . " done]");
}
$this->log("Starting Piwik reports archiving...");
$this->log("Done archiving!");
$this->logSection("SUMMARY");
$this->log("Total daily visits archived: " . $this->visits);
$totalWebsites = count($this->allWebsites);
$skipped = $totalWebsites - $websitesWithVisitsSinceLastRun;
$this->log("Archived today's reports for {$websitesWithVisitsSinceLastRun} websites");
$this->log("Archived week/month/year for {$archivedPeriodsArchivesWebsite} websites");
$this->log("Skipped {$skipped} websites: no new visit since the last script execution");
$this->log("Skipped {$skippedDayArchivesWebsites} websites day archiving: existing daily reports are less than {$this->todayArchiveTimeToLive} seconds old");
$this->log("Skipped {$skippedPeriodsArchivesWebsite} websites week/month/year archiving: existing periods reports are less than {$this->processPeriodsMaximumEverySeconds} seconds old");
$this->log("Total API requests: {$this->requests}");
//DONE: done/total, visits, wtoday, wperiods, reqs, time, errors[count]: first eg.
$percent = count($this->websites) == 0 ? "" : " " . round($processed * 100 / count($this->websites), 0) . "%";
$this->log("done: " . $processed . "/" . count($this->websites) . "" . $percent . ", " . $this->visits . " v, {$websitesWithVisitsSinceLastRun} wtoday, {$archivedPeriodsArchivesWebsite} wperiods, " . $this->requests . " req, " . round($timer->getTimeMs()) . " ms, " . (empty($this->errors) ? "no error" : count($this->errors) . " errors. eg. '" . reset($this->errors) . "'"));
$this->log($timer->__toString());
}