Example #1
0
 /**
  * Record query profile
  *
  * @param string $query
  * @param Timer $timer
  */
 protected function recordQueryProfile($query, $timer)
 {
     if (!isset($this->queriesProfiling[$query])) {
         $this->queriesProfiling[$query] = array('sum_time_ms' => 0, 'count' => 0);
     }
     $time = $timer->getTimeMs(2);
     $time += $this->queriesProfiling[$query]['sum_time_ms'];
     $count = $this->queriesProfiling[$query]['count'] + 1;
     $this->queriesProfiling[$query] = array('sum_time_ms' => $time, 'count' => $count);
 }
Example #2
0
 /**
  * Main function, runs archiving on all websites with new activity
  */
 public function run()
 {
     $timer = new Timer();
     $this->logSection("START");
     $this->logger->info("Starting Piwik reports archiving...");
     do {
         $idSite = $this->websites->getNextSiteId();
         if (null === $idSite) {
             break;
         }
         flush();
         $requestsBefore = $this->requests;
         if ($idSite <= 0) {
             continue;
         }
         $skipWebsiteForced = in_array($idSite, $this->shouldSkipSpecifiedSites);
         if ($skipWebsiteForced) {
             $this->logger->info("Skipped website id {$idSite}, found in --skip-idsites ");
             $this->skipped++;
             continue;
         }
         $shouldCheckIfArchivingIsNeeded = !$this->shouldArchiveSpecifiedSites && !$this->shouldArchiveAllSites;
         $hasWebsiteDayFinishedSinceLastRun = in_array($idSite, $this->websiteDayHasFinishedSinceLastRun);
         $isOldReportInvalidatedForWebsite = $this->isOldReportInvalidatedForWebsite($idSite);
         if ($shouldCheckIfArchivingIsNeeded) {
             // if not specific sites and not all websites should be archived, we check whether we actually have
             // to process the archives for this website (only if there were visits since midnight)
             if (!$hasWebsiteDayFinishedSinceLastRun && !$isOldReportInvalidatedForWebsite) {
                 if ($this->isWebsiteUsingTheTracker($idSite) && !$this->hadWebsiteTrafficSinceMidnightInTimezone($idSite)) {
                     $this->logger->info("Will skip website {$idSite} as archiving is not needed");
                     $this->skipped++;
                     continue;
                 }
             } elseif ($hasWebsiteDayFinishedSinceLastRun) {
                 $this->logger->info("Day has finished for website {$idSite} since last run");
             } elseif ($isOldReportInvalidatedForWebsite) {
                 $this->logger->info("Old report was invalidated for website {$idSite}");
             }
         }
         /**
          * This event is triggered before the cron archiving process starts archiving data for a single
          * site.
          *
          * @param int $idSite The ID of the site we're archiving data for.
          */
         Piwik::postEvent('CronArchive.archiveSingleSite.start', array($idSite));
         $completed = $this->archiveSingleSite($idSite, $requestsBefore);
         /**
          * This event is triggered immediately after the cron archiving process starts archiving data for a single
          * site.
          *
          * @param int $idSite The ID of the site we're archiving data for.
          */
         Piwik::postEvent('CronArchive.archiveSingleSite.finish', array($idSite, $completed));
     } while (!empty($idSite));
     $this->logger->info("Done archiving!");
     $this->logSection("SUMMARY");
     $this->logger->info("Total visits for today across archived websites: " . $this->visitsToday);
     $totalWebsites = count($this->allWebsites);
     $this->skipped = $totalWebsites - $this->websitesWithVisitsSinceLastRun;
     $this->logger->info("Archived today's reports for {$this->websitesWithVisitsSinceLastRun} websites");
     $this->logger->info("Archived week/month/year for {$this->archivedPeriodsArchivesWebsite} websites");
     $this->logger->info("Skipped {$this->skipped} websites: no new visit since the last script execution");
     $this->logger->info("Skipped {$this->skippedDayArchivesWebsites} websites day archiving: existing daily reports are less than {$this->todayArchiveTimeToLive} seconds old");
     $this->logger->info("Skipped {$this->skippedPeriodsArchivesWebsite} websites week/month/year archiving: existing periods reports are less than {$this->processPeriodsMaximumEverySeconds} seconds old");
     $this->logger->info("Total API requests: {$this->requests}");
     //DONE: done/total, visits, wtoday, wperiods, reqs, time, errors[count]: first eg.
     $percent = $this->websites->getNumSites() == 0 ? "" : " " . round($this->processed * 100 / $this->websites->getNumSites(), 0) . "%";
     $this->logger->info("done: " . $this->processed . "/" . $this->websites->getNumSites() . "" . $percent . ", " . $this->visitsToday . " vtoday, {$this->websitesWithVisitsSinceLastRun} wtoday, {$this->archivedPeriodsArchivesWebsite} wperiods, " . $this->requests . " req, " . round($timer->getTimeMs()) . " ms, " . (empty($this->errors) ? self::NO_ERROR : count($this->errors) . " errors."));
     $this->logger->info($timer->__toString());
 }
Example #3
0
 /**
  * Main function, runs archiving on all websites with new activity
  */
 public function run()
 {
     $timer = new Timer();
     $this->logSection("START");
     $this->log("Starting Piwik reports archiving...");
     do {
         $idSite = $this->websites->getNextSiteId();
         if (null === $idSite) {
             break;
         }
         flush();
         $requestsBefore = $this->requests;
         if ($idSite <= 0) {
             continue;
         }
         $skipWebsiteForced = in_array($idSite, $this->shouldSkipSpecifiedSites);
         if ($skipWebsiteForced) {
             $this->log("Skipped website id {$idSite}, found in --skip-idsites ");
             $this->skipped++;
             continue;
         }
         /**
          * This event is triggered before the cron archiving process starts archiving data for a single
          * site.
          *
          * @param int $idSite The ID of the site we're archiving data for.
          */
         Piwik::postEvent('CronArchive.archiveSingleSite.start', array($idSite));
         $completed = $this->archiveSingleSite($idSite, $requestsBefore);
         /**
          * This event is triggered immediately after the cron archiving process starts archiving data for a single
          * site.
          *
          * @param int $idSite The ID of the site we're archiving data for.
          */
         Piwik::postEvent('CronArchive.archiveSingleSite.finish', array($idSite, $completed));
     } while (!empty($idSite));
     $this->log("Done archiving!");
     $this->logSection("SUMMARY");
     $this->log("Total visits for today across archived websites: " . $this->visitsToday);
     $totalWebsites = count($this->allWebsites);
     $this->skipped = $totalWebsites - $this->websitesWithVisitsSinceLastRun;
     $this->log("Archived today's reports for {$this->websitesWithVisitsSinceLastRun} websites");
     $this->log("Archived week/month/year for {$this->archivedPeriodsArchivesWebsite} websites");
     $this->log("Skipped {$this->skipped} websites: no new visit since the last script execution");
     $this->log("Skipped {$this->skippedDayArchivesWebsites} websites day archiving: existing daily reports are less than {$this->todayArchiveTimeToLive} seconds old");
     $this->log("Skipped {$this->skippedPeriodsArchivesWebsite} websites week/month/year archiving: existing periods reports are less than {$this->processPeriodsMaximumEverySeconds} seconds old");
     $this->log("Total API requests: {$this->requests}");
     //DONE: done/total, visits, wtoday, wperiods, reqs, time, errors[count]: first eg.
     $percent = $this->websites->getNumSites() == 0 ? "" : " " . round($this->processed * 100 / $this->websites->getNumSites(), 0) . "%";
     $this->log("done: " . $this->processed . "/" . $this->websites->getNumSites() . "" . $percent . ", " . $this->visitsToday . " vtoday, {$this->websitesWithVisitsSinceLastRun} wtoday, {$this->archivedPeriodsArchivesWebsite} wperiods, " . $this->requests . " req, " . round($timer->getTimeMs()) . " ms, " . (empty($this->errors) ? self::NO_ERROR : count($this->errors) . " errors."));
     $this->log($timer->__toString());
 }
 public function getProcessedReport($idSite, $period, $date, $apiModule, $apiAction, $segment = false, $apiParameters = false, $idGoal = false, $language = false, $showTimer = true, $hideMetricsDoc = false, $idSubtable = false, $showRawMetrics = false, $formatMetrics = null)
 {
     $timer = new Timer();
     if (empty($apiParameters)) {
         $apiParameters = array();
     }
     if (!empty($idGoal) && empty($apiParameters['idGoal'])) {
         $apiParameters['idGoal'] = $idGoal;
     }
     // Is this report found in the Metadata available reports?
     $reportMetadata = $this->getMetadata($idSite, $apiModule, $apiAction, $apiParameters, $language, $period, $date, $hideMetricsDoc, $showSubtableReports = true);
     if (empty($reportMetadata)) {
         throw new Exception("Requested report {$apiModule}.{$apiAction} for Website id={$idSite} not found in the list of available reports. \n");
     }
     $reportMetadata = reset($reportMetadata);
     // Generate Api call URL passing custom parameters
     $parameters = array_merge($apiParameters, array('method' => $apiModule . '.' . $apiAction, 'idSite' => $idSite, 'period' => $period, 'date' => $date, 'format' => 'original', 'serialize' => '0', 'language' => $language, 'idSubtable' => $idSubtable));
     if (!empty($segment)) {
         $parameters['segment'] = $segment;
     }
     if (!empty($reportMetadata['processedMetrics']) && !empty($reportMetadata['metrics']['nb_visits']) && @$reportMetadata['category'] != Piwik::translate('Goals_Ecommerce') && $apiModule !== 'MultiSites') {
         $deleteRowsWithNoVisits = empty($reportMetadata['constantRowsCount']) ? '1' : '0';
         $parameters['filter_add_columns_when_show_all_columns'] = $deleteRowsWithNoVisits;
     }
     $url = Url::getQueryStringFromParameters($parameters);
     $request = new Request($url);
     try {
         /** @var DataTable */
         $dataTable = $request->process();
     } catch (Exception $e) {
         throw new Exception("API returned an error: " . $e->getMessage() . " at " . basename($e->getFile()) . ":" . $e->getLine() . "\n");
     }
     list($newReport, $columns, $rowsMetadata, $totals) = $this->handleTableReport($idSite, $dataTable, $reportMetadata, $showRawMetrics, $formatMetrics);
     foreach ($columns as &$name) {
         $name = ucfirst($name);
     }
     $website = new Site($idSite);
     $period = Period\Factory::build($period, $date);
     $period = $period->getLocalizedLongString();
     $return = array('website' => $website->getName(), 'prettyDate' => $period, 'metadata' => $reportMetadata, 'columns' => $columns, 'reportData' => $newReport, 'reportMetadata' => $rowsMetadata, 'reportTotal' => $totals);
     if ($showTimer) {
         $return['timerMillis'] = $timer->getTimeMs(0);
     }
     return $return;
 }
 /**
  * Main function, runs archiving on all websites with new activity
  */
 public function run()
 {
     $websitesWithVisitsSinceLastRun = $skippedPeriodsArchivesWebsite = $skippedDayArchivesWebsites = $skipped = $processed = $archivedPeriodsArchivesWebsite = 0;
     $timer = new Timer();
     $this->logSection("START");
     foreach ($this->websites as $idsite) {
         flush();
         $requestsBefore = $this->requests;
         if ($idsite <= 0) {
             continue;
         }
         $skipWebsiteForced = in_array($idsite, $this->shouldSkipSpecifiedSites);
         if ($skipWebsiteForced) {
             $this->log("Skipped website id {$idsite}, found in --skip-idsites ");
             $skipped++;
             continue;
         }
         $timerWebsite = new Timer();
         $lastTimestampWebsiteProcessedPeriods = $lastTimestampWebsiteProcessedDay = false;
         if ($this->archiveAndRespectTTL) {
             $lastTimestampWebsiteProcessedPeriods = Option::get($this->lastRunKey($idsite, "periods"));
             $lastTimestampWebsiteProcessedDay = Option::get($this->lastRunKey($idsite, "day"));
         }
         // For period other than days, we only re-process the reports at most
         // 1) every $processPeriodsMaximumEverySeconds
         $secondsSinceLastExecution = time() - $lastTimestampWebsiteProcessedPeriods;
         // if timeout is more than 10 min, we account for a 5 min processing time, and allow trigger 1 min earlier
         if ($this->processPeriodsMaximumEverySeconds > 10 * 60) {
             $secondsSinceLastExecution += 5 * 60;
         }
         $shouldArchivePeriods = $secondsSinceLastExecution > $this->processPeriodsMaximumEverySeconds;
         if (empty($lastTimestampWebsiteProcessedPeriods)) {
             // 2) OR always if script never executed for this website before
             $shouldArchivePeriods = true;
         }
         // (*) If the website is archived because it is a new day in its timezone
         // We make sure all periods are archived, even if there is 0 visit today
         $dayHasEndedMustReprocess = in_array($idsite, $this->websiteDayHasFinishedSinceLastRun);
         if ($dayHasEndedMustReprocess) {
             $shouldArchivePeriods = true;
         }
         // (*) If there was some old reports invalidated for this website
         // we make sure all these old reports are triggered at least once
         $websiteIsOldDataInvalidate = in_array($idsite, $this->idSitesInvalidatedOldReports);
         if ($websiteIsOldDataInvalidate) {
             $shouldArchivePeriods = true;
         }
         $websiteIdIsForced = in_array($idsite, $this->shouldArchiveSpecifiedSites);
         if ($websiteIdIsForced) {
             $shouldArchivePeriods = true;
         }
         // Test if we should process this website at all
         $elapsedSinceLastArchiving = time() - $lastTimestampWebsiteProcessedDay;
         // Skip this day archive if last archive was older than TTL
         $existingArchiveIsValid = $elapsedSinceLastArchiving < $this->todayArchiveTimeToLive;
         $skipDayArchive = $existingArchiveIsValid;
         // Invalidate old website forces the archiving for this site
         $skipDayArchive = $skipDayArchive && !$websiteIsOldDataInvalidate;
         // Also reprocess when day has ended since last run
         if ($dayHasEndedMustReprocess && !$existingArchiveIsValid) {
             $skipDayArchive = false;
         }
         if ($websiteIdIsForced) {
             $skipDayArchive = false;
         }
         if ($skipDayArchive) {
             $this->log("Skipped website id {$idsite}, already processed today's report in recent run, " . \Piwik\MetricsFormatter::getPrettyTimeFromSeconds($elapsedSinceLastArchiving, true, $isHtml = false) . " ago, " . $timerWebsite->__toString());
             $skippedDayArchivesWebsites++;
             $skipped++;
             continue;
         }
         // Fake that the request is already done, so that other archive.php
         // running do not grab the same website from the queue
         Option::set($this->lastRunKey($idsite, "day"), time());
         // when some data was purged from this website
         // we make sure we query all previous days/weeks/months
         $processDaysSince = $lastTimestampWebsiteProcessedDay;
         if ($websiteIsOldDataInvalidate || $this->shouldArchiveAllSites) {
             $processDaysSince = false;
         }
         $url = $this->getVisitsRequestUrl($idsite, "day", $processDaysSince);
         $content = $this->request($url);
         $response = @unserialize($content);
         if (empty($content) || !is_array($response) || count($response) == 0) {
             // cancel the succesful run flag
             Option::set($this->lastRunKey($idsite, "day"), 0);
             $this->log("WARNING: Empty or invalid response '{$content}' for website id {$idsite}, " . $timerWebsite->__toString() . ", skipping");
             $skipped++;
             continue;
         }
         $visitsToday = end($response);
         if (empty($visitsToday)) {
             $visitsToday = 0;
         }
         $this->requests++;
         $processed++;
         // If there is no visit today and we don't need to process this website, we can skip remaining archives
         if ($visitsToday == 0 && !$shouldArchivePeriods) {
             $this->log("Skipped website id {$idsite}, no visit today, " . $timerWebsite->__toString());
             $skipped++;
             continue;
         }
         $visitsAllDays = array_sum($response);
         if ($visitsAllDays == 0 && !$shouldArchivePeriods && $this->shouldArchiveAllSites) {
             $this->log("Skipped website id {$idsite}, no visits in the last " . count($response) . " days, " . $timerWebsite->__toString());
             $skipped++;
             continue;
         }
         $this->visits += $visitsToday;
         $websitesWithVisitsSinceLastRun++;
         $this->archiveVisitsAndSegments($idsite, "day", $lastTimestampWebsiteProcessedDay, $timerWebsite);
         if (!$shouldArchivePeriods) {
             $this->log("Skipped website id {$idsite}, already processed period reports in recent run, " . \Piwik\MetricsFormatter::getPrettyTimeFromSeconds($elapsedSinceLastArchiving, true, $isHtml = false) . " ago, " . $timerWebsite->__toString());
             $skippedDayArchivesWebsites++;
             $skipped++;
             continue;
         }
         $success = true;
         foreach (array('week', 'month', 'year') as $period) {
             $success = $this->archiveVisitsAndSegments($idsite, $period, $lastTimestampWebsiteProcessedPeriods) && $success;
         }
         // Record succesful run of this website's periods archiving
         if ($success) {
             Option::set($this->lastRunKey($idsite, "periods"), time());
             // Remove this website from the list of websites to be invalidated
             // since it's now just been re-processing the reports, job is done!
             if ($websiteIsOldDataInvalidate) {
                 $this->setSiteIsArchived($idsite);
             }
         }
         $archivedPeriodsArchivesWebsite++;
         $requestsWebsite = $this->requests - $requestsBefore;
         $debug = $this->shouldArchiveAllSites ? ", last days = {$visitsAllDays} visits" : "";
         Log::info("Archived website id = {$idsite}, today = {$visitsToday} visits" . $debug . ", {$requestsWebsite} API requests, " . $timerWebsite->__toString() . " [" . ($websitesWithVisitsSinceLastRun + $skipped) . "/" . count($this->websites) . " done]");
     }
     $this->log("Starting Piwik reports archiving...");
     $this->log("Done archiving!");
     $this->logSection("SUMMARY");
     $this->log("Total daily visits archived: " . $this->visits);
     $totalWebsites = count($this->allWebsites);
     $skipped = $totalWebsites - $websitesWithVisitsSinceLastRun;
     $this->log("Archived today's reports for {$websitesWithVisitsSinceLastRun} websites");
     $this->log("Archived week/month/year for {$archivedPeriodsArchivesWebsite} websites");
     $this->log("Skipped {$skipped} websites: no new visit since the last script execution");
     $this->log("Skipped {$skippedDayArchivesWebsites} websites day archiving: existing daily reports are less than {$this->todayArchiveTimeToLive} seconds old");
     $this->log("Skipped {$skippedPeriodsArchivesWebsite} websites week/month/year archiving: existing periods reports are less than {$this->processPeriodsMaximumEverySeconds} seconds old");
     $this->log("Total API requests: {$this->requests}");
     //DONE: done/total, visits, wtoday, wperiods, reqs, time, errors[count]: first eg.
     $percent = count($this->websites) == 0 ? "" : " " . round($processed * 100 / count($this->websites), 0) . "%";
     $this->log("done: " . $processed . "/" . count($this->websites) . "" . $percent . ", " . $this->visits . " v, {$websitesWithVisitsSinceLastRun} wtoday, {$archivedPeriodsArchivesWebsite} wperiods, " . $this->requests . " req, " . round($timer->getTimeMs()) . " ms, " . (empty($this->errors) ? "no error" : count($this->errors) . " errors. eg. '" . reset($this->errors) . "'"));
     $this->log($timer->__toString());
 }