/**
* Record query profile
*
* @param string $query
* @param Timer $timer
*/
protected function recordQueryProfile($query, $timer)
{
if (!isset($this->queriesProfiling[$query])) {
$this->queriesProfiling[$query] = array('sum_time_ms' => 0, 'count' => 0);
}
$time = $timer->getTimeMs(2);
$time += $this->queriesProfiling[$query]['sum_time_ms'];
$count = $this->queriesProfiling[$query]['count'] + 1;
$this->queriesProfiling[$query] = array('sum_time_ms' => $time, 'count' => $count);
}
/**
* Instantiates the Archiver class in each plugin that defines it,
* and triggers Aggregation processing on these plugins.
*/
public function callAggregateAllPlugins($visits, $visitsConverted)
{
Log::debug("PluginsArchiver::%s: Initializing archiving process for all plugins [visits = %s, visits converted = %s]", __FUNCTION__, $visits, $visitsConverted);
$this->archiveProcessor->setNumberOfVisits($visits, $visitsConverted);
$archivers = $this->getPluginArchivers();
foreach ($archivers as $pluginName => $archiverClass) {
// We clean up below all tables created during this function call (and recursive calls)
$latestUsedTableId = Manager::getInstance()->getMostRecentTableId();
/** @var Archiver $archiver */
$archiver = new $archiverClass($this->archiveProcessor);
if (!$archiver->isEnabled()) {
Log::debug("PluginsArchiver::%s: Skipping archiving for plugin '%s'.", __FUNCTION__, $pluginName);
continue;
}
if ($this->shouldProcessReportsForPlugin($pluginName)) {
$timer = new Timer();
if ($this->isSingleSiteDayArchive) {
Log::debug("PluginsArchiver::%s: Archiving day reports for plugin '%s'.", __FUNCTION__, $pluginName);
$archiver->aggregateDayReport();
} else {
Log::debug("PluginsArchiver::%s: Archiving period reports for plugin '%s'.", __FUNCTION__, $pluginName);
$archiver->aggregateMultipleReports();
}
Log::debug("PluginsArchiver::%s: %s while archiving %s reports for plugin '%s'.", __FUNCTION__, $timer->getMemoryLeak(), $this->params->getPeriod()->getLabel(), $pluginName);
} else {
Log::debug("PluginsArchiver::%s: Not archiving reports for plugin '%s'.", __FUNCTION__, $pluginName);
}
Manager::getInstance()->deleteAll($latestUsedTableId);
unset($archiver);
}
}
/**
* Print information about progress.
* @param OutputInterface $output
*/
protected function onVisitProcessed(OutputInterface $output)
{
++$this->processed;
$percent = ceil($this->processed / $this->amountOfVisits * 100);
if ($percent > $this->processedPercent && $percent % $this->percentStep === 0) {
$output->writeln(sprintf('%d%% processed. <comment>%s</comment>', $percent, $this->timer->__toString()));
$this->processedPercent = $percent;
}
}
/**
* @param $idSite
* @param $period
* @param $date
* @param $segmentsCount
* @param $visitsInLastPeriods
* @param $visitsToday
* @param $timer
*/
private function logArchivedWebsite($idSite, $period, $date, $segmentsCount, $visitsInLastPeriods, $visitsToday, Timer $timer)
{
if (strpos($date, 'last') === 0 || strpos($date, 'previous') === 0) {
$humanReadable = $this->formatReadableDateRange($date);
$visitsInLastPeriods = (int) $visitsInLastPeriods . " visits in {$humanReadable} " . $period . "s, ";
$thisPeriod = $period == "day" ? "today" : "this " . $period;
$visitsInLastPeriod = (int) $visitsToday . " visits " . $thisPeriod . ", ";
} else {
$visitsInLastPeriods = (int) $visitsInLastPeriods . " visits in " . $period . "s included in: {$date}, ";
$visitsInLastPeriod = '';
}
$this->logger->info("Archived website id = {$idSite}, period = {$period}, {$segmentsCount} segments, " . $visitsInLastPeriods . $visitsInLastPeriod . $timer->__toString());
}
private function fixDuplicateActionsInTable(OutputInterface $output, $table, $toIdAction, $fromIdActions)
{
$timer = new Timer();
$archivesAffected = $this->duplicateActionRemover->getSitesAndDatesOfRowsUsingDuplicates($table, $fromIdActions);
$this->duplicateActionRemover->fixDuplicateActionsInTable($table, $toIdAction, $fromIdActions);
$output->writeln("\tFixed duplicates in " . Common::prefixTable($table) . ". <comment>" . $timer->__toString() . "</comment>.");
return $archivesAffected;
}
public function getProcessedReport($idSite, $period, $date, $apiModule, $apiAction, $segment = false, $apiParameters = false, $idGoal = false, $language = false, $showTimer = true, $hideMetricsDoc = false, $idSubtable = false, $showRawMetrics = false, $formatMetrics = null)
{
$timer = new Timer();
if (empty($apiParameters)) {
$apiParameters = array();
}
if (!empty($idGoal) && empty($apiParameters['idGoal'])) {
$apiParameters['idGoal'] = $idGoal;
}
// Is this report found in the Metadata available reports?
$reportMetadata = $this->getMetadata($idSite, $apiModule, $apiAction, $apiParameters, $language, $period, $date, $hideMetricsDoc, $showSubtableReports = true);
if (empty($reportMetadata)) {
throw new Exception("Requested report {$apiModule}.{$apiAction} for Website id={$idSite} not found in the list of available reports. \n");
}
$reportMetadata = reset($reportMetadata);
// Generate Api call URL passing custom parameters
$parameters = array_merge($apiParameters, array('method' => $apiModule . '.' . $apiAction, 'idSite' => $idSite, 'period' => $period, 'date' => $date, 'format' => 'original', 'serialize' => '0', 'language' => $language, 'idSubtable' => $idSubtable));
if (!empty($segment)) {
$parameters['segment'] = $segment;
}
if (!empty($reportMetadata['processedMetrics']) && !empty($reportMetadata['metrics']['nb_visits']) && @$reportMetadata['category'] != Piwik::translate('Goals_Ecommerce') && $apiModule !== 'MultiSites') {
$deleteRowsWithNoVisits = empty($reportMetadata['constantRowsCount']) ? '1' : '0';
$parameters['filter_add_columns_when_show_all_columns'] = $deleteRowsWithNoVisits;
}
$url = Url::getQueryStringFromParameters($parameters);
$request = new Request($url);
try {
/** @var DataTable */
$dataTable = $request->process();
} catch (Exception $e) {
throw new Exception("API returned an error: " . $e->getMessage() . " at " . basename($e->getFile()) . ":" . $e->getLine() . "\n");
}
list($newReport, $columns, $rowsMetadata, $totals) = $this->handleTableReport($idSite, $dataTable, $reportMetadata, $showRawMetrics, $formatMetrics);
foreach ($columns as &$name) {
$name = ucfirst($name);
}
$website = new Site($idSite);
$period = Period\Factory::build($period, $date);
$period = $period->getLocalizedLongString();
$return = array('website' => $website->getName(), 'prettyDate' => $period, 'metadata' => $reportMetadata, 'columns' => $columns, 'reportData' => $newReport, 'reportMetadata' => $rowsMetadata, 'reportTotal' => $totals);
if ($showTimer) {
$return['timerMillis'] = $timer->getTimeMs(0);
}
return $return;
}
/**
* Executes the given taks
*
* @param ScheduledTask $task
* @return string
*/
private static function executeTask($task)
{
try {
$timer = new Timer();
call_user_func(array($task->getObjectInstance(), $task->getMethodName()), $task->getMethodParameter());
$message = $timer->__toString();
} catch (Exception $e) {
$message = 'ERROR: ' . $e->getMessage();
}
return $message;
}
/**
* @param $idSite
* @param $period
* @param $date
* @param $visitsInLastPeriods
* @param $visitsToday
* @param $timer
*/
private function logArchivedWebsite($idSite, $period, $date, $visitsInLastPeriods, $visitsToday, Timer $timer)
{
if (substr($date, 0, 4) === 'last') {
$visitsInLastPeriods = (int) $visitsInLastPeriods . " visits in last " . $date . " " . $period . "s, ";
$thisPeriod = $period == "day" ? "today" : "this " . $period;
$visitsInLastPeriod = (int) $visitsToday . " visits " . $thisPeriod . ", ";
} else {
$visitsInLastPeriods = (int) $visitsInLastPeriods . " visits in " . $period . "s included in: {$date}, ";
$visitsInLastPeriod = '';
}
$this->log("Archived website id = {$idSite}, period = {$period}, " . $visitsInLastPeriods . $visitsInLastPeriod . $timer->__toString());
}
/**
* Executes the given task
*
* @param Task $task
* @return string
*/
private function executeTask($task)
{
$this->logger->info("Scheduler: executing task {taskName}...", array('taskName' => $task->getName()));
$this->isRunningTask = true;
$timer = new Timer();
try {
$callable = array($task->getObjectInstance(), $task->getMethodName());
call_user_func($callable, $task->getMethodParameter());
$message = $timer->__toString();
} catch (Exception $e) {
$message = 'ERROR: ' . $e->getMessage();
}
$this->isRunningTask = false;
$this->logger->info("Scheduler: finished. {timeElapsed}", array('timeElapsed' => $timer));
return $message;
}
/**
* Executes the given task
*
* @param Task $task
* @return string
*/
private function executeTask($task)
{
$this->logger->debug('Running task {task}', array('task' => $task->getName()));
try {
$timer = new Timer();
call_user_func(array($task->getObjectInstance(), $task->getMethodName()), $task->getMethodParameter());
$message = $timer->__toString();
} catch (Exception $e) {
$message = 'ERROR: ' . $e->getMessage();
}
return $message;
}
/**
* Will trigger API requests for the specified Website $idsite,
* for the specified $period, for all segments that are pre-processed for this website.
* Requests are triggered using cURL multi handle
*
* @param $idsite int
* @param $period
* @param $lastTimestampWebsiteProcessed
* @param Timer $timerWebsite
* @return bool True on success, false if some request failed
*/
private function archiveVisitsAndSegments($idsite, $period, $lastTimestampWebsiteProcessed, Timer $timerWebsite = null)
{
$timer = new Timer();
$aCurl = array();
$mh = false;
$url = $this->piwikUrl;
$url .= $this->getVisitsRequestUrl($idsite, $period, $lastTimestampWebsiteProcessed);
$url .= self::APPEND_TO_API_REQUEST;
// already processed above for "day"
if ($period != "day") {
$ch = $this->getNewCurlHandle($url);
$this->addCurlHandleToMulti($mh, $ch);
$aCurl[$url] = $ch;
$this->requests++;
}
$urlNoSegment = $url;
foreach ($this->getSegmentsForSite($idsite) as $segment) {
$segmentUrl = $url . '&segment=' . urlencode($segment);
$ch = $this->getNewCurlHandle($segmentUrl);
$this->addCurlHandleToMulti($mh, $ch);
$aCurl[$segmentUrl] = $ch;
$this->requests++;
}
$success = true;
$visitsAllDaysInPeriod = false;
if (!empty($aCurl)) {
$running = null;
do {
usleep(1000);
curl_multi_exec($mh, $running);
} while ($running > 0);
foreach ($aCurl as $url => $ch) {
$content = curl_multi_getcontent($ch);
$successResponse = $this->checkResponse($content, $url);
$success = $successResponse && $success;
if ($url == $urlNoSegment && $successResponse) {
$stats = @unserialize($content);
if (!is_array($stats)) {
$this->logError("Error unserializing the following response from {$url}: " . $content);
}
$visitsAllDaysInPeriod = @array_sum($stats);
}
}
foreach ($aCurl as $ch) {
curl_multi_remove_handle($mh, $ch);
}
curl_multi_close($mh);
}
$this->log("Archived website id = {$idsite}, period = {$period}, " . ($period != "day" ? (int) $visitsAllDaysInPeriod . " visits, " : "") . (!empty($timerWebsite) ? $timerWebsite->__toString() : $timer->__toString()));
return $success;
}
/**
* @param $idSite
* @param $period
* @param $dateLast
* @param $visitsInLastPeriods
* @param $visitsToday
* @param $timer
*/
private function logArchivedWebsite($idSite, $period, $dateLast, $visitsInLastPeriods, $visitsToday, Timer $timer)
{
$thisPeriod = $period == "day" ? "today" : "this " . $period;
$this->log("Archived website id = {$idSite}, period = {$period}, " . (int) $visitsInLastPeriods . " visits in last " . $dateLast . " " . $period . "s, " . (int) $visitsToday . " visits " . $thisPeriod . ", " . $timer->__toString());
}
private function performTimedPurging(OutputInterface $output, $startMessage, $callback)
{
$timer = new Timer();
$output->write($startMessage);
$callback();
$output->writeln("Done. <comment>[" . $timer->__toString() . "]</comment>");
}