/**
* Record query profile
*
* @param string $query
* @param Piwik_Timer $timer
*/
protected function recordQueryProfile( $query, $timer )
{
if(!isset($this->queriesProfiling[$query])) $this->queriesProfiling[$query] = array('sum_time_ms' => 0, 'count' => 0);
$time = $timer->getTimeMs(2);
$time += $this->queriesProfiling[$query]['sum_time_ms'];
$count = $this->queriesProfiling[$query]['count'] + 1;
$this->queriesProfiling[$query] = array('sum_time_ms' => $time, 'count' => $count);
}
/**
* Will execute $className->$methodName($parametersValues)
* If any error is detected (wrong number of parameters, method not found, class not found, etc.)
* it will throw an exception
*
* It also logs the API calls, with the parameters values, the returned value, the performance, etc.
* You can enable logging in config/global.ini.php (log_api_call)
*
* @param string The class name (eg. Piwik_Referers_API)
* @param string The method name
* @param array The parameters pairs (name=>value)
*
* @throws Piwik_Access_NoAccessException
*/
public function call($className, $methodName, $parametersRequest)
{
$returnedValue = null;
try {
$this->registerClass($className);
// instanciate the object
$object = call_user_func(array($className, "getInstance"));
// check method exists
$this->checkMethodExists($className, $methodName);
// get the list of parameters required by the method
$parameterNamesDefaultValues = $this->getParametersList($className, $methodName);
// load parameters in the right order, etc.
$finalParameters = $this->getRequestParametersArray($parameterNamesDefaultValues, $parametersRequest);
// all parameters to the function call must be non null
$this->checkParametersAreNotNull($className, $methodName, $finalParameters);
// start the timer
$timer = new Piwik_Timer();
// call the method
$returnedValue = call_user_func_array(array($object, $methodName), $finalParameters);
// log the API Call
Zend_Registry::get('logger_api_call')->log($className, $methodName, $parameterNamesDefaultValues, $finalParameters, $timer->getTimeMs(), $returnedValue);
} catch (Piwik_Access_NoAccessException $e) {
throw $e;
}
return $returnedValue;
}
public function getProcessedReport($idSite, $period, $date, $apiModule, $apiAction, $segment = false, $apiParameters = false, $idGoal = false, $language = false, $showTimer = true)
{
$timer = new Piwik_Timer();
if ($apiParameters === false) {
$apiParameters = array();
}
if (!empty($idGoal) && empty($apiParameters['idGoal'])) {
$apiParameters['idGoal'] = $idGoal;
}
// Is this report found in the Metadata available reports?
$reportMetadata = $this->getMetadata($idSite, $apiModule, $apiAction, $apiParameters, $language, $period, $date);
if (empty($reportMetadata)) {
throw new Exception("Requested report {$apiModule}.{$apiAction} for Website id={$idSite} not found in the list of available reports. \n");
}
$reportMetadata = reset($reportMetadata);
// Generate Api call URL passing custom parameters
$parameters = array_merge($apiParameters, array('method' => $apiModule . '.' . $apiAction, 'idSite' => $idSite, 'period' => $period, 'date' => $date, 'format' => 'original', 'serialize' => '0', 'language' => $language));
if (!empty($segment)) {
$parameters['segment'] = $segment;
}
$url = Piwik_Url::getQueryStringFromParameters($parameters);
$request = new Piwik_API_Request($url);
try {
/** @var Piwik_DataTable */
$dataTable = $request->process();
} catch (Exception $e) {
throw new Exception("API returned an error: " . $e->getMessage() . "\n");
}
list($newReport, $columns, $rowsMetadata) = $this->handleTableReport($idSite, $dataTable, $reportMetadata, isset($reportMetadata['dimension']));
foreach ($columns as $columnId => &$name) {
$name = ucfirst($name);
}
$website = new Piwik_Site($idSite);
// $segment = new Piwik_Segment($segment, $idSite);
if (Piwik_Archive::isMultiplePeriod($date, $period)) {
$period = new Piwik_Period_Range($period, $date);
} else {
if ($period == 'range') {
$period = new Piwik_Period_Range($period, $date);
} else {
$period = Piwik_Period::factory($period, Piwik_Date::factory($date));
}
}
$period = $period->getLocalizedLongString();
$return = array('website' => $website->getName(), 'prettyDate' => $period, 'metadata' => $reportMetadata, 'columns' => $columns, 'reportData' => $newReport, 'reportMetadata' => $rowsMetadata);
if ($showTimer) {
$return['timerMillis'] = $timer->getTimeMs(0);
}
return $return;
}
/**
* Will execute $className->$methodName($parametersValues)
* If any error is detected (wrong number of parameters, method not found, class not found, etc.)
* it will throw an exception
*
* It also logs the API calls, with the parameters values, the returned value, the performance, etc.
* You can enable logging in config/global.ini.php (log_api_call)
*
* @param string $className The class name (eg. Piwik_Referers_API)
* @param string $methodName The method name
* @param array $parametersRequest The parameters pairs (name=>value)
*
* @return mixed|null
* @throws Exception|Piwik_Access_NoAccessException
*/
public function call($className, $methodName, $parametersRequest)
{
$returnedValue = null;
// Temporarily sets the Request array to this API call context
$saveGET = $_GET;
foreach ($parametersRequest as $param => $value) {
$_GET[$param] = $value;
}
try {
$this->registerClass($className);
// instanciate the object
$object = call_user_func(array($className, "getInstance"));
// check method exists
$this->checkMethodExists($className, $methodName);
// get the list of parameters required by the method
$parameterNamesDefaultValues = $this->getParametersList($className, $methodName);
// load parameters in the right order, etc.
$finalParameters = $this->getRequestParametersArray($parameterNamesDefaultValues, $parametersRequest);
// start the timer
$timer = new Piwik_Timer();
// call the method
$returnedValue = call_user_func_array(array($object, $methodName), $finalParameters);
// allow plugins to manipulate the value
if (substr($className, 0, 6) == 'Piwik_' && substr($className, -4) == '_API') {
$pluginName = substr($className, 6, -4);
Piwik_PostEvent('API.Proxy.processReturnValue', $returnedValue, array('className' => $className, 'module' => $pluginName, 'action' => $methodName, 'parameters' => &$parametersRequest));
}
// Restore the request
$_GET = $saveGET;
// log the API Call
try {
Zend_Registry::get('logger_api_call')->logEvent($className, $methodName, $parameterNamesDefaultValues, $finalParameters, $timer->getTimeMs(), $returnedValue);
} catch (Exception $e) {
// logger can fail (eg. Tracker request)
}
} catch (Exception $e) {
$_GET = $saveGET;
throw $e;
}
return $returnedValue;
}
/**
* Main function, runs archiving on all websites with new activity
*/
public function run()
{
$websitesWithVisitsSinceLastRun = $skippedPeriodsArchivesWebsite = $skippedDayArchivesWebsites = $skipped = $processed = $archivedPeriodsArchivesWebsite = 0;
$timer = new Piwik_Timer();
$this->logSection("START");
$this->log("Starting Piwik reports archiving...");
$this->websites = array_unique($this->websites);
foreach ($this->websites as $idsite) {
flush();
$requestsBefore = $this->requests;
if ($idsite <= 0) {
continue;
}
$timerWebsite = new Piwik_Timer();
$lastTimestampWebsiteProcessedPeriods = $lastTimestampWebsiteProcessedDay = false;
if (!$this->shouldResetState) {
$lastTimestampWebsiteProcessedPeriods = Piwik_GetOption($this->lastRunKey($idsite, "periods"));
$lastTimestampWebsiteProcessedDay = Piwik_GetOption($this->lastRunKey($idsite, "day"));
}
// For period other than days, we only re-process the reports at most
// 1) every $processPeriodsMaximumEverySeconds
$secondsSinceLastExecution = time() - $lastTimestampWebsiteProcessedPeriods;
// if timeout is more than 10 min, we account for a 5 min processing time, and allow trigger 1 min earlier
if ($this->processPeriodsMaximumEverySeconds > 10 * 60) {
$secondsSinceLastExecution += 5 * 60;
}
$shouldArchivePeriods = $secondsSinceLastExecution > $this->processPeriodsMaximumEverySeconds;
if (empty($lastTimestampWebsiteProcessedPeriods)) {
// 2) OR always if script never executed for this website before
$shouldArchivePeriods = true;
}
// (*) If the website is archived because it is a new day in its timezone
// We make sure all periods are archived, even if there is 0 visit today
$dayHasEndedMustReprocess = in_array($idsite, $this->websiteDayHasFinishedSinceLastRun);
if ($dayHasEndedMustReprocess) {
$shouldArchivePeriods = true;
}
// (*) If there was some old reports invalidated for this website
// we make sure all these old reports are triggered at least once
$websiteIsOldDataInvalidate = in_array($idsite, $this->idSitesInvalidatedOldReports);
if ($websiteIsOldDataInvalidate) {
$shouldArchivePeriods = true;
}
// Test if we should process this website at all
$elapsedSinceLastArchiving = time() - $lastTimestampWebsiteProcessedDay;
if (!$websiteIsOldDataInvalidate && !$dayHasEndedMustReprocess && $elapsedSinceLastArchiving < $this->todayArchiveTimeToLive) {
$this->log("Skipped website id {$idsite}, already processed today's report in recent run, " . Piwik::getPrettyTimeFromSeconds($elapsedSinceLastArchiving, true, $isHtml = false) . " ago, " . $timerWebsite->__toString());
$skippedDayArchivesWebsites++;
$skipped++;
continue;
}
// Fake that the request is already done, so that other archive.php
// running do not grab the same website from the queue
Piwik_SetOption($this->lastRunKey($idsite, "day"), time());
$url = $this->getVisitsRequestUrl($idsite, "day", $websiteIsOldDataInvalidate || $this->shouldArchiveAllWebsites ? false : $lastTimestampWebsiteProcessedDay);
$content = $this->request($url);
$response = @unserialize($content);
if (empty($content) || !is_array($response) || count($response) == 0) {
// cancel the succesful run flag
Piwik_SetOption($this->lastRunKey($idsite, "day"), 0);
$this->log("WARNING: Empty or invalid response '{$content}' for website id {$idsite}, " . $timerWebsite->__toString() . ", skipping");
$skipped++;
continue;
}
$visitsToday = end($response);
$this->requests++;
$processed++;
// If there is no visit today and we don't need to process this website, we can skip remaining archives
if ($visitsToday <= 0 && !$shouldArchivePeriods) {
$this->log("Skipped website id {$idsite}, no visit today, " . $timerWebsite->__toString());
$skipped++;
continue;
}
$visitsAllDays = array_sum($response);
if ($visitsAllDays == 0 && !$shouldArchivePeriods && $this->shouldArchiveAllWebsites) {
$this->log("Skipped website id {$idsite}, no visits in the last " . count($response) . " days, " . $timerWebsite->__toString());
$skipped++;
continue;
}
$this->visits += $visitsToday;
$websitesWithVisitsSinceLastRun++;
$this->archiveVisitsAndSegments($idsite, "day", $lastTimestampWebsiteProcessedDay, $timerWebsite);
if ($shouldArchivePeriods) {
$success = true;
foreach (array('week', 'month', 'year') as $period) {
$success = $this->archiveVisitsAndSegments($idsite, $period, $lastTimestampWebsiteProcessedPeriods) && $success;
}
// Record succesful run of this website's periods archiving
if ($success) {
Piwik_SetOption($this->lastRunKey($idsite, "periods"), time());
// Remove this website from the list of websites to be invalidated
// since it's now just been re-processing the reports, job is done!
if ($websiteIsOldDataInvalidate) {
$websiteIdsInvalidated = Piwik_CoreAdminHome_API::getWebsiteIdsToInvalidate();
if (count($websiteIdsInvalidated)) {
$found = array_search($idsite, $websiteIdsInvalidated);
if ($found !== false) {
unset($websiteIdsInvalidated[$found]);
// $this->log("Websites left to invalidate: " . implode(", ", $websiteIdsInvalidated));
Piwik_SetOption(Piwik_CoreAdminHome_API::OPTION_INVALIDATED_IDSITES, serialize($websiteIdsInvalidated));
}
}
}
}
}
$archivedPeriodsArchivesWebsite++;
$requestsWebsite = $this->requests - $requestsBefore;
$debug = $this->shouldArchiveAllWebsites ? ", last days = {$visitsAllDays} visits" : "";
Piwik::log("Archived website id = {$idsite}, today = {$visitsToday} visits" . $debug . ", {$requestsWebsite} API requests, " . $timerWebsite->__toString() . " [" . ($websitesWithVisitsSinceLastRun + $skipped) . "/" . count($this->websites) . " done]");
}
$this->log("Done archiving!");
$this->logSection("SUMMARY");
$this->log("Total daily visits archived: " . $this->visits);
$totalWebsites = count($this->allWebsites);
$skipped = $totalWebsites - $websitesWithVisitsSinceLastRun;
$this->log("Archived today's reports for {$websitesWithVisitsSinceLastRun} websites");
$this->log("Archived week/month/year for {$archivedPeriodsArchivesWebsite} websites. ");
$this->log("Skipped {$skipped} websites: no new visit since the last script execution");
$this->log("Skipped {$skippedDayArchivesWebsites} websites day archiving: existing daily reports are less than {$this->todayArchiveTimeToLive} seconds old");
$this->log("Skipped {$skippedPeriodsArchivesWebsite} websites week/month/year archiving: existing periods reports are less than {$this->processPeriodsMaximumEverySeconds} seconds old");
$this->log("Total API requests: {$this->requests}");
//DONE: done/total, visits, wtoday, wperiods, reqs, time, errors[count]: first eg.
$percent = count($this->websites) == 0 ? "" : " " . round($processed * 100 / count($this->websites), 0) . "%";
$otherInParallel = $skippedDayArchivesWebsites;
$this->log("done: " . $processed . "/" . count($this->websites) . "" . $percent . ", " . $this->visits . " v, {$websitesWithVisitsSinceLastRun} wtoday, {$archivedPeriodsArchivesWebsite} wperiods, " . $this->requests . " req, " . round($timer->getTimeMs()) . " ms, " . (empty($this->errors) ? "no error" : count($this->errors) . " errors. eg. '" . reset($this->errors) . "'"));
$this->log($timer->__toString());
$this->logSection("SCHEDULED TASKS");
$this->log("Starting Scheduled tasks... ");
$tasksOutput = $this->request("?module=API&method=CoreAdminHome.runScheduledTasks&format=csv&convertToUnicode=0&token_auth=" . $this->token_auth);
if ($tasksOutput == "No data available") {
$tasksOutput = " No task to run";
}
$this->log($tasksOutput);
$this->log("done");
}
/**
* Will execute $className->$methodName($parametersValues)
* If any error is detected (wrong number of parameters, method not found, class not found, etc.)
* it will throw an exception
*
* It also logs the API calls, with the parameters values, the returned value, the performance, etc.
* You can enable logging in config/global.ini.php (log_api_call)
*
* @param string The class name (eg. Piwik_Referers_API)
* @param string The method name
* @param array The parameters pairs (name=>value)
*
* @throws Piwik_Access_NoAccessException
*/
public function call($className, $methodName, $parametersRequest )
{
$returnedValue = null;
// Temporarily sets the Request array to this API call context
$saveGET = $_GET;
foreach($parametersRequest as $param => $value) {
$_GET[$param] = $value;
}
try {
$this->registerClass($className);
// instanciate the object
$object = call_user_func(array($className, "getInstance"));
// check method exists
$this->checkMethodExists($className, $methodName);
// get the list of parameters required by the method
$parameterNamesDefaultValues = $this->getParametersList($className, $methodName);
// load parameters in the right order, etc.
$finalParameters = $this->getRequestParametersArray( $parameterNamesDefaultValues, $parametersRequest );
// start the timer
$timer = new Piwik_Timer();
// call the method
$returnedValue = call_user_func_array(array($object, $methodName), $finalParameters);
// Restore the request
$_GET = $saveGET;
// log the API Call
try {
Zend_Registry::get('logger_api_call')->logEvent(
$className,
$methodName,
$parameterNamesDefaultValues,
$finalParameters,
$timer->getTimeMs(),
$returnedValue
);
} catch (Exception $e) {
// logger can fail (eg. Tracker request)
}
} catch (Exception $e) {
$_GET = $saveGET;
throw $e;
}
return $returnedValue;
}
/**
* Method always called when making an API request.
* It checks several things before actually calling the real method on the given module.
*
* It also logs the API calls, with the parameters values, the returned value, the performance, etc.
* You can enable logging in config/global.ini.php (log_api_call)
*
* @param string The method name
* @param array The parameters
*
* @throws Piwik_Access_NoAccessException
*/
public function __call($methodName, $parameterValues)
{
$returnedValue = null;
try {
$this->registerClass(self::$classCalled);
$className = $this->getClassNameFromModule(self::$classCalled);
// instanciate the object
$object = call_user_func(array($className, "getInstance"));
// check method exists
$this->checkMethodExists($className, $methodName);
// first check number of parameters do match
$this->checkNumberOfParametersMatch($className, $methodName, $parameterValues);
// start the timer
$timer = new Piwik_Timer();
// call the method
$returnedValue = call_user_func_array(array($object, $methodName), $parameterValues);
// log the API Call
$parameterNamesDefaultValues = $this->getParametersList($className, $methodName);
Zend_Registry::get('logger_api_call')->log(self::$classCalled, $methodName, $parameterNamesDefaultValues, $parameterValues, $timer->getTimeMs(), $returnedValue);
} catch (Piwik_Access_NoAccessException $e) {
throw $e;
}
self::$classCalled = null;
return $returnedValue;
}
