/** * Record query profile * * @param string $query * @param Piwik_Timer $timer */ protected function recordQueryProfile( $query, $timer ) { if(!isset($this->queriesProfiling[$query])) $this->queriesProfiling[$query] = array('sum_time_ms' => 0, 'count' => 0); $time = $timer->getTimeMs(2); $time += $this->queriesProfiling[$query]['sum_time_ms']; $count = $this->queriesProfiling[$query]['count'] + 1; $this->queriesProfiling[$query] = array('sum_time_ms' => $time, 'count' => $count); }
static public function runTasks() { // Gets the array where rescheduled timetables are stored $option = Piwik_GetOption(self::TIMETABLE_OPTION_STRING); $timetable = self::getTimetableFromOption($option); if($timetable === false) { return; } if(isset($GLOBALS['PIWIK_TRACKER_DEBUG_FORCE_SCHEDULED_TASKS']) && $GLOBALS['PIWIK_TRACKER_DEBUG_FORCE_SCHEDULED_TASKS']) { $timetable = array(); } // Collects tasks Piwik_PostEvent(self::GET_TASKS_EVENT, $tasks); $return = array(); // Loop through each task foreach ($tasks as $task) { $scheduledTime = $task->getScheduledTime(); $className = $task->getClassName(); $methodName = $task->getMethodName(); $fullyQualifiedMethodName = get_class($className) . '.' . $methodName; /* * Task has to be executed if : * - it is the first time, ie. rescheduledTime is not set * - that task has already been executed and the current system time is greater than the * rescheduled time. */ if ( !isset($timetable[$fullyQualifiedMethodName]) || (isset($timetable[$fullyQualifiedMethodName]) && time() >= $timetable[$fullyQualifiedMethodName]) ) { // Updates the rescheduled time $timetable[$fullyQualifiedMethodName] = $scheduledTime->getRescheduledTime(); Piwik_SetOption(self::TIMETABLE_OPTION_STRING, serialize($timetable)); // Run the task try { $timer = new Piwik_Timer; call_user_func ( array($className,$methodName) ); $message = $timer->__toString(); } catch(Exception $e) { $message = 'ERROR: '.$e->getMessage(); } $return[] = array('task' => $fullyQualifiedMethodName, 'output' => $message); } } return $return; }
/** * runTasks collects tasks defined within piwik plugins, runs them if they are scheduled and reschedules * the tasks that have been executed. * * @return array */ public static function runTasks() { // Gets the array where rescheduled timetables are stored $option = Piwik_GetOption(self::TIMETABLE_OPTION_STRING); $timetable = self::getTimetableFromOption($option); if ($timetable === false) { return; } $forceScheduledTasks = false; if (isset($GLOBALS['PIWIK_TRACKER_DEBUG_FORCE_SCHEDULED_TASKS']) && $GLOBALS['PIWIK_TRACKER_DEBUG_FORCE_SCHEDULED_TASKS'] || DEBUG_FORCE_SCHEDULED_TASKS) { $forceScheduledTasks = true; $timetable = array(); } // Collects tasks Piwik_PostEvent(self::GET_TASKS_EVENT, $tasks); $return = array(); // for every priority level, starting with the highest and concluding with the lowest for ($priority = Piwik_ScheduledTask::HIGHEST_PRIORITY; $priority <= Piwik_ScheduledTask::LOWEST_PRIORITY; ++$priority) { // Loop through each task foreach ($tasks as $task) { // if the task does not have the current priority level, don't execute it yet if ($task->getPriority() != $priority) { continue; } $scheduledTime = $task->getScheduledTime(); $className = $task->getClassName(); $methodName = $task->getMethodName(); $fullyQualifiedMethodName = get_class($className) . '.' . $methodName; /* * Task has to be executed if : * - it is the first time, ie. rescheduledTime is not set * - that task has already been executed and the current system time is greater than the * rescheduled time. */ if (!isset($timetable[$fullyQualifiedMethodName]) || isset($timetable[$fullyQualifiedMethodName]) && time() >= $timetable[$fullyQualifiedMethodName] || $forceScheduledTasks) { // Updates the rescheduled time $timetable[$fullyQualifiedMethodName] = $scheduledTime->getRescheduledTime(); Piwik_SetOption(self::TIMETABLE_OPTION_STRING, serialize($timetable)); self::$running = true; // Run the task try { $timer = new Piwik_Timer(); call_user_func(array($className, $methodName)); $message = $timer->__toString(); } catch (Exception $e) { $message = 'ERROR: ' . $e->getMessage(); } self::$running = false; $return[] = array('task' => $fullyQualifiedMethodName, 'output' => $message); } } } return $return; }
/** * Will execute $className->$methodName($parametersValues) * If any error is detected (wrong number of parameters, method not found, class not found, etc.) * it will throw an exception * * It also logs the API calls, with the parameters values, the returned value, the performance, etc. * You can enable logging in config/global.ini.php (log_api_call) * * @param string The class name (eg. Piwik_Referers_API) * @param string The method name * @param array The parameters pairs (name=>value) * * @throws Piwik_Access_NoAccessException */ public function call($className, $methodName, $parametersRequest) { $returnedValue = null; try { $this->registerClass($className); // instanciate the object $object = call_user_func(array($className, "getInstance")); // check method exists $this->checkMethodExists($className, $methodName); // get the list of parameters required by the method $parameterNamesDefaultValues = $this->getParametersList($className, $methodName); // load parameters in the right order, etc. $finalParameters = $this->getRequestParametersArray($parameterNamesDefaultValues, $parametersRequest); // all parameters to the function call must be non null $this->checkParametersAreNotNull($className, $methodName, $finalParameters); // start the timer $timer = new Piwik_Timer(); // call the method $returnedValue = call_user_func_array(array($object, $methodName), $finalParameters); // log the API Call Zend_Registry::get('logger_api_call')->log($className, $methodName, $parameterNamesDefaultValues, $finalParameters, $timer->getTimeMs(), $returnedValue); } catch (Piwik_Access_NoAccessException $e) { throw $e; } return $returnedValue; }
//Piwik_PluginsManager::getInstance()->unloadPlugins(); // we have to unload the Provider plugin otherwise it tries to lookup the IP for ahostname, and there is no dns server here if (Piwik_PluginsManager::getInstance()->isPluginActivated('Provider')) { Piwik_PluginsManager::getInstance()->unloadPlugin('Provider'); } // we set the DO NOT load plugins so that the Tracker generator doesn't load the plugins we've just disabled. // if for some reasons you want to load the plugins, comment this line, and disable the plugin Provider in the plugins interface Piwik_PluginsManager::getInstance()->doNotLoadPlugins(); $generator = new Piwik_Tracker_Generator(); $generator->setMaximumUrlDepth(3); //$generator->disableProfiler(); $generator->setIdSite($idSite); $nbActionsTotal = 0; //$generator->emptyAllLogTables(); $generator->init(); $t = new Piwik_Timer(); $startTime = time() - ($daysToCompute - 1) * 86400; while ($startTime <= time()) { $visitors = rand($minVisitors, $maxVisitors); $actions = $nbActions; $generator->setTimestampToUse($startTime); $nbActionsTotalThisDay = $generator->generate($visitors, $actions); $actionsPerVisit = round($nbActionsTotalThisDay / $visitors); print "Generated {$visitors} unique visitors and {$actionsPerVisit} actions per visit for the " . date("Y-m-d", $startTime) . "<br>\n"; $startTime += 86400; $nbActionsTotal += $nbActionsTotalThisDay; sleep(1); } echo "<br>Total actions: {$nbActionsTotal}"; echo "<br>Total requests per sec: " . round($nbActionsTotal / $t->getTime(), 0); echo "<br>" . $t;
public function getProcessedReport($idSite, $period, $date, $apiModule, $apiAction, $segment = false, $apiParameters = false, $idGoal = false, $language = false, $showTimer = true) { $timer = new Piwik_Timer(); if ($apiParameters === false) { $apiParameters = array(); } if (!empty($idGoal) && empty($apiParameters['idGoal'])) { $apiParameters['idGoal'] = $idGoal; } // Is this report found in the Metadata available reports? $reportMetadata = $this->getMetadata($idSite, $apiModule, $apiAction, $apiParameters, $language, $period, $date); if (empty($reportMetadata)) { throw new Exception("Requested report {$apiModule}.{$apiAction} for Website id={$idSite} not found in the list of available reports. \n"); } $reportMetadata = reset($reportMetadata); // Generate Api call URL passing custom parameters $parameters = array_merge($apiParameters, array('method' => $apiModule . '.' . $apiAction, 'idSite' => $idSite, 'period' => $period, 'date' => $date, 'format' => 'original', 'serialize' => '0', 'language' => $language)); if (!empty($segment)) { $parameters['segment'] = $segment; } $url = Piwik_Url::getQueryStringFromParameters($parameters); $request = new Piwik_API_Request($url); try { /** @var Piwik_DataTable */ $dataTable = $request->process(); } catch (Exception $e) { throw new Exception("API returned an error: " . $e->getMessage() . "\n"); } list($newReport, $columns, $rowsMetadata) = $this->handleTableReport($idSite, $dataTable, $reportMetadata, isset($reportMetadata['dimension'])); foreach ($columns as $columnId => &$name) { $name = ucfirst($name); } $website = new Piwik_Site($idSite); // $segment = new Piwik_Segment($segment, $idSite); if (Piwik_Archive::isMultiplePeriod($date, $period)) { $period = new Piwik_Period_Range($period, $date); } else { if ($period == 'range') { $period = new Piwik_Period_Range($period, $date); } else { $period = Piwik_Period::factory($period, Piwik_Date::factory($date)); } } $period = $period->getLocalizedLongString(); $return = array('website' => $website->getName(), 'prettyDate' => $period, 'metadata' => $reportMetadata, 'columns' => $columns, 'reportData' => $newReport, 'reportMetadata' => $rowsMetadata); if ($showTimer) { $return['timerMillis'] = $timer->getTimeMs(0); } return $return; }
/** * Will execute $className->$methodName($parametersValues) * If any error is detected (wrong number of parameters, method not found, class not found, etc.) * it will throw an exception * * It also logs the API calls, with the parameters values, the returned value, the performance, etc. * You can enable logging in config/global.ini.php (log_api_call) * * @param string $className The class name (eg. Piwik_Referers_API) * @param string $methodName The method name * @param array $parametersRequest The parameters pairs (name=>value) * * @return mixed|null * @throws Exception|Piwik_Access_NoAccessException */ public function call($className, $methodName, $parametersRequest) { $returnedValue = null; // Temporarily sets the Request array to this API call context $saveGET = $_GET; foreach ($parametersRequest as $param => $value) { $_GET[$param] = $value; } try { $this->registerClass($className); // instanciate the object $object = call_user_func(array($className, "getInstance")); // check method exists $this->checkMethodExists($className, $methodName); // get the list of parameters required by the method $parameterNamesDefaultValues = $this->getParametersList($className, $methodName); // load parameters in the right order, etc. $finalParameters = $this->getRequestParametersArray($parameterNamesDefaultValues, $parametersRequest); // start the timer $timer = new Piwik_Timer(); // call the method $returnedValue = call_user_func_array(array($object, $methodName), $finalParameters); // allow plugins to manipulate the value if (substr($className, 0, 6) == 'Piwik_' && substr($className, -4) == '_API') { $pluginName = substr($className, 6, -4); Piwik_PostEvent('API.Proxy.processReturnValue', $returnedValue, array('className' => $className, 'module' => $pluginName, 'action' => $methodName, 'parameters' => &$parametersRequest)); } // Restore the request $_GET = $saveGET; // log the API Call try { Zend_Registry::get('logger_api_call')->logEvent($className, $methodName, $parameterNamesDefaultValues, $finalParameters, $timer->getTimeMs(), $returnedValue); } catch (Exception $e) { // logger can fail (eg. Tracker request) } } catch (Exception $e) { $_GET = $saveGET; throw $e; } return $returnedValue; }
/** * @return bool True on success, false if some request failed */ private function archiveVisitsAndSegments($idsite, $period, $lastTimestampWebsiteProcessed, Piwik_Timer $timerWebsite = null) { $timer = new Piwik_Timer(); $aCurl = array(); $mh = curl_multi_init(); $url = $this->piwikUrl . $this->getVisitsRequestUrl($idsite, $period, $lastTimestampWebsiteProcessed) . $this->requestPrepend; // already processed above for "day" if ($period != "day") { $ch = $this->getNewCurlHandle($url); curl_multi_add_handle($mh, $ch); $aCurl[$url] = $ch; $this->requests++; } $urlNoSegment = $url; foreach ($this->segments as $segment) { $segmentUrl = $url . '&segment=' . urlencode($segment); $ch = $this->getNewCurlHandle($segmentUrl); curl_multi_add_handle($mh, $ch); $aCurl[$segmentUrl] = $ch; $this->requests++; } $running = null; do { usleep(10000); curl_multi_exec($mh, $running); } while ($running > 0); $success = true; $visitsAllDaysInPeriod = false; foreach ($aCurl as $url => $ch) { $content = curl_multi_getcontent($ch); $successResponse = $this->checkResponse($content, $url); $success = $successResponse && $success; if ($url == $urlNoSegment && $successResponse) { $stats = unserialize($content); if (!is_array($stats)) { $this->logError("Error unserializing the following response: " . $content); } $visitsAllDaysInPeriod = @array_sum($stats); } } foreach ($aCurl as $ch) { curl_multi_remove_handle($mh, $ch); } curl_multi_close($mh); $this->log("Archived website id = {$idsite}, period = {$period}, " . ($period != "day" ? (int) $visitsAllDaysInPeriod . " visits, " : "") . (!empty($timerWebsite) ? $timerWebsite->__toString() : $timer->__toString())); return $success; }
/** * Will execute $className->$methodName($parametersValues) * If any error is detected (wrong number of parameters, method not found, class not found, etc.) * it will throw an exception * * It also logs the API calls, with the parameters values, the returned value, the performance, etc. * You can enable logging in config/global.ini.php (log_api_call) * * @param string The class name (eg. Piwik_Referers_API) * @param string The method name * @param array The parameters pairs (name=>value) * * @throws Piwik_Access_NoAccessException */ public function call($className, $methodName, $parametersRequest ) { $returnedValue = null; // Temporarily sets the Request array to this API call context $saveGET = $_GET; foreach($parametersRequest as $param => $value) { $_GET[$param] = $value; } try { $this->registerClass($className); // instanciate the object $object = call_user_func(array($className, "getInstance")); // check method exists $this->checkMethodExists($className, $methodName); // get the list of parameters required by the method $parameterNamesDefaultValues = $this->getParametersList($className, $methodName); // load parameters in the right order, etc. $finalParameters = $this->getRequestParametersArray( $parameterNamesDefaultValues, $parametersRequest ); // start the timer $timer = new Piwik_Timer(); // call the method $returnedValue = call_user_func_array(array($object, $methodName), $finalParameters); // Restore the request $_GET = $saveGET; // log the API Call try { Zend_Registry::get('logger_api_call')->logEvent( $className, $methodName, $parameterNamesDefaultValues, $finalParameters, $timer->getTimeMs(), $returnedValue ); } catch (Exception $e) { // logger can fail (eg. Tracker request) } } catch (Exception $e) { $_GET = $saveGET; throw $e; } return $returnedValue; }
<?php require_once SIMPLE_TEST . 'unit_tester.php'; require_once SIMPLE_TEST . 'reporter.php'; $test =& new GroupTest('Piwik - running all tests'); $toInclude = array(); foreach (globr(PIWIK_INCLUDE_PATH . '/tests/core/', '*.php') as $file) { $toInclude[] = $file; } foreach ($toInclude as $file) { if (substr_count($file, 'test.php') == 0) { print "The file '{$file}' is not valid: doesn't end with '.test.php' extension. \n<br>"; continue; } $test->addFile($file); } $timer = new Piwik_Timer(); $test->run(new HtmlReporter()); echo $timer . "<br>"; echo $timer->getMemoryLeak(); /* assertTrue($x) Fail if $x is false assertFalse($x) Fail if $x is true assertNull($x) Fail if $x is set assertNotNull($x) Fail if $x not set assertIsA($x, $t) Fail if $x is not the class or type $t assertNotA($x, $t) Fail if $x is of the class or type $t assertEqual($x, $y) Fail if $x == $y is false assertNotEqual($x, $y) Fail if $x == $y is true assertWithinMargin($x, $y, $m) Fail if abs($x - $y) < $m is false assertOutsideMargin($x, $y, $m) Fail if abs($x - $y) < $m is true assertIdentical($x, $y) Fail if $x == $y is false or a type mismatch
/** * Main function, runs archiving on all websites with new activity */ public function run() { $websitesWithVisitsSinceLastRun = $skippedPeriodsArchivesWebsite = $skippedDayArchivesWebsites = $skipped = $processed = $archivedPeriodsArchivesWebsite = 0; $timer = new Piwik_Timer(); $this->logSection("START"); $this->log("Starting Piwik reports archiving..."); foreach ($this->websites as $idsite) { flush(); $requestsBefore = $this->requests; if ($idsite <= 0) { continue; } $timerWebsite = new $timer(); $lastTimestampWebsiteProcessedPeriods = $lastTimestampWebsiteProcessedDay = false; if (!$this->shouldResetState) { $lastTimestampWebsiteProcessedPeriods = Piwik_GetOption($this->lastRunKey($idsite, "periods")); $lastTimestampWebsiteProcessedDay = Piwik_GetOption($this->lastRunKey($idsite, "day")); } // For period other than days, we only re-process the reports at most // 1) every $processPeriodsMaximumEverySeconds $secondsSinceLastExecution = time() - $lastTimestampWebsiteProcessedPeriods; // if timeout is more than 10 min, we account for a 5 min processing time, and allow trigger 1 min earlier if ($this->processPeriodsMaximumEverySeconds > 10 * 60) { $secondsSinceLastExecution += 5 * 60; } $shouldArchivePeriods = $secondsSinceLastExecution > $this->processPeriodsMaximumEverySeconds; if (empty($lastTimestampWebsiteProcessedPeriods)) { // 2) OR always if script never executed for this website before $shouldArchivePeriods = true; } // If the website is archived because it is a new day in its timezone // We make sure all periods are archived, even if there is 0 visit today if (!$shouldArchivePeriods && in_array($idsite, $this->websiteDayHasFinishedSinceLastRun)) { $shouldArchivePeriods = true; } // Test if we should process this website at all $elapsedSinceLastArchiving = time() - $lastTimestampWebsiteProcessedDay; if (!$shouldArchivePeriods && $elapsedSinceLastArchiving < $this->todayArchiveTimeToLive) { $this->log("Skipped website id {$idsite}, already processed today's report in recent run, " . Piwik::getPrettyTimeFromSeconds($elapsedSinceLastArchiving, true, $isHtml = false) . " ago, " . $timerWebsite); $skippedDayArchivesWebsites++; $skipped++; continue; } // Fake that the request is already done, so that other archive.php // running do not grab the same website from the queue Piwik_SetOption($this->lastRunKey($idsite, "day"), time()); $url = $this->getVisitsRequestUrl($idsite, "day", $this->shouldArchiveAllWebsites ? false : $lastTimestampWebsiteProcessedDay); $content = $this->request($url); $response = @unserialize($content); if (empty($content) || !is_array($response) || count($response) == 0) { // cancel the succesful run flag Piwik_SetOption($this->lastRunKey($idsite, "day"), 0); $this->log("WARNING: Empty or invalid response '{$content}' for website id {$idsite}, " . $timerWebsite . ", skipping"); $skipped++; continue; } $visitsToday = end($response); $this->requests++; $processed++; // If there is no visit today and we don't need to process this website, we can skip remaining archives if ($visitsToday <= 0 && !$shouldArchivePeriods) { $this->log("Skipped website id {$idsite}, no visit today, " . $timerWebsite); $skipped++; continue; } $visitsAllDays = array_sum($response); if ($visitsAllDays == 0 && !$shouldArchivePeriods && $this->shouldArchiveAllWebsites) { $this->log("Skipped website id {$idsite}, no visits in the last " . count($response) . " days, " . $timerWebsite); $skipped++; continue; } $this->visits += $visitsToday; $websitesWithVisitsSinceLastRun++; $this->archiveVisitsAndSegments($idsite, "day", $lastTimestampWebsiteProcessedDay, $timerWebsite); if ($shouldArchivePeriods) { $success = true; foreach (array('week', 'month', 'year') as $period) { $success = $this->archiveVisitsAndSegments($idsite, $period, $lastTimestampWebsiteProcessedPeriods) && $success; } // Record succesful run of this website's periods archiving if ($success) { Piwik_SetOption($this->lastRunKey($idsite, "periods"), time()); } $archivedPeriodsArchivesWebsite++; } else { $skippedPeriodsArchivesWebsite++; } $requestsWebsite = $this->requests - $requestsBefore; $debug = $this->shouldArchiveAllWebsites ? ", last days = {$visitsAllDays} visits" : ""; Piwik::log("Archived website id = {$idsite}, today = {$visitsToday} visits" . $debug . ", {$requestsWebsite} API requests, " . $timerWebsite . " [" . ($websitesWithVisitsSinceLastRun + $skipped) . "/" . count($this->websites) . " done]"); } $this->log("Done archiving!"); $this->logSection("SUMMARY"); $this->log("Total daily visits archived: " . $this->visits); $totalWebsites = count($this->allWebsites); $skipped = $totalWebsites - $websitesWithVisitsSinceLastRun; $this->log("Archived today's reports for {$websitesWithVisitsSinceLastRun} websites"); $this->log("Archived week/month/year for {$archivedPeriodsArchivesWebsite} websites. "); $this->log("Skipped {$skipped} websites: no new visit since the last script execution"); $this->log("Skipped {$skippedDayArchivesWebsites} websites day archiving: existing daily reports are less than {$this->todayArchiveTimeToLive} seconds old"); $this->log("Skipped {$skippedPeriodsArchivesWebsite} websites week/month/year archiving: existing periods reports are less than {$this->processPeriodsMaximumEverySeconds} seconds old"); $this->log("Total API requests: {$this->requests}"); //DONE: done/total, visits, wtoday, wperiods, reqs, time, errors[count]: first eg. $percent = count($this->websites) == 0 ? "" : " " . round($processed * 100 / count($this->websites), 0) . "%"; $otherInParallel = $skippedDayArchivesWebsites; $this->log("done: " . $processed . "/" . count($this->websites) . "" . $percent . ", " . $this->visits . " v, {$websitesWithVisitsSinceLastRun} wtoday, {$archivedPeriodsArchivesWebsite} wperiods, " . $this->requests . " req, " . round($timer->getTimeMs()) . " ms, " . (empty($this->errors) ? "no error" : count($this->errors) . " errors. eg. '" . reset($this->errors) . "'")); $this->log($timer); $this->logSection("SCHEDULED TASKS"); $this->log("Starting Scheduled tasks... "); $tasksOutput = $this->request("?module=API&method=CoreAdminHome.runScheduledTasks&format=csv&convertToUnicode=0&token_auth=" . $this->token_auth); if ($tasksOutput == "No data available") { $tasksOutput = " No task to run"; } $this->log($tasksOutput); $this->log("done"); }
/** * Method always called when making an API request. * It checks several things before actually calling the real method on the given module. * * It also logs the API calls, with the parameters values, the returned value, the performance, etc. * You can enable logging in config/global.ini.php (log_api_call) * * @param string The method name * @param array The parameters * * @throws Piwik_Access_NoAccessException */ public function __call($methodName, $parameterValues) { $returnedValue = null; try { $this->registerClass(self::$classCalled); $className = $this->getClassNameFromModule(self::$classCalled); // instanciate the object $object = call_user_func(array($className, "getInstance")); // check method exists $this->checkMethodExists($className, $methodName); // first check number of parameters do match $this->checkNumberOfParametersMatch($className, $methodName, $parameterValues); // start the timer $timer = new Piwik_Timer(); // call the method $returnedValue = call_user_func_array(array($object, $methodName), $parameterValues); // log the API Call $parameterNamesDefaultValues = $this->getParametersList($className, $methodName); Zend_Registry::get('logger_api_call')->log(self::$classCalled, $methodName, $parameterNamesDefaultValues, $parameterValues, $timer->getTimeMs(), $returnedValue); } catch (Piwik_Access_NoAccessException $e) { throw $e; } self::$classCalled = null; return $returnedValue; }