Esempio n. 1
0
 public function test_purgeOutdatedArchives_Purges_WhenBrowserArchivingEnabled_AndCronArchiveTriggerPresent()
 {
     Rules::setBrowserTriggerArchiving(false);
     $_GET['trigger'] = 'archivephp';
     $wasPurged = $this->tasks->purgeOutdatedArchives();
     $this->assertTrue($wasPurged);
 }
 /**
  * @group        Benchmarks
  */
 public function testArchivingProcess()
 {
     if ($this->archivingLaunched) {
         echo "NOTE: Had to archive data, memory results will not be accurate. Run again for better results.";
     }
     Rules::$archivingDisabledByTests = true;
     APIMultiSites::getInstance()->getAll(self::$fixture->period, self::$fixture->date);
 }
Esempio n. 3
0
File: API.php Progetto: piwik/piwik
 /**
  * @internal
  */
 public function setArchiveSettings($enableBrowserTriggerArchiving, $todayArchiveTimeToLive)
 {
     Piwik::checkUserHasSuperUserAccess();
     if (!Controller::isGeneralSettingsAdminEnabled()) {
         throw new Exception('General settings admin is ont enabled');
     }
     Rules::setBrowserTriggerArchiving((bool) $enableBrowserTriggerArchiving);
     Rules::setTodayArchiveTimeToLive($todayArchiveTimeToLive);
     return true;
 }
Esempio n. 4
0
 /**
  * @group        Benchmarks
  */
 public function testArchivingProcess()
 {
     if ($this->archivingLaunched) {
         echo "NOTE: Had to archive data, memory results will not be accurate. Run again for better results.";
     }
     Rules::$archivingDisabledByTests = true;
     $period = Period\Factory::build(self::$fixture->period, Date::factory(self::$fixture->date));
     $dateRange = $period->getDateStart() . ',' . $period->getDateEnd();
     API::getInstance()->get(self::$fixture->idSite, 'day', $dateRange);
 }
 public function __construct(ArchiveProcessor\Parameters $params, $isArchiveTemporary)
 {
     $this->idArchive = false;
     $this->idSite = $params->getSite()->getId();
     $this->segment = $params->getSegment();
     $this->period = $params->getPeriod();
     $this->doneFlag = Rules::getDoneStringFlagFor($this->segment, $this->period->getLabel(), $params->getRequestedPlugin());
     $this->isArchiveTemporary = $isArchiveTemporary;
     $this->dateStart = $this->period->getDateStart();
 }
Esempio n. 6
0
 public static function setUpBeforeClass()
 {
     parent::setUpBeforeClass();
     // Temporarily disable the purge of old archives so that getNumeric('nb_visits')
     // in _addReportData does not trigger the data purge of data we've just imported
     Rules::$purgeDisabledByTests = true;
     self::_addLogData();
     self::_addReportData();
     Rules::$purgeDisabledByTests = false;
     self::$dbData = self::getDbTablesWithData();
 }
Esempio n. 7
0
 public static function setUpBeforeClass()
 {
     parent::setUpBeforeClass();
     // Temporarily disable the purge of old archives so that getNumeric('nb_visits')
     // in _addReportData does not trigger the data purge of data we've just imported
     \Piwik\ArchiveProcessor\Rules::disablePurgeOutdatedArchives();
     self::_addLogData();
     self::_addReportData();
     \Piwik\ArchiveProcessor\Rules::enablePurgeOutdatedArchives();
     self::$dbData = self::getDbTablesWithData();
 }
Esempio n. 8
0
 public static function purgeOutdatedArchives(Date $dateStart)
 {
     $purgeArchivesOlderThan = Rules::shouldPurgeOutdatedArchives($dateStart);
     if (!$purgeArchivesOlderThan) {
         return;
     }
     $idArchivesToDelete = self::getTemporaryArchiveIdsOlderThan($dateStart, $purgeArchivesOlderThan);
     if (!empty($idArchivesToDelete)) {
         self::deleteArchiveIds($dateStart, $idArchivesToDelete);
     }
     self::deleteArchivesWithPeriodRange($dateStart);
     Log::debug("Purging temporary archives: done [ purged archives older than %s in %s ] [Deleted IDs: %s]", $purgeArchivesOlderThan, $dateStart->toString("Y-m"), implode(',', $idArchivesToDelete));
 }
Esempio n. 9
0
 public function setUp()
 {
     // drop all tables
     Db::dropAllTables();
     // download data dump if url supplied
     if (is_file($this->dumpUrl)) {
         $dumpPath = $this->dumpUrl;
     } else {
         $dumpPath = PIWIK_INCLUDE_PATH . '/tmp/logdump.sql.gz';
         $bufferSize = 1024 * 1024;
         $dump = fopen($this->dumpUrl, 'rb');
         $outfile = fopen($dumpPath, 'wb');
         $bytesRead = 0;
         while (!feof($dump)) {
             fwrite($outfile, fread($dump, $bufferSize), $bufferSize);
             $bytesRead += $bufferSize;
         }
         fclose($dump);
         fclose($outfile);
         if ($bytesRead <= 40 * 1024 * 1024) {
             // sanity check
             throw new Exception("Could not download sql dump!");
         }
     }
     // unzip the dump
     if (substr($dumpPath, -3) === ".gz") {
         $deflatedDumpPath = PIWIK_INCLUDE_PATH . '/tmp/logdump.sql';
         // TODO: should depend on name of URL
         exec("gunzip -c \"" . $dumpPath . "\" > \"{$deflatedDumpPath}\"", $output, $return);
         if ($return !== 0) {
             throw new Exception("gunzip failed: " . implode("\n", $output));
         }
     } else {
         $deflatedDumpPath = $dumpPath;
     }
     // load the data into the correct database
     $user = Config::getInstance()->database['username'];
     $password = Config::getInstance()->database['password'];
     Config::getInstance()->database['tables_prefix'] = $this->tablesPrefix;
     $cmd = "mysql -u \"{$user}\" \"--password={$password}\" {$this->dbName} < \"" . $deflatedDumpPath . "\" 2>&1";
     exec($cmd, $output, $return);
     if ($return !== 0) {
         throw new Exception("Failed to load sql dump: " . implode("\n", $output));
     }
     // make sure archiving will be called
     Rules::setBrowserTriggerArchiving(true);
     // reload access
     Piwik::setUserHasSuperUserAccess();
     $this->getTestEnvironment()->configOverride = array('database' => array('tables_prefix' => $this->tablesPrefix));
     $this->getTestEnvironment()->save();
 }
Esempio n. 10
0
 public function setUp()
 {
     // drop all tables
     Db::dropAllTables();
     // download data dump if url supplied
     if (is_file($this->dumpUrl)) {
         $dumpPath = $this->dumpUrl;
     } else {
         $dumpPath = PIWIK_INCLUDE_PATH . '/tmp/logdump.sql.gz';
         $bytesRead = $this->downloadDumpInPath($dumpPath);
         // sanity check
         if ($bytesRead <= 40 * 1024 * 1024) {
             $str = "Could not download sql dump! You can manually download %s into %s";
             throw new Exception(sprintf($str, $this->dumpUrl, $dumpPath));
         }
     }
     // unzip the dump
     if (substr($dumpPath, -3) === ".gz") {
         $deflatedDumpPath = PIWIK_INCLUDE_PATH . '/tmp/logdump.sql';
         // TODO: should depend on name of URL
         exec("gunzip -c \"" . $dumpPath . "\" > \"{$deflatedDumpPath}\"", $output, $return);
         if ($return !== 0) {
             throw new Exception("gunzip failed: " . implode("\n", $output));
         }
     } else {
         $deflatedDumpPath = $dumpPath;
     }
     // load the data into the correct database
     $user = Config::getInstance()->database['username'];
     $password = Config::getInstance()->database['password'];
     $host = Config::getInstance()->database['host'];
     Config::getInstance()->database['tables_prefix'] = $this->tablesPrefix;
     $cmd = "mysql -h \"{$host}\" -u \"{$user}\" \"--password={$password}\" {$this->dbName} < \"" . $deflatedDumpPath . "\" 2>&1";
     exec($cmd, $output, $return);
     if ($return !== 0) {
         throw new Exception("Failed to load sql dump: " . implode("\n", $output));
     }
     // make sure archiving will be called
     Rules::setBrowserTriggerArchiving(true);
     // reload access
     Access::getInstance()->reloadAccess();
     $testVars = new TestingEnvironmentVariables();
     $testVars->configOverride = array('database' => array('tables_prefix' => $this->tablesPrefix));
     $testVars->save();
 }
Esempio n. 11
0
 /**
  * Constructor.
  */
 public function __construct($idSite = false)
 {
     parent::__construct();
     $this->jsClass = "SegmentSelectorControl";
     $this->cssIdentifier = "segmentEditorPanel";
     $this->cssClass = "piwikTopControl borderedControl piwikSelector";
     $this->idSite = $idSite ?: Common::getRequestVar('idSite', false, 'int');
     $this->selectedSegment = Common::getRequestVar('segment', false, 'string');
     $formatter = StaticContainer::get('Piwik\\Plugins\\SegmentEditor\\SegmentFormatter');
     $this->segmentDescription = $formatter->getHumanReadable(Request::getRawSegmentFromRequest(), $this->idSite);
     $this->isAddingSegmentsForAllWebsitesEnabled = SegmentEditor::isAddingSegmentsForAllWebsitesEnabled();
     $segments = APIMetadata::getInstance()->getSegmentsMetadata($this->idSite);
     $visitTitle = Piwik::translate('General_Visit');
     $segmentsByCategory = array();
     foreach ($segments as $segment) {
         if ($segment['category'] == $visitTitle && ($segment['type'] == 'metric' && $segment['segment'] != 'visitIp')) {
             $metricsLabel = Piwik::translate('General_Metrics');
             $metricsLabel[0] = Common::mb_strtolower($metricsLabel[0]);
             $segment['category'] .= ' (' . $metricsLabel . ')';
         }
         $segmentsByCategory[$segment['category']][] = $segment;
     }
     $this->createRealTimeSegmentsIsEnabled = Config::getInstance()->General['enable_create_realtime_segments'];
     $this->segmentsByCategory = $segmentsByCategory;
     $this->nameOfCurrentSegment = '';
     $this->isSegmentNotAppliedBecauseBrowserArchivingIsDisabled = 0;
     $this->availableSegments = API::getInstance()->getAll($this->idSite);
     foreach ($this->availableSegments as &$savedSegment) {
         $savedSegment['name'] = Common::sanitizeInputValue($savedSegment['name']);
         if (!empty($this->selectedSegment) && $this->selectedSegment == $savedSegment['definition']) {
             $this->nameOfCurrentSegment = $savedSegment['name'];
             $this->isSegmentNotAppliedBecauseBrowserArchivingIsDisabled = $this->wouldApplySegment($savedSegment) ? 0 : 1;
         }
     }
     $this->authorizedToCreateSegments = SegmentEditorAPI::getInstance()->isUserCanAddNewSegment($this->idSite);
     $this->isUserAnonymous = Piwik::isUserIsAnonymous();
     $this->segmentTranslations = $this->getTranslations();
     $this->segmentProcessedOnRequest = Rules::isBrowserArchivingAvailableForSegments();
     $this->hideSegmentDefinitionChangeMessage = UsersManagerAPI::getInstance()->getUserPreference(Piwik::getCurrentUserLogin(), 'hideSegmentDefinitionChangeMessage');
 }
Esempio n. 12
0
 /**
  * Returns the done string flag for a plugin using this instance's segment & periods.
  * @param string $plugin
  * @return string
  */
 private function getDoneStringForPlugin($plugin)
 {
     return Rules::getDoneStringFlagFor($this->params->getIdSites(), $this->params->getSegment(), $this->getPeriodLabel(), $plugin, $this->params->isSkipAggregationOfSubTables());
 }
Esempio n. 13
0
 /**
  * Returns a timestamp indicating outdated archives older than this timestamp (processed before) can be purged.
  *
  * @return int|bool  Outdated archives older than this timestamp should be purged
  */
 protected function getOldestTemporaryArchiveToKeepThreshold()
 {
     $temporaryArchivingTimeout = Rules::getTodayArchiveTimeToLive();
     if (Rules::isBrowserTriggerEnabled()) {
         // If Browser Archiving is enabled, it is likely there are many more temporary archives
         // We delete more often which is safe, since reports are re-processed on demand
         return Date::factory($this->now - 2 * $temporaryArchivingTimeout)->getDateTime();
     }
     // If cron core:archive command is building the reports, we should keep all temporary reports from today
     return $this->yesterday->getDateTime();
 }
Esempio n. 14
0
 /**
  * Returns the SQL condition used to find successfully completed archives that
  * this instance is querying for.
  *
  * @return string
  */
 private static function getPossibleValues()
 {
     $possibleValues = array(ArchiveWriter::DONE_OK, ArchiveWriter::DONE_OK_TEMPORARY);
     if (!Rules::isRequestAuthorizedToArchive()) {
         //If request is not authorized to archive then fetch also invalidated archives
         $possibleValues[] = ArchiveWriter::DONE_INVALIDATED;
     }
     return $possibleValues;
 }
Esempio n. 15
0
 static function update()
 {
     $returningMetrics = array('nb_visits_returning', 'nb_actions_returning', 'max_actions_returning', 'sum_visit_length_returning', 'bounce_count_returning', 'nb_visits_converted_returning', 'nb_uniq_visitors_returning');
     $now = Date::factory('now')->getDatetime();
     $archiveNumericTables = Db::get()->fetchCol("SHOW TABLES LIKE '%archive_numeric%'");
     // for each numeric archive table, copy *_returning metrics to VisitsSummary metrics w/ the appropriate
     // returning visit segment
     foreach ($archiveNumericTables as $table) {
         // get archives w/ *._returning
         $sql = "SELECT idarchive, idsite, period, date1, date2\n                      FROM {$table}\n                     WHERE name IN ('" . implode("','", $returningMetrics) . "')\n                  GROUP BY idarchive";
         $idArchivesWithReturning = Db::fetchAll($sql);
         // get archives for visitssummary returning visitor segment
         $sql = "SELECT idarchive, idsite, period, date1, date2\n                      FROM {$table}\n                     WHERE name = ?\n                  GROUP BY idarchive";
         $visitSummaryReturningSegmentDone = Rules::getDoneFlagArchiveContainsOnePlugin(new Segment(VisitFrequencyApi::RETURNING_VISITOR_SEGMENT, $idSites = array()), 'VisitsSummary');
         $idArchivesWithVisitReturningSegment = Db::fetchAll($sql, array($visitSummaryReturningSegmentDone));
         // collect info for new visitssummary archives have to be created to match archives w/ *._returning
         // metrics
         $missingIdArchives = array();
         $idArchiveMappings = array();
         foreach ($idArchivesWithReturning as $row) {
             $withMetricsIdArchive = $row['idarchive'];
             foreach ($idArchivesWithVisitReturningSegment as $segmentRow) {
                 if ($row['idsite'] == $segmentRow['idsite'] && $row['period'] == $segmentRow['period'] && $row['date1'] == $segmentRow['date1'] && $row['date2'] == $segmentRow['date2']) {
                     $idArchiveMappings[$withMetricsIdArchive] = $segmentRow['idarchive'];
                 }
             }
             if (!isset($idArchiveMappings[$withMetricsIdArchive])) {
                 $missingIdArchives[$withMetricsIdArchive] = $row;
             }
         }
         // if there are missing idarchives, fill out new archive row values
         if (!empty($missingIdArchives)) {
             $newIdArchiveStart = Db::fetchOne("SELECT MAX(idarchive) FROM {$table}") + 1;
             foreach ($missingIdArchives as $withMetricsIdArchive => &$rowToInsert) {
                 $idArchiveMappings[$withMetricsIdArchive] = $newIdArchiveStart;
                 $rowToInsert['idarchive'] = $newIdArchiveStart;
                 $rowToInsert['ts_archived'] = $now;
                 $rowToInsert['name'] = $visitSummaryReturningSegmentDone;
                 $rowToInsert['value'] = ArchiveWriter::DONE_OK;
                 ++$newIdArchiveStart;
             }
             // add missing archives
             try {
                 $params = array();
                 foreach ($missingIdArchives as $missingIdArchive) {
                     $params[] = array_values($missingIdArchive);
                 }
                 BatchInsert::tableInsertBatch($table, array_keys(reset($missingIdArchives)), $params, $throwException = false);
             } catch (\Exception $ex) {
                 Updater::handleQueryError($ex, "<batch insert>", false, __FILE__);
             }
         }
         // update idarchive & name columns in rows with *._returning metrics
         $updateSqlPrefix = "UPDATE {$table}\n                                   SET idarchive = CASE idarchive ";
         $updateSqlSuffix = " END, name = CASE name ";
         foreach ($returningMetrics as $metric) {
             $newMetricName = substr($metric, 0, strlen($metric) - strlen(VisitFrequencyApi::COLUMN_SUFFIX));
             $updateSqlSuffix .= "WHEN '{$metric}' THEN '" . $newMetricName . "' ";
         }
         $updateSqlSuffix .= " END WHERE idarchive IN (%s)\n                                        AND name IN ('" . implode("','", $returningMetrics) . "')";
         // update only 1000 rows at a time so we don't send too large an SQL query to MySQL
         foreach (array_chunk($missingIdArchives, 1000, $preserveKeys = true) as $chunk) {
             $idArchives = array();
             $updateSql = $updateSqlPrefix;
             foreach ($chunk as $withMetricsIdArchive => $row) {
                 $updateSql .= "WHEN {$withMetricsIdArchive} THEN {$row['idarchive']} ";
                 $idArchives[] = $withMetricsIdArchive;
             }
             $updateSql .= sprintf($updateSqlSuffix, implode(',', $idArchives));
             Updater::executeMigrationQuery($updateSql, false, __FILE__);
         }
     }
 }
Esempio n. 16
0
 /**
  * Runs API tests.
  */
 protected function runApiTests($api, $params)
 {
     $testConfig = new ApiTestConfig($params);
     $testName = 'test_' . static::getOutputPrefix();
     $this->missingExpectedFiles = array();
     $this->comparisonFailures = array();
     if ($testConfig->disableArchiving) {
         Rules::$archivingDisabledByTests = true;
         Config::getInstance()->General['browser_archiving_disabled_enforce'] = 1;
     } else {
         Rules::$archivingDisabledByTests = false;
         Config::getInstance()->General['browser_archiving_disabled_enforce'] = 0;
     }
     if ($testConfig->language) {
         $this->changeLanguage($testConfig->language);
     }
     $testRequests = $this->getTestRequestsCollection($api, $testConfig, $api);
     foreach ($testRequests->getRequestUrls() as $apiId => $requestUrl) {
         $this->_testApiUrl($testName . $testConfig->testSuffix, $apiId, $requestUrl, $testConfig->compareAgainst, $params);
     }
     // change the language back to en
     if ($this->lastLanguage != 'en') {
         $this->changeLanguage('en');
     }
     if (!empty($this->missingExpectedFiles)) {
         $expectedDir = dirname(reset($this->missingExpectedFiles));
         $this->fail(" ERROR: Could not find expected API output '" . implode("', '", $this->missingExpectedFiles) . "'. For new tests, to pass the test, you can copy files from the processed/ directory into" . " {$expectedDir}  after checking that the output is valid. %s ");
     }
     // Display as one error all sub-failures
     if (!empty($this->comparisonFailures)) {
         $this->printComparisonFailures();
         throw reset($this->comparisonFailures);
     }
     return count($this->comparisonFailures) == 0;
 }
Esempio n. 17
0
 private function handleGeneralSettingsAdmin($view)
 {
     // Whether to display or not the general settings (cron, beta, smtp)
     $view->isGeneralSettingsAdminEnabled = self::isGeneralSettingsAdminEnabled();
     if ($view->isGeneralSettingsAdminEnabled) {
         $this->displayWarningIfConfigFileNotWritable();
     }
     $enableBrowserTriggerArchiving = Rules::isBrowserTriggerEnabled();
     $todayArchiveTimeToLive = Rules::getTodayArchiveTimeToLive();
     $showWarningCron = false;
     if (!$enableBrowserTriggerArchiving && $todayArchiveTimeToLive < 3600) {
         $showWarningCron = true;
     }
     $view->showWarningCron = $showWarningCron;
     $view->todayArchiveTimeToLive = $todayArchiveTimeToLive;
     $view->enableBrowserTriggerArchiving = $enableBrowserTriggerArchiving;
     $view->enableBetaReleaseCheck = Config::getInstance()->Debug['allow_upgrades_to_beta'];
     $view->mail = Config::getInstance()->mail;
     $pluginUpdateCommunication = new UpdateCommunication();
     $view->canUpdateCommunication = $pluginUpdateCommunication->canBeEnabled();
     $view->enableSendPluginUpdateCommunication = $pluginUpdateCommunication->isEnabled();
 }
Esempio n. 18
0
 private function oneClick_Copy()
 {
     /*
      * Make sure the execute bit is set for this shell script
      */
     if (!Rules::isBrowserTriggerEnabled()) {
         @chmod($this->pathRootExtractedPiwik . '/misc/cron/archive.sh', 0755);
     }
     $model = new Model();
     /*
      * Copy all files to PIWIK_INCLUDE_PATH.
      * These files are accessed through the dispatcher.
      */
     Filesystem::copyRecursive($this->pathRootExtractedPiwik, PIWIK_INCLUDE_PATH);
     $model->removeGoneFiles($this->pathRootExtractedPiwik, PIWIK_INCLUDE_PATH);
     /*
      * These files are visible in the web root and are generally
      * served directly by the web server.  May be shared.
      */
     if (PIWIK_INCLUDE_PATH !== PIWIK_DOCUMENT_ROOT) {
         /*
          * Copy PHP files that expect to be in the document root
          */
         $specialCases = array('/index.php', '/piwik.php', '/js/index.php');
         foreach ($specialCases as $file) {
             Filesystem::copy($this->pathRootExtractedPiwik . $file, PIWIK_DOCUMENT_ROOT . $file);
         }
         /*
          * Copy the non-PHP files (e.g., images, css, javascript)
          */
         Filesystem::copyRecursive($this->pathRootExtractedPiwik, PIWIK_DOCUMENT_ROOT, true);
         $model->removeGoneFiles($this->pathRootExtractedPiwik, PIWIK_DOCUMENT_ROOT);
     }
     /*
      * Config files may be user (account) specific
      */
     if (PIWIK_INCLUDE_PATH !== PIWIK_USER_PATH) {
         Filesystem::copyRecursive($this->pathRootExtractedPiwik . '/config', PIWIK_USER_PATH . '/config');
     }
     Filesystem::unlinkRecursive($this->pathRootExtractedPiwik, true);
     Filesystem::clearPhpCaches();
 }
Esempio n. 19
0
 /**
  * Runs API tests.
  */
 protected function runApiTests($api, $params)
 {
     $testConfig = new ApiTestConfig($params);
     // make sure that the reports we process here are not directly deleted in ArchiveProcessor/PluginsArchiver
     // (because we process reports in the past, they would sometimes be invalid, and would have been deleted)
     \Piwik\ArchiveProcessor\Rules::disablePurgeOutdatedArchives();
     $testName = 'test_' . static::getOutputPrefix();
     $this->missingExpectedFiles = array();
     $this->comparisonFailures = array();
     if ($testConfig->disableArchiving) {
         Rules::$archivingDisabledByTests = true;
         Config::getInstance()->General['browser_archiving_disabled_enforce'] = 1;
     } else {
         Rules::$archivingDisabledByTests = false;
         Config::getInstance()->General['browser_archiving_disabled_enforce'] = 0;
     }
     if ($testConfig->language) {
         $this->changeLanguage($testConfig->language);
     }
     $testRequests = new TestRequestCollection($api, $testConfig, $api);
     foreach ($testRequests->getRequestUrls() as $apiId => $requestUrl) {
         $this->_testApiUrl($testName . $testConfig->testSuffix, $apiId, $requestUrl, $testConfig->compareAgainst, $testConfig->xmlFieldsToRemove, $params);
     }
     // Restore normal purge behavior
     \Piwik\ArchiveProcessor\Rules::enablePurgeOutdatedArchives();
     // change the language back to en
     if ($this->lastLanguage != 'en') {
         $this->changeLanguage('en');
     }
     if (!empty($this->missingExpectedFiles)) {
         $expectedDir = dirname(reset($this->missingExpectedFiles));
         $this->fail(" ERROR: Could not find expected API output '" . implode("', '", $this->missingExpectedFiles) . "'. For new tests, to pass the test, you can copy files from the processed/ directory into" . " {$expectedDir}  after checking that the output is valid. %s ");
     }
     // Display as one error all sub-failures
     if (!empty($this->comparisonFailures)) {
         $this->printComparisonFailures();
         throw reset($this->comparisonFailures);
     }
     return count($this->comparisonFailures) == 0;
 }
Esempio n. 20
0
 /**
  * Runs API tests.
  */
 protected function runApiTests($api, $params)
 {
     $testConfig = new ApiTestConfig($params);
     $testName = 'test_' . static::getOutputPrefix();
     $this->missingExpectedFiles = array();
     $this->comparisonFailures = array();
     if ($testConfig->disableArchiving) {
         Rules::$archivingDisabledByTests = true;
         Config::getInstance()->General['browser_archiving_disabled_enforce'] = 1;
     } else {
         Rules::$archivingDisabledByTests = false;
         Config::getInstance()->General['browser_archiving_disabled_enforce'] = 0;
     }
     if ($testConfig->language) {
         $this->changeLanguage($testConfig->language);
     }
     $testRequests = $this->getTestRequestsCollection($api, $testConfig, $api);
     foreach ($testRequests->getRequestUrls() as $apiId => $requestUrl) {
         $this->_testApiUrl($testName . $testConfig->testSuffix, $apiId, $requestUrl, $testConfig->compareAgainst, $params);
     }
     // change the language back to en
     if ($this->lastLanguage != 'en') {
         $this->changeLanguage('en');
     }
     $this->printApiTestFailures();
     return count($this->comparisonFailures) == 0;
 }
Esempio n. 21
0
 public function __construct(Parameters $params, ArchiveWriter $archiveWriter, LogAggregator $logAggregator)
 {
     $this->params = $params;
     $this->logAggregator = $logAggregator;
     $this->archiveWriter = $archiveWriter;
     $this->skipUniqueVisitorsCalculationForMultipleSites = Rules::shouldSkipUniqueVisitorsCalculationForMultipleSites();
 }
Esempio n. 22
0
 public function test_ArchiveIsLaunched_WhenForceOnBrowserRequest_IsFalse_AndArchivePhpTriggered()
 {
     $this->archiveDataForIndividualDays();
     Config::getInstance()->General['archiving_range_force_on_browser_request'] = 0;
     $_GET['trigger'] = 'archivephp';
     Rules::setBrowserTriggerArchiving(false);
     $data = $this->initiateArchivingForRange();
     $this->assertNotEmpty($data);
     $this->assertArchiveTablesAreNotEmpty('2010_03');
 }
Esempio n. 23
0
 /**
  * Returns the SQL condition used to find successfully completed archives that
  * this instance is querying for.
  *
  * @param array $plugins
  * @param Segment $segment
  * @param bool $isSkipAggregationOfSubTables
  * @return string
  */
 private static function getNameCondition(array $plugins, Segment $segment, $isSkipAggregationOfSubTables)
 {
     // the flags used to tell how the archiving process for a specific archive was completed,
     // if it was completed
     $doneFlags = Rules::getDoneFlags($plugins, $segment, $isSkipAggregationOfSubTables);
     $allDoneFlags = "'" . implode("','", $doneFlags) . "'";
     $possibleValues = array(ArchiveWriter::DONE_OK, ArchiveWriter::DONE_OK_TEMPORARY);
     if (!Rules::isRequestAuthorizedToArchive()) {
         //If request is not authorized to archive then fetch also invalidated archives
         $possibleValues[] = ArchiveWriter::DONE_INVALIDATED;
     }
     // create the SQL to find archives that are DONE
     return "((name IN ({$allDoneFlags})) AND " . " (value IN (" . implode(',', $possibleValues) . ")))";
 }
Esempio n. 24
0
 private function insertArchiveRow($idSite, $date, $periodLabel)
 {
     $periodObject = \Piwik\Period\Factory::build($periodLabel, $date);
     $dateStart = $periodObject->getDateStart();
     $dateEnd = $periodObject->getDateEnd();
     $table = ArchiveTableCreator::getNumericTable($dateStart);
     $idArchive = (int) Db::fetchOne("SELECT MAX(idarchive) FROM {$table} WHERE name LIKE 'done%'");
     $idArchive = $idArchive + 1;
     $periodId = Piwik::$idPeriods[$periodLabel];
     $doneFlag = 'done';
     if ($idArchive % 5 == 1) {
         $doneFlag = Rules::getDoneFlagArchiveContainsAllPlugins(self::$segment1);
     } else {
         if ($idArchive % 5 == 2) {
             $doneFlag .= '.VisitsSummary';
         } else {
             if ($idArchive % 5 == 3) {
                 $doneFlag = Rules::getDoneFlagArchiveContainsOnePlugin(self::$segment1, 'UserCountry');
             } else {
                 if ($idArchive % 5 == 4) {
                     $doneFlag = Rules::getDoneFlagArchiveContainsAllPlugins(self::$segment2);
                 }
             }
         }
     }
     $sql = "INSERT INTO {$table} (idarchive, name, idsite, date1, date2, period, ts_archived)\n                     VALUES ({$idArchive}, 'nb_visits', {$idSite}, '{$dateStart}', '{$dateEnd}', {$periodId}, NOW()),\n                            ({$idArchive}, '{$doneFlag}', {$idSite}, '{$dateStart}', '{$dateEnd}', {$periodId}, NOW())";
     Db::query($sql);
 }
Esempio n. 25
0
 private function logArchiveTimeoutInfo()
 {
     $this->logSection("NOTES");
     // Recommend to disable browser archiving when using this script
     if (Rules::isBrowserTriggerEnabled()) {
         $this->logger->info("- If you execute this script at least once per hour (or more often) in a crontab, you may disable 'Browser trigger archiving' in Piwik UI > Settings > General Settings.");
         $this->logger->info("  See the doc at: http://piwik.org/docs/setup-auto-archiving/");
     }
     $this->logger->info("- Reports for today will be processed at most every " . $this->todayArchiveTimeToLive . " seconds. You can change this value in Piwik UI > Settings > General Settings.");
     $this->logger->info("- Reports for the current week/month/year will be refreshed at most every " . $this->processPeriodsMaximumEverySeconds . " seconds.");
     // Try and not request older data we know is already archived
     if ($this->lastSuccessRunTimestamp !== false) {
         $dateLast = time() - $this->lastSuccessRunTimestamp;
         $this->logger->info("- Archiving was last executed without error " . $this->formatter->getPrettyTimeFromSeconds($dateLast, true) . " ago");
     }
 }
Esempio n. 26
0
 /**
  * Returns contents of general (global) cache.
  * If the cache file tmp/cache/tracker/general.php does not exist yet, create it
  *
  * @return array
  */
 public static function getCacheGeneral()
 {
     $cache = self::getCache();
     $cacheContent = $cache->fetch(self::$cacheIdGeneral);
     if (false !== $cacheContent) {
         return $cacheContent;
     }
     Tracker::initCorePiwikInTrackerMode();
     $cacheContent = array('isBrowserTriggerEnabled' => Rules::isBrowserTriggerEnabled(), 'lastTrackerCronRun' => Option::get('lastTrackerCronRun'));
     /**
      * Triggered before the [general tracker cache](/guides/all-about-tracking#the-tracker-cache)
      * is saved to disk. This event can be used to add extra content to the cache.
      *
      * Data that is used during tracking but is expensive to compute/query should be
      * cached to keep tracking efficient. One example of such data are options
      * that are stored in the piwik_option table. Querying data for each tracking
      * request means an extra unnecessary database query for each visitor action. Using
      * a cache solves this problem.
      *
      * **Example**
      *
      *     public function setTrackerCacheGeneral(&$cacheContent)
      *     {
      *         $cacheContent['MyPlugin.myCacheKey'] = Option::get('MyPlugin_myOption');
      *     }
      *
      * @param array &$cacheContent Array of cached data. Each piece of data must be
      *                             mapped by name.
      */
     Piwik::postEvent('Tracker.setTrackerCacheGeneral', array(&$cacheContent));
     self::setCacheGeneral($cacheContent);
     Common::printDebug("General tracker cache was re-created.");
     Tracker::restoreTrackerPlugins();
     return $cacheContent;
 }
Esempio n. 27
0
 /**
  * @param string $archiveTable Prefixed table name
  * @param int[] $idSites
  * @param string[][] $datesByPeriodType
  * @param Segment $segment
  * @return \Zend_Db_Statement
  * @throws Exception
  */
 public function updateArchiveAsInvalidated($archiveTable, $idSites, $datesByPeriodType, Segment $segment = null)
 {
     $idSites = array_map('intval', $idSites);
     $bind = array();
     $periodConditions = array();
     foreach ($datesByPeriodType as $periodType => $dates) {
         $dateConditions = array();
         foreach ($dates as $date) {
             $dateConditions[] = "(date1 <= ? AND ? <= date2)";
             $bind[] = $date;
             $bind[] = $date;
         }
         $dateConditionsSql = implode(" OR ", $dateConditions);
         if (empty($periodType) || $periodType == Period\Day::PERIOD_ID) {
             // invalidate all periods if no period supplied or period is day
             $periodConditions[] = "({$dateConditionsSql})";
         } else {
             if ($periodType == Period\Range::PERIOD_ID) {
                 $periodConditions[] = "(period = " . Period\Range::PERIOD_ID . " AND ({$dateConditionsSql}))";
             } else {
                 // for non-day periods, invalidate greater periods, but not range periods
                 $periodConditions[] = "(period >= " . (int) $periodType . " AND period < " . Period\Range::PERIOD_ID . " AND ({$dateConditionsSql}))";
             }
         }
     }
     if ($segment) {
         $nameCondition = "name LIKE '" . Rules::getDoneFlagArchiveContainsAllPlugins($segment) . "%'";
     } else {
         $nameCondition = "name LIKE 'done%'";
     }
     $sql = "UPDATE {$archiveTable} SET value = " . ArchiveWriter::DONE_INVALIDATED . " WHERE {$nameCondition}\n                   AND idsite IN (" . implode(", ", $idSites) . ")\n                   AND (" . implode(" OR ", $periodConditions) . ")";
     return Db::query($sql, $bind);
 }
Esempio n. 28
0
 public static function isRequestAuthorizedToArchive()
 {
     return Rules::isBrowserTriggerEnabled() || SettingsServer::isArchivePhpTriggered();
 }
Esempio n. 29
0
 /**
  * test of validity of an archive, for today's archive with toronto's timezone
  * @group Core
  */
 public function testInitTodayToronto()
 {
     if (!SettingsServer::isTimezoneSupportEnabled()) {
         $this->markTestSkipped('timezones needs to be supported');
     }
     $now = time();
     $siteTimezone = 'America/Toronto';
     $timestamp = Date::factory('now', $siteTimezone)->getTimestamp();
     $dateLabel = date('Y-m-d', $timestamp);
     Rules::setBrowserTriggerArchiving(true);
     $archiveProcessor = $this->_createArchiveProcessor('day', $dateLabel, $siteTimezone);
     $archiveProcessor->time = $now;
     // we look at anything processed within the time to live range
     $dateMinArchived = $now - Rules::getTodayArchiveTimeToLive();
     $this->compareTimestamps($dateMinArchived, $archiveProcessor->public_getMinTimeArchiveProcessed());
     $this->assertTrue($archiveProcessor->public_isArchiveTemporary());
     // when browsers don't trigger archives...
     Rules::setBrowserTriggerArchiving(false);
     // ...we force ArchiveProcessor to fetch any of the most recent archive
     $dateMinArchived = false;
     $this->compareTimestamps($dateMinArchived, $archiveProcessor->public_getMinTimeArchiveProcessed());
     // this test varies with DST
     $this->assertTrue($archiveProcessor->getParams()->getDateStart()->getDateStartUTC() == date('Y-m-d', $timestamp) . ' 04:00:00' || $archiveProcessor->getParams()->getDateStart()->getDateStartUTC() == date('Y-m-d', $timestamp) . ' 05:00:00');
     $this->assertTrue($archiveProcessor->getParams()->getDateEnd()->getDateEndUTC() == date('Y-m-d', $timestamp + 86400) . ' 03:59:59' || $archiveProcessor->getParams()->getDateEnd()->getDateEndUTC() == date('Y-m-d', $timestamp + 86400) . ' 04:59:59');
     $this->assertTrue($archiveProcessor->public_isArchiveTemporary());
 }
Esempio n. 30
0
 /**
  * Whether the specified plugin's reports should be archived
  * @param string $pluginName
  * @return bool
  */
 protected function shouldProcessReportsForPlugin($pluginName)
 {
     if ($this->params->getRequestedPlugin() == $pluginName) {
         return true;
     }
     if (Rules::shouldProcessReportsAllPlugins($this->params->getIdSites(), $this->params->getSegment(), $this->params->getPeriod()->getLabel())) {
         return true;
     }
     if (!\Piwik\Plugin\Manager::getInstance()->isPluginLoaded($this->params->getRequestedPlugin())) {
         return true;
     }
     return false;
 }