public function test_purgeOutdatedArchives_Purges_WhenBrowserArchivingEnabled_AndCronArchiveTriggerPresent()
{
Rules::setBrowserTriggerArchiving(false);
$_GET['trigger'] = 'archivephp';
$wasPurged = $this->tasks->purgeOutdatedArchives();
$this->assertTrue($wasPurged);
}
/**
* @group Benchmarks
*/
public function testArchivingProcess()
{
if ($this->archivingLaunched) {
echo "NOTE: Had to archive data, memory results will not be accurate. Run again for better results.";
}
Rules::$archivingDisabledByTests = true;
APIMultiSites::getInstance()->getAll(self::$fixture->period, self::$fixture->date);
}
/**
* @internal
*/
public function setArchiveSettings($enableBrowserTriggerArchiving, $todayArchiveTimeToLive)
{
Piwik::checkUserHasSuperUserAccess();
if (!Controller::isGeneralSettingsAdminEnabled()) {
throw new Exception('General settings admin is ont enabled');
}
Rules::setBrowserTriggerArchiving((bool) $enableBrowserTriggerArchiving);
Rules::setTodayArchiveTimeToLive($todayArchiveTimeToLive);
return true;
}
/**
* @group Benchmarks
*/
public function testArchivingProcess()
{
if ($this->archivingLaunched) {
echo "NOTE: Had to archive data, memory results will not be accurate. Run again for better results.";
}
Rules::$archivingDisabledByTests = true;
$period = Period\Factory::build(self::$fixture->period, Date::factory(self::$fixture->date));
$dateRange = $period->getDateStart() . ',' . $period->getDateEnd();
API::getInstance()->get(self::$fixture->idSite, 'day', $dateRange);
}
public function __construct(ArchiveProcessor\Parameters $params, $isArchiveTemporary)
{
$this->idArchive = false;
$this->idSite = $params->getSite()->getId();
$this->segment = $params->getSegment();
$this->period = $params->getPeriod();
$this->doneFlag = Rules::getDoneStringFlagFor($this->segment, $this->period->getLabel(), $params->getRequestedPlugin());
$this->isArchiveTemporary = $isArchiveTemporary;
$this->dateStart = $this->period->getDateStart();
}
public static function setUpBeforeClass()
{
parent::setUpBeforeClass();
// Temporarily disable the purge of old archives so that getNumeric('nb_visits')
// in _addReportData does not trigger the data purge of data we've just imported
Rules::$purgeDisabledByTests = true;
self::_addLogData();
self::_addReportData();
Rules::$purgeDisabledByTests = false;
self::$dbData = self::getDbTablesWithData();
}
public static function setUpBeforeClass()
{
parent::setUpBeforeClass();
// Temporarily disable the purge of old archives so that getNumeric('nb_visits')
// in _addReportData does not trigger the data purge of data we've just imported
\Piwik\ArchiveProcessor\Rules::disablePurgeOutdatedArchives();
self::_addLogData();
self::_addReportData();
\Piwik\ArchiveProcessor\Rules::enablePurgeOutdatedArchives();
self::$dbData = self::getDbTablesWithData();
}
public static function purgeOutdatedArchives(Date $dateStart)
{
$purgeArchivesOlderThan = Rules::shouldPurgeOutdatedArchives($dateStart);
if (!$purgeArchivesOlderThan) {
return;
}
$idArchivesToDelete = self::getTemporaryArchiveIdsOlderThan($dateStart, $purgeArchivesOlderThan);
if (!empty($idArchivesToDelete)) {
self::deleteArchiveIds($dateStart, $idArchivesToDelete);
}
self::deleteArchivesWithPeriodRange($dateStart);
Log::debug("Purging temporary archives: done [ purged archives older than %s in %s ] [Deleted IDs: %s]", $purgeArchivesOlderThan, $dateStart->toString("Y-m"), implode(',', $idArchivesToDelete));
}
public function setUp()
{
// drop all tables
Db::dropAllTables();
// download data dump if url supplied
if (is_file($this->dumpUrl)) {
$dumpPath = $this->dumpUrl;
} else {
$dumpPath = PIWIK_INCLUDE_PATH . '/tmp/logdump.sql.gz';
$bufferSize = 1024 * 1024;
$dump = fopen($this->dumpUrl, 'rb');
$outfile = fopen($dumpPath, 'wb');
$bytesRead = 0;
while (!feof($dump)) {
fwrite($outfile, fread($dump, $bufferSize), $bufferSize);
$bytesRead += $bufferSize;
}
fclose($dump);
fclose($outfile);
if ($bytesRead <= 40 * 1024 * 1024) {
// sanity check
throw new Exception("Could not download sql dump!");
}
}
// unzip the dump
if (substr($dumpPath, -3) === ".gz") {
$deflatedDumpPath = PIWIK_INCLUDE_PATH . '/tmp/logdump.sql';
// TODO: should depend on name of URL
exec("gunzip -c \"" . $dumpPath . "\" > \"{$deflatedDumpPath}\"", $output, $return);
if ($return !== 0) {
throw new Exception("gunzip failed: " . implode("\n", $output));
}
} else {
$deflatedDumpPath = $dumpPath;
}
// load the data into the correct database
$user = Config::getInstance()->database['username'];
$password = Config::getInstance()->database['password'];
Config::getInstance()->database['tables_prefix'] = $this->tablesPrefix;
$cmd = "mysql -u \"{$user}\" \"--password={$password}\" {$this->dbName} < \"" . $deflatedDumpPath . "\" 2>&1";
exec($cmd, $output, $return);
if ($return !== 0) {
throw new Exception("Failed to load sql dump: " . implode("\n", $output));
}
// make sure archiving will be called
Rules::setBrowserTriggerArchiving(true);
// reload access
Piwik::setUserHasSuperUserAccess();
$this->getTestEnvironment()->configOverride = array('database' => array('tables_prefix' => $this->tablesPrefix));
$this->getTestEnvironment()->save();
}
public function setUp()
{
// drop all tables
Db::dropAllTables();
// download data dump if url supplied
if (is_file($this->dumpUrl)) {
$dumpPath = $this->dumpUrl;
} else {
$dumpPath = PIWIK_INCLUDE_PATH . '/tmp/logdump.sql.gz';
$bytesRead = $this->downloadDumpInPath($dumpPath);
// sanity check
if ($bytesRead <= 40 * 1024 * 1024) {
$str = "Could not download sql dump! You can manually download %s into %s";
throw new Exception(sprintf($str, $this->dumpUrl, $dumpPath));
}
}
// unzip the dump
if (substr($dumpPath, -3) === ".gz") {
$deflatedDumpPath = PIWIK_INCLUDE_PATH . '/tmp/logdump.sql';
// TODO: should depend on name of URL
exec("gunzip -c \"" . $dumpPath . "\" > \"{$deflatedDumpPath}\"", $output, $return);
if ($return !== 0) {
throw new Exception("gunzip failed: " . implode("\n", $output));
}
} else {
$deflatedDumpPath = $dumpPath;
}
// load the data into the correct database
$user = Config::getInstance()->database['username'];
$password = Config::getInstance()->database['password'];
$host = Config::getInstance()->database['host'];
Config::getInstance()->database['tables_prefix'] = $this->tablesPrefix;
$cmd = "mysql -h \"{$host}\" -u \"{$user}\" \"--password={$password}\" {$this->dbName} < \"" . $deflatedDumpPath . "\" 2>&1";
exec($cmd, $output, $return);
if ($return !== 0) {
throw new Exception("Failed to load sql dump: " . implode("\n", $output));
}
// make sure archiving will be called
Rules::setBrowserTriggerArchiving(true);
// reload access
Access::getInstance()->reloadAccess();
$testVars = new TestingEnvironmentVariables();
$testVars->configOverride = array('database' => array('tables_prefix' => $this->tablesPrefix));
$testVars->save();
}
/**
* Constructor.
*/
public function __construct($idSite = false)
{
parent::__construct();
$this->jsClass = "SegmentSelectorControl";
$this->cssIdentifier = "segmentEditorPanel";
$this->cssClass = "piwikTopControl borderedControl piwikSelector";
$this->idSite = $idSite ?: Common::getRequestVar('idSite', false, 'int');
$this->selectedSegment = Common::getRequestVar('segment', false, 'string');
$formatter = StaticContainer::get('Piwik\\Plugins\\SegmentEditor\\SegmentFormatter');
$this->segmentDescription = $formatter->getHumanReadable(Request::getRawSegmentFromRequest(), $this->idSite);
$this->isAddingSegmentsForAllWebsitesEnabled = SegmentEditor::isAddingSegmentsForAllWebsitesEnabled();
$segments = APIMetadata::getInstance()->getSegmentsMetadata($this->idSite);
$visitTitle = Piwik::translate('General_Visit');
$segmentsByCategory = array();
foreach ($segments as $segment) {
if ($segment['category'] == $visitTitle && ($segment['type'] == 'metric' && $segment['segment'] != 'visitIp')) {
$metricsLabel = Piwik::translate('General_Metrics');
$metricsLabel[0] = Common::mb_strtolower($metricsLabel[0]);
$segment['category'] .= ' (' . $metricsLabel . ')';
}
$segmentsByCategory[$segment['category']][] = $segment;
}
$this->createRealTimeSegmentsIsEnabled = Config::getInstance()->General['enable_create_realtime_segments'];
$this->segmentsByCategory = $segmentsByCategory;
$this->nameOfCurrentSegment = '';
$this->isSegmentNotAppliedBecauseBrowserArchivingIsDisabled = 0;
$this->availableSegments = API::getInstance()->getAll($this->idSite);
foreach ($this->availableSegments as &$savedSegment) {
$savedSegment['name'] = Common::sanitizeInputValue($savedSegment['name']);
if (!empty($this->selectedSegment) && $this->selectedSegment == $savedSegment['definition']) {
$this->nameOfCurrentSegment = $savedSegment['name'];
$this->isSegmentNotAppliedBecauseBrowserArchivingIsDisabled = $this->wouldApplySegment($savedSegment) ? 0 : 1;
}
}
$this->authorizedToCreateSegments = SegmentEditorAPI::getInstance()->isUserCanAddNewSegment($this->idSite);
$this->isUserAnonymous = Piwik::isUserIsAnonymous();
$this->segmentTranslations = $this->getTranslations();
$this->segmentProcessedOnRequest = Rules::isBrowserArchivingAvailableForSegments();
$this->hideSegmentDefinitionChangeMessage = UsersManagerAPI::getInstance()->getUserPreference(Piwik::getCurrentUserLogin(), 'hideSegmentDefinitionChangeMessage');
}
/**
* Returns the done string flag for a plugin using this instance's segment & periods.
* @param string $plugin
* @return string
*/
private function getDoneStringForPlugin($plugin)
{
return Rules::getDoneStringFlagFor($this->params->getIdSites(), $this->params->getSegment(), $this->getPeriodLabel(), $plugin, $this->params->isSkipAggregationOfSubTables());
}
/**
* Returns a timestamp indicating outdated archives older than this timestamp (processed before) can be purged.
*
* @return int|bool Outdated archives older than this timestamp should be purged
*/
protected function getOldestTemporaryArchiveToKeepThreshold()
{
$temporaryArchivingTimeout = Rules::getTodayArchiveTimeToLive();
if (Rules::isBrowserTriggerEnabled()) {
// If Browser Archiving is enabled, it is likely there are many more temporary archives
// We delete more often which is safe, since reports are re-processed on demand
return Date::factory($this->now - 2 * $temporaryArchivingTimeout)->getDateTime();
}
// If cron core:archive command is building the reports, we should keep all temporary reports from today
return $this->yesterday->getDateTime();
}
/**
* Returns the SQL condition used to find successfully completed archives that
* this instance is querying for.
*
* @return string
*/
private static function getPossibleValues()
{
$possibleValues = array(ArchiveWriter::DONE_OK, ArchiveWriter::DONE_OK_TEMPORARY);
if (!Rules::isRequestAuthorizedToArchive()) {
//If request is not authorized to archive then fetch also invalidated archives
$possibleValues[] = ArchiveWriter::DONE_INVALIDATED;
}
return $possibleValues;
}
static function update()
{
$returningMetrics = array('nb_visits_returning', 'nb_actions_returning', 'max_actions_returning', 'sum_visit_length_returning', 'bounce_count_returning', 'nb_visits_converted_returning', 'nb_uniq_visitors_returning');
$now = Date::factory('now')->getDatetime();
$archiveNumericTables = Db::get()->fetchCol("SHOW TABLES LIKE '%archive_numeric%'");
// for each numeric archive table, copy *_returning metrics to VisitsSummary metrics w/ the appropriate
// returning visit segment
foreach ($archiveNumericTables as $table) {
// get archives w/ *._returning
$sql = "SELECT idarchive, idsite, period, date1, date2\n FROM {$table}\n WHERE name IN ('" . implode("','", $returningMetrics) . "')\n GROUP BY idarchive";
$idArchivesWithReturning = Db::fetchAll($sql);
// get archives for visitssummary returning visitor segment
$sql = "SELECT idarchive, idsite, period, date1, date2\n FROM {$table}\n WHERE name = ?\n GROUP BY idarchive";
$visitSummaryReturningSegmentDone = Rules::getDoneFlagArchiveContainsOnePlugin(new Segment(VisitFrequencyApi::RETURNING_VISITOR_SEGMENT, $idSites = array()), 'VisitsSummary');
$idArchivesWithVisitReturningSegment = Db::fetchAll($sql, array($visitSummaryReturningSegmentDone));
// collect info for new visitssummary archives have to be created to match archives w/ *._returning
// metrics
$missingIdArchives = array();
$idArchiveMappings = array();
foreach ($idArchivesWithReturning as $row) {
$withMetricsIdArchive = $row['idarchive'];
foreach ($idArchivesWithVisitReturningSegment as $segmentRow) {
if ($row['idsite'] == $segmentRow['idsite'] && $row['period'] == $segmentRow['period'] && $row['date1'] == $segmentRow['date1'] && $row['date2'] == $segmentRow['date2']) {
$idArchiveMappings[$withMetricsIdArchive] = $segmentRow['idarchive'];
}
}
if (!isset($idArchiveMappings[$withMetricsIdArchive])) {
$missingIdArchives[$withMetricsIdArchive] = $row;
}
}
// if there are missing idarchives, fill out new archive row values
if (!empty($missingIdArchives)) {
$newIdArchiveStart = Db::fetchOne("SELECT MAX(idarchive) FROM {$table}") + 1;
foreach ($missingIdArchives as $withMetricsIdArchive => &$rowToInsert) {
$idArchiveMappings[$withMetricsIdArchive] = $newIdArchiveStart;
$rowToInsert['idarchive'] = $newIdArchiveStart;
$rowToInsert['ts_archived'] = $now;
$rowToInsert['name'] = $visitSummaryReturningSegmentDone;
$rowToInsert['value'] = ArchiveWriter::DONE_OK;
++$newIdArchiveStart;
}
// add missing archives
try {
$params = array();
foreach ($missingIdArchives as $missingIdArchive) {
$params[] = array_values($missingIdArchive);
}
BatchInsert::tableInsertBatch($table, array_keys(reset($missingIdArchives)), $params, $throwException = false);
} catch (\Exception $ex) {
Updater::handleQueryError($ex, "<batch insert>", false, __FILE__);
}
}
// update idarchive & name columns in rows with *._returning metrics
$updateSqlPrefix = "UPDATE {$table}\n SET idarchive = CASE idarchive ";
$updateSqlSuffix = " END, name = CASE name ";
foreach ($returningMetrics as $metric) {
$newMetricName = substr($metric, 0, strlen($metric) - strlen(VisitFrequencyApi::COLUMN_SUFFIX));
$updateSqlSuffix .= "WHEN '{$metric}' THEN '" . $newMetricName . "' ";
}
$updateSqlSuffix .= " END WHERE idarchive IN (%s)\n AND name IN ('" . implode("','", $returningMetrics) . "')";
// update only 1000 rows at a time so we don't send too large an SQL query to MySQL
foreach (array_chunk($missingIdArchives, 1000, $preserveKeys = true) as $chunk) {
$idArchives = array();
$updateSql = $updateSqlPrefix;
foreach ($chunk as $withMetricsIdArchive => $row) {
$updateSql .= "WHEN {$withMetricsIdArchive} THEN {$row['idarchive']} ";
$idArchives[] = $withMetricsIdArchive;
}
$updateSql .= sprintf($updateSqlSuffix, implode(',', $idArchives));
Updater::executeMigrationQuery($updateSql, false, __FILE__);
}
}
}
/**
* Runs API tests.
*/
protected function runApiTests($api, $params)
{
$testConfig = new ApiTestConfig($params);
$testName = 'test_' . static::getOutputPrefix();
$this->missingExpectedFiles = array();
$this->comparisonFailures = array();
if ($testConfig->disableArchiving) {
Rules::$archivingDisabledByTests = true;
Config::getInstance()->General['browser_archiving_disabled_enforce'] = 1;
} else {
Rules::$archivingDisabledByTests = false;
Config::getInstance()->General['browser_archiving_disabled_enforce'] = 0;
}
if ($testConfig->language) {
$this->changeLanguage($testConfig->language);
}
$testRequests = $this->getTestRequestsCollection($api, $testConfig, $api);
foreach ($testRequests->getRequestUrls() as $apiId => $requestUrl) {
$this->_testApiUrl($testName . $testConfig->testSuffix, $apiId, $requestUrl, $testConfig->compareAgainst, $params);
}
// change the language back to en
if ($this->lastLanguage != 'en') {
$this->changeLanguage('en');
}
if (!empty($this->missingExpectedFiles)) {
$expectedDir = dirname(reset($this->missingExpectedFiles));
$this->fail(" ERROR: Could not find expected API output '" . implode("', '", $this->missingExpectedFiles) . "'. For new tests, to pass the test, you can copy files from the processed/ directory into" . " {$expectedDir} after checking that the output is valid. %s ");
}
// Display as one error all sub-failures
if (!empty($this->comparisonFailures)) {
$this->printComparisonFailures();
throw reset($this->comparisonFailures);
}
return count($this->comparisonFailures) == 0;
}
private function handleGeneralSettingsAdmin($view)
{
// Whether to display or not the general settings (cron, beta, smtp)
$view->isGeneralSettingsAdminEnabled = self::isGeneralSettingsAdminEnabled();
if ($view->isGeneralSettingsAdminEnabled) {
$this->displayWarningIfConfigFileNotWritable();
}
$enableBrowserTriggerArchiving = Rules::isBrowserTriggerEnabled();
$todayArchiveTimeToLive = Rules::getTodayArchiveTimeToLive();
$showWarningCron = false;
if (!$enableBrowserTriggerArchiving && $todayArchiveTimeToLive < 3600) {
$showWarningCron = true;
}
$view->showWarningCron = $showWarningCron;
$view->todayArchiveTimeToLive = $todayArchiveTimeToLive;
$view->enableBrowserTriggerArchiving = $enableBrowserTriggerArchiving;
$view->enableBetaReleaseCheck = Config::getInstance()->Debug['allow_upgrades_to_beta'];
$view->mail = Config::getInstance()->mail;
$pluginUpdateCommunication = new UpdateCommunication();
$view->canUpdateCommunication = $pluginUpdateCommunication->canBeEnabled();
$view->enableSendPluginUpdateCommunication = $pluginUpdateCommunication->isEnabled();
}
private function oneClick_Copy()
{
/*
* Make sure the execute bit is set for this shell script
*/
if (!Rules::isBrowserTriggerEnabled()) {
@chmod($this->pathRootExtractedPiwik . '/misc/cron/archive.sh', 0755);
}
$model = new Model();
/*
* Copy all files to PIWIK_INCLUDE_PATH.
* These files are accessed through the dispatcher.
*/
Filesystem::copyRecursive($this->pathRootExtractedPiwik, PIWIK_INCLUDE_PATH);
$model->removeGoneFiles($this->pathRootExtractedPiwik, PIWIK_INCLUDE_PATH);
/*
* These files are visible in the web root and are generally
* served directly by the web server. May be shared.
*/
if (PIWIK_INCLUDE_PATH !== PIWIK_DOCUMENT_ROOT) {
/*
* Copy PHP files that expect to be in the document root
*/
$specialCases = array('/index.php', '/piwik.php', '/js/index.php');
foreach ($specialCases as $file) {
Filesystem::copy($this->pathRootExtractedPiwik . $file, PIWIK_DOCUMENT_ROOT . $file);
}
/*
* Copy the non-PHP files (e.g., images, css, javascript)
*/
Filesystem::copyRecursive($this->pathRootExtractedPiwik, PIWIK_DOCUMENT_ROOT, true);
$model->removeGoneFiles($this->pathRootExtractedPiwik, PIWIK_DOCUMENT_ROOT);
}
/*
* Config files may be user (account) specific
*/
if (PIWIK_INCLUDE_PATH !== PIWIK_USER_PATH) {
Filesystem::copyRecursive($this->pathRootExtractedPiwik . '/config', PIWIK_USER_PATH . '/config');
}
Filesystem::unlinkRecursive($this->pathRootExtractedPiwik, true);
Filesystem::clearPhpCaches();
}
/**
* Runs API tests.
*/
protected function runApiTests($api, $params)
{
$testConfig = new ApiTestConfig($params);
// make sure that the reports we process here are not directly deleted in ArchiveProcessor/PluginsArchiver
// (because we process reports in the past, they would sometimes be invalid, and would have been deleted)
\Piwik\ArchiveProcessor\Rules::disablePurgeOutdatedArchives();
$testName = 'test_' . static::getOutputPrefix();
$this->missingExpectedFiles = array();
$this->comparisonFailures = array();
if ($testConfig->disableArchiving) {
Rules::$archivingDisabledByTests = true;
Config::getInstance()->General['browser_archiving_disabled_enforce'] = 1;
} else {
Rules::$archivingDisabledByTests = false;
Config::getInstance()->General['browser_archiving_disabled_enforce'] = 0;
}
if ($testConfig->language) {
$this->changeLanguage($testConfig->language);
}
$testRequests = new TestRequestCollection($api, $testConfig, $api);
foreach ($testRequests->getRequestUrls() as $apiId => $requestUrl) {
$this->_testApiUrl($testName . $testConfig->testSuffix, $apiId, $requestUrl, $testConfig->compareAgainst, $testConfig->xmlFieldsToRemove, $params);
}
// Restore normal purge behavior
\Piwik\ArchiveProcessor\Rules::enablePurgeOutdatedArchives();
// change the language back to en
if ($this->lastLanguage != 'en') {
$this->changeLanguage('en');
}
if (!empty($this->missingExpectedFiles)) {
$expectedDir = dirname(reset($this->missingExpectedFiles));
$this->fail(" ERROR: Could not find expected API output '" . implode("', '", $this->missingExpectedFiles) . "'. For new tests, to pass the test, you can copy files from the processed/ directory into" . " {$expectedDir} after checking that the output is valid. %s ");
}
// Display as one error all sub-failures
if (!empty($this->comparisonFailures)) {
$this->printComparisonFailures();
throw reset($this->comparisonFailures);
}
return count($this->comparisonFailures) == 0;
}
/**
* Runs API tests.
*/
protected function runApiTests($api, $params)
{
$testConfig = new ApiTestConfig($params);
$testName = 'test_' . static::getOutputPrefix();
$this->missingExpectedFiles = array();
$this->comparisonFailures = array();
if ($testConfig->disableArchiving) {
Rules::$archivingDisabledByTests = true;
Config::getInstance()->General['browser_archiving_disabled_enforce'] = 1;
} else {
Rules::$archivingDisabledByTests = false;
Config::getInstance()->General['browser_archiving_disabled_enforce'] = 0;
}
if ($testConfig->language) {
$this->changeLanguage($testConfig->language);
}
$testRequests = $this->getTestRequestsCollection($api, $testConfig, $api);
foreach ($testRequests->getRequestUrls() as $apiId => $requestUrl) {
$this->_testApiUrl($testName . $testConfig->testSuffix, $apiId, $requestUrl, $testConfig->compareAgainst, $params);
}
// change the language back to en
if ($this->lastLanguage != 'en') {
$this->changeLanguage('en');
}
$this->printApiTestFailures();
return count($this->comparisonFailures) == 0;
}
public function __construct(Parameters $params, ArchiveWriter $archiveWriter, LogAggregator $logAggregator)
{
$this->params = $params;
$this->logAggregator = $logAggregator;
$this->archiveWriter = $archiveWriter;
$this->skipUniqueVisitorsCalculationForMultipleSites = Rules::shouldSkipUniqueVisitorsCalculationForMultipleSites();
}
public function test_ArchiveIsLaunched_WhenForceOnBrowserRequest_IsFalse_AndArchivePhpTriggered()
{
$this->archiveDataForIndividualDays();
Config::getInstance()->General['archiving_range_force_on_browser_request'] = 0;
$_GET['trigger'] = 'archivephp';
Rules::setBrowserTriggerArchiving(false);
$data = $this->initiateArchivingForRange();
$this->assertNotEmpty($data);
$this->assertArchiveTablesAreNotEmpty('2010_03');
}
/**
* Returns the SQL condition used to find successfully completed archives that
* this instance is querying for.
*
* @param array $plugins
* @param Segment $segment
* @param bool $isSkipAggregationOfSubTables
* @return string
*/
private static function getNameCondition(array $plugins, Segment $segment, $isSkipAggregationOfSubTables)
{
// the flags used to tell how the archiving process for a specific archive was completed,
// if it was completed
$doneFlags = Rules::getDoneFlags($plugins, $segment, $isSkipAggregationOfSubTables);
$allDoneFlags = "'" . implode("','", $doneFlags) . "'";
$possibleValues = array(ArchiveWriter::DONE_OK, ArchiveWriter::DONE_OK_TEMPORARY);
if (!Rules::isRequestAuthorizedToArchive()) {
//If request is not authorized to archive then fetch also invalidated archives
$possibleValues[] = ArchiveWriter::DONE_INVALIDATED;
}
// create the SQL to find archives that are DONE
return "((name IN ({$allDoneFlags})) AND " . " (value IN (" . implode(',', $possibleValues) . ")))";
}
private function insertArchiveRow($idSite, $date, $periodLabel)
{
$periodObject = \Piwik\Period\Factory::build($periodLabel, $date);
$dateStart = $periodObject->getDateStart();
$dateEnd = $periodObject->getDateEnd();
$table = ArchiveTableCreator::getNumericTable($dateStart);
$idArchive = (int) Db::fetchOne("SELECT MAX(idarchive) FROM {$table} WHERE name LIKE 'done%'");
$idArchive = $idArchive + 1;
$periodId = Piwik::$idPeriods[$periodLabel];
$doneFlag = 'done';
if ($idArchive % 5 == 1) {
$doneFlag = Rules::getDoneFlagArchiveContainsAllPlugins(self::$segment1);
} else {
if ($idArchive % 5 == 2) {
$doneFlag .= '.VisitsSummary';
} else {
if ($idArchive % 5 == 3) {
$doneFlag = Rules::getDoneFlagArchiveContainsOnePlugin(self::$segment1, 'UserCountry');
} else {
if ($idArchive % 5 == 4) {
$doneFlag = Rules::getDoneFlagArchiveContainsAllPlugins(self::$segment2);
}
}
}
}
$sql = "INSERT INTO {$table} (idarchive, name, idsite, date1, date2, period, ts_archived)\n VALUES ({$idArchive}, 'nb_visits', {$idSite}, '{$dateStart}', '{$dateEnd}', {$periodId}, NOW()),\n ({$idArchive}, '{$doneFlag}', {$idSite}, '{$dateStart}', '{$dateEnd}', {$periodId}, NOW())";
Db::query($sql);
}
private function logArchiveTimeoutInfo()
{
$this->logSection("NOTES");
// Recommend to disable browser archiving when using this script
if (Rules::isBrowserTriggerEnabled()) {
$this->logger->info("- If you execute this script at least once per hour (or more often) in a crontab, you may disable 'Browser trigger archiving' in Piwik UI > Settings > General Settings.");
$this->logger->info(" See the doc at: http://piwik.org/docs/setup-auto-archiving/");
}
$this->logger->info("- Reports for today will be processed at most every " . $this->todayArchiveTimeToLive . " seconds. You can change this value in Piwik UI > Settings > General Settings.");
$this->logger->info("- Reports for the current week/month/year will be refreshed at most every " . $this->processPeriodsMaximumEverySeconds . " seconds.");
// Try and not request older data we know is already archived
if ($this->lastSuccessRunTimestamp !== false) {
$dateLast = time() - $this->lastSuccessRunTimestamp;
$this->logger->info("- Archiving was last executed without error " . $this->formatter->getPrettyTimeFromSeconds($dateLast, true) . " ago");
}
}
/**
* Returns contents of general (global) cache.
* If the cache file tmp/cache/tracker/general.php does not exist yet, create it
*
* @return array
*/
public static function getCacheGeneral()
{
$cache = self::getCache();
$cacheContent = $cache->fetch(self::$cacheIdGeneral);
if (false !== $cacheContent) {
return $cacheContent;
}
Tracker::initCorePiwikInTrackerMode();
$cacheContent = array('isBrowserTriggerEnabled' => Rules::isBrowserTriggerEnabled(), 'lastTrackerCronRun' => Option::get('lastTrackerCronRun'));
/**
* Triggered before the [general tracker cache](/guides/all-about-tracking#the-tracker-cache)
* is saved to disk. This event can be used to add extra content to the cache.
*
* Data that is used during tracking but is expensive to compute/query should be
* cached to keep tracking efficient. One example of such data are options
* that are stored in the piwik_option table. Querying data for each tracking
* request means an extra unnecessary database query for each visitor action. Using
* a cache solves this problem.
*
* **Example**
*
* public function setTrackerCacheGeneral(&$cacheContent)
* {
* $cacheContent['MyPlugin.myCacheKey'] = Option::get('MyPlugin_myOption');
* }
*
* @param array &$cacheContent Array of cached data. Each piece of data must be
* mapped by name.
*/
Piwik::postEvent('Tracker.setTrackerCacheGeneral', array(&$cacheContent));
self::setCacheGeneral($cacheContent);
Common::printDebug("General tracker cache was re-created.");
Tracker::restoreTrackerPlugins();
return $cacheContent;
}
/**
* @param string $archiveTable Prefixed table name
* @param int[] $idSites
* @param string[][] $datesByPeriodType
* @param Segment $segment
* @return \Zend_Db_Statement
* @throws Exception
*/
public function updateArchiveAsInvalidated($archiveTable, $idSites, $datesByPeriodType, Segment $segment = null)
{
$idSites = array_map('intval', $idSites);
$bind = array();
$periodConditions = array();
foreach ($datesByPeriodType as $periodType => $dates) {
$dateConditions = array();
foreach ($dates as $date) {
$dateConditions[] = "(date1 <= ? AND ? <= date2)";
$bind[] = $date;
$bind[] = $date;
}
$dateConditionsSql = implode(" OR ", $dateConditions);
if (empty($periodType) || $periodType == Period\Day::PERIOD_ID) {
// invalidate all periods if no period supplied or period is day
$periodConditions[] = "({$dateConditionsSql})";
} else {
if ($periodType == Period\Range::PERIOD_ID) {
$periodConditions[] = "(period = " . Period\Range::PERIOD_ID . " AND ({$dateConditionsSql}))";
} else {
// for non-day periods, invalidate greater periods, but not range periods
$periodConditions[] = "(period >= " . (int) $periodType . " AND period < " . Period\Range::PERIOD_ID . " AND ({$dateConditionsSql}))";
}
}
}
if ($segment) {
$nameCondition = "name LIKE '" . Rules::getDoneFlagArchiveContainsAllPlugins($segment) . "%'";
} else {
$nameCondition = "name LIKE 'done%'";
}
$sql = "UPDATE {$archiveTable} SET value = " . ArchiveWriter::DONE_INVALIDATED . " WHERE {$nameCondition}\n AND idsite IN (" . implode(", ", $idSites) . ")\n AND (" . implode(" OR ", $periodConditions) . ")";
return Db::query($sql, $bind);
}
public static function isRequestAuthorizedToArchive()
{
return Rules::isBrowserTriggerEnabled() || SettingsServer::isArchivePhpTriggered();
}
/**
* test of validity of an archive, for today's archive with toronto's timezone
* @group Core
*/
public function testInitTodayToronto()
{
if (!SettingsServer::isTimezoneSupportEnabled()) {
$this->markTestSkipped('timezones needs to be supported');
}
$now = time();
$siteTimezone = 'America/Toronto';
$timestamp = Date::factory('now', $siteTimezone)->getTimestamp();
$dateLabel = date('Y-m-d', $timestamp);
Rules::setBrowserTriggerArchiving(true);
$archiveProcessor = $this->_createArchiveProcessor('day', $dateLabel, $siteTimezone);
$archiveProcessor->time = $now;
// we look at anything processed within the time to live range
$dateMinArchived = $now - Rules::getTodayArchiveTimeToLive();
$this->compareTimestamps($dateMinArchived, $archiveProcessor->public_getMinTimeArchiveProcessed());
$this->assertTrue($archiveProcessor->public_isArchiveTemporary());
// when browsers don't trigger archives...
Rules::setBrowserTriggerArchiving(false);
// ...we force ArchiveProcessor to fetch any of the most recent archive
$dateMinArchived = false;
$this->compareTimestamps($dateMinArchived, $archiveProcessor->public_getMinTimeArchiveProcessed());
// this test varies with DST
$this->assertTrue($archiveProcessor->getParams()->getDateStart()->getDateStartUTC() == date('Y-m-d', $timestamp) . ' 04:00:00' || $archiveProcessor->getParams()->getDateStart()->getDateStartUTC() == date('Y-m-d', $timestamp) . ' 05:00:00');
$this->assertTrue($archiveProcessor->getParams()->getDateEnd()->getDateEndUTC() == date('Y-m-d', $timestamp + 86400) . ' 03:59:59' || $archiveProcessor->getParams()->getDateEnd()->getDateEndUTC() == date('Y-m-d', $timestamp + 86400) . ' 04:59:59');
$this->assertTrue($archiveProcessor->public_isArchiveTemporary());
}
/**
* Whether the specified plugin's reports should be archived
* @param string $pluginName
* @return bool
*/
protected function shouldProcessReportsForPlugin($pluginName)
{
if ($this->params->getRequestedPlugin() == $pluginName) {
return true;
}
if (Rules::shouldProcessReportsAllPlugins($this->params->getIdSites(), $this->params->getSegment(), $this->params->getPeriod()->getLabel())) {
return true;
}
if (!\Piwik\Plugin\Manager::getInstance()->isPluginLoaded($this->params->getRequestedPlugin())) {
return true;
}
return false;
}