/** Extract values from ArchiveProcessing */ private function extractArchiveProcessing(Piwik_ArchiveProcessing $archive) { $this->archiveProcessing = $archive; $this->idsite = intval($archive->idsite); $this->period = $archive->period; $this->startDate = $archive->getStartDatetimeUTC(); $this->endDate = $archive->getEndDatetimeUTC(); }
public function setGeneralSettings() { Piwik::checkUserIsSuperUser(); $response = new Piwik_API_ResponseBuilder(Piwik_Common::getRequestVar('format')); try { $this->checkTokenInUrl(); $enableBrowserTriggerArchiving = Piwik_Common::getRequestVar('enableBrowserTriggerArchiving'); $todayArchiveTimeToLive = Piwik_Common::getRequestVar('todayArchiveTimeToLive'); Piwik_ArchiveProcessing::setBrowserTriggerArchiving((bool) $enableBrowserTriggerArchiving); Piwik_ArchiveProcessing::setTodayArchiveTimeToLive($todayArchiveTimeToLive); // Update email settings $mail = Zend_Registry::get('config')->mail; $mail->transport = Piwik_Common::getRequestVar('mailUseSmtp') == '1' ? 'smtp' : ''; $mail->port = Piwik_Common::getRequestVar('mailPort', ''); $mail->host = Piwik_Common::getRequestVar('mailHost', ''); $mail->type = Piwik_Common::getRequestVar('mailType', ''); $mail->username = Piwik_Common::getRequestVar('mailUsername', ''); $mail->password = Piwik_Common::getRequestVar('mailPassword', ''); $mail->encryption = Piwik_Common::getRequestVar('mailEncryption', ''); Zend_Registry::get('config')->mail = $mail->toArray(); $toReturn = $response->getResponse(); } catch (Exception $e) { $toReturn = $response->getResponseException($e); } echo $toReturn; }
public function setGeneralSettings() { Piwik::checkUserIsSuperUser(); $response = new Piwik_API_ResponseBuilder(Piwik_Common::getRequestVar('format')); try { $this->checkTokenInUrl(); $enableBrowserTriggerArchiving = Piwik_Common::getRequestVar('enableBrowserTriggerArchiving'); $todayArchiveTimeToLive = Piwik_Common::getRequestVar('todayArchiveTimeToLive'); Piwik_ArchiveProcessing::setBrowserTriggerArchiving((bool) $enableBrowserTriggerArchiving); Piwik_ArchiveProcessing::setTodayArchiveTimeToLive($todayArchiveTimeToLive); // Update email settings $mail = array(); $mail['transport'] = Piwik_Common::getRequestVar('mailUseSmtp') == '1' ? 'smtp' : ''; $mail['port'] = Piwik_Common::getRequestVar('mailPort', ''); $mail['host'] = Piwik_Common::unsanitizeInputValue(Piwik_Common::getRequestVar('mailHost', '')); $mail['type'] = Piwik_Common::getRequestVar('mailType', ''); $mail['username'] = Piwik_Common::unsanitizeInputValue(Piwik_Common::getRequestVar('mailUsername', '')); $mail['password'] = Piwik_Common::unsanitizeInputValue(Piwik_Common::getRequestVar('mailPassword', '')); $mail['encryption'] = Piwik_Common::getRequestVar('mailEncryption', ''); Piwik_Config::getInstance()->mail = $mail; // update branding settings Piwik_Config::getInstance()->branding['use_custom_logo'] = Piwik_Common::getRequestVar('useCustomLogo', '0'); $toReturn = $response->getResponse(); } catch (Exception $e) { $toReturn = $response->getResponseException($e); } echo $toReturn; }
/** * Called at the end of the archiving process. * Does some cleaning job in the database. * * @return void */ protected function postCompute() { parent::postCompute(); // we delete out of date records // = archives that for day N computed on day N (means they are only partial) $blobTable = $this->tableArchiveBlob->getTableName(); $numericTable = $this->tableArchiveNumeric->getTableName(); $query = "\tDELETE \n\t\t\t\t\tFROM %s\n\t\t\t\t\tWHERE period = ? \n\t\t\t\t\t\tAND date1 = DATE(ts_archived)\n\t\t\t\t\t\tAND DATE(ts_archived) <> CURRENT_DATE()\n\t\t\t\t\t"; Zend_Registry::get('db')->query(sprintf($query, $blobTable), $this->periodId); Zend_Registry::get('db')->query(sprintf($query, $numericTable), $this->periodId); }
/** * Fetches all blob fields name_* at once for the current archive for performance reasons. * * @return void */ public function preFetchBlob($name) { if (!$this->isThereSomeVisits) { return false; } $tableBlob = $this->archiveProcessing->getTableArchiveBlobName(); $db = Zend_Registry::get('db'); $query = $db->query("SELECT value, name\n\t\t\t\t\t\t\t\tFROM {$tableBlob}\n\t\t\t\t\t\t\t\tWHERE idarchive = ?\n\t\t\t\t\t\t\t\t\tAND name LIKE '{$name}%'", array($this->idArchive)); while ($row = $query->fetch()) { $value = $row['value']; $name = $row['name']; $this->blobCached[$name] = gzuncompress($value); } }
/** * Build DataTable from array and archive it * @return id of the datatable */ private function archiveDataArray($keyword, &$data, $addSearchTermMetaData = false, $addUrlMetaData = false) { $dataTable = new Piwik_DataTable(); foreach ($data as &$row) { $rowData = array(Piwik_DataTable_Row::COLUMNS => $row); if ($addSearchTermMetaData) { $rowData[Piwik_DataTable_Row::METADATA] = array('idSearch' => $row[$addSearchTermMetaData], 'searchTerm' => $row[self::SEARCH_TERM]); } if ($addUrlMetaData) { $rowData[Piwik_DataTable_Row::METADATA]['url'] = $row[self::URL]; } $dataTable->addRow(new Piwik_SiteSearch_ExtendedDataTableRow($rowData)); } $id = $dataTable->getId(); $name = 'SiteSearch_' . $keyword; $this->archiveProcessing->insertBlobRecord($name, $dataTable->getSerialized()); destroy($dataTable); return $id; }
/** * Returns datatable describing the number of visits for each day of the week. * * @param string $idSite The site ID. Cannot refer to multiple sites. * @param string $period The period type: day, week, year, range... * @param string $date The start date of the period. Cannot refer to multiple dates. * @param string $segment The segment. * @return Piwik_DataTable */ public function getByDayOfWeek($idSite, $period, $date, $segment = false) { Piwik::checkUserHasViewAccess($idSite); // disabled for multiple sites/dates if (Piwik_Archive::isMultipleSites($idSite)) { throw new Exception("VisitTime.getByDayOfWeek does not support multiple sites."); } if (Piwik_Archive::isMultiplePeriod($date, $period)) { throw new Exception("VisitTime.getByDayOfWeek does not support multiple dates."); } // metrics to query $metrics = Piwik_ArchiveProcessing::getCoreMetrics(); // get metric data for every day within the supplied period $oSite = new Piwik_Site($idSite); $oPeriod = Piwik_Archive::makePeriodFromQueryParams($oSite, $period, $date); $dateRange = $oPeriod->getDateStart()->toString() . ',' . $oPeriod->getDateEnd()->toString(); $archive = Piwik_Archive::build($idSite, 'day', $dateRange, $segment); $dataTable = $archive->getDataTableFromNumeric($metrics)->mergeChildren(); // if there's no data for this report, don't bother w/ anything else if ($dataTable->getRowsCount() == 0) { return $dataTable; } // group by the day of the week (see below for dayOfWeekFromDate function) $dataTable->filter('GroupBy', array('label', 'Piwik_VisitTime_dayOfWeekFromDate')); // create new datatable w/ empty rows, then add calculated datatable $rows = array(); foreach (array(1, 2, 3, 4, 5, 6, 7) as $day) { $rows[] = array('label' => $day, 'nb_visits' => 0); } $result = new Piwik_DataTable(); $result->addRowsFromSimpleArray($rows); $result->addDataTable($dataTable); // set day of week integer as metadata $result->filter('ColumnCallbackAddMetadata', array('label', 'day_of_week')); // translate labels $result->filter('ColumnCallbackReplace', array('label', 'Piwik_VisitTime_translateDayOfWeek')); // set datatable metadata for period start & finish $result->setMetadata('date_start', $oPeriod->getDateStart()); $result->setMetadata('date_end', $oPeriod->getDateEnd()); return $result; }
public function setUp() { $dumpPath = PIWIK_INCLUDE_PATH . '/tmp/logdump.sql.gz'; $deflatedDumpPath = PIWIK_INCLUDE_PATH . '/tmp/logdump.sql'; $bufferSize = 1024 * 1024; // drop all tables Piwik::dropTables(); // download data dump $dump = fopen(self::$dumpUrl, 'rb'); $outfile = fopen($dumpPath, 'wb'); $bytesRead = 0; while (!feof($dump)) { fwrite($outfile, fread($dump, $bufferSize), $bufferSize); $bytesRead += $bufferSize; } fclose($dump); fclose($outfile); if ($bytesRead <= 40 * 1024 * 1024) { throw new Exception("Could not download sql dump!"); } // unzip the dump exec("gunzip -c \"" . $dumpPath . "\" > \"{$deflatedDumpPath}\"", $output, $return); if ($return !== 0) { throw new Exception("gunzip failed: " . implode("\n", $output)); } // load the data into the correct database $user = Piwik_Config::getInstance()->database['username']; $password = Piwik_Config::getInstance()->database['password']; $dbName = Piwik_Config::getInstance()->database['dbname']; Piwik_Config::getInstance()->database['tables_prefix'] = 'piwik_'; Piwik_Common::$cachedTablePrefix = null; exec("mysql -u \"{$user}\" \"--password={$password}\" {$dbName} < \"" . $deflatedDumpPath . "\" 2>&1", $output, $return); if ($return !== 0) { throw new Exception("Failed to load sql dump: " . implode("\n", $output)); } // make sure archiving will be called Piwik_ArchiveProcessing::setBrowserTriggerArchiving(true); }
private function oneClick_Copy() { /* * Make sure the execute bit is set for this shell script */ if (!Piwik_ArchiveProcessing::isBrowserTriggerArchivingEnabled()) { @chmod($this->pathRootExtractedPiwik . '/misc/cron/archive.sh', 0755); } /* * Copy all files to PIWIK_INCLUDE_PATH. * These files are accessed through the dispatcher. */ Piwik::copyRecursive($this->pathRootExtractedPiwik, PIWIK_INCLUDE_PATH); /* * These files are visible in the web root and are generally * served directly by the web server. May be shared. */ if (PIWIK_INCLUDE_PATH !== PIWIK_DOCUMENT_ROOT) { /* * Copy PHP files that expect to be in the document root */ $specialCases = array('/index.php', '/piwik.php', '/js/index.php'); foreach ($specialCases as $file) { Piwik::copy($this->pathRootExtractedPiwik . $file, PIWIK_DOCUMENT_ROOT . $file); } /* * Copy the non-PHP files (e.g., images, css, javascript) */ Piwik::copyRecursive($this->pathRootExtractedPiwik, PIWIK_DOCUMENT_ROOT, true); } /* * Config files may be user (account) specific */ if (PIWIK_INCLUDE_PATH !== PIWIK_USER_PATH) { Piwik::copyRecursive($this->pathRootExtractedPiwik . '/config', PIWIK_USER_PATH . '/config'); } Piwik::unlinkRecursive($this->pathRootExtractedPiwik, true); }
/** * @param $select * @param $from * @param $where * @param $orderBy * @param $groupBy * @param $sprintfField * @param Piwik_ArchiveProcessing $archiveProcessing * @return int */ protected function archiveDayQueryProcess($select, $from, $where, $orderBy, $groupBy, $sprintfField, $archiveProcessing) { // idaction field needs to be set in select clause before calling getSelectQuery(). // if a complex segmentation join is needed, the field needs to be propagated // to the outer select. therefore, $segment needs to know about it. $select = sprintf($select, $sprintfField); // get query with segmentation $bind = array(); $orderBy = false; $query = $archiveProcessing->getSegment()->getSelectQuery($select, $from, $where, $bind, $orderBy, $groupBy); // replace the rest of the %s $querySql = str_replace("%s", $sprintfField, $query['sql']); // extend bindings $bind = array_merge(array($archiveProcessing->getStartDatetimeUTC(), $archiveProcessing->getEndDatetimeUTC(), $archiveProcessing->idsite), $query['bind']); // get result $resultSet = $archiveProcessing->db->query($querySql, $bind); $modified = $this->updateActionsTableWithRowQuery($resultSet, $sprintfField); return $modified; }
protected function getRequestedReport() { // Core metrics are always processed in Core, for the requested date/period/segment if(in_array($this->requestedReport, Piwik_ArchiveProcessing::getCoreMetrics()) || $this->requestedReport == 'max_actions') { return 'VisitsSummary_CoreMetrics'; } // VisitFrequency metrics don't follow the same naming convention (HACK) if(strpos($this->requestedReport, '_returning') > 0 // ignore Goal_visitor_returning_1_1_nb_conversions && strpos($this->requestedReport, 'Goal_') === false) { return 'VisitFrequency_Metrics'; } // Goal_* metrics are processed by the Goals plugin (HACK) if(strpos($this->requestedReport, 'Goal_') === 0) { return 'Goals_Metrics'; } return $this->requestedReport; }
function __construct() { parent::__construct(); $this->db = Zend_Registry::get('db'); $this->debugAlwaysArchive = Zend_Registry::get('config')->Debug->always_archive_data_day; }
/** * Called at the end of the archiving process. * Does some cleaning job in the database. * * @return void */ protected function postCompute() { parent::postCompute(); // we delete records that are now out of date // in the case of a period we delete archives that were archived before the end of the period // and only if they are at least 1 day old (so we don't delete archives computed today that may be stil valid) $blobTable = $this->tableArchiveBlob->getTableName(); $numericTable = $this->tableArchiveNumeric->getTableName(); $query = "\tDELETE \n\t\t\t\t\tFROM %s\n\t\t\t\t\tWHERE period > ? \n\t\t\t\t\t\tAND DATE(ts_archived) <= date2\n\t\t\t\t\t\tAND date(ts_archived) < date_sub(CURRENT_DATE(), INTERVAL 1 DAY)\n\t\t\t\t\t"; Zend_Registry::get('db')->query(sprintf($query, $blobTable), Piwik::$idPeriods['day']); Zend_Registry::get('db')->query(sprintf($query, $numericTable), Piwik::$idPeriods['day']); }
protected function initStateFromParameters() { // Detect parameters $reset = $this->isParameterSet("force-all-periods", $valuePossible = true); $forceAll = $this->isParameterSet("force-all-websites"); $forceTimeoutPeriod = $this->isParameterSet("force-timeout-for-periods", $valuePossible = true); if (!empty($forceTimeoutPeriod) && $forceTimeoutPeriod !== true) { // Ensure the cache for periods is at least as high as cache for today $todayTTL = Piwik_ArchiveProcessing::getTodayArchiveTimeToLive(); if ($forceTimeoutPeriod < $todayTTL) { $this->log("WARNING: Automatically increasing --force-timeout-for-periods from {$forceTimeoutPeriod} to " . $todayTTL . " to match the cache timeout for Today's report specified in Piwik UI > Settings > General Settings"); $forceTimeoutPeriod = $todayTTL; } $this->processPeriodsMaximumEverySeconds = $forceTimeoutPeriod; } // Recommend to disable browser archiving when using this script if (Piwik_ArchiveProcessing::isBrowserTriggerArchivingEnabled()) { $this->log("NOTE: if you execute this script at least once per hour (or more often) in a crontab, you may disable 'Browser trigger archiving' in Piwik UI > Settings > General Settings. "); $this->log(" see doc at: http://piwik.org/docs/setup-auto-archiving/"); } if ($reset) { $this->log("--force-all-periods was detected: the script will run as if it was its first run, and will trigger archiving for all periods."); $this->shouldResetState = true; if (!$forceAll && is_numeric($reset) && $reset > 0) { $this->firstRunActiveWebsitesWithTraffic = (int) $reset; } } if ($forceAll) { $this->log("--force-all-websites was detected: the script will archive all websites and all periods sequentially"); $this->shouldArchiveAllWebsites = true; } $this->timeLastCompleted = Piwik_GetOption(self::OPTION_ARCHIVING_FINISHED_TS); if ($this->shouldResetState) { $this->timeLastCompleted = false; } }
/** * Returns true if Piwik can launch the archiving process for this archive, * false if otherwise. * * @return bool */ public function isArchivingDisabled() { return Piwik_ArchiveProcessing::isArchivingDisabledFor($this->segment, $this->period); }
/** * Called at the end of the archiving process. * Does some cleaning job in the database. */ protected function postCompute() { parent::postCompute(); $blobTable = $this->tableArchiveBlob->getTableName(); $numericTable = $this->tableArchiveNumeric->getTableName(); $key = 'lastPurge_' . $blobTable; $timestamp = Piwik_GetOption($key); if (!$timestamp || $timestamp < time() - 86400) { Piwik_SetOption($key, time()); // we delete out of date daily archives from table, maximum once per day // we only delete archives processed that are older than 1 day, to not delete archives we just processed $yesterday = Piwik_Date::factory('yesterday')->getDateTime(); $result = Piwik_FetchAll("\n\t\t\t\t\t\t\tSELECT idarchive\n\t\t\t\t\t\t\tFROM {$numericTable}\n\t\t\t\t\t\t\tWHERE name LIKE 'done%'\n\t\t\t\t\t\t\t\tAND value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . "\n\t\t\t\t\t\t\t\tAND ts_archived < ?", array($yesterday)); $idArchivesToDelete = array(); if (!empty($result)) { foreach ($result as $row) { $idArchivesToDelete[] = $row['idarchive']; } $query = "DELETE \n \t\t\t\t\t\tFROM %s\n \t\t\t\t\t\tWHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ")\n \t\t\t\t\t\t"; Piwik_Query(sprintf($query, $blobTable)); Piwik_Query(sprintf($query, $numericTable)); } Piwik::log("Purging temporary archives: done [ purged archives older than {$yesterday} from {$blobTable} and {$numericTable} ] [Deleted IDs: " . implode(',', $idArchivesToDelete) . "]"); // Deleting "Custom Date Range" reports after 1 day, since they can be re-processed // and would take up unecessary space $query = "DELETE \n \t\t\t\t\tFROM %s\n \t\t\t\t\tWHERE period = ?\n \t\t\t\t\t\tAND ts_archived < ?"; $bind = array(Piwik::$idPeriods['range'], $yesterday); Piwik_Query(sprintf($query, $blobTable), $bind); Piwik_Query(sprintf($query, $numericTable), $bind); } else { Piwik::log("Purging temporary archives: skipped."); } if (!isset($this->archives)) { return; } foreach ($this->archives as $archive) { destroy($archive); } $this->archives = array(); }
/** * Runs API tests. */ protected function runApiTests($api, $params) { $testName = 'test_' . $this->getOutputPrefix(); $this->missingExpectedFiles = array(); $this->comparisonFailures = array(); $this->_setCallableApi($api); if (isset($params['disableArchiving']) && $params['disableArchiving'] === true) { Piwik_ArchiveProcessing::$forceDisableArchiving = true; } else { Piwik_ArchiveProcessing::$forceDisableArchiving = false; } if (isset($params['language'])) { $this->changeLanguage($params['language']); } $testSuffix = isset($params['testSuffix']) ? $params['testSuffix'] : ''; $requestUrls = $this->_generateApiUrls(isset($params['format']) ? $params['format'] : 'xml', isset($params['idSite']) ? $params['idSite'] : false, isset($params['date']) ? $params['date'] : false, isset($params['periods']) ? $params['periods'] : false, isset($params['setDateLastN']) ? $params['setDateLastN'] : false, isset($params['language']) ? $params['language'] : false, isset($params['segment']) ? $params['segment'] : false, isset($params['visitorId']) ? $params['visitorId'] : false, isset($params['abandonedCarts']) ? $params['abandonedCarts'] : false, isset($params['idGoal']) ? $params['idGoal'] : false, isset($params['apiModule']) ? $params['apiModule'] : false, isset($params['apiAction']) ? $params['apiAction'] : false, isset($params['otherRequestParameters']) ? $params['otherRequestParameters'] : array(), isset($params['supertableApi']) ? $params['supertableApi'] : false, isset($params['fileExtension']) ? $params['fileExtension'] : false); foreach ($requestUrls as $apiId => $requestUrl) { $this->_testApiUrl($testName . $testSuffix, $apiId, $requestUrl); } // change the language back to en if ($this->lastLanguage != 'en') { $this->changeLanguage('en'); } if (!empty($this->missingExpectedFiles)) { $expectedDir = dirname(reset($this->missingExpectedFiles)); $this->markTestIncomplete(" ERROR: Could not find expected API output '" . implode("', '", $this->missingExpectedFiles) . "'. For new tests, to pass the test, you can copy files from the processed/ directory into" . " {$expectedDir} after checking that the output is valid. %s "); } if (!empty($this->comparisonFailures)) { throw reset($this->comparisonFailures); } }
/** * Records the daily stats (numeric or datatable blob) into the archive tables. * * @param Piwik_ArchiveProcessing $archiveProcessing * @return void */ protected function archiveDayRecordInDatabase($archiveProcessing) { $numericRecords = array('Referers_distinctSearchEngines' => count($this->interestBySearchEngineAndKeyword), 'Referers_distinctKeywords' => count($this->interestByKeywordAndSearchEngine), 'Referers_distinctCampaigns' => count($this->interestByCampaign), 'Referers_distinctWebsites' => count($this->interestByWebsite), 'Referers_distinctWebsitesUrls' => count($this->distinctUrls)); foreach ($numericRecords as $name => $value) { $archiveProcessing->insertNumericRecord($name, $value); } $dataTable = $archiveProcessing->getDataTableSerialized($this->interestByType); $archiveProcessing->insertBlobRecord('Referers_type', $dataTable); destroy($dataTable); $blobRecords = array('Referers_keywordBySearchEngine' => $archiveProcessing->getDataTableWithSubtablesFromArraysIndexedByLabel($this->interestBySearchEngineAndKeyword, $this->interestBySearchEngine), 'Referers_searchEngineByKeyword' => $archiveProcessing->getDataTableWithSubtablesFromArraysIndexedByLabel($this->interestByKeywordAndSearchEngine, $this->interestByKeyword), 'Referers_keywordByCampaign' => $archiveProcessing->getDataTableWithSubtablesFromArraysIndexedByLabel($this->interestByCampaignAndKeyword, $this->interestByCampaign), 'Referers_urlByWebsite' => $archiveProcessing->getDataTableWithSubtablesFromArraysIndexedByLabel($this->interestByWebsiteAndUrl, $this->interestByWebsite)); foreach ($blobRecords as $recordName => $table) { $blob = $table->getSerialized($this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation); $archiveProcessing->insertBlobRecord($recordName, $blob); destroy($table); } }
/** * Called at the end of the archiving process. * Does some cleaning job in the database. * * @return void */ protected function postCompute() { parent::postCompute(); $blobTable = $this->tableArchiveBlob->getTableName(); $numericTable = $this->tableArchiveNumeric->getTableName(); // delete out of date records maximum once per day (DELETE request is costly) $key = 'lastPurge_' . $blobTable; $timestamp = Piwik_GetOption($key); if (!$timestamp || $timestamp < time() - 86400) { // we delete out of date daily archives from table, maximum once per day // those for day N that were processed on day N (means the archives are only partial as the day wasn't finished) $query = "/* SHARDING_ID_SITE = " . $this->idsite . " */ \tDELETE \n\t\t\t\t\t\tFROM %s\n\t\t\t\t\t\tWHERE period = ? \n\t\t\t\t\t\t\tAND date1 = DATE(ts_archived)\n\t\t\t\t\t\t\tAND DATE(ts_archived) <> CURRENT_DATE()\n\t\t\t\t\t\t"; Piwik_Query(sprintf($query, $blobTable), Piwik::$idPeriods['day']); Piwik_Query(sprintf($query, $numericTable), Piwik::$idPeriods['day']); // we delete out of date Period records (week/month/etc) // we delete archives that were archived before the end of the period // and only if they are at least 1 day old (so we don't delete archives computed today that may be stil valid) $query = "\tDELETE \n\t\t\t\t\t\tFROM %s\n\t\t\t\t\t\tWHERE period > ? \n\t\t\t\t\t\t\tAND DATE(ts_archived) <= date2\n\t\t\t\t\t\t\tAND date(ts_archived) < date_sub(CURRENT_DATE(), INTERVAL 1 DAY)\n\t\t\t\t\t\t"; Piwik_Query(sprintf($query, $blobTable), Piwik::$idPeriods['day']); Piwik_Query(sprintf($query, $numericTable), Piwik::$idPeriods['day']); Piwik_SetOption($key, time()); } foreach ($this->archives as $archive) { destroy($archive); } $this->archives = array(); }
/** * Called at the end of the archiving process. * Does some cleaning job in the database. */ protected function postCompute() { parent::postCompute(); $blobTable = $this->tableArchiveBlob->getTableName(); $numericTable = $this->tableArchiveNumeric->getTableName(); $key = 'lastPurge_' . $blobTable; $timestamp = Piwik_GetOption($key); // we shall purge temporary archives after their timeout is finished, plus an extra 2 hours // in case archiving is disabled and is late to run, we give it this extra time to run and re-process more recent records $temporaryArchivingTimeout = self::getTodayArchiveTimeToLive(); $purgeEveryNSeconds = $temporaryArchivingTimeout + 2 * 3600; // we only delete archives if we are able to process them, otherwise, the browser might process reports // when &segment= is specified (or custom date range) and would below, delete temporary archives that the // browser is not able to process until next cron run (which could be more than 1 hour away) if ($this->isRequestAuthorizedToArchive() && (!$timestamp || $timestamp < time() - $purgeEveryNSeconds)) { Piwik_SetOption($key, time()); $purgeArchivesOlderThan = Piwik_Date::factory(time() - $purgeEveryNSeconds)->getDateTime(); $result = Piwik_FetchAll("\n\t\t\t\t\t\t\tSELECT idarchive\n\t\t\t\t\t\t\tFROM {$numericTable}\n\t\t\t\t\t\t\tWHERE name LIKE 'done%'\n\t\t\t\t\t\t\t\tAND value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . "\n\t\t\t\t\t\t\t\tAND ts_archived < ?", array($purgeArchivesOlderThan)); $idArchivesToDelete = array(); if (!empty($result)) { foreach ($result as $row) { $idArchivesToDelete[] = $row['idarchive']; } $query = "DELETE \n \t\t\t\t\t\tFROM %s\n \t\t\t\t\t\tWHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ")\n \t\t\t\t\t\t"; Piwik_Query(sprintf($query, $blobTable)); Piwik_Query(sprintf($query, $numericTable)); } Piwik::log("Purging temporary archives: done [ purged archives older than {$purgeArchivesOlderThan} from {$blobTable} and {$numericTable} ] [Deleted IDs: " . implode(',', $idArchivesToDelete) . "]"); // Deleting "Custom Date Range" reports after 1 day, since they can be re-processed // and would take up unecessary space $yesterday = Piwik_Date::factory('yesterday')->getDateTime(); $query = "DELETE \n \t\t\t\t\tFROM %s\n \t\t\t\t\tWHERE period = ?\n \t\t\t\t\t\tAND ts_archived < ?"; $bind = array(Piwik::$idPeriods['range'], $yesterday); Piwik::log("Purging Custom Range archives: done [ purged archives older than {$yesterday} from {$blobTable} and {$numericTable} ]"); Piwik_Query(sprintf($query, $blobTable), $bind); Piwik_Query(sprintf($query, $numericTable), $bind); // these tables will be OPTIMIZEd daily in a scheduled task, to claim lost space } else { Piwik::log("Purging temporary archives: skipped."); } if (!isset($this->archives)) { return; } foreach ($this->archives as $archive) { destroy($archive); } $this->archives = array(); }
/** * Gets the name of the database table that holds the numeric archive data for * this archive. * * @return string */ private function getNumericTableName() { if (is_null($this->tableName)) { $table = Piwik_ArchiveProcessing::makeNumericArchiveTable($this->getFirstArchive()->getPeriod()); $this->tableName = $table->getTableName(); } return $this->tableName; }
/** * Runs API tests. */ private function runApiTests() { $apiToTest = $this->getApiToTest(); $testName = 'test_' . $this->getOutputPrefix(); foreach ($apiToTest as $test) { list($api, $params) = $test; if ($api == 'all') { $this->setApiToCall(array()); $this->setApiNotToCall(Test_Integration::$defaultApiNotToCall); } else { if (!is_array($api)) { $api = array($api); } $this->setApiToCall($api); $this->setApiNotToCall(array()); } if (isset($params['disableArchiving']) && $params['disableArchiving'] === true) { Piwik_ArchiveProcessing::$forceDisableArchiving = true; } else { Piwik_ArchiveProcessing::$forceDisableArchiving = false; } if (isset($params['language'])) { $this->changeLanguage($params['language']); } $testSuffix = isset($params['testSuffix']) ? $params['testSuffix'] : ''; $this->callGetApiCompareOutput($testName . $testSuffix, isset($params['format']) ? $params['format'] : 'xml', isset($params['idSite']) ? $params['idSite'] : false, isset($params['date']) ? $params['date'] : false, isset($params['periods']) ? $params['periods'] : false, isset($params['setDateLastN']) ? $params['setDateLastN'] : false, isset($params['language']) ? $params['language'] : false, isset($params['segment']) ? $params['segment'] : false, isset($params['visitorId']) ? $params['visitorId'] : false, isset($params['abandonedCarts']) ? $params['abandonedCarts'] : false, isset($params['idGoal']) ? $params['idGoal'] : false, isset($params['apiModule']) ? $params['apiModule'] : false, isset($params['apiAction']) ? $params['apiAction'] : false, isset($params['otherRequestParameters']) ? $params['otherRequestParameters'] : array()); // change the language back to en if ($this->lastLanguage != 'en') { $this->changeLanguage('en'); } } }
/** * Runs API tests. */ protected function runApiTests($api, $params) { $testName = 'test_' . $this->getOutputPrefix(); $this->_setCallableApi($api); if (isset($params['disableArchiving']) && $params['disableArchiving'] === true) { Piwik_ArchiveProcessing::$forceDisableArchiving = true; } else { Piwik_ArchiveProcessing::$forceDisableArchiving = false; } if (isset($params['language'])) { $this->changeLanguage($params['language']); } $testSuffix = isset($params['testSuffix']) ? $params['testSuffix'] : ''; $requestUrls = $this->_generateApiUrls(isset($params['format']) ? $params['format'] : 'xml', isset($params['idSite']) ? $params['idSite'] : false, isset($params['date']) ? $params['date'] : false, isset($params['periods']) ? $params['periods'] : false, isset($params['setDateLastN']) ? $params['setDateLastN'] : false, isset($params['language']) ? $params['language'] : false, isset($params['segment']) ? $params['segment'] : false, isset($params['visitorId']) ? $params['visitorId'] : false, isset($params['abandonedCarts']) ? $params['abandonedCarts'] : false, isset($params['idGoal']) ? $params['idGoal'] : false, isset($params['apiModule']) ? $params['apiModule'] : false, isset($params['apiAction']) ? $params['apiAction'] : false, isset($params['otherRequestParameters']) ? $params['otherRequestParameters'] : array(), isset($params['supertableApi']) ? $params['supertableApi'] : false, isset($params['fileExtension']) ? $params['fileExtension'] : false); foreach ($requestUrls as $apiId => $requestUrl) { $this->_testApiUrl($testName . $testSuffix, $apiId, $requestUrl); } // change the language back to en if ($this->lastLanguage != 'en') { $this->changeLanguage('en'); } }
/** * Returns contents of general (global) cache * * @return array */ protected static function getCacheGeneral() { $cache = self::getTrackerCache(); $cacheId = 'general'; $expectedRows = 3; if (($cacheContent = $cache->get($cacheId)) !== false && count($cacheContent) == $expectedRows) { return $cacheContent; } self::initCorePiwikInTrackerMode(); $cacheContent = array('isBrowserTriggerArchivingEnabled' => Piwik_ArchiveProcessing::isBrowserTriggerArchivingEnabled(), 'lastTrackerCronRun' => Piwik_GetOption('lastTrackerCronRun'), 'currentLocationProviderId' => Piwik_UserCountry_LocationProvider::getCurrentProviderId()); return $cacheContent; }
/** * @param Piwik_ArchiveProcessing $archiveProcessing * @return void */ protected function archiveDayRecordInDatabase($archiveProcessing) { $recordName = 'CustomVariables_valueByName'; $table = $archiveProcessing->getDataTableWithSubtablesFromArraysIndexedByLabel($this->interestByCustomVariablesAndValue, $this->interestByCustomVariables); $blob = $table->getSerialized($this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable); $archiveProcessing->insertBlobRecord($recordName, $blob); destroy($table); }
/** * test of validity of an archive, for today's archive with toronto's timezone * @group Core * @group ArchiveProcessing */ public function testInitTodayToronto() { if (!Piwik::isTimezoneSupportEnabled()) { $this->markTestSkipped('timezones needs to be supported'); } $now = time(); $siteTimezone = 'America/Toronto'; $timestamp = Piwik_Date::factory('now', $siteTimezone)->getTimestamp(); $dateLabel = date('Y-m-d', $timestamp); Piwik_ArchiveProcessing::setBrowserTriggerArchiving(true); $archiveProcessing = $this->_createArchiveProcessing('day', $dateLabel, $siteTimezone); $archiveProcessing->time = $now; // we look at anything processed within the time to live range $dateMinArchived = $now - Piwik_ArchiveProcessing::getTodayArchiveTimeToLive(); $this->assertEquals($archiveProcessing->getMinTimeArchivedProcessed(), $dateMinArchived); $this->assertTrue($archiveProcessing->isArchiveTemporary()); // when browsers don't trigger archives, we force ArchiveProcessing // to fetch any of the most recent archive Piwik_ArchiveProcessing::setBrowserTriggerArchiving(false); // see isArchivingDisabled() // Running in CLI doesn't impact the time to live today's archive we are loading // From CLI, we will not return data that is 'stale' if (!Piwik_Common::isPhpCliMode()) { $dateMinArchived = 0; } $this->assertEquals($archiveProcessing->getMinTimeArchivedProcessed(), $dateMinArchived); // this test varies with DST $this->assertTrue($archiveProcessing->getStartDatetimeUTC() == date('Y-m-d', $timestamp) . ' 04:00:00' || $archiveProcessing->getStartDatetimeUTC() == date('Y-m-d', $timestamp) . ' 05:00:00'); $this->assertTrue($archiveProcessing->getEndDatetimeUTC() == date('Y-m-d', $timestamp + 86400) . ' 03:59:59' || $archiveProcessing->getEndDatetimeUTC() == date('Y-m-d', $timestamp + 86400) . ' 04:59:59'); $this->assertTrue($archiveProcessing->isArchiveTemporary()); }
/** * Constructor */ function __construct() { parent::__construct(); $this->db = Zend_Registry::get('db'); }
/** * Called at the end of the archiving process. * Does some cleaning job in the database. */ protected function postCompute() { parent::postCompute(); $numericTable = $this->tableArchiveNumeric->getTableName(); self::doPurgeOutdatedArchives($numericTable, $this->isArchiveTemporary()); if (!isset($this->archives)) { return; } foreach ($this->archives as $archive) { destroy($archive); } $this->archives = array(); }
/** * @param $select * @param $from * @param $where * @param $orderBy * @param $groupBy * @param $sprintfField * @param Piwik_ArchiveProcessing $archiveProcessing * @param Piwik_RankingQuery|false $rankingQuery * @return int */ protected function archiveDayQueryProcess($select, $from, $where, $orderBy, $groupBy, $sprintfField, $archiveProcessing, $rankingQuery = false) { // idaction field needs to be set in select clause before calling getSelectQuery(). // if a complex segmentation join is needed, the field needs to be propagated // to the outer select. therefore, $segment needs to know about it. $select = sprintf($select, $sprintfField); $bind = array(); // get query with segmentation $query = $archiveProcessing->getSegment()->getSelectQuery($select, $from, $where, $bind, $orderBy, $groupBy); // extend bindings $bind = array_merge(array($archiveProcessing->getStartDatetimeUTC(), $archiveProcessing->getEndDatetimeUTC(), $archiveProcessing->idsite), $query['bind']); // replace the rest of the %s $querySql = str_replace("%s", $sprintfField, $query['sql']); // apply ranking query if ($rankingQuery) { $querySql = $rankingQuery->generateQuery($querySql); } // echo '<pre>';var_dump($querySql); // var_dump($bind); // get result $resultSet = $archiveProcessing->db->query($querySql, $bind); $modified = Piwik_Actions_ArchivingHelper::updateActionsTableWithRowQuery($resultSet, $sprintfField, $this->actionsTablesByType); return $modified; }