/** Extract values from ArchiveProcessing */
private function extractArchiveProcessing(Piwik_ArchiveProcessing $archive)
{
$this->archiveProcessing = $archive;
$this->idsite = intval($archive->idsite);
$this->period = $archive->period;
$this->startDate = $archive->getStartDatetimeUTC();
$this->endDate = $archive->getEndDatetimeUTC();
}
public function setGeneralSettings()
{
Piwik::checkUserIsSuperUser();
$response = new Piwik_API_ResponseBuilder(Piwik_Common::getRequestVar('format'));
try {
$this->checkTokenInUrl();
$enableBrowserTriggerArchiving = Piwik_Common::getRequestVar('enableBrowserTriggerArchiving');
$todayArchiveTimeToLive = Piwik_Common::getRequestVar('todayArchiveTimeToLive');
Piwik_ArchiveProcessing::setBrowserTriggerArchiving((bool) $enableBrowserTriggerArchiving);
Piwik_ArchiveProcessing::setTodayArchiveTimeToLive($todayArchiveTimeToLive);
// Update email settings
$mail = Zend_Registry::get('config')->mail;
$mail->transport = Piwik_Common::getRequestVar('mailUseSmtp') == '1' ? 'smtp' : '';
$mail->port = Piwik_Common::getRequestVar('mailPort', '');
$mail->host = Piwik_Common::getRequestVar('mailHost', '');
$mail->type = Piwik_Common::getRequestVar('mailType', '');
$mail->username = Piwik_Common::getRequestVar('mailUsername', '');
$mail->password = Piwik_Common::getRequestVar('mailPassword', '');
$mail->encryption = Piwik_Common::getRequestVar('mailEncryption', '');
Zend_Registry::get('config')->mail = $mail->toArray();
$toReturn = $response->getResponse();
} catch (Exception $e) {
$toReturn = $response->getResponseException($e);
}
echo $toReturn;
}
public function setGeneralSettings()
{
Piwik::checkUserIsSuperUser();
$response = new Piwik_API_ResponseBuilder(Piwik_Common::getRequestVar('format'));
try {
$this->checkTokenInUrl();
$enableBrowserTriggerArchiving = Piwik_Common::getRequestVar('enableBrowserTriggerArchiving');
$todayArchiveTimeToLive = Piwik_Common::getRequestVar('todayArchiveTimeToLive');
Piwik_ArchiveProcessing::setBrowserTriggerArchiving((bool) $enableBrowserTriggerArchiving);
Piwik_ArchiveProcessing::setTodayArchiveTimeToLive($todayArchiveTimeToLive);
// Update email settings
$mail = array();
$mail['transport'] = Piwik_Common::getRequestVar('mailUseSmtp') == '1' ? 'smtp' : '';
$mail['port'] = Piwik_Common::getRequestVar('mailPort', '');
$mail['host'] = Piwik_Common::unsanitizeInputValue(Piwik_Common::getRequestVar('mailHost', ''));
$mail['type'] = Piwik_Common::getRequestVar('mailType', '');
$mail['username'] = Piwik_Common::unsanitizeInputValue(Piwik_Common::getRequestVar('mailUsername', ''));
$mail['password'] = Piwik_Common::unsanitizeInputValue(Piwik_Common::getRequestVar('mailPassword', ''));
$mail['encryption'] = Piwik_Common::getRequestVar('mailEncryption', '');
Piwik_Config::getInstance()->mail = $mail;
// update branding settings
Piwik_Config::getInstance()->branding['use_custom_logo'] = Piwik_Common::getRequestVar('useCustomLogo', '0');
$toReturn = $response->getResponse();
} catch (Exception $e) {
$toReturn = $response->getResponseException($e);
}
echo $toReturn;
}
/**
* Called at the end of the archiving process.
* Does some cleaning job in the database.
*
* @return void
*/
protected function postCompute()
{
parent::postCompute();
// we delete out of date records
// = archives that for day N computed on day N (means they are only partial)
$blobTable = $this->tableArchiveBlob->getTableName();
$numericTable = $this->tableArchiveNumeric->getTableName();
$query = "\tDELETE \n\t\t\t\t\tFROM %s\n\t\t\t\t\tWHERE period = ? \n\t\t\t\t\t\tAND date1 = DATE(ts_archived)\n\t\t\t\t\t\tAND DATE(ts_archived) <> CURRENT_DATE()\n\t\t\t\t\t";
Zend_Registry::get('db')->query(sprintf($query, $blobTable), $this->periodId);
Zend_Registry::get('db')->query(sprintf($query, $numericTable), $this->periodId);
}
/**
* Fetches all blob fields name_* at once for the current archive for performance reasons.
*
* @return void
*/
public function preFetchBlob($name)
{
if (!$this->isThereSomeVisits) {
return false;
}
$tableBlob = $this->archiveProcessing->getTableArchiveBlobName();
$db = Zend_Registry::get('db');
$query = $db->query("SELECT value, name\n\t\t\t\t\t\t\t\tFROM {$tableBlob}\n\t\t\t\t\t\t\t\tWHERE idarchive = ?\n\t\t\t\t\t\t\t\t\tAND name LIKE '{$name}%'", array($this->idArchive));
while ($row = $query->fetch()) {
$value = $row['value'];
$name = $row['name'];
$this->blobCached[$name] = gzuncompress($value);
}
}
/**
* Build DataTable from array and archive it
* @return id of the datatable
*/
private function archiveDataArray($keyword, &$data, $addSearchTermMetaData = false, $addUrlMetaData = false)
{
$dataTable = new Piwik_DataTable();
foreach ($data as &$row) {
$rowData = array(Piwik_DataTable_Row::COLUMNS => $row);
if ($addSearchTermMetaData) {
$rowData[Piwik_DataTable_Row::METADATA] = array('idSearch' => $row[$addSearchTermMetaData], 'searchTerm' => $row[self::SEARCH_TERM]);
}
if ($addUrlMetaData) {
$rowData[Piwik_DataTable_Row::METADATA]['url'] = $row[self::URL];
}
$dataTable->addRow(new Piwik_SiteSearch_ExtendedDataTableRow($rowData));
}
$id = $dataTable->getId();
$name = 'SiteSearch_' . $keyword;
$this->archiveProcessing->insertBlobRecord($name, $dataTable->getSerialized());
destroy($dataTable);
return $id;
}
/**
* Returns datatable describing the number of visits for each day of the week.
*
* @param string $idSite The site ID. Cannot refer to multiple sites.
* @param string $period The period type: day, week, year, range...
* @param string $date The start date of the period. Cannot refer to multiple dates.
* @param string $segment The segment.
* @return Piwik_DataTable
*/
public function getByDayOfWeek($idSite, $period, $date, $segment = false)
{
Piwik::checkUserHasViewAccess($idSite);
// disabled for multiple sites/dates
if (Piwik_Archive::isMultipleSites($idSite)) {
throw new Exception("VisitTime.getByDayOfWeek does not support multiple sites.");
}
if (Piwik_Archive::isMultiplePeriod($date, $period)) {
throw new Exception("VisitTime.getByDayOfWeek does not support multiple dates.");
}
// metrics to query
$metrics = Piwik_ArchiveProcessing::getCoreMetrics();
// get metric data for every day within the supplied period
$oSite = new Piwik_Site($idSite);
$oPeriod = Piwik_Archive::makePeriodFromQueryParams($oSite, $period, $date);
$dateRange = $oPeriod->getDateStart()->toString() . ',' . $oPeriod->getDateEnd()->toString();
$archive = Piwik_Archive::build($idSite, 'day', $dateRange, $segment);
$dataTable = $archive->getDataTableFromNumeric($metrics)->mergeChildren();
// if there's no data for this report, don't bother w/ anything else
if ($dataTable->getRowsCount() == 0) {
return $dataTable;
}
// group by the day of the week (see below for dayOfWeekFromDate function)
$dataTable->filter('GroupBy', array('label', 'Piwik_VisitTime_dayOfWeekFromDate'));
// create new datatable w/ empty rows, then add calculated datatable
$rows = array();
foreach (array(1, 2, 3, 4, 5, 6, 7) as $day) {
$rows[] = array('label' => $day, 'nb_visits' => 0);
}
$result = new Piwik_DataTable();
$result->addRowsFromSimpleArray($rows);
$result->addDataTable($dataTable);
// set day of week integer as metadata
$result->filter('ColumnCallbackAddMetadata', array('label', 'day_of_week'));
// translate labels
$result->filter('ColumnCallbackReplace', array('label', 'Piwik_VisitTime_translateDayOfWeek'));
// set datatable metadata for period start & finish
$result->setMetadata('date_start', $oPeriod->getDateStart());
$result->setMetadata('date_end', $oPeriod->getDateEnd());
return $result;
}
public function setUp()
{
$dumpPath = PIWIK_INCLUDE_PATH . '/tmp/logdump.sql.gz';
$deflatedDumpPath = PIWIK_INCLUDE_PATH . '/tmp/logdump.sql';
$bufferSize = 1024 * 1024;
// drop all tables
Piwik::dropTables();
// download data dump
$dump = fopen(self::$dumpUrl, 'rb');
$outfile = fopen($dumpPath, 'wb');
$bytesRead = 0;
while (!feof($dump)) {
fwrite($outfile, fread($dump, $bufferSize), $bufferSize);
$bytesRead += $bufferSize;
}
fclose($dump);
fclose($outfile);
if ($bytesRead <= 40 * 1024 * 1024) {
throw new Exception("Could not download sql dump!");
}
// unzip the dump
exec("gunzip -c \"" . $dumpPath . "\" > \"{$deflatedDumpPath}\"", $output, $return);
if ($return !== 0) {
throw new Exception("gunzip failed: " . implode("\n", $output));
}
// load the data into the correct database
$user = Piwik_Config::getInstance()->database['username'];
$password = Piwik_Config::getInstance()->database['password'];
$dbName = Piwik_Config::getInstance()->database['dbname'];
Piwik_Config::getInstance()->database['tables_prefix'] = 'piwik_';
Piwik_Common::$cachedTablePrefix = null;
exec("mysql -u \"{$user}\" \"--password={$password}\" {$dbName} < \"" . $deflatedDumpPath . "\" 2>&1", $output, $return);
if ($return !== 0) {
throw new Exception("Failed to load sql dump: " . implode("\n", $output));
}
// make sure archiving will be called
Piwik_ArchiveProcessing::setBrowserTriggerArchiving(true);
}
private function oneClick_Copy()
{
/*
* Make sure the execute bit is set for this shell script
*/
if (!Piwik_ArchiveProcessing::isBrowserTriggerArchivingEnabled()) {
@chmod($this->pathRootExtractedPiwik . '/misc/cron/archive.sh', 0755);
}
/*
* Copy all files to PIWIK_INCLUDE_PATH.
* These files are accessed through the dispatcher.
*/
Piwik::copyRecursive($this->pathRootExtractedPiwik, PIWIK_INCLUDE_PATH);
/*
* These files are visible in the web root and are generally
* served directly by the web server. May be shared.
*/
if (PIWIK_INCLUDE_PATH !== PIWIK_DOCUMENT_ROOT) {
/*
* Copy PHP files that expect to be in the document root
*/
$specialCases = array('/index.php', '/piwik.php', '/js/index.php');
foreach ($specialCases as $file) {
Piwik::copy($this->pathRootExtractedPiwik . $file, PIWIK_DOCUMENT_ROOT . $file);
}
/*
* Copy the non-PHP files (e.g., images, css, javascript)
*/
Piwik::copyRecursive($this->pathRootExtractedPiwik, PIWIK_DOCUMENT_ROOT, true);
}
/*
* Config files may be user (account) specific
*/
if (PIWIK_INCLUDE_PATH !== PIWIK_USER_PATH) {
Piwik::copyRecursive($this->pathRootExtractedPiwik . '/config', PIWIK_USER_PATH . '/config');
}
Piwik::unlinkRecursive($this->pathRootExtractedPiwik, true);
}
/**
* @param $select
* @param $from
* @param $where
* @param $orderBy
* @param $groupBy
* @param $sprintfField
* @param Piwik_ArchiveProcessing $archiveProcessing
* @return int
*/
protected function archiveDayQueryProcess($select, $from, $where, $orderBy, $groupBy, $sprintfField, $archiveProcessing)
{
// idaction field needs to be set in select clause before calling getSelectQuery().
// if a complex segmentation join is needed, the field needs to be propagated
// to the outer select. therefore, $segment needs to know about it.
$select = sprintf($select, $sprintfField);
// get query with segmentation
$bind = array();
$orderBy = false;
$query = $archiveProcessing->getSegment()->getSelectQuery($select, $from, $where, $bind, $orderBy, $groupBy);
// replace the rest of the %s
$querySql = str_replace("%s", $sprintfField, $query['sql']);
// extend bindings
$bind = array_merge(array($archiveProcessing->getStartDatetimeUTC(), $archiveProcessing->getEndDatetimeUTC(), $archiveProcessing->idsite), $query['bind']);
// get result
$resultSet = $archiveProcessing->db->query($querySql, $bind);
$modified = $this->updateActionsTableWithRowQuery($resultSet, $sprintfField);
return $modified;
}
protected function getRequestedReport()
{
// Core metrics are always processed in Core, for the requested date/period/segment
if(in_array($this->requestedReport, Piwik_ArchiveProcessing::getCoreMetrics())
|| $this->requestedReport == 'max_actions')
{
return 'VisitsSummary_CoreMetrics';
}
// VisitFrequency metrics don't follow the same naming convention (HACK)
if(strpos($this->requestedReport, '_returning') > 0
// ignore Goal_visitor_returning_1_1_nb_conversions
&& strpos($this->requestedReport, 'Goal_') === false)
{
return 'VisitFrequency_Metrics';
}
// Goal_* metrics are processed by the Goals plugin (HACK)
if(strpos($this->requestedReport, 'Goal_') === 0)
{
return 'Goals_Metrics';
}
return $this->requestedReport;
}
function __construct()
{
parent::__construct();
$this->db = Zend_Registry::get('db');
$this->debugAlwaysArchive = Zend_Registry::get('config')->Debug->always_archive_data_day;
}
/**
* Called at the end of the archiving process.
* Does some cleaning job in the database.
*
* @return void
*/
protected function postCompute()
{
parent::postCompute();
// we delete records that are now out of date
// in the case of a period we delete archives that were archived before the end of the period
// and only if they are at least 1 day old (so we don't delete archives computed today that may be stil valid)
$blobTable = $this->tableArchiveBlob->getTableName();
$numericTable = $this->tableArchiveNumeric->getTableName();
$query = "\tDELETE \n\t\t\t\t\tFROM %s\n\t\t\t\t\tWHERE period > ? \n\t\t\t\t\t\tAND DATE(ts_archived) <= date2\n\t\t\t\t\t\tAND date(ts_archived) < date_sub(CURRENT_DATE(), INTERVAL 1 DAY)\n\t\t\t\t\t";
Zend_Registry::get('db')->query(sprintf($query, $blobTable), Piwik::$idPeriods['day']);
Zend_Registry::get('db')->query(sprintf($query, $numericTable), Piwik::$idPeriods['day']);
}
protected function initStateFromParameters()
{
// Detect parameters
$reset = $this->isParameterSet("force-all-periods", $valuePossible = true);
$forceAll = $this->isParameterSet("force-all-websites");
$forceTimeoutPeriod = $this->isParameterSet("force-timeout-for-periods", $valuePossible = true);
if (!empty($forceTimeoutPeriod) && $forceTimeoutPeriod !== true) {
// Ensure the cache for periods is at least as high as cache for today
$todayTTL = Piwik_ArchiveProcessing::getTodayArchiveTimeToLive();
if ($forceTimeoutPeriod < $todayTTL) {
$this->log("WARNING: Automatically increasing --force-timeout-for-periods from {$forceTimeoutPeriod} to " . $todayTTL . " to match the cache timeout for Today's report specified in Piwik UI > Settings > General Settings");
$forceTimeoutPeriod = $todayTTL;
}
$this->processPeriodsMaximumEverySeconds = $forceTimeoutPeriod;
}
// Recommend to disable browser archiving when using this script
if (Piwik_ArchiveProcessing::isBrowserTriggerArchivingEnabled()) {
$this->log("NOTE: if you execute this script at least once per hour (or more often) in a crontab, you may disable 'Browser trigger archiving' in Piwik UI > Settings > General Settings. ");
$this->log(" see doc at: http://piwik.org/docs/setup-auto-archiving/");
}
if ($reset) {
$this->log("--force-all-periods was detected: the script will run as if it was its first run, and will trigger archiving for all periods.");
$this->shouldResetState = true;
if (!$forceAll && is_numeric($reset) && $reset > 0) {
$this->firstRunActiveWebsitesWithTraffic = (int) $reset;
}
}
if ($forceAll) {
$this->log("--force-all-websites was detected: the script will archive all websites and all periods sequentially");
$this->shouldArchiveAllWebsites = true;
}
$this->timeLastCompleted = Piwik_GetOption(self::OPTION_ARCHIVING_FINISHED_TS);
if ($this->shouldResetState) {
$this->timeLastCompleted = false;
}
}
/**
* Returns true if Piwik can launch the archiving process for this archive,
* false if otherwise.
*
* @return bool
*/
public function isArchivingDisabled()
{
return Piwik_ArchiveProcessing::isArchivingDisabledFor($this->segment, $this->period);
}
/**
* Called at the end of the archiving process.
* Does some cleaning job in the database.
*/
protected function postCompute()
{
parent::postCompute();
$blobTable = $this->tableArchiveBlob->getTableName();
$numericTable = $this->tableArchiveNumeric->getTableName();
$key = 'lastPurge_' . $blobTable;
$timestamp = Piwik_GetOption($key);
if (!$timestamp || $timestamp < time() - 86400) {
Piwik_SetOption($key, time());
// we delete out of date daily archives from table, maximum once per day
// we only delete archives processed that are older than 1 day, to not delete archives we just processed
$yesterday = Piwik_Date::factory('yesterday')->getDateTime();
$result = Piwik_FetchAll("\n\t\t\t\t\t\t\tSELECT idarchive\n\t\t\t\t\t\t\tFROM {$numericTable}\n\t\t\t\t\t\t\tWHERE name LIKE 'done%'\n\t\t\t\t\t\t\t\tAND value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . "\n\t\t\t\t\t\t\t\tAND ts_archived < ?", array($yesterday));
$idArchivesToDelete = array();
if (!empty($result)) {
foreach ($result as $row) {
$idArchivesToDelete[] = $row['idarchive'];
}
$query = "DELETE \n \t\t\t\t\t\tFROM %s\n \t\t\t\t\t\tWHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ")\n \t\t\t\t\t\t";
Piwik_Query(sprintf($query, $blobTable));
Piwik_Query(sprintf($query, $numericTable));
}
Piwik::log("Purging temporary archives: done [ purged archives older than {$yesterday} from {$blobTable} and {$numericTable} ] [Deleted IDs: " . implode(',', $idArchivesToDelete) . "]");
// Deleting "Custom Date Range" reports after 1 day, since they can be re-processed
// and would take up unecessary space
$query = "DELETE \n \t\t\t\t\tFROM %s\n \t\t\t\t\tWHERE period = ?\n \t\t\t\t\t\tAND ts_archived < ?";
$bind = array(Piwik::$idPeriods['range'], $yesterday);
Piwik_Query(sprintf($query, $blobTable), $bind);
Piwik_Query(sprintf($query, $numericTable), $bind);
} else {
Piwik::log("Purging temporary archives: skipped.");
}
if (!isset($this->archives)) {
return;
}
foreach ($this->archives as $archive) {
destroy($archive);
}
$this->archives = array();
}
/**
* Runs API tests.
*/
protected function runApiTests($api, $params)
{
$testName = 'test_' . $this->getOutputPrefix();
$this->missingExpectedFiles = array();
$this->comparisonFailures = array();
$this->_setCallableApi($api);
if (isset($params['disableArchiving']) && $params['disableArchiving'] === true) {
Piwik_ArchiveProcessing::$forceDisableArchiving = true;
} else {
Piwik_ArchiveProcessing::$forceDisableArchiving = false;
}
if (isset($params['language'])) {
$this->changeLanguage($params['language']);
}
$testSuffix = isset($params['testSuffix']) ? $params['testSuffix'] : '';
$requestUrls = $this->_generateApiUrls(isset($params['format']) ? $params['format'] : 'xml', isset($params['idSite']) ? $params['idSite'] : false, isset($params['date']) ? $params['date'] : false, isset($params['periods']) ? $params['periods'] : false, isset($params['setDateLastN']) ? $params['setDateLastN'] : false, isset($params['language']) ? $params['language'] : false, isset($params['segment']) ? $params['segment'] : false, isset($params['visitorId']) ? $params['visitorId'] : false, isset($params['abandonedCarts']) ? $params['abandonedCarts'] : false, isset($params['idGoal']) ? $params['idGoal'] : false, isset($params['apiModule']) ? $params['apiModule'] : false, isset($params['apiAction']) ? $params['apiAction'] : false, isset($params['otherRequestParameters']) ? $params['otherRequestParameters'] : array(), isset($params['supertableApi']) ? $params['supertableApi'] : false, isset($params['fileExtension']) ? $params['fileExtension'] : false);
foreach ($requestUrls as $apiId => $requestUrl) {
$this->_testApiUrl($testName . $testSuffix, $apiId, $requestUrl);
}
// change the language back to en
if ($this->lastLanguage != 'en') {
$this->changeLanguage('en');
}
if (!empty($this->missingExpectedFiles)) {
$expectedDir = dirname(reset($this->missingExpectedFiles));
$this->markTestIncomplete(" ERROR: Could not find expected API output '" . implode("', '", $this->missingExpectedFiles) . "'. For new tests, to pass the test, you can copy files from the processed/ directory into" . " {$expectedDir} after checking that the output is valid. %s ");
}
if (!empty($this->comparisonFailures)) {
throw reset($this->comparisonFailures);
}
}
/**
* Records the daily stats (numeric or datatable blob) into the archive tables.
*
* @param Piwik_ArchiveProcessing $archiveProcessing
* @return void
*/
protected function archiveDayRecordInDatabase($archiveProcessing)
{
$numericRecords = array('Referers_distinctSearchEngines' => count($this->interestBySearchEngineAndKeyword), 'Referers_distinctKeywords' => count($this->interestByKeywordAndSearchEngine), 'Referers_distinctCampaigns' => count($this->interestByCampaign), 'Referers_distinctWebsites' => count($this->interestByWebsite), 'Referers_distinctWebsitesUrls' => count($this->distinctUrls));
foreach ($numericRecords as $name => $value) {
$archiveProcessing->insertNumericRecord($name, $value);
}
$dataTable = $archiveProcessing->getDataTableSerialized($this->interestByType);
$archiveProcessing->insertBlobRecord('Referers_type', $dataTable);
destroy($dataTable);
$blobRecords = array('Referers_keywordBySearchEngine' => $archiveProcessing->getDataTableWithSubtablesFromArraysIndexedByLabel($this->interestBySearchEngineAndKeyword, $this->interestBySearchEngine), 'Referers_searchEngineByKeyword' => $archiveProcessing->getDataTableWithSubtablesFromArraysIndexedByLabel($this->interestByKeywordAndSearchEngine, $this->interestByKeyword), 'Referers_keywordByCampaign' => $archiveProcessing->getDataTableWithSubtablesFromArraysIndexedByLabel($this->interestByCampaignAndKeyword, $this->interestByCampaign), 'Referers_urlByWebsite' => $archiveProcessing->getDataTableWithSubtablesFromArraysIndexedByLabel($this->interestByWebsiteAndUrl, $this->interestByWebsite));
foreach ($blobRecords as $recordName => $table) {
$blob = $table->getSerialized($this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation);
$archiveProcessing->insertBlobRecord($recordName, $blob);
destroy($table);
}
}
/**
* Called at the end of the archiving process.
* Does some cleaning job in the database.
*
* @return void
*/
protected function postCompute()
{
parent::postCompute();
$blobTable = $this->tableArchiveBlob->getTableName();
$numericTable = $this->tableArchiveNumeric->getTableName();
// delete out of date records maximum once per day (DELETE request is costly)
$key = 'lastPurge_' . $blobTable;
$timestamp = Piwik_GetOption($key);
if (!$timestamp || $timestamp < time() - 86400) {
// we delete out of date daily archives from table, maximum once per day
// those for day N that were processed on day N (means the archives are only partial as the day wasn't finished)
$query = "/* SHARDING_ID_SITE = " . $this->idsite . " */ \tDELETE \n\t\t\t\t\t\tFROM %s\n\t\t\t\t\t\tWHERE period = ? \n\t\t\t\t\t\t\tAND date1 = DATE(ts_archived)\n\t\t\t\t\t\t\tAND DATE(ts_archived) <> CURRENT_DATE()\n\t\t\t\t\t\t";
Piwik_Query(sprintf($query, $blobTable), Piwik::$idPeriods['day']);
Piwik_Query(sprintf($query, $numericTable), Piwik::$idPeriods['day']);
// we delete out of date Period records (week/month/etc)
// we delete archives that were archived before the end of the period
// and only if they are at least 1 day old (so we don't delete archives computed today that may be stil valid)
$query = "\tDELETE \n\t\t\t\t\t\tFROM %s\n\t\t\t\t\t\tWHERE period > ? \n\t\t\t\t\t\t\tAND DATE(ts_archived) <= date2\n\t\t\t\t\t\t\tAND date(ts_archived) < date_sub(CURRENT_DATE(), INTERVAL 1 DAY)\n\t\t\t\t\t\t";
Piwik_Query(sprintf($query, $blobTable), Piwik::$idPeriods['day']);
Piwik_Query(sprintf($query, $numericTable), Piwik::$idPeriods['day']);
Piwik_SetOption($key, time());
}
foreach ($this->archives as $archive) {
destroy($archive);
}
$this->archives = array();
}
/**
* Called at the end of the archiving process.
* Does some cleaning job in the database.
*/
protected function postCompute()
{
parent::postCompute();
$blobTable = $this->tableArchiveBlob->getTableName();
$numericTable = $this->tableArchiveNumeric->getTableName();
$key = 'lastPurge_' . $blobTable;
$timestamp = Piwik_GetOption($key);
// we shall purge temporary archives after their timeout is finished, plus an extra 2 hours
// in case archiving is disabled and is late to run, we give it this extra time to run and re-process more recent records
$temporaryArchivingTimeout = self::getTodayArchiveTimeToLive();
$purgeEveryNSeconds = $temporaryArchivingTimeout + 2 * 3600;
// we only delete archives if we are able to process them, otherwise, the browser might process reports
// when &segment= is specified (or custom date range) and would below, delete temporary archives that the
// browser is not able to process until next cron run (which could be more than 1 hour away)
if ($this->isRequestAuthorizedToArchive() && (!$timestamp || $timestamp < time() - $purgeEveryNSeconds)) {
Piwik_SetOption($key, time());
$purgeArchivesOlderThan = Piwik_Date::factory(time() - $purgeEveryNSeconds)->getDateTime();
$result = Piwik_FetchAll("\n\t\t\t\t\t\t\tSELECT idarchive\n\t\t\t\t\t\t\tFROM {$numericTable}\n\t\t\t\t\t\t\tWHERE name LIKE 'done%'\n\t\t\t\t\t\t\t\tAND value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . "\n\t\t\t\t\t\t\t\tAND ts_archived < ?", array($purgeArchivesOlderThan));
$idArchivesToDelete = array();
if (!empty($result)) {
foreach ($result as $row) {
$idArchivesToDelete[] = $row['idarchive'];
}
$query = "DELETE \n \t\t\t\t\t\tFROM %s\n \t\t\t\t\t\tWHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ")\n \t\t\t\t\t\t";
Piwik_Query(sprintf($query, $blobTable));
Piwik_Query(sprintf($query, $numericTable));
}
Piwik::log("Purging temporary archives: done [ purged archives older than {$purgeArchivesOlderThan} from {$blobTable} and {$numericTable} ] [Deleted IDs: " . implode(',', $idArchivesToDelete) . "]");
// Deleting "Custom Date Range" reports after 1 day, since they can be re-processed
// and would take up unecessary space
$yesterday = Piwik_Date::factory('yesterday')->getDateTime();
$query = "DELETE \n \t\t\t\t\tFROM %s\n \t\t\t\t\tWHERE period = ?\n \t\t\t\t\t\tAND ts_archived < ?";
$bind = array(Piwik::$idPeriods['range'], $yesterday);
Piwik::log("Purging Custom Range archives: done [ purged archives older than {$yesterday} from {$blobTable} and {$numericTable} ]");
Piwik_Query(sprintf($query, $blobTable), $bind);
Piwik_Query(sprintf($query, $numericTable), $bind);
// these tables will be OPTIMIZEd daily in a scheduled task, to claim lost space
} else {
Piwik::log("Purging temporary archives: skipped.");
}
if (!isset($this->archives)) {
return;
}
foreach ($this->archives as $archive) {
destroy($archive);
}
$this->archives = array();
}
/**
* Gets the name of the database table that holds the numeric archive data for
* this archive.
*
* @return string
*/
private function getNumericTableName()
{
if (is_null($this->tableName)) {
$table = Piwik_ArchiveProcessing::makeNumericArchiveTable($this->getFirstArchive()->getPeriod());
$this->tableName = $table->getTableName();
}
return $this->tableName;
}
/**
* Runs API tests.
*/
private function runApiTests()
{
$apiToTest = $this->getApiToTest();
$testName = 'test_' . $this->getOutputPrefix();
foreach ($apiToTest as $test) {
list($api, $params) = $test;
if ($api == 'all') {
$this->setApiToCall(array());
$this->setApiNotToCall(Test_Integration::$defaultApiNotToCall);
} else {
if (!is_array($api)) {
$api = array($api);
}
$this->setApiToCall($api);
$this->setApiNotToCall(array());
}
if (isset($params['disableArchiving']) && $params['disableArchiving'] === true) {
Piwik_ArchiveProcessing::$forceDisableArchiving = true;
} else {
Piwik_ArchiveProcessing::$forceDisableArchiving = false;
}
if (isset($params['language'])) {
$this->changeLanguage($params['language']);
}
$testSuffix = isset($params['testSuffix']) ? $params['testSuffix'] : '';
$this->callGetApiCompareOutput($testName . $testSuffix, isset($params['format']) ? $params['format'] : 'xml', isset($params['idSite']) ? $params['idSite'] : false, isset($params['date']) ? $params['date'] : false, isset($params['periods']) ? $params['periods'] : false, isset($params['setDateLastN']) ? $params['setDateLastN'] : false, isset($params['language']) ? $params['language'] : false, isset($params['segment']) ? $params['segment'] : false, isset($params['visitorId']) ? $params['visitorId'] : false, isset($params['abandonedCarts']) ? $params['abandonedCarts'] : false, isset($params['idGoal']) ? $params['idGoal'] : false, isset($params['apiModule']) ? $params['apiModule'] : false, isset($params['apiAction']) ? $params['apiAction'] : false, isset($params['otherRequestParameters']) ? $params['otherRequestParameters'] : array());
// change the language back to en
if ($this->lastLanguage != 'en') {
$this->changeLanguage('en');
}
}
}
/**
* Runs API tests.
*/
protected function runApiTests($api, $params)
{
$testName = 'test_' . $this->getOutputPrefix();
$this->_setCallableApi($api);
if (isset($params['disableArchiving']) && $params['disableArchiving'] === true) {
Piwik_ArchiveProcessing::$forceDisableArchiving = true;
} else {
Piwik_ArchiveProcessing::$forceDisableArchiving = false;
}
if (isset($params['language'])) {
$this->changeLanguage($params['language']);
}
$testSuffix = isset($params['testSuffix']) ? $params['testSuffix'] : '';
$requestUrls = $this->_generateApiUrls(isset($params['format']) ? $params['format'] : 'xml', isset($params['idSite']) ? $params['idSite'] : false, isset($params['date']) ? $params['date'] : false, isset($params['periods']) ? $params['periods'] : false, isset($params['setDateLastN']) ? $params['setDateLastN'] : false, isset($params['language']) ? $params['language'] : false, isset($params['segment']) ? $params['segment'] : false, isset($params['visitorId']) ? $params['visitorId'] : false, isset($params['abandonedCarts']) ? $params['abandonedCarts'] : false, isset($params['idGoal']) ? $params['idGoal'] : false, isset($params['apiModule']) ? $params['apiModule'] : false, isset($params['apiAction']) ? $params['apiAction'] : false, isset($params['otherRequestParameters']) ? $params['otherRequestParameters'] : array(), isset($params['supertableApi']) ? $params['supertableApi'] : false, isset($params['fileExtension']) ? $params['fileExtension'] : false);
foreach ($requestUrls as $apiId => $requestUrl) {
$this->_testApiUrl($testName . $testSuffix, $apiId, $requestUrl);
}
// change the language back to en
if ($this->lastLanguage != 'en') {
$this->changeLanguage('en');
}
}
/**
* Returns contents of general (global) cache
*
* @return array
*/
protected static function getCacheGeneral()
{
$cache = self::getTrackerCache();
$cacheId = 'general';
$expectedRows = 3;
if (($cacheContent = $cache->get($cacheId)) !== false && count($cacheContent) == $expectedRows) {
return $cacheContent;
}
self::initCorePiwikInTrackerMode();
$cacheContent = array('isBrowserTriggerArchivingEnabled' => Piwik_ArchiveProcessing::isBrowserTriggerArchivingEnabled(), 'lastTrackerCronRun' => Piwik_GetOption('lastTrackerCronRun'), 'currentLocationProviderId' => Piwik_UserCountry_LocationProvider::getCurrentProviderId());
return $cacheContent;
}
/**
* @param Piwik_ArchiveProcessing $archiveProcessing
* @return void
*/
protected function archiveDayRecordInDatabase($archiveProcessing)
{
$recordName = 'CustomVariables_valueByName';
$table = $archiveProcessing->getDataTableWithSubtablesFromArraysIndexedByLabel($this->interestByCustomVariablesAndValue, $this->interestByCustomVariables);
$blob = $table->getSerialized($this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable);
$archiveProcessing->insertBlobRecord($recordName, $blob);
destroy($table);
}
/**
* test of validity of an archive, for today's archive with toronto's timezone
* @group Core
* @group ArchiveProcessing
*/
public function testInitTodayToronto()
{
if (!Piwik::isTimezoneSupportEnabled()) {
$this->markTestSkipped('timezones needs to be supported');
}
$now = time();
$siteTimezone = 'America/Toronto';
$timestamp = Piwik_Date::factory('now', $siteTimezone)->getTimestamp();
$dateLabel = date('Y-m-d', $timestamp);
Piwik_ArchiveProcessing::setBrowserTriggerArchiving(true);
$archiveProcessing = $this->_createArchiveProcessing('day', $dateLabel, $siteTimezone);
$archiveProcessing->time = $now;
// we look at anything processed within the time to live range
$dateMinArchived = $now - Piwik_ArchiveProcessing::getTodayArchiveTimeToLive();
$this->assertEquals($archiveProcessing->getMinTimeArchivedProcessed(), $dateMinArchived);
$this->assertTrue($archiveProcessing->isArchiveTemporary());
// when browsers don't trigger archives, we force ArchiveProcessing
// to fetch any of the most recent archive
Piwik_ArchiveProcessing::setBrowserTriggerArchiving(false);
// see isArchivingDisabled()
// Running in CLI doesn't impact the time to live today's archive we are loading
// From CLI, we will not return data that is 'stale'
if (!Piwik_Common::isPhpCliMode()) {
$dateMinArchived = 0;
}
$this->assertEquals($archiveProcessing->getMinTimeArchivedProcessed(), $dateMinArchived);
// this test varies with DST
$this->assertTrue($archiveProcessing->getStartDatetimeUTC() == date('Y-m-d', $timestamp) . ' 04:00:00' || $archiveProcessing->getStartDatetimeUTC() == date('Y-m-d', $timestamp) . ' 05:00:00');
$this->assertTrue($archiveProcessing->getEndDatetimeUTC() == date('Y-m-d', $timestamp + 86400) . ' 03:59:59' || $archiveProcessing->getEndDatetimeUTC() == date('Y-m-d', $timestamp + 86400) . ' 04:59:59');
$this->assertTrue($archiveProcessing->isArchiveTemporary());
}
/**
* Constructor
*/
function __construct()
{
parent::__construct();
$this->db = Zend_Registry::get('db');
}
/**
* Called at the end of the archiving process.
* Does some cleaning job in the database.
*/
protected function postCompute()
{
parent::postCompute();
$numericTable = $this->tableArchiveNumeric->getTableName();
self::doPurgeOutdatedArchives($numericTable, $this->isArchiveTemporary());
if (!isset($this->archives)) {
return;
}
foreach ($this->archives as $archive) {
destroy($archive);
}
$this->archives = array();
}
/**
* @param $select
* @param $from
* @param $where
* @param $orderBy
* @param $groupBy
* @param $sprintfField
* @param Piwik_ArchiveProcessing $archiveProcessing
* @param Piwik_RankingQuery|false $rankingQuery
* @return int
*/
protected function archiveDayQueryProcess($select, $from, $where, $orderBy, $groupBy, $sprintfField, $archiveProcessing, $rankingQuery = false)
{
// idaction field needs to be set in select clause before calling getSelectQuery().
// if a complex segmentation join is needed, the field needs to be propagated
// to the outer select. therefore, $segment needs to know about it.
$select = sprintf($select, $sprintfField);
$bind = array();
// get query with segmentation
$query = $archiveProcessing->getSegment()->getSelectQuery($select, $from, $where, $bind, $orderBy, $groupBy);
// extend bindings
$bind = array_merge(array($archiveProcessing->getStartDatetimeUTC(), $archiveProcessing->getEndDatetimeUTC(), $archiveProcessing->idsite), $query['bind']);
// replace the rest of the %s
$querySql = str_replace("%s", $sprintfField, $query['sql']);
// apply ranking query
if ($rankingQuery) {
$querySql = $rankingQuery->generateQuery($querySql);
}
// echo '<pre>';var_dump($querySql);
// var_dump($bind);
// get result
$resultSet = $archiveProcessing->db->query($querySql, $bind);
$modified = Piwik_Actions_ArchivingHelper::updateActionsTableWithRowQuery($resultSet, $sprintfField, $this->actionsTablesByType);
return $modified;
}