/**
* See {@link PatternRecursive}.
*
* @param DataTable $table
* @return int The number of deleted rows.
*/
public function filter($table)
{
$rows = $table->getRows();
foreach ($rows as $key => $row) {
// A row is deleted if
// 1 - its label doesnt contain the pattern
// AND 2 - the label is not found in the children
$patternNotFoundInChildren = false;
try {
$idSubTable = $row->getIdSubDataTable();
$subTable = Manager::getInstance()->getTable($idSubTable);
// we delete the row if we couldn't find the pattern in any row in the
// children hierarchy
if ($this->filter($subTable) == 0) {
$patternNotFoundInChildren = true;
}
} catch (Exception $e) {
// there is no subtable loaded for example
$patternNotFoundInChildren = true;
}
if ($patternNotFoundInChildren && !Pattern::match($this->patternToSearchQuoted, $row->getColumn($this->columnToFilter), $invertedMatch = false)) {
$table->deleteRow($key);
}
}
return $table->getRowsCount();
}
/**
* Reset this row to an empty one and sums the associated subtable again.
*/
public function recalculate()
{
$id = $this->getIdSubDataTable();
if ($id !== null) {
$subTable = Manager::getInstance()->getTable($id);
$this->sumTable($subTable);
}
}
/**
* Filters a row's subtable, if one exists and is loaded in memory.
*
* @param Row $row The row whose subtable should be filter.
*/
public function filterSubTable(Row $row)
{
if (!$this->enableRecursive) {
return;
}
if ($row->isSubtableLoaded()) {
$subTable = Manager::getInstance()->getTable($row->getIdSubDataTable());
$this->filter($subTable);
}
}
/**
* See {@link ReplaceSummaryRowLabel}.
*
* @param DataTable $table
*/
public function filter($table)
{
$rows = $table->getRows();
foreach ($rows as $id => $row) {
if ($row->getColumn('label') == DataTable::LABEL_SUMMARY_ROW || $id == DataTable::ID_SUMMARY_ROW) {
$row->setColumn('label', $this->newLabel);
break;
}
}
// recurse
foreach ($rows as $row) {
if ($row->isSubtableLoaded()) {
$subTable = Manager::getInstance()->getTable($row->getIdSubDataTable());
$this->filter($subTable);
}
}
}
/**
* Instantiates the Archiver class in each plugin that defines it,
* and triggers Aggregation processing on these plugins.
*/
public function callAggregateAllPlugins($visits, $visitsConverted)
{
$this->archiveProcessor->setNumberOfVisits($visits, $visitsConverted);
$archivers = $this->getPluginArchivers();
foreach ($archivers as $pluginName => $archiverClass) {
// We clean up below all tables created during this function call (and recursive calls)
$latestUsedTableId = Manager::getInstance()->getMostRecentTableId();
/** @var Archiver $archiver */
$archiver = new $archiverClass($this->archiveProcessor);
if (!$archiver->isEnabled()) {
continue;
}
if ($this->shouldProcessReportsForPlugin($pluginName)) {
if ($this->isSingleSiteDayArchive) {
$archiver->aggregateDayReport();
} else {
$archiver->aggregateMultipleReports();
}
}
Manager::getInstance()->deleteAll($latestUsedTableId);
unset($archiver);
}
}
/**
* Returns the associated subtable, if one exists. Returns `false` if none exists.
*
* @return DataTable|bool
*/
public function getSubtable()
{
if ($this->isSubtableLoaded()) {
return Manager::getInstance()->getTable($this->getIdSubDataTable());
}
return false;
}
/**
* Add the external referrers to the report:
* direct entries, websites, campaigns, search engines
*
* @param LogAggregator $logAggregator
* @param $report
* @param $idaction
* @param string $actionType
* @param $limitBeforeGrouping
*/
private function addExternalReferrers($logAggregator, &$report, $idaction, $actionType, $limitBeforeGrouping)
{
$data = $this->queryExternalReferrers($idaction, $actionType, $logAggregator, $limitBeforeGrouping);
$report['pageMetrics']['entries'] = 0;
$report['referrers'] = array();
foreach ($data->getRows() as $row) {
$referrerId = $row->getColumn('label');
$visits = $row->getColumn(Metrics::INDEX_NB_VISITS);
if ($visits) {
// load details (i.e. subtables)
$details = array();
if ($idSubTable = $row->getIdSubDataTable()) {
$subTable = Manager::getInstance()->getTable($idSubTable);
foreach ($subTable->getRows() as $subRow) {
$details[] = array('label' => $subRow->getColumn('label'), 'referrals' => $subRow->getColumn(Metrics::INDEX_NB_VISITS));
}
}
$report['referrers'][] = array('label' => $this->getReferrerLabel($referrerId), 'shortName' => \Piwik\Plugins\Referrers\getReferrerTypeFromShortName($referrerId), 'visits' => $visits, 'details' => $details);
$report['pageMetrics']['entries'] += $visits;
}
}
// if there's no data for referrers, ResponseBuilder::handleMultiDimensionalArray
// does not detect the multi dimensional array and the data is rendered differently, which
// causes an exception.
if (count($report['referrers']) == 0) {
$report['referrers'][] = array('label' => $this->getReferrerLabel(Common::REFERRER_TYPE_DIRECT_ENTRY), 'shortName' => \Piwik\Plugins\Referrers\getReferrerTypeLabel(Common::REFERRER_TYPE_DIRECT_ENTRY), 'visits' => 0);
}
}
public function setUp()
{
$this->builder = $this->makeBuilder(array('method' => 'MultiSites_getAll'));
DataTable\Manager::getInstance()->deleteAll();
}
<?php
/**
* Proxy to normal piwik.php, but in testing mode
*
* - Use the tests database to record Tracking data
* - Allows to overwrite the Visitor IP, and Server datetime
*
*/
use Piwik\DataTable\Manager;
use Piwik\Option;
use Piwik\Plugins\UserCountry\LocationProvider\GeoIp;
use Piwik\Site;
use Piwik\Tracker\Cache;
use Piwik\Tracker;
require realpath(dirname(__FILE__)) . "/includes.php";
// Wrapping the request inside ob_start() calls to ensure that the Test
// calling us waits for the full request to process before unblocking
ob_start();
try {
Piwik_TestingEnvironment::addHooks();
GeoIp::$geoIPDatabaseDir = 'tests/lib/geoip-files';
Tracker::setTestEnvironment();
Manager::getInstance()->deleteAll();
Option::clearCache();
Site::clearCache();
include PIWIK_INCLUDE_PATH . '/piwik.php';
} catch (Exception $ex) {
echo "Unexpected error during tracking: " . $ex->getMessage() . "\n" . $ex->getTraceAsString() . "\n";
}
ob_end_flush();
public function setUp()
{
$this->builder = $this->makeBuilder(array());
DataTable\Manager::getInstance()->deleteAll();
}
/**
* Aggregates the $row columns to this table.
*
* $row must have a column "label". The $row will be summed to this table's row with the same label.
*
* @param $row
* @throws \Exception
*/
protected function aggregateRowWithLabel(Row $row, $doAggregateSubTables = true)
{
$labelToLookFor = $row->getColumn('label');
if ($labelToLookFor === false) {
throw new Exception("Label column not found in the table to add in addDataTable()");
}
$rowFound = $this->getRowFromLabel($labelToLookFor);
if ($rowFound === false) {
if ($labelToLookFor === self::LABEL_SUMMARY_ROW) {
$this->addSummaryRow($row);
} else {
$this->addRow($row);
}
} else {
$rowFound->sumRow($row, $copyMeta = true, $this->getMetadata(self::COLUMN_AGGREGATION_OPS_METADATA_NAME));
if ($doAggregateSubTables) {
// if the row to add has a subtable whereas the current row doesn't
// we simply add it (cloning the subtable)
// if the row has the subtable already
// then we have to recursively sum the subtables
if (($idSubTable = $row->getIdSubDataTable()) !== null) {
$subTable = Manager::getInstance()->getTable($idSubTable);
$subTable->metadata[self::COLUMN_AGGREGATION_OPS_METADATA_NAME] = $this->getMetadata(self::COLUMN_AGGREGATION_OPS_METADATA_NAME);
$rowFound->sumSubtable($subTable);
}
}
}
}
public function clearInMemoryCaches()
{
Archive::clearStaticCache();
DataTableManager::getInstance()->deleteAll();
Option::clearCache();
Site::clearCache();
Cache::deleteTrackerCache();
PiwikCache::getTransientCache()->flushAll();
PiwikCache::getEagerCache()->flushAll();
PiwikCache::getLazyCache()->flushAll();
ArchiveTableCreator::clear();
\Piwik\Plugins\ScheduledReports\API::$cache = array();
Singleton::clearAll();
PluginsArchiver::$archivers = array();
$_GET = $_REQUEST = array();
Translate::reset();
self::getConfig()->Plugins;
// make sure Plugins exists in a config object for next tests that use Plugin\Manager
// since Plugin\Manager uses getFromGlobalConfig which doesn't init the config object
}
private function generateOverviewReport($method, $idSite, $period, $date, $segment, array $defaultParams)
{
$tableManager = DataTable\Manager::getInstance();
/** @var DataTable[] $tables */
$tables = array();
foreach ($this->getOverviewReports() as $reportId => $reportParams) {
if (!empty($reportParams)) {
foreach ($defaultParams as $key => $defaultParam) {
if (!array_key_exists($key, $reportParams)) {
$reportParams[$key] = $defaultParam;
}
}
}
$firstTableId = $tableManager->getMostRecentTableId();
$table = $this->requestApiMethod($method, $idSite, $period, $date, $reportId, $segment, $reportParams);
$reportTableIds[] = $table->getId();
$tableManager->deleteTablesExceptIgnored($reportTableIds, $firstTableId);
$tables[] = $table;
}
$map = new DataTable\Map();
foreach ($tables as $table) {
$map->addTable($table, $table->getMetadata('reportName'));
}
return $map;
}
/**
* Converts the given data table to an array
*
* @param DataTable $table
* @return array
*/
protected function renderTable($table)
{
$array = array();
foreach ($table->getRows() as $id => $row) {
$newRow = array('columns' => $row->getColumns(), 'metadata' => $row->getMetadata(), 'idsubdatatable' => $row->getIdSubDataTable());
if ($id == DataTable::ID_SUMMARY_ROW) {
$newRow['issummaryrow'] = true;
}
if ($this->isRenderSubtables() && $row->isSubtableLoaded()) {
$subTable = $this->renderTable(Manager::getInstance()->getTable($row->getIdSubDataTable()));
$newRow['subtable'] = $subTable;
if ($this->hideIdSubDatatable === false && isset($newRow['metadata']['idsubdatatable_in_db'])) {
$newRow['columns']['idsubdatatable'] = $newRow['metadata']['idsubdatatable_in_db'];
}
unset($newRow['metadata']['idsubdatatable_in_db']);
}
if ($this->hideIdSubDatatable !== false) {
unset($newRow['idsubdatatable']);
}
$array[] = $newRow;
}
return $array;
}
/**
* Computes the given dataTable output and returns the string/binary
*
* @param DataTable $table data table to render
* @param string $prefix prefix to output before table data
* @return string
*/
protected function renderTable($table, $prefix = "")
{
if (is_array($table)) {
$table = DataTable::makeFromSimpleArray($table);
}
if ($table instanceof DataTable\Map) {
return $this->renderDataTableMap($table, $prefix);
}
if ($table->getRowsCount() == 0) {
return "Empty table<br />\n";
}
static $depth = 0;
$output = '';
$i = 1;
foreach ($table->getRows() as $row) {
$dataTableMapBreak = false;
$columns = array();
foreach ($row->getColumns() as $column => $value) {
if ($value instanceof DataTable\Map) {
$output .= $this->renderDataTableMap($value, $prefix);
$dataTableMapBreak = true;
break;
}
if (is_string($value)) {
$value = "'{$value}'";
} elseif (is_array($value)) {
$value = var_export($value, true);
}
$columns[] = "'{$column}' => {$value}";
}
if ($dataTableMapBreak === true) {
continue;
}
$columns = implode(", ", $columns);
$metadata = array();
foreach ($row->getMetadata() as $name => $value) {
if (is_string($value)) {
$value = "'{$value}'";
} elseif (is_array($value)) {
$value = var_export($value, true);
}
$metadata[] = "'{$name}' => {$value}";
}
$metadata = implode(", ", $metadata);
$output .= str_repeat($this->prefixRows, $depth) . "- {$i} [" . $columns . "] [" . $metadata . "] [idsubtable = " . $row->getIdSubDataTable() . "]<br />\n";
if (!is_null($row->getIdSubDataTable())) {
if ($row->isSubtableLoaded()) {
$depth++;
$output .= $this->renderTable(Manager::getInstance()->getTable($row->getIdSubDataTable()), $prefix . ' ');
$depth--;
} else {
$output .= "-- Sub DataTable not loaded<br />\n";
}
}
$i++;
}
$metadata = $table->getAllTableMetadata();
if (!empty($metadata)) {
$output .= "<hr />Metadata<br />";
foreach ($metadata as $id => $metadataIn) {
$output .= "<br />";
$output .= $prefix . " <b>{$id}</b><br />";
if (is_array($metadataIn)) {
foreach ($metadataIn as $name => $value) {
$output .= $prefix . $prefix . "{$name} => {$value}";
}
}
}
}
return $output;
}
public function clearInMemoryCaches()
{
DataTableManager::getInstance()->deleteAll();
Option::clearCache();
Site::clearCache();
Cache::deleteTrackerCache();
Config::getInstance()->clear();
ArchiveTableCreator::clear();
\Piwik\Plugins\ScheduledReports\API::$cache = array();
\Piwik\Registry::unsetInstance();
\Piwik\EventDispatcher::getInstance()->clearAllObservers();
$_GET = $_REQUEST = array();
Translate::unloadEnglishTranslation();
Config::unsetInstance();
\Piwik\Config::getInstance()->Plugins;
// make sure Plugins exists in a config object for next tests that use Plugin\Manager
// since Plugin\Manager uses getFromGlobalConfig which doesn't init the config object
}
public function test_serializeFails_onSubTableNotFound()
{
// create a simple table with a subtable
$table1 = $this->_getDataTable1ForTest();
$table2 = $this->_getDataTable2ForTest();
$table2->getFirstRow()->setSubtable($table1);
$idSubtable = 1;
// subtableIds are consecutive, we cannot use $table->getId()
/* Check it looks good:
$renderer = DataTable\Renderer::factory('xml');
$renderer->setTable($table2);
$renderer->setRenderSubTables(true);
echo $renderer->render();
*/
// test serialize:
// - subtable is serialized as expected
$serializedStrings = $table2->getSerialized();
// both the main table and the sub table are serialized
$this->assertEquals(sizeof($serializedStrings), 2);
// the serialized string references the id subtable
$unserialized = unserialize($serializedStrings[0]);
$this->assertSame($idSubtable, $unserialized[0][3], "not found the id sub table in the serialized, not expected");
// KABOOM, we delete the subtable, reproducing a "random data issue"
Manager::getInstance()->deleteTable($table1->getId());
// Now we will serialize this "broken datatable" and check it works.
// - it does not throw an exception
$serializedStrings = $table2->getSerialized();
// - the serialized table does NOT contain the sub table
$this->assertEquals(sizeof($serializedStrings), 1);
// main table only is serialized
$unserialized = unserialize($serializedStrings[0]);
// - the serialized string does NOT contain the id subtable (the row was cleaned up as expected)
$this->assertNull($unserialized[0][3], "found the id sub table in the serialized, not expected");
}
/**
* Sums records for every subperiod of the current period and inserts the result as the record
* for this period.
*
* DataTables are summed recursively so subtables will be summed as well.
*
* @param string|array $recordNames Name(s) of the report we are aggregating, eg, `'Referrers_type'`.
* @param int $maximumRowsInDataTableLevelZero Maximum number of rows allowed in the top level DataTable.
* @param int $maximumRowsInSubDataTable Maximum number of rows allowed in each subtable.
* @param string $columnToSortByBeforeTruncation The name of the column to sort by before truncating a DataTable.
* @param array $columnsAggregationOperation Operations for aggregating columns, see {@link Row::sumRow()}.
* @param array $columnsToRenameAfterAggregation Columns mapped to new names for columns that must change names
* when summed because they cannot be summed, eg,
* `array('nb_uniq_visitors' => 'sum_daily_nb_uniq_visitors')`.
* @return array Returns the row counts of each aggregated report before truncation, eg,
*
* array(
* 'report1' => array('level0' => $report1->getRowsCount,
* 'recursive' => $report1->getRowsCountRecursive()),
* 'report2' => array('level0' => $report2->getRowsCount,
* 'recursive' => $report2->getRowsCountRecursive()),
* ...
* )
* @api
*/
public function aggregateDataTableRecords($recordNames, $maximumRowsInDataTableLevelZero = null, $maximumRowsInSubDataTable = null, $columnToSortByBeforeTruncation = null, &$columnsAggregationOperation = null, $columnsToRenameAfterAggregation = null)
{
if (!is_array($recordNames)) {
$recordNames = array($recordNames);
}
$nameToCount = array();
foreach ($recordNames as $recordName) {
$latestUsedTableId = Manager::getInstance()->getMostRecentTableId();
$table = $this->aggregateDataTableRecord($recordName, $columnsAggregationOperation, $columnsToRenameAfterAggregation);
$rowsCount = $table->getRowsCount();
$nameToCount[$recordName]['level0'] = $rowsCount;
$rowsCountRecursive = $rowsCount;
if ($this->isAggregateSubTables()) {
$rowsCountRecursive = $table->getRowsCountRecursive();
}
$nameToCount[$recordName]['recursive'] = $rowsCountRecursive;
$blob = $table->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation);
Common::destroy($table);
$this->insertBlobRecord($recordName, $blob);
unset($blob);
DataTable\Manager::getInstance()->deleteAll($latestUsedTableId);
}
return $nameToCount;
}
public function tearDown()
{
parent::tearDown();
Manager::getInstance()->deleteAll();
Option::clearCache();
Site::clearCache();
Cache::deleteTrackerCache();
ArchiveTableCreator::clear();
$tempTableName = Common::prefixTable(LogDataPurger::TEMP_TABLE_NAME);
Db::query("DROP TABLE IF EXISTS " . $tempTableName);
}
public function performTearDown()
{
// Note: avoid run SQL in the *tearDown() metohds because it randomly fails on Travis CI
// with error Error while sending QUERY packet. PID=XX
$this->tearDown();
self::unloadAllPlugins();
if ($this->dropDatabaseInTearDown) {
$this->dropDatabase();
}
DataTableManager::getInstance()->deleteAll();
Option::clearCache();
Site::clearCache();
Cache::deleteTrackerCache();
Config::getInstance()->clear();
ArchiveTableCreator::clear();
\Piwik\Plugins\ScheduledReports\API::$cache = array();
\Piwik\Registry::unsetInstance();
\Piwik\EventDispatcher::getInstance()->clearAllObservers();
$_GET = $_REQUEST = array();
Translate::unloadEnglishTranslation();
Config::unsetInstance();
\Piwik\Config::getInstance()->Plugins;
// make sure Plugins exists in a config object for next tests that use Plugin\Manager
// since Plugin\Manager uses getFromGlobalConfig which doesn't init the config object
}
/**
* Instantiates the Archiver class in each plugin that defines it,
* and triggers Aggregation processing on these plugins.
*/
public function callAggregateAllPlugins($visits, $visitsConverted)
{
Log::debug("PluginsArchiver::%s: Initializing archiving process for all plugins [visits = %s, visits converted = %s]", __FUNCTION__, $visits, $visitsConverted);
$this->archiveProcessor->setNumberOfVisits($visits, $visitsConverted);
$archivers = $this->getPluginArchivers();
foreach ($archivers as $pluginName => $archiverClass) {
// We clean up below all tables created during this function call (and recursive calls)
$latestUsedTableId = Manager::getInstance()->getMostRecentTableId();
/** @var Archiver $archiver */
$archiver = new $archiverClass($this->archiveProcessor);
if (!$archiver->isEnabled()) {
Log::debug("PluginsArchiver::%s: Skipping archiving for plugin '%s'.", __FUNCTION__, $pluginName);
continue;
}
if ($this->shouldProcessReportsForPlugin($pluginName)) {
$this->logAggregator->setQueryOriginHint($pluginName);
try {
$timer = new Timer();
if ($this->isSingleSiteDayArchive) {
Log::debug("PluginsArchiver::%s: Archiving day reports for plugin '%s'.", __FUNCTION__, $pluginName);
$archiver->aggregateDayReport();
} else {
Log::debug("PluginsArchiver::%s: Archiving period reports for plugin '%s'.", __FUNCTION__, $pluginName);
$archiver->aggregateMultipleReports();
}
$this->logAggregator->setQueryOriginHint('');
Log::debug("PluginsArchiver::%s: %s while archiving %s reports for plugin '%s'.", __FUNCTION__, $timer->getMemoryLeak(), $this->params->getPeriod()->getLabel(), $pluginName);
} catch (Exception $e) {
$className = get_class($e);
$exception = new $className($e->getMessage() . " - caused by plugin {$pluginName}", $e->getCode(), $e);
throw $exception;
}
} else {
Log::debug("PluginsArchiver::%s: Not archiving reports for plugin '%s'.", __FUNCTION__, $pluginName);
}
Manager::getInstance()->deleteAll($latestUsedTableId);
unset($archiver);
}
}
public function setUp()
{
parent::setUp();
Manager::getInstance()->deleteAll();
}
public function setUp()
{
parent::setUp();
Config::getInstance()->setTestEnvironment();
Manager::getInstance()->deleteAll();
}
/**
* Clone. Called when cloning the datatable. We need to make sure to create a new datatableId.
* If we do not increase tableId it can result in segmentation faults when destructing a datatable.
*/
public function __clone()
{
// registers this instance to the manager
$this->currentId = Manager::getInstance()->addTable($this);
}
/**
* Returns the associated subtable, if one exists. Returns `false` if none exists.
*
* @return DataTable|bool
*/
public function getSubtable()
{
if ($this->isSubtableLoaded) {
try {
return Manager::getInstance()->getTable($this->subtableId);
} catch (TableNotFoundException $e) {
// edge case
}
}
return false;
}
/**
* @param array $alert
* @param int $idSite
* @param int $subPeriodN
*
* @return array
*/
public function getValueForAlertInPast($alert, $idSite, $subPeriodN)
{
$processedReport = new ProcessedReport();
$report = $processedReport->getReportMetadataByUniqueId($idSite, $alert['report']);
$dateInPast = $this->getDateForAlertInPast($idSite, $alert['period'], $subPeriodN);
$params = array('method' => $report['module'] . '.' . $report['action'], 'format' => 'original', 'idSite' => $idSite, 'period' => $alert['period'], 'date' => $dateInPast, 'flat' => 1, 'disable_queued_filters' => 1, 'filter_limit' => -1);
if (!empty($report['parameters'])) {
$params = array_merge($params, $report['parameters']);
}
$subtableId = DataTable\Manager::getInstance()->getMostRecentTableId();
$request = new ApiRequest($params);
$table = $request->process();
$value = $this->aggregateToOneValue($table, $alert['metric'], $alert['report_condition'], $alert['report_matched']);
DataTable\Manager::getInstance()->deleteAll($subtableId);
return $value;
}
/**
* For rows which have subtables (eg. directories with sub pages),
* deletes columns which don't make sense when all values of sub pages are summed.
*
* @param $dataTable DataTable
*/
public static function deleteInvalidSummedColumnsFromDataTable($dataTable)
{
foreach ($dataTable->getRows() as $id => $row) {
if (($idSubtable = $row->getIdSubDataTable()) !== null || $id === DataTable::ID_SUMMARY_ROW) {
if ($idSubtable !== null) {
$subtable = Manager::getInstance()->getTable($idSubtable);
self::deleteInvalidSummedColumnsFromDataTable($subtable);
}
if ($row instanceof DataTableSummaryRow) {
$row->recalculate();
}
foreach (Archiver::$columnsToDeleteAfterAggregation as $name) {
$row->deleteColumn($name);
}
}
}
// And this as well
ArchivingHelper::removeEmptyColumns($dataTable);
}