DataTable rows contain columns, metadata and a subtable ID. Columns and metadata
are stored as an array of name => value mappings.
/**
* @param Row $row
* @param DataTable $dataTable
* @param string $labelPrefix
* @param bool $parentLogo
*/
private function flattenRow(Row $row, DataTable $dataTable, $labelPrefix = '', $parentLogo = false)
{
$label = $row->getColumn('label');
if ($label !== false) {
$label = trim($label);
if (substr($label, 0, 1) == '/' && $this->recursiveLabelSeparator == '/') {
$label = substr($label, 1);
}
$label = $labelPrefix . $label;
$row->setColumn('label', $label);
}
$logo = $row->getMetadata('logo');
if ($logo === false && $parentLogo !== false) {
$logo = $parentLogo;
$row->setMetadata('logo', $logo);
}
$subTable = $this->loadSubtable($dataTable, $row);
$row->removeSubtable();
if ($subTable === null) {
if ($this->includeAggregateRows) {
$row->setMetadata('is_aggregate', 0);
}
$dataTable->addRow($row);
} else {
if ($this->includeAggregateRows) {
$row->setMetadata('is_aggregate', 1);
$dataTable->addRow($row);
}
$prefix = $label . $this->recursiveLabelSeparator;
foreach ($subTable->getRows() as $row) {
$this->flattenRow($row, $dataTable, $prefix, $logo);
}
}
}
private function tryToAddColumn(Row $row, $column, $callable)
{
try {
$row->addColumn($column, $callable);
} catch (\Exception $e) {
}
}
private function addRowWithMetadata($metadata)
{
$row = new Row(array(Row::COLUMNS => array('label' => 'val1')));
foreach ($metadata as $name => $value) {
$row->setMetadata($name, $value);
}
$this->table->addRow($row);
}
private function buildRowWithMetadata($metadata)
{
$row = new Row(array(Row::COLUMNS => array('label' => 'val1')));
foreach ($metadata as $name => $value) {
$row->setMetadata($name, $value);
}
return $row;
}
/**
* Filters a row's subtable, if one exists and is loaded in memory.
*
* @param Row $row The row whose subtable should be filter.
*/
public function filterSubTable(Row $row)
{
if (!$this->enableRecursive) {
return;
}
$subTable = $row->getSubtable();
if ($subTable) {
$this->filter($subTable);
}
}
/**
* Filters a row's subtable, if one exists and is loaded in memory.
*
* @param Row $row The row whose subtable should be filter.
*/
public function filterSubTable(Row $row)
{
if (!$this->enableRecursive) {
return;
}
if ($row->isSubtableLoaded()) {
$subTable = Manager::getInstance()->getTable($row->getIdSubDataTable());
$this->filter($subTable);
}
}
public function sort(Row $a, Row $b)
{
foreach ($this->columnsToCheck as $column) {
if ($column) {
$valA = $a->getColumn($column);
$valB = $b->getColumn($column);
$sort = $this->sortVal($valA, $valB);
if (isset($sort)) {
return $sort;
}
}
}
return 0;
}
private function addRow(DataTable $table, DataTable\Row $row, $growthPercentage, $newValue, $oldValue, $difference, $disappeared = false, $isNew = false, $isMover = false)
{
$columns = $row->getColumns();
$columns['growth_percent'] = $growthPercentage;
$columns['growth_percent_numeric'] = str_replace('%', '', $growthPercentage);
$columns['grown'] = '-' != substr($growthPercentage, 0, 1);
$columns['value_old'] = $oldValue;
$columns['value_new'] = $newValue;
$columns['difference'] = $difference;
$columns['importance'] = abs($difference);
$columns['isDisappeared'] = $disappeared;
$columns['isNew'] = $isNew;
$columns['isMover'] = $isMover;
$table->addRowFromArray(array(DataTable\Row::COLUMNS => $columns));
}
private function addSummaryRow($table)
{
$table->filter('Sort', array($this->columnToSortByBeforeTruncating, 'desc'));
if ($table->getRowsCount() <= $this->truncateAfter + 1) {
return;
}
$rows = $table->getRows();
$count = $table->getRowsCount();
$newRow = new Row(array(Row::COLUMNS => array('label' => DataTable::LABEL_SUMMARY_ROW)));
for ($i = $this->truncateAfter; $i < $count; $i++) {
if (!isset($rows[$i])) {
// case when the last row is a summary row, it is not indexed by $cout but by DataTable::ID_SUMMARY_ROW
$summaryRow = $table->getRowFromId(DataTable::ID_SUMMARY_ROW);
//FIXME: I'm not sure why it could return false, but it was reported in: http://forum.piwik.org/read.php?2,89324,page=1#msg-89442
if ($summaryRow) {
$newRow->sumRow($summaryRow, $enableCopyMetadata = false, $table->getMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME));
}
} else {
$newRow->sumRow($rows[$i], $enableCopyMetadata = false, $table->getMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME));
}
}
$table->filter('Limit', array(0, $this->truncateAfter));
$table->addSummaryRow($newRow);
unset($rows);
}
/**
* Constructor.
*
* @param DataTable|null $subTable The subtable of this row. This parameter is mostly for
* convenience. If set, its rows will be summed to this one,
* but it will not be set as this row's subtable (so
* getSubtable() will return false).
*/
public function __construct($subTable = null)
{
parent::__construct();
if ($subTable !== null) {
$this->sumTable($subTable);
}
}
/**
* @param Row $row
* @param DataTable $dataTable
* @param string $labelPrefix
* @param bool $parentLogo
*/
private function flattenRow(Row $row, $rowId, DataTable $dataTable, $labelPrefix = '', $parentLogo = false)
{
$label = $row->getColumn('label');
if ($label !== false) {
$label = trim($label);
if ($this->recursiveLabelSeparator == '/') {
if (substr($label, 0, 1) == '/') {
$label = substr($label, 1);
} elseif ($rowId === DataTable::ID_SUMMARY_ROW && $labelPrefix && $label != DataTable::LABEL_SUMMARY_ROW) {
$label = ' - ' . $label;
}
}
$label = $labelPrefix . $label;
$row->setColumn('label', $label);
}
$logo = $row->getMetadata('logo');
if ($logo === false && $parentLogo !== false) {
$logo = $parentLogo;
$row->setMetadata('logo', $logo);
}
/** @var DataTable $subTable */
$subTable = $row->getSubtable();
if ($subTable) {
$subTable->applyQueuedFilters();
$row->deleteMetadata('idsubdatatable_in_db');
} else {
$subTable = $this->loadSubtable($dataTable, $row);
}
$row->removeSubtable();
if ($subTable === null) {
if ($this->includeAggregateRows) {
$row->setMetadata('is_aggregate', 0);
}
$dataTable->addRow($row);
} else {
if ($this->includeAggregateRows) {
$row->setMetadata('is_aggregate', 1);
$dataTable->addRow($row);
}
$prefix = $label . $this->recursiveLabelSeparator;
foreach ($subTable->getRows() as $rowId => $row) {
$this->flattenRow($row, $rowId, $dataTable, $prefix, $logo);
}
}
}
/**
* Returns column from a given row.
* Will work with 2 types of datatable
* - raw datatables coming from the archive DB, which columns are int indexed
* - datatables processed resulting of API calls, which columns have human readable english names
*
* @param Row|array $row
* @param int $columnIdRaw see consts in Archive::
* @param bool|array $mappingIdToName
* @return mixed Value of column, false if not found
*/
public function getColumn($row, $columnIdRaw, $mappingIdToName = false)
{
if (empty($mappingIdToName)) {
$mappingIdToName = Metrics::$mappingFromIdToName;
}
$columnIdReadable = $mappingIdToName[$columnIdRaw];
if ($row instanceof Row) {
$raw = $row->getColumn($columnIdRaw);
if ($raw !== false) {
return $raw;
}
return $row->getColumn($columnIdReadable);
}
if (isset($row[$columnIdRaw])) {
return $row[$columnIdRaw];
}
if (isset($row[$columnIdReadable])) {
return $row[$columnIdReadable];
}
return false;
}
/**
* Aggregates the $row columns to this table.
*
* $row must have a column "label". The $row will be summed to this table's row with the same label.
*
* @param $row
* @params null|array $columnAggregationOps
* @throws \Exception
*/
protected function aggregateRowWithLabel(Row $row, $columnAggregationOps)
{
$labelToLookFor = $row->getColumn('label');
if ($labelToLookFor === false) {
throw new Exception("Label column not found in the table to add in addDataTable()");
}
$rowFound = $this->getRowFromLabel($labelToLookFor);
if ($rowFound === false) {
if ($labelToLookFor === self::LABEL_SUMMARY_ROW) {
$this->addSummaryRow($row);
} else {
$this->addRow($row);
}
} else {
$rowFound->sumRow($row, $copyMeta = true, $columnAggregationOps);
// if the row to add has a subtable whereas the current row doesn't
// we simply add it (cloning the subtable)
// if the row has the subtable already
// then we have to recursively sum the subtables
$subTable = $row->getSubtable();
if ($subTable) {
$subTable->metadata[self::COLUMN_AGGREGATION_OPS_METADATA_NAME] = $columnAggregationOps;
$rowFound->sumSubtable($subTable);
}
}
}
/**
* Utility function. Returns the current row in the past DataTable.
*
* @param Row $row The row in the 'current' DataTable.
* @return bool|Row
*/
protected function getPastRowFromCurrent($row)
{
return $this->pastDataTable->getRowFromLabel($row->getColumn('label'));
}
public function testWhenRowsInRandomOrderButSortSpecifiedShouldComputeSummaryRowAfterSort()
{
$table = new DataTable();
$table->addRow($this->getRow3());
$table->addRow($this->getRow2());
$table->addRow($this->getRow4());
$table->addRow($this->getRow1());
$table->addRow($this->getRow0());
$filter = new Truncate($table, 2, DataTable::LABEL_SUMMARY_ROW, $columnToSortBy = 'nb');
$filter->filter($table);
$this->assertEquals(3, $table->getRowsCount());
$expectedRow = new Row(array(Row::COLUMNS => array('label' => DataTable::LABEL_SUMMARY_ROW, 'nb' => 111)));
$this->assertTrue(Row::isEqual($table->getLastRow(), $expectedRow));
}
protected function enrichWithUniqueVisitorsMetric(Row $row)
{
if (!$this->getParams()->isSingleSite()) {
// we only compute unique visitors for a single site
return;
}
if ($row->getColumn('nb_uniq_visitors') !== false) {
if (SettingsPiwik::isUniqueVisitorsEnabled($this->getParams()->getPeriod()->getLabel())) {
$uniqueVisitors = (double) $this->computeNbUniqVisitors();
$row->setColumn('nb_uniq_visitors', $uniqueVisitors);
} else {
$row->deleteColumn('nb_uniq_visitors');
}
}
}
/**
* Sets the column to be used for Excluding low population
*
* @param DataTable\Row $row
* @return int
*/
private function selectColumnToExclude($columnToFilter, $row)
{
if ($row->hasColumn($columnToFilter)) {
return $columnToFilter;
}
// filter_excludelowpop=nb_visits but the column name is still Metrics::INDEX_NB_VISITS in the table
$columnIdToName = Metrics::getMappingFromNameToId();
if (isset($columnIdToName[$columnToFilter])) {
$column = $columnIdToName[$columnToFilter];
if ($row->hasColumn($column)) {
return $column;
}
}
return $columnToFilter;
}
/**
* @group Core
*/
public function testSubDataTableIsDestructed()
{
$mockedDataTable = $this->getMock('\\Piwik\\DataTable', array('__destruct'));
$mockedDataTable->expects($this->once())->method('__destruct');
$rowBeingDestructed = new Row();
$rowBeingDestructed->setSubtable($mockedDataTable);
Common::destroy($rowBeingDestructed);
}
/**
* Helper function that tests if two rows are equal.
*
* Two rows are equal if:
*
* - they have exactly the same columns / metadata
* - they have a subDataTable associated, then we check that both of them are the same.
*
* Column order is not important.
*
* @param \Piwik\DataTable\Row $row1 first to compare
* @param \Piwik\DataTable\Row $row2 second to compare
* @return bool
*/
public static function isEqual(Row $row1, Row $row2)
{
//same columns
$cols1 = $row1->getColumns();
$cols2 = $row2->getColumns();
$diff1 = array_udiff($cols1, $cols2, array(__CLASS__, 'compareElements'));
$diff2 = array_udiff($cols2, $cols1, array(__CLASS__, 'compareElements'));
if ($diff1 != $diff2) {
return false;
}
$dets1 = $row1->getMetadata();
$dets2 = $row2->getMetadata();
ksort($dets1);
ksort($dets2);
if ($dets1 != $dets2) {
return false;
}
// either both are null
// or both have a value
if (!(is_null($row1->getIdSubDataTable()) && is_null($row2->getIdSubDataTable()))) {
$subtable1 = $row1->getSubtable();
$subtable2 = $row2->getSubtable();
if (!DataTable::isEqual($subtable1, $subtable2)) {
return false;
}
}
return true;
}
/**
* Returns the element that should be replaced
*
* @param Row $row
* @param string $columnToFilter
* @return mixed
*/
protected function getElementToReplace($row, $columnToFilter)
{
return $row->getColumn($columnToFilter);
}
private function generateDataTableWithManySubtables($numSubtables)
{
$dataTable = new DataTable();
for ($i = 1; $i <= $numSubtables; $i++) {
$row = new Row(array(Row::COLUMNS => array('label' => 'Label Test ' . $i, 'nb_visits' => $i)));
$subtable = DataTable::makeFromSimpleArray(array(array('label' => 'subtable' . $i, 'nb_visits' => $i)));
$row->setSubtable($subtable);
$dataTable->addRow($row);
}
return $dataTable->getSerialized();
}
/**
* Load the subtable for a row.
* Returns null if none is found.
*
* @param DataTable $dataTable
* @param Row $row
*
* @return DataTable
*/
protected function loadSubtable($dataTable, $row)
{
if (!($this->apiModule && $this->apiMethod && count($this->request))) {
return null;
}
$request = $this->request;
$idSubTable = $row->getIdSubDataTable();
if ($idSubTable === null) {
return null;
}
$request['idSubtable'] = $idSubTable;
if ($dataTable) {
$period = $dataTable->getMetadata(DataTableFactory::TABLE_METADATA_PERIOD_INDEX);
if ($period instanceof Range) {
$request['date'] = $period->getDateStart() . ',' . $period->getDateEnd();
} else {
$request['date'] = $period->getDateStart()->toString();
}
}
$method = $this->getApiMethodForSubtable();
return $this->callApiAndReturnDataTable($this->apiModule, $method, $request);
}
private function mergeChildren_checkRow(Row $row, $expectedLabel, $expectedColumnValue)
{
$this->assertEquals($expectedLabel, $row->getColumn('label'));
$this->assertEquals($expectedColumnValue, $row->getColumn('col1'));
}
private function assertSegment($expected, Row $row)
{
$segment = $row->getMetadata('segment');
$this->assertSame($expected, $segment);
}
/**
* Convert a dimension-less report to a multi-row two-column data table
*
* @static
* @param $reportMetadata array
* @param $report DataTable
* @param $reportColumns array
* @return array DataTable $report & array $columns
*/
protected static function processTableFormat($reportMetadata, $report, $reportColumns)
{
$finalReport = $report;
if (empty($reportMetadata['dimension'])) {
$simpleReportMetrics = $report->getFirstRow();
if ($simpleReportMetrics) {
$finalReport = new Simple();
foreach ($simpleReportMetrics->getColumns() as $metricId => $metric) {
$newRow = new Row();
$newRow->addColumn("label", $reportColumns[$metricId]);
$newRow->addColumn("value", $metric);
$finalReport->addRow($newRow);
}
}
$reportColumns = array('label' => Piwik::translate('General_Name'), 'value' => Piwik::translate('General_Value'));
}
return array($finalReport, $reportColumns);
}
protected function enrichWithUniqueVisitorsMetric(Row $row)
{
// skip unique visitors metrics calculation if calculating for multiple sites is disabled
if (!$this->getParams()->isSingleSite() && $this->skipUniqueVisitorsCalculationForMultipleSites) {
return;
}
if ($row->getColumn('nb_uniq_visitors') === false && $row->getColumn('nb_users') === false) {
return;
}
if (!SettingsPiwik::isUniqueVisitorsEnabled($this->getParams()->getPeriod()->getLabel())) {
$row->deleteColumn('nb_uniq_visitors');
$row->deleteColumn('nb_users');
return;
}
$metrics = array(Metrics::INDEX_NB_USERS);
if ($this->getParams()->isSingleSite()) {
$uniqueVisitorsMetric = Metrics::INDEX_NB_UNIQ_VISITORS;
} else {
if (!SettingsPiwik::isSameFingerprintAcrossWebsites()) {
throw new Exception("Processing unique visitors across websites is enabled for this instance,\n but to process this metric you must first set enable_fingerprinting_across_websites=1\n in the config file, under the [Tracker] section.");
}
$uniqueVisitorsMetric = Metrics::INDEX_NB_UNIQ_FINGERPRINTS;
}
$metrics[] = $uniqueVisitorsMetric;
$uniques = $this->computeNbUniques($metrics);
// see edge case as described in https://github.com/piwik/piwik/issues/9357 where uniq_visitors might be higher
// than visits because we archive / process it after nb_visits. Between archiving nb_visits and nb_uniq_visitors
// there could have been a new visit leading to a higher nb_unique_visitors than nb_visits which is not possible
// by definition. In this case we simply use the visits metric instead of unique visitors metric.
$visits = $row->getColumn('nb_visits');
if ($visits !== false && $uniques[$uniqueVisitorsMetric] !== false) {
$uniques[$uniqueVisitorsMetric] = min($uniques[$uniqueVisitorsMetric], $visits);
}
$row->setColumn('nb_uniq_visitors', $uniques[$uniqueVisitorsMetric]);
$row->setColumn('nb_users', $uniques[Metrics::INDEX_NB_USERS]);
}
/** Get row evolution for a multiple labels */
private function getMultiRowEvolution(DataTable\Map $dataTable, $metadata, $apiModule, $apiAction, $labels, $column, $legendAppendMetric = true, $labelUseAbsoluteUrl = true)
{
if (!isset($metadata['metrics'][$column])) {
// invalid column => use the first one that's available
$metrics = array_keys($metadata['metrics']);
$column = reset($metrics);
}
// get the processed label and logo (if any) for every requested label
$actualLabels = $logos = array();
foreach ($labels as $labelIdx => $label) {
foreach ($dataTable->getDataTables() as $table) {
$labelRow = $this->getRowEvolutionRowFromLabelIdx($table, $labelIdx);
if ($labelRow) {
$actualLabels[$labelIdx] = $this->getRowUrlForEvolutionLabel($labelRow, $apiModule, $apiAction, $labelUseAbsoluteUrl);
$logos[$labelIdx] = $labelRow->getMetadata('logo');
if (!empty($actualLabels[$labelIdx])) {
break;
}
}
}
if (empty($actualLabels[$labelIdx])) {
$actualLabels[$labelIdx] = $this->cleanOriginalLabel($label);
}
}
// convert rows to be array($column.'_'.$labelIdx => $value) as opposed to
// array('label' => $label, 'column' => $value).
$dataTableMulti = $dataTable->getEmptyClone();
foreach ($dataTable->getDataTables() as $tableLabel => $table) {
$newRow = new Row();
foreach ($labels as $labelIdx => $label) {
$row = $this->getRowEvolutionRowFromLabelIdx($table, $labelIdx);
$value = 0;
if ($row) {
$value = $row->getColumn($column);
$value = floatVal(str_replace(',', '.', $value));
}
if ($value == '') {
$value = 0;
}
$newLabel = $column . '_' . (int) $labelIdx;
$newRow->addColumn($newLabel, $value);
}
$newTable = $table->getEmptyClone();
if (!empty($labels)) {
// only add a row if the row has data (no labels === no data)
$newTable->addRow($newRow);
}
$dataTableMulti->addTable($newTable, $tableLabel);
}
// the available metrics for the report are returned as metadata / columns
$metadata['columns'] = $metadata['metrics'];
// metadata / metrics should document the rows that are compared
// this way, UI code can be reused
$metadata['metrics'] = array();
foreach ($actualLabels as $labelIndex => $label) {
if ($legendAppendMetric) {
$label .= ' (' . $metadata['columns'][$column] . ')';
}
$metricName = $column . '_' . $labelIndex;
$metadata['metrics'][$metricName] = SafeDecodeLabel::decodeLabelSafe($label);
if (!empty($logos[$labelIndex])) {
$metadata['logos'][$metricName] = $logos[$labelIndex];
}
}
$this->enhanceRowEvolutionMetaData($metadata, $dataTableMulti);
return array('column' => $column, 'reportData' => $dataTableMulti, 'metadata' => $metadata);
}
/**
* Returns the divisor to use when calculating the new column value. Can
* be overridden by descendent classes to customize behavior.
*
* @param Row $row The row being modified.
* @return int|float
*/
protected function getDivisor($row)
{
if (!is_null($this->totalValueUsedAsDivisor)) {
return $this->totalValueUsedAsDivisor;
} else {
if ($this->getDivisorFromSummaryRow) {
$summaryRow = $this->table->getRowFromId(DataTable::ID_SUMMARY_ROW);
return $summaryRow->getColumn($this->columnNameUsedAsDivisor);
} else {
return $row->getColumn($this->columnNameUsedAsDivisor);
}
}
}
/**
* @param $columnRow
* @param $pivotColumn
* @return false|mixed
*/
private function getColumnValue(Row $columnRow, $pivotColumn)
{
$value = $columnRow->getColumn($pivotColumn);
if (empty($value) && !empty($this->metricIndexValue)) {
$value = $columnRow->getColumn($this->metricIndexValue);
}
return $value;
}
private function addColumn(Row $row, $columnName, $callback)
{
$this->expectedColumns[$columnName] = true;
$row->addColumn($columnName, $callback);
}
