private function aggregateByLabel($labelSQL, $recordName)
{
$metrics = $this->getLogAggregator()->getMetricsFromVisitByDimension($labelSQL);
if (in_array($recordName, array(self::DEVICE_TYPE_RECORD_NAME, self::DEVICE_BRAND_RECORD_NAME, self::DEVICE_MODEL_RECORD_NAME))) {
$labelSQL = str_replace('log_visit.', 'log_conversion.', $labelSQL);
$query = $this->getLogAggregator()->queryConversionsByDimension(array($labelSQL));
if ($query === false) {
return;
}
while ($conversionRow = $query->fetch()) {
$metrics->sumMetricsGoals($conversionRow[$labelSQL], $conversionRow);
}
$metrics->enrichMetricsWithConversions();
}
$table = $metrics->asDataTable();
$report = $table->getSerialized($this->maximumRows, null, Metrics::INDEX_NB_VISITS);
Common::destroy($table);
$this->getProcessor()->insertBlobRecord($recordName, $report);
unset($table, $report);
}
/**
* Deletes (unsets) the datatable given its id and removes it from the manager
* Subsequent get for this table will fail
*
* @param int $id
*/
public function deleteTable($id)
{
if (isset($this->tables[$id])) {
Common::destroy($this->tables[$id]);
$this->setTableDeleted($id);
}
}
/**
* Pivots to table.
*
* @param DataTable $table The table to manipulate.
*/
public function filter($table)
{
// set of all column names in the pivoted table mapped with the sum of all column
// values. used later in truncating and ordering the pivoted table's columns.
$columnSet = array();
// if no pivot column was set, use the first one found in the row
if (empty($this->pivotColumn)) {
$this->pivotColumn = $this->getNameOfFirstNonLabelColumnInTable($table);
}
Log::debug("PivotByDimension::%s: pivoting table with pivot column = %s", __FUNCTION__, $this->pivotColumn);
foreach ($table->getRows() as $row) {
$row->setColumns(array('label' => $row->getColumn('label')));
$associatedTable = $this->getIntersectedTable($table, $row);
if (!empty($associatedTable)) {
foreach ($associatedTable->getRows() as $columnRow) {
$pivotTableColumn = $columnRow->getColumn('label');
$columnValue = $this->getColumnValue($columnRow, $this->pivotColumn);
if (isset($columnSet[$pivotTableColumn])) {
$columnSet[$pivotTableColumn] += $columnValue;
} else {
$columnSet[$pivotTableColumn] = $columnValue;
}
$row->setColumn($pivotTableColumn, $columnValue);
}
Common::destroy($associatedTable);
unset($associatedTable);
}
}
Log::debug("PivotByDimension::%s: pivoted columns set: %s", __FUNCTION__, $columnSet);
$others = Piwik::translate('General_Others');
$defaultRow = $this->getPivotTableDefaultRowFromColumnSummary($columnSet, $others);
Log::debug("PivotByDimension::%s: un-prepended default row: %s", __FUNCTION__, $defaultRow);
// post process pivoted datatable
foreach ($table->getRows() as $row) {
// remove subtables from rows
$row->removeSubtable();
$row->deleteMetadata('idsubdatatable_in_db');
// use default row to ensure column ordering and add missing columns/aggregate cut-off columns
$orderedColumns = $defaultRow;
foreach ($row->getColumns() as $name => $value) {
if (isset($orderedColumns[$name])) {
$orderedColumns[$name] = $value;
} else {
$orderedColumns[$others] += $value;
}
}
$row->setColumns($orderedColumns);
}
$table->clearQueuedFilters();
// TODO: shouldn't clear queued filters, but we can't wait for them to be run
// since generic filters are run before them. remove after refactoring
// processed metrics.
// prepend numerals to columns in a queued filter (this way, disable_queued_filters can be used
// to get machine readable data from the API if needed)
$prependedColumnNames = $this->getOrderedColumnsWithPrependedNumerals($defaultRow, $others);
Log::debug("PivotByDimension::%s: prepended column name mapping: %s", __FUNCTION__, $prependedColumnNames);
$table->queueFilter(function (DataTable $table) use($prependedColumnNames) {
foreach ($table->getRows() as $row) {
$row->setColumns(array_combine($prependedColumnNames, $row->getColumns()));
}
});
}
/**
* Destructor. Makes sure DataTable memory will be cleaned up.
*/
public function __destruct()
{
static $depth = 0;
// destruct can be called several times
if ($depth < self::$maximumDepthLevelAllowed && isset($this->rows)) {
$depth++;
foreach ($this->rows as $row) {
Common::destroy($row);
}
if (isset($this->summaryRow)) {
Common::destroy($this->summaryRow);
}
unset($this->rows);
Manager::getInstance()->setTableDeleted($this->currentId);
$depth--;
}
}
/**
* Sums records for every subperiod of the current period and inserts the result as the record
* for this period.
*
* DataTables are summed recursively so subtables will be summed as well.
*
* @param string|array $recordNames Name(s) of the report we are aggregating, eg, `'Referrers_type'`.
* @param int $maximumRowsInDataTableLevelZero Maximum number of rows allowed in the top level DataTable.
* @param int $maximumRowsInSubDataTable Maximum number of rows allowed in each subtable.
* @param string $columnToSortByBeforeTruncation The name of the column to sort by before truncating a DataTable.
* @param array $columnsAggregationOperation Operations for aggregating columns, see {@link Row::sumRow()}.
* @param array $columnsToRenameAfterAggregation Columns mapped to new names for columns that must change names
* when summed because they cannot be summed, eg,
* `array('nb_uniq_visitors' => 'sum_daily_nb_uniq_visitors')`.
* @return array Returns the row counts of each aggregated report before truncation, eg,
*
* array(
* 'report1' => array('level0' => $report1->getRowsCount,
* 'recursive' => $report1->getRowsCountRecursive()),
* 'report2' => array('level0' => $report2->getRowsCount,
* 'recursive' => $report2->getRowsCountRecursive()),
* ...
* )
* @api
*/
public function aggregateDataTableRecords($recordNames, $maximumRowsInDataTableLevelZero = null, $maximumRowsInSubDataTable = null, $columnToSortByBeforeTruncation = null, &$columnsAggregationOperation = null, $columnsToRenameAfterAggregation = null)
{
if (!is_array($recordNames)) {
$recordNames = array($recordNames);
}
$nameToCount = array();
foreach ($recordNames as $recordName) {
$latestUsedTableId = Manager::getInstance()->getMostRecentTableId();
$table = $this->aggregateDataTableRecord($recordName, $columnsAggregationOperation, $columnsToRenameAfterAggregation);
$rowsCount = $table->getRowsCount();
$nameToCount[$recordName]['level0'] = $rowsCount;
$rowsCountRecursive = $rowsCount;
if ($this->isAggregateSubTables()) {
$rowsCountRecursive = $table->getRowsCountRecursive();
}
$nameToCount[$recordName]['recursive'] = $rowsCountRecursive;
$blob = $table->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation);
Common::destroy($table);
$this->insertBlobRecord($recordName, $blob);
unset($blob);
DataTable\Manager::getInstance()->deleteAll($latestUsedTableId);
}
return $nameToCount;
}
/**
* @group Core
*/
public function testSubDataTableIsDestructed()
{
$mockedDataTable = $this->getMock('\\Piwik\\DataTable', array('__destruct'));
$mockedDataTable->expects($this->once())->method('__destruct');
$rowBeingDestructed = new Row();
$rowBeingDestructed->setSubtable($mockedDataTable);
Common::destroy($rowBeingDestructed);
}
/**
* Merges the rows of every child {@link DataTable} into a new one and
* returns it. This function will also set the label of the merged rows
* to the label of the {@link DataTable} they were originally from.
*
* The result of this function is determined by the type of DataTable
* this instance holds. If this DataTable\Map instance holds an array
* of DataTables, this function will transform it from:
*
* Label 0:
* DataTable(row1)
* Label 1:
* DataTable(row2)
*
* to:
*
* DataTable(row1[label = 'Label 0'], row2[label = 'Label 1'])
*
* If this instance holds an array of DataTable\Maps, this function will
* transform it from:
*
* Outer Label 0: // the outer DataTable\Map
* Inner Label 0: // one of the inner DataTable\Maps
* DataTable(row1)
* Inner Label 1:
* DataTable(row2)
* Outer Label 1:
* Inner Label 0:
* DataTable(row3)
* Inner Label 1:
* DataTable(row4)
*
* to:
*
* Inner Label 0:
* DataTable(row1[label = 'Outer Label 0'], row3[label = 'Outer Label 1'])
* Inner Label 1:
* DataTable(row2[label = 'Outer Label 0'], row4[label = 'Outer Label 1'])
*
* If this instance holds an array of DataTable\Maps, the
* metadata of the first child is used as the metadata of the result.
*
* This function can be used, for example, to smoosh IndexedBySite archive
* query results into one DataTable w/ different rows differentiated by site ID.
*
* Note: This DataTable/Map will be destroyed and will be no longer usable after the tables have been merged into
* the new dataTable to reduce memory usage. Destroying all DataTables witihn the Map also seems to fix a
* Segmentation Fault that occurred in the AllWebsitesDashboard when having > 16k sites.
*
* @return DataTable|Map
*/
public function mergeChildren()
{
$firstChild = reset($this->array);
if ($firstChild instanceof Map) {
$result = $firstChild->getEmptyClone();
/** @var $subDataTableMap Map */
foreach ($this->getDataTables() as $label => $subDataTableMap) {
foreach ($subDataTableMap->getDataTables() as $innerLabel => $subTable) {
if (!isset($result->array[$innerLabel])) {
$dataTable = new DataTable();
$dataTable->setMetadataValues($subTable->getAllTableMetadata());
$result->addTable($dataTable, $innerLabel);
}
$this->copyRowsAndSetLabel($result->array[$innerLabel], $subTable, $label);
}
}
} else {
$result = new DataTable();
foreach ($this->getDataTables() as $label => $subTable) {
$this->copyRowsAndSetLabel($result, $subTable, $label);
Common::destroy($subTable);
}
$this->array = array();
}
return $result;
}
/**
* @param int $idSite
* @param string $period
* @param string $date
* @param string|false $segment
* @param bool $expanded
* @param DataTable $dataTable
*/
private function buildExpandedTableForFlattenGetSocials($idSite, $period, $date, $segment, $expanded, $dataTable)
{
$urlsTable = Archive::createDataTableFromArchive(Archiver::WEBSITES_RECORD_NAME, $idSite, $period, $date, $segment, $expanded, $flat = true);
$urlsTable->filter('ColumnCallbackDeleteRow', array('label', function ($url) {
return !Social::getInstance()->isSocialUrl($url);
}));
$urlsTable = $urlsTable->mergeSubtables();
foreach ($dataTable->getRows() as $row) {
$row->removeSubtable();
$social = $row->getColumn('label');
$newTable = $urlsTable->getEmptyClone();
$rows = $urlsTable->getRows();
foreach ($rows as $id => $urlsTableRow) {
$url = $urlsTableRow->getColumn('label');
if (Social::getInstance()->isSocialUrl($url, $social)) {
$newTable->addRow($urlsTableRow);
$urlsTable->deleteRow($id);
}
}
if ($newTable->getRowsCount()) {
$newTable->filter('Piwik\\Plugins\\Referrers\\DataTable\\Filter\\UrlsForSocial', array($expanded));
$row->setSubtable($newTable);
}
}
Common::destroy($urlsTable);
$urlsTable = null;
}
private function buildDataTable($idSitesOrIdSite, $period, $date, $segment, $_restrictSitesToLogin, $enhanced, $multipleWebsitesRequested)
{
$allWebsitesRequested = $idSitesOrIdSite == 'all';
if ($allWebsitesRequested) {
// First clear cache
Site::clearCache();
// Then, warm the cache with only the data we should have access to
if (Piwik::hasUserSuperUserAccess() && !TaskScheduler::isTaskBeingExecuted()) {
APISitesManager::getInstance()->getAllSites();
} else {
APISitesManager::getInstance()->getSitesWithAtLeastViewAccess($limit = false, $_restrictSitesToLogin);
}
// Both calls above have called Site::setSitesFromArray. We now get these sites:
$sitesToProblablyAdd = Site::getSites();
} else {
$sitesToProblablyAdd = array(APISitesManager::getInstance()->getSiteFromId($idSitesOrIdSite));
}
// build the archive type used to query archive data
$archive = Archive::build($idSitesOrIdSite, $period, $date, $segment, $_restrictSitesToLogin);
// determine what data will be displayed
$fieldsToGet = array();
$columnNameRewrites = array();
$apiECommerceMetrics = array();
$apiMetrics = API::getApiMetrics($enhanced);
foreach ($apiMetrics as $metricName => $metricSettings) {
$fieldsToGet[] = $metricSettings[self::METRIC_RECORD_NAME_KEY];
$columnNameRewrites[$metricSettings[self::METRIC_RECORD_NAME_KEY]] = $metricName;
if ($metricSettings[self::METRIC_IS_ECOMMERCE_KEY]) {
$apiECommerceMetrics[$metricName] = $metricSettings;
}
}
// get the data
// $dataTable instanceOf Set
$dataTable = $archive->getDataTableFromNumeric($fieldsToGet);
$dataTable = $this->mergeDataTableMapAndPopulateLabel($idSitesOrIdSite, $multipleWebsitesRequested, $dataTable);
if ($dataTable instanceof DataTable\Map) {
foreach ($dataTable->getDataTables() as $table) {
$this->addMissingWebsites($table, $fieldsToGet, $sitesToProblablyAdd);
}
} else {
$this->addMissingWebsites($dataTable, $fieldsToGet, $sitesToProblablyAdd);
}
// calculate total visits/actions/revenue
$this->setMetricsTotalsMetadata($dataTable, $apiMetrics);
// if the period isn't a range & a lastN/previousN date isn't used, we get the same
// data for the last period to show the evolution of visits/actions/revenue
list($strLastDate, $lastPeriod) = Range::getLastDate($date, $period);
if ($strLastDate !== false) {
if ($lastPeriod !== false) {
// NOTE: no easy way to set last period date metadata when range of dates is requested.
// will be easier if DataTable\Map::metadata is removed, and metadata that is
// put there is put directly in DataTable::metadata.
$dataTable->setMetadata(self::getLastPeriodMetadataName('date'), $lastPeriod);
}
$pastArchive = Archive::build($idSitesOrIdSite, $period, $strLastDate, $segment, $_restrictSitesToLogin);
$pastData = $pastArchive->getDataTableFromNumeric($fieldsToGet);
$pastData = $this->mergeDataTableMapAndPopulateLabel($idSitesOrIdSite, $multipleWebsitesRequested, $pastData);
// use past data to calculate evolution percentages
$this->calculateEvolutionPercentages($dataTable, $pastData, $apiMetrics);
Common::destroy($pastData);
}
// remove eCommerce related metrics on non eCommerce Piwik sites
// note: this is not optimal in terms of performance: those metrics should not be retrieved in the first place
if ($enhanced) {
if ($dataTable instanceof DataTable\Map) {
foreach ($dataTable->getDataTables() as $table) {
$this->removeEcommerceRelatedMetricsOnNonEcommercePiwikSites($table, $apiECommerceMetrics);
}
} else {
$this->removeEcommerceRelatedMetricsOnNonEcommercePiwikSites($dataTable, $apiECommerceMetrics);
}
}
// move the site id to a metadata column
$dataTable->filter('ColumnCallbackAddMetadata', array('label', 'group', array('\\Piwik\\Site', 'getGroupFor'), array()));
$dataTable->filter('ColumnCallbackAddMetadata', array('label', 'main_url', array('\\Piwik\\Site', 'getMainUrlFor'), array()));
$dataTable->filter('ColumnCallbackAddMetadata', array('label', 'idsite'));
// set the label of each row to the site name
if ($multipleWebsitesRequested) {
$dataTable->filter('ColumnCallbackReplace', array('label', '\\Piwik\\Site::getNameFor'));
} else {
$dataTable->filter('ColumnDelete', array('label'));
}
Site::clearCache();
// replace record names with user friendly metric names
$dataTable->filter('ReplaceColumnNames', array($columnNameRewrites));
// Ensures data set sorted, for Metadata output
$dataTable->filter('Sort', array(self::NB_VISITS_METRIC, 'desc', $naturalSort = false));
// filter rows without visits
// note: if only one website is queried and there are no visits, we can not remove the row otherwise
// ResponseBuilder throws 'Call to a member function getColumns() on a non-object'
if ($multipleWebsitesRequested && !$enhanced) {
$dataTable->filter('ColumnCallbackDeleteRow', array(self::NB_VISITS_METRIC, function ($value) {
return $value == 0;
}));
}
return $dataTable;
}