public function testTransfer()
{
$cycleID = DWHInspector::getCycle('LOADED');
$ds_lines = array();
$files = DWHInspector::getFiles($cycleID);
$dsTablesToFactTables = $this->getDSTablesToFactTables();
foreach ($files as $fileID) {
foreach (array_keys($dsTablesToFactTables) as $dsTable) {
$ds_lines[$fileID][$dsTable] = DWHInspector::countRows('kalturadw_ds.' . $dsTable, $fileID);
}
}
DWHInspector::markAllAsAggregated();
$transferParams = $this->getTransferParams();
KettleRunner::execute($this->getTransferJob(), $transferParams);
$this->assertEquals($cycleID, DWHInspector::getCycle('DONE'));
$this->isCycleDirExists($cycleID, false);
$files = DWHInspector::getFiles($cycleID);
foreach ($files as $fileID) {
foreach ($dsTablesToFactTables as $dsTable => $factTable) {
// compare rows in ds_events and dwh_fact_events
$this->assertEquals($ds_lines[$fileID][$dsTable], DWHInspector::countRows('kalturadw.' . $factTable, $fileID));
// make sure ds_events was emptied
$this->assertEquals(0, DWHInspector::countRows('kalturadw_ds.' . $dsTable, $fileID));
}
}
foreach (DWHInspector::getAggrDatesAndHours($cycleID) as $table => $arr) {
$minDateID = DWHInspector::getResetAggregationsMinDateID($cycleID, $table);
$postTransferAggregationTypes = DWHInspector::getPostTransferAggregationTypes($transferParams[self::TRANSFER_PARAM_PROCESS_ID], $table);
foreach ($arr as $dateID => $hours) {
foreach ($hours as $hourID) {
foreach ($postTransferAggregationTypes as $aggrType) {
$filter = 'aggr_name = \'' . $aggrType . '\' and date_id = ' . $dateID . ' and hour_id = ' . $hourID . ' and ifnull(start_time,date(19700101)) < data_insert_time';
$rowExists = DWHInspector::rowExists('kalturadw.aggr_managment', $filter);
$this->assertEquals($dateID >= $minDateID, $rowExists, "Row Date: {$dateID} {$hourID} {$aggrType} . Min Date: {$minDateID}");
}
}
}
}
}
public function testAggregation()
{
// create entries for aggregation according to fact
DWHInspector::createEntriesFromFact();
parent::testAggregation();
$cycleID = DWHInspector::getCycle('DONE');
$factsToHours = DWHInspector::getAggrDatesAndHours($cycleID);
$factTable = 'kalturadw.dwh_fact_events';
$minDateID = DWHInspector::getResetAggregationsMinDateID($cycleID, $factTable);
foreach ($factsToHours[$factTable] as $dateID => $hours) {
if ($dateID < $minDateID) {
continue;
}
foreach ($hours as $hourID) {
$this->compareAggregation(array(new ComparedTable('partner_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('partner_id', 'kalturadw.dwh_hourly_partner', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
$this->compareAggregation(array(new ComparedTable('entry_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('entry_id', 'kalturadw.dwh_hourly_events_entry', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
$this->compareAggregation(array(new ComparedTable('domain_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('domain_id', 'kalturadw.dwh_hourly_events_domain', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
$this->compareAggregation(array(new ComparedTable('referrer_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('referrer_id', 'kalturadw.dwh_hourly_events_domain_referrer', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
$this->compareAggregation(array(new ComparedTable('location_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('location_id', 'kalturadw.dwh_hourly_events_country', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
$this->compareAggregation(array(new ComparedTable('country_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('country_id', 'kalturadw.dwh_hourly_events_country', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
$this->compareAggregation(array(new ComparedTable('widget_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('widget_id', 'kalturadw.dwh_hourly_events_widget', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
$this->compareAggregation(array(new ComparedTable('os_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('os_id', 'kalturadw.dwh_hourly_events_devices', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
$this->compareAggregation(array(new ComparedTable('browser_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('browser_id', 'kalturadw.dwh_hourly_events_devices', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
}
$this->compareAggregation(array(new ComparedTable('partner_id', 'kalturadw.dwh_fact_bandwidth_usage', '(bandwidth_bytes/1024)'), new ComparedTable('session_partner_id', 'kalturadw.dwh_fact_fms_sessions', '(total_bytes/1024)')), array(new ComparedTable('partner_id', 'kalturadw.dwh_hourly_partner_usage', 'ifnull(count_bandwidth_kb, 0)')), 1, 'activity_date_id = ' . $dateID, 'date_id = ' . $dateID);
$this->compareAggregation(array(new ComparedTable('bandwidth_source_id', 'kalturadw.dwh_fact_bandwidth_usage', '(bandwidth_bytes/1024)'), new ComparedTable('bandwidth_source_id', 'kalturadw.dwh_fact_fms_sessions', '(total_bytes/1024)')), array(new ComparedTable('bandwidth_source_id', 'kalturadw.dwh_hourly_partner_usage', 'ifnull(count_bandwidth_kb,0)')), 1, 'activity_date_id = ' . $dateID, 'date_id = ' . $dateID);
$this->compareAggregation(array(new ComparedTable('location_id', 'kalturadw.dwh_fact_bandwidth_usage', '(bandwidth_bytes/1024)'), new ComparedTable('location_id', 'kalturadw.dwh_fact_fms_sessions', '(total_bytes/1024)')), array(new ComparedTable('location_id', 'kalturadw.dwh_hourly_events_devices', 'ifnull(count_bandwidth_kb, 0)')), 1, 'activity_date_id = ' . $dateID, 'date_id = ' . $dateID);
$this->compareAggregation(array(new ComparedTable('country_id', 'kalturadw.dwh_fact_bandwidth_usage', '(bandwidth_bytes/1024)'), new ComparedTable('country_id', 'kalturadw.dwh_fact_fms_sessions', '(total_bytes/1024)')), array(new ComparedTable('country_id', 'kalturadw.dwh_hourly_events_devices', 'ifnull(count_bandwidth_kb, 0)')), 1, 'activity_date_id = ' . $dateID, 'date_id = ' . $dateID);
}
$this->compareAggregation(array(new ComparedTable('entry_id', 'kalturadw.dwh_hourly_events_entry', 'ifnull(count_plays,0)')), array(new ComparedTable('entry_id', 'kalturadw.dwh_entry_plays_views', 'ifnull(plays,0)')));
$this->compareAggregation(array(new ComparedTable('entry_id', 'kalturadw.dwh_hourly_events_entry', 'ifnull(count_loads,0)')), array(new ComparedTable('entry_id', 'kalturadw.dwh_entry_plays_views', 'ifnull(views,0)')));
}
