public function testTransfer() { $cycleID = DWHInspector::getCycle('LOADED'); $ds_lines = array(); $files = DWHInspector::getFiles($cycleID); $dsTablesToFactTables = $this->getDSTablesToFactTables(); foreach ($files as $fileID) { foreach (array_keys($dsTablesToFactTables) as $dsTable) { $ds_lines[$fileID][$dsTable] = DWHInspector::countRows('kalturadw_ds.' . $dsTable, $fileID); } } DWHInspector::markAllAsAggregated(); $transferParams = $this->getTransferParams(); KettleRunner::execute($this->getTransferJob(), $transferParams); $this->assertEquals($cycleID, DWHInspector::getCycle('DONE')); $this->isCycleDirExists($cycleID, false); $files = DWHInspector::getFiles($cycleID); foreach ($files as $fileID) { foreach ($dsTablesToFactTables as $dsTable => $factTable) { // compare rows in ds_events and dwh_fact_events $this->assertEquals($ds_lines[$fileID][$dsTable], DWHInspector::countRows('kalturadw.' . $factTable, $fileID)); // make sure ds_events was emptied $this->assertEquals(0, DWHInspector::countRows('kalturadw_ds.' . $dsTable, $fileID)); } } foreach (DWHInspector::getAggrDatesAndHours($cycleID) as $table => $arr) { $minDateID = DWHInspector::getResetAggregationsMinDateID($cycleID, $table); $postTransferAggregationTypes = DWHInspector::getPostTransferAggregationTypes($transferParams[self::TRANSFER_PARAM_PROCESS_ID], $table); foreach ($arr as $dateID => $hours) { foreach ($hours as $hourID) { foreach ($postTransferAggregationTypes as $aggrType) { $filter = 'aggr_name = \'' . $aggrType . '\' and date_id = ' . $dateID . ' and hour_id = ' . $hourID . ' and ifnull(start_time,date(19700101)) < data_insert_time'; $rowExists = DWHInspector::rowExists('kalturadw.aggr_managment', $filter); $this->assertEquals($dateID >= $minDateID, $rowExists, "Row Date: {$dateID} {$hourID} {$aggrType} . Min Date: {$minDateID}"); } } } } }
public function testAggregation() { // create entries for aggregation according to fact DWHInspector::createEntriesFromFact(); parent::testAggregation(); $cycleID = DWHInspector::getCycle('DONE'); $factsToHours = DWHInspector::getAggrDatesAndHours($cycleID); $factTable = 'kalturadw.dwh_fact_events'; $minDateID = DWHInspector::getResetAggregationsMinDateID($cycleID, $factTable); foreach ($factsToHours[$factTable] as $dateID => $hours) { if ($dateID < $minDateID) { continue; } foreach ($hours as $hourID) { $this->compareAggregation(array(new ComparedTable('partner_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('partner_id', 'kalturadw.dwh_hourly_partner', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID); $this->compareAggregation(array(new ComparedTable('entry_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('entry_id', 'kalturadw.dwh_hourly_events_entry', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID); $this->compareAggregation(array(new ComparedTable('domain_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('domain_id', 'kalturadw.dwh_hourly_events_domain', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID); $this->compareAggregation(array(new ComparedTable('referrer_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('referrer_id', 'kalturadw.dwh_hourly_events_domain_referrer', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID); $this->compareAggregation(array(new ComparedTable('location_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('location_id', 'kalturadw.dwh_hourly_events_country', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID); $this->compareAggregation(array(new ComparedTable('country_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('country_id', 'kalturadw.dwh_hourly_events_country', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID); $this->compareAggregation(array(new ComparedTable('widget_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('widget_id', 'kalturadw.dwh_hourly_events_widget', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID); $this->compareAggregation(array(new ComparedTable('os_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('os_id', 'kalturadw.dwh_hourly_events_devices', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID); $this->compareAggregation(array(new ComparedTable('browser_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('browser_id', 'kalturadw.dwh_hourly_events_devices', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID); } $this->compareAggregation(array(new ComparedTable('partner_id', 'kalturadw.dwh_fact_bandwidth_usage', '(bandwidth_bytes/1024)'), new ComparedTable('session_partner_id', 'kalturadw.dwh_fact_fms_sessions', '(total_bytes/1024)')), array(new ComparedTable('partner_id', 'kalturadw.dwh_hourly_partner_usage', 'ifnull(count_bandwidth_kb, 0)')), 1, 'activity_date_id = ' . $dateID, 'date_id = ' . $dateID); $this->compareAggregation(array(new ComparedTable('bandwidth_source_id', 'kalturadw.dwh_fact_bandwidth_usage', '(bandwidth_bytes/1024)'), new ComparedTable('bandwidth_source_id', 'kalturadw.dwh_fact_fms_sessions', '(total_bytes/1024)')), array(new ComparedTable('bandwidth_source_id', 'kalturadw.dwh_hourly_partner_usage', 'ifnull(count_bandwidth_kb,0)')), 1, 'activity_date_id = ' . $dateID, 'date_id = ' . $dateID); $this->compareAggregation(array(new ComparedTable('location_id', 'kalturadw.dwh_fact_bandwidth_usage', '(bandwidth_bytes/1024)'), new ComparedTable('location_id', 'kalturadw.dwh_fact_fms_sessions', '(total_bytes/1024)')), array(new ComparedTable('location_id', 'kalturadw.dwh_hourly_events_devices', 'ifnull(count_bandwidth_kb, 0)')), 1, 'activity_date_id = ' . $dateID, 'date_id = ' . $dateID); $this->compareAggregation(array(new ComparedTable('country_id', 'kalturadw.dwh_fact_bandwidth_usage', '(bandwidth_bytes/1024)'), new ComparedTable('country_id', 'kalturadw.dwh_fact_fms_sessions', '(total_bytes/1024)')), array(new ComparedTable('country_id', 'kalturadw.dwh_hourly_events_devices', 'ifnull(count_bandwidth_kb, 0)')), 1, 'activity_date_id = ' . $dateID, 'date_id = ' . $dateID); } $this->compareAggregation(array(new ComparedTable('entry_id', 'kalturadw.dwh_hourly_events_entry', 'ifnull(count_plays,0)')), array(new ComparedTable('entry_id', 'kalturadw.dwh_entry_plays_views', 'ifnull(plays,0)'))); $this->compareAggregation(array(new ComparedTable('entry_id', 'kalturadw.dwh_hourly_events_entry', 'ifnull(count_loads,0)')), array(new ComparedTable('entry_id', 'kalturadw.dwh_entry_plays_views', 'ifnull(views,0)'))); }