public function testTransfer()
{
$cycleID = DWHInspector::getCycle('LOADED');
$files = DWHInspector::getFiles($cycleID);
$dsSessions = array();
foreach ($files as $fileID) {
$dsSessions[$fileID] = DWHInspector::getFullDSFMSSessions($fileID, implode(",", $this->ignoredPartners));
}
parent::testTransfer();
foreach ($files as $fileID) {
$factSessions = DWHInspector::getFactFMSSessions($fileID);
$this->assertEquals(count($dsSessions[$fileID]), count($factSessions));
foreach ($dsSessions[$fileID] as $sessionID => $dsSessionDictionary) {
$this->assertEquals($dsSessionDictionary, $factSessions[$sessionID]);
}
}
}
public function testProcess()
{
parent::testProcess();
global $CONF;
$cycleID = DWHInspector::getCycle('LOADED');
$files = DWHInspector::getFiles($cycleID);
foreach ($files as $fileID) {
$filename = $CONF->ProcessPath . "/" . $cycleID . '/' . DWHInspector::getFileName($fileID);
// compare rows in ds_bandwidth_usage to rows in file
$this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_bandwidth_usage', $fileID), $this->countRows($filename, array($this, 'validBWLine')));
$this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_bandwidth_usage', $fileID, 'and bandwidth_source_id = ' . $this->getBandwidthSourceID()), $this->countRows($filename, array($this, 'validBWLine')));
// compare bandwidth_bytes in ds_bandwidth_usage to bandwidth bytes consumed in file
$this->assertEquals(DWHInspector::sumRows('kalturadw_ds.ds_bandwidth_usage', $fileID, "bandwidth_bytes"), $this->sumBytes($filename, array($this, 'validBWLine'), $this->getBWRegex()));
// compare bw consumption per partner
$bwPartners = $this->countBWEventsPerPartner($filename);
$this->assertEquals(count($bwPartners), DWHInspector::countDistinct('kalturadw_ds.ds_bandwidth_usage', $fileID, 'partner_id'));
foreach ($bwPartners as $partner => $val) {
$res = DWHInspector::sumRows('kalturadw_ds.ds_bandwidth_usage', $fileID, 'bandwidth_bytes', ' and partner_id=\'' . $partner . '\'');
$this->assertEquals($res, $val);
}
// make sure there are very little invalid lines
$this->assertEquals($this->countInvalidLines($filename, array($this, 'validBWLine'), array($this, 'ignoredInvalidBWLine')), DWHInspector::countRows('kalturadw_ds.invalid_ds_lines', $fileID));
}
}
public function testTransfer()
{
$cycleID = DWHInspector::getCycle('LOADED');
$ds_lines = array();
$files = DWHInspector::getFiles($cycleID);
$dsTablesToFactTables = $this->getDSTablesToFactTables();
foreach ($files as $fileID) {
foreach (array_keys($dsTablesToFactTables) as $dsTable) {
$ds_lines[$fileID][$dsTable] = DWHInspector::countRows('kalturadw_ds.' . $dsTable, $fileID);
}
}
DWHInspector::markAllAsAggregated();
$transferParams = $this->getTransferParams();
KettleRunner::execute($this->getTransferJob(), $transferParams);
$this->assertEquals($cycleID, DWHInspector::getCycle('DONE'));
$this->isCycleDirExists($cycleID, false);
$files = DWHInspector::getFiles($cycleID);
foreach ($files as $fileID) {
foreach ($dsTablesToFactTables as $dsTable => $factTable) {
// compare rows in ds_events and dwh_fact_events
$this->assertEquals($ds_lines[$fileID][$dsTable], DWHInspector::countRows('kalturadw.' . $factTable, $fileID));
// make sure ds_events was emptied
$this->assertEquals(0, DWHInspector::countRows('kalturadw_ds.' . $dsTable, $fileID));
}
}
foreach (DWHInspector::getAggrDatesAndHours($cycleID) as $table => $arr) {
$minDateID = DWHInspector::getResetAggregationsMinDateID($cycleID, $table);
$postTransferAggregationTypes = DWHInspector::getPostTransferAggregationTypes($transferParams[self::TRANSFER_PARAM_PROCESS_ID], $table);
foreach ($arr as $dateID => $hours) {
foreach ($hours as $hourID) {
foreach ($postTransferAggregationTypes as $aggrType) {
$filter = 'aggr_name = \'' . $aggrType . '\' and date_id = ' . $dateID . ' and hour_id = ' . $hourID . ' and ifnull(start_time,date(19700101)) < data_insert_time';
$rowExists = DWHInspector::rowExists('kalturadw.aggr_managment', $filter);
$this->assertEquals($dateID >= $minDateID, $rowExists, "Row Date: {$dateID} {$hourID} {$aggrType} . Min Date: {$minDateID}");
}
}
}
}
}
public function testTransfer()
{
$cycleID = DWHInspector::getCycle('LOADED');
$files = DWHInspector::getFiles($cycleID);
$unifiedApiCalls = DWHInspector::getUnifiedAPICalls($cycleID);
$errornousUnifiedApiCalls = DWHInspector::getUnifiedAPICalls($cycleID, true);
parent::testTransfer();
foreach ($unifiedApiCalls as $call) {
$this->assertEquals(DWHInspector::countRows('kalturadw.dwh_fact_api_calls', '%', "and CONCAT(session_id,'_',request_index) = '" . $call->getID() . "'"), 1, "APICall ID " . $call->getID());
$this->assertEquals(DWHInspector::countRows('kalturadw.dwh_fact_incomplete_api_calls', '%', "and CONCAT(session_id,'_',request_index) = '" . $call->getID() . "'"), 0, "APICall ID " . $call->getID());
$this->assertEquals(DWHInspector::countRows('kalturadw.dwh_fact_errors', '%', "and error_object_id = '" . $call->getID() . "'"), 1, "APICall ID " . $call->getID());
}
}
public function testAggregation()
{
// create entries for aggregation according to fact
DWHInspector::createEntriesFromFact();
parent::testAggregation();
$cycleID = DWHInspector::getCycle('DONE');
$factsToHours = DWHInspector::getAggrDatesAndHours($cycleID);
$factTable = 'kalturadw.dwh_fact_events';
$minDateID = DWHInspector::getResetAggregationsMinDateID($cycleID, $factTable);
foreach ($factsToHours[$factTable] as $dateID => $hours) {
if ($dateID < $minDateID) {
continue;
}
foreach ($hours as $hourID) {
$this->compareAggregation(array(new ComparedTable('partner_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('partner_id', 'kalturadw.dwh_hourly_partner', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
$this->compareAggregation(array(new ComparedTable('entry_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('entry_id', 'kalturadw.dwh_hourly_events_entry', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
$this->compareAggregation(array(new ComparedTable('domain_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('domain_id', 'kalturadw.dwh_hourly_events_domain', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
$this->compareAggregation(array(new ComparedTable('referrer_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('referrer_id', 'kalturadw.dwh_hourly_events_domain_referrer', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
$this->compareAggregation(array(new ComparedTable('location_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('location_id', 'kalturadw.dwh_hourly_events_country', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
$this->compareAggregation(array(new ComparedTable('country_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('country_id', 'kalturadw.dwh_hourly_events_country', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
$this->compareAggregation(array(new ComparedTable('widget_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('widget_id', 'kalturadw.dwh_hourly_events_widget', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
$this->compareAggregation(array(new ComparedTable('os_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('os_id', 'kalturadw.dwh_hourly_events_devices', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
$this->compareAggregation(array(new ComparedTable('browser_id', 'kalturadw.dwh_fact_events', 'if(event_type_id=3,1,0)')), array(new ComparedTable('browser_id', 'kalturadw.dwh_hourly_events_devices', 'ifnull(count_plays, 0)')), 0, 'event_date_id = ' . $dateID . ' and event_hour_id = ' . $hourID, 'date_id = ' . $dateID . ' and hour_id = ' . $hourID);
}
$this->compareAggregation(array(new ComparedTable('partner_id', 'kalturadw.dwh_fact_bandwidth_usage', '(bandwidth_bytes/1024)'), new ComparedTable('session_partner_id', 'kalturadw.dwh_fact_fms_sessions', '(total_bytes/1024)')), array(new ComparedTable('partner_id', 'kalturadw.dwh_hourly_partner_usage', 'ifnull(count_bandwidth_kb, 0)')), 1, 'activity_date_id = ' . $dateID, 'date_id = ' . $dateID);
$this->compareAggregation(array(new ComparedTable('bandwidth_source_id', 'kalturadw.dwh_fact_bandwidth_usage', '(bandwidth_bytes/1024)'), new ComparedTable('bandwidth_source_id', 'kalturadw.dwh_fact_fms_sessions', '(total_bytes/1024)')), array(new ComparedTable('bandwidth_source_id', 'kalturadw.dwh_hourly_partner_usage', 'ifnull(count_bandwidth_kb,0)')), 1, 'activity_date_id = ' . $dateID, 'date_id = ' . $dateID);
$this->compareAggregation(array(new ComparedTable('location_id', 'kalturadw.dwh_fact_bandwidth_usage', '(bandwidth_bytes/1024)'), new ComparedTable('location_id', 'kalturadw.dwh_fact_fms_sessions', '(total_bytes/1024)')), array(new ComparedTable('location_id', 'kalturadw.dwh_hourly_events_devices', 'ifnull(count_bandwidth_kb, 0)')), 1, 'activity_date_id = ' . $dateID, 'date_id = ' . $dateID);
$this->compareAggregation(array(new ComparedTable('country_id', 'kalturadw.dwh_fact_bandwidth_usage', '(bandwidth_bytes/1024)'), new ComparedTable('country_id', 'kalturadw.dwh_fact_fms_sessions', '(total_bytes/1024)')), array(new ComparedTable('country_id', 'kalturadw.dwh_hourly_events_devices', 'ifnull(count_bandwidth_kb, 0)')), 1, 'activity_date_id = ' . $dateID, 'date_id = ' . $dateID);
}
$this->compareAggregation(array(new ComparedTable('entry_id', 'kalturadw.dwh_hourly_events_entry', 'ifnull(count_plays,0)')), array(new ComparedTable('entry_id', 'kalturadw.dwh_entry_plays_views', 'ifnull(plays,0)')));
$this->compareAggregation(array(new ComparedTable('entry_id', 'kalturadw.dwh_hourly_events_entry', 'ifnull(count_loads,0)')), array(new ComparedTable('entry_id', 'kalturadw.dwh_entry_plays_views', 'ifnull(views,0)')));
}
