public function testTransfer() { $cycleID = DWHInspector::getCycle('LOADED'); $files = DWHInspector::getFiles($cycleID); $dsSessions = array(); foreach ($files as $fileID) { $dsSessions[$fileID] = DWHInspector::getFullDSFMSSessions($fileID, implode(",", $this->ignoredPartners)); } parent::testTransfer(); foreach ($files as $fileID) { $factSessions = DWHInspector::getFactFMSSessions($fileID); $this->assertEquals(count($dsSessions[$fileID]), count($factSessions)); foreach ($dsSessions[$fileID] as $sessionID => $dsSessionDictionary) { $this->assertEquals($dsSessionDictionary, $factSessions[$sessionID]); } } }
public function testProcess() { parent::testProcess(); global $CONF; $cycleID = DWHInspector::getCycle('LOADED'); $files = DWHInspector::getFiles($cycleID); foreach ($files as $fileID) { $filename = $CONF->ProcessPath . "/" . $cycleID . '/' . DWHInspector::getFileName($fileID); // compare rows in ds_events to rows in file $this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_events', $fileID), $this->countRows($filename, array($this, 'validKDPLine'))); // compare plays in ds_events to plays in file $this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_events', $fileID, ' and event_type_id=3'), $this->countPlays($filename)); // compare per entry $entries = $this->countPerEntry($filename); $this->assertEquals(count($entries), DWHInspector::countDistinct('kalturadw_ds.ds_events', $fileID, 'entry_id')); foreach ($entries as $entry => $val) { $res = DWHInspector::countRows('kalturadw_ds.ds_events', $fileID, " and entry_id='" . $entry . "'"); $this->assertEquals($res, $val); } // compare kdp events per partner $kdpEventsPartners = $this->countKDPEventsPerPartner($filename); $this->assertEquals(count($kdpEventsPartners), DWHInspector::countDistinct('kalturadw_ds.ds_events', $fileID, 'partner_id')); foreach ($kdpEventsPartners as $partner => $val) { $res = DWHInspector::countRows('kalturadw_ds.ds_events', $fileID, " and partner_id='" . $partner . "'"); $this->assertEquals($res, $val); } // compare rows in ds_bandwidth_usage to rows in file $this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_bandwidth_usage', $fileID), $this->countRows($filename, array($this, 'validBWLine'))); // compare bandwidth_bytes in ds_bandwidth_usage to bandwidth bytes consumed in file $dbBytes = DWHInspector::sumRows('kalturadw_ds.ds_bandwidth_usage', $fileID, "bandwidth_bytes"); $this->assertEquals(is_null($dbBytes) ? 0 : $dbBytes, $this->sumBytes($filename, array($this, 'validBWLine'), self::BW_REGEX)); // compare bw consumption per partner $bwPartners = $this->countBWEventsPerPartner($filename); $this->assertEquals(count($bwPartners), DWHInspector::countDistinct('kalturadw_ds.ds_bandwidth_usage', $fileID, 'partner_id')); foreach ($bwPartners as $partner => $val) { $res = DWHInspector::sumRows('kalturadw_ds.ds_bandwidth_usage', $fileID, 'bandwidth_bytes', ' and partner_id=\'' . $partner . '\''); $this->assertEquals($res, $val); } // make sure there are very little invalid lines $this->assertEquals($this->countInvalidLines($filename, array($this, 'validKDPLine'), array($this, 'ignoredInvalidKDPLine')) + $this->countInvalidLines($filename, array($this, 'validBWLine'), array($this, 'ignoredInvalidBWLine')), DWHInspector::countRows('kalturadw_ds.invalid_ds_lines', $fileID)); } }
public function testProcess() { parent::testProcess(); global $CONF; $cycleID = DWHInspector::getCycle('LOADED'); $files = DWHInspector::getFiles($cycleID); foreach ($files as $fileID) { $filename = $CONF->ProcessPath . "/" . $cycleID . '/' . DWHInspector::getFileName($fileID); // compare rows in ds_bandwidth_usage to rows in file $this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_bandwidth_usage', $fileID), $this->countRows($filename, array($this, 'validBWLine'))); $this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_bandwidth_usage', $fileID, 'and bandwidth_source_id = ' . $this->getBandwidthSourceID()), $this->countRows($filename, array($this, 'validBWLine'))); // compare bandwidth_bytes in ds_bandwidth_usage to bandwidth bytes consumed in file $this->assertEquals(DWHInspector::sumRows('kalturadw_ds.ds_bandwidth_usage', $fileID, "bandwidth_bytes"), $this->sumBytes($filename, array($this, 'validBWLine'), $this->getBWRegex())); // compare bw consumption per partner $bwPartners = $this->countBWEventsPerPartner($filename); $this->assertEquals(count($bwPartners), DWHInspector::countDistinct('kalturadw_ds.ds_bandwidth_usage', $fileID, 'partner_id')); foreach ($bwPartners as $partner => $val) { $res = DWHInspector::sumRows('kalturadw_ds.ds_bandwidth_usage', $fileID, 'bandwidth_bytes', ' and partner_id=\'' . $partner . '\''); $this->assertEquals($res, $val); } // make sure there are very little invalid lines $this->assertEquals($this->countInvalidLines($filename, array($this, 'validBWLine'), array($this, 'ignoredInvalidBWLine')), DWHInspector::countRows('kalturadw_ds.invalid_ds_lines', $fileID)); } }
public function testTransfer() { $cycleID = DWHInspector::getCycle('LOADED'); $ds_lines = array(); $files = DWHInspector::getFiles($cycleID); $dsTablesToFactTables = $this->getDSTablesToFactTables(); foreach ($files as $fileID) { foreach (array_keys($dsTablesToFactTables) as $dsTable) { $ds_lines[$fileID][$dsTable] = DWHInspector::countRows('kalturadw_ds.' . $dsTable, $fileID); } } DWHInspector::markAllAsAggregated(); $transferParams = $this->getTransferParams(); KettleRunner::execute($this->getTransferJob(), $transferParams); $this->assertEquals($cycleID, DWHInspector::getCycle('DONE')); $this->isCycleDirExists($cycleID, false); $files = DWHInspector::getFiles($cycleID); foreach ($files as $fileID) { foreach ($dsTablesToFactTables as $dsTable => $factTable) { // compare rows in ds_events and dwh_fact_events $this->assertEquals($ds_lines[$fileID][$dsTable], DWHInspector::countRows('kalturadw.' . $factTable, $fileID)); // make sure ds_events was emptied $this->assertEquals(0, DWHInspector::countRows('kalturadw_ds.' . $dsTable, $fileID)); } } foreach (DWHInspector::getAggrDatesAndHours($cycleID) as $table => $arr) { $minDateID = DWHInspector::getResetAggregationsMinDateID($cycleID, $table); $postTransferAggregationTypes = DWHInspector::getPostTransferAggregationTypes($transferParams[self::TRANSFER_PARAM_PROCESS_ID], $table); foreach ($arr as $dateID => $hours) { foreach ($hours as $hourID) { foreach ($postTransferAggregationTypes as $aggrType) { $filter = 'aggr_name = \'' . $aggrType . '\' and date_id = ' . $dateID . ' and hour_id = ' . $hourID . ' and ifnull(start_time,date(19700101)) < data_insert_time'; $rowExists = DWHInspector::rowExists('kalturadw.aggr_managment', $filter); $this->assertEquals($dateID >= $minDateID, $rowExists, "Row Date: {$dateID} {$hourID} {$aggrType} . Min Date: {$minDateID}"); } } } } }
public function testTransfer() { $cycleID = DWHInspector::getCycle('LOADED'); $files = DWHInspector::getFiles($cycleID); $unifiedApiCalls = DWHInspector::getUnifiedAPICalls($cycleID); $errornousUnifiedApiCalls = DWHInspector::getUnifiedAPICalls($cycleID, true); parent::testTransfer(); foreach ($unifiedApiCalls as $call) { $this->assertEquals(DWHInspector::countRows('kalturadw.dwh_fact_api_calls', '%', "and CONCAT(session_id,'_',request_index) = '" . $call->getID() . "'"), 1, "APICall ID " . $call->getID()); $this->assertEquals(DWHInspector::countRows('kalturadw.dwh_fact_incomplete_api_calls', '%', "and CONCAT(session_id,'_',request_index) = '" . $call->getID() . "'"), 0, "APICall ID " . $call->getID()); $this->assertEquals(DWHInspector::countRows('kalturadw.dwh_fact_errors', '%', "and error_object_id = '" . $call->getID() . "'"), 1, "APICall ID " . $call->getID()); } }