public function testProcess()
{
parent::testProcess();
global $CONF;
$cycleID = DWHInspector::getCycle('LOADED');
$files = DWHInspector::getFiles($cycleID);
foreach ($files as $fileID) {
$filename = $CONF->ProcessPath . "/" . $cycleID . '/' . DWHInspector::getFileName($fileID);
// compare rows in ds_events to rows in file
$this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_events', $fileID), $this->countRows($filename, array($this, 'validKDPLine')));
// compare plays in ds_events to plays in file
$this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_events', $fileID, ' and event_type_id=3'), $this->countPlays($filename));
// compare per entry
$entries = $this->countPerEntry($filename);
$this->assertEquals(count($entries), DWHInspector::countDistinct('kalturadw_ds.ds_events', $fileID, 'entry_id'));
foreach ($entries as $entry => $val) {
$res = DWHInspector::countRows('kalturadw_ds.ds_events', $fileID, " and entry_id='" . $entry . "'");
$this->assertEquals($res, $val);
}
// compare kdp events per partner
$kdpEventsPartners = $this->countKDPEventsPerPartner($filename);
$this->assertEquals(count($kdpEventsPartners), DWHInspector::countDistinct('kalturadw_ds.ds_events', $fileID, 'partner_id'));
foreach ($kdpEventsPartners as $partner => $val) {
$res = DWHInspector::countRows('kalturadw_ds.ds_events', $fileID, " and partner_id='" . $partner . "'");
$this->assertEquals($res, $val);
}
// compare rows in ds_bandwidth_usage to rows in file
$this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_bandwidth_usage', $fileID), $this->countRows($filename, array($this, 'validBWLine')));
// compare bandwidth_bytes in ds_bandwidth_usage to bandwidth bytes consumed in file
$dbBytes = DWHInspector::sumRows('kalturadw_ds.ds_bandwidth_usage', $fileID, "bandwidth_bytes");
$this->assertEquals(is_null($dbBytes) ? 0 : $dbBytes, $this->sumBytes($filename, array($this, 'validBWLine'), self::BW_REGEX));
// compare bw consumption per partner
$bwPartners = $this->countBWEventsPerPartner($filename);
$this->assertEquals(count($bwPartners), DWHInspector::countDistinct('kalturadw_ds.ds_bandwidth_usage', $fileID, 'partner_id'));
foreach ($bwPartners as $partner => $val) {
$res = DWHInspector::sumRows('kalturadw_ds.ds_bandwidth_usage', $fileID, 'bandwidth_bytes', ' and partner_id=\'' . $partner . '\'');
$this->assertEquals($res, $val);
}
// make sure there are very little invalid lines
$this->assertEquals($this->countInvalidLines($filename, array($this, 'validKDPLine'), array($this, 'ignoredInvalidKDPLine')) + $this->countInvalidLines($filename, array($this, 'validBWLine'), array($this, 'ignoredInvalidBWLine')), DWHInspector::countRows('kalturadw_ds.invalid_ds_lines', $fileID));
}
}
public function testProcess()
{
parent::testProcess();
global $CONF;
$cycleID = DWHInspector::getCycle('LOADED');
$files = DWHInspector::getFiles($cycleID);
foreach ($files as $fileID) {
$filename = $CONF->ProcessPath . "/" . $cycleID . '/' . DWHInspector::getFileName($fileID);
// compare rows in ds_bandwidth_usage to rows in file
$this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_bandwidth_usage', $fileID), $this->countRows($filename, array($this, 'validBWLine')));
$this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_bandwidth_usage', $fileID, 'and bandwidth_source_id = ' . $this->getBandwidthSourceID()), $this->countRows($filename, array($this, 'validBWLine')));
// compare bandwidth_bytes in ds_bandwidth_usage to bandwidth bytes consumed in file
$this->assertEquals(DWHInspector::sumRows('kalturadw_ds.ds_bandwidth_usage', $fileID, "bandwidth_bytes"), $this->sumBytes($filename, array($this, 'validBWLine'), $this->getBWRegex()));
// compare bw consumption per partner
$bwPartners = $this->countBWEventsPerPartner($filename);
$this->assertEquals(count($bwPartners), DWHInspector::countDistinct('kalturadw_ds.ds_bandwidth_usage', $fileID, 'partner_id'));
foreach ($bwPartners as $partner => $val) {
$res = DWHInspector::sumRows('kalturadw_ds.ds_bandwidth_usage', $fileID, 'bandwidth_bytes', ' and partner_id=\'' . $partner . '\'');
$this->assertEquals($res, $val);
}
// make sure there are very little invalid lines
$this->assertEquals($this->countInvalidLines($filename, array($this, 'validBWLine'), array($this, 'ignoredInvalidBWLine')), DWHInspector::countRows('kalturadw_ds.invalid_ds_lines', $fileID));
}
}
public function testTransfer()
{
$cycleID = DWHInspector::getCycle('LOADED');
$ds_lines = array();
$files = DWHInspector::getFiles($cycleID);
$dsTablesToFactTables = $this->getDSTablesToFactTables();
foreach ($files as $fileID) {
foreach (array_keys($dsTablesToFactTables) as $dsTable) {
$ds_lines[$fileID][$dsTable] = DWHInspector::countRows('kalturadw_ds.' . $dsTable, $fileID);
}
}
DWHInspector::markAllAsAggregated();
$transferParams = $this->getTransferParams();
KettleRunner::execute($this->getTransferJob(), $transferParams);
$this->assertEquals($cycleID, DWHInspector::getCycle('DONE'));
$this->isCycleDirExists($cycleID, false);
$files = DWHInspector::getFiles($cycleID);
foreach ($files as $fileID) {
foreach ($dsTablesToFactTables as $dsTable => $factTable) {
// compare rows in ds_events and dwh_fact_events
$this->assertEquals($ds_lines[$fileID][$dsTable], DWHInspector::countRows('kalturadw.' . $factTable, $fileID));
// make sure ds_events was emptied
$this->assertEquals(0, DWHInspector::countRows('kalturadw_ds.' . $dsTable, $fileID));
}
}
foreach (DWHInspector::getAggrDatesAndHours($cycleID) as $table => $arr) {
$minDateID = DWHInspector::getResetAggregationsMinDateID($cycleID, $table);
$postTransferAggregationTypes = DWHInspector::getPostTransferAggregationTypes($transferParams[self::TRANSFER_PARAM_PROCESS_ID], $table);
foreach ($arr as $dateID => $hours) {
foreach ($hours as $hourID) {
foreach ($postTransferAggregationTypes as $aggrType) {
$filter = 'aggr_name = \'' . $aggrType . '\' and date_id = ' . $dateID . ' and hour_id = ' . $hourID . ' and ifnull(start_time,date(19700101)) < data_insert_time';
$rowExists = DWHInspector::rowExists('kalturadw.aggr_managment', $filter);
$this->assertEquals($dateID >= $minDateID, $rowExists, "Row Date: {$dateID} {$hourID} {$aggrType} . Min Date: {$minDateID}");
}
}
}
}
}
private function AssertFMSEntity($countPerEntityCallBack, $filename, $fileID, $tableEntityName)
{
$collection = call_user_func($countPerEntityCallBack, $filename);
$this->assertEquals(count($collection), DWHInspector::countDistinct('kalturadw_ds.ds_fms_session_events', $fileID, $tableEntityName), $countPerEntityCallBack[1]);
foreach ($collection as $objectID => $val) {
$res = DWHInspector::countRows('kalturadw_ds.ds_fms_session_events', $fileID, " and {$tableEntityName} = '{$objectID}'");
$this->assertEquals($res, $val, "Expected(db): {$res}, Actual(file): {$val} {$tableEntityName}: {$objectID}");
}
}
