public function testProcess() { parent::testProcess(); global $CONF; $cycleID = DWHInspector::getCycle('LOADED'); $files = DWHInspector::getFiles($cycleID); foreach ($files as $fileID) { $filename = $CONF->ProcessPath . "/" . $cycleID . '/' . DWHInspector::getFileName($fileID); // compare rows in ds_fms_session_events to rows in file $this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_fms_session_events', $fileID), $this->countRows($filename, array($this, 'validFMSLine'))); // compare number of entries and number of rows per entry $this->AssertFMSEntity(array($this, 'countPerEntry'), $filename, $fileID, 'entry_id'); // compare number of partners and number of rows per partner $this->AssertFMSEntity(array($this, 'countPerPartner'), $filename, $fileID, 'partner_id'); // compare number of session and number of rows per session $this->AssertFMSEntity(array($this, 'countPerSession'), $filename, $fileID, 'session_id'); $fullFileSessions = $this->getFullSessions($filename); $fullDBSessions = array_keys(DWHInspector::getFullDSFMSSessions($fileID, implode(",", $this->ignoredPartners))); // compare number of full sessions $this->assertEquals(count($fullFileSessions), count($fullDBSessions)); // comapre the session IDs $this->assertEquals(count(array_intersect($fullFileSessions, $fullDBSessions)), count($fullFileSessions)); // make sure there are very little invalid lines $this->assertEquals($this->countInvalidLines($filename, array($this, 'validFMSLine'), array($this, 'ignoredInvalidFMSLine')), DWHInspector::countRows('kalturadw_ds.invalid_ds_lines', $fileID)); } }
public function testProcess() { parent::testProcess(); global $CONF; $cycleID = DWHInspector::getCycle('LOADED'); $files = DWHInspector::getFiles($cycleID); foreach ($files as $fileID) { $filename = $CONF->ProcessPath . "/" . $cycleID . '/' . DWHInspector::getFileName($fileID); // compare rows in ds_events to rows in file $this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_events', $fileID), $this->countRows($filename, array($this, 'validKDPLine'))); // compare plays in ds_events to plays in file $this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_events', $fileID, ' and event_type_id=3'), $this->countPlays($filename)); // compare per entry $entries = $this->countPerEntry($filename); $this->assertEquals(count($entries), DWHInspector::countDistinct('kalturadw_ds.ds_events', $fileID, 'entry_id')); foreach ($entries as $entry => $val) { $res = DWHInspector::countRows('kalturadw_ds.ds_events', $fileID, " and entry_id='" . $entry . "'"); $this->assertEquals($res, $val); } // compare kdp events per partner $kdpEventsPartners = $this->countKDPEventsPerPartner($filename); $this->assertEquals(count($kdpEventsPartners), DWHInspector::countDistinct('kalturadw_ds.ds_events', $fileID, 'partner_id')); foreach ($kdpEventsPartners as $partner => $val) { $res = DWHInspector::countRows('kalturadw_ds.ds_events', $fileID, " and partner_id='" . $partner . "'"); $this->assertEquals($res, $val); } // compare rows in ds_bandwidth_usage to rows in file $this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_bandwidth_usage', $fileID), $this->countRows($filename, array($this, 'validBWLine'))); // compare bandwidth_bytes in ds_bandwidth_usage to bandwidth bytes consumed in file $dbBytes = DWHInspector::sumRows('kalturadw_ds.ds_bandwidth_usage', $fileID, "bandwidth_bytes"); $this->assertEquals(is_null($dbBytes) ? 0 : $dbBytes, $this->sumBytes($filename, array($this, 'validBWLine'), self::BW_REGEX)); // compare bw consumption per partner $bwPartners = $this->countBWEventsPerPartner($filename); $this->assertEquals(count($bwPartners), DWHInspector::countDistinct('kalturadw_ds.ds_bandwidth_usage', $fileID, 'partner_id')); foreach ($bwPartners as $partner => $val) { $res = DWHInspector::sumRows('kalturadw_ds.ds_bandwidth_usage', $fileID, 'bandwidth_bytes', ' and partner_id=\'' . $partner . '\''); $this->assertEquals($res, $val); } // make sure there are very little invalid lines $this->assertEquals($this->countInvalidLines($filename, array($this, 'validKDPLine'), array($this, 'ignoredInvalidKDPLine')) + $this->countInvalidLines($filename, array($this, 'validBWLine'), array($this, 'ignoredInvalidBWLine')), DWHInspector::countRows('kalturadw_ds.invalid_ds_lines', $fileID)); } }
public function testProcess() { parent::testProcess(); global $CONF; $cycleID = DWHInspector::getCycle('LOADED'); $files = DWHInspector::getFiles($cycleID); foreach ($files as $fileID) { $filename = $CONF->ProcessPath . "/" . $cycleID . '/' . DWHInspector::getFileName($fileID); // compare rows in ds_bandwidth_usage to rows in file $this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_bandwidth_usage', $fileID), $this->countRows($filename, array($this, 'validBWLine'))); $this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_bandwidth_usage', $fileID, 'and bandwidth_source_id = ' . $this->getBandwidthSourceID()), $this->countRows($filename, array($this, 'validBWLine'))); // compare bandwidth_bytes in ds_bandwidth_usage to bandwidth bytes consumed in file $this->assertEquals(DWHInspector::sumRows('kalturadw_ds.ds_bandwidth_usage', $fileID, "bandwidth_bytes"), $this->sumBytes($filename, array($this, 'validBWLine'), $this->getBWRegex())); // compare bw consumption per partner $bwPartners = $this->countBWEventsPerPartner($filename); $this->assertEquals(count($bwPartners), DWHInspector::countDistinct('kalturadw_ds.ds_bandwidth_usage', $fileID, 'partner_id')); foreach ($bwPartners as $partner => $val) { $res = DWHInspector::sumRows('kalturadw_ds.ds_bandwidth_usage', $fileID, 'bandwidth_bytes', ' and partner_id=\'' . $partner . '\''); $this->assertEquals($res, $val); } // make sure there are very little invalid lines $this->assertEquals($this->countInvalidLines($filename, array($this, 'validBWLine'), array($this, 'ignoredInvalidBWLine')), DWHInspector::countRows('kalturadw_ds.invalid_ds_lines', $fileID)); } }
public function testProcess() { parent::testProcess(); global $CONF; $cycleID = DWHInspector::getCycle('LOADED'); $files = DWHInspector::getFiles($cycleID); foreach ($files as $fileID) { $filename = $CONF->ProcessPath . "/" . $cycleID . '/' . DWHInspector::getFileName($fileID); // compare rows in full api_calls to the ones in the file to rows in file $this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_api_calls', $fileID), count($this->getFileApiFullCalls($filename))); // compare rows in partial api_calls to the ones in the file to rows in file $this->assertEquals(DWHInspector::countRows('kalturadw_ds.ds_incomplete_api_calls', $fileID), count($this->getFileApiIncompleteCalls($filename))); // compare number of distinct actions and number of rows per action $collection = $this->getFullCallsPerEntity($filename, "ACTION", "unknown"); $this->assertEquals(DWHInspector::countDistinct('kalturadw_ds.ds_api_calls', $fileID, 'action_name', 'kalturadw.dwh_dim_api_actions', 'action_id'), count($collection)); foreach ($collection as $objectID => $val) { $res = DWHInspector::countRows('kalturadw_ds.ds_api_calls', $fileID, " and action_name = '{$objectID}'", 'kalturadw.dwh_dim_api_actions', 'action_id'); $this->assertEquals($res, $val, "Expected(db): {$res}, Actual(file): {$val} action_name: {$objectID}"); } // compare number of distinct serivces and number of rows per service $collection = $this->getFullCallsPerEntity($filename, "SERVICE", "unknown"); $this->assertEquals(DWHInspector::countDistinct('kalturadw_ds.ds_api_calls', $fileID, 'service_name', 'kalturadw.dwh_dim_api_actions', 'action_id'), count($collection)); foreach ($collection as $objectID => $val) { $res = DWHInspector::countRows('kalturadw_ds.ds_api_calls', $fileID, " and service_name = '{$objectID}'", 'kalturadw.dwh_dim_api_actions', 'action_id'); $this->assertEquals($res, $val, "Expected(db): {$res}, Actual(file): {$val} service_name: {$objectID}"); } // compare number of distinct partners and number of rows per partner $collection = $this->getFullCallsPerEntity($filename, "PARTNER_ID"); $this->assertEquals(DWHInspector::countDistinct('kalturadw_ds.ds_api_calls', $fileID, 'partner_id'), count($collection)); foreach ($collection as $objectID => $val) { $res = DWHInspector::countRows('kalturadw_ds.ds_api_calls', $fileID, " and partner_id = '{$objectID}'"); $this->assertEquals($res, $val, "Expected(db): {$res}, Actual(file): {$val} partner_id: {$objectID}"); } $collection = $this->sumFullCallsPerEntity($filename, "PARTNER_ID", "DURATION"); foreach ($collection as $partnerID => $duration) { $res = DWHInspector::sumRows('kalturadw_ds.ds_api_calls', $fileID, 'duration_msecs', " and partner_id = '{$partnerID}'"); $maxDiffInPercent = 1; $this->assertLessThanOrEqual($maxDiffInPercent, abs(100 - $duration / $res * 100), "Diff is bigger than {$maxDiffInPercent} percent - Expected(db): {$res}, Actual(file): {$duration} partner_id: {$partnerID}"); } // compare number of distinct tags and number of rows per tags $collection = $this->getFullCallsPerEntity($filename, "CLIENT_TAG", "unknown"); $this->assertEquals(DWHInspector::countDistinct('kalturadw_ds.ds_api_calls', $fileID, 'client_tag_name', 'kalturadw.dwh_dim_client_tags', 'client_tag_id'), count($collection)); foreach ($collection as $objectID => $val) { $res = DWHInspector::countRows('kalturadw_ds.ds_api_calls', $fileID, " and client_tag_name = '{$objectID}'", 'kalturadw.dwh_dim_client_tags', 'client_tag_id'); $this->assertEquals($res, $val, "Expected(db): {$res}, Actual(file): {$val} client_tag_name: {$objectID}"); } // compare number of distinct is_admins and number of rows per is_admin $collection = $this->getFullCallsPerEntity($filename, "IS_ADMIN"); $this->assertEquals(DWHInspector::countDistinct('kalturadw_ds.ds_api_calls', $fileID, 'is_admin'), count($collection)); foreach ($collection as $objectID => $val) { $res = DWHInspector::countRows('kalturadw_ds.ds_api_calls', $fileID, " and is_admin = '{$objectID}'"); $this->assertEquals($res, $val, "Expected(db): {$res}, Actual(file): {$val} is_admin: {$objectID}"); } // compare number of distinct partners and number of rows per partner $collection = $this->getFullCallsPerEntity($filename, "PARTNER_ID", '', true); $this->assertEquals(DWHInspector::countDistinct('kalturadw_ds.ds_errors', $fileID, 'partner_id'), count($collection)); foreach ($collection as $objectID => $val) { $res = DWHInspector::countRows('kalturadw_ds.ds_errors', $fileID, " and partner_id = '{$objectID}'"); $this->assertEquals($res, $val, "Expected(db): {$res}, Actual(file): {$val} partner_id: {$objectID}"); } $collection = $this->getFullCallsPerEntity($filename, "ERROR_CODE", '', true); $this->assertEquals(DWHInspector::countDistinct('kalturadw_ds.ds_errors', $fileID, 'error_code_name', 'kalturadw.dwh_dim_error_codes', 'error_code_id'), count($collection)); foreach ($collection as $objectID => $val) { $res = DWHInspector::countRows('kalturadw_ds.ds_errors', $fileID, " and error_code_name = '{$objectID}'", 'kalturadw.dwh_dim_error_codes', 'error_code_id'); $this->assertEquals($res, $val, "Expected(db): {$res}, Actual(file): {$val} error_code_name: {$objectID}"); } // make sure there are very little invalid lines $this->assertEquals($this->countInvalidLines($filename, array($this, 'validLine'), array($this, 'ignoredLine')), DWHInspector::countRows('kalturadw_ds.invalid_ds_lines', $fileID)); } }