/**
* @param string[][][] &$filesWithLicenses
* @param ItemTreeBounds $itemTreeBounds
*/
protected function addScannerResults(&$filesWithLicenses, ItemTreeBounds $itemTreeBounds)
{
$uploadId = $itemTreeBounds->getUploadId();
$scannerAgents = array_keys($this->agentNames);
$scanJobProxy = new ScanJobProxy($this->container->get('dao.agent'), $uploadId);
$scanJobProxy->createAgentStatus($scannerAgents);
$scannerIds = $scanJobProxy->getLatestSuccessfulAgentIds();
if (empty($scannerIds)) {
return;
}
$selectedScanners = '{' . implode(',', $scannerIds) . '}';
$tableName = $itemTreeBounds->getUploadTreeTableName();
$stmt = __METHOD__ . '.scanner_findings';
$sql = "SELECT DISTINCT uploadtree_pk,rf_fk FROM {$tableName} ut, license_file\n WHERE ut.pfile_fk=license_file.pfile_fk AND rf_fk IS NOT NULL AND agent_fk=any(\$1)";
$param = array($selectedScanners);
if ($tableName == 'uploadtree_a') {
$param[] = $uploadId;
$sql .= " AND upload_fk=\$" . count($param);
$stmt .= $tableName;
}
$sql .= " GROUP BY uploadtree_pk,rf_fk";
$this->dbManager->prepare($stmt, $sql);
$res = $this->dbManager->execute($stmt, $param);
while ($row = $this->dbManager->fetchArray($res)) {
$reportedLicenseId = $this->licenseMap->getProjectedId($row['rf_fk']);
$shortName = $this->licenseMap->getProjectedShortname($reportedLicenseId);
if ($shortName != 'No_license_found' && $shortName != 'Void') {
$filesWithLicenses[$row['uploadtree_pk']]['scanner'][] = $shortName;
$this->includedLicenseIds[$reportedLicenseId] = true;
}
}
$this->dbManager->freeResult($res);
return "licenseInfoInFile determined by Scanners {$selectedScanners}";
}
/**
* @param ItemTreeBounds $itemTreeBounds
* @param int $groupId
* @param int[] $additionalEventIds additional event ids to include, indexed by licenseId
* @param null|LicenseMap $licenseMap if given then license are considered as equal iff mapped to same
* @return bool
*/
public function hasUnhandledScannerDetectedLicenses(ItemTreeBounds $itemTreeBounds, $groupId, $additionalEventIds = array(), $licenseMap = null)
{
if (!empty($licenseMap) && !$licenseMap instanceof LicenseMap) {
throw new Exception('invalid license map');
}
$userEvents = $this->clearingDao->getRelevantClearingEvents($itemTreeBounds, $groupId);
$usageId = empty($licenseMap) ? LicenseMap::TRIVIAL : $licenseMap->getUsage();
$scannerDetectedEvents = $this->agentLicenseEventProcessor->getScannerEvents($itemTreeBounds, $usageId);
$eventLicenceIds = array();
foreach (array_keys($userEvents) as $licenseId) {
$eventLicenceIds[empty($licenseMap) ? $licenseId : $licenseMap->getProjectedId($licenseId)] = $licenseId;
}
foreach (array_keys($additionalEventIds) as $licenseId) {
$eventLicenceIds[empty($licenseMap) ? $licenseId : $licenseMap->getProjectedId($licenseId)] = $licenseId;
}
foreach (array_keys($scannerDetectedEvents) as $licenseId) {
if (!array_key_exists(empty($licenseMap) ? $licenseId : $licenseMap->getProjectedId($licenseId), $eventLicenceIds)) {
return true;
}
}
return false;
}
protected function getStatements($uploadId, $uploadTreeTableName, $groupId = null)
{
$itemTreeBounds = $this->uploadDao->getParentItemBounds($uploadId, $uploadTreeTableName);
$clearingDecisions = $this->clearingDao->getFileClearingsFolder($itemTreeBounds, $groupId);
$dbManager = $GLOBALS['container']->get('db.manager');
$licenseMap = new LicenseMap($dbManager, $groupId, LicenseMap::REPORT);
$mainLicIds = $this->clearingDao->getMainLicenseIds($uploadId, $groupId);
$ungroupedStatements = array();
foreach ($clearingDecisions as $clearingDecision) {
if ($clearingDecision->getType() == DecisionTypes::IRRELEVANT) {
continue;
}
/** @var ClearingDecision $clearingDecision */
foreach ($clearingDecision->getClearingLicenses() as $clearingLicense) {
if ($clearingLicense->isRemoved()) {
continue;
}
if ($this->onlyComments && !($comment = $clearingLicense->getComment())) {
continue;
}
$originLicenseId = $clearingLicense->getLicenseId();
$licenseId = $licenseMap->getProjectedId($originLicenseId);
if (!$this->onlyComments && in_array($licenseId, $mainLicIds)) {
continue;
}
if ($this->onlyComments) {
$text = $comment;
} else {
$reportInfo = $clearingLicense->getReportInfo();
$text = $reportInfo ?: $this->getCachedLicenseText($licenseId, $groupId);
}
$ungroupedStatements[] = array('content' => $licenseMap->getProjectedShortname($originLicenseId, $clearingLicense->getShortName()), 'uploadtree_pk' => $clearingDecision->getUploadTreeId(), 'text' => $text);
}
}
return $ungroupedStatements;
}
protected function remapByProjectedId($matches)
{
$remapped = array();
foreach ($matches as $licenseId => $licenseMatches) {
$projectedId = $this->licenseMap->getProjectedId($licenseId);
foreach ($licenseMatches as $agent => $agentMatches) {
$haveId = array_key_exists($projectedId, $remapped);
$haveAgent = $haveId && array_key_exists($agent, $remapped[$projectedId]);
if ($haveAgent) {
$remapped[$projectedId][$agent] = array_merge($remapped[$projectedId][$agent], $agentMatches);
} else {
$remapped[$projectedId][$agent] = $agentMatches;
}
}
}
return $remapped;
}
/**
* \brief get all the licenses for a single file or uploadtree
*
* @param ItemTreeBounds $itemTreeBounds
* @param int
* @return LicenseMatch[]
*/
function getAgentFileLicenseMatches(ItemTreeBounds $itemTreeBounds, $usageId = LicenseMap::TRIVIAL)
{
$uploadTreeTableName = $itemTreeBounds->getUploadTreeTableName();
$statementName = __METHOD__ . ".{$uploadTreeTableName}.{$usageId}";
$params = array($itemTreeBounds->getUploadId(), $itemTreeBounds->getLeft(), $itemTreeBounds->getRight());
if ($usageId == LicenseMap::TRIVIAL) {
$licenseJoin = "ONLY license_ref mlr ON license_file.rf_fk = mlr.rf_pk";
} else {
$params[] = $usageId;
$licenseMapCte = LicenseMap::getMappedLicenseRefView('$4');
$licenseJoin = "({$licenseMapCte}) AS mlr ON license_file.rf_fk = mlr.rf_origin";
}
$this->dbManager->prepare($statementName, "SELECT LFR.rf_shortname AS license_shortname,\n LFR.rf_fullname AS license_fullname,\n LFR.rf_pk AS license_id,\n LFR.fl_pk AS license_file_id,\n LFR.pfile_fk as file_id,\n LFR.rf_match_pct AS percent_match,\n AG.agent_name AS agent_name,\n AG.agent_pk AS agent_id,\n AG.agent_rev AS agent_revision\n FROM ( SELECT mlr.rf_fullname, mlr.rf_shortname, mlr.rf_pk, license_file.fl_pk, license_file.agent_fk, license_file.pfile_fk, license_file.rf_match_pct\n FROM license_file JOIN {$licenseJoin}) as LFR\n INNER JOIN {$uploadTreeTableName} as UT ON UT.pfile_fk = LFR.pfile_fk\n INNER JOIN agent as AG ON AG.agent_pk = LFR.agent_fk\n WHERE AG.agent_enabled='true' and\n UT.upload_fk=\$1 AND UT.lft BETWEEN \$2 and \$3\n ORDER BY license_shortname ASC, percent_match DESC");
$result = $this->dbManager->execute($statementName, $params);
$matches = array();
while ($row = $this->dbManager->fetchArray($result)) {
$licenseRef = new LicenseRef(intval($row['license_id']), $row['license_shortname'], $row['license_fullname']);
$agentRef = new AgentRef(intval($row['agent_id']), $row['agent_name'], $row['agent_revision']);
$matches[] = new LicenseMatch(intval($row['file_id']), $licenseRef, $agentRef, intval($row['license_file_id']), intval($row['percent_match']));
}
$this->dbManager->freeResult($result);
return $matches;
}
/**
* \brief Add a new license_ref to the database
*
* \return An add status string
*/
function Adddb()
{
$rf_shortname = trim($_POST['rf_shortname']);
$rf_fullname = trim($_POST['rf_fullname']);
$rf_url = $_POST['rf_url'];
$rf_notes = $_POST['rf_notes'];
$rf_text = trim($_POST['rf_text']);
$parent = $_POST['rf_parent'];
$report = $_POST['rf_report'];
$riskLvl = intval($_POST['risk_level']);
if (empty($rf_shortname)) {
$text = _("ERROR: The license shortname is empty.");
return "<b>{$text}</b><p>";
}
if ($this->isShortnameBlocked(0, $rf_shortname, $rf_text)) {
$text = _("ERROR: The shortname or license text already exist in the license list. License not added.");
return "<b>{$text}</b><p>";
}
$md5term = empty($rf_text) || stristr($rf_text, "License by Nomos") ? 'null' : 'md5($7)';
$stmt = __METHOD__ . '.rf';
$sql = "INSERT into license_ref (\n rf_active, marydone, rf_shortname, rf_fullname,\n rf_url, rf_notes, rf_md5, rf_text, rf_text_updatable,\n rf_detector_type, rf_risk) \n VALUES (\n \$1, \$2, \$3, \$4, \$5, \$6, {$md5term}, \$7, \$8, \$9, \$10) RETURNING rf_pk";
$this->dbManager->prepare($stmt, $sql);
$res = $this->dbManager->execute($stmt, array($_POST['rf_active'], $_POST['marydone'], $rf_shortname, $rf_fullname, $rf_url, $rf_notes, $rf_text, $_POST['rf_text_updatable'], $_POST['rf_detector_type'], $riskLvl));
$row = $this->dbManager->fetchArray($res);
$rfId = $row['rf_pk'];
$parentMap = new LicenseMap($this->dbManager, 0, LicenseMap::CONCLUSION);
$parentLicenses = $parentMap->getTopLevelLicenseRefs();
if (array_key_exists($parent, $parentLicenses)) {
$this->dbManager->insertTableRow('license_map', array('rf_fk' => $rfId, 'rf_parent' => $parent, 'usage' => LicenseMap::CONCLUSION));
}
$reportMap = new LicenseMap($this->dbManager, 0, LicenseMap::REPORT);
$reportLicenses = $reportMap->getTopLevelLicenseRefs();
if (array_key_exists($report, $reportLicenses)) {
$this->dbManager->insertTableRow('license_map', array('rf_fk' => $rfId, 'rf_parent' => $report, 'usage' => LicenseMap::REPORT));
}
$ob = "License {$_POST['rf_shortname']} (id={$rfId}) added.<p>";
return $ob;
}
public function testGetMappedLicenseRefView()
{
$this->testDb = new TestPgDb();
$this->testDb->createPlainTables(array('license_ref', 'license_map'));
$this->dbManager = $this->testDb->getDbManager();
$this->dbManager->queryOnce("CREATE TABLE license_candidate (group_fk integer) INHERITS (license_ref)");
$this->dbManager->insertTableRow('license_map', array('license_map_pk' => 0, 'rf_fk' => 2, 'rf_parent' => 1, 'usage' => LicenseMap::CONCLUSION));
$this->dbManager->insertTableRow('license_ref', array('rf_pk' => 1, 'rf_shortname' => 'One', 'rf_fullname' => 'One-1'));
$this->dbManager->insertTableRow('license_ref', array('rf_pk' => 2, 'rf_shortname' => 'Two', 'rf_fullname' => 'Two-2'));
$this->dbManager->insertTableRow('license_candidate', array('rf_pk' => 3, 'rf_shortname' => 'Three', 'rf_fullname' => 'Three-3', 'group_fk' => $this->groupId));
$this->assertCountBefore = \Hamcrest\MatcherAssert::getCount();
$view = LicenseMap::getMappedLicenseRefView(LicenseMap::CONCLUSION);
$stmt = __METHOD__;
$this->dbManager->prepare($stmt, $view);
$res = $this->dbManager->execute($stmt);
$map = $this->dbManager->fetchAll($res);
$this->dbManager->freeResult($res);
assertThat($map, is(arrayWithSize(2)));
$expected = array(array('rf_origin' => 1, 'rf_pk' => 1, 'rf_shortname' => 'One', 'rf_fullname' => 'One-1'), array('rf_origin' => 2, 'rf_pk' => 1, 'rf_shortname' => 'One', 'rf_fullname' => 'One-1'));
assertThat($map, containsInAnyOrder($expected));
}
/**
* @param $tagId
* @param ItemTreeBounds $itemTreeBounds
* @param $UniqueTagArray
* @param $selectedAgentId
* @param int $groupId
* @param ScanJobProxy $scanJobProxy
* @return array
*/
private function createFileListing($tagId, ItemTreeBounds $itemTreeBounds, &$UniqueTagArray, $selectedAgentId, $groupId, $scanJobProxy)
{
if (!empty($selectedAgentId)) {
$agentName = $this->agentDao->getAgentName($selectedAgentId);
$selectedScanners = array($agentName => $selectedAgentId);
} else {
$selectedScanners = $scanJobProxy->getLatestSuccessfulAgentIds();
}
/** change the license result when selecting one version of nomos */
$uploadId = $itemTreeBounds->getUploadId();
$isFlat = isset($_GET['flatten']);
if ($isFlat) {
$options = array(UploadTreeProxy::OPT_RANGE => $itemTreeBounds);
} else {
$options = array(UploadTreeProxy::OPT_REALPARENT => $itemTreeBounds->getItemId());
}
$descendantView = new UploadTreeProxy($uploadId, $options, $itemTreeBounds->getUploadTreeTableName(), 'uberItems');
$vars['iTotalDisplayRecords'] = $descendantView->count();
$columnNamesInDatabase = array($isFlat ? 'ufile_name' : 'lft');
$defaultOrder = array(array(0, "asc"));
$orderString = $this->getObject('utils.data_tables_utility')->getSortingString($_GET, $columnNamesInDatabase, $defaultOrder);
$offset = GetParm('iDisplayStart', PARM_INTEGER);
$limit = GetParm('iDisplayLength', PARM_INTEGER);
if ($offset) {
$orderString .= " OFFSET {$offset}";
}
if ($limit) {
$orderString .= " LIMIT {$limit}";
}
/* Get ALL the items under this Uploadtree_pk */
$sql = $descendantView->getDbViewQuery() . " {$orderString}";
$dbManager = $this->getObject('db.manager');
$dbManager->prepare($stmt = __METHOD__ . $orderString, $sql);
$res = $dbManager->execute($stmt, $descendantView->getParams());
$descendants = $dbManager->fetchAll($res);
$dbManager->freeResult($res);
/* Filter out Children that don't have tag */
if (!empty($tagId)) {
TagFilter($descendants, $tagId, $itemTreeBounds->getUploadTreeTableName());
}
if (empty($descendants)) {
$vars['fileData'] = array();
return $vars;
}
if ($isFlat) {
$firstChild = reset($descendants);
$lastChild = end($descendants);
$nameRange = array($firstChild['ufile_name'], $lastChild['ufile_name']);
} else {
$nameRange = array();
}
/******* File Listing ************/
$pfileLicenses = array();
foreach ($selectedScanners as $agentName => $agentId) {
$licensePerPfile = $this->licenseDao->getLicenseIdPerPfileForAgentId($itemTreeBounds, $agentId, $isFlat, $nameRange);
foreach ($licensePerPfile as $pfile => $licenseRow) {
foreach ($licenseRow as $licId => $row) {
$lic = $this->licenseProjector->getProjectedShortname($licId);
$pfileLicenses[$pfile][$lic][$agentName] = $row;
}
}
}
$baseUri = Traceback_uri() . '?mod=fileBrowse' . Traceback_parm_keep(array('upload', 'folder', 'show'));
$tableData = array();
global $Plugins;
$latestSuccessfulAgentIds = $scanJobProxy->getLatestSuccessfulAgentIds();
foreach ($descendants as $child) {
if (empty($child)) {
continue;
}
$tableData[] = $this->createFileDataRow($child, $uploadId, $selectedAgentId, $pfileLicenses, $groupId, $baseUri, $UniqueTagArray, $isFlat, $latestSuccessfulAgentIds);
}
$vars['fileData'] = $tableData;
return $vars;
}
/**
* @param array $child
* @param int $uploadId
* @param int $selectedAgentId
* @param array $pfileLicenses
* @param int $groupId
* @param ClearingDecision[][] $editedMappedLicenses
* @param string $uri
* @param null|ClearingView $ModLicView
* @param array $UniqueTagArray
* @param boolean $isFlat
* @param int[] $latestSuccessfulAgentIds
* @return array
*/
private function createFileDataRow($child, $uploadId, $selectedAgentId, $pfileLicenses, $groupId, $editedMappedLicenses, $uri, $ModLicView, &$UniqueTagArray, $isFlat, $latestSuccessfulAgentIds)
{
$fileId = $child['pfile_fk'];
$childUploadTreeId = $child['uploadtree_pk'];
$linkUri = '';
if (!empty($fileId) && !empty($ModLicView)) {
$linkUri = Traceback_uri();
$linkUri .= "?mod=view-license&upload={$uploadId}&item={$childUploadTreeId}";
if ($selectedAgentId) {
$linkUri .= "&agentId={$selectedAgentId}";
}
}
/* Determine link for containers */
$isContainer = Iscontainer($child['ufile_mode']);
if ($isContainer && !$isFlat) {
$fatChild = $this->uploadDao->getFatItemArray($child['uploadtree_pk'], $uploadId, $this->uploadtree_tablename);
$uploadtree_pk = $fatChild['item_id'];
$linkUri = "{$uri}&item=" . $uploadtree_pk;
if ($selectedAgentId) {
$linkUri .= "&agentId={$selectedAgentId}";
}
$child['ufile_name'] = $fatChild['ufile_name'];
if (!Iscontainer($fatChild['ufile_mode'])) {
$isContainer = false;
}
} else {
if ($isContainer) {
$uploadtree_pk = Isartifact($child['ufile_mode']) ? DirGetNonArtifact($childUploadTreeId, $this->uploadtree_tablename) : $childUploadTreeId;
$linkUri = "{$uri}&item=" . $uploadtree_pk;
if ($selectedAgentId) {
$linkUri .= "&agentId={$selectedAgentId}";
}
}
}
/* Populate the output ($VF) - file list */
/* id of each element is its uploadtree_pk */
$fileName = $child['ufile_name'];
if ($isContainer) {
$fileName = "<a href='{$linkUri}'><span style='color: darkblue'> <b>{$fileName}</b> </span></a>";
} else {
if (!empty($linkUri)) {
$fileName = "<a href='{$linkUri}'>{$fileName}</a>";
}
}
/* show licenses under file name */
$childItemTreeBounds = new ItemTreeBounds($childUploadTreeId, $this->uploadtree_tablename, $child['upload_fk'], $child['lft'], $child['rgt']);
if ($isContainer) {
$agentFilter = $selectedAgentId ? array($selectedAgentId) : $latestSuccessfulAgentIds;
$licenseEntries = $this->licenseDao->getLicenseShortnamesContained($childItemTreeBounds, $agentFilter, array());
$editedLicenses = $this->clearingDao->getClearedLicenses($childItemTreeBounds, $groupId);
} else {
$licenseEntries = array();
if (array_key_exists($fileId, $pfileLicenses)) {
foreach ($pfileLicenses[$fileId] as $shortName => $rfInfo) {
$agentEntries = array();
foreach ($rfInfo as $agent => $match) {
$agentName = $this->agentNames[$agent];
$agentEntry = "<a href='?mod=view-license&upload={$child['upload_fk']}&item={$childUploadTreeId}&format=text&agentId={$match['agent_id']}&licenseId={$match['license_id']}#highlight'>" . $agentName . "</a>";
if ($match['match_percentage'] > 0) {
$agentEntry .= ": {$match['match_percentage']}%";
}
$agentEntries[] = $agentEntry;
}
$licenseEntries[] = $shortName . " [" . implode("][", $agentEntries) . "]";
}
}
/* @var $decision ClearingDecision */
if (false !== ($decision = $this->clearingFilter->getDecisionOf($editedMappedLicenses, $childUploadTreeId, $fileId))) {
$editedLicenses = $decision->getPositiveLicenses();
} else {
$editedLicenses = array();
}
}
$concludedLicenses = array();
/** @var LicenseRef $licenseRef */
foreach ($editedLicenses as $licenseRef) {
$projectedId = $this->licenseProjector->getProjectedId($licenseRef->getId());
$projectedName = $this->licenseProjector->getProjectedShortname($licenseRef->getId(), $licenseRef->getShortName());
$concludedLicenses[$projectedId] = $projectedName;
}
$editedLicenseList = implode(', ', $concludedLicenses);
$licenseList = implode(', ', $licenseEntries);
$fileListLinks = FileListLinks($uploadId, $childUploadTreeId, 0, $fileId, true, $UniqueTagArray, $this->uploadtree_tablename, !$isFlat);
$getTextEditUser = _("Edit");
$fileListLinks .= "[<a href='#' onclick='openUserModal({$childUploadTreeId})' >{$getTextEditUser}</a>]";
if ($isContainer) {
$getTextEditBulk = _("Bulk");
$fileListLinks .= "[<a href='#' onclick='openBulkModal({$childUploadTreeId})' >{$getTextEditBulk}</a>]";
}
$filesThatShouldStillBeCleared = array_key_exists($childItemTreeBounds->getItemId(), $this->filesThatShouldStillBeCleared) ? $this->filesThatShouldStillBeCleared[$childItemTreeBounds->getItemId()] : 0;
$filesToBeCleared = array_key_exists($childItemTreeBounds->getItemId(), $this->filesToBeCleared) ? $this->filesToBeCleared[$childItemTreeBounds->getItemId()] : 0;
$filesCleared = $filesToBeCleared - $filesThatShouldStillBeCleared;
$img = $filesCleared == $filesToBeCleared ? 'green' : 'red';
return array($fileName, $licenseList, $editedLicenseList, $img, "{$filesCleared}/{$filesToBeCleared}", $fileListLinks);
}
public function testGetUnhandledScannerDetectedLicensesWithMappedMatch()
{
/** @var LicenseRef $licenseRef */
list($scannerResults, $licenseRef) = $this->createScannerDetectedLicenses();
$offset = 0;
$clearingEvent = $this->createClearingEvent($eventId = 123, $this->timestamp, $licenseRef->getId() + $offset, $licenseRef->getShortName(), $licenseRef->getFullName());
$this->clearingDao->shouldReceive("getRelevantClearingEvents")->with($this->itemTreeBounds, $this->groupId)->andReturn(array($clearingEvent->getLicenseId() => $clearingEvent));
$this->agentLicenseEventProcessor->shouldReceive("getScannerEvents")->with($this->itemTreeBounds, LicenseMap::CONCLUSION)->andReturn($scannerResults);
$licenseMap = M::mock(LicenseMap::classname());
$licenseMap->shouldReceive('getProjectedId')->andReturnUsing(function ($id) {
return $id;
});
$licenseMap->shouldReceive('getUsage')->andReturn(LicenseMap::CONCLUSION);
$hasUnhandledScannerDetectedLicenses = $this->clearingDecisionProcessor->hasUnhandledScannerDetectedLicenses($this->itemTreeBounds, $this->groupId, array(), $licenseMap);
assertThat($hasUnhandledScannerDetectedLicenses, is(False));
}
