private function runnerDeciderRealShouldMakeNoDecisionForIrrelevantFiles($runner)
{
$this->setUpTables();
$this->setUpRepo();
$monkAgentId = 5;
$licenseRef1 = $this->licenseDao->getLicenseByShortName("GPL-3.0")->getRef();
$licId1 = $licenseRef1->getId();
$pfile = 4;
$jobId = 16;
$groupId = 2;
$userId = 2;
$itemId = 7;
$itemTreeBounds = new ItemTreeBounds($itemId, 'uploadtree_a', $uploadId = 1, 15, 16);
$this->dbManager->queryOnce("DELETE FROM license_file");
/* insert NoLicenseKnown decisions */
$this->dbManager->queryOnce("INSERT INTO clearing_decision (clearing_decision_pk, uploadtree_fk, pfile_fk, user_fk, group_fk, decision_type, scope, date_added)" . " VALUES (2, {$itemId}, {$pfile}, {$userId}, {$groupId}, " . DecisionTypes::IRRELEVANT . ", " . DecisionScopes::ITEM . ", '2015-05-04 11:43:18.276425+02')");
$lastDecision = $this->clearingDao->getRelevantClearingDecision($itemTreeBounds, $groupId);
$lastClearingId = $lastDecision->getClearingId();
$this->dbManager->queryOnce("INSERT INTO license_file (fl_pk,rf_fk,pfile_fk,agent_fk) VALUES(12222,{$licId1},{$pfile},{$monkAgentId})");
$this->dbManager->queryOnce("INSERT INTO jobqueue (jq_pk, jq_job_fk, jq_type, jq_args, jq_starttime, jq_endtime, jq_endtext, jq_end_bits, jq_schedinfo, jq_itemsprocessed, jq_log, jq_runonpfile, jq_host, jq_cmd_args)" . " VALUES ({$jobId}, 2, 'decider', '2', '2015-06-07 09:57:27.718312+00', NULL, '', 0, NULL, 6, NULL, NULL, NULL, '-r8')");
list($success, $output, $retCode) = $runner->run($uploadId, $userId, $groupId, $jobId, '');
$this->assertTrue($success, 'cannot run runner');
$this->assertEquals($retCode, 0, 'decider failed (did you make test?): ' . $output);
$newDecision = $this->clearingDao->getRelevantClearingDecision($itemTreeBounds, $groupId);
assertThat($newDecision->getClearingId(), equalTo($lastClearingId));
$this->rmRepo();
}
/**
* @param ItemTreeBounds $itemTreeBounds
* @return string[][][] $filesWithLicenses mapping item->'concluded'->(array of shortnames)
*/
protected function getFilesWithLicensesFromClearings(ItemTreeBounds $itemTreeBounds)
{
$clearingDecisions = $this->clearingDao->getFileClearingsFolder($itemTreeBounds, $this->groupId);
$filesWithLicenses = array();
$clearingsProceeded = 0;
foreach ($clearingDecisions as $clearingDecision) {
$clearingsProceeded += 1;
if (($clearingsProceeded & 2047) == 0) {
$this->heartbeat(0);
}
if ($clearingDecision->getType() == DecisionTypes::IRRELEVANT) {
continue;
}
foreach ($clearingDecision->getClearingEvents() as $clearingEvent) {
$clearingLicense = $clearingEvent->getClearingLicense();
if ($clearingLicense->isRemoved()) {
continue;
}
if ($clearingEvent->getReportinfo()) {
$customLicenseText = $clearingEvent->getReportinfo();
$reportedLicenseShortname = $this->licenseMap->getProjectedShortname($this->licenseMap->getProjectedId($clearingLicense->getLicenseId())) . '-' . md5($customLicenseText);
$this->includedLicenseIds[$reportedLicenseShortname] = $customLicenseText;
$filesWithLicenses[$clearingDecision->getUploadTreeId()]['concluded'][] = $reportedLicenseShortname;
} else {
$reportedLicenseId = $this->licenseMap->getProjectedId($clearingLicense->getLicenseId());
$this->includedLicenseIds[$reportedLicenseId] = true;
$filesWithLicenses[$clearingDecision->getUploadTreeId()]['concluded'][] = $this->licenseMap->getProjectedShortname($reportedLicenseId);
}
}
}
return $filesWithLicenses;
}
function doEdit($userId, $groupId, $itemId)
{
$licenses = GetParm("licenseNumbersToBeSubmitted", PARM_RAW);
$removed = $_POST['removed'] === 't' || $_POST['removed'] === 'true';
$itemTreeBounds = $this->uploadDao->getItemTreeBounds($itemId);
$uploadId = $itemTreeBounds->getUploadId();
$upload = $this->uploadDao->getUpload($uploadId);
$uploadName = $upload->getFilename();
$jobId = JobAddJob($userId, $groupId, $uploadName, $uploadId);
if (isset($licenses)) {
if (!is_array($licenses)) {
return $this->errorJson("bad license array");
}
foreach ($licenses as $licenseId) {
if (intval($licenseId) <= 0) {
return $this->errorJson("bad license");
}
$this->clearingDao->insertClearingEvent($itemId, $userId, $groupId, $licenseId, $removed, ClearingEventTypes::USER, $reportInfo = '', $comment = '', $jobId);
}
}
/** @var agent_fodecider $deciderPlugin */
$deciderPlugin = plugin_find("agent_deciderjob");
$conflictStrategyId = null;
$errorMsg = "";
$jq_pk = $deciderPlugin->AgentAdd($jobId, $uploadId, $errorMsg, array(), $conflictStrategyId);
/** after changing one license, purge all the report cache */
ReportCachePurgeAll();
//Todo: Change sql statement of fossology/src/buckets/agent/leaf.c line 124 to take the newest valid license, then uncomment this line
// $this->ChangeBuckets(); // change bucket accordingly
if (empty($errorMsg) && $jq_pk > 0) {
return new JsonResponse(array("jqid" => $jq_pk));
} else {
return $this->errorJson($errorMsg, 500);
}
}
/** @group Functional */
public function testRunMonkBulkScanWithAnEmptySearchText()
{
$this->setUpTables();
$this->setUpRepo();
$userId = 2;
$groupId = 2;
$uploadTreeId = 1;
$licenseId = 225;
$removing = false;
$refText = "";
$bulkId = $this->licenseDao->insertBulkLicense($userId, $groupId, $uploadTreeId, array($licenseId => $removing), $refText);
$this->assertGreaterThan($expected = 0, $bulkId);
$jobId = 64;
list($output, $retCode) = $this->runBulkMonk($userId, $groupId, $jobId, $bulkId);
$this->rmRepo();
$this->assertEquals($retCode, 0, 'monk bulk failed: ' . $output);
$bounds6 = new ItemTreeBounds(6, 'uploadtree_a', 1, 17, 18);
$bounds7 = new ItemTreeBounds(7, 'uploadtree_a', 1, 15, 16);
$relevantDecisionsItem6 = $this->clearingDao->getRelevantClearingEvents($bounds6, $groupId);
$relevantDecisionsItem7 = $this->clearingDao->getRelevantClearingEvents($bounds7, $groupId);
assertThat(count($relevantDecisionsItem6), is(equalTo(0)));
assertThat(count($relevantDecisionsItem7), is(equalTo(0)));
$bulkHighlights = $this->highlightDao->getHighlightBulk(6);
assertThat(count($bulkHighlights), is(equalTo(0)));
}
/**
* @param int $itemId
* @param ClearingDecision $clearingDecisionToCopy
*/
protected function createCopyOfClearingDecision($itemId, $userId, $groupId, $clearingDecisionToCopy)
{
$clearingEventIdsToCopy = array();
/** @var ClearingEvent $clearingEvent */
foreach ($clearingDecisionToCopy->getClearingEvents() as $clearingEvent) {
$clearingEventIdsToCopy[] = $clearingEvent->getEventId();
}
$this->clearingDao->createDecisionFromEvents($itemId, $userId, $groupId, $clearingDecisionToCopy->getType(), $clearingDecisionToCopy->getScope(), $clearingEventIdsToCopy);
}
/**
* @param Request $request
* @return Response
*/
protected function handle(Request $request)
{
$uploadId = intval($request->get('upload'));
if (empty($uploadId)) {
return;
}
$uploadTreeId = intval($request->get('item'));
if (empty($uploadTreeId)) {
return;
}
$onlyTried = !$request->get('all');
$uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId);
/** @var ItemTreeBounds */
$itemTreeBounds = $this->uploadDao->getItemTreeBounds($uploadTreeId, $uploadTreeTableName);
/** @var Session */
$session = $this->getObject('session');
$groupId = $session->get(Auth::GROUP_ID);
$bulkHistory = $this->clearingDao->getBulkHistory($itemTreeBounds, $groupId, $onlyTried);
return $this->render("bulk-history.html.twig", $this->mergeWithDefault(array('bulkHistory' => $bulkHistory)));
}
protected function setUp()
{
global $container;
$this->highlightDao = M::mock(HighlightDao::classname());
$container->shouldReceive('get')->withArgs(array('dao.highlight'))->andReturn($this->highlightDao);
$container->shouldReceive('get')->withArgs(array('dao.upload'))->andReturn(M::mock(UploadDao::classname()));
$container->shouldReceive('get')->withArgs(array('dao.clearing'))->andReturn(M::mock(ClearingDao::classname()));
$container->shouldReceive('get')->withArgs(array('decision.types'))->andReturn(M::mock(DecisionTypes::classname()));
$container->shouldReceive('get')->withArgs(array('businessrules.clearing_decision_processor'))->andReturn(M::mock(ClearingDecisionProcessor::classname()));
$container->shouldReceive('get')->withArgs(array('businessrules.agent_license_event_processor'))->andReturn(M::mock(AgentLicenseEventProcessor::classname()));
$this->assertCountBefore = \Hamcrest\MatcherAssert::getCount();
}
public function testupdateClearingEvent()
{
$this->testDb->createSequences(array('clearing_event_clearing_event_pk_seq'));
$this->testDb->createConstraints(array('clearing_event_pkey'));
$this->dbManager->queryOnce("ALTER TABLE clearing_event ALTER COLUMN clearing_event_pk SET DEFAULT nextval('clearing_event_clearing_event_pk_seq'::regclass)");
$this->clearingDao->updateClearingEvent($uploadTreeId = 301, $userId = 1, $groupId = 1, $licenseId = 402, $what = 'comment', $changeCom = 'abc123');
$rowPast = $this->dbManager->getSingleRow('SELECT * FROM clearing_event WHERE uploadtree_fk=$1 AND rf_fk=$2 ORDER BY clearing_event_pk DESC LIMIT 1', array($uploadTreeId, $licenseId), __METHOD__ . 'beforeReportinfo');
assertThat($rowPast['comment'], equalTo($changeCom));
$this->clearingDao->updateClearingEvent($uploadTreeId, $userId, $groupId, $licenseId, $what = 'reportinfo', $changeRep = 'def456');
$rowFuture = $this->dbManager->getSingleRow('SELECT * FROM clearing_event WHERE uploadtree_fk=$1 AND rf_fk=$2 ORDER BY clearing_event_pk DESC LIMIT 1', array($uploadTreeId, $licenseId), __METHOD__ . 'afterReportinfo');
assertThat($rowFuture['comment'], equalTo($changeCom));
assertThat($rowFuture['reportinfo'], equalTo($changeRep));
}
/**
* @param ItemTreeBounds $itemTreeBounds
* @param int $groupId
* @param boolean $orderAscending
* @return array
*/
protected function getCurrentSelectedLicensesTableData(ItemTreeBounds $itemTreeBounds, $groupId, $orderAscending)
{
$uploadTreeId = $itemTreeBounds->getItemId();
$uploadId = $itemTreeBounds->getUploadId();
$uberUri = Traceback_uri() . "?mod=view-license" . Traceback_parm_keep(array('upload', 'folder'));
list($addedClearingResults, $removedLicenses) = $this->clearingDecisionEventProcessor->getCurrentClearings($itemTreeBounds, $groupId, LicenseMap::CONCLUSION);
$licenseEventTypes = new ClearingEventTypes();
$mainLicIds = $this->clearingDao->getMainLicenseIds($uploadId, $groupId);
$table = array();
/* @var $clearingResult ClearingResult */
foreach ($addedClearingResults as $licenseShortName => $clearingResult) {
$licenseId = $clearingResult->getLicenseId();
$types = $this->getAgentInfo($clearingResult, $uberUri, $uploadTreeId);
$reportInfo = "";
$comment = "";
if ($clearingResult->hasClearingEvent()) {
$licenseDecisionEvent = $clearingResult->getClearingEvent();
$types[] = $this->getEventInfo($licenseDecisionEvent, $uberUri, $uploadTreeId, $licenseEventTypes);
$reportInfo = $licenseDecisionEvent->getReportinfo();
$comment = $licenseDecisionEvent->getComment();
}
$licenseShortNameWithLink = $this->urlBuilder->getLicenseTextUrl($clearingResult->getLicenseRef());
$actionLink = "<a href=\"javascript:;\" onclick=\"removeLicense({$uploadId}, {$uploadTreeId}, {$licenseId});\"><img class=\"delete\" src=\"images/space_16.png\" alt=\"\"/></a>";
if (in_array($clearingResult->getLicenseId(), $mainLicIds)) {
$tooltip = _('This is a main license for the upload. Click to discard selection.');
$actionLink .= " <a href=\"javascript:;\" onclick=\"removeMainLicense({$uploadId}, {$licenseId});\"><img src=\"images/icons/star_filled_16.png\" alt=\"mainLicense\" title=\"{$tooltip}\" border=\"0\"/></a>";
} else {
$tooltip = _('Click to select this as a main license for the upload.');
$actionLink .= " <a href=\"javascript:;\" onclick=\"makeMainLicense({$uploadId}, {$licenseId});\"><img src=\"images/icons/star_16.png\" alt=\"noMainLicense\" title=\"{$tooltip}\" border=\"0\"/></a>";
}
$reportInfoField = nl2br(htmlspecialchars($reportInfo));
$commentField = nl2br(htmlspecialchars($comment));
$id = "{$uploadTreeId},{$licenseId}";
$table[$licenseShortName] = array('DT_RowId' => $id, '0' => $licenseShortNameWithLink, '1' => implode("<br/>", $types), '2' => $reportInfoField, '3' => $commentField, '4' => $actionLink);
}
foreach ($removedLicenses as $licenseShortName => $clearingResult) {
if ($clearingResult->getAgentDecisionEvents()) {
$agents = $this->getAgentInfo($clearingResult, $uberUri, $uploadTreeId);
$licenseShortNameWithLink = $this->urlBuilder->getLicenseTextUrl($clearingResult->getLicenseRef());
$licenseId = $clearingResult->getLicenseId();
$actionLink = "<a href=\"javascript:;\" onclick=\"addLicense({$uploadId}, {$uploadTreeId}, {$licenseId});\"><img class=\"add\" src=\"images/space_16.png\" alt=\"\"/></a>";
$filled = in_array($clearingResult->getLicenseId(), $mainLicIds) ? 'filled_' : '';
$actionLink .= ' <img src="images/icons/star_' . $filled . '16.png" alt="mainLicense"/>';
$idArray = array($uploadTreeId, $licenseId);
$id = implode(',', $idArray);
$table[$licenseShortName] = array('DT_RowId' => $id, 'DT_RowClass' => 'removed', '0' => $licenseShortNameWithLink, '1' => implode("<br/>", $agents), '2' => "-", '3' => "-", '4' => $actionLink);
}
}
$valueTable = array_values($this->sortByKeys($table, $orderAscending));
return $valueTable;
}
protected function getStatements($uploadId, $uploadTreeTableName, $groupId = null)
{
$itemTreeBounds = $this->uploadDao->getParentItemBounds($uploadId, $uploadTreeTableName);
$clearingDecisions = $this->clearingDao->getFileClearingsFolder($itemTreeBounds, $groupId);
$dbManager = $GLOBALS['container']->get('db.manager');
$licenseMap = new LicenseMap($dbManager, $groupId, LicenseMap::REPORT);
$mainLicIds = $this->clearingDao->getMainLicenseIds($uploadId, $groupId);
$ungroupedStatements = array();
foreach ($clearingDecisions as $clearingDecision) {
if ($clearingDecision->getType() == DecisionTypes::IRRELEVANT) {
continue;
}
/** @var ClearingDecision $clearingDecision */
foreach ($clearingDecision->getClearingLicenses() as $clearingLicense) {
if ($clearingLicense->isRemoved()) {
continue;
}
if ($this->onlyComments && !($comment = $clearingLicense->getComment())) {
continue;
}
$originLicenseId = $clearingLicense->getLicenseId();
$licenseId = $licenseMap->getProjectedId($originLicenseId);
if (!$this->onlyComments && in_array($licenseId, $mainLicIds)) {
continue;
}
if ($this->onlyComments) {
$text = $comment;
} else {
$reportInfo = $clearingLicense->getReportInfo();
$text = $reportInfo ?: $this->getCachedLicenseText($licenseId, $groupId);
}
$ungroupedStatements[] = array('content' => $licenseMap->getProjectedShortname($originLicenseId, $clearingLicense->getShortName()), 'uploadtree_pk' => $clearingDecision->getUploadTreeId(), 'text' => $text);
}
}
return $ungroupedStatements;
}
public function testGetUnhandledScannerDetectedLicensesWithMappedMatch()
{
/** @var LicenseRef $licenseRef */
list($scannerResults, $licenseRef) = $this->createScannerDetectedLicenses();
$offset = 0;
$clearingEvent = $this->createClearingEvent($eventId = 123, $this->timestamp, $licenseRef->getId() + $offset, $licenseRef->getShortName(), $licenseRef->getFullName());
$this->clearingDao->shouldReceive("getRelevantClearingEvents")->with($this->itemTreeBounds, $this->groupId)->andReturn(array($clearingEvent->getLicenseId() => $clearingEvent));
$this->agentLicenseEventProcessor->shouldReceive("getScannerEvents")->with($this->itemTreeBounds, LicenseMap::CONCLUSION)->andReturn($scannerResults);
$licenseMap = M::mock(LicenseMap::classname());
$licenseMap->shouldReceive('getProjectedId')->andReturnUsing(function ($id) {
return $id;
});
$licenseMap->shouldReceive('getUsage')->andReturn(LicenseMap::CONCLUSION);
$hasUnhandledScannerDetectedLicenses = $this->clearingDecisionProcessor->hasUnhandledScannerDetectedLicenses($this->itemTreeBounds, $this->groupId, array(), $licenseMap);
assertThat($hasUnhandledScannerDetectedLicenses, is(False));
}
private function runnerReuserScanWithARepoClearingEnhanced($runner)
{
$this->setUpTables();
$this->setUpRepo();
$originallyClearedItemId = 23;
/* upload 3 in the test db is the same as upload 2 -> items 13-24 in upload 2 correspond to 33-44 */
$reusingUploadItemShift = 20;
$this->dbManager->queryOnce("UPDATE uploadtree_a SET pfile_fk=351 WHERE uploadtree_pk={$originallyClearedItemId}+{$reusingUploadItemShift}", __METHOD__ . '.minorChange');
$this->uploadDao->addReusedUpload($uploadId = 3, $reusedUpload = 2, $this->groupId, $this->groupId, $reuseMode = 1);
$repoPath = $this->testDb->getFossSysConf() . '/repo/files/';
$this->treeDao->shouldReceive('getRepoPathOfPfile')->with(4)->andReturn($repoPath . '04621571bcbabce75c4dd1c6445b87dec0995734.59cacdfce5051cd8a1d8a1f2dcce40a5.12320');
$this->treeDao->shouldReceive('getRepoPathOfPfile')->with(351)->andReturn($repoPath . 'c518ce1658140b65fa0132ad1130cb91512416bf.8e913e594d24ff3aeabe350107d97815.35829');
list($clearingLicense1, $clearingLicense2, $addedEventIds) = $this->insertDecisionFromTwoEvents(DecisionScopes::REPO, $originallyClearedItemId);
$clearingLicenses = array($clearingLicense1, $clearingLicense2);
list($success, $output, $retCode) = $runner->run($uploadId, $this->userId, $this->groupId);
$this->assertTrue($success, 'cannot run runner');
$this->assertEquals($retCode, 0, 'reuser failed: ' . $output);
$newUploadClearings = $this->getFilteredClearings($uploadId, $this->groupId);
$potentiallyReusableClearings = $this->getFilteredClearings($reusedUpload, $this->groupId);
assertThat($newUploadClearings, is(arrayWithSize(1)));
assertThat($potentiallyReusableClearings, is(arrayWithSize(1)));
/** @var ClearingDecision */
$potentiallyReusableClearing = $potentiallyReusableClearings[0];
/** @var ClearingDecision */
$newClearing = $newUploadClearings[0];
/* they are actually the same ClearingDecision
* only sameFolder and sameUpload are different */
assertThat($newClearing, not(equalTo($potentiallyReusableClearing)));
assertThat($newClearing->getClearingLicenses(), arrayContainingInAnyOrder($clearingLicenses));
assertThat($newClearing->getType(), equalTo($potentiallyReusableClearing->getType()));
assertThat($newClearing->getScope(), equalTo($potentiallyReusableClearing->getScope()));
assertThat($newClearing->getUploadTreeId(), equalTo($potentiallyReusableClearing->getUploadTreeId() + $reusingUploadItemShift));
/* reuser should have not created a correct local event history */
$bounds = $this->uploadDao->getItemTreeBounds($originallyClearedItemId + $reusingUploadItemShift);
$newEvents = $this->clearingDao->getRelevantClearingEvents($bounds, $this->groupId);
assertThat($newEvents, is(arrayWithSize(count($clearingLicenses))));
/** @var ClearingEvent $newEvent */
foreach ($newEvents as $newEvent) {
assertThat($newEvent->getEventId(), anyOf($addedEventIds));
assertThat($newEvent->getClearingLicense(), anyOf($clearingLicenses));
}
$this->rmRepo();
}
/**
* @param ItemTreeBounds $itemTreeBounds
* @param int $groupId
* @return array
* @throws Exception
*/
public function getCurrentClearings(ItemTreeBounds $itemTreeBounds, $groupId, $usageId = LicenseMap::TRIVIAL)
{
$agentEvents = $this->agentLicenseEventProcessor->getScannerEvents($itemTreeBounds, $usageId);
$events = $this->clearingDao->getRelevantClearingEvents($itemTreeBounds, $groupId);
$addedResults = array();
$removedResults = array();
foreach (array_unique(array_merge(array_keys($events), array_keys($agentEvents))) as $licenseId) {
$licenseDecisionEvent = array_key_exists($licenseId, $events) ? $events[$licenseId] : null;
$agentClearingEvents = array_key_exists($licenseId, $agentEvents) ? $agentEvents[$licenseId] : array();
if ($licenseDecisionEvent === null && count($agentClearingEvents) == 0) {
throw new Exception('not in merge');
}
$licenseDecisionResult = new ClearingResult($licenseDecisionEvent, $agentClearingEvents);
if ($licenseDecisionResult->isRemoved()) {
$removedResults[$licenseId] = $licenseDecisionResult;
} else {
$addedResults[$licenseId] = $licenseDecisionResult;
}
}
return array($addedResults, $removedResults);
}
/**
* @param ItemTreeBounds $itemTreeBounds
* @param int $userId
*/
protected function processClearingEventsForItem(ItemTreeBounds $itemTreeBounds, $userId, $groupId, $additionalEventsFromThisJob)
{
$this->dbManager->begin();
$itemId = $itemTreeBounds->getItemId();
switch ($this->conflictStrategyId) {
case self::FORCE_DECISION:
$createDecision = true;
break;
default:
$createDecision = !$this->clearingDecisionProcessor->hasUnhandledScannerDetectedLicenses($itemTreeBounds, $groupId, $additionalEventsFromThisJob, $this->licenseMap);
}
if ($createDecision) {
$this->clearingDecisionProcessor->makeDecisionFromLastEvents($itemTreeBounds, $userId, $groupId, DecisionTypes::IDENTIFIED, $this->decisionIsGlobal, $additionalEventsFromThisJob);
} else {
foreach ($additionalEventsFromThisJob as $eventId) {
$this->clearingDao->copyEventIdTo($eventId, $itemId, $userId, $groupId);
}
$this->clearingDao->markDecisionAsWip($itemId, $userId, $groupId);
}
$this->heartbeat(1);
$this->dbManager->commit();
}
protected function renderPackage($uploadId)
{
$uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId);
$itemTreeBounds = $this->uploadDao->getParentItemBounds($uploadId, $uploadTreeTableName);
$clearingDecisions = $this->clearingDao->getFileClearingsFolder($itemTreeBounds, $this->groupId);
$this->heartbeat(0);
$filesWithLicenses = $this->getFilesWithLicensesFromClearings($clearingDecisions);
$licenseComment = $this->addScannerResults($filesWithLicenses, $itemTreeBounds);
$this->addCopyrightResults($filesWithLicenses, $uploadId);
$this->heartbeat(0);
$upload = $this->uploadDao->getUpload($uploadId);
$fileNodes = $this->generateFileNodes($filesWithLicenses, $upload->getTreeTableName());
$mainLicenseIds = $this->clearingDao->getMainLicenseIds($uploadId, $this->groupId);
$mainLicenses = array();
foreach ($mainLicenseIds as $licId) {
$reportedLicenseId = $this->licenseMap->getProjectedId($licId);
$this->includedLicenseIds[$reportedLicenseId] = $reportedLicenseId;
$mainLicenses[] = $this->licenseMap->getProjectedShortname($reportedLicenseId);
}
$hashes = $this->uploadDao->getUploadHashes($uploadId);
return $this->renderString($this->getTemplateFile('package'), array('uploadId' => $uploadId, 'uri' => $this->uri, 'packageName' => $upload->getFilename(), 'uploadName' => $upload->getFilename(), 'sha1' => $hashes['sha1'], 'md5' => $hashes['md5'], 'verificationCode' => $this->getVerificationCode($upload), 'mainLicenses' => $mainLicenses, 'licenseComments' => $licenseComment, 'fileNodes' => $fileNodes));
}
private function processItem(Item $item)
{
$itemTreeBounds = $item->getItemTreeBounds();
$unMappedMatches = $this->agentLicenseEventProcessor->getLatestScannerDetectedMatches($itemTreeBounds);
$projectedScannerMatches = $this->remapByProjectedId($unMappedMatches);
$lastDecision = $this->clearingDao->getRelevantClearingDecision($itemTreeBounds, $this->groupId);
if (null !== $lastDecision && $lastDecision->getType() == DecisionTypes::IRRELEVANT) {
return 0;
}
$currentEvents = $this->clearingDao->getRelevantClearingEvents($itemTreeBounds, $this->groupId);
$markAsWip = false;
if (null !== $lastDecision && $projectedScannerMatches && ($this->activeRules & self::RULES_WIP_SCANNER_UPDATES) == self::RULES_WIP_SCANNER_UPDATES) {
$licensesFromDecision = array();
foreach ($lastDecision->getClearingLicenses() as $clearingLicense) {
$licenseIdFromEvent = $this->licenseMap->getProjectedId($clearingLicense->getLicenseId());
$licensesFromDecision[$licenseIdFromEvent] = $licenseIdFromEvent;
}
$markAsWip = $this->existsUnhandledMatch($projectedScannerMatches, $licensesFromDecision);
}
if (null !== $lastDecision && $markAsWip) {
$this->clearingDao->markDecisionAsWip($item->getId(), $this->userId, $this->groupId);
return 1;
}
if (null !== $lastDecision || 0 < count($currentEvents)) {
return 0;
}
$haveDecided = false;
if (($this->activeRules & self::RULES_NOMOS_IN_MONK) == self::RULES_NOMOS_IN_MONK) {
$haveDecided = $this->autodecideNomosMatchesInsideMonk($itemTreeBounds, $projectedScannerMatches);
}
if (!$haveDecided && ($this->activeRules & self::RULES_NOMOS_MONK_NINKA) == self::RULES_NOMOS_MONK_NINKA) {
$haveDecided = $this->autodecideNomosMonkNinka($itemTreeBounds, $projectedScannerMatches);
}
if (!$haveDecided && $markAsWip) {
$this->clearingDao->markDecisionAsWip($item->getId(), $this->userId, $this->groupId);
}
return $haveDecided || $markAsWip ? 1 : 0;
}
/**
* @param $uploadTreeId
* @param $tagId
* @param ItemTreeBounds $itemTreeBounds
* @param int|int[] $agentIds
* @param ClearingDecision []
* @return array
*/
private function createLicenseHistogram($uploadTreeId, $tagId, ItemTreeBounds $itemTreeBounds, $agentIds, $groupId)
{
$fileCount = $this->uploadDao->countPlainFiles($itemTreeBounds);
$licenseHistogram = $this->licenseDao->getLicenseHistogram($itemTreeBounds, $agentIds);
$editedLicensesHist = $this->clearingDao->getClearedLicenseIdAndMultiplicities($itemTreeBounds, $groupId);
$agentId = GetParm('agentId', PARM_INTEGER);
$licListUri = Traceback_uri() . "?mod=license_list_files&item={$uploadTreeId}";
if ($tagId) {
$licListUri .= "&tag={$tagId}";
}
if ($agentId) {
$licListUri .= "&agentId={$agentId}";
}
/* Write license histogram to $VLic */
list($tableData, $totalScannerLicenseCount, $editedTotalLicenseCount) = $this->createLicenseHistogramJSarray($licenseHistogram, $editedLicensesHist, $licListUri);
$uniqueLicenseCount = count($tableData);
$scannerUniqueLicenseCount = count($licenseHistogram);
$editedUniqueLicenseCount = count($editedLicensesHist);
$noScannerLicenseFoundCount = array_key_exists(LicenseDao::NO_LICENSE_FOUND, $licenseHistogram) ? $licenseHistogram[LicenseDao::NO_LICENSE_FOUND]['count'] : 0;
$editedNoLicenseFoundCount = array_key_exists(LicenseDao::NO_LICENSE_FOUND, $editedLicensesHist) ? $editedLicensesHist[LicenseDao::NO_LICENSE_FOUND]['count'] : 0;
$vars = array('tableDataJson' => json_encode($tableData), 'uniqueLicenseCount' => $uniqueLicenseCount, 'fileCount' => $fileCount, 'scannerUniqueLicenseCount' => $scannerUniqueLicenseCount, 'editedUniqueLicenseCount' => $editedUniqueLicenseCount, 'scannerLicenseCount' => $totalScannerLicenseCount, 'editedLicenseCount' => $editedTotalLicenseCount - $editedNoLicenseFoundCount, 'noScannerLicenseFoundCount' => $noScannerLicenseFoundCount, 'editedNoLicenseFoundCount' => $editedNoLicenseFoundCount, 'scannerLicenses' => $licenseHistogram, 'editedLicenses' => $editedLicensesHist);
return $vars;
}
private function runnerDeciderScanWithForceDecision($runner)
{
$this->setUpTables();
$this->setUpRepo();
$jobId = 42;
$licenseRef1 = $this->licenseDao->getLicenseByShortName("GPL-3.0")->getRef();
$licenseRef2 = $this->licenseDao->getLicenseByShortName("3DFX")->getRef();
$agentLicId = $this->licenseDao->getLicenseByShortName("Adaptec")->getRef()->getId();
$addedLicenses = array($licenseRef1, $licenseRef2);
assertThat($addedLicenses, not(arrayContaining(null)));
$agentId = 5;
$pfile = 4;
$this->dbManager->queryOnce("INSERT INTO license_file (fl_pk,rf_fk,pfile_fk,agent_fk) VALUES(12222,{$agentLicId},{$pfile},{$agentId})");
$itemTreeBounds = $this->uploadDao->getItemTreeBounds($itemId = 23);
assertThat($this->agentLicenseEventProcessor->getScannerEvents($itemTreeBounds), is(not(emptyArray())));
$eventId1 = $this->clearingDao->insertClearingEvent($itemId, $userId = 2, $groupId = 3, $licenseRef1->getId(), false);
$eventId2 = $this->clearingDao->insertClearingEvent($itemId, 5, $groupId, $licenseRef2->getId(), true);
$this->dbManager->queryOnce("UPDATE clearing_event SET job_fk={$jobId}");
$addedEventIds = array($eventId1, $eventId2);
list($success, $output, $retCode) = $runner->run($uploadId = 2, $userId, $groupId, $jobId, $args = "-k1");
$this->assertTrue($success, 'cannot run runner');
$this->assertEquals($retCode, 0, 'decider failed: ' . $output);
assertThat($this->getHeartCount($output), equalTo(1));
$uploadBounds = $this->uploadDao->getParentItemBounds($uploadId);
$decisions = $this->clearingDao->getFileClearingsFolder($uploadBounds, $groupId);
assertThat($decisions, is(arrayWithSize(1)));
/** @var ClearingDecision $deciderMadeDecision */
$deciderMadeDecision = $decisions[0];
$eventIds = array();
foreach ($deciderMadeDecision->getClearingEvents() as $event) {
$eventIds[] = $event->getEventId();
}
assertThat($eventIds, arrayValue($addedEventIds[0]));
assertThat($eventIds, arrayValue($addedEventIds[1]));
assertThat($eventIds, arrayWithSize(1 + count($addedEventIds)));
$this->rmRepo();
}
/**
* @param array $child
* @param int $uploadId
* @param int $selectedAgentId
* @param array $pfileLicenses
* @param int $groupId
* @param ClearingDecision[][] $editedMappedLicenses
* @param string $uri
* @param null|ClearingView $ModLicView
* @param array $UniqueTagArray
* @param boolean $isFlat
* @param int[] $latestSuccessfulAgentIds
* @return array
*/
private function createFileDataRow($child, $uploadId, $selectedAgentId, $pfileLicenses, $groupId, $editedMappedLicenses, $uri, $ModLicView, &$UniqueTagArray, $isFlat, $latestSuccessfulAgentIds)
{
$fileId = $child['pfile_fk'];
$childUploadTreeId = $child['uploadtree_pk'];
$linkUri = '';
if (!empty($fileId) && !empty($ModLicView)) {
$linkUri = Traceback_uri();
$linkUri .= "?mod=view-license&upload={$uploadId}&item={$childUploadTreeId}";
if ($selectedAgentId) {
$linkUri .= "&agentId={$selectedAgentId}";
}
}
/* Determine link for containers */
$isContainer = Iscontainer($child['ufile_mode']);
if ($isContainer && !$isFlat) {
$fatChild = $this->uploadDao->getFatItemArray($child['uploadtree_pk'], $uploadId, $this->uploadtree_tablename);
$uploadtree_pk = $fatChild['item_id'];
$linkUri = "{$uri}&item=" . $uploadtree_pk;
if ($selectedAgentId) {
$linkUri .= "&agentId={$selectedAgentId}";
}
$child['ufile_name'] = $fatChild['ufile_name'];
if (!Iscontainer($fatChild['ufile_mode'])) {
$isContainer = false;
}
} else {
if ($isContainer) {
$uploadtree_pk = Isartifact($child['ufile_mode']) ? DirGetNonArtifact($childUploadTreeId, $this->uploadtree_tablename) : $childUploadTreeId;
$linkUri = "{$uri}&item=" . $uploadtree_pk;
if ($selectedAgentId) {
$linkUri .= "&agentId={$selectedAgentId}";
}
}
}
/* Populate the output ($VF) - file list */
/* id of each element is its uploadtree_pk */
$fileName = $child['ufile_name'];
if ($isContainer) {
$fileName = "<a href='{$linkUri}'><span style='color: darkblue'> <b>{$fileName}</b> </span></a>";
} else {
if (!empty($linkUri)) {
$fileName = "<a href='{$linkUri}'>{$fileName}</a>";
}
}
/* show licenses under file name */
$childItemTreeBounds = new ItemTreeBounds($childUploadTreeId, $this->uploadtree_tablename, $child['upload_fk'], $child['lft'], $child['rgt']);
if ($isContainer) {
$agentFilter = $selectedAgentId ? array($selectedAgentId) : $latestSuccessfulAgentIds;
$licenseEntries = $this->licenseDao->getLicenseShortnamesContained($childItemTreeBounds, $agentFilter, array());
$editedLicenses = $this->clearingDao->getClearedLicenses($childItemTreeBounds, $groupId);
} else {
$licenseEntries = array();
if (array_key_exists($fileId, $pfileLicenses)) {
foreach ($pfileLicenses[$fileId] as $shortName => $rfInfo) {
$agentEntries = array();
foreach ($rfInfo as $agent => $match) {
$agentName = $this->agentNames[$agent];
$agentEntry = "<a href='?mod=view-license&upload={$child['upload_fk']}&item={$childUploadTreeId}&format=text&agentId={$match['agent_id']}&licenseId={$match['license_id']}#highlight'>" . $agentName . "</a>";
if ($match['match_percentage'] > 0) {
$agentEntry .= ": {$match['match_percentage']}%";
}
$agentEntries[] = $agentEntry;
}
$licenseEntries[] = $shortName . " [" . implode("][", $agentEntries) . "]";
}
}
/* @var $decision ClearingDecision */
if (false !== ($decision = $this->clearingFilter->getDecisionOf($editedMappedLicenses, $childUploadTreeId, $fileId))) {
$editedLicenses = $decision->getPositiveLicenses();
} else {
$editedLicenses = array();
}
}
$concludedLicenses = array();
/** @var LicenseRef $licenseRef */
foreach ($editedLicenses as $licenseRef) {
$projectedId = $this->licenseProjector->getProjectedId($licenseRef->getId());
$projectedName = $this->licenseProjector->getProjectedShortname($licenseRef->getId(), $licenseRef->getShortName());
$concludedLicenses[$projectedId] = $projectedName;
}
$editedLicenseList = implode(', ', $concludedLicenses);
$licenseList = implode(', ', $licenseEntries);
$fileListLinks = FileListLinks($uploadId, $childUploadTreeId, 0, $fileId, true, $UniqueTagArray, $this->uploadtree_tablename, !$isFlat);
$getTextEditUser = _("Edit");
$fileListLinks .= "[<a href='#' onclick='openUserModal({$childUploadTreeId})' >{$getTextEditUser}</a>]";
if ($isContainer) {
$getTextEditBulk = _("Bulk");
$fileListLinks .= "[<a href='#' onclick='openBulkModal({$childUploadTreeId})' >{$getTextEditBulk}</a>]";
}
$filesThatShouldStillBeCleared = array_key_exists($childItemTreeBounds->getItemId(), $this->filesThatShouldStillBeCleared) ? $this->filesThatShouldStillBeCleared[$childItemTreeBounds->getItemId()] : 0;
$filesToBeCleared = array_key_exists($childItemTreeBounds->getItemId(), $this->filesToBeCleared) ? $this->filesToBeCleared[$childItemTreeBounds->getItemId()] : 0;
$filesCleared = $filesToBeCleared - $filesThatShouldStillBeCleared;
$img = $filesCleared == $filesToBeCleared ? 'green' : 'red';
return array($fileName, $licenseList, $editedLicenseList, $img, "{$filesCleared}/{$filesToBeCleared}", $fileListLinks);
}
/**
* @brief extends standard Output to handle empty uploads
*/
function Output()
{
if ($this->invalidParm) {
$this->vars['content'] = 'This upload contains no files!<br><a href="' . Traceback_uri() . '?mod=browse">Go back to browse view</a>';
return $this->render("include/base.html.twig");
}
$uploadId = GetParm("upload", PARM_INTEGER);
if (empty($uploadId)) {
return new Response("", Response::HTTP_BAD_REQUEST);
}
$uploadTreeId = GetParm("item", PARM_INTEGER);
if (empty($uploadTreeId)) {
return new Response("", Response::HTTP_BAD_REQUEST);
}
$userId = Auth::getUserId();
$groupId = Auth::getGroupId();
$lastItem = GetParm("lastItem", PARM_INTEGER);
if (!empty($lastItem)) {
$this->updateLastItem($userId, $groupId, $lastItem);
}
$uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId);
$itemTreeBounds = $this->uploadDao->getItemTreeBounds($uploadTreeId, $uploadTreeTableName);
$this->vars['micromenu'] = Dir2Browse('license', $uploadTreeId, NULL, $showBox = 0, "View", -1, '', '', $uploadTreeTableName);
global $Plugins;
/** @var ui_view $view */
$view =& $Plugins[plugin_find_id("view")];
$licenseId = GetParm("licenseId", PARM_INTEGER);
$selectedAgentId = GetParm("agentId", PARM_INTEGER);
$highlightId = GetParm("highlightId", PARM_INTEGER);
$clearingId = GetParm("clearingId", PARM_INTEGER);
if ($clearingId !== null) {
$highlightId = -1;
} else {
if ($highlightId !== null) {
$clearingId = -1;
}
}
$baseUri = Traceback_uri();
$this->vars['baseuri'] = $baseUri;
$this->vars['uri'] = $baseUri . "?mod=" . $this->Name . Traceback_parm_keep(array('upload', 'folder'));
$this->vars['bulkHistoryHighlightUri'] = $this->vars['uri'];
$this->vars['optionName'] = "skipFile";
$this->vars['formName'] = "uiClearingForm";
$this->vars['ajaxAction'] = "setNextPrev";
$highlights = $this->getSelectedHighlighting($itemTreeBounds, $licenseId, $selectedAgentId, $highlightId, $clearingId, $uploadId);
$isSingleFile = !$itemTreeBounds->containsFiles();
$hasWritePermission = $this->uploadDao->isEditable($uploadId, $groupId);
$clearingDecisions = null;
if ($isSingleFile || $hasWritePermission) {
$clearingDecisions = $this->clearingDao->getFileClearings($itemTreeBounds, $groupId, false);
}
if ($isSingleFile && $hasWritePermission) {
$this->vars['bulkUri'] = Traceback_uri() . "?mod=popup-license";
$licenseArray = $this->licenseDao->getLicenseArray($groupId);
// $clearingDecision = $this->clearingDao->getRelevantClearingDecision($itemTreeBounds, $groupId);
list($addedResults, $removedResults) = $this->clearingDecisionEventProcessor->getCurrentClearings($itemTreeBounds, $groupId, LicenseMap::CONCLUSION);
if (count($addedResults) + count($removedResults) > 0) {
array_unshift($licenseArray, array('id' => 0, 'fullname' => '', 'shortname' => '------'));
}
/** @var ClearingResult $result */
foreach ($removedResults as $result) {
array_unshift($licenseArray, array('id' => $result->getLicenseId(), 'fullname' => $result->getLicenseFullName(), 'shortname' => $result->getLicenseShortName()));
}
/** @var ClearingResult $result */
foreach ($addedResults as $result) {
array_unshift($licenseArray, array('id' => $result->getLicenseId(), 'fullname' => $result->getLicenseFullName(), 'shortname' => $result->getLicenseShortName()));
}
$this->vars['licenseArray'] = $licenseArray;
} elseif ($isSingleFile) {
$this->vars['auditDenied'] = true;
}
$clearingHistory = array();
$selectedClearingType = false;
if ($hasWritePermission) {
$clearingHistory = $this->getClearingHistory($clearingDecisions);
}
if (count($clearingHistory) > 0) {
$selectedClearingType = $this->decisionTypes->getTypeByName($clearingHistory[0]['type']);
}
$bulkHistory = $this->clearingDao->getBulkHistory($itemTreeBounds, $groupId);
$ModBack = GetParm("modback", PARM_STRING) ?: "license";
list($pageMenu, $textView) = $view->getView(NULL, $ModBack, 0, "", $highlights, false, true);
$this->vars['uploadId'] = $uploadId;
$this->vars['itemId'] = $uploadTreeId;
$this->vars['pageMenu'] = $pageMenu;
$this->vars['textView'] = $textView;
$this->vars['legendData'] = $this->highlightRenderer->getLegendData($selectedAgentId || $clearingId);
$this->vars['clearingTypes'] = $this->decisionTypes->getMap();
$this->vars['selectedClearingType'] = $selectedClearingType;
$this->vars['tmpClearingType'] = $this->clearingDao->isDecisionWip($uploadTreeId, $groupId);
$this->vars['clearingHistory'] = $clearingHistory;
$this->vars['bulkHistory'] = $bulkHistory;
$noLicenseUploadTreeView = new UploadTreeProxy($uploadId, $options = array(UploadTreeProxy::OPT_SKIP_THESE => "noLicense", UploadTreeProxy::OPT_GROUP_ID => $groupId), $uploadTreeTableName, $viewName = 'no_license_uploadtree' . $uploadId);
$filesOfInterest = $noLicenseUploadTreeView->count();
$nonClearedUploadTreeView = new UploadTreeProxy($uploadId, $options = array(UploadTreeProxy::OPT_SKIP_THESE => "alreadyCleared", UploadTreeProxy::OPT_GROUP_ID => $groupId), $uploadTreeTableName, $viewName = 'already_cleared_uploadtree' . $uploadId);
$filesToBeCleared = $nonClearedUploadTreeView->count();
$filesAlreadyCleared = $filesOfInterest - $filesToBeCleared;
$this->vars['message'] = _("Cleared") . ": {$filesAlreadyCleared}/{$filesOfInterest}";
return $this->render("ui-clearing-view.html.twig");
}
