From fda811aafae547a0971807dd328f5a3794b0ff5b Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 14 Nov 2024 12:30:42 +0100 Subject: [PATCH 01/28] feat: SP-1844 parse replace option from bom configuration file --- pom.xml | 8 + .../com/scanoss/ScannerPostProcessor.java | 129 +++++++++++ .../ScannerPostProcessorException.java | 51 +++++ .../scanoss/settings/BomConfiguration.java | 40 ++++ .../java/com/scanoss/utils/JsonUtils.java | 13 ++ src/test/java/com/scanoss/TestConstants.java | 28 +++ src/test/java/com/scanoss/TestJsonUtils.java | 27 ++- .../com/scanoss/TestScannerPostProcessor.java | 201 ++++++++++++++++++ 8 files changed, 495 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/scanoss/ScannerPostProcessor.java create mode 100644 src/main/java/com/scanoss/exceptions/ScannerPostProcessorException.java create mode 100644 src/main/java/com/scanoss/settings/BomConfiguration.java create mode 100644 src/test/java/com/scanoss/TestScannerPostProcessor.java diff --git a/pom.xml b/pom.xml index 57ad66f..fb9785d 100644 --- a/pom.xml +++ b/pom.xml @@ -121,6 +121,11 @@ 2.10.1 compile + + com.github.package-url + packageurl-java + 1.5.0 + @@ -130,6 +135,9 @@ 3.8.1 11 + 21 + 21 + --enable-preview diff --git a/src/main/java/com/scanoss/ScannerPostProcessor.java b/src/main/java/com/scanoss/ScannerPostProcessor.java new file mode 100644 index 0000000..ab38e2e --- /dev/null +++ b/src/main/java/com/scanoss/ScannerPostProcessor.java @@ -0,0 +1,129 @@ +package com.scanoss; + +import com.scanoss.dto.ScanFileResult; +import com.scanoss.exceptions.ScannerPostProcessorException; +import com.scanoss.settings.BomConfiguration; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public class ScannerPostProcessor { + + + /** + * Processes scan results according to BOM configuration rules. + * Applies remove, and replace rules as specified in the configuration. + * + * @param scanFileResults List of scan results to process + * @param bomConfiguration Configuration containing BOM rules + * @return List of processed scan results + */ + public List process(List scanFileResults, BomConfiguration bomConfiguration) { + if (scanFileResults == null || bomConfiguration == null) { + throw new ScannerPostProcessorException("Scan results and BOM configuration cannot be null"); + } + + List processedResults = new ArrayList<>(scanFileResults); + + // Apply remove rules + if (bomConfiguration.getBom().getRemove() != null && !bomConfiguration.getBom().getRemove().isEmpty()) { + processedResults = applyRemoveRules(processedResults, bomConfiguration.getBom().getRemove()); + } + + return processedResults; + } + + /** + * Applies remove rules to the scan results. + * A result will be removed if: + * 1. The remove rule has both path and purl, and both match the result + * 2. The remove rule has only purl (no path), and the purl matches the result + */ + private List applyRemoveRules(List results, List removeRules) { + if (results == null || removeRules == null) { + return results; + } + + List resultsList = new ArrayList<>(results); + + resultsList.removeIf(result -> shouldRemoveResult(result, removeRules)); + return resultsList; + } + + /** + * Determines if a result should be removed based on the remove rules. + * Returns true if the result should be removed, false if it should be kept. + */ + private boolean shouldRemoveResult(ScanFileResult result, List removeRules) { + for (BomConfiguration.Component rule : removeRules) { + if (isMatchingRule(rule, result)) { + return true; // Found a matching rule, remove the result + } + } + + return false; // No matching remove rules found, keep the result + } + + /** + * Checks if a rule matches a scan result. + * A rule matches if: + * 1. It has both path and purl, and both match the result + * 2. It has only a purl (no path), and the purl matches the result + * 3. It has only a path (no purl), and the path matches the result + */ + private boolean isMatchingRule(BomConfiguration.Component rule, ScanFileResult result) { + boolean hasPath = rule.getPath() != null && !rule.getPath().isEmpty(); + boolean hasPurl = rule.getPurl() != null && !rule.getPurl().isEmpty(); + + if (hasPath && hasPurl) { + return isPathAndPurlMatch(rule, result); + } else if (hasPath) { + return isPathOnlyMatch(rule, result); + } else if (hasPurl) { + return isPurlOnlyMatch(rule, result); + } + + return false; // Neither path nor purl specified + } + + /** + * Checks if both path and purl of the rule match the result + */ + private boolean isPathAndPurlMatch(BomConfiguration.Component rule, ScanFileResult result) { + return Objects.equals(rule.getPath(), result.getFilePath()) && + isPurlMatch(rule.getPurl(), result.getFileDetails().get(0).getPurls()); + } + + + /** + * Checks if the rule's path matches the result (ignoring purl) + */ + private boolean isPathOnlyMatch(BomConfiguration.Component rule, ScanFileResult result) { + return Objects.equals(rule.getPath(), result.getFilePath()); + } + + /** + * Checks if the rule's purl matches the result (ignoring path) + */ + private boolean isPurlOnlyMatch(BomConfiguration.Component rule, ScanFileResult result) { + return isPurlMatch(rule.getPurl(), result.getFileDetails().get(0).getPurls()); + } + + /** + * Checks if a specific purl exists in an array of purls + */ + private boolean isPurlMatch(String rulePurl, String[] resultPurls) { + if (rulePurl == null || resultPurls == null) { + return false; + } + + for (String resultPurl : resultPurls) { + if (Objects.equals(rulePurl, resultPurl)) { + return true; + } + } + return false; + } + +} diff --git a/src/main/java/com/scanoss/exceptions/ScannerPostProcessorException.java b/src/main/java/com/scanoss/exceptions/ScannerPostProcessorException.java new file mode 100644 index 0000000..42ae9e5 --- /dev/null +++ b/src/main/java/com/scanoss/exceptions/ScannerPostProcessorException.java @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2023, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.scanoss.exceptions; + +/** + * SCANOSS Post Processor Exception Class + *

+ * This exception will be used by the ScannerPostProcessor class to alert processing issues + *

+ */ +public class ScannerPostProcessorException extends RuntimeException { + + /** + * Winnowing Exception + * + * @param errorMessage error message + */ + public ScannerPostProcessorException(String errorMessage) { + super(errorMessage); + } + + /** + * Nested Winnowing Exception + * + * @param errorMessage error message + * @param err nested exception + */ + public ScannerPostProcessorException(String errorMessage, Throwable err) { + super(errorMessage, err); + } +} diff --git a/src/main/java/com/scanoss/settings/BomConfiguration.java b/src/main/java/com/scanoss/settings/BomConfiguration.java new file mode 100644 index 0000000..93a2f27 --- /dev/null +++ b/src/main/java/com/scanoss/settings/BomConfiguration.java @@ -0,0 +1,40 @@ +package com.scanoss.settings; + +import com.github.packageurl.PackageURL; +import com.google.gson.annotations.SerializedName; +import com.scanoss.dto.ScanFileResult; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; + +@Data +public class BomConfiguration { + private Bom bom; + + @Data + public static class Bom { + private List include; + private List remove; + private List replace; + } + + @Data + public static class Component { + private String path; + private String purl; + + public boolean doesMatchScanFileResult(ScanFileResult scanFileResult) { + //TODO: Implement matching logic here + return true; + } + } + + + @EqualsAndHashCode(callSuper = true) //NOTE: This will check both 'replaceWith' AND the parent class fields (path and purl) when comparing objects + @Data + public static class ReplaceComponent extends Component { + @SerializedName("replace_with") + private String replaceWith; + } +} diff --git a/src/main/java/com/scanoss/utils/JsonUtils.java b/src/main/java/com/scanoss/utils/JsonUtils.java index e0ba703..4f80f16 100644 --- a/src/main/java/com/scanoss/utils/JsonUtils.java +++ b/src/main/java/com/scanoss/utils/JsonUtils.java @@ -26,6 +26,7 @@ import com.google.gson.reflect.TypeToken; import com.scanoss.dto.ScanFileDetails; import com.scanoss.dto.ScanFileResult; +import com.scanoss.settings.BomConfiguration; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -195,6 +196,18 @@ public static List toScanFileResultsFromObject(@NonNull JsonObje return results; } + /** + * Convert the given JSON Object to a Settings object + * + * @param jsonObject JSON Object + * @return Settings + */ + public static BomConfiguration toBomConfigurationFromObject(@NonNull JsonObject jsonObject) { + + Gson gson = new Gson(); + return gson.fromJson(jsonObject, BomConfiguration.class); + } + /** * Determine if the given string is a boolean true/false * diff --git a/src/test/java/com/scanoss/TestConstants.java b/src/test/java/com/scanoss/TestConstants.java index 516a9be..ae87a8e 100644 --- a/src/test/java/com/scanoss/TestConstants.java +++ b/src/test/java/com/scanoss/TestConstants.java @@ -256,6 +256,34 @@ public class TestConstants { " ]\n" + "}\n"; + + static final String BOM_CONFIGURATION_MOCK = "{\n" + + " \"bom\": {\n" + + " \"include\": [\n" + + " {\n" + + " \"path\": \"src/main.c\",\n" + + " \"purl\": \"pkg:github/scanoss/scanner.c\"\n" + + " }\n" + + " ],\n" + + " \"remove\": [\n" + + " {\n" + + " \"path\": \"src/spdx.h\",\n" + + " \"purl\": \"pkg:github/scanoss/scanoss.py\"\n" + + " },\n" + + " {\n" + + " \"purl\": \"pkg:github/scanoss/ldb\"\n" + + " }\n" + + " ],\n" + + " \"replace\": [\n" + + " {\n" + + " \"path\": \"src/winnowing.c\",\n" + + " \"purl\": \"pkg:github/scanoss/core\",\n" + + " \"replace_with\": \"pkg:github/scanoss/\"\n" + + " }\n" + + " ]\n" + + " }\n" + + "}"; + // Custom self-signed certificate static final String customSelfSignedCertificate = "-----BEGIN CERTIFICATE-----\n" + diff --git a/src/test/java/com/scanoss/TestJsonUtils.java b/src/test/java/com/scanoss/TestJsonUtils.java index 00e287c..e86e42b 100644 --- a/src/test/java/com/scanoss/TestJsonUtils.java +++ b/src/test/java/com/scanoss/TestJsonUtils.java @@ -23,6 +23,7 @@ package com.scanoss; import com.google.gson.JsonObject; +import com.scanoss.settings.BomConfiguration; import com.scanoss.utils.JsonUtils; import com.scanoss.dto.ScanFileResult; import lombok.extern.slf4j.Slf4j; @@ -34,8 +35,7 @@ import java.util.List; import static com.scanoss.TestConstants.*; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.*; @Slf4j public class TestJsonUtils { @@ -87,4 +87,27 @@ public void TestRawResultsPositive() { log.info("Finished {} -->", methodName); } + + @Test() + public void TestBomConfiguration() { + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + + JsonObject jsonObject = JsonUtils.toJsonObject(BOM_CONFIGURATION_MOCK); + assertNotNull(jsonObject); + assertFalse("Should have decoded JSON Objects", jsonObject.isEmpty()); + log.info("JSON Objects: {}", jsonObject); + + + + BomConfiguration bomConfiguration = JsonUtils.toBomConfigurationFromObject(jsonObject); + assertNotNull(bomConfiguration); + log.info("Bom Configuration: {}", bomConfiguration); + + + log.info("Finished {} -->", methodName); + + } } diff --git a/src/test/java/com/scanoss/TestScannerPostProcessor.java b/src/test/java/com/scanoss/TestScannerPostProcessor.java new file mode 100644 index 0000000..b72b829 --- /dev/null +++ b/src/test/java/com/scanoss/TestScannerPostProcessor.java @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2023, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.scanoss; +import com.google.gson.JsonObject; +import com.scanoss.exceptions.ScannerPostProcessorException; +import com.scanoss.utils.JsonUtils; +import lombok.extern.slf4j.Slf4j; +import org.junit.Before; +import org.junit.Test; +import static com.scanoss.TestConstants.jsonResultsString; +import static org.junit.Assert.assertFalse; +import com.scanoss.dto.ScanFileResult; +import com.scanoss.settings.BomConfiguration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import static org.junit.Assert.*; + +@Slf4j +public class TestScannerPostProcessor { + private ScannerPostProcessor scannerPostProcessor; + private BomConfiguration bomConfiguration; + private List sampleScanResults; + + @Before + public void Setup() { + log.info("Starting ScannerPostProcessor test cases..."); + scannerPostProcessor = new ScannerPostProcessor(); + setupBomConfiguration(); + setupSampleScanResults(); + } + + private void setupBomConfiguration() { + bomConfiguration = new BomConfiguration(); + BomConfiguration.Bom bom = new BomConfiguration.Bom(); + bomConfiguration.setBom(bom); + } + + private void setupSampleScanResults() { + JsonObject jsonObject = JsonUtils.toJsonObject(jsonResultsString); + sampleScanResults = JsonUtils.toScanFileResultsFromObject(jsonObject); + } + + @Test + public void TestNullParameters() { + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + try { + scannerPostProcessor.process(null, bomConfiguration); + fail("Should throw ScannerPostProcessorException when scan results is null"); + } catch (Exception e) { + assertTrue("Wrong exception type thrown: " + e.getClass().getSimpleName(), e instanceof ScannerPostProcessorException); + } + + try { + scannerPostProcessor.process(sampleScanResults, null); + fail("Should throw ScannerPostProcessorException when BOM configuration is null"); + } catch (Exception e) { + assertTrue("Wrong exception type thrown: " + e.getClass().getSimpleName(), e instanceof ScannerPostProcessorException); + } + + log.info("Finished {} -->", methodName); + } + + @Test + public void TestRemoveRuleWithPathAndPurl() { + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + // Setup remove rule with both path and purl + BomConfiguration.Component removeRule = new BomConfiguration.Component(); + removeRule.setPath("CMSsite/admin/js/npm.js"); + removeRule.setPurl("pkg:github/twbs/bootstrap"); + bomConfiguration.getBom().setRemove(Collections.singletonList(removeRule)); + + // Process results + List results = scannerPostProcessor.process(sampleScanResults, bomConfiguration); + + // Verify + assertEquals("Should have one result less after removal", sampleScanResults.size()-1, results.size()); + log.info("Finished {} -->", methodName); + } + + @Test + public void TestRemoveRuleWithPurlOnly() { + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + // Setup remove rule with only purl + BomConfiguration.Component removeRule = new BomConfiguration.Component(); + removeRule.setPurl("pkg:npm/mip-bootstrap"); + bomConfiguration.getBom().setRemove(Collections.singletonList(removeRule)); + + // Process results + List results = scannerPostProcessor.process(sampleScanResults, bomConfiguration); + + // Verify + assertEquals("Size should decrease by 1 after removal", + sampleScanResults.size()-1, + results.size()); + + assertFalse("Should remove file CMSsite/admin/js/npm.js", + results.stream().anyMatch(r -> r.getFilePath().matches("CMSsite/admin/js/npm.js"))); + + log.info("Finished {} -->", methodName); + } + + @Test + public void TestNoMatchingRemoveRules() { + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + // Setup non-matching remove rule + BomConfiguration.Component removeRule = new BomConfiguration.Component(); + removeRule.setPath("non/existing/path.c"); + removeRule.setPurl("pkg:github/non-existing/lib@1.0.0"); + bomConfiguration.getBom().setRemove(Collections.singletonList(removeRule)); + + // Process results + List results = scannerPostProcessor.process(sampleScanResults, bomConfiguration); + + // Verify + assertEquals("Should keep all results", sampleScanResults.size(), results.size()); + + log.info("Finished {} -->", methodName); + } + + @Test + public void TestMultipleRemoveRules() { + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + // Setup multiple remove rules + List removeRules = new ArrayList<>(); + + BomConfiguration.Component rule1 = new BomConfiguration.Component(); + rule1.setPath("CMSsite/admin/js/npm.js"); + rule1.setPurl("pkg:npm/myoneui"); + + BomConfiguration.Component rule2 = new BomConfiguration.Component(); + rule2.setPurl("pkg:pypi/scanoss"); + + BomConfiguration.Component rule3 = new BomConfiguration.Component(); + rule3.setPath("scanoss/__init__.py"); + + removeRules.add(rule1); + removeRules.add(rule2); + removeRules.add(rule3); + bomConfiguration.getBom().setRemove(removeRules); + + // Process results + List results = scannerPostProcessor.process(sampleScanResults, bomConfiguration); + + // Verify + assertTrue("Should remove all results", results.isEmpty()); + + log.info("Finished {} -->", methodName); + } + + @Test + public void TestEmptyRemoveRules() { + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + // Process results with empty remove rules + List results = scannerPostProcessor.process(sampleScanResults, bomConfiguration); + + // Verify + assertEquals("Should keep all results", sampleScanResults.size(), results.size()); + assertEquals("Results should match original", sampleScanResults, results); + + log.info("Finished {} -->", methodName); + } +} \ No newline at end of file From fc90ba6c9f83f9f472b1ebc8ad84334171353472 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 14 Nov 2024 17:06:37 +0100 Subject: [PATCH 02/28] feat: SP-1847 parse replace rule --- .../com/scanoss/ScannerPostProcessor.java | 136 +++++++++++++----- .../java/com/scanoss/dto/ScanFileDetails.java | 4 +- .../scanoss/settings/BomConfiguration.java | 17 +-- .../com/scanoss/TestScannerPostProcessor.java | 62 ++++++-- 4 files changed, 161 insertions(+), 58 deletions(-) diff --git a/src/main/java/com/scanoss/ScannerPostProcessor.java b/src/main/java/com/scanoss/ScannerPostProcessor.java index ab38e2e..2116b12 100644 --- a/src/main/java/com/scanoss/ScannerPostProcessor.java +++ b/src/main/java/com/scanoss/ScannerPostProcessor.java @@ -1,15 +1,16 @@ package com.scanoss; +import com.scanoss.dto.ScanFileDetails; import com.scanoss.dto.ScanFileResult; import com.scanoss.exceptions.ScannerPostProcessorException; import com.scanoss.settings.BomConfiguration; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; +import java.util.*; +import java.util.stream.Collectors; public class ScannerPostProcessor { + private Map indexPurlToScanFileDetails = new HashMap<>(); /** * Processes scan results according to BOM configuration rules. @@ -24,6 +25,8 @@ public List process(List scanFileResults, BomCon throw new ScannerPostProcessorException("Scan results and BOM configuration cannot be null"); } + createIndexPurlToScanFileDetails(scanFileResults); + List processedResults = new ArrayList<>(scanFileResults); // Apply remove rules @@ -31,66 +34,129 @@ public List process(List scanFileResults, BomCon processedResults = applyRemoveRules(processedResults, bomConfiguration.getBom().getRemove()); } + //Apply replace rules. First loads the indexPurlToScanFileDetails + if (bomConfiguration.getBom().getReplace() != null && !bomConfiguration.getBom().getReplace().isEmpty()) { + processedResults = applyReplaceRules(processedResults, bomConfiguration.getBom().getReplace()); + } + return processedResults; } /** - * Applies remove rules to the scan results. - * A result will be removed if: - * 1. The remove rule has both path and purl, and both match the result - * 2. The remove rule has only purl (no path), and the purl matches the result + * Creates a map of PURL (Package URL) to ScanFileDetails from a list of scan results. + * + * @param scanFileResults List of scan results to process + * @return Map where keys are PURLs and values are corresponding ScanFileDetails */ - private List applyRemoveRules(List results, List removeRules) { - if (results == null || removeRules == null) { + private void createIndexPurlToScanFileDetails(List scanFileResults) { + if (scanFileResults == null) { + this.indexPurlToScanFileDetails = new HashMap<>(); + return; + } + + this.indexPurlToScanFileDetails = scanFileResults.stream() + .filter(result -> result != null && result.getFileDetails() != null) + .flatMap(result -> result.getFileDetails().stream()) + .filter(details -> details != null && details.getPurls() != null) + .flatMap(details -> Arrays.stream(details.getPurls()) + .filter(purl -> purl != null && !purl.trim().isEmpty()) + .map(purl -> Map.entry(purl.trim(), details))) + .collect(Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue, + (existing, replacement) -> existing, // Keep first occurrence in case of duplicates + HashMap::new + )); } + + + /** + * Applies replacement rules to scan results, updating their PURLs (Package URLs) based on matching rules. + * If a cached component exists for a replacement PURL, it will be used instead of creating a new one. + * + * @param results The original list of scan results to process + * @param replaceRules The list of replacement rules to apply + * @return A new list containing the processed scan results with updated PURLs + */ + private List applyReplaceRules(List results, List replaceRules) { + if (results == null || replaceRules == null) { return results; } List resultsList = new ArrayList<>(results); - resultsList.removeIf(result -> shouldRemoveResult(result, removeRules)); + for (ScanFileResult result : resultsList) { + findMatchingRule(result, replaceRules).ifPresent(matchedRule -> { + + String replacementPurl = matchedRule.getReplaceWith(); + if (replacementPurl == null || replacementPurl.trim().isEmpty()) { + return; //Empty replacement PURL found + } + + // Try to get cached component first + ScanFileDetails cachedComponent = this.indexPurlToScanFileDetails.get(replacementPurl); + if (cachedComponent != null) { + result.getFileDetails().set(0, cachedComponent); // Use cached component if available + } else { + result.getFileDetails().get(0).setPurls(new String[] { replacementPurl.trim() }); // Create new PURL array if no cached component exists + } + }); + } return resultsList; } + /** - * Determines if a result should be removed based on the remove rules. - * Returns true if the result should be removed, false if it should be kept. + * Applies remove rules to the scan results. + * A result will be removed if: + * 1. The remove rule has both path and purl, and both match the result + * 2. The remove rule has only purl (no path), and the purl matches the result */ - private boolean shouldRemoveResult(ScanFileResult result, List removeRules) { - for (BomConfiguration.Component rule : removeRules) { - if (isMatchingRule(rule, result)) { - return true; // Found a matching rule, remove the result - } + private List applyRemoveRules(List results, List removeRules) { + if (results == null || removeRules == null) { + return results; } - return false; // No matching remove rules found, keep the result + List resultsList = new ArrayList<>(results); + + resultsList.removeIf(result -> findMatchingRule(result, removeRules).isPresent()); + return resultsList; } /** - * Checks if a rule matches a scan result. + * Finds and returns the first matching rule for a scan result. * A rule matches if: * 1. It has both path and purl, and both match the result * 2. It has only a purl (no path), and the purl matches the result * 3. It has only a path (no purl), and the path matches the result + * + * @param The rule type. Must extend Rule class + * @param result The scan result to check + * @param rules List of rules to check against + * @return Optional containing the first matching rule, or empty if no match found */ - private boolean isMatchingRule(BomConfiguration.Component rule, ScanFileResult result) { - boolean hasPath = rule.getPath() != null && !rule.getPath().isEmpty(); - boolean hasPurl = rule.getPurl() != null && !rule.getPurl().isEmpty(); - - if (hasPath && hasPurl) { - return isPathAndPurlMatch(rule, result); - } else if (hasPath) { - return isPathOnlyMatch(rule, result); - } else if (hasPurl) { - return isPurlOnlyMatch(rule, result); - } - - return false; // Neither path nor purl specified + private Optional findMatchingRule(ScanFileResult result, List rules) { + return rules.stream() + .filter(rule -> { + boolean hasPath = rule.getPath() != null && !rule.getPath().isEmpty(); + boolean hasPurl = rule.getPurl() != null && !rule.getPurl().isEmpty(); + + if (hasPath && hasPurl) { + return isPathAndPurlMatch(rule, result); + } else if (hasPath) { + return isPathOnlyMatch(rule, result); + } else if (hasPurl) { + return isPurlOnlyMatch(rule, result); + } + + return false; // Neither path nor purl specified + }) + .findFirst(); } /** * Checks if both path and purl of the rule match the result */ - private boolean isPathAndPurlMatch(BomConfiguration.Component rule, ScanFileResult result) { + private boolean isPathAndPurlMatch(BomConfiguration.Rule rule, ScanFileResult result) { return Objects.equals(rule.getPath(), result.getFilePath()) && isPurlMatch(rule.getPurl(), result.getFileDetails().get(0).getPurls()); } @@ -99,14 +165,14 @@ private boolean isPathAndPurlMatch(BomConfiguration.Component rule, ScanFileResu /** * Checks if the rule's path matches the result (ignoring purl) */ - private boolean isPathOnlyMatch(BomConfiguration.Component rule, ScanFileResult result) { + private boolean isPathOnlyMatch(BomConfiguration.Rule rule, ScanFileResult result) { return Objects.equals(rule.getPath(), result.getFilePath()); } /** * Checks if the rule's purl matches the result (ignoring path) */ - private boolean isPurlOnlyMatch(BomConfiguration.Component rule, ScanFileResult result) { + private boolean isPurlOnlyMatch(BomConfiguration.Rule rule, ScanFileResult result) { return isPurlMatch(rule.getPurl(), result.getFileDetails().get(0).getPurls()); } diff --git a/src/main/java/com/scanoss/dto/ScanFileDetails.java b/src/main/java/com/scanoss/dto/ScanFileDetails.java index b343d40..4bbdb24 100644 --- a/src/main/java/com/scanoss/dto/ScanFileDetails.java +++ b/src/main/java/com/scanoss/dto/ScanFileDetails.java @@ -31,7 +31,7 @@ @Data public class ScanFileDetails { private final String id; - private final String component; + private String component; private final String vendor; private final String version; private final String latest; @@ -53,7 +53,7 @@ public class ScanFileDetails { @SerializedName("source_hash") private final String sourceHash; @SerializedName("purl") - private final String[] purls; + private String[] purls; @SerializedName("server") private final ServerDetails serverDetails; @SerializedName("licenses") diff --git a/src/main/java/com/scanoss/settings/BomConfiguration.java b/src/main/java/com/scanoss/settings/BomConfiguration.java index 93a2f27..7f1d3dd 100644 --- a/src/main/java/com/scanoss/settings/BomConfiguration.java +++ b/src/main/java/com/scanoss/settings/BomConfiguration.java @@ -1,6 +1,5 @@ package com.scanoss.settings; -import com.github.packageurl.PackageURL; import com.google.gson.annotations.SerializedName; import com.scanoss.dto.ScanFileResult; import lombok.Data; @@ -14,27 +13,23 @@ public class BomConfiguration { @Data public static class Bom { - private List include; - private List remove; - private List replace; + private List include; + private List remove; + private List replace; } @Data - public static class Component { + public static class Rule { private String path; private String purl; - - public boolean doesMatchScanFileResult(ScanFileResult scanFileResult) { - //TODO: Implement matching logic here - return true; - } } @EqualsAndHashCode(callSuper = true) //NOTE: This will check both 'replaceWith' AND the parent class fields (path and purl) when comparing objects @Data - public static class ReplaceComponent extends Component { + public static class ReplaceRule extends Rule { @SerializedName("replace_with") private String replaceWith; + private String license; } } diff --git a/src/test/java/com/scanoss/TestScannerPostProcessor.java b/src/test/java/com/scanoss/TestScannerPostProcessor.java index b72b829..1d9ce44 100644 --- a/src/test/java/com/scanoss/TestScannerPostProcessor.java +++ b/src/test/java/com/scanoss/TestScannerPostProcessor.java @@ -31,9 +31,9 @@ import static org.junit.Assert.assertFalse; import com.scanoss.dto.ScanFileResult; import com.scanoss.settings.BomConfiguration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; + +import java.util.*; + import static org.junit.Assert.*; @Slf4j @@ -61,6 +61,7 @@ private void setupSampleScanResults() { sampleScanResults = JsonUtils.toScanFileResultsFromObject(jsonObject); } + /** TESTING REMOVE RULES**/ @Test public void TestNullParameters() { String methodName = new Object() { @@ -91,7 +92,7 @@ public void TestRemoveRuleWithPathAndPurl() { log.info("<-- Starting {}", methodName); // Setup remove rule with both path and purl - BomConfiguration.Component removeRule = new BomConfiguration.Component(); + BomConfiguration.Rule removeRule = new BomConfiguration.Rule(); removeRule.setPath("CMSsite/admin/js/npm.js"); removeRule.setPurl("pkg:github/twbs/bootstrap"); bomConfiguration.getBom().setRemove(Collections.singletonList(removeRule)); @@ -111,7 +112,7 @@ public void TestRemoveRuleWithPurlOnly() { log.info("<-- Starting {}", methodName); // Setup remove rule with only purl - BomConfiguration.Component removeRule = new BomConfiguration.Component(); + BomConfiguration.Rule removeRule = new BomConfiguration.Rule(); removeRule.setPurl("pkg:npm/mip-bootstrap"); bomConfiguration.getBom().setRemove(Collections.singletonList(removeRule)); @@ -136,7 +137,7 @@ public void TestNoMatchingRemoveRules() { log.info("<-- Starting {}", methodName); // Setup non-matching remove rule - BomConfiguration.Component removeRule = new BomConfiguration.Component(); + BomConfiguration.Rule removeRule = new BomConfiguration.Rule(); removeRule.setPath("non/existing/path.c"); removeRule.setPurl("pkg:github/non-existing/lib@1.0.0"); bomConfiguration.getBom().setRemove(Collections.singletonList(removeRule)); @@ -157,16 +158,16 @@ public void TestMultipleRemoveRules() { log.info("<-- Starting {}", methodName); // Setup multiple remove rules - List removeRules = new ArrayList<>(); + List removeRules = new ArrayList<>(); - BomConfiguration.Component rule1 = new BomConfiguration.Component(); + BomConfiguration.Rule rule1 = new BomConfiguration.Rule(); rule1.setPath("CMSsite/admin/js/npm.js"); rule1.setPurl("pkg:npm/myoneui"); - BomConfiguration.Component rule2 = new BomConfiguration.Component(); + BomConfiguration.Rule rule2 = new BomConfiguration.Rule(); rule2.setPurl("pkg:pypi/scanoss"); - BomConfiguration.Component rule3 = new BomConfiguration.Component(); + BomConfiguration.Rule rule3 = new BomConfiguration.Rule(); rule3.setPath("scanoss/__init__.py"); removeRules.add(rule1); @@ -198,4 +199,45 @@ public void TestEmptyRemoveRules() { log.info("Finished {} -->", methodName); } + + + /** TESTING REPLACE RULES**/ + @Test + public void TestReplaceRuleWithEmptyPurl() { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + // Setup replace rule with empty PURL + BomConfiguration.ReplaceRule replaceRule = new BomConfiguration.ReplaceRule(); + replaceRule.setPurl("pkg:github/scanoss/scanoss.py"); + replaceRule.setReplaceWith(""); + bomConfiguration.getBom().setReplace(Collections.singletonList(replaceRule)); + + // Find the specific result for scanoss.py + Optional originalResult = sampleScanResults.stream() + .filter(r -> r.getFilePath().equals("scanoss/api/__init__.py")) + .findFirst(); + + assertTrue("Original result should exist", originalResult.isPresent()); + String originalPurl = originalResult.get().getFileDetails().get(0).getPurls()[0]; + + + // Process results + List results = scannerPostProcessor.process(sampleScanResults, bomConfiguration); + + Optional processedResult = results.stream() + .filter(r -> r.getFilePath().equals("scanoss/api/__init__.py")) + .findFirst(); + + assertTrue("Processed result should exist", processedResult.isPresent()); + + // Verify original PURL remains unchanged + String resultPurl = processedResult.get().getFileDetails().get(0).getPurls()[0]; + assertEquals("PURL should remain unchanged with empty replacement", originalPurl, resultPurl); + + log.info("Finished {} -->", methodName); + } + + + } \ No newline at end of file From 66ac3ad0bd713df9608bbf228fcaaabd78b16097 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 15 Nov 2024 12:37:08 +0100 Subject: [PATCH 03/28] chore: add test case with multiple purls --- .../com/scanoss/TestScannerPostProcessor.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/test/java/com/scanoss/TestScannerPostProcessor.java b/src/test/java/com/scanoss/TestScannerPostProcessor.java index 1d9ce44..fb63b0d 100644 --- a/src/test/java/com/scanoss/TestScannerPostProcessor.java +++ b/src/test/java/com/scanoss/TestScannerPostProcessor.java @@ -238,6 +238,36 @@ public void TestReplaceRuleWithEmptyPurl() { log.info("Finished {} -->", methodName); } + @Test() + public void TestReplaceRuleWithPurl() { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + + // Setup replace rule + BomConfiguration.ReplaceRule replaceRule = new BomConfiguration.ReplaceRule(); + replaceRule.setPurl("pkg:github/scanoss/scanoss.py"); + replaceRule.setReplaceWith("pkg:github/scanoss/scanner.c"); + bomConfiguration.getBom().setReplace(Collections.singletonList(replaceRule)); + + List results = scannerPostProcessor.process(sampleScanResults, bomConfiguration); + + Optional processedResult = results.stream() + .filter(r -> r.getFilePath().equals("scanoss/api/__init__.py")) + .findFirst(); + + assertTrue("Processed result should exist", processedResult.isPresent()); + + // Verify exactly one PURL exists and it's the correct one + String[] processedPurls = processedResult.get().getFileDetails().get(0).getPurls(); + assertEquals("Should have exactly one PURL", 1, processedPurls.length); + assertEquals("PURL should be scanner.c", + "pkg:github/scanoss/scanner.c", processedPurls[0]); + + log.info("Finished {} -->", methodName); + + } + } \ No newline at end of file From f9274ef5e1e39426efc60663e2ad43802cd6acf6 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 18 Nov 2024 10:01:19 +0100 Subject: [PATCH 04/28] fix: scan command --- pom.xml | 5 +- .../com/scanoss/ScannerPostProcessor.java | 16 ++--- .../java/com/scanoss/cli/ScanCommandLine.java | 26 ++++++++- .../scanoss/settings/BomConfiguration.java | 23 +++++++- .../java/com/scanoss/utils/JsonUtils.java | 23 +++++++- .../com/scanoss/TestBomConfiguration.java | 58 +++++++++++++++++++ 6 files changed, 133 insertions(+), 18 deletions(-) create mode 100644 src/test/java/com/scanoss/TestBomConfiguration.java diff --git a/pom.xml b/pom.xml index fb9785d..5d7a74e 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.scanoss scanoss - 0.7.3 + 0.8.0 jar scanoss.java https://github.com/scanoss/scanoss.java @@ -135,9 +135,6 @@ 3.8.1 11 - 21 - 21 - --enable-preview diff --git a/src/main/java/com/scanoss/ScannerPostProcessor.java b/src/main/java/com/scanoss/ScannerPostProcessor.java index 2116b12..70196ed 100644 --- a/src/main/java/com/scanoss/ScannerPostProcessor.java +++ b/src/main/java/com/scanoss/ScannerPostProcessor.java @@ -10,7 +10,7 @@ public class ScannerPostProcessor { - private Map indexPurlToScanFileDetails = new HashMap<>(); + private Map componentIndex = new HashMap<>(); /** * Processes scan results according to BOM configuration rules. @@ -25,7 +25,7 @@ public List process(List scanFileResults, BomCon throw new ScannerPostProcessorException("Scan results and BOM configuration cannot be null"); } - createIndexPurlToScanFileDetails(scanFileResults); + createComponentIndex(scanFileResults); List processedResults = new ArrayList<>(scanFileResults); @@ -34,7 +34,7 @@ public List process(List scanFileResults, BomCon processedResults = applyRemoveRules(processedResults, bomConfiguration.getBom().getRemove()); } - //Apply replace rules. First loads the indexPurlToScanFileDetails + //Apply replace rules if (bomConfiguration.getBom().getReplace() != null && !bomConfiguration.getBom().getReplace().isEmpty()) { processedResults = applyReplaceRules(processedResults, bomConfiguration.getBom().getReplace()); } @@ -48,13 +48,13 @@ public List process(List scanFileResults, BomCon * @param scanFileResults List of scan results to process * @return Map where keys are PURLs and values are corresponding ScanFileDetails */ - private void createIndexPurlToScanFileDetails(List scanFileResults) { + private void createComponentIndex(List scanFileResults) { if (scanFileResults == null) { - this.indexPurlToScanFileDetails = new HashMap<>(); + this.componentIndex = new HashMap<>(); return; } - this.indexPurlToScanFileDetails = scanFileResults.stream() + this.componentIndex = scanFileResults.stream() .filter(result -> result != null && result.getFileDetails() != null) .flatMap(result -> result.getFileDetails().stream()) .filter(details -> details != null && details.getPurls() != null) @@ -64,7 +64,7 @@ private void createIndexPurlToScanFileDetails(List scanFileResul .collect(Collectors.toMap( Map.Entry::getKey, Map.Entry::getValue, - (existing, replacement) -> existing, // Keep first occurrence in case of duplicates + (existing, replacement) -> existing, HashMap::new )); } @@ -93,7 +93,7 @@ private List applyReplaceRules(List results, Lis } // Try to get cached component first - ScanFileDetails cachedComponent = this.indexPurlToScanFileDetails.get(replacementPurl); + ScanFileDetails cachedComponent = this.componentIndex.get(replacementPurl); if (cachedComponent != null) { result.getFileDetails().set(0, cachedComponent); // Use cached component if available } else { diff --git a/src/main/java/com/scanoss/cli/ScanCommandLine.java b/src/main/java/com/scanoss/cli/ScanCommandLine.java index 86ff835..bc443b9 100644 --- a/src/main/java/com/scanoss/cli/ScanCommandLine.java +++ b/src/main/java/com/scanoss/cli/ScanCommandLine.java @@ -23,6 +23,8 @@ package com.scanoss.cli; import com.scanoss.Scanner; +import com.scanoss.ScannerPostProcessor; +import com.scanoss.dto.ScanFileResult; import com.scanoss.exceptions.ScannerException; import com.scanoss.exceptions.WinnowingException; import com.scanoss.utils.JsonUtils; @@ -39,6 +41,7 @@ import static com.scanoss.ScanossConstants.*; import static com.scanoss.cli.CommandLine.printDebug; import static com.scanoss.cli.CommandLine.printMsg; +import static com.scanoss.utils.JsonUtils.toScanFileResultJsonObject; /** * Scan Command Line Processor Class @@ -91,6 +94,9 @@ class ScanCommandLine implements Runnable { @picocli.CommandLine.Option(names = {"-n", "--ignore"}, description = "Ignore components specified in the SBOM file") private String ignoreSbom; + @picocli.CommandLine.Option(names = {"--settings"}, description = "Settings file to use for scanning (optional - default scanoss.json)") + private String settings; + @picocli.CommandLine.Option(names = {"--snippet-limit"}, description = "Length of single line snippet limit (0 for unlimited, default 1000)") private int snippetLimit = 1000; @@ -108,6 +114,8 @@ class ScanCommandLine implements Runnable { private Scanner scanner; + private List scanFileResults; + /** * Run the 'scan' command */ @@ -165,6 +173,7 @@ public void run() { .retryLimit(retryLimit).timeout(Duration.ofSeconds(timeoutLimit)).scanFlags(scanFlags) .sbomType(sbomType).sbom(sbom).snippetLimit(snippetLimit).customCert(caCertPem).proxy(proxy).hpsm(enableHpsm) .build(); + File f = new File(fileFolder); if (!f.exists()) { throw new RuntimeException(String.format("Error: File or folder does not exist: %s\n", fileFolder)); @@ -176,6 +185,18 @@ public void run() { } else { throw new RuntimeException(String.format("Error: Specified path is not a file or a folder: %s\n", fileFolder)); } + + if (settings != null && !settings.isEmpty()) { + try { + ScannerPostProcessor scannerPostProcessor = new ScannerPostProcessor(); + scanFileResults = scannerPostProcessor.process(scanFileResults, JsonUtils.toBomConfigurationFromFilePath(settings)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + var out = spec.commandLine().getOut(); + JsonUtils.writeJsonPretty(toScanFileResultJsonObject(scanFileResults), null); // Uses System.out } /** @@ -201,13 +222,12 @@ private String loadFileToString(@NonNull String filename) { * @param file file to scan */ private void scanFile(String file) { - var out = spec.commandLine().getOut(); var err = spec.commandLine().getErr(); try { printMsg(err, String.format("Scanning %s...", file)); String result = scanner.scanFile(file); if (result != null && !result.isEmpty()) { - JsonUtils.writeJsonPretty(JsonUtils.toJsonObject(result), out); + scanFileResults = JsonUtils.toScanFileResultsFromObject(JsonUtils.toJsonObject(result)); return; } else { err.println("Warning: No results returned."); @@ -235,7 +255,7 @@ private void scanFolder(String folder) { if (results != null && !results.isEmpty()) { printMsg(err, String.format("Found %d results.", results.size())); printDebug(err, "Converting to JSON..."); - JsonUtils.writeJsonPretty(JsonUtils.joinJsonObjects(JsonUtils.toJsonObjects(results)), out); + scanFileResults = JsonUtils.toScanFileResultsFromObject(JsonUtils.joinJsonObjects(JsonUtils.toJsonObjects(results))); return; } else { err.println("Error: No results return."); diff --git a/src/main/java/com/scanoss/settings/BomConfiguration.java b/src/main/java/com/scanoss/settings/BomConfiguration.java index 7f1d3dd..c8c245d 100644 --- a/src/main/java/com/scanoss/settings/BomConfiguration.java +++ b/src/main/java/com/scanoss/settings/BomConfiguration.java @@ -1,5 +1,7 @@ package com.scanoss.settings; +import com.github.packageurl.MalformedPackageURLException; +import com.github.packageurl.PackageURL; import com.google.gson.annotations.SerializedName; import com.scanoss.dto.ScanFileResult; import lombok.Data; @@ -22,14 +24,31 @@ public static class Bom { public static class Rule { private String path; private String purl; - } + public void setPurl(String purl) { + validatePurl(purl); + this.purl = purl; + } + + private void validatePurl(String purl) { + if (purl != null && !purl.trim().isEmpty()) { + try { + new PackageURL(purl); + } catch (MalformedPackageURLException e) { + throw new IllegalArgumentException("Invalid PURL: " + e.getMessage()); + } + } + } - @EqualsAndHashCode(callSuper = true) //NOTE: This will check both 'replaceWith' AND the parent class fields (path and purl) when comparing objects + } + + @EqualsAndHashCode(callSuper = true) + //NOTE: This will check both 'replaceWith' AND the parent class fields (path and purl) when comparing objects @Data public static class ReplaceRule extends Rule { @SerializedName("replace_with") private String replaceWith; private String license; } + } diff --git a/src/main/java/com/scanoss/utils/JsonUtils.java b/src/main/java/com/scanoss/utils/JsonUtils.java index 4f80f16..7d22b89 100644 --- a/src/main/java/com/scanoss/utils/JsonUtils.java +++ b/src/main/java/com/scanoss/utils/JsonUtils.java @@ -30,8 +30,11 @@ import lombok.NonNull; import lombok.extern.slf4j.Slf4j; +import java.io.IOException; import java.io.PrintWriter; import java.lang.reflect.Type; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -180,6 +183,19 @@ public static List toScanFileResults(@NonNull List resul return scanFileResults; } + public static JsonObject toScanFileResultJsonObject(List scanFileResults) { + JsonObject root = new JsonObject(); + Gson gson = new Gson(); + + scanFileResults.forEach(result -> { + JsonElement detailsJson = gson.toJsonTree(result.getFileDetails()); + root.add(result.getFilePath(), detailsJson); + }); + + return root; + } + + /** * Convert the given JSON Object to a list of Scan File Results * @@ -203,11 +219,16 @@ public static List toScanFileResultsFromObject(@NonNull JsonObje * @return Settings */ public static BomConfiguration toBomConfigurationFromObject(@NonNull JsonObject jsonObject) { - Gson gson = new Gson(); return gson.fromJson(jsonObject, BomConfiguration.class); } + public static BomConfiguration toBomConfigurationFromFilePath(@NonNull String path) throws IOException { + String settingsContent = Files.readString(Path.of(path)); + JsonObject settingsJson = JsonUtils.toJsonObject(settingsContent); + return JsonUtils.toBomConfigurationFromObject(settingsJson); + } + /** * Determine if the given string is a boolean true/false * diff --git a/src/test/java/com/scanoss/TestBomConfiguration.java b/src/test/java/com/scanoss/TestBomConfiguration.java new file mode 100644 index 0000000..346205e --- /dev/null +++ b/src/test/java/com/scanoss/TestBomConfiguration.java @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2023, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.scanoss; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.scanoss.dto.*; +import com.scanoss.settings.BomConfiguration; +import com.scanoss.utils.JsonUtils; +import lombok.extern.slf4j.Slf4j; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; + +import static com.scanoss.TestConstants.*; +import static org.junit.Assert.*; + +@Slf4j +public class TestBomConfiguration { + @Before + public void Setup() { + log.info("Starting Bom Configuration test cases..."); + log.debug("Logging debug enabled"); + log.trace("Logging trace enabled"); + } + + @Test + public void TestInvalidPurl() { + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + log.info("Finished {} -->", methodName); + } + +} From 0918b15338a5b208ca0eeec9a15f9887922aec7e Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 18 Nov 2024 17:13:28 +0100 Subject: [PATCH 05/28] chore: use lombok builder pattern for package settings --- .../com/scanoss/ScannerPostProcessor.java | 39 ++--- .../java/com/scanoss/cli/ScanCommandLine.java | 12 +- .../java/com/scanoss/dto/ScanFileResult.java | 1 + src/main/java/com/scanoss/settings/Bom.java | 28 ++++ .../scanoss/settings/BomConfiguration.java | 54 ------- .../com/scanoss/settings/ReplaceRule.java | 15 ++ src/main/java/com/scanoss/settings/Rule.java | 24 +++ .../java/com/scanoss/settings/Settings.java | 11 ++ .../java/com/scanoss/utils/JsonUtils.java | 19 --- .../com/scanoss/TestBomConfiguration.java | 58 ------- src/test/java/com/scanoss/TestJsonUtils.java | 23 --- .../com/scanoss/TestScannerPostProcessor.java | 152 +++++++++--------- 12 files changed, 178 insertions(+), 258 deletions(-) create mode 100644 src/main/java/com/scanoss/settings/Bom.java delete mode 100644 src/main/java/com/scanoss/settings/BomConfiguration.java create mode 100644 src/main/java/com/scanoss/settings/ReplaceRule.java create mode 100644 src/main/java/com/scanoss/settings/Rule.java create mode 100644 src/main/java/com/scanoss/settings/Settings.java delete mode 100644 src/test/java/com/scanoss/TestBomConfiguration.java diff --git a/src/main/java/com/scanoss/ScannerPostProcessor.java b/src/main/java/com/scanoss/ScannerPostProcessor.java index 70196ed..2be9bac 100644 --- a/src/main/java/com/scanoss/ScannerPostProcessor.java +++ b/src/main/java/com/scanoss/ScannerPostProcessor.java @@ -2,8 +2,11 @@ import com.scanoss.dto.ScanFileDetails; import com.scanoss.dto.ScanFileResult; -import com.scanoss.exceptions.ScannerPostProcessorException; -import com.scanoss.settings.BomConfiguration; +import com.scanoss.settings.Bom; +import com.scanoss.settings.ReplaceRule; +import com.scanoss.settings.Rule; +import com.scanoss.settings.Settings; +import org.jetbrains.annotations.NotNull; import java.util.*; import java.util.stream.Collectors; @@ -17,26 +20,22 @@ public class ScannerPostProcessor { * Applies remove, and replace rules as specified in the configuration. * * @param scanFileResults List of scan results to process - * @param bomConfiguration Configuration containing BOM rules + * @param bom Bom containing BOM rules * @return List of processed scan results */ - public List process(List scanFileResults, BomConfiguration bomConfiguration) { - if (scanFileResults == null || bomConfiguration == null) { - throw new ScannerPostProcessorException("Scan results and BOM configuration cannot be null"); - } - + public List process(@NotNull List scanFileResults, @NotNull Bom bom) { createComponentIndex(scanFileResults); List processedResults = new ArrayList<>(scanFileResults); // Apply remove rules - if (bomConfiguration.getBom().getRemove() != null && !bomConfiguration.getBom().getRemove().isEmpty()) { - processedResults = applyRemoveRules(processedResults, bomConfiguration.getBom().getRemove()); + if (bom.getRemove() != null && !bom.getRemove().isEmpty()) { + processedResults = applyRemoveRules(processedResults, bom.getRemove()); } //Apply replace rules - if (bomConfiguration.getBom().getReplace() != null && !bomConfiguration.getBom().getReplace().isEmpty()) { - processedResults = applyReplaceRules(processedResults, bomConfiguration.getBom().getReplace()); + if (bom.getReplace() != null && !bom.getReplace().isEmpty()) { + processedResults = applyReplaceRules(processedResults, bom.getReplace()); } return processedResults; @@ -77,7 +76,7 @@ private void createComponentIndex(List scanFileResults) { * @param replaceRules The list of replacement rules to apply * @return A new list containing the processed scan results with updated PURLs */ - private List applyReplaceRules(List results, List replaceRules) { + private List applyReplaceRules(List results, List replaceRules) { if (results == null || replaceRules == null) { return results; } @@ -111,11 +110,7 @@ private List applyReplaceRules(List results, Lis * 1. The remove rule has both path and purl, and both match the result * 2. The remove rule has only purl (no path), and the purl matches the result */ - private List applyRemoveRules(List results, List removeRules) { - if (results == null || removeRules == null) { - return results; - } - + private List applyRemoveRules(@NotNull List results, @NotNull List removeRules) { List resultsList = new ArrayList<>(results); resultsList.removeIf(result -> findMatchingRule(result, removeRules).isPresent()); @@ -134,7 +129,7 @@ private List applyRemoveRules(List results, List * @param rules List of rules to check against * @return Optional containing the first matching rule, or empty if no match found */ - private Optional findMatchingRule(ScanFileResult result, List rules) { + private Optional findMatchingRule(ScanFileResult result, List rules) { return rules.stream() .filter(rule -> { boolean hasPath = rule.getPath() != null && !rule.getPath().isEmpty(); @@ -156,7 +151,7 @@ private Optional findMatchingRule(ScanFileR /** * Checks if both path and purl of the rule match the result */ - private boolean isPathAndPurlMatch(BomConfiguration.Rule rule, ScanFileResult result) { + private boolean isPathAndPurlMatch(Rule rule, ScanFileResult result) { return Objects.equals(rule.getPath(), result.getFilePath()) && isPurlMatch(rule.getPurl(), result.getFileDetails().get(0).getPurls()); } @@ -165,14 +160,14 @@ private boolean isPathAndPurlMatch(BomConfiguration.Rule rule, ScanFileResult re /** * Checks if the rule's path matches the result (ignoring purl) */ - private boolean isPathOnlyMatch(BomConfiguration.Rule rule, ScanFileResult result) { + private boolean isPathOnlyMatch(Rule rule, ScanFileResult result) { return Objects.equals(rule.getPath(), result.getFilePath()); } /** * Checks if the rule's purl matches the result (ignoring path) */ - private boolean isPurlOnlyMatch(BomConfiguration.Rule rule, ScanFileResult result) { + private boolean isPurlOnlyMatch(Rule rule, ScanFileResult result) { return isPurlMatch(rule.getPurl(), result.getFileDetails().get(0).getPurls()); } diff --git a/src/main/java/com/scanoss/cli/ScanCommandLine.java b/src/main/java/com/scanoss/cli/ScanCommandLine.java index bc443b9..92b9b57 100644 --- a/src/main/java/com/scanoss/cli/ScanCommandLine.java +++ b/src/main/java/com/scanoss/cli/ScanCommandLine.java @@ -187,12 +187,12 @@ public void run() { } if (settings != null && !settings.isEmpty()) { - try { - ScannerPostProcessor scannerPostProcessor = new ScannerPostProcessor(); - scanFileResults = scannerPostProcessor.process(scanFileResults, JsonUtils.toBomConfigurationFromFilePath(settings)); - } catch (Exception e) { - throw new RuntimeException(e); - } +// try { +// ScannerPostProcessor scannerPostProcessor = new ScannerPostProcessor(); +// scanFileResults = scannerPostProcessor.process(scanFileResults, JsonUtils.toBomConfigurationFromFilePath(settings)); +// } catch (Exception e) { +// throw new RuntimeException(e); +// } } var out = spec.commandLine().getOut(); diff --git a/src/main/java/com/scanoss/dto/ScanFileResult.java b/src/main/java/com/scanoss/dto/ScanFileResult.java index 5783873..814eb7f 100644 --- a/src/main/java/com/scanoss/dto/ScanFileResult.java +++ b/src/main/java/com/scanoss/dto/ScanFileResult.java @@ -34,3 +34,4 @@ public class ScanFileResult { private final String filePath; private final List fileDetails; } + diff --git a/src/main/java/com/scanoss/settings/Bom.java b/src/main/java/com/scanoss/settings/Bom.java new file mode 100644 index 0000000..5d849d0 --- /dev/null +++ b/src/main/java/com/scanoss/settings/Bom.java @@ -0,0 +1,28 @@ +package com.scanoss.settings; + +import lombok.Builder; +import lombok.Data; +import org.jetbrains.annotations.NotNull; + +import java.util.List; + +@Data +@Builder +public class Bom { + private List include; + private List remove; + private List replace; + + + public void addInclude(@NotNull Rule rule) { + this.include.add(rule); + } + + public void addRemove(@NotNull Rule rule) { + this.include.add(rule); + } + + +} + + diff --git a/src/main/java/com/scanoss/settings/BomConfiguration.java b/src/main/java/com/scanoss/settings/BomConfiguration.java deleted file mode 100644 index c8c245d..0000000 --- a/src/main/java/com/scanoss/settings/BomConfiguration.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.scanoss.settings; - -import com.github.packageurl.MalformedPackageURLException; -import com.github.packageurl.PackageURL; -import com.google.gson.annotations.SerializedName; -import com.scanoss.dto.ScanFileResult; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.List; - -@Data -public class BomConfiguration { - private Bom bom; - - @Data - public static class Bom { - private List include; - private List remove; - private List replace; - } - - @Data - public static class Rule { - private String path; - private String purl; - - public void setPurl(String purl) { - validatePurl(purl); - this.purl = purl; - } - - private void validatePurl(String purl) { - if (purl != null && !purl.trim().isEmpty()) { - try { - new PackageURL(purl); - } catch (MalformedPackageURLException e) { - throw new IllegalArgumentException("Invalid PURL: " + e.getMessage()); - } - } - } - - } - - @EqualsAndHashCode(callSuper = true) - //NOTE: This will check both 'replaceWith' AND the parent class fields (path and purl) when comparing objects - @Data - public static class ReplaceRule extends Rule { - @SerializedName("replace_with") - private String replaceWith; - private String license; - } - -} diff --git a/src/main/java/com/scanoss/settings/ReplaceRule.java b/src/main/java/com/scanoss/settings/ReplaceRule.java new file mode 100644 index 0000000..b193c92 --- /dev/null +++ b/src/main/java/com/scanoss/settings/ReplaceRule.java @@ -0,0 +1,15 @@ +package com.scanoss.settings; + +import com.google.gson.annotations.SerializedName; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.SuperBuilder; + +@EqualsAndHashCode(callSuper = true) +@Data +@SuperBuilder +public class ReplaceRule extends Rule { + @SerializedName("replace_with") + private String replaceWith; + private String license; +} \ No newline at end of file diff --git a/src/main/java/com/scanoss/settings/Rule.java b/src/main/java/com/scanoss/settings/Rule.java new file mode 100644 index 0000000..f44c81e --- /dev/null +++ b/src/main/java/com/scanoss/settings/Rule.java @@ -0,0 +1,24 @@ +package com.scanoss.settings; + +import com.github.packageurl.MalformedPackageURLException; +import com.github.packageurl.PackageURL; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.SuperBuilder; +import lombok.extern.slf4j.Slf4j; + +@Data +@Slf4j +@SuperBuilder +public class Rule { + private String path; + private String purl; + + public void setPurl(String purl) throws MalformedPackageURLException { + new PackageURL(purl); + this.purl = purl; + } + +} + + diff --git a/src/main/java/com/scanoss/settings/Settings.java b/src/main/java/com/scanoss/settings/Settings.java new file mode 100644 index 0000000..acdcacf --- /dev/null +++ b/src/main/java/com/scanoss/settings/Settings.java @@ -0,0 +1,11 @@ +package com.scanoss.settings; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class Settings { + private Bom bom; +} + diff --git a/src/main/java/com/scanoss/utils/JsonUtils.java b/src/main/java/com/scanoss/utils/JsonUtils.java index 7d22b89..e11ba0b 100644 --- a/src/main/java/com/scanoss/utils/JsonUtils.java +++ b/src/main/java/com/scanoss/utils/JsonUtils.java @@ -26,15 +26,11 @@ import com.google.gson.reflect.TypeToken; import com.scanoss.dto.ScanFileDetails; import com.scanoss.dto.ScanFileResult; -import com.scanoss.settings.BomConfiguration; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import java.io.IOException; import java.io.PrintWriter; import java.lang.reflect.Type; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -212,22 +208,7 @@ public static List toScanFileResultsFromObject(@NonNull JsonObje return results; } - /** - * Convert the given JSON Object to a Settings object - * - * @param jsonObject JSON Object - * @return Settings - */ - public static BomConfiguration toBomConfigurationFromObject(@NonNull JsonObject jsonObject) { - Gson gson = new Gson(); - return gson.fromJson(jsonObject, BomConfiguration.class); - } - public static BomConfiguration toBomConfigurationFromFilePath(@NonNull String path) throws IOException { - String settingsContent = Files.readString(Path.of(path)); - JsonObject settingsJson = JsonUtils.toJsonObject(settingsContent); - return JsonUtils.toBomConfigurationFromObject(settingsJson); - } /** * Determine if the given string is a boolean true/false diff --git a/src/test/java/com/scanoss/TestBomConfiguration.java b/src/test/java/com/scanoss/TestBomConfiguration.java deleted file mode 100644 index 346205e..0000000 --- a/src/test/java/com/scanoss/TestBomConfiguration.java +++ /dev/null @@ -1,58 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright (c) 2023, SCANOSS - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -package com.scanoss; - -import com.google.gson.Gson; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.scanoss.dto.*; -import com.scanoss.settings.BomConfiguration; -import com.scanoss.utils.JsonUtils; -import lombok.extern.slf4j.Slf4j; -import org.junit.Before; -import org.junit.Test; - -import java.util.List; - -import static com.scanoss.TestConstants.*; -import static org.junit.Assert.*; - -@Slf4j -public class TestBomConfiguration { - @Before - public void Setup() { - log.info("Starting Bom Configuration test cases..."); - log.debug("Logging debug enabled"); - log.trace("Logging trace enabled"); - } - - @Test - public void TestInvalidPurl() { - String methodName = new Object() { - }.getClass().getEnclosingMethod().getName(); - log.info("<-- Starting {}", methodName); - - log.info("Finished {} -->", methodName); - } - -} diff --git a/src/test/java/com/scanoss/TestJsonUtils.java b/src/test/java/com/scanoss/TestJsonUtils.java index e86e42b..8e2c7fd 100644 --- a/src/test/java/com/scanoss/TestJsonUtils.java +++ b/src/test/java/com/scanoss/TestJsonUtils.java @@ -23,7 +23,6 @@ package com.scanoss; import com.google.gson.JsonObject; -import com.scanoss.settings.BomConfiguration; import com.scanoss.utils.JsonUtils; import com.scanoss.dto.ScanFileResult; import lombok.extern.slf4j.Slf4j; @@ -88,26 +87,4 @@ public void TestRawResultsPositive() { log.info("Finished {} -->", methodName); } - @Test() - public void TestBomConfiguration() { - String methodName = new Object() { - }.getClass().getEnclosingMethod().getName(); - log.info("<-- Starting {}", methodName); - - - JsonObject jsonObject = JsonUtils.toJsonObject(BOM_CONFIGURATION_MOCK); - assertNotNull(jsonObject); - assertFalse("Should have decoded JSON Objects", jsonObject.isEmpty()); - log.info("JSON Objects: {}", jsonObject); - - - - BomConfiguration bomConfiguration = JsonUtils.toBomConfigurationFromObject(jsonObject); - assertNotNull(bomConfiguration); - log.info("Bom Configuration: {}", bomConfiguration); - - - log.info("Finished {} -->", methodName); - - } } diff --git a/src/test/java/com/scanoss/TestScannerPostProcessor.java b/src/test/java/com/scanoss/TestScannerPostProcessor.java index fb63b0d..a33a504 100644 --- a/src/test/java/com/scanoss/TestScannerPostProcessor.java +++ b/src/test/java/com/scanoss/TestScannerPostProcessor.java @@ -22,15 +22,15 @@ */ package com.scanoss; import com.google.gson.JsonObject; -import com.scanoss.exceptions.ScannerPostProcessorException; +import com.scanoss.settings.Bom; +import com.scanoss.settings.ReplaceRule; +import com.scanoss.settings.Rule; import com.scanoss.utils.JsonUtils; import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; import static com.scanoss.TestConstants.jsonResultsString; -import static org.junit.Assert.assertFalse; import com.scanoss.dto.ScanFileResult; -import com.scanoss.settings.BomConfiguration; import java.util.*; @@ -39,51 +39,20 @@ @Slf4j public class TestScannerPostProcessor { private ScannerPostProcessor scannerPostProcessor; - private BomConfiguration bomConfiguration; private List sampleScanResults; @Before public void Setup() { log.info("Starting ScannerPostProcessor test cases..."); scannerPostProcessor = new ScannerPostProcessor(); - setupBomConfiguration(); - setupSampleScanResults(); - } - - private void setupBomConfiguration() { - bomConfiguration = new BomConfiguration(); - BomConfiguration.Bom bom = new BomConfiguration.Bom(); - bomConfiguration.setBom(bom); - } - private void setupSampleScanResults() { JsonObject jsonObject = JsonUtils.toJsonObject(jsonResultsString); sampleScanResults = JsonUtils.toScanFileResultsFromObject(jsonObject); - } - - /** TESTING REMOVE RULES**/ - @Test - public void TestNullParameters() { - String methodName = new Object() { - }.getClass().getEnclosingMethod().getName(); - log.info("<-- Starting {}", methodName); - try { - scannerPostProcessor.process(null, bomConfiguration); - fail("Should throw ScannerPostProcessorException when scan results is null"); - } catch (Exception e) { - assertTrue("Wrong exception type thrown: " + e.getClass().getSimpleName(), e instanceof ScannerPostProcessorException); - } + } - try { - scannerPostProcessor.process(sampleScanResults, null); - fail("Should throw ScannerPostProcessorException when BOM configuration is null"); - } catch (Exception e) { - assertTrue("Wrong exception type thrown: " + e.getClass().getSimpleName(), e instanceof ScannerPostProcessorException); - } - log.info("Finished {} -->", methodName); - } + /** TESTING REMOVE RULES**/ @Test public void TestRemoveRuleWithPathAndPurl() { @@ -91,14 +60,18 @@ public void TestRemoveRuleWithPathAndPurl() { }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); - // Setup remove rule with both path and purl - BomConfiguration.Rule removeRule = new BomConfiguration.Rule(); - removeRule.setPath("CMSsite/admin/js/npm.js"); - removeRule.setPurl("pkg:github/twbs/bootstrap"); - bomConfiguration.getBom().setRemove(Collections.singletonList(removeRule)); + Rule removeRule = Rule.builder() + .purl("pkg:github/twbs/bootstrap") + .path("CMSsite/admin/js/npm.js") + .build(); + + Bom bom = Bom.builder(). + remove(Arrays.asList(removeRule)) + .build(); + // Process results - List results = scannerPostProcessor.process(sampleScanResults, bomConfiguration); + List results = scannerPostProcessor.process(sampleScanResults, bom); // Verify assertEquals("Should have one result less after removal", sampleScanResults.size()-1, results.size()); @@ -112,12 +85,17 @@ public void TestRemoveRuleWithPurlOnly() { log.info("<-- Starting {}", methodName); // Setup remove rule with only purl - BomConfiguration.Rule removeRule = new BomConfiguration.Rule(); - removeRule.setPurl("pkg:npm/mip-bootstrap"); - bomConfiguration.getBom().setRemove(Collections.singletonList(removeRule)); + Rule removeRule = Rule.builder() + .purl("pkg:npm/mip-bootstrap") + .path("CMSsite/admin/js/npm.js") + .build(); + + Bom bom = Bom.builder(). + remove(Arrays.asList(removeRule)) + .build(); // Process results - List results = scannerPostProcessor.process(sampleScanResults, bomConfiguration); + List results = scannerPostProcessor.process(sampleScanResults, bom); // Verify assertEquals("Size should decrease by 1 after removal", @@ -136,14 +114,20 @@ public void TestNoMatchingRemoveRules() { }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); + // Setup non-matching remove rule - BomConfiguration.Rule removeRule = new BomConfiguration.Rule(); - removeRule.setPath("non/existing/path.c"); - removeRule.setPurl("pkg:github/non-existing/lib@1.0.0"); - bomConfiguration.getBom().setRemove(Collections.singletonList(removeRule)); + Rule removeRule = Rule.builder() + .purl("pkg:github/non-existing/lib@1.0.0") + .path("non/existing/path.c") + .build(); + + Bom bom = Bom.builder(). + remove(Arrays.asList(removeRule)) + .build(); + // Process results - List results = scannerPostProcessor.process(sampleScanResults, bomConfiguration); + List results = scannerPostProcessor.process(sampleScanResults, bom); // Verify assertEquals("Should keep all results", sampleScanResults.size(), results.size()); @@ -158,25 +142,28 @@ public void TestMultipleRemoveRules() { log.info("<-- Starting {}", methodName); // Setup multiple remove rules - List removeRules = new ArrayList<>(); - BomConfiguration.Rule rule1 = new BomConfiguration.Rule(); - rule1.setPath("CMSsite/admin/js/npm.js"); - rule1.setPurl("pkg:npm/myoneui"); + Rule removeRule1 = Rule.builder() + .purl("pkg:npm/myoneui") + .path("CMSsite/admin/js/npm.js") + .build(); + + Rule removeRule2 = Rule.builder() + .purl("pkg:pypi/scanoss") + .build(); - BomConfiguration.Rule rule2 = new BomConfiguration.Rule(); - rule2.setPurl("pkg:pypi/scanoss"); + Rule removeRule3 = Rule.builder() + .path("scanoss/__init__.py") + .build(); - BomConfiguration.Rule rule3 = new BomConfiguration.Rule(); - rule3.setPath("scanoss/__init__.py"); - removeRules.add(rule1); - removeRules.add(rule2); - removeRules.add(rule3); - bomConfiguration.getBom().setRemove(removeRules); + Bom bom = Bom.builder(). + remove(Arrays.asList(removeRule1, removeRule2, removeRule3)) + .build(); + // Process results - List results = scannerPostProcessor.process(sampleScanResults, bomConfiguration); + List results = scannerPostProcessor.process(sampleScanResults, bom); // Verify assertTrue("Should remove all results", results.isEmpty()); @@ -190,8 +177,11 @@ public void TestEmptyRemoveRules() { }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); + Bom bom = Bom.builder() + .build(); + // Process results with empty remove rules - List results = scannerPostProcessor.process(sampleScanResults, bomConfiguration); + List results = scannerPostProcessor.process(sampleScanResults, bom); // Verify assertEquals("Should keep all results", sampleScanResults.size(), results.size()); @@ -208,10 +198,15 @@ public void TestReplaceRuleWithEmptyPurl() { log.info("<-- Starting {}", methodName); // Setup replace rule with empty PURL - BomConfiguration.ReplaceRule replaceRule = new BomConfiguration.ReplaceRule(); - replaceRule.setPurl("pkg:github/scanoss/scanoss.py"); - replaceRule.setReplaceWith(""); - bomConfiguration.getBom().setReplace(Collections.singletonList(replaceRule)); + ReplaceRule replace = ReplaceRule.builder() + .purl("pkg:github/scanoss/scanoss.py") + .replaceWith("") + .build(); + + Bom bom = Bom.builder() + .replace(Arrays.asList(replace)) + .build(); + // Find the specific result for scanoss.py Optional originalResult = sampleScanResults.stream() @@ -223,7 +218,7 @@ public void TestReplaceRuleWithEmptyPurl() { // Process results - List results = scannerPostProcessor.process(sampleScanResults, bomConfiguration); + List results = scannerPostProcessor.process(sampleScanResults, bom); Optional processedResult = results.stream() .filter(r -> r.getFilePath().equals("scanoss/api/__init__.py")) @@ -244,13 +239,18 @@ public void TestReplaceRuleWithPurl() { log.info("<-- Starting {}", methodName); - // Setup replace rule - BomConfiguration.ReplaceRule replaceRule = new BomConfiguration.ReplaceRule(); - replaceRule.setPurl("pkg:github/scanoss/scanoss.py"); - replaceRule.setReplaceWith("pkg:github/scanoss/scanner.c"); - bomConfiguration.getBom().setReplace(Collections.singletonList(replaceRule)); + // Setup replace rule with empty PURL + ReplaceRule replace = ReplaceRule.builder() + .purl("pkg:github/scanoss/scanoss.py") + .replaceWith("pkg:github/scanoss/scanner.c") + .build(); + + Bom bom = Bom.builder() + .replace(Arrays.asList(replace)) + .build(); + - List results = scannerPostProcessor.process(sampleScanResults, bomConfiguration); + List results = scannerPostProcessor.process(sampleScanResults, bom); Optional processedResult = results.stream() .filter(r -> r.getFilePath().equals("scanoss/api/__init__.py")) From 06011799d943f50a14345f70a908b3771099b4d3 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Tue, 19 Nov 2024 15:35:27 +0100 Subject: [PATCH 06/28] feat: use cached components for replacement rules --- pom.xml | 6 + .../com/scanoss/ScannerPostProcessor.java | 102 ++++++++++----- .../java/com/scanoss/cli/ScanCommandLine.java | 16 ++- .../java/com/scanoss/dto/LicenseDetails.java | 20 ++- .../java/com/scanoss/dto/ScanFileDetails.java | 46 ++++--- src/main/java/com/scanoss/settings/Bom.java | 20 +-- .../com/scanoss/settings/ReplaceRule.java | 1 + src/main/java/com/scanoss/settings/Rule.java | 13 +- .../java/com/scanoss/settings/Settings.java | 36 ++++++ .../com/scanoss/TestScannerPostProcessor.java | 120 ++++++++++++++++-- src/test/java/com/scanoss/TestSettings.java | 102 +++++++++++++++ 11 files changed, 382 insertions(+), 100 deletions(-) create mode 100644 src/test/java/com/scanoss/TestSettings.java diff --git a/pom.xml b/pom.xml index 5d7a74e..3625071 100644 --- a/pom.xml +++ b/pom.xml @@ -126,6 +126,12 @@ packageurl-java 1.5.0 + + org.junit.jupiter + junit-jupiter + RELEASE + test + diff --git a/src/main/java/com/scanoss/ScannerPostProcessor.java b/src/main/java/com/scanoss/ScannerPostProcessor.java index 2be9bac..73fb95c 100644 --- a/src/main/java/com/scanoss/ScannerPostProcessor.java +++ b/src/main/java/com/scanoss/ScannerPostProcessor.java @@ -1,16 +1,19 @@ package com.scanoss; -import com.scanoss.dto.ScanFileDetails; -import com.scanoss.dto.ScanFileResult; +import com.github.packageurl.MalformedPackageURLException; +import com.github.packageurl.PackageURL; +import com.scanoss.dto.*; import com.scanoss.settings.Bom; import com.scanoss.settings.ReplaceRule; import com.scanoss.settings.Rule; -import com.scanoss.settings.Settings; +import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; +import java.lang.reflect.Array; import java.util.*; import java.util.stream.Collectors; +@Slf4j public class ScannerPostProcessor { private Map componentIndex = new HashMap<>(); @@ -20,7 +23,7 @@ public class ScannerPostProcessor { * Applies remove, and replace rules as specified in the configuration. * * @param scanFileResults List of scan results to process - * @param bom Bom containing BOM rules + * @param bom Bom containing BOM rules * @return List of processed scan results */ public List process(@NotNull List scanFileResults, @NotNull Bom bom) { @@ -28,12 +31,10 @@ public List process(@NotNull List scanFileResult List processedResults = new ArrayList<>(scanFileResults); - // Apply remove rules if (bom.getRemove() != null && !bom.getRemove().isEmpty()) { processedResults = applyRemoveRules(processedResults, bom.getRemove()); } - //Apply replace rules if (bom.getReplace() != null && !bom.getReplace().isEmpty()) { processedResults = applyReplaceRules(processedResults, bom.getReplace()); } @@ -65,7 +66,8 @@ private void createComponentIndex(List scanFileResults) { Map.Entry::getValue, (existing, replacement) -> existing, HashMap::new - )); } + )); + } /** @@ -73,32 +75,66 @@ private void createComponentIndex(List scanFileResults) { * If a cached component exists for a replacement PURL, it will be used instead of creating a new one. * * @param results The original list of scan results to process - * @param replaceRules The list of replacement rules to apply + * @param rules The list of replacement rules to apply * @return A new list containing the processed scan results with updated PURLs */ - private List applyReplaceRules(List results, List replaceRules) { - if (results == null || replaceRules == null) { - return results; - } - + private List applyReplaceRules(@NotNull List results, @NotNull List rules) { List resultsList = new ArrayList<>(results); for (ScanFileResult result : resultsList) { - findMatchingRule(result, replaceRules).ifPresent(matchedRule -> { - String replacementPurl = matchedRule.getReplaceWith(); - if (replacementPurl == null || replacementPurl.trim().isEmpty()) { - return; //Empty replacement PURL found + for (ReplaceRule rule : this.findMatchingRules(result, rules)) { + + + + PackageURL newPurl; + try { + newPurl = new PackageURL(rule.getReplaceWith()); + } catch (MalformedPackageURLException e) { + log.error("ERROR: Parsing purl from rule: {} - {}", rule, e.getMessage()); + continue; } - // Try to get cached component first - ScanFileDetails cachedComponent = this.componentIndex.get(replacementPurl); - if (cachedComponent != null) { - result.getFileDetails().set(0, cachedComponent); // Use cached component if available + LicenseDetails[] licenseDetails = new LicenseDetails[]{LicenseDetails.builder().name(rule.getLicense()).build()}; + + ScanFileDetails cachedFileDetails = this.componentIndex.get(newPurl.toString()); + ScanFileDetails currentFileDetails = result.getFileDetails().get(0); + ScanFileDetails newFileDetails; + + if (cachedFileDetails != null) { + + newFileDetails = ScanFileDetails.builder() + .id(currentFileDetails.getId()) + .file(currentFileDetails.getFile()) + .fileHash(currentFileDetails.getFileHash()) + .fileUrl(currentFileDetails.getFileUrl()) + .lines(currentFileDetails.getLines()) + .matched(currentFileDetails.getMatched()) + .licenseDetails(licenseDetails) + .component(cachedFileDetails.getComponent()) + .vendor(cachedFileDetails.getVendor()) + .url(cachedFileDetails.getUrl()) + .purls(new String[]{newPurl.toString()}) + .build(); + + + result.getFileDetails().set(0, cachedFileDetails); } else { - result.getFileDetails().get(0).setPurls(new String[] { replacementPurl.trim() }); // Create new PURL array if no cached component exists + + newFileDetails = currentFileDetails; + + newFileDetails.setCopyrightDetails(new CopyrightDetails[]{}); + newFileDetails.setLicenseDetails(new LicenseDetails[]{}); + newFileDetails.setVulnerabilityDetails(new VulnerabilityDetails[]{}); + newFileDetails.setPurls(new String[]{newPurl.toString()}); + newFileDetails.setUrl(""); + + newFileDetails.setComponent(newPurl.getName()); + newFileDetails.setVendor(newPurl.getNamespace()); } - }); + + result.getFileDetails().set(0, newFileDetails); + } } return resultsList; } @@ -109,27 +145,32 @@ private List applyReplaceRules(List results, Lis * A result will be removed if: * 1. The remove rule has both path and purl, and both match the result * 2. The remove rule has only purl (no path), and the purl matches the result + * 3. The remove rule has only path (no purl), and the path matches the result + * + * @param results The list of scan results to process + * @param removeRules The list of remove rules to apply + * @return A new list with matching results removed */ private List applyRemoveRules(@NotNull List results, @NotNull List removeRules) { List resultsList = new ArrayList<>(results); - resultsList.removeIf(result -> findMatchingRule(result, removeRules).isPresent()); + resultsList.removeIf(result -> !findMatchingRules(result, removeRules).isEmpty()); return resultsList; } /** - * Finds and returns the first matching rule for a scan result. + * Finds and returns a list of matching rules for a scan result. * A rule matches if: * 1. It has both path and purl, and both match the result * 2. It has only a purl (no path), and the purl matches the result * 3. It has only a path (no purl), and the path matches the result * - * @param The rule type. Must extend Rule class + * @param The rule type. Must extend Rule class * @param result The scan result to check - * @param rules List of rules to check against - * @return Optional containing the first matching rule, or empty if no match found + * @param rules List of rules to check against + * @return List of matching rules, empty list if no matches found */ - private Optional findMatchingRule(ScanFileResult result, List rules) { + private List findMatchingRules(@NotNull ScanFileResult result, @NotNull List rules) { return rules.stream() .filter(rule -> { boolean hasPath = rule.getPath() != null && !rule.getPath().isEmpty(); @@ -144,8 +185,7 @@ private Optional findMatchingRule(ScanFileResult result, Lis } return false; // Neither path nor purl specified - }) - .findFirst(); + }).collect(Collectors.toList()); } /** diff --git a/src/main/java/com/scanoss/cli/ScanCommandLine.java b/src/main/java/com/scanoss/cli/ScanCommandLine.java index 92b9b57..ae92855 100644 --- a/src/main/java/com/scanoss/cli/ScanCommandLine.java +++ b/src/main/java/com/scanoss/cli/ScanCommandLine.java @@ -27,6 +27,7 @@ import com.scanoss.dto.ScanFileResult; import com.scanoss.exceptions.ScannerException; import com.scanoss.exceptions.WinnowingException; +import com.scanoss.settings.Settings; import com.scanoss.utils.JsonUtils; import com.scanoss.utils.ProxyUtils; import lombok.NonNull; @@ -35,6 +36,8 @@ import java.io.IOException; import java.net.Proxy; import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.time.Duration; import java.util.List; @@ -187,12 +190,13 @@ public void run() { } if (settings != null && !settings.isEmpty()) { -// try { -// ScannerPostProcessor scannerPostProcessor = new ScannerPostProcessor(); -// scanFileResults = scannerPostProcessor.process(scanFileResults, JsonUtils.toBomConfigurationFromFilePath(settings)); -// } catch (Exception e) { -// throw new RuntimeException(e); -// } + try { + Path path = Paths.get(settings); + ScannerPostProcessor scannerPostProcessor = new ScannerPostProcessor(); + scanFileResults = scannerPostProcessor.process(scanFileResults, Settings.fromPath(path).getBom()); + } catch (Exception e) { + throw new RuntimeException(e); + } } var out = spec.commandLine().getOut(); diff --git a/src/main/java/com/scanoss/dto/LicenseDetails.java b/src/main/java/com/scanoss/dto/LicenseDetails.java index a230a6c..14cbc05 100644 --- a/src/main/java/com/scanoss/dto/LicenseDetails.java +++ b/src/main/java/com/scanoss/dto/LicenseDetails.java @@ -23,7 +23,10 @@ package com.scanoss.dto; import com.google.gson.annotations.SerializedName; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; import static com.scanoss.utils.JsonUtils.checkBooleanString; @@ -31,17 +34,20 @@ * Scan Results Match License Details */ @Data +@Builder +@NoArgsConstructor +@AllArgsConstructor public class LicenseDetails { - private final String name; - private final String source; - private final String copyleft; + private String name; + private String source; + private String copyleft; @SerializedName("patent_hints") - private final String patentHints; - private final String url; + private String patentHints; + private String url; @SerializedName("checklist_url") - private final String checklistUrl; + private String checklistUrl; @SerializedName("osadl_updated") - private final String osadlUpdated; + private String osadlUpdated; /** * Determine if the license is Copyleft or not diff --git a/src/main/java/com/scanoss/dto/ScanFileDetails.java b/src/main/java/com/scanoss/dto/ScanFileDetails.java index 4bbdb24..a62e6b7 100644 --- a/src/main/java/com/scanoss/dto/ScanFileDetails.java +++ b/src/main/java/com/scanoss/dto/ScanFileDetails.java @@ -23,45 +23,51 @@ package com.scanoss.dto; import com.google.gson.annotations.SerializedName; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; /** * Scan File Result Detailed Information */ @Data +@Builder +@NoArgsConstructor +@AllArgsConstructor public class ScanFileDetails { - private final String id; + private String id; private String component; - private final String vendor; - private final String version; - private final String latest; - private final String url; - private final String status; - private final String matched; - private final String file; - private final String lines; + private String vendor; + private String version; + private String latest; + private String url; + private String status; + private String matched; + private String file; + private String lines; @SerializedName("oss_lines") - private final String ossLines; + private String ossLines; @SerializedName("file_hash") - private final String fileHash; + private String fileHash; @SerializedName("file_url") - private final String fileUrl; + private String fileUrl; @SerializedName("url_hash") - private final String urlHash; + private String urlHash; @SerializedName("release_date") - private final String releaseDate; + private String releaseDate; @SerializedName("source_hash") - private final String sourceHash; + private String sourceHash; @SerializedName("purl") private String[] purls; @SerializedName("server") - private final ServerDetails serverDetails; + private ServerDetails serverDetails; @SerializedName("licenses") - private final LicenseDetails[] licenseDetails; + private LicenseDetails[] licenseDetails; @SerializedName("quality") - private final QualityDetails[] qualityDetails; + private QualityDetails[] qualityDetails; @SerializedName("vulnerabilities") - private final VulnerabilityDetails[] vulnerabilityDetails; + private VulnerabilityDetails[] vulnerabilityDetails; @SerializedName("copyrights") - private final CopyrightDetails[] copyrightDetails; + private CopyrightDetails[] copyrightDetails; } diff --git a/src/main/java/com/scanoss/settings/Bom.java b/src/main/java/com/scanoss/settings/Bom.java index 5d849d0..041225f 100644 --- a/src/main/java/com/scanoss/settings/Bom.java +++ b/src/main/java/com/scanoss/settings/Bom.java @@ -2,27 +2,17 @@ import lombok.Builder; import lombok.Data; -import org.jetbrains.annotations.NotNull; +import lombok.Singular; import java.util.List; @Data @Builder public class Bom { - private List include; - private List remove; - private List replace; - - - public void addInclude(@NotNull Rule rule) { - this.include.add(rule); - } - - public void addRemove(@NotNull Rule rule) { - this.include.add(rule); - } - - + private @Singular("include") List include; + private @Singular("ignore") List ignore; + private @Singular("remove") List remove; + private @Singular("replace") List replace; } diff --git a/src/main/java/com/scanoss/settings/ReplaceRule.java b/src/main/java/com/scanoss/settings/ReplaceRule.java index b193c92..e0ebb45 100644 --- a/src/main/java/com/scanoss/settings/ReplaceRule.java +++ b/src/main/java/com/scanoss/settings/ReplaceRule.java @@ -1,5 +1,6 @@ package com.scanoss.settings; +import com.github.packageurl.PackageURL; import com.google.gson.annotations.SerializedName; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/src/main/java/com/scanoss/settings/Rule.java b/src/main/java/com/scanoss/settings/Rule.java index f44c81e..129da9b 100644 --- a/src/main/java/com/scanoss/settings/Rule.java +++ b/src/main/java/com/scanoss/settings/Rule.java @@ -1,24 +1,15 @@ package com.scanoss.settings; - -import com.github.packageurl.MalformedPackageURLException; import com.github.packageurl.PackageURL; -import lombok.Builder; import lombok.Data; import lombok.experimental.SuperBuilder; import lombok.extern.slf4j.Slf4j; @Data @Slf4j -@SuperBuilder +@SuperBuilder() public class Rule { private String path; - private String purl; - - public void setPurl(String purl) throws MalformedPackageURLException { - new PackageURL(purl); - this.purl = purl; - } - + private String purl; //TODO: Add validation with PackageURL } diff --git a/src/main/java/com/scanoss/settings/Settings.java b/src/main/java/com/scanoss/settings/Settings.java index acdcacf..3da603d 100644 --- a/src/main/java/com/scanoss/settings/Settings.java +++ b/src/main/java/com/scanoss/settings/Settings.java @@ -1,11 +1,47 @@ package com.scanoss.settings; +import com.google.gson.Gson; import lombok.Builder; import lombok.Data; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; @Data @Builder public class Settings { private Bom bom; + + + + /** + * Creates a Settings object from a JSON string + * + * @param json The JSON string to parse + * @return A new Settings object + */ + public static Settings fromJSON(@NotNull String json) { + Gson gson = new Gson(); + return gson.fromJson(json, Settings.class); + } + + /** + * Creates a Settings object from a JSON file + * + * @param path The path to the JSON file + * @return A new Settings object + * @throws IOException If there's an error reading the file + */ + public static Settings fromPath(@NotNull Path path) throws IOException { + try { + String json = Files.readString(path, StandardCharsets.UTF_8); + return fromJSON(json); + } catch (IOException e) { + throw new IOException("Failed to read settings file: " + path, e); + } + } } diff --git a/src/test/java/com/scanoss/TestScannerPostProcessor.java b/src/test/java/com/scanoss/TestScannerPostProcessor.java index a33a504..0cdb721 100644 --- a/src/test/java/com/scanoss/TestScannerPostProcessor.java +++ b/src/test/java/com/scanoss/TestScannerPostProcessor.java @@ -32,7 +32,14 @@ import static com.scanoss.TestConstants.jsonResultsString; import com.scanoss.dto.ScanFileResult; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.*; +import java.util.stream.Collectors; import static org.junit.Assert.*; @@ -40,35 +47,45 @@ public class TestScannerPostProcessor { private ScannerPostProcessor scannerPostProcessor; private List sampleScanResults; + private List longScanResults; @Before - public void Setup() { + public void Setup() throws URISyntaxException, IOException { log.info("Starting ScannerPostProcessor test cases..."); scannerPostProcessor = new ScannerPostProcessor(); - JsonObject jsonObject = JsonUtils.toJsonObject(jsonResultsString); sampleScanResults = JsonUtils.toScanFileResultsFromObject(jsonObject); + + var resource = getClass().getClassLoader().getResource("results.json"); + if (resource == null) { + throw new IllegalStateException( + "Required test resource 'results.json' not found. Please ensure it exists in src/test/resources/data/" + ); + } + + + String json = Files.readString(Paths.get(resource.toURI()), StandardCharsets.UTF_8); + longScanResults = JsonUtils.toScanFileResultsFromObject(JsonUtils.toJsonObject(json)); + } - /** TESTING REMOVE RULES**/ + + /** TESTING REMOVE RULES**/ @Test public void TestRemoveRuleWithPathAndPurl() { String methodName = new Object() { }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); - Rule removeRule = Rule.builder() + Rule rule = Rule.builder() .purl("pkg:github/twbs/bootstrap") .path("CMSsite/admin/js/npm.js") .build(); - Bom bom = Bom.builder(). - remove(Arrays.asList(removeRule)) - .build(); - + Bom bom = Bom.builder().remove(rule).build(); // Process results List results = scannerPostProcessor.process(sampleScanResults, bom); @@ -127,10 +144,10 @@ public void TestNoMatchingRemoveRules() { // Process results - List results = scannerPostProcessor.process(sampleScanResults, bom); + List results = scannerPostProcessor.process(longScanResults, bom); // Verify - assertEquals("Should keep all results", sampleScanResults.size(), results.size()); + assertEquals("Should keep all results", longScanResults.size(), results.size()); log.info("Finished {} -->", methodName); } @@ -270,4 +287,87 @@ public void TestReplaceRuleWithPurl() { + @Test() + public void TestOriginalPurlExistsWhenNoReplacementRule() { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + String originalPurl = "pkg:github/scanoss/scanner.c"; + + // Setup BOM without replace rule - expecting original PURL to exist + Bom bom = Bom.builder() + .build(); + + List results = scannerPostProcessor.process(longScanResults, bom); + + assertNotNull("Results should not be null", results); + assertFalse("Results should not be empty", results.isEmpty()); + + List allPurls = results.stream() + .map(result -> result.getFileDetails().get(0).getPurls()) + .flatMap(Arrays::stream) + .collect(Collectors.toList()); + + log.info("All PURLs found: {}", allPurls); + log.info("Original PURL we're looking for: '{}'", originalPurl); + + + boolean hasOriginalPurl = results.stream() + .map(result -> result.getFileDetails().get(0).getPurls()) + .flatMap(Arrays::stream) + .anyMatch(purl -> { + log.info("Comparing: '{}' with '{}' = {}", + purl, originalPurl, purl.equals(originalPurl)); + return purl.equals(originalPurl); + }); + + assertTrue("Original PURL should exist since no replacement rule was set", hasOriginalPurl); + + log.info("Finished {} -->", methodName); + } + + + @Test + public void TestOriginalPurlNotExistsWhenReplacementRuleDefined() { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + String originalPurl = "pkg:github/scanoss/scanner.c"; + String replacementPurl = "pkg:maven/com.scanoss/scanoss"; + + // Setup replace rule + ReplaceRule replace = ReplaceRule.builder() + .purl(originalPurl) + .replaceWith(replacementPurl) + .build(); + + Bom bom = Bom.builder() + .replace(Arrays.asList(replace)) + .build(); + + List results = scannerPostProcessor.process(longScanResults, bom); + + assertNotNull("Results should not be null", results); + assertFalse("Results should not be empty", results.isEmpty()); + + boolean hasOriginalPurl = results.stream() + .map(result -> result.getFileDetails().get(0).getPurls()) + .flatMap(Arrays::stream) + .anyMatch(purl -> purl.equals(originalPurl)); + + assertFalse("Original PURL should not exist when replacement rule is set", hasOriginalPurl); + + boolean hasReplacementPurl = results.stream() + .map(result -> result.getFileDetails().get(0).getPurls()) + .flatMap(Arrays::stream) + .anyMatch(purl -> purl.equals(replacementPurl)); + + assertTrue("Replacement PURL should exist", hasReplacementPurl); + + log.info("Finished {} -->", methodName); + } + + + + } \ No newline at end of file diff --git a/src/test/java/com/scanoss/TestSettings.java b/src/test/java/com/scanoss/TestSettings.java new file mode 100644 index 0000000..6362fb6 --- /dev/null +++ b/src/test/java/com/scanoss/TestSettings.java @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2023, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.scanoss; + +import com.scanoss.settings.Settings; +import com.scanoss.utils.ProxyUtils; +import lombok.extern.slf4j.Slf4j; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.net.Proxy; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; + +import static org.junit.Assert.*; + +@Slf4j +public class TestSettings { + private Path existingSettingsPath; + private Path nonExistentSettingsPath; + @Before + public void Setup() throws URISyntaxException { + log.info("Starting Settings test cases..."); + log.debug("Logging debug enabled"); + log.trace("Logging trace enabled"); + + // Check if resource exists before attempting to get its path + var resource = getClass().getClassLoader().getResource("scanoss.json"); + if (resource == null) { + throw new IllegalStateException( + "Required test resource 'scanoss.json' not found. Please ensure it exists in src/test/resources/data/" + ); + } + + existingSettingsPath = Paths.get(resource.toURI()); + nonExistentSettingsPath = Paths.get("non-existent-settings.json"); + + // Verify the file actually exists + if (!Files.exists(existingSettingsPath)) { + throw new IllegalStateException( + "Test file exists as resource but cannot be accessed at path: " + + existingSettingsPath + ); + } + } + + + @Test + public void testSettingsFromExistingFile() { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + try { + Settings settings = Settings.fromPath(existingSettingsPath); + assertNotNull("Settings should not be null", settings); + + assertEquals("scanner.c", settings.getBom().getRemove().get(0).getPath()); + assertEquals("pkg:github/scanoss/scanner.c", settings.getBom().getRemove().get(0).getPurl()); + + } catch (IOException e) { + fail("Should not throw IOException for existing file: " + e.getMessage()); + } + + log.info("Finished {} -->", methodName); + } + + @Test(expected = IOException.class) + public void testSettingsFromNonExistentFile() throws IOException { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + Settings.fromPath(nonExistentSettingsPath); + // Should throw IOException before reaching this point + + log.info("Finished {} -->", methodName); + } + +} From d3d8c81eaf98ca8448fc69e177829b0250602d46 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 2 Dec 2024 21:59:03 +0100 Subject: [PATCH 07/28] adds testing files --- src/test/resources/results.json | 392 ++++++++++++++++++++++++++++++++ src/test/resources/scanoss.json | 12 + 2 files changed, 404 insertions(+) create mode 100644 src/test/resources/results.json create mode 100644 src/test/resources/scanoss.json diff --git a/src/test/resources/results.json b/src/test/resources/results.json new file mode 100644 index 0000000..75c2f98 --- /dev/null +++ b/src/test/resources/results.json @@ -0,0 +1,392 @@ +{ + "external/inc/json.h": [ + { + "component": "scanner.c", + "file": "scanner.c-1.3.3/external/inc/json.h", + "file_hash": "e91a03b850651dd56dd979ba92668a19", + "file_url": "https://api.osskb.org/file_contents/e91a03b850651dd56dd979ba92668a19", + "id": "file", + "latest": "1.3.4", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/BSD-2-Clause.txt", + "copyleft": "no", + "name": "BSD-2-Clause", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "no", + "source": "file_header", + "url": "https://spdx.org/licenses/BSD-2-Clause.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/BSD-2-Clause.txt", + "copyleft": "no", + "name": "BSD-2-Clause", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "no", + "source": "scancode", + "url": "https://spdx.org/licenses/BSD-2-Clause.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-only", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "yes", + "source": "component_declared", + "url": "https://spdx.org/licenses/GPL-2.0-only.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/scanoss/scanner.c" + ], + "release_date": "2021-05-26", + "server": { + "kb_version": { + "daily": "24.11.13", + "monthly": "24.10" + }, + "version": "5.4.8" + }, + "source_hash": "e91a03b850651dd56dd979ba92668a19", + "status": "pending", + "url": "https://github.com/scanoss/scanner.c", + "url_hash": "2d1700ba496453d779d4987255feb5f2", + "url_stats": {}, + "vendor": "scanoss", + "version": "1.3.3" + } + ], + "src/cyclonedx.c": [ + { + "component": "scanner.c", + "file": "scanner.c-1.3.3/src/cyclonedx.c", + "file_hash": "342bab2935f4817281eb262c23a4bdd9", + "file_url": "https://api.osskb.org/file_contents/342bab2935f4817281eb262c23a4bdd9", + "id": "snippet", + "latest": "1.3.3", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-or-later.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-or-later", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "yes", + "source": "scancode", + "url": "https://spdx.org/licenses/GPL-2.0-or-later.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-1.0-or-later.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-1.0-or-later", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "no", + "source": "scancode", + "url": "https://spdx.org/licenses/GPL-1.0-or-later.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-or-later.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-or-later", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "yes", + "source": "file_spdx_tag", + "url": "https://spdx.org/licenses/GPL-2.0-or-later.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-only", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "yes", + "source": "component_declared", + "url": "https://spdx.org/licenses/GPL-2.0-only.html" + } + ], + "lines": "9-9,32-96", + "matched": "66%", + "oss_lines": "7-7,29-93", + "purl": [ + "pkg:github/scanoss/scanner.c" + ], + "release_date": "2021-05-26", + "server": { + "kb_version": { + "daily": "24.11.13", + "monthly": "24.10" + }, + "version": "5.4.8" + }, + "source_hash": "33bbeaa1f27d48d11a6b81e0d7292562", + "status": "pending", + "url": "https://github.com/scanoss/scanner.c", + "url_hash": "2d1700ba496453d779d4987255feb5f2", + "url_stats": {}, + "vendor": "scanoss", + "version": "1.3.3" + } + ], + "src/format_utils.c": [ + { + "component": "scanner.c", + "file": "scanner.c-1.3.3/src/format_utils.c", + "file_hash": "2691bb31301bbc70edbe7960673b7ca7", + "file_url": "https://api.osskb.org/file_contents/2691bb31301bbc70edbe7960673b7ca7", + "id": "snippet", + "latest": "1.3.3", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-only", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "yes", + "source": "component_declared", + "url": "https://spdx.org/licenses/GPL-2.0-only.html" + } + ], + "lines": "8-18,41-294", + "matched": "84%", + "oss_lines": "8-18,48-301", + "purl": [ + "pkg:github/scanoss/scanner.c" + ], + "release_date": "2021-05-26", + "server": { + "kb_version": { + "daily": "24.11.13", + "monthly": "24.10" + }, + "version": "5.4.8" + }, + "source_hash": "1b07d074ce3d81ca10990baed612d5cf", + "status": "pending", + "url": "https://github.com/scanoss/scanner.c", + "url_hash": "2d1700ba496453d779d4987255feb5f2", + "url_stats": {}, + "vendor": "scanoss", + "version": "1.3.3" + } + ], + "src/main.c": [ + { + "component": "scanner.c", + "file": "scanner.c-1.3.2/src/main.c", + "file_hash": "30d93e53539a3ca8db58e8f852fc1c30", + "file_url": "https://api.osskb.org/file_contents/30d93e53539a3ca8db58e8f852fc1c30", + "id": "snippet", + "latest": "1.3.2", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-or-later.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-or-later", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "yes", + "source": "scancode", + "url": "https://spdx.org/licenses/GPL-2.0-or-later.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-1.0-or-later.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-1.0-or-later", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "no", + "source": "scancode", + "url": "https://spdx.org/licenses/GPL-1.0-or-later.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-or-later.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-or-later", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "yes", + "source": "file_spdx_tag", + "url": "https://spdx.org/licenses/GPL-2.0-or-later.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-only", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "yes", + "source": "component_declared", + "url": "https://spdx.org/licenses/GPL-2.0-only.html" + } + ], + "lines": "7-9,50-84,102-161", + "matched": "49%", + "oss_lines": "7-9,45-79,96-155", + "purl": [ + "pkg:github/scanoss/scanner.c" + ], + "release_date": "2021-05-17", + "server": { + "kb_version": { + "daily": "24.11.13", + "monthly": "24.10" + }, + "version": "5.4.8" + }, + "source_hash": "54cbf9c7b4ec540b6b9dd08f1c45dd47", + "status": "pending", + "url": "https://github.com/scanoss/scanner.c", + "url_hash": "bcb91cab3d56bc463bb857a257289f1f", + "url_stats": {}, + "vendor": "scanoss", + "version": "1.3.2" + } + ], + "src/scanner.c": [ + { + "component": "scanoss.java", + "file": "testing/data/test-folder-ignore/test-no-ignore/scanner.c", + "file_hash": "f83589f71d6c31a2afb8d374953292f1", + "file_url": "https://api.osskb.org/file_contents/f83589f71d6c31a2afb8d374953292f1", + "id": "file", + "latest": "8598c47", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/MIT.txt", + "copyleft": "no", + "name": "MIT", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "no", + "source": "component_declared", + "url": "https://spdx.org/licenses/MIT.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-or-later.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-or-later", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "yes", + "source": "file_spdx_tag", + "url": "https://spdx.org/licenses/GPL-2.0-or-later.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/MIT.txt", + "copyleft": "no", + "name": "MIT", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "no", + "source": "license_file", + "url": "https://spdx.org/licenses/MIT.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/scanoss/scanoss.java", + "pkg:maven/com.scanoss/scanoss" + ], + "release_date": "2024-04-12", + "server": { + "kb_version": { + "daily": "24.11.13", + "monthly": "24.10" + }, + "version": "5.4.8" + }, + "source_hash": "f83589f71d6c31a2afb8d374953292f1", + "status": "pending", + "url": "https://github.com/scanoss/scanoss.java", + "url_hash": "7197978c914a0cb767cad47aaa1c8276", + "url_stats": {}, + "vendor": "scanoss", + "version": "0.7.1" + } + ], + "src/spdx.c": [ + { + "component": "scanner.c", + "file": "scanner.c-1.3.3/src/spdx.c", + "file_hash": "00693585177fc51a8d16b2b890f39277", + "file_url": "https://api.osskb.org/file_contents/00693585177fc51a8d16b2b890f39277", + "id": "snippet", + "latest": "1.3.4", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-1.0-or-later.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-1.0-or-later", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "no", + "source": "scancode", + "url": "https://spdx.org/licenses/GPL-1.0-or-later.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-or-later.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-or-later", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "yes", + "source": "scancode", + "url": "https://spdx.org/licenses/GPL-2.0-or-later.html" + }, + { + "name": "CC0-1.0", + "source": "scancode", + "url": "https://spdx.org/licenses/CC0-1.0.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-or-later.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-or-later", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "yes", + "source": "file_spdx_tag", + "url": "https://spdx.org/licenses/GPL-2.0-or-later.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-only", + "osadl_updated": "2024-11-04T13:45:00+0000", + "patent_hints": "yes", + "source": "component_declared", + "url": "https://spdx.org/licenses/GPL-2.0-only.html" + } + ], + "lines": "30-79", + "matched": "62%", + "oss_lines": "28-77", + "purl": [ + "pkg:github/scanoss/scanner.c" + ], + "release_date": "2021-05-26", + "server": { + "kb_version": { + "daily": "24.11.13", + "monthly": "24.10" + }, + "version": "5.4.8" + }, + "source_hash": "920066098c63d986a663132d9ec73e03", + "status": "pending", + "url": "https://github.com/scanoss/scanner.c", + "url_hash": "2d1700ba496453d779d4987255feb5f2", + "url_stats": {}, + "vendor": "scanoss", + "version": "1.3.3" + } + ] +} diff --git a/src/test/resources/scanoss.json b/src/test/resources/scanoss.json new file mode 100644 index 0000000..2d65680 --- /dev/null +++ b/src/test/resources/scanoss.json @@ -0,0 +1,12 @@ +{ + "bom": { + "include": [], + "remove": [ + { + "path": "scanner.c", + "purl": "pkg:github/scanoss/scanner.c" + } + ] + } +} + From 0de844d06d1fd1f1f42b2d1f5a4d814331d8601b Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Tue, 3 Dec 2024 15:12:30 +0100 Subject: [PATCH 08/28] feat: SP-1876 Adds lines range on remove rule --- .../com/scanoss/ScannerPostProcessor.java | 56 ++++-- .../java/com/scanoss/dto/ScanFileDetails.java | 4 +- .../java/com/scanoss/dto/enums/MatchType.java | 7 + src/main/java/com/scanoss/settings/Bom.java | 2 +- .../java/com/scanoss/settings/RemoveRule.java | 16 ++ .../java/com/scanoss/utils/LineRange.java | 24 +++ .../com/scanoss/utils/LineRangeUtils.java | 67 +++++++ src/test/java/com/scanoss/TestConstants.java | 104 ++++++++--- .../java/com/scanoss/TestLineRangeUtils.java | 163 ++++++++++++++++++ .../com/scanoss/TestScannerPostProcessor.java | 154 +++++++++++++---- 10 files changed, 521 insertions(+), 76 deletions(-) create mode 100644 src/main/java/com/scanoss/dto/enums/MatchType.java create mode 100644 src/main/java/com/scanoss/settings/RemoveRule.java create mode 100644 src/main/java/com/scanoss/utils/LineRange.java create mode 100644 src/main/java/com/scanoss/utils/LineRangeUtils.java create mode 100644 src/test/java/com/scanoss/TestLineRangeUtils.java diff --git a/src/main/java/com/scanoss/ScannerPostProcessor.java b/src/main/java/com/scanoss/ScannerPostProcessor.java index 73fb95c..f1b8502 100644 --- a/src/main/java/com/scanoss/ScannerPostProcessor.java +++ b/src/main/java/com/scanoss/ScannerPostProcessor.java @@ -4,12 +4,14 @@ import com.github.packageurl.PackageURL; import com.scanoss.dto.*; import com.scanoss.settings.Bom; +import com.scanoss.settings.RemoveRule; import com.scanoss.settings.ReplaceRule; import com.scanoss.settings.Rule; +import com.scanoss.utils.LineRange; +import com.scanoss.utils.LineRangeUtils; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; -import java.lang.reflect.Array; import java.util.*; import java.util.stream.Collectors; @@ -86,7 +88,6 @@ private List applyReplaceRules(@NotNull List res for (ReplaceRule rule : this.findMatchingRules(result, rules)) { - PackageURL newPurl; try { newPurl = new PackageURL(rule.getReplaceWith()); @@ -104,7 +105,7 @@ private List applyReplaceRules(@NotNull List res if (cachedFileDetails != null) { newFileDetails = ScanFileDetails.builder() - .id(currentFileDetails.getId()) + .matchType(currentFileDetails.getMatchType()) .file(currentFileDetails.getFile()) .fileHash(currentFileDetails.getFileHash()) .fileUrl(currentFileDetails.getFileUrl()) @@ -141,23 +142,49 @@ private List applyReplaceRules(@NotNull List res /** - * Applies remove rules to the scan results. - * A result will be removed if: - * 1. The remove rule has both path and purl, and both match the result - * 2. The remove rule has only purl (no path), and the purl matches the result - * 3. The remove rule has only path (no purl), and the path matches the result + * Applies remove rules to scan results, filtering out matches based on certain criteria. + * + * First, matches are found based on path and/or purl: + * - Rule must match either both path and purl, just the path, or just the purl * - * @param results The list of scan results to process - * @param removeRules The list of remove rules to apply - * @return A new list with matching results removed + * Then, for each matched result: + * 1. If none of the matching rules define line ranges -> Remove the result + * 2. If any matching rules define line ranges -> Only remove if the result's lines overlap with any rule's line range + * + * @param results The list of scan results to process + * @param rules The list of remove rules to apply + * @return A filtered list with matching results removed based on the above criteria */ - private List applyRemoveRules(@NotNull List results, @NotNull List removeRules) { + private List applyRemoveRules(@NotNull List results, @NotNull List rules) { List resultsList = new ArrayList<>(results); - resultsList.removeIf(result -> !findMatchingRules(result, removeRules).isEmpty()); + resultsList.removeIf(result -> { + List matchingRules = findMatchingRules(result, rules); + if (matchingRules.isEmpty()) { + return false; + } + + // Check if any matching rules have line ranges defined + List rulesWithLineRanges = matchingRules.stream() + .filter(rule -> rule.getStartLine() != null && rule.getEndLine() != null) + .collect(Collectors.toList()); + + // If no rules have line ranges, remove the result + if (rulesWithLineRanges.isEmpty()) { + return true; + } + + // If we have line ranges, check for overlaps + String resultLineRangesString = result.getFileDetails().get(0).getLines(); + List resultLineRanges = LineRangeUtils.parseLineRanges(resultLineRangesString); + + return rulesWithLineRanges.stream() + .map(rule -> new LineRange(rule.getStartLine(), rule.getEndLine())) + .anyMatch(ruleLineRange -> LineRangeUtils.hasOverlappingRanges(resultLineRanges, ruleLineRange)); + }); + return resultsList; } - /** * Finds and returns a list of matching rules for a scan result. * A rule matches if: @@ -188,6 +215,7 @@ private List findMatchingRules(@NotNull ScanFileResult resul }).collect(Collectors.toList()); } + /** * Checks if both path and purl of the rule match the result */ diff --git a/src/main/java/com/scanoss/dto/ScanFileDetails.java b/src/main/java/com/scanoss/dto/ScanFileDetails.java index a62e6b7..24ecc12 100644 --- a/src/main/java/com/scanoss/dto/ScanFileDetails.java +++ b/src/main/java/com/scanoss/dto/ScanFileDetails.java @@ -23,6 +23,7 @@ package com.scanoss.dto; import com.google.gson.annotations.SerializedName; +import com.scanoss.dto.enums.MatchType; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -36,7 +37,8 @@ @NoArgsConstructor @AllArgsConstructor public class ScanFileDetails { - private String id; + @SerializedName("id") + private MatchType matchType; private String component; private String vendor; private String version; diff --git a/src/main/java/com/scanoss/dto/enums/MatchType.java b/src/main/java/com/scanoss/dto/enums/MatchType.java new file mode 100644 index 0000000..fe27f68 --- /dev/null +++ b/src/main/java/com/scanoss/dto/enums/MatchType.java @@ -0,0 +1,7 @@ +package com.scanoss.dto.enums; + +public enum MatchType { + file, + snippet, + none; +} \ No newline at end of file diff --git a/src/main/java/com/scanoss/settings/Bom.java b/src/main/java/com/scanoss/settings/Bom.java index 041225f..74e90ee 100644 --- a/src/main/java/com/scanoss/settings/Bom.java +++ b/src/main/java/com/scanoss/settings/Bom.java @@ -11,7 +11,7 @@ public class Bom { private @Singular("include") List include; private @Singular("ignore") List ignore; - private @Singular("remove") List remove; + private @Singular("remove") List remove; private @Singular("replace") List replace; } diff --git a/src/main/java/com/scanoss/settings/RemoveRule.java b/src/main/java/com/scanoss/settings/RemoveRule.java new file mode 100644 index 0000000..96b1c3f --- /dev/null +++ b/src/main/java/com/scanoss/settings/RemoveRule.java @@ -0,0 +1,16 @@ +package com.scanoss.settings; + +import com.google.gson.annotations.SerializedName; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.SuperBuilder; + +@EqualsAndHashCode(callSuper = true) +@Data +@SuperBuilder +public class RemoveRule extends Rule { + @SerializedName("start_line") + private final Integer startLine; + @SerializedName("end_line") + private final Integer endLine; +} \ No newline at end of file diff --git a/src/main/java/com/scanoss/utils/LineRange.java b/src/main/java/com/scanoss/utils/LineRange.java new file mode 100644 index 0000000..cf20d93 --- /dev/null +++ b/src/main/java/com/scanoss/utils/LineRange.java @@ -0,0 +1,24 @@ +package com.scanoss.utils; + +import lombok.Getter; + +/** + * Represents a line range with start and end lines + */ +@Getter +public class LineRange { + private final int start; + private final int end; + + public LineRange(int start, int end) { + this.start = start; + this.end = end; + } + + /** + * Checks if this interval overlaps with another interval + */ + public boolean overlaps(LineRange other) { + return this.start <= other.end && this.end >= other.start; + } +} \ No newline at end of file diff --git a/src/main/java/com/scanoss/utils/LineRangeUtils.java b/src/main/java/com/scanoss/utils/LineRangeUtils.java new file mode 100644 index 0000000..cb9e38c --- /dev/null +++ b/src/main/java/com/scanoss/utils/LineRangeUtils.java @@ -0,0 +1,67 @@ +package com.scanoss.utils; + + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Utility class for handling line range operations + */ +public class LineRangeUtils { + /** + * Parses a line range string into a list of intervals + * @param lineRanges String in format "1-5,7-10" + * @return List of LineInterval objects + */ + public static List parseLineRanges(String lineRanges) { + if (lineRanges == null || lineRanges.trim().isEmpty()) { + return Collections.emptyList(); + } + + List intervals = new ArrayList<>(); + String[] ranges = lineRanges.split(","); + + for (String range : ranges) { + String[] bounds = range.trim().split("-"); + if (bounds.length == 2) { + try { + int start = Integer.parseInt(bounds[0].trim()); + int end = Integer.parseInt(bounds[1].trim()); + intervals.add(new LineRange(start, end)); + } catch (NumberFormatException e) { + // Skip invalid intervals + continue; + } + } + } + + return intervals; + } + + /** + * Checks if two sets of line ranges overlap + * @param ranges1 First set of line ranges + * @param ranges2 Second set of line ranges + * @return true if any intervals overlap + */ + public static boolean hasOverlappingRanges(List ranges1, List ranges2) { + for (LineRange interval1 : ranges1) { + for (LineRange interval2 : ranges2) { + if (interval1.overlaps(interval2)) { + return true; + } + } + } + return false; + } + + public static boolean hasOverlappingRanges(List ranges, LineRange range) { + for (LineRange interval1 : ranges) { + if (interval1.overlaps(range)) { + return true; + } + } + return false; + } +} \ No newline at end of file diff --git a/src/test/java/com/scanoss/TestConstants.java b/src/test/java/com/scanoss/TestConstants.java index ae87a8e..9aeb8aa 100644 --- a/src/test/java/com/scanoss/TestConstants.java +++ b/src/test/java/com/scanoss/TestConstants.java @@ -165,6 +165,84 @@ public class TestConstants { " ]\n" + " }\n" + " ],\n" + + " \"src/spdx.c\": [\n" + + " {\n" + + " \"component\": \"scanner.c\",\n" + + " \"file\": \"scanner.c-1.3.3/src/spdx.c\",\n" + + " \"file_hash\": \"00693585177fc51a8d16b2b890f39277\",\n" + + " \"file_url\": \"https://api.osskb.org/file_contents/00693585177fc51a8d16b2b890f39277\",\n" + + " \"id\": \"snippet\",\n" + + " \"latest\": \"1.3.4\",\n" + + " \"licenses\": [\n" + + " {\n" + + " \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-1.0-or-later.txt\",\n" + + " \"copyleft\": \"yes\",\n" + + " \"incompatible_with\": \"Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1\",\n" + + " \"name\": \"GPL-1.0-or-later\",\n" + + " \"osadl_updated\": \"2024-11-29T15:09:00+0000\",\n" + + " \"patent_hints\": \"no\",\n" + + " \"source\": \"scancode\",\n" + + " \"url\": \"https://spdx.org/licenses/GPL-1.0-or-later.html\"\n" + + " },\n" + + " {\n" + + " \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-or-later.txt\",\n" + + " \"copyleft\": \"yes\",\n" + + " \"incompatible_with\": \"Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1\",\n" + + " \"name\": \"GPL-2.0-or-later\",\n" + + " \"osadl_updated\": \"2024-11-29T15:09:00+0000\",\n" + + " \"patent_hints\": \"yes\",\n" + + " \"source\": \"scancode\",\n" + + " \"url\": \"https://spdx.org/licenses/GPL-2.0-or-later.html\"\n" + + " },\n" + + " {\n" + + " \"name\": \"CC0-1.0\",\n" + + " \"source\": \"scancode\",\n" + + " \"url\": \"https://spdx.org/licenses/CC0-1.0.html\"\n" + + " },\n" + + " {\n" + + " \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-or-later.txt\",\n" + + " \"copyleft\": \"yes\",\n" + + " \"incompatible_with\": \"Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1\",\n" + + " \"name\": \"GPL-2.0-or-later\",\n" + + " \"osadl_updated\": \"2024-11-29T15:09:00+0000\",\n" + + " \"patent_hints\": \"yes\",\n" + + " \"source\": \"file_spdx_tag\",\n" + + " \"url\": \"https://spdx.org/licenses/GPL-2.0-or-later.html\"\n" + + " },\n" + + " {\n" + + " \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt\",\n" + + " \"copyleft\": \"yes\",\n" + + " \"incompatible_with\": \"Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1\",\n" + + " \"name\": \"GPL-2.0-only\",\n" + + " \"osadl_updated\": \"2024-11-29T15:09:00+0000\",\n" + + " \"patent_hints\": \"yes\",\n" + + " \"source\": \"component_declared\",\n" + + " \"url\": \"https://spdx.org/licenses/GPL-2.0-only.html\"\n" + + " }\n" + + " ],\n" + + " \"lines\": \"11-52,81-123\",\n" + + " \"matched\": \"63%\",\n" + + " \"oss_lines\": \"28-69,28-70\",\n" + + " \"purl\": [\n" + + " \"pkg:github/scanoss/scanner.c\"\n" + + " ],\n" + + " \"release_date\": \"2021-05-26\",\n" + + " \"server\": {\n" + + " \"kb_version\": {\n" + + " \"daily\": \"24.12.03\",\n" + + " \"monthly\": \"24.11\"\n" + + " },\n" + + " \"version\": \"5.4.8\"\n" + + " },\n" + + " \"source_hash\": \"0bcee0405fbf27bc6a9fc6eb8fb58642\",\n" + + " \"status\": \"pending\",\n" + + " \"url\": \"https://github.com/scanoss/scanner.c\",\n" + + " \"url_hash\": \"2d1700ba496453d779d4987255feb5f2\",\n" + + " \"url_stats\": {},\n" + + " \"vendor\": \"scanoss\",\n" + + " \"version\": \"1.3.3\"\n" + + " }\n" + + " ],\n" + " \"scanoss/api/__init__.py\": [\n" + " {\n" + " \"component\": \"scanoss.py\",\n" + @@ -257,32 +335,6 @@ public class TestConstants { "}\n"; - static final String BOM_CONFIGURATION_MOCK = "{\n" + - " \"bom\": {\n" + - " \"include\": [\n" + - " {\n" + - " \"path\": \"src/main.c\",\n" + - " \"purl\": \"pkg:github/scanoss/scanner.c\"\n" + - " }\n" + - " ],\n" + - " \"remove\": [\n" + - " {\n" + - " \"path\": \"src/spdx.h\",\n" + - " \"purl\": \"pkg:github/scanoss/scanoss.py\"\n" + - " },\n" + - " {\n" + - " \"purl\": \"pkg:github/scanoss/ldb\"\n" + - " }\n" + - " ],\n" + - " \"replace\": [\n" + - " {\n" + - " \"path\": \"src/winnowing.c\",\n" + - " \"purl\": \"pkg:github/scanoss/core\",\n" + - " \"replace_with\": \"pkg:github/scanoss/\"\n" + - " }\n" + - " ]\n" + - " }\n" + - "}"; // Custom self-signed certificate static final String customSelfSignedCertificate = diff --git a/src/test/java/com/scanoss/TestLineRangeUtils.java b/src/test/java/com/scanoss/TestLineRangeUtils.java new file mode 100644 index 0000000..e399b28 --- /dev/null +++ b/src/test/java/com/scanoss/TestLineRangeUtils.java @@ -0,0 +1,163 @@ +package com.scanoss; + +import com.scanoss.utils.LineRange; +import com.scanoss.utils.LineRangeUtils; +import lombok.extern.slf4j.Slf4j; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.*; + +@Slf4j +public class TestLineRangeUtils { + + @Before + public void Setup() { + log.info("Starting Line Range Utils test cases..."); + log.debug("Logging debug enabled"); + log.trace("Logging trace enabled"); + } + + @Test + public void testSingleRangeOverlap() { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + LineRange range1 = new LineRange(1, 10); + LineRange range2 = new LineRange(5, 15); + + assertTrue("Overlapping ranges should return true", range1.overlaps(range2)); + assertTrue("Overlap should be commutative", range2.overlaps(range1)); + + log.info("Finished {} -->", methodName); + } + + @Test + public void testNonOverlappingRanges() { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + LineRange range1 = new LineRange(1, 5); + LineRange range2 = new LineRange(6, 10); + + assertFalse("Non-overlapping ranges should return false", range1.overlaps(range2)); + assertFalse("Non-overlap should be commutative", range2.overlaps(range1)); + + log.info("Finished {} -->", methodName); + } + + @Test + public void testAdjacentRanges() { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + LineRange range1 = new LineRange(1, 5); + LineRange range2 = new LineRange(5, 10); + + assertTrue("Adjacent ranges should be considered overlapping", range1.overlaps(range2)); + assertTrue("Adjacent overlap should be commutative", range2.overlaps(range1)); + + log.info("Finished {} -->", methodName); + } + + @Test + public void testParseValidLineRanges() { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + String rangesStr = "11-52,81-123"; + List ranges = LineRangeUtils.parseLineRanges(rangesStr); + + assertEquals("Should parse two ranges", 2, ranges.size()); + assertEquals("First range should start at 11", 11, ranges.get(0).getStart()); + assertEquals("First range should end at 52", 52, ranges.get(0).getEnd()); + assertEquals("Second range should start at 81", 81, ranges.get(1).getStart()); + assertEquals("Second range should end at 123", 123, ranges.get(1).getEnd()); + + log.info("Finished {} -->", methodName); + } + + @Test + public void testParseEmptyInput() { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + List ranges = LineRangeUtils.parseLineRanges(""); + assertTrue("Empty input should return empty list", ranges.isEmpty()); + + ranges = LineRangeUtils.parseLineRanges(null); + assertTrue("Null input should return empty list", ranges.isEmpty()); + + log.info("Finished {} -->", methodName); + } + + @Test + public void testParseInvalidFormat() { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + List ranges = LineRangeUtils.parseLineRanges("11-52-81"); + assertTrue("Invalid format should be skipped", ranges.isEmpty()); + + ranges = LineRangeUtils.parseLineRanges("abc-def"); + assertTrue("Non-numeric ranges should be skipped", ranges.isEmpty()); + + ranges = LineRangeUtils.parseLineRanges("11-52,invalid,81-123"); + assertEquals("Should parse valid ranges and skip invalid ones", 2, ranges.size()); + + log.info("Finished {} -->", methodName); + } + + @Test + public void testHasOverlappingRanges() { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + String ranges1Str = "1-10,20-30"; + String ranges2Str = "5-15,25-35"; + + List ranges1 = LineRangeUtils.parseLineRanges(ranges1Str); + List ranges2 = LineRangeUtils.parseLineRanges(ranges2Str); + + assertTrue("Should detect overlapping ranges", + LineRangeUtils.hasOverlappingRanges(ranges1, ranges2)); + + log.info("Finished {} -->", methodName); + } + + @Test + public void testNoOverlappingRanges() { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + String ranges1Str = "1-10,20-30"; + String ranges2Str = "40-50,60-70"; + + List ranges1 = LineRangeUtils.parseLineRanges(ranges1Str); + List ranges2 = LineRangeUtils.parseLineRanges(ranges2Str); + + assertFalse("Should not detect overlapping ranges", + LineRangeUtils.hasOverlappingRanges(ranges1, ranges2)); + + log.info("Finished {} -->", methodName); + } + + @Test + public void testSingleLineRanges() { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + String rangesStr = "5-5,10-10"; + List ranges = LineRangeUtils.parseLineRanges(rangesStr); + + assertEquals("Should parse two single-line ranges", 2, ranges.size()); + assertEquals("First range should be single line", + ranges.get(0).getStart(), ranges.get(0).getEnd()); + assertEquals("Second range should be single line", + ranges.get(1).getStart(), ranges.get(1).getEnd()); + + log.info("Finished {} -->", methodName); + } +} \ No newline at end of file diff --git a/src/test/java/com/scanoss/TestScannerPostProcessor.java b/src/test/java/com/scanoss/TestScannerPostProcessor.java index 0cdb721..35ac463 100644 --- a/src/test/java/com/scanoss/TestScannerPostProcessor.java +++ b/src/test/java/com/scanoss/TestScannerPostProcessor.java @@ -23,8 +23,8 @@ package com.scanoss; import com.google.gson.JsonObject; import com.scanoss.settings.Bom; +import com.scanoss.settings.RemoveRule; import com.scanoss.settings.ReplaceRule; -import com.scanoss.settings.Rule; import com.scanoss.utils.JsonUtils; import lombok.extern.slf4j.Slf4j; import org.junit.Before; @@ -54,7 +54,7 @@ public void Setup() throws URISyntaxException, IOException { log.info("Starting ScannerPostProcessor test cases..."); scannerPostProcessor = new ScannerPostProcessor(); JsonObject jsonObject = JsonUtils.toJsonObject(jsonResultsString); - sampleScanResults = JsonUtils.toScanFileResultsFromObject(jsonObject); + sampleScanResults = JsonUtils.toScanFileResultsFromObject(jsonObject); //TODO: Create sampleScanResults with a helper function var resource = getClass().getClassLoader().getResource("results.json"); @@ -71,16 +71,16 @@ public void Setup() throws URISyntaxException, IOException { } - - - /** TESTING REMOVE RULES**/ + /** + * TESTING REMOVE RULES + **/ @Test public void TestRemoveRuleWithPathAndPurl() { String methodName = new Object() { }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); - Rule rule = Rule.builder() + RemoveRule rule = RemoveRule.builder() .purl("pkg:github/twbs/bootstrap") .path("CMSsite/admin/js/npm.js") .build(); @@ -91,7 +91,7 @@ public void TestRemoveRuleWithPathAndPurl() { List results = scannerPostProcessor.process(sampleScanResults, bom); // Verify - assertEquals("Should have one result less after removal", sampleScanResults.size()-1, results.size()); + assertEquals("Should have one result less after removal", sampleScanResults.size() - 1, results.size()); log.info("Finished {} -->", methodName); } @@ -102,9 +102,8 @@ public void TestRemoveRuleWithPurlOnly() { log.info("<-- Starting {}", methodName); // Setup remove rule with only purl - Rule removeRule = Rule.builder() + RemoveRule removeRule = RemoveRule.builder() .purl("pkg:npm/mip-bootstrap") - .path("CMSsite/admin/js/npm.js") .build(); Bom bom = Bom.builder(). @@ -116,7 +115,7 @@ public void TestRemoveRuleWithPurlOnly() { // Verify assertEquals("Size should decrease by 1 after removal", - sampleScanResults.size()-1, + sampleScanResults.size() - 1, results.size()); assertFalse("Should remove file CMSsite/admin/js/npm.js", @@ -133,7 +132,7 @@ public void TestNoMatchingRemoveRules() { // Setup non-matching remove rule - Rule removeRule = Rule.builder() + RemoveRule removeRule = RemoveRule.builder() .purl("pkg:github/non-existing/lib@1.0.0") .path("non/existing/path.c") .build(); @@ -159,26 +158,27 @@ public void TestMultipleRemoveRules() { log.info("<-- Starting {}", methodName); // Setup multiple remove rules - - Rule removeRule1 = Rule.builder() - .purl("pkg:npm/myoneui") - .path("CMSsite/admin/js/npm.js") - .build(); - - Rule removeRule2 = Rule.builder() - .purl("pkg:pypi/scanoss") - .build(); - - Rule removeRule3 = Rule.builder() - .path("scanoss/__init__.py") - .build(); - - Bom bom = Bom.builder(). - remove(Arrays.asList(removeRule1, removeRule2, removeRule3)) + remove(Arrays.asList( + RemoveRule.builder() + .purl("pkg:npm/myoneui") + .path("CMSsite/admin/js/npm.js") + .build(), + + RemoveRule.builder() + .purl("pkg:pypi/scanoss") + .build(), + + RemoveRule.builder() + .path("scanoss/__init__.py") + .build(), + + RemoveRule.builder() + .path("src/spdx.c") + .build() + )) .build(); - // Process results List results = scannerPostProcessor.process(sampleScanResults, bom); @@ -207,13 +207,97 @@ public void TestEmptyRemoveRules() { log.info("Finished {} -->", methodName); } + @Test + public void testRemoveRuleWithNonOverlappingLineRanges() { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + // Setup remove rule with non-overlapping line ranges + Bom bom = Bom.builder() + .remove(Collections.singletonList( + RemoveRule.builder() + .path("src/spdx.c") + .startLine(1) + .endLine(10) // Before the first range + .build() + )) + .build(); + + + // Process results + List results = scannerPostProcessor.process(sampleScanResults, bom); + + // Verify - should keep because lines don't overlap + assertEquals("Results should match original", sampleScanResults, results); + + log.info("Finished {} -->", methodName); + } - /** TESTING REPLACE RULES**/ @Test - public void TestReplaceRuleWithEmptyPurl() { + public void testRemoveRuleWithOverlappingLineRanges() { String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); + // Setup remove rule with overlapping line ranges + Bom bom = Bom.builder() + .remove(Collections.singletonList( + RemoveRule.builder() + .path("src/spdx.c") + .startLine(40) + .endLine(60) // Overlaps with 11-52 + .build() + )) + .build(); + + // Process results + List results = scannerPostProcessor.process(sampleScanResults, bom); + + // Verify - should remove because lines overlap + assertEquals("Should have one result less after removal", sampleScanResults.size() - 1, results.size()); + + log.info("Finished {} -->", methodName); + } + + @Test + public void testMultipleRemoveRulesWithMixedLineRanges() { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + // Setup multiple remove rules with different line range configurations + Bom bom = Bom.builder() + .remove(Arrays.asList( + RemoveRule.builder() + .path("src/spdx.c") + .startLine(1) + .endLine(10) // Non-overlapping + .build(), + RemoveRule.builder() + .path("src/spdx.c") + .startLine(40) + .endLine(60) // Overlapping + .build() + )) + .build(); + + + // Process results + List results = scannerPostProcessor.process(sampleScanResults, bom); + + assertEquals("Should have one result less after removal", sampleScanResults.size() - 1, results.size()); + + log.info("Finished {} -->", methodName); + } + + + /** + * TESTING REPLACE RULES + **/ + @Test + public void TestReplaceRuleWithEmptyPurl() { + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + // Setup replace rule with empty PURL ReplaceRule replace = ReplaceRule.builder() .purl("pkg:github/scanoss/scanoss.py") @@ -252,7 +336,8 @@ public void TestReplaceRuleWithEmptyPurl() { @Test() public void TestReplaceRuleWithPurl() { - String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); @@ -286,10 +371,10 @@ public void TestReplaceRuleWithPurl() { } - @Test() public void TestOriginalPurlExistsWhenNoReplacementRule() { - String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); String originalPurl = "pkg:github/scanoss/scanner.c"; @@ -329,7 +414,8 @@ public void TestOriginalPurlExistsWhenNoReplacementRule() { @Test public void TestOriginalPurlNotExistsWhenReplacementRuleDefined() { - String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); String originalPurl = "pkg:github/scanoss/scanner.c"; From 8754c75248373bd4efae5f29d156e3ed2201ea96 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Tue, 3 Dec 2024 15:22:23 +0100 Subject: [PATCH 09/28] chore: remove unused imports & adds license headers --- .../com/scanoss/ScannerPostProcessor.java | 5 +-- .../java/com/scanoss/cli/ScanCommandLine.java | 1 + .../java/com/scanoss/dto/enums/MatchType.java | 24 ++++++++++++- .../ScannerPostProcessorException.java | 2 +- src/main/java/com/scanoss/settings/Bom.java | 22 ++++++++++++ .../java/com/scanoss/settings/RemoveRule.java | 30 +++++++++++++--- .../com/scanoss/settings/ReplaceRule.java | 29 +++++++++++++--- src/main/java/com/scanoss/settings/Rule.java | 24 ++++++++++++- .../java/com/scanoss/settings/Settings.java | 23 ++++++++++++- .../java/com/scanoss/utils/LineRange.java | 22 ++++++++++++ .../com/scanoss/utils/LineRangeUtils.java | 30 ++++++++++++++-- .../java/com/scanoss/TestLineRangeUtils.java | 27 ++++++++++----- .../com/scanoss/TestScannerPostProcessor.java | 34 +++++++++++-------- src/test/java/com/scanoss/TestSettings.java | 12 +++---- 14 files changed, 238 insertions(+), 47 deletions(-) diff --git a/src/main/java/com/scanoss/ScannerPostProcessor.java b/src/main/java/com/scanoss/ScannerPostProcessor.java index f1b8502..b15b6c8 100644 --- a/src/main/java/com/scanoss/ScannerPostProcessor.java +++ b/src/main/java/com/scanoss/ScannerPostProcessor.java @@ -143,10 +143,10 @@ private List applyReplaceRules(@NotNull List res /** * Applies remove rules to scan results, filtering out matches based on certain criteria. - * + *

* First, matches are found based on path and/or purl: * - Rule must match either both path and purl, just the path, or just the purl - * + *

* Then, for each matched result: * 1. If none of the matching rules define line ranges -> Remove the result * 2. If any matching rules define line ranges -> Only remove if the result's lines overlap with any rule's line range @@ -185,6 +185,7 @@ private List applyRemoveRules(@NotNull List resu return resultsList; } + /** * Finds and returns a list of matching rules for a scan result. * A rule matches if: diff --git a/src/main/java/com/scanoss/cli/ScanCommandLine.java b/src/main/java/com/scanoss/cli/ScanCommandLine.java index ae92855..4cc5ed1 100644 --- a/src/main/java/com/scanoss/cli/ScanCommandLine.java +++ b/src/main/java/com/scanoss/cli/ScanCommandLine.java @@ -205,6 +205,7 @@ public void run() { /** * Load the specified file into a string + * * @param filename filename to load * @return loaded string */ diff --git a/src/main/java/com/scanoss/dto/enums/MatchType.java b/src/main/java/com/scanoss/dto/enums/MatchType.java index fe27f68..8ed000f 100644 --- a/src/main/java/com/scanoss/dto/enums/MatchType.java +++ b/src/main/java/com/scanoss/dto/enums/MatchType.java @@ -1,7 +1,29 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2024, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package com.scanoss.dto.enums; public enum MatchType { file, snippet, - none; + none } \ No newline at end of file diff --git a/src/main/java/com/scanoss/exceptions/ScannerPostProcessorException.java b/src/main/java/com/scanoss/exceptions/ScannerPostProcessorException.java index 42ae9e5..14a15d8 100644 --- a/src/main/java/com/scanoss/exceptions/ScannerPostProcessorException.java +++ b/src/main/java/com/scanoss/exceptions/ScannerPostProcessorException.java @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT /* - * Copyright (c) 2023, SCANOSS + * Copyright (c) 2024, SCANOSS * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/src/main/java/com/scanoss/settings/Bom.java b/src/main/java/com/scanoss/settings/Bom.java index 74e90ee..099657f 100644 --- a/src/main/java/com/scanoss/settings/Bom.java +++ b/src/main/java/com/scanoss/settings/Bom.java @@ -1,3 +1,25 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2024, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package com.scanoss.settings; import lombok.Builder; diff --git a/src/main/java/com/scanoss/settings/RemoveRule.java b/src/main/java/com/scanoss/settings/RemoveRule.java index 96b1c3f..12010d0 100644 --- a/src/main/java/com/scanoss/settings/RemoveRule.java +++ b/src/main/java/com/scanoss/settings/RemoveRule.java @@ -1,3 +1,25 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2024, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package com.scanoss.settings; import com.google.gson.annotations.SerializedName; @@ -9,8 +31,8 @@ @Data @SuperBuilder public class RemoveRule extends Rule { - @SerializedName("start_line") - private final Integer startLine; - @SerializedName("end_line") - private final Integer endLine; + @SerializedName("start_line") + private final Integer startLine; + @SerializedName("end_line") + private final Integer endLine; } \ No newline at end of file diff --git a/src/main/java/com/scanoss/settings/ReplaceRule.java b/src/main/java/com/scanoss/settings/ReplaceRule.java index e0ebb45..874c6cb 100644 --- a/src/main/java/com/scanoss/settings/ReplaceRule.java +++ b/src/main/java/com/scanoss/settings/ReplaceRule.java @@ -1,6 +1,27 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2024, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package com.scanoss.settings; -import com.github.packageurl.PackageURL; import com.google.gson.annotations.SerializedName; import lombok.Data; import lombok.EqualsAndHashCode; @@ -10,7 +31,7 @@ @Data @SuperBuilder public class ReplaceRule extends Rule { - @SerializedName("replace_with") - private String replaceWith; - private String license; + @SerializedName("replace_with") + private String replaceWith; + private String license; } \ No newline at end of file diff --git a/src/main/java/com/scanoss/settings/Rule.java b/src/main/java/com/scanoss/settings/Rule.java index 129da9b..cc6836a 100644 --- a/src/main/java/com/scanoss/settings/Rule.java +++ b/src/main/java/com/scanoss/settings/Rule.java @@ -1,5 +1,27 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2024, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package com.scanoss.settings; -import com.github.packageurl.PackageURL; + import lombok.Data; import lombok.experimental.SuperBuilder; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/com/scanoss/settings/Settings.java b/src/main/java/com/scanoss/settings/Settings.java index 3da603d..0916611 100644 --- a/src/main/java/com/scanoss/settings/Settings.java +++ b/src/main/java/com/scanoss/settings/Settings.java @@ -1,3 +1,25 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2024, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package com.scanoss.settings; import com.google.gson.Gson; @@ -16,7 +38,6 @@ public class Settings { private Bom bom; - /** * Creates a Settings object from a JSON string * diff --git a/src/main/java/com/scanoss/utils/LineRange.java b/src/main/java/com/scanoss/utils/LineRange.java index cf20d93..0dc9fde 100644 --- a/src/main/java/com/scanoss/utils/LineRange.java +++ b/src/main/java/com/scanoss/utils/LineRange.java @@ -1,3 +1,25 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2024, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package com.scanoss.utils; import lombok.Getter; diff --git a/src/main/java/com/scanoss/utils/LineRangeUtils.java b/src/main/java/com/scanoss/utils/LineRangeUtils.java index cb9e38c..d2e16c2 100644 --- a/src/main/java/com/scanoss/utils/LineRangeUtils.java +++ b/src/main/java/com/scanoss/utils/LineRangeUtils.java @@ -1,3 +1,25 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2024, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package com.scanoss.utils; @@ -11,6 +33,7 @@ public class LineRangeUtils { /** * Parses a line range string into a list of intervals + * * @param lineRanges String in format "1-5,7-10" * @return List of LineInterval objects */ @@ -41,6 +64,7 @@ public static List parseLineRanges(String lineRanges) { /** * Checks if two sets of line ranges overlap + * * @param ranges1 First set of line ranges * @param ranges2 Second set of line ranges * @return true if any intervals overlap @@ -58,9 +82,9 @@ public static boolean hasOverlappingRanges(List ranges1, List ranges, LineRange range) { for (LineRange interval1 : ranges) { - if (interval1.overlaps(range)) { - return true; - } + if (interval1.overlaps(range)) { + return true; + } } return false; } diff --git a/src/test/java/com/scanoss/TestLineRangeUtils.java b/src/test/java/com/scanoss/TestLineRangeUtils.java index e399b28..f863339 100644 --- a/src/test/java/com/scanoss/TestLineRangeUtils.java +++ b/src/test/java/com/scanoss/TestLineRangeUtils.java @@ -22,7 +22,8 @@ public void Setup() { @Test public void testSingleRangeOverlap() { - String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); LineRange range1 = new LineRange(1, 10); @@ -36,7 +37,8 @@ public void testSingleRangeOverlap() { @Test public void testNonOverlappingRanges() { - String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); LineRange range1 = new LineRange(1, 5); @@ -50,7 +52,8 @@ public void testNonOverlappingRanges() { @Test public void testAdjacentRanges() { - String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); LineRange range1 = new LineRange(1, 5); @@ -64,7 +67,8 @@ public void testAdjacentRanges() { @Test public void testParseValidLineRanges() { - String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); String rangesStr = "11-52,81-123"; @@ -81,7 +85,8 @@ public void testParseValidLineRanges() { @Test public void testParseEmptyInput() { - String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); List ranges = LineRangeUtils.parseLineRanges(""); @@ -95,7 +100,8 @@ public void testParseEmptyInput() { @Test public void testParseInvalidFormat() { - String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); List ranges = LineRangeUtils.parseLineRanges("11-52-81"); @@ -112,7 +118,8 @@ public void testParseInvalidFormat() { @Test public void testHasOverlappingRanges() { - String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); String ranges1Str = "1-10,20-30"; @@ -129,7 +136,8 @@ public void testHasOverlappingRanges() { @Test public void testNoOverlappingRanges() { - String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); String ranges1Str = "1-10,20-30"; @@ -146,7 +154,8 @@ public void testNoOverlappingRanges() { @Test public void testSingleLineRanges() { - String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); String rangesStr = "5-5,10-10"; diff --git a/src/test/java/com/scanoss/TestScannerPostProcessor.java b/src/test/java/com/scanoss/TestScannerPostProcessor.java index 35ac463..1c3d891 100644 --- a/src/test/java/com/scanoss/TestScannerPostProcessor.java +++ b/src/test/java/com/scanoss/TestScannerPostProcessor.java @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT /* - * Copyright (c) 2023, SCANOSS + * Copyright (c) 2024, SCANOSS * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -21,7 +21,9 @@ * THE SOFTWARE. */ package com.scanoss; + import com.google.gson.JsonObject; +import com.scanoss.dto.ScanFileResult; import com.scanoss.settings.Bom; import com.scanoss.settings.RemoveRule; import com.scanoss.settings.ReplaceRule; @@ -29,18 +31,19 @@ import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; -import static com.scanoss.TestConstants.jsonResultsString; -import com.scanoss.dto.ScanFileResult; - import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; +import static com.scanoss.TestConstants.jsonResultsString; import static org.junit.Assert.*; @Slf4j @@ -107,7 +110,7 @@ public void TestRemoveRuleWithPurlOnly() { .build(); Bom bom = Bom.builder(). - remove(Arrays.asList(removeRule)) + remove(Collections.singletonList(removeRule)) .build(); // Process results @@ -138,7 +141,7 @@ public void TestNoMatchingRemoveRules() { .build(); Bom bom = Bom.builder(). - remove(Arrays.asList(removeRule)) + remove(Collections.singletonList(removeRule)) .build(); @@ -209,7 +212,8 @@ public void TestEmptyRemoveRules() { @Test public void testRemoveRuleWithNonOverlappingLineRanges() { - String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); // Setup remove rule with non-overlapping line ranges @@ -235,7 +239,8 @@ public void testRemoveRuleWithNonOverlappingLineRanges() { @Test public void testRemoveRuleWithOverlappingLineRanges() { - String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); // Setup remove rule with overlapping line ranges @@ -260,7 +265,8 @@ public void testRemoveRuleWithOverlappingLineRanges() { @Test public void testMultipleRemoveRulesWithMixedLineRanges() { - String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); // Setup multiple remove rules with different line range configurations @@ -305,7 +311,7 @@ public void TestReplaceRuleWithEmptyPurl() { .build(); Bom bom = Bom.builder() - .replace(Arrays.asList(replace)) + .replace(Collections.singletonList(replace)) .build(); @@ -348,7 +354,7 @@ public void TestReplaceRuleWithPurl() { .build(); Bom bom = Bom.builder() - .replace(Arrays.asList(replace)) + .replace(Collections.singletonList(replace)) .build(); @@ -428,7 +434,7 @@ public void TestOriginalPurlNotExistsWhenReplacementRuleDefined() { .build(); Bom bom = Bom.builder() - .replace(Arrays.asList(replace)) + .replace(Collections.singletonList(replace)) .build(); List results = scannerPostProcessor.process(longScanResults, bom); @@ -454,6 +460,4 @@ public void TestOriginalPurlNotExistsWhenReplacementRuleDefined() { } - - } \ No newline at end of file diff --git a/src/test/java/com/scanoss/TestSettings.java b/src/test/java/com/scanoss/TestSettings.java index 6362fb6..0fcfdfd 100644 --- a/src/test/java/com/scanoss/TestSettings.java +++ b/src/test/java/com/scanoss/TestSettings.java @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT /* - * Copyright (c) 2023, SCANOSS + * Copyright (c) 2024, SCANOSS * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -23,18 +23,15 @@ package com.scanoss; import com.scanoss.settings.Settings; -import com.scanoss.utils.ProxyUtils; import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; import java.io.IOException; -import java.net.Proxy; import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Objects; import static org.junit.Assert.*; @@ -42,6 +39,7 @@ public class TestSettings { private Path existingSettingsPath; private Path nonExistentSettingsPath; + @Before public void Setup() throws URISyntaxException { log.info("Starting Settings test cases..."); @@ -71,7 +69,8 @@ public void Setup() throws URISyntaxException { @Test public void testSettingsFromExistingFile() { - String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); try { @@ -90,7 +89,8 @@ public void testSettingsFromExistingFile() { @Test(expected = IOException.class) public void testSettingsFromNonExistentFile() throws IOException { - String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String methodName = new Object() { + }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); Settings.fromPath(nonExistentSettingsPath); From 9ca0c0709b4f088d39622113c32c2166fc975d3f Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 9 Dec 2024 12:53:14 +0100 Subject: [PATCH 10/28] chore: adds license headers, use final on dtos, add log info ScannerPostProcessor, implement start_line and end_line on RemoveRule --- .../com/scanoss/ScannerPostProcessor.java | 65 ++++++++++++++----- .../java/com/scanoss/cli/ScanCommandLine.java | 6 +- .../java/com/scanoss/dto/ScanFileDetails.java | 47 +++++++------- .../ScannerPostProcessorException.java | 51 --------------- src/main/java/com/scanoss/settings/Bom.java | 8 +-- .../com/scanoss/settings/ReplaceRule.java | 4 +- src/main/java/com/scanoss/settings/Rule.java | 4 +- .../java/com/scanoss/settings/Settings.java | 8 +-- .../com/scanoss/TestScannerPostProcessor.java | 2 +- 9 files changed, 83 insertions(+), 112 deletions(-) delete mode 100644 src/main/java/com/scanoss/exceptions/ScannerPostProcessorException.java diff --git a/src/main/java/com/scanoss/ScannerPostProcessor.java b/src/main/java/com/scanoss/ScannerPostProcessor.java index b15b6c8..e590963 100644 --- a/src/main/java/com/scanoss/ScannerPostProcessor.java +++ b/src/main/java/com/scanoss/ScannerPostProcessor.java @@ -9,6 +9,7 @@ import com.scanoss.settings.Rule; import com.scanoss.utils.LineRange; import com.scanoss.utils.LineRangeUtils; +import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -16,9 +17,10 @@ import java.util.stream.Collectors; @Slf4j +@Builder public class ScannerPostProcessor { - private Map componentIndex = new HashMap<>(); + private Map componentIndex; /** * Processes scan results according to BOM configuration rules. @@ -29,18 +31,27 @@ public class ScannerPostProcessor { * @return List of processed scan results */ public List process(@NotNull List scanFileResults, @NotNull Bom bom) { + log.info("Starting scan results processing with {} results", scanFileResults.size()); + log.debug("BOM configuration - Remove rules: {}, Replace rules: {}", + bom.getRemove() != null ? bom.getRemove().size() : 0, + bom.getReplace() != null ? bom.getReplace().size() : 0); + createComponentIndex(scanFileResults); List processedResults = new ArrayList<>(scanFileResults); if (bom.getRemove() != null && !bom.getRemove().isEmpty()) { + log.info("Applying {} remove rules to scan results", bom.getRemove().size()); processedResults = applyRemoveRules(processedResults, bom.getRemove()); } if (bom.getReplace() != null && !bom.getReplace().isEmpty()) { + log.info("Applying {} replace rules to scan results", bom.getReplace().size()); processedResults = applyReplaceRules(processedResults, bom.getReplace()); } + log.info("Scan results processing completed. Original results: {}, Processed results: {}", + scanFileResults.size(), processedResults.size()); return processedResults; } @@ -51,7 +62,10 @@ public List process(@NotNull List scanFileResult * @return Map where keys are PURLs and values are corresponding ScanFileDetails */ private void createComponentIndex(List scanFileResults) { + log.debug("Creating component index from scan results"); + if (scanFileResults == null) { + log.warn("Received null scan results, creating empty component index"); this.componentIndex = new HashMap<>(); return; } @@ -69,6 +83,7 @@ private void createComponentIndex(List scanFileResults) { (existing, replacement) -> existing, HashMap::new )); + log.debug("Component index created with {} entries", componentIndex.size()); } @@ -81,18 +96,18 @@ private void createComponentIndex(List scanFileResults) { * @return A new list containing the processed scan results with updated PURLs */ private List applyReplaceRules(@NotNull List results, @NotNull List rules) { + log.debug("Starting replace rules application"); List resultsList = new ArrayList<>(results); for (ScanFileResult result : resultsList) { - for (ReplaceRule rule : this.findMatchingRules(result, rules)) { - + log.debug("Applying replace rule: {} to file: {}", rule, result.getFilePath()); PackageURL newPurl; try { newPurl = new PackageURL(rule.getReplaceWith()); } catch (MalformedPackageURLException e) { - log.error("ERROR: Parsing purl from rule: {} - {}", rule, e.getMessage()); + log.warn("Failed to parse PURL from replace rule: {}. Skipping", rule); continue; } @@ -102,8 +117,13 @@ private List applyReplaceRules(@NotNull List res ScanFileDetails currentFileDetails = result.getFileDetails().get(0); ScanFileDetails newFileDetails; - if (cachedFileDetails != null) { + log.trace("Processing replacement - Cached details found: {}, Current PURL: {}, New PURL: {}", + cachedFileDetails != null, + currentFileDetails.getPurls()[0], + newPurl); + if (cachedFileDetails != null) { + log.debug("Using cached component details for PURL: {}", newPurl); newFileDetails = ScanFileDetails.builder() .matchType(currentFileDetails.getMatchType()) .file(currentFileDetails.getFile()) @@ -121,17 +141,16 @@ private List applyReplaceRules(@NotNull List res result.getFileDetails().set(0, cachedFileDetails); } else { - - newFileDetails = currentFileDetails; - - newFileDetails.setCopyrightDetails(new CopyrightDetails[]{}); - newFileDetails.setLicenseDetails(new LicenseDetails[]{}); - newFileDetails.setVulnerabilityDetails(new VulnerabilityDetails[]{}); - newFileDetails.setPurls(new String[]{newPurl.toString()}); - newFileDetails.setUrl(""); - - newFileDetails.setComponent(newPurl.getName()); - newFileDetails.setVendor(newPurl.getNamespace()); + log.debug("Creating new component details for PURL: {}", newPurl); + newFileDetails = currentFileDetails.toBuilder() + .copyrightDetails(new CopyrightDetails[]{}) + .licenseDetails(new LicenseDetails[]{}) + .vulnerabilityDetails(new VulnerabilityDetails[]{}) + .purls(new String[]{newPurl.toString()}) + .url("") + .component(newPurl.getName()) + .vendor(newPurl.getNamespace()) + .build(); } result.getFileDetails().set(0, newFileDetails); @@ -156,10 +175,12 @@ private List applyReplaceRules(@NotNull List res * @return A filtered list with matching results removed based on the above criteria */ private List applyRemoveRules(@NotNull List results, @NotNull List rules) { + log.debug("Starting remove rules application to {} results", results.size()); List resultsList = new ArrayList<>(results); resultsList.removeIf(result -> { List matchingRules = findMatchingRules(result, rules); + log.trace("Found {} matching remove rules for file: {}", matchingRules.size(), result.getFilePath()); if (matchingRules.isEmpty()) { return false; } @@ -171,6 +192,7 @@ private List applyRemoveRules(@NotNull List resu // If no rules have line ranges, remove the result if (rulesWithLineRanges.isEmpty()) { + log.debug("Removing entire file - no line ranges specified in rules for file {}", result.getFilePath()); return true; } @@ -178,11 +200,18 @@ private List applyRemoveRules(@NotNull List resu String resultLineRangesString = result.getFileDetails().get(0).getLines(); List resultLineRanges = LineRangeUtils.parseLineRanges(resultLineRangesString); - return rulesWithLineRanges.stream() + boolean shouldRemove = rulesWithLineRanges.stream() .map(rule -> new LineRange(rule.getStartLine(), rule.getEndLine())) .anyMatch(ruleLineRange -> LineRangeUtils.hasOverlappingRanges(resultLineRanges, ruleLineRange)); + + if (shouldRemove) { + log.debug("Removing file {} due to overlapping line ranges", result.getFilePath()); + } + + return shouldRemove; }); + log.debug("Remove rules application completed. Results remaining: {}", resultsList.size()); return resultsList; } @@ -199,6 +228,7 @@ private List applyRemoveRules(@NotNull List resu * @return List of matching rules, empty list if no matches found */ private List findMatchingRules(@NotNull ScanFileResult result, @NotNull List rules) { + log.trace("Finding matching rules for file: {}", result.getFilePath()); return rules.stream() .filter(rule -> { boolean hasPath = rule.getPath() != null && !rule.getPath().isEmpty(); @@ -212,6 +242,7 @@ private List findMatchingRules(@NotNull ScanFileResult resul return isPurlOnlyMatch(rule, result); } + log.warn("Rule {} has neither path nor PURL specified", rule); return false; // Neither path nor purl specified }).collect(Collectors.toList()); } diff --git a/src/main/java/com/scanoss/cli/ScanCommandLine.java b/src/main/java/com/scanoss/cli/ScanCommandLine.java index 4cc5ed1..aedf9ea 100644 --- a/src/main/java/com/scanoss/cli/ScanCommandLine.java +++ b/src/main/java/com/scanoss/cli/ScanCommandLine.java @@ -190,13 +190,9 @@ public void run() { } if (settings != null && !settings.isEmpty()) { - try { Path path = Paths.get(settings); - ScannerPostProcessor scannerPostProcessor = new ScannerPostProcessor(); + ScannerPostProcessor scannerPostProcessor = ScannerPostProcessor.builder().build(); scanFileResults = scannerPostProcessor.process(scanFileResults, Settings.fromPath(path).getBom()); - } catch (Exception e) { - throw new RuntimeException(e); - } } var out = spec.commandLine().getOut(); diff --git a/src/main/java/com/scanoss/dto/ScanFileDetails.java b/src/main/java/com/scanoss/dto/ScanFileDetails.java index 24ecc12..9cbfb53 100644 --- a/src/main/java/com/scanoss/dto/ScanFileDetails.java +++ b/src/main/java/com/scanoss/dto/ScanFileDetails.java @@ -33,43 +33,42 @@ * Scan File Result Detailed Information */ @Data -@Builder -@NoArgsConstructor +@Builder(toBuilder = true) @AllArgsConstructor public class ScanFileDetails { @SerializedName("id") - private MatchType matchType; - private String component; - private String vendor; - private String version; - private String latest; - private String url; - private String status; - private String matched; - private String file; - private String lines; + private final MatchType matchType; + private final String component; + private final String vendor; + private final String version; + private final String latest; + private final String url; + private final String status; + private final String matched; + private final String file; + private final String lines; @SerializedName("oss_lines") - private String ossLines; + private final String ossLines; @SerializedName("file_hash") - private String fileHash; + private final String fileHash; @SerializedName("file_url") - private String fileUrl; + private final String fileUrl; @SerializedName("url_hash") - private String urlHash; + private final String urlHash; @SerializedName("release_date") - private String releaseDate; + private final String releaseDate; @SerializedName("source_hash") - private String sourceHash; + private final String sourceHash; @SerializedName("purl") - private String[] purls; + private final String[] purls; @SerializedName("server") - private ServerDetails serverDetails; + private final ServerDetails serverDetails; @SerializedName("licenses") - private LicenseDetails[] licenseDetails; + private final LicenseDetails[] licenseDetails; @SerializedName("quality") - private QualityDetails[] qualityDetails; + private final QualityDetails[] qualityDetails; @SerializedName("vulnerabilities") - private VulnerabilityDetails[] vulnerabilityDetails; + private final VulnerabilityDetails[] vulnerabilityDetails; @SerializedName("copyrights") - private CopyrightDetails[] copyrightDetails; + private final CopyrightDetails[] copyrightDetails; } diff --git a/src/main/java/com/scanoss/exceptions/ScannerPostProcessorException.java b/src/main/java/com/scanoss/exceptions/ScannerPostProcessorException.java deleted file mode 100644 index 14a15d8..0000000 --- a/src/main/java/com/scanoss/exceptions/ScannerPostProcessorException.java +++ /dev/null @@ -1,51 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright (c) 2024, SCANOSS - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -package com.scanoss.exceptions; - -/** - * SCANOSS Post Processor Exception Class - *

- * This exception will be used by the ScannerPostProcessor class to alert processing issues - *

- */ -public class ScannerPostProcessorException extends RuntimeException { - - /** - * Winnowing Exception - * - * @param errorMessage error message - */ - public ScannerPostProcessorException(String errorMessage) { - super(errorMessage); - } - - /** - * Nested Winnowing Exception - * - * @param errorMessage error message - * @param err nested exception - */ - public ScannerPostProcessorException(String errorMessage, Throwable err) { - super(errorMessage, err); - } -} diff --git a/src/main/java/com/scanoss/settings/Bom.java b/src/main/java/com/scanoss/settings/Bom.java index 099657f..ea07330 100644 --- a/src/main/java/com/scanoss/settings/Bom.java +++ b/src/main/java/com/scanoss/settings/Bom.java @@ -31,10 +31,10 @@ @Data @Builder public class Bom { - private @Singular("include") List include; - private @Singular("ignore") List ignore; - private @Singular("remove") List remove; - private @Singular("replace") List replace; + private final @Singular("include") List include; + private final @Singular("ignore") List ignore; + private final @Singular("remove") List remove; + private final @Singular("replace") List replace; } diff --git a/src/main/java/com/scanoss/settings/ReplaceRule.java b/src/main/java/com/scanoss/settings/ReplaceRule.java index 874c6cb..4526fd8 100644 --- a/src/main/java/com/scanoss/settings/ReplaceRule.java +++ b/src/main/java/com/scanoss/settings/ReplaceRule.java @@ -32,6 +32,6 @@ @SuperBuilder public class ReplaceRule extends Rule { @SerializedName("replace_with") - private String replaceWith; - private String license; + private final String replaceWith; + private final String license; } \ No newline at end of file diff --git a/src/main/java/com/scanoss/settings/Rule.java b/src/main/java/com/scanoss/settings/Rule.java index cc6836a..f34a150 100644 --- a/src/main/java/com/scanoss/settings/Rule.java +++ b/src/main/java/com/scanoss/settings/Rule.java @@ -30,8 +30,8 @@ @Slf4j @SuperBuilder() public class Rule { - private String path; - private String purl; //TODO: Add validation with PackageURL + private final String path; + private final String purl; //TODO: Add validation with PackageURL } diff --git a/src/main/java/com/scanoss/settings/Settings.java b/src/main/java/com/scanoss/settings/Settings.java index 0916611..8f5d0bf 100644 --- a/src/main/java/com/scanoss/settings/Settings.java +++ b/src/main/java/com/scanoss/settings/Settings.java @@ -35,7 +35,7 @@ @Data @Builder public class Settings { - private Bom bom; + private final Bom bom; /** @@ -56,13 +56,9 @@ public static Settings fromJSON(@NotNull String json) { * @return A new Settings object * @throws IOException If there's an error reading the file */ - public static Settings fromPath(@NotNull Path path) throws IOException { - try { + public static Settings fromPath(@NotNull Path path) { String json = Files.readString(path, StandardCharsets.UTF_8); return fromJSON(json); - } catch (IOException e) { - throw new IOException("Failed to read settings file: " + path, e); - } } } diff --git a/src/test/java/com/scanoss/TestScannerPostProcessor.java b/src/test/java/com/scanoss/TestScannerPostProcessor.java index 1c3d891..3c36ba5 100644 --- a/src/test/java/com/scanoss/TestScannerPostProcessor.java +++ b/src/test/java/com/scanoss/TestScannerPostProcessor.java @@ -55,7 +55,7 @@ public class TestScannerPostProcessor { @Before public void Setup() throws URISyntaxException, IOException { log.info("Starting ScannerPostProcessor test cases..."); - scannerPostProcessor = new ScannerPostProcessor(); + scannerPostProcessor = ScannerPostProcessor.builder().build(); JsonObject jsonObject = JsonUtils.toJsonObject(jsonResultsString); sampleScanResults = JsonUtils.toScanFileResultsFromObject(jsonObject); //TODO: Create sampleScanResults with a helper function From 98ef79dbb536fef60cd85929e494676bf2911785 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 9 Dec 2024 18:04:14 +0100 Subject: [PATCH 11/28] chore: update dependencies, update unit tests, handle errors with invalid settings json --- pom.xml | 14 ++++--------- .../java/com/scanoss/cli/ScanCommandLine.java | 17 ++++++++++----- .../java/com/scanoss/settings/Settings.java | 14 ++++++++++--- src/test/java/com/scanoss/TestCli.java | 16 ++++++++++++++ src/test/java/com/scanoss/TestSettings.java | 21 +++++++++---------- 5 files changed, 53 insertions(+), 29 deletions(-) diff --git a/pom.xml b/pom.xml index 3625071..94eda5d 100644 --- a/pom.xml +++ b/pom.xml @@ -77,7 +77,7 @@ org.projectlombok lombok - 1.18.28 + 1.18.36 true @@ -108,17 +108,17 @@ org.apache.tika tika-core - 2.8.0 + 2.9.2 info.picocli picocli - 4.7.4 + 4.7.6 com.google.code.gson gson - 2.10.1 + 2.11.0 compile @@ -126,12 +126,6 @@ packageurl-java 1.5.0 - - org.junit.jupiter - junit-jupiter - RELEASE - test - diff --git a/src/main/java/com/scanoss/cli/ScanCommandLine.java b/src/main/java/com/scanoss/cli/ScanCommandLine.java index aedf9ea..2dd1c41 100644 --- a/src/main/java/com/scanoss/cli/ScanCommandLine.java +++ b/src/main/java/com/scanoss/cli/ScanCommandLine.java @@ -98,7 +98,7 @@ class ScanCommandLine implements Runnable { private String ignoreSbom; @picocli.CommandLine.Option(names = {"--settings"}, description = "Settings file to use for scanning (optional - default scanoss.json)") - private String settings; + private String settingsPath; @picocli.CommandLine.Option(names = {"--snippet-limit"}, description = "Length of single line snippet limit (0 for unlimited, default 1000)") private int snippetLimit = 1000; @@ -119,6 +119,7 @@ class ScanCommandLine implements Runnable { private List scanFileResults; + private Settings settings; /** * Run the 'scan' command */ @@ -151,6 +152,12 @@ public void run() { throw new RuntimeException("Error: Failed to setup proxy config"); } } + + if(settingsPath != null && !settingsPath.isEmpty()) { + settings = Settings.createFromPath(Paths.get(settingsPath)); + if (settings == null) throw new RuntimeException("Error: Failed to read settings file"); + } + if (com.scanoss.cli.CommandLine.debug) { if (numThreads != DEFAULT_WORKER_THREADS) { printMsg(err, String.format("Running with %d threads.", numThreads)); @@ -189,12 +196,12 @@ public void run() { throw new RuntimeException(String.format("Error: Specified path is not a file or a folder: %s\n", fileFolder)); } - if (settings != null && !settings.isEmpty()) { - Path path = Paths.get(settings); - ScannerPostProcessor scannerPostProcessor = ScannerPostProcessor.builder().build(); - scanFileResults = scannerPostProcessor.process(scanFileResults, Settings.fromPath(path).getBom()); + if (settings != null && settings.getBom() != null) { + ScannerPostProcessor scannerPostProcessor = ScannerPostProcessor.builder().build(); + scanFileResults = scannerPostProcessor.process(scanFileResults, settings.getBom()); } + var out = spec.commandLine().getOut(); JsonUtils.writeJsonPretty(toScanFileResultJsonObject(scanFileResults), null); // Uses System.out } diff --git a/src/main/java/com/scanoss/settings/Settings.java b/src/main/java/com/scanoss/settings/Settings.java index 8f5d0bf..7376a0e 100644 --- a/src/main/java/com/scanoss/settings/Settings.java +++ b/src/main/java/com/scanoss/settings/Settings.java @@ -25,6 +25,7 @@ import com.google.gson.Gson; import lombok.Builder; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; import java.io.IOException; @@ -32,6 +33,7 @@ import java.nio.file.Files; import java.nio.file.Path; +@Slf4j @Data @Builder public class Settings { @@ -44,7 +46,7 @@ public class Settings { * @param json The JSON string to parse * @return A new Settings object */ - public static Settings fromJSON(@NotNull String json) { + public static Settings createFromJsonString(@NotNull String json) { Gson gson = new Gson(); return gson.fromJson(json, Settings.class); } @@ -56,9 +58,15 @@ public static Settings fromJSON(@NotNull String json) { * @return A new Settings object * @throws IOException If there's an error reading the file */ - public static Settings fromPath(@NotNull Path path) { + public static Settings createFromPath(@NotNull Path path) { + try { String json = Files.readString(path, StandardCharsets.UTF_8); - return fromJSON(json); + return createFromJsonString(json); + } catch (IOException e) { + log.error("Cannot read settings file - {}", e.getMessage()); + return null; + } + } } diff --git a/src/test/java/com/scanoss/TestCli.java b/src/test/java/com/scanoss/TestCli.java index be975dc..166f01e 100644 --- a/src/test/java/com/scanoss/TestCli.java +++ b/src/test/java/com/scanoss/TestCli.java @@ -138,9 +138,15 @@ public void TestScanCommandPositive() { exitCode = new picocli.CommandLine(new CommandLine()).execute(args2); assertEquals("command should not fail", 0, exitCode); + + String[] args3 = new String[]{"-d", "scan", "src/test/java/com", "--settings", "src/test/resources/scanoss.json" + }; + exitCode = new picocli.CommandLine(new CommandLine()).execute(args3); + assertEquals("command should not fail", 0, exitCode); log.info("Finished {} -->", methodName); } + @Test public void TestScanCommandNegative() { String methodName = new Object() { @@ -168,6 +174,16 @@ public void TestScanCommandNegative() { exitCode = new picocli.CommandLine(new CommandLine()).execute(args5); assertTrue("command should fail", exitCode != 0); + String[] args6 = new String[]{"-d", "scan", "src/test/java/com", "--settings", "does-not-exist.json"}; + exitCode = new picocli.CommandLine(new CommandLine()).execute(args6); + assertTrue("command should fail", exitCode != 0); + + + String[] args7 = new String[]{"-d", "scan", "src/test/java/com", "--settings", "src/test/resources/scanoss-broken.json"}; + exitCode = new picocli.CommandLine(new CommandLine()).execute(args7); + assertTrue("command should fail", exitCode != 0); + + log.info("Finished {} -->", methodName); } diff --git a/src/test/java/com/scanoss/TestSettings.java b/src/test/java/com/scanoss/TestSettings.java index 0fcfdfd..8be98ac 100644 --- a/src/test/java/com/scanoss/TestSettings.java +++ b/src/test/java/com/scanoss/TestSettings.java @@ -73,28 +73,27 @@ public void testSettingsFromExistingFile() { }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); - try { - Settings settings = Settings.fromPath(existingSettingsPath); - assertNotNull("Settings should not be null", settings); - assertEquals("scanner.c", settings.getBom().getRemove().get(0).getPath()); - assertEquals("pkg:github/scanoss/scanner.c", settings.getBom().getRemove().get(0).getPurl()); + Settings settings = Settings.createFromPath(existingSettingsPath); + assertNotNull("Settings should not be null", settings); + + assertEquals("scanner.c", settings.getBom().getRemove().get(0).getPath()); + assertEquals("pkg:github/scanoss/scanner.c", settings.getBom().getRemove().get(0).getPurl()); + - } catch (IOException e) { - fail("Should not throw IOException for existing file: " + e.getMessage()); - } log.info("Finished {} -->", methodName); } - @Test(expected = IOException.class) + @Test public void testSettingsFromNonExistentFile() throws IOException { String methodName = new Object() { }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); - Settings.fromPath(nonExistentSettingsPath); - // Should throw IOException before reaching this point + Settings settings = Settings.createFromPath(nonExistentSettingsPath); + + assertNull("Settings should be null", settings); log.info("Finished {} -->", methodName); } From 2a9211aa407ffe0461bb111022b1245ac0b6c179 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 12 Dec 2024 14:43:16 +0100 Subject: [PATCH 12/28] chore: fix javadoc error --- src/main/java/com/scanoss/settings/Settings.java | 1 - src/main/java/com/scanoss/utils/LineRange.java | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/scanoss/settings/Settings.java b/src/main/java/com/scanoss/settings/Settings.java index 7376a0e..bdd4a6d 100644 --- a/src/main/java/com/scanoss/settings/Settings.java +++ b/src/main/java/com/scanoss/settings/Settings.java @@ -56,7 +56,6 @@ public static Settings createFromJsonString(@NotNull String json) { * * @param path The path to the JSON file * @return A new Settings object - * @throws IOException If there's an error reading the file */ public static Settings createFromPath(@NotNull Path path) { try { diff --git a/src/main/java/com/scanoss/utils/LineRange.java b/src/main/java/com/scanoss/utils/LineRange.java index 0dc9fde..d6c09ac 100644 --- a/src/main/java/com/scanoss/utils/LineRange.java +++ b/src/main/java/com/scanoss/utils/LineRange.java @@ -39,6 +39,8 @@ public LineRange(int start, int end) { /** * Checks if this interval overlaps with another interval + * @param other object to compare with + * @return true if overlaps */ public boolean overlaps(LineRange other) { return this.start <= other.end && this.end >= other.start; From a1d3fc357edb53cd1d9bea8243ca2bee75291810 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 12 Dec 2024 15:49:12 +0100 Subject: [PATCH 13/28] chore: apply PR comments --- .../com/scanoss/ScannerPostProcessor.java | 23 ++++++++++++++++++- .../java/com/scanoss/cli/ScanCommandLine.java | 2 -- .../java/com/scanoss/utils/JsonUtils.java | 7 ++++++ .../java/com/scanoss/utils/LineRange.java | 12 +++++++--- .../com/scanoss/utils/LineRangeUtils.java | 12 ++++++---- 5 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/scanoss/ScannerPostProcessor.java b/src/main/java/com/scanoss/ScannerPostProcessor.java index e590963..1e484e4 100644 --- a/src/main/java/com/scanoss/ScannerPostProcessor.java +++ b/src/main/java/com/scanoss/ScannerPostProcessor.java @@ -1,3 +1,25 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2024, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package com.scanoss; import com.github.packageurl.MalformedPackageURLException; @@ -59,7 +81,6 @@ public List process(@NotNull List scanFileResult * Creates a map of PURL (Package URL) to ScanFileDetails from a list of scan results. * * @param scanFileResults List of scan results to process - * @return Map where keys are PURLs and values are corresponding ScanFileDetails */ private void createComponentIndex(List scanFileResults) { log.debug("Creating component index from scan results"); diff --git a/src/main/java/com/scanoss/cli/ScanCommandLine.java b/src/main/java/com/scanoss/cli/ScanCommandLine.java index 2dd1c41..06a48b8 100644 --- a/src/main/java/com/scanoss/cli/ScanCommandLine.java +++ b/src/main/java/com/scanoss/cli/ScanCommandLine.java @@ -36,7 +36,6 @@ import java.io.IOException; import java.net.Proxy; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; import java.util.List; @@ -201,7 +200,6 @@ public void run() { scanFileResults = scannerPostProcessor.process(scanFileResults, settings.getBom()); } - var out = spec.commandLine().getOut(); JsonUtils.writeJsonPretty(toScanFileResultJsonObject(scanFileResults), null); // Uses System.out } diff --git a/src/main/java/com/scanoss/utils/JsonUtils.java b/src/main/java/com/scanoss/utils/JsonUtils.java index e11ba0b..bc8f0da 100644 --- a/src/main/java/com/scanoss/utils/JsonUtils.java +++ b/src/main/java/com/scanoss/utils/JsonUtils.java @@ -179,6 +179,13 @@ public static List toScanFileResults(@NonNull List resul return scanFileResults; } + /** + * Converts a list of ScanFileResult objects into a JSON object where the file paths are keys + * and the corresponding file details are the values + * + * @param scanFileResults List of ScanFileResult objects to convert + * @return JsonObject containing file paths as keys and file details as JSON elements + */ public static JsonObject toScanFileResultJsonObject(List scanFileResults) { JsonObject root = new JsonObject(); Gson gson = new Gson(); diff --git a/src/main/java/com/scanoss/utils/LineRange.java b/src/main/java/com/scanoss/utils/LineRange.java index d6c09ac..8626fcf 100644 --- a/src/main/java/com/scanoss/utils/LineRange.java +++ b/src/main/java/com/scanoss/utils/LineRange.java @@ -38,9 +38,15 @@ public LineRange(int start, int end) { } /** - * Checks if this interval overlaps with another interval - * @param other object to compare with - * @return true if overlaps + * Determines if this line range overlaps with another line range. + * Two ranges overlap if any line numbers are shared between them. + * For example: + * - LineRange(1,5) overlaps with LineRange(3,7) + * - LineRange(1,3) overlaps with LineRange(3,5) + * - LineRange(1,3) does not overlap with LineRange(4,6) + * + * @param other the LineRange to check for overlap with this range + * @return true if the ranges share any line numbers, false otherwise */ public boolean overlaps(LineRange other) { return this.start <= other.end && this.end >= other.start; diff --git a/src/main/java/com/scanoss/utils/LineRangeUtils.java b/src/main/java/com/scanoss/utils/LineRangeUtils.java index d2e16c2..8569fbc 100644 --- a/src/main/java/com/scanoss/utils/LineRangeUtils.java +++ b/src/main/java/com/scanoss/utils/LineRangeUtils.java @@ -23,6 +23,9 @@ package com.scanoss.utils; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; + import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -30,6 +33,7 @@ /** * Utility class for handling line range operations */ +@Slf4j public class LineRangeUtils { /** * Parses a line range string into a list of intervals @@ -42,8 +46,8 @@ public static List parseLineRanges(String lineRanges) { return Collections.emptyList(); } - List intervals = new ArrayList<>(); String[] ranges = lineRanges.split(","); + List intervals = new ArrayList<>(ranges.length); for (String range : ranges) { String[] bounds = range.trim().split("-"); @@ -54,7 +58,7 @@ public static List parseLineRanges(String lineRanges) { intervals.add(new LineRange(start, end)); } catch (NumberFormatException e) { // Skip invalid intervals - continue; + log.debug("Invalid interval format: {} in range {}", range, e.getMessage()); } } } @@ -69,7 +73,7 @@ public static List parseLineRanges(String lineRanges) { * @param ranges2 Second set of line ranges * @return true if any intervals overlap */ - public static boolean hasOverlappingRanges(List ranges1, List ranges2) { + public static boolean hasOverlappingRanges(@NotNull List ranges1, @NotNull List ranges2) { for (LineRange interval1 : ranges1) { for (LineRange interval2 : ranges2) { if (interval1.overlaps(interval2)) { @@ -80,7 +84,7 @@ public static boolean hasOverlappingRanges(List ranges1, List ranges, LineRange range) { + public static boolean hasOverlappingRanges(@NotNull List ranges, @NotNull LineRange range) { for (LineRange interval1 : ranges) { if (interval1.overlaps(range)) { return true; From c5787fa4d18714d340c92a7efbd7034b6cd87842 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 16 Dec 2024 12:57:49 +0100 Subject: [PATCH 14/28] chore: add missing JavaDocs --- .../com/scanoss/ScannerPostProcessor.java | 15 +++++++++ .../java/com/scanoss/dto/enums/MatchType.java | 14 +++++++++ src/main/java/com/scanoss/rest/ScanApi.java | 11 +++++++ src/main/java/com/scanoss/settings/Bom.java | 31 ++++++++++++++++++- .../java/com/scanoss/settings/RemoveRule.java | 3 ++ .../com/scanoss/settings/ReplaceRule.java | 3 ++ src/main/java/com/scanoss/settings/Rule.java | 13 ++++++-- .../java/com/scanoss/settings/Settings.java | 13 +++++++- .../java/com/scanoss/utils/LineRange.java | 7 +++++ .../com/scanoss/utils/LineRangeUtils.java | 8 +++++ 10 files changed, 113 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/scanoss/ScannerPostProcessor.java b/src/main/java/com/scanoss/ScannerPostProcessor.java index 1e484e4..c11b203 100644 --- a/src/main/java/com/scanoss/ScannerPostProcessor.java +++ b/src/main/java/com/scanoss/ScannerPostProcessor.java @@ -38,6 +38,21 @@ import java.util.*; import java.util.stream.Collectors; +/** + * Post-processor for SCANOSS scanner results that applies BOM (Bill of Materials) rules + * to modify scan results after the scanning process. This processor handles two main + * operations: + *

+ * 1. Removing components based on remove rules + * 2. Replacing components based on replace rules + *

+ * + * The processor maintains an internal component index for efficient lookup and + * transformation of components during the post-processing phase. + * @see Bom + * @see ScanFileResult + * @see ReplaceRule + */ @Slf4j @Builder public class ScannerPostProcessor { diff --git a/src/main/java/com/scanoss/dto/enums/MatchType.java b/src/main/java/com/scanoss/dto/enums/MatchType.java index 8ed000f..0d8bb33 100644 --- a/src/main/java/com/scanoss/dto/enums/MatchType.java +++ b/src/main/java/com/scanoss/dto/enums/MatchType.java @@ -22,8 +22,22 @@ */ package com.scanoss.dto.enums; +/** + * Represents the type of match found during SCANOSS scanning. + */ public enum MatchType { + /** + * Indicates a complete file match + */ file, + + /** + * Indicates a partial code snippet match + */ snippet, + + /** + * Indicates no match was found + */ none } \ No newline at end of file diff --git a/src/main/java/com/scanoss/rest/ScanApi.java b/src/main/java/com/scanoss/rest/ScanApi.java index a843763..92c41ef 100644 --- a/src/main/java/com/scanoss/rest/ScanApi.java +++ b/src/main/java/com/scanoss/rest/ScanApi.java @@ -238,8 +238,19 @@ private RequestBody multipartData(Map data, String uuid) { } private static final int RETRY_FAIL_SLEEP_TIME = 5; // Time to sleep between failed scan requests + + /** + * Base URL for the SCANOSS OSSKB (Open Source Knowledge Base) free API. + * This endpoint provides access to the free tier of SCANOSS scanning services. + */ public static final String DEFAULT_BASE_URL = "https://api.osskb.org"; + + /** + * Base URL for the SCANOSS Premium API. + * This endpoint is used for premium/enterprise level scanning services. + */ public static final String DEFAULT_BASE_URL2 = "https://api.scanoss.com"; + static final String DEFAULT_SCAN_PATH = "scan/direct"; static final String DEFAULT_SCAN_URL = String.format( "%s/%s", DEFAULT_BASE_URL, DEFAULT_SCAN_PATH ); // Free OSS OSSKB URL static final String DEFAULT_SCAN_URL2 = String.format( "%s/%s", DEFAULT_BASE_URL2, DEFAULT_SCAN_PATH ); // Standard SCANOSS Premium URL diff --git a/src/main/java/com/scanoss/settings/Bom.java b/src/main/java/com/scanoss/settings/Bom.java index ea07330..fece0c6 100644 --- a/src/main/java/com/scanoss/settings/Bom.java +++ b/src/main/java/com/scanoss/settings/Bom.java @@ -28,13 +28,42 @@ import java.util.List; + +/** + * Represents the Bill of Materials (BOM) rules configuration for the SCANOSS scanner. + * This class defines rules for modifying the scan results through include, + * ignore, remove, and replace operations. + */ @Data @Builder public class Bom { + + /** + * List of include rules for adding context when scanning. + * These rules are sent to the SCANOSS API and have a higher chance of being + * considered part of the resulting scan. + */ private final @Singular("include") List include; + + /** + * List of ignore rules for excluding certain components . + * These rules are sent to the SCANOSS API. + */ private final @Singular("ignore") List ignore; + + /** + * List of remove rules for excluding components from results after scanning. + * These rules are applied to the results file after scanning and are processed + * on the client side. + */ private final @Singular("remove") List remove; + + /** + * List of replace rules for substituting components after scanning. + * These rules are applied to the results file after scanning and are processed + * on the client side. Each rule can specify a new PURL and license for the + * replacement component. + */ private final @Singular("replace") List replace; } - diff --git a/src/main/java/com/scanoss/settings/RemoveRule.java b/src/main/java/com/scanoss/settings/RemoveRule.java index 12010d0..3e35188 100644 --- a/src/main/java/com/scanoss/settings/RemoveRule.java +++ b/src/main/java/com/scanoss/settings/RemoveRule.java @@ -27,6 +27,9 @@ import lombok.EqualsAndHashCode; import lombok.experimental.SuperBuilder; +/** + * Rule for removing specific components from scan results. + */ @EqualsAndHashCode(callSuper = true) @Data @SuperBuilder diff --git a/src/main/java/com/scanoss/settings/ReplaceRule.java b/src/main/java/com/scanoss/settings/ReplaceRule.java index 4526fd8..1e88265 100644 --- a/src/main/java/com/scanoss/settings/ReplaceRule.java +++ b/src/main/java/com/scanoss/settings/ReplaceRule.java @@ -27,6 +27,9 @@ import lombok.EqualsAndHashCode; import lombok.experimental.SuperBuilder; +/** + * Rule for replacing components in scan results. + */ @EqualsAndHashCode(callSuper = true) @Data @SuperBuilder diff --git a/src/main/java/com/scanoss/settings/Rule.java b/src/main/java/com/scanoss/settings/Rule.java index f34a150..cea6b77 100644 --- a/src/main/java/com/scanoss/settings/Rule.java +++ b/src/main/java/com/scanoss/settings/Rule.java @@ -26,12 +26,19 @@ import lombok.experimental.SuperBuilder; import lombok.extern.slf4j.Slf4j; +/** + * Base class for SCANOSS BOM rules. Rules are used to modify scan results + * based on file paths and Package URLs (PURLs). + *

+ * Rules support two types of matching: + * - Full match: Both path and PURL match + * - Partial match: Either path or PURL matches + *

+ */ @Data @Slf4j @SuperBuilder() public class Rule { private final String path; private final String purl; //TODO: Add validation with PackageURL -} - - +} \ No newline at end of file diff --git a/src/main/java/com/scanoss/settings/Settings.java b/src/main/java/com/scanoss/settings/Settings.java index bdd4a6d..99ff99a 100644 --- a/src/main/java/com/scanoss/settings/Settings.java +++ b/src/main/java/com/scanoss/settings/Settings.java @@ -33,12 +33,23 @@ import java.nio.file.Files; import java.nio.file.Path; + +/** + * Represents the SCANOSS scanner settings configuration. + * Provides functionality to load and manage scanner settings from JSON files or strings. + * Settings include BOM (Bill of Materials) rules for modifying components before and after scanning. + */ @Slf4j @Data @Builder public class Settings { - private final Bom bom; + /** + * The Bill of Materials (BOM) configuration containing rules for component handling. + * Includes rules for including, ignoring, removing, and replacing components + * during and after the scanning process. + */ + private final Bom bom; /** * Creates a Settings object from a JSON string diff --git a/src/main/java/com/scanoss/utils/LineRange.java b/src/main/java/com/scanoss/utils/LineRange.java index 8626fcf..47f647e 100644 --- a/src/main/java/com/scanoss/utils/LineRange.java +++ b/src/main/java/com/scanoss/utils/LineRange.java @@ -32,6 +32,13 @@ public class LineRange { private final int start; private final int end; + + /** + * Creates a new line range with the specified start and end lines. + * + * @param start the starting line number (inclusive) + * @param end the ending line number (inclusive) + */ public LineRange(int start, int end) { this.start = start; this.end = end; diff --git a/src/main/java/com/scanoss/utils/LineRangeUtils.java b/src/main/java/com/scanoss/utils/LineRangeUtils.java index 8569fbc..c0eb5bc 100644 --- a/src/main/java/com/scanoss/utils/LineRangeUtils.java +++ b/src/main/java/com/scanoss/utils/LineRangeUtils.java @@ -84,6 +84,14 @@ public static boolean hasOverlappingRanges(@NotNull List ranges1, @No return false; } + /** + * Checks if a list of line ranges overlaps with a single range + * + * @param ranges List of line ranges to check against + * @param range Single line range to check for overlap + * @return true if any interval from the list overlaps with the given range + * @throws NullPointerException if either parameter is null + */ public static boolean hasOverlappingRanges(@NotNull List ranges, @NotNull LineRange range) { for (LineRange interval1 : ranges) { if (interval1.overlaps(range)) { From 1864599fb172770f052515a28e7496d9d4dbc69c Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 16 Dec 2024 16:02:38 +0100 Subject: [PATCH 15/28] chore: creates new target excluding slf4j dependencies --- pom.xml | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 94eda5d..625be9c 100644 --- a/pom.xml +++ b/pom.xml @@ -153,9 +153,10 @@ org.apache.maven.plugins maven-assembly-plugin - 3.6.0 + 3.7.1 + with-all-dependencies package single @@ -171,6 +172,40 @@ + + without-slf4j + package + + single + + + + + ${exec.mainClass} + + + + + with-dependencies-slf4j-excluded + + jar + + false + + + / + true + true + + org.slf4j:slf4j-api + org.slf4j:slf4j-simple + + + + + + + From af335cc9b81fb180446286124be6ef7d3a3c54d0 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 19 Dec 2024 18:42:56 +0100 Subject: [PATCH 16/28] Apply changes based on feedback --- src/main/java/com/scanoss/Scanner.java | 1 + .../com/scanoss/ScannerPostProcessor.java | 391 +++++++++++------- src/main/java/com/scanoss/settings/Bom.java | 30 ++ src/main/java/com/scanoss/settings/Rule.java | 2 +- .../com/scanoss/settings/RuleComparator.java | 48 +++ src/test/java/com/scanoss/TestBom.java | 136 ++++++ 6 files changed, 463 insertions(+), 145 deletions(-) create mode 100644 src/main/java/com/scanoss/settings/RuleComparator.java create mode 100644 src/test/java/com/scanoss/TestBom.java diff --git a/src/main/java/com/scanoss/Scanner.java b/src/main/java/com/scanoss/Scanner.java index c859228..3686311 100644 --- a/src/main/java/com/scanoss/Scanner.java +++ b/src/main/java/com/scanoss/Scanner.java @@ -399,6 +399,7 @@ public String scanFile(@NonNull String filename) throws ScannerException, Winnow * @param folder folder to scan * @return List of scan result strings (in JSON format) */ + //TODO: Include postProcessing stage public List scanFolder(@NonNull String folder) { return processFolder(folder, scanFileProcessor); } diff --git a/src/main/java/com/scanoss/ScannerPostProcessor.java b/src/main/java/com/scanoss/ScannerPostProcessor.java index c11b203..cdffd1c 100644 --- a/src/main/java/com/scanoss/ScannerPostProcessor.java +++ b/src/main/java/com/scanoss/ScannerPostProcessor.java @@ -36,18 +36,17 @@ import org.jetbrains.annotations.NotNull; import java.util.*; -import java.util.stream.Collectors; /** * Post-processor for SCANOSS scanner results that applies BOM (Bill of Materials) rules * to modify scan results after the scanning process. This processor handles two main * operations: *

- * 1. Removing components based on remove rules - * 2. Replacing components based on replace rules + * 1. Remove rules + * 2. Replace rules *

* - * The processor maintains an internal component index for efficient lookup and + * The processor maintains an internal Purl2ComponentMap for efficient lookup and * transformation of components during the post-processing phase. * @see Bom * @see ScanFileResult @@ -57,7 +56,10 @@ @Builder public class ScannerPostProcessor { - private Map componentIndex; + /** + * Maps purl to Component (ScanFileDetail) + */ + private Map purl2ComponentDetailsMap; /** * Processes scan results according to BOM configuration rules. @@ -68,14 +70,16 @@ public class ScannerPostProcessor { * @return List of processed scan results */ public List process(@NotNull List scanFileResults, @NotNull Bom bom) { + List processedResults = new ArrayList<>(scanFileResults); + log.info("Starting scan results processing with {} results", scanFileResults.size()); log.debug("BOM configuration - Remove rules: {}, Replace rules: {}", bom.getRemove() != null ? bom.getRemove().size() : 0, bom.getReplace() != null ? bom.getReplace().size() : 0); - createComponentIndex(scanFileResults); - List processedResults = new ArrayList<>(scanFileResults); + purl2ComponentDetailsMap = buildPurl2ComponentDetailsMap(scanFileResults); + if (bom.getRemove() != null && !bom.getRemove().isEmpty()) { log.info("Applying {} remove rules to scan results", bom.getRemove().size()); @@ -84,7 +88,7 @@ public List process(@NotNull List scanFileResult if (bom.getReplace() != null && !bom.getReplace().isEmpty()) { log.info("Applying {} replace rules to scan results", bom.getReplace().size()); - processedResults = applyReplaceRules(processedResults, bom.getReplace()); + processedResults = applyReplaceRules(processedResults, bom.getReplaceRulesByPriority()); } log.info("Scan results processing completed. Original results: {}, Processed results: {}", @@ -93,33 +97,45 @@ public List process(@NotNull List scanFileResult } /** - * Creates a map of PURL (Package URL) to ScanFileDetails from a list of scan results. + * Creates a lookup map that links PURLs to their corresponding component details. + * This map enables efficient component lookup during the replacement process. * * @param scanFileResults List of scan results to process + * @return Map where keys are PURLs and values are their associated component details */ - private void createComponentIndex(List scanFileResults) { - log.debug("Creating component index from scan results"); + private Map buildPurl2ComponentDetailsMap(@NotNull List scanFileResults) { + log.debug("Creating Purl Component Map from scan results"); - if (scanFileResults == null) { - log.warn("Received null scan results, creating empty component index"); - this.componentIndex = new HashMap<>(); - return; + Map index = new HashMap<>(); + + for (ScanFileResult result : scanFileResults) { + if (result == null || result.getFileDetails() == null) { + continue; + } + + // Iterate through file details + for (ScanFileDetails details : result.getFileDetails()) { + if (details == null || details.getPurls() == null) { + continue; + } + + // Iterate through purls for each detail + for (String purl : details.getPurls()) { + if (purl == null || purl.trim().isEmpty()) { + continue; + } + + // Only store if purl not already in map + String trimmedPurl = purl.trim(); + if (!index.containsKey(trimmedPurl)) { + index.put(trimmedPurl, details); + } + } + } } - this.componentIndex = scanFileResults.stream() - .filter(result -> result != null && result.getFileDetails() != null) - .flatMap(result -> result.getFileDetails().stream()) - .filter(details -> details != null && details.getPurls() != null) - .flatMap(details -> Arrays.stream(details.getPurls()) - .filter(purl -> purl != null && !purl.trim().isEmpty()) - .map(purl -> Map.entry(purl.trim(), details))) - .collect(Collectors.toMap( - Map.Entry::getKey, - Map.Entry::getValue, - (existing, replacement) -> existing, - HashMap::new - )); - log.debug("Component index created with {} entries", componentIndex.size()); + log.debug("Purl Component Map created with {} entries", index.size()); + return index; } @@ -127,72 +143,134 @@ private void createComponentIndex(List scanFileResults) { * Applies replacement rules to scan results, updating their PURLs (Package URLs) based on matching rules. * If a cached component exists for a replacement PURL, it will be used instead of creating a new one. * - * @param results The original list of scan results to process + * @param results The list of scan results to process and modify * @param rules The list of replacement rules to apply - * @return A new list containing the processed scan results with updated PURLs + * @return The modified input list of scan results with updated PURLs */ private List applyReplaceRules(@NotNull List results, @NotNull List rules) { - log.debug("Starting replace rules application"); - List resultsList = new ArrayList<>(results); + log.debug("Starting replace rules application for {} results with {} rules", results.size(), rules.size()); + results.forEach(result -> applyReplaceRulesOnResult(result, rules)); + return results; + } - for (ScanFileResult result : resultsList) { - for (ReplaceRule rule : this.findMatchingRules(result, rules)) { - log.debug("Applying replace rule: {} to file: {}", rule, result.getFilePath()); - PackageURL newPurl; - try { - newPurl = new PackageURL(rule.getReplaceWith()); - } catch (MalformedPackageURLException e) { - log.warn("Failed to parse PURL from replace rule: {}. Skipping", rule); - continue; - } - LicenseDetails[] licenseDetails = new LicenseDetails[]{LicenseDetails.builder().name(rule.getLicense()).build()}; - - ScanFileDetails cachedFileDetails = this.componentIndex.get(newPurl.toString()); - ScanFileDetails currentFileDetails = result.getFileDetails().get(0); - ScanFileDetails newFileDetails; - - log.trace("Processing replacement - Cached details found: {}, Current PURL: {}, New PURL: {}", - cachedFileDetails != null, - currentFileDetails.getPurls()[0], - newPurl); - - if (cachedFileDetails != null) { - log.debug("Using cached component details for PURL: {}", newPurl); - newFileDetails = ScanFileDetails.builder() - .matchType(currentFileDetails.getMatchType()) - .file(currentFileDetails.getFile()) - .fileHash(currentFileDetails.getFileHash()) - .fileUrl(currentFileDetails.getFileUrl()) - .lines(currentFileDetails.getLines()) - .matched(currentFileDetails.getMatched()) - .licenseDetails(licenseDetails) - .component(cachedFileDetails.getComponent()) - .vendor(cachedFileDetails.getVendor()) - .url(cachedFileDetails.getUrl()) - .purls(new String[]{newPurl.toString()}) - .build(); - - - result.getFileDetails().set(0, cachedFileDetails); - } else { - log.debug("Creating new component details for PURL: {}", newPurl); - newFileDetails = currentFileDetails.toBuilder() - .copyrightDetails(new CopyrightDetails[]{}) - .licenseDetails(new LicenseDetails[]{}) - .vulnerabilityDetails(new VulnerabilityDetails[]{}) - .purls(new String[]{newPurl.toString()}) - .url("") - .component(newPurl.getName()) - .vendor(newPurl.getNamespace()) - .build(); - } + /** + * Processes a single scan result against all replacement rules. + * Applies the first matching rule found to update the result's package information. + * + * @param result The scan result to process + * @param rules List of replacement rules to check against + */ + private void applyReplaceRulesOnResult(@NotNull ScanFileResult result, @NotNull List rules) { + if (!isValidScanResult(result)) { + log.warn("Invalid scan result structure for file: {}", result.getFilePath()); + return; + } + + + // Find the first matching rule and apply its replacement + // This ensures only one rule is applied per result, maintaining consistency + rules.stream() + .filter(rule -> isMatchingRule(result, rule)) + .findFirst() + .ifPresent(rule -> updateResultWithReplaceRule(result, rule)); + } + + /** + * Updates a scan result using the specified replacement rule. + * Creates a new package URL from the rule and updates all component details + * within the scan result to use the new package information. + * + * @param result The scan result to update + * @param rule The replacement rule containing the new package URL + */ + private void updateResultWithReplaceRule(@NotNull ScanFileResult result, @NotNull ReplaceRule rule) { + PackageURL newPurl = createPackageUrl(rule); + if (newPurl == null) return; + + + List componentDetails = result.getFileDetails(); + for (ScanFileDetails componentDetail : componentDetails ) { - result.getFileDetails().set(0, newFileDetails); + if (componentDetail == null) { + continue; } + + ScanFileDetails newFileDetails = createUpdatedResultDetails(componentDetail, newPurl); + result.getFileDetails().set(0, newFileDetails); + + log.debug("Updated package URL from {} to {} for file: {}", + componentDetail.getPurls()[0], + newPurl, + result.getFilePath()); + } + + + } + + + /** + * Creates a PackageURL object from a replacement rule's target URL. + * + * @param rule The replacement rule containing the new package URL + * @return A new PackageURL object, or null if the URL is malformed + */ + private PackageURL createPackageUrl(ReplaceRule rule) { + try { + return new PackageURL(rule.getReplaceWith()); + } catch (MalformedPackageURLException e) { + log.warn("Failed to parse PURL from replace rule: {}. Skipping", rule); + return null; } - return resultsList; + } + + + /** + * Updates the component details with new package information while preserving existing metadata. + * Takes the existing component as the base and only overrides specific fields (component name, + * vendor, PURLs) based on the new package URL. License details will be updated if specified + * in the replacement rule. + * + * @param existingComponent The current component details to use as a base + * @param newPurl The new package URL containing updated package information + * @return Updated component details with specific fields overridden + */ + private ScanFileDetails createUpdatedResultDetails(ScanFileDetails existingComponent, + PackageURL newPurl) { + + // Check for cached component + ScanFileDetails cached = purl2ComponentDetailsMap.get(newPurl.toString()); + + if (cached != null) { + //TODO: Clarification on copyright, Vulns, etc + //currentComponent.toBuilder().component().vendor().purls().licenseDetails() + + + //Version if we have a package url with version + //pkg:github/scanoss@1.0.0 + + + return cached.toBuilder() + .file(existingComponent.getFile()) + .fileHash(existingComponent.getFileHash()) + .fileUrl(existingComponent.getFileUrl()) + .purls(new String[]{newPurl.toString()}) + .component(newPurl.getName()) + .vendor(newPurl.getNamespace()) + .build(); + } + + // If no cached info, create minimal version + return existingComponent.toBuilder() + .copyrightDetails(new CopyrightDetails[]{}) //TODO: Check if we need the empty Object + .licenseDetails(new LicenseDetails[]{}) + .vulnerabilityDetails(new VulnerabilityDetails[]{}) + .purls(new String[]{newPurl.toString()}) + .url("") // TODO: Implement purl2Url in PackageURL upstream library + .component(newPurl.getName()) + .vendor(newPurl.getNamespace()) + .build(); } @@ -210,80 +288,94 @@ private List applyReplaceRules(@NotNull List res * @param rules The list of remove rules to apply * @return A filtered list with matching results removed based on the above criteria */ - private List applyRemoveRules(@NotNull List results, @NotNull List rules) { + public List applyRemoveRules(@NotNull List results, @NotNull List rules) { log.debug("Starting remove rules application to {} results", results.size()); List resultsList = new ArrayList<>(results); + resultsList.removeIf(result -> matchesRemovalCriteria(result, rules)); + log.debug("Remove rules application completed. Results remaining: {}", resultsList.size()); + return resultsList; + } - resultsList.removeIf(result -> { - List matchingRules = findMatchingRules(result, rules); - log.trace("Found {} matching remove rules for file: {}", matchingRules.size(), result.getFilePath()); - if (matchingRules.isEmpty()) { - return false; - } - - // Check if any matching rules have line ranges defined - List rulesWithLineRanges = matchingRules.stream() - .filter(rule -> rule.getStartLine() != null && rule.getEndLine() != null) - .collect(Collectors.toList()); - - // If no rules have line ranges, remove the result - if (rulesWithLineRanges.isEmpty()) { - log.debug("Removing entire file - no line ranges specified in rules for file {}", result.getFilePath()); - return true; - } - - // If we have line ranges, check for overlaps - String resultLineRangesString = result.getFileDetails().get(0).getLines(); - List resultLineRanges = LineRangeUtils.parseLineRanges(resultLineRangesString); - - boolean shouldRemove = rulesWithLineRanges.stream() - .map(rule -> new LineRange(rule.getStartLine(), rule.getEndLine())) - .anyMatch(ruleLineRange -> LineRangeUtils.hasOverlappingRanges(resultLineRanges, ruleLineRange)); - if (shouldRemove) { - log.debug("Removing file {} due to overlapping line ranges", result.getFilePath()); - } + /** + * Determines if a scan result should be excluded based on the removal rules. + *

+ * For each rule, checks: + * 1. If the result matches the rule's path/purl patterns + * 2. Then applies line range logic: + * - If rule has no line range specified: Result is removed + * - If rule has line range specified: Result is only removed if its lines overlap with the rule's range + *

+ * @param result The scan result to evaluate + * @param rules List of removal rules to check against + * @return true if the result should be removed, false otherwise + */ + private Boolean matchesRemovalCriteria(@NotNull ScanFileResult result, @NotNull List rules) { - return shouldRemove; - }); + if (!isValidScanResult(result)) { + log.warn("Invalid scan result structure for file: {}", result.getFilePath()); + return false; + } - log.debug("Remove rules application completed. Results remaining: {}", resultsList.size()); - return resultsList; + return rules.stream() + .filter(rule -> isMatchingRule(result, rule)) + .anyMatch(rule -> { + // Process line range conditions: + // - returns true if rule has no line range specified + // - returns true if rule has line range AND result overlaps with it + // - returns false otherwise (continue checking remaining rules) + boolean ruleHasLineRange = rule.getStartLine() != null && rule.getEndLine() != null; + return !ruleHasLineRange || isLineRangeMatch(rule, result); + }); } + /** - * Finds and returns a list of matching rules for a scan result. - * A rule matches if: - * 1. It has both path and purl, and both match the result - * 2. It has only a purl (no path), and the purl matches the result - * 3. It has only a path (no purl), and the path matches the result + * Checks if a scan result matches the path and/or PURL patterns defined in a rule. + * + * The match is considered successful if any of these conditions are met: + * 1. Rule has both path and PURL defined: Both must match the result + * 2. Rule has only PURL defined: PURL must match the result + * 3. Rule has only path defined: Path must match the result * - * @param The rule type. Must extend Rule class * @param result The scan result to check - * @param rules List of rules to check against - * @return List of matching rules, empty list if no matches found + * @param rule The rule containing the patterns to match against + * @param Type parameter extending Rule class + * @return true if the result matches the rule's patterns according to above conditions */ - private List findMatchingRules(@NotNull ScanFileResult result, @NotNull List rules) { - log.trace("Finding matching rules for file: {}", result.getFilePath()); - return rules.stream() - .filter(rule -> { - boolean hasPath = rule.getPath() != null && !rule.getPath().isEmpty(); - boolean hasPurl = rule.getPurl() != null && !rule.getPurl().isEmpty(); - - if (hasPath && hasPurl) { - return isPathAndPurlMatch(rule, result); - } else if (hasPath) { - return isPathOnlyMatch(rule, result); - } else if (hasPurl) { - return isPurlOnlyMatch(rule, result); - } + private Boolean isMatchingRule(@NotNull ScanFileResult result, @NotNull T rule) { + // Check if rule has valid path and/or PURL patterns + boolean hasPath = rule.getPath() != null && !rule.getPath().isEmpty(); + boolean hasPurl = rule.getPurl() != null && !rule.getPurl().isEmpty(); + + // Check three possible matching conditions: + // 1. Both path and PURL match + if (hasPath && hasPurl && isPathAndPurlMatch(rule, result)) { + return true; + // 2. Only PURL match required and matches + } else if (hasPurl && isPurlOnlyMatch(rule, result)) { + return true; + // 3. Only path match required and matches + } if (hasPath && isPathOnlyMatch(rule, result)) { + return true; + } - log.warn("Rule {} has neither path nor PURL specified", rule); - return false; // Neither path nor purl specified - }).collect(Collectors.toList()); + return false; } + /** + * Checks if line range of the remove rule match the result + */ + private boolean isLineRangeMatch(RemoveRule rule, ScanFileResult result) { + LineRange ruleLineRange = new LineRange(rule.getStartLine(), rule.getEndLine()); + + String lines = result.getFileDetails().get(0).getLines(); + List resultLineRanges = LineRangeUtils.parseLineRanges(lines); + + return LineRangeUtils.hasOverlappingRanges(resultLineRanges,ruleLineRange); + } + /** * Checks if both path and purl of the rule match the result */ @@ -323,4 +415,15 @@ private boolean isPurlMatch(String rulePurl, String[] resultPurls) { return false; } + /** + * Validates if a scan result has the required fields for rule processing. + * + * @param result The scan result to validate + * @return true if the result has valid file details and PURLs + */ + private boolean isValidScanResult(@NotNull ScanFileResult result) { + return result.getFileDetails() != null + && !result.getFileDetails().isEmpty() + && result.getFileDetails().get(0) != null; + } } diff --git a/src/main/java/com/scanoss/settings/Bom.java b/src/main/java/com/scanoss/settings/Bom.java index fece0c6..d98615c 100644 --- a/src/main/java/com/scanoss/settings/Bom.java +++ b/src/main/java/com/scanoss/settings/Bom.java @@ -26,6 +26,7 @@ import lombok.Data; import lombok.Singular; +import java.util.ArrayList; import java.util.List; @@ -65,5 +66,34 @@ public class Bom { * replacement component. */ private final @Singular("replace") List replace; + + + /** + * Cached list of replace rules sorted by priority. + * This list is lazily initialized when first accessed through + * getReplaceRulesByPriority(). + * + * @see #getReplaceRulesByPriority() + */ + private final List sortedReplace; + + + /** + * Sorts replace rules by priority from highest to lowest: + * 1. Rules with both purl/path (most specific) + * 3. Rules with only purl + * 4. Rules with only path (least specific) + * + * @return A new list containing the sorted replace rules + */ + public List getReplaceRulesByPriority() { + if (sortedReplace == null) { + List sortedReplace = new ArrayList<>(replace); + sortedReplace.sort(new RuleComparator()); + return sortedReplace; + } + return sortedReplace; + } + } diff --git a/src/main/java/com/scanoss/settings/Rule.java b/src/main/java/com/scanoss/settings/Rule.java index cea6b77..0b18942 100644 --- a/src/main/java/com/scanoss/settings/Rule.java +++ b/src/main/java/com/scanoss/settings/Rule.java @@ -40,5 +40,5 @@ @SuperBuilder() public class Rule { private final String path; - private final String purl; //TODO: Add validation with PackageURL + private final String purl; } \ No newline at end of file diff --git a/src/main/java/com/scanoss/settings/RuleComparator.java b/src/main/java/com/scanoss/settings/RuleComparator.java new file mode 100644 index 0000000..6d66a42 --- /dev/null +++ b/src/main/java/com/scanoss/settings/RuleComparator.java @@ -0,0 +1,48 @@ +package com.scanoss.settings; + +import java.util.Comparator; + +public class RuleComparator implements Comparator { + @Override + public int compare(Rule r1, Rule r2) { + int score1 = calculatePriorityScore(r1); + int score2 = calculatePriorityScore(r2); + + // If scores are different, use the score comparison + if (score1 != score2) { + return Integer.compare(score2, score1); + } + + String r1Path = r1.getPath(); + String r2Path = r2.getPath(); + + // If both rules have paths and scores are equal, compare path lengths + if (r1Path != null && r2Path != null) { + return Integer.compare(r2Path.length(), r1Path.length()); + } + + return 0; + } + + private int calculatePriorityScore(Rule rule) { + int score = 0; + + boolean hasPurl = rule.getPurl() != null; + boolean hasPath = rule.getPath() != null; + + // Highest priority: has all fields + if (hasPurl && hasPath) { + score = 4; + } + // Second priority: has purl only + else if (hasPurl) { + score = 2; + } + // Lowest priority: has path only + else if (hasPath) { + score = 1; + } + + return score; + } +} \ No newline at end of file diff --git a/src/test/java/com/scanoss/TestBom.java b/src/test/java/com/scanoss/TestBom.java new file mode 100644 index 0000000..c6c5828 --- /dev/null +++ b/src/test/java/com/scanoss/TestBom.java @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2024, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.scanoss; + +import com.scanoss.settings.Bom; +import com.scanoss.settings.ReplaceRule; +import com.scanoss.settings.RuleComparator; +import lombok.extern.slf4j.Slf4j; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.*; + + +@Slf4j +public class TestBom { + + @Before + public void Setup() { + log.info("Starting Bom test cases..."); + } + + @Test + public void testReplaceRulesSortingAllCombinations() { + log.info("<-- Starting testReplaceRulesSortingAllCombinations"); + + // Create rules with different combinations of path and purl + ReplaceRule bothFieldsLongPath = ReplaceRule.builder() + .path("/very/long/path/to/specific/file.txt") + .purl("pkg:maven/org.example/library@1.0.0") + .build(); + + ReplaceRule bothFieldsShortPath = ReplaceRule.builder() + .path("/short/path") + .purl("pkg:maven/org.example/another@1.0.0") + .build(); + + ReplaceRule purlOnly = ReplaceRule.builder() + .purl("pkg:maven/org.example/another@2.0.0") + .build(); + + ReplaceRule pathOnly = ReplaceRule.builder() + .path("/another/path") + .build(); + + // Create Bom with rules in random order + Bom bom = Bom.builder() + .replace(pathOnly) + .replace(bothFieldsShortPath) + .replace(purlOnly) + .replace(bothFieldsLongPath) + .build(); + + // Get sorted rules + List sortedRules = bom.getReplaceRulesByPriority(); + + // Verify order + assertEquals("Rule with both fields and longer path should be first", bothFieldsLongPath, sortedRules.get(0)); + assertEquals("Rule with both fields and shorter path should be second", bothFieldsShortPath, sortedRules.get(1)); + assertEquals("Rule with purl only should be third", purlOnly, sortedRules.get(2)); + assertEquals("Rule with path only should be last", pathOnly, sortedRules.get(3)); + + log.info("Finished testReplaceRulesSortingAllCombinations -->"); + } + + + @Test + public void testReplaceRulesSortingWithDuplicatePaths() { + log.info("<-- Starting testReplaceRulesSortingWithDuplicatePaths"); + + // Create rules with same path length but different paths + ReplaceRule pathRule1 = ReplaceRule.builder() + .path("/path/to/first") + .build(); + + ReplaceRule pathRule2 = ReplaceRule.builder() + .path("/path/to/other") + .build(); + + // Create Bom with rules + Bom bom = Bom.builder() + .replace(pathRule1) + .replace(pathRule2) + .build(); + + // Get sorted rules + List sortedRules = bom.getReplaceRulesByPriority(); + + // Verify the rules with same path length maintain original order + assertEquals("Size should be 2", 2, sortedRules.size()); + assertTrue("Both rules should have same priority", + new RuleComparator().compare(sortedRules.get(0), sortedRules.get(1)) == 0); + + log.info("Finished testReplaceRulesSortingWithDuplicatePaths -->"); + } + + + @Test + public void testReplaceRulesSortingEmptyList() { + log.info("<-- Starting testReplaceRulesSortingEmptyList"); + + // Create Bom with no rules + Bom bom = Bom.builder().build(); + + // Get sorted rules + List sortedRules = bom.getReplaceRulesByPriority(); + + // Verify + assertNotNull("Sorted list should not be null", sortedRules); + assertTrue("Sorted list should be empty", sortedRules.isEmpty()); + + log.info("Finished testReplaceRulesSortingEmptyList -->"); + } +} \ No newline at end of file From 9ffd7847b05b3bc5710f64b31e7e48a10948eb45 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 20 Dec 2024 15:27:56 +0100 Subject: [PATCH 17/28] SP-1982 adds purl2url convertion --- .../com/scanoss/ScannerPostProcessor.java | 86 +++-- .../com/scanoss/settings/RuleComparator.java | 51 +++ src/main/java/com/scanoss/utils/Purl2Url.java | 83 +++++ src/test/java/com/scanoss/TestPurl2Url.java | 350 ++++++++++++++++++ 4 files changed, 538 insertions(+), 32 deletions(-) create mode 100644 src/main/java/com/scanoss/utils/Purl2Url.java create mode 100644 src/test/java/com/scanoss/TestPurl2Url.java diff --git a/src/main/java/com/scanoss/ScannerPostProcessor.java b/src/main/java/com/scanoss/ScannerPostProcessor.java index cdffd1c..ac2ac56 100644 --- a/src/main/java/com/scanoss/ScannerPostProcessor.java +++ b/src/main/java/com/scanoss/ScannerPostProcessor.java @@ -31,6 +31,7 @@ import com.scanoss.settings.Rule; import com.scanoss.utils.LineRange; import com.scanoss.utils.LineRangeUtils; +import com.scanoss.utils.Purl2Url; import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -197,7 +198,7 @@ private void updateResultWithReplaceRule(@NotNull ScanFileResult result, @NotNul continue; } - ScanFileDetails newFileDetails = createUpdatedResultDetails(componentDetail, newPurl); + ScanFileDetails newFileDetails = createUpdatedResultDetails(componentDetail, newPurl, rule); result.getFileDetails().set(0, newFileDetails); log.debug("Updated package URL from {} to {} for file: {}", @@ -233,46 +234,67 @@ private PackageURL createPackageUrl(ReplaceRule rule) { * in the replacement rule. * * @param existingComponent The current component details to use as a base - * @param newPurl The new package URL containing updated package information + * @param newPackageUrl The new package URL containing updated package information + * @param replacementRule The rule to extract the license, if exist * @return Updated component details with specific fields overridden */ private ScanFileDetails createUpdatedResultDetails(ScanFileDetails existingComponent, - PackageURL newPurl) { - - // Check for cached component - ScanFileDetails cached = purl2ComponentDetailsMap.get(newPurl.toString()); - - if (cached != null) { - //TODO: Clarification on copyright, Vulns, etc - //currentComponent.toBuilder().component().vendor().purls().licenseDetails() - - - //Version if we have a package url with version - //pkg:github/scanoss@1.0.0 - - - return cached.toBuilder() - .file(existingComponent.getFile()) - .fileHash(existingComponent.getFileHash()) - .fileUrl(existingComponent.getFileUrl()) - .purls(new String[]{newPurl.toString()}) - .component(newPurl.getName()) - .vendor(newPurl.getNamespace()) + PackageURL newPackageUrl, + @NotNull ReplaceRule replacementRule) { + + + // Check if we already have processed this package URL + ScanFileDetails cachedComponent = purl2ComponentDetailsMap.get(newPackageUrl.toString()); + + // Extract license information from the replacement rule + String definedLicenseName = replacementRule.getLicense(); + LicenseDetails newLicense = definedLicenseName != null + ? LicenseDetails.builder().name(definedLicenseName).build() + : null; + + if (cachedComponent != null) { + // Update existing component with cached information + return existingComponent.toBuilder() + .copyrightDetails(null) + .licenseDetails(definedLicenseName != null + ? new LicenseDetails[]{ newLicense } + : cachedComponent.getLicenseDetails()) + .version(newPackageUrl.getVersion() != null + ? newPackageUrl.getVersion() + : existingComponent.getVersion()) + .purls(new String[]{ newPackageUrl.toString() }) + .component(newPackageUrl.getName()) + .vendor(newPackageUrl.getNamespace()) .build(); } - // If no cached info, create minimal version - return existingComponent.toBuilder() - .copyrightDetails(new CopyrightDetails[]{}) //TODO: Check if we need the empty Object - .licenseDetails(new LicenseDetails[]{}) - .vulnerabilityDetails(new VulnerabilityDetails[]{}) - .purls(new String[]{newPurl.toString()}) - .url("") // TODO: Implement purl2Url in PackageURL upstream library - .component(newPurl.getName()) - .vendor(newPurl.getNamespace()) + // Create new component when no cached version exists + return ScanFileDetails.builder() + .licenseDetails(newLicense != null + ? new LicenseDetails[]{ newLicense } + : null) + .purls(new String[]{ newPackageUrl.toString() }) + .url(Purl2Url.convert(newPackageUrl)) + .version(determineVersion(newPackageUrl, existingComponent)) + .component(newPackageUrl.getName()) + .vendor(newPackageUrl.getNamespace()) .build(); } + /** + * Determines the version to use by checking if the new PURL has a version specified. + * If the new PURL has no version, falls back to the existing component's version. + * + * @param newPurl The new PURL containing potential version information + * @param existingComponent The existing component with fallback version information + * @return The determined version string + */ + private String determineVersion(PackageURL newPurl, ScanFileDetails existingComponent) { + return newPurl.getVersion() != null + ? newPurl.getVersion() + : existingComponent.getVersion(); + } + /** * Applies remove rules to scan results, filtering out matches based on certain criteria. diff --git a/src/main/java/com/scanoss/settings/RuleComparator.java b/src/main/java/com/scanoss/settings/RuleComparator.java index 6d66a42..d9a7a60 100644 --- a/src/main/java/com/scanoss/settings/RuleComparator.java +++ b/src/main/java/com/scanoss/settings/RuleComparator.java @@ -2,7 +2,43 @@ import java.util.Comparator; + +/** + * A comparator implementation for sorting {@link Rule} objects based on their attributes + * and a defined priority system. + * + *

The comparison is performed using the following criteria in order:

+ *
    + *
  1. Priority score based on the presence of PURL and path attributes
    2. + *
  2. Path length comparison (if priority scores are equal)
    3. + *
+ * + *

Priority scoring system:

+ *
    + *
  • Score 4: Rule has both PURL and path
    • + *
  • Score 2: Rule has only PURL
    • + *
  • Score 1: Rule has only path
    • + *
  • Score 0: Rule has neither PURL nor path
    • + *
+ */ public class RuleComparator implements Comparator { + + /** + * Compares two Rule objects based on their priority scores and path lengths. + * + *

The comparison follows these steps:

+ *
    + *
  1. Calculate priority scores for both rules
    2. + *
  2. If scores differ, higher score takes precedence
    3. + *
  3. If scores are equal and both rules have paths, longer path takes precedence
    4. + *
  4. If no differentiation is possible, rules are considered equal
    5. + *
+ * + * @param r1 the first Rule to compare + * @param r2 the second Rule to compare + * @return a negative integer, zero, or a positive integer if the first rule is less than, + * equal to, or greater than the second rule respectively + */ @Override public int compare(Rule r1, Rule r2) { int score1 = calculatePriorityScore(r1); @@ -24,6 +60,21 @@ public int compare(Rule r1, Rule r2) { return 0; } + + /** + * Calculates a priority score for a rule based on its attributes. + * + *

The scoring system is as follows:

+ *
    + *
  • 4 points: Rule has both PURL and path
    • + *
  • 2 points: Rule has only PURL
    • + *
  • 1 point: Rule has only path
    • + *
  • 0 points: Rule has neither PURL nor path
    • + *
+ * + * @param rule the Rule object to calculate the score for + * @return an integer representing the priority score of the rule + */ private int calculatePriorityScore(Rule rule) { int score = 0; diff --git a/src/main/java/com/scanoss/utils/Purl2Url.java b/src/main/java/com/scanoss/utils/Purl2Url.java new file mode 100644 index 0000000..d56a4f8 --- /dev/null +++ b/src/main/java/com/scanoss/utils/Purl2Url.java @@ -0,0 +1,83 @@ +package com.scanoss.utils; + +import com.github.packageurl.PackageURL; +import lombok.Getter; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Converts Package URLs (purls) to their corresponding browsable web URLs for + * different package management systems and source code repositories. + */ +public class Purl2Url { + private static final Logger log = LoggerFactory.getLogger(Purl2Url.class); + + @Getter + public enum PurlType { + GITHUB("github", "https://github.com/%s"), + NPM("npm", "https://www.npmjs.com/package/%s"), + MAVEN("maven", "https://mvnrepository.com/artifact/%s"), + GEM("gem", "https://rubygems.org/gems/%s"), + PYPI("pypi", "https://pypi.org/project/%s"), + GOLANG("golang", "https://pkg.go.dev/%s"), + NUGET("nuget", "https://www.nuget.org/packages/%s"); + + + private final String type; + private final String urlPattern; + + PurlType(String type, String urlPattern) { + this.type = type; + this.urlPattern = urlPattern; + } + } + + /** + * Checks if the given PackageURL is supported for conversion. + * + * @param purl The PackageURL to check + * @return true if the PackageURL can be converted to a browsable URL + */ + public static boolean isSupported(@NotNull PackageURL purl) { + try { + findPurlType(purl.getType()); + return true; + } catch (Exception e) { + return false; + } + } + + /** + * Converts a PackageURL to its browsable web URL. + * Returns null if the conversion is not possible. + * + * @param purl The PackageURL to convert + * @return The browsable web URL or null if conversion fails + */ + @Nullable + public static String convert(@NotNull PackageURL purl) { + try { + PurlType purlType = findPurlType(purl.getType()); + String fullName = purl.getNamespace() != null ? + purl.getNamespace() + "/" + purl.getName() : + purl.getName(); + return String.format(purlType.getUrlPattern(), fullName); + } catch (Exception e) { + log.debug("Failed to convert purl to URL: {}", purl, e); + return null; + } + } + + private static PurlType findPurlType(String type) { + for (PurlType purlType : PurlType.values()) { + if (purlType.getType().equals(type)) { + return purlType; + } + } + throw new IllegalArgumentException( + String.format("Unsupported package type: %s", type) + ); + } +} \ No newline at end of file diff --git a/src/test/java/com/scanoss/TestPurl2Url.java b/src/test/java/com/scanoss/TestPurl2Url.java new file mode 100644 index 0000000..32ecc17 --- /dev/null +++ b/src/test/java/com/scanoss/TestPurl2Url.java @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2023, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.scanoss; +import com.github.packageurl.PackageURL; +import com.scanoss.utils.Purl2Url; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.*; + +@Slf4j +public class TestPurl2Url { + + @Test + public void testValidGithubUrl() throws Exception { + log.info("<-- Starting testValidGithubUrl"); + + // Create test data - using React repository + PackageURL purl = new PackageURL("pkg:github/facebook/react"); + + // Verify support + assertTrue("React GitHub repo should be supported", Purl2Url.isSupported(purl)); + + // Generate URL + String url = Purl2Url.convert(purl); + + // Verify + assertNotNull("Generated URL should not be null", url); + assertEquals("URL should match React GitHub repo", + "https://github.com/facebook/react", url); + + log.info("Finished testValidGithubUrl -->"); + } + + @Test + public void testValidNpmUrl() throws Exception { + log.info("<-- Starting testValidNpmUrl"); + + // Create test data - using Express.js + PackageURL purl = new PackageURL("pkg:npm/express"); + + // Verify support + assertTrue("Express npm package should be supported", Purl2Url.isSupported(purl)); + + // Generate URL + String url = Purl2Url.convert(purl); + + // Verify + assertNotNull("Generated URL should not be null", url); + assertEquals("URL should match Express npm package", + "https://www.npmjs.com/package/express", url); + + log.info("Finished testValidNpmUrl -->"); + } + + @Test + public void testScopedNpmPackage() throws Exception { + log.info("<-- Starting testScopedNpmPackage"); + + // Create test data - using Angular core package + PackageURL purl = new PackageURL("pkg:npm/%40angular/core"); + + // Verify support + assertTrue("Angular core package should be supported", Purl2Url.isSupported(purl)); + + // Generate URL + String url = Purl2Url.convert(purl); + + // Verify + assertNotNull("Generated URL should not be null", url); + assertEquals("URL should match Angular core package", + "https://www.npmjs.com/package/@angular/core", url); + + log.info("Finished testScopedNpmPackage -->"); + } + + @Test + public void testValidMavenUrl() throws Exception { + log.info("<-- Starting testValidMavenUrl"); + + // Create test data - using Spring Boot + PackageURL purl = new PackageURL("pkg:maven/org.springframework.boot/spring-boot"); + + // Verify support + assertTrue("Spring Boot Maven artifact should be supported", Purl2Url.isSupported(purl)); + + // Generate URL + String url = Purl2Url.convert(purl); + + // Verify + assertNotNull("Generated URL should not be null", url); + assertEquals("URL should match Spring Boot Maven artifact", + "https://mvnrepository.com/artifact/org.springframework.boot/spring-boot", url); + + log.info("Finished testValidMavenUrl -->"); + } + + @Test + public void testValidPypiUrl() throws Exception { + log.info("<-- Starting testValidPypiUrl"); + + // Create test data - using Django + PackageURL purl = new PackageURL("pkg:pypi/django"); + + // Verify support + assertTrue("Django PyPI package should be supported", Purl2Url.isSupported(purl)); + + // Generate URL + String url = Purl2Url.convert(purl); + + // Verify + assertNotNull("Generated URL should not be null", url); + assertEquals("URL should match Django PyPI package", + "https://pypi.org/project/django", url); + + log.info("Finished testValidPypiUrl -->"); + } + + @Test + public void testValidNugetUrl() throws Exception { + log.info("<-- Starting testValidNugetUrl"); + + // Create test data - using Newtonsoft.Json + PackageURL purl = new PackageURL("pkg:nuget/Newtonsoft.Json"); + + // Verify support + assertTrue("Newtonsoft.Json NuGet package should be supported", Purl2Url.isSupported(purl)); + + // Generate URL + String url = Purl2Url.convert(purl); + + // Verify + assertNotNull("Generated URL should not be null", url); + assertEquals("URL should match Newtonsoft.Json NuGet package", + "https://www.nuget.org/packages/Newtonsoft.Json", url); + + log.info("Finished testValidNugetUrl -->"); + } + + @Test + public void testUnsupportedType() throws Exception { + log.info("<-- Starting testUnsupportedType"); + + // Create test data - using an unsupported type + PackageURL purl = new PackageURL("pkg:unknown/test-package"); + + // Verify non-support + assertFalse("Unknown package type should not be supported", Purl2Url.isSupported(purl)); + + // Generate URL + String url = Purl2Url.convert(purl); + + // Verify + assertNull("Generated URL should be null for unsupported type", url); + + log.info("Finished testUnsupportedType -->"); + } + + @Test + public void testAllPackageTypes() throws Exception { + log.info("<-- Starting testAllPackageTypes"); + + class PackageTestCase { + final String purl; + final String expectedUrl; + final String description; + + PackageTestCase(String purl, String expectedUrl, String description) { + this.purl = purl; + this.expectedUrl = expectedUrl; + this.description = description; + } + } + + List testCases = Arrays.asList( + // GitHub Cases + new PackageTestCase( + "pkg:github/facebook/react", + "https://github.com/facebook/react", + "Standard GitHub repository" + ), + new PackageTestCase( + "pkg:github/apache/kafka", + "https://github.com/apache/kafka", + "GitHub repo with no special chars" + ), + new PackageTestCase( + "pkg:github/spring-projects/spring-boot", + "https://github.com/spring-projects/spring-boot", + "GitHub repo with hyphen" + ), + + // NPM Cases + new PackageTestCase( + "pkg:npm/express", + "https://www.npmjs.com/package/express", + "Simple NPM package" + ), + new PackageTestCase( + "pkg:npm/%40angular/core", + "https://www.npmjs.com/package/@angular/core", + "Scoped NPM package" + ), + new PackageTestCase( + "pkg:npm/%40types/node", + "https://www.npmjs.com/package/@types/node", + "TypeScript definitions package" + ), + + // Maven Cases + new PackageTestCase( + "pkg:maven/org.springframework.boot/spring-boot", + "https://mvnrepository.com/artifact/org.springframework.boot/spring-boot", + "Standard Maven artifact" + ), + new PackageTestCase( + "pkg:maven/com.google.guava/guava", + "https://mvnrepository.com/artifact/com.google.guava/guava", + "Google Guava Maven artifact" + ), + new PackageTestCase( + "pkg:maven/io.quarkus/quarkus-core", + "https://mvnrepository.com/artifact/io.quarkus/quarkus-core", + "Quarkus Maven artifact" + ), + + // PyPI Cases + new PackageTestCase( + "pkg:pypi/requests", + "https://pypi.org/project/requests", + "Simple PyPI package" + ), + new PackageTestCase( + "pkg:pypi/django", + "https://pypi.org/project/django", + "Django Python package" + ), + new PackageTestCase( + "pkg:pypi/python-dateutil", + "https://pypi.org/project/python-dateutil", + "PyPI package with hyphen" + ), + + // Gem Cases + new PackageTestCase( + "pkg:gem/rails", + "https://rubygems.org/gems/rails", + "Simple Ruby gem" + ), + new PackageTestCase( + "pkg:gem/activerecord", + "https://rubygems.org/gems/activerecord", + "ActiveRecord Ruby gem" + ), + new PackageTestCase( + "pkg:gem/devise", + "https://rubygems.org/gems/devise", + "Devise authentication gem" + ), + + // Golang Cases + new PackageTestCase( + "pkg:golang/golang.org/x/text", + "https://pkg.go.dev/golang.org/x/text", + "Official Go package" + ), + new PackageTestCase( + "pkg:golang/github.com/gin-gonic/gin", + "https://pkg.go.dev/github.com/gin-gonic/gin", + "Gin web framework" + ), + new PackageTestCase( + "pkg:golang/google.golang.org/grpc", + "https://pkg.go.dev/google.golang.org/grpc", + "gRPC Go package" + ), + + // NuGet Cases + new PackageTestCase( + "pkg:nuget/Newtonsoft.Json", + "https://www.nuget.org/packages/Newtonsoft.Json", + "Popular JSON library" + ), + new PackageTestCase( + "pkg:nuget/Microsoft.AspNetCore.App", + "https://www.nuget.org/packages/Microsoft.AspNetCore.App", + "Microsoft ASP.NET Core package" + ), + new PackageTestCase( + "pkg:nuget/Serilog.Sinks.Console", + "https://www.nuget.org/packages/Serilog.Sinks.Console", + "NuGet package with dots" + ) + ); + + // Run all test cases + for (PackageTestCase tc : testCases) { + log.info("Testing: {}", tc.description); + + PackageURL purl = new PackageURL(tc.purl); + + // Verify support + assertTrue( + String.format("%s should be supported", tc.description), + Purl2Url.isSupported(purl) + ); + + // Generate URL + String url = Purl2Url.convert(purl); + + // Verify + assertNotNull( + String.format("Generated URL should not be null for %s", tc.description), + url + ); + assertEquals( + String.format("URL should match expected for %s", tc.description), + tc.expectedUrl, + url + ); + } + + log.info("Finished testAllPackageTypes -->"); + } + +} \ No newline at end of file From 5574ed7c92a69b7a2a0feba58870613dc4eb3f4d Mon Sep 17 00:00:00 2001 From: eeisegn Date: Tue, 31 Dec 2024 12:31:24 +0000 Subject: [PATCH 18/28] documentation and error checking cleanup --- src/main/java/com/scanoss/utils/Purl2Url.java | 97 +++++++++++++++---- 1 file changed, 76 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/scanoss/utils/Purl2Url.java b/src/main/java/com/scanoss/utils/Purl2Url.java index d56a4f8..aa52209 100644 --- a/src/main/java/com/scanoss/utils/Purl2Url.java +++ b/src/main/java/com/scanoss/utils/Purl2Url.java @@ -1,33 +1,83 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2024, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package com.scanoss.utils; import com.github.packageurl.PackageURL; import lombok.Getter; -import org.jetbrains.annotations.NotNull; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Converts Package URLs (purls) to their corresponding browsable web URLs for * different package management systems and source code repositories. */ +@Slf4j public class Purl2Url { - private static final Logger log = LoggerFactory.getLogger(Purl2Url.class); + /** + * PURL type to URL enum + */ @Getter public enum PurlType { + /** + * GitHub URL + */ GITHUB("github", "https://github.com/%s"), + /** + * Node URL + */ NPM("npm", "https://www.npmjs.com/package/%s"), + /** + * Maven Central URL + */ MAVEN("maven", "https://mvnrepository.com/artifact/%s"), + /** + * Ruby Gems URL + */ GEM("gem", "https://rubygems.org/gems/%s"), + /** + * Python PyPI URL + */ PYPI("pypi", "https://pypi.org/project/%s"), + /** + * Golang URL + */ GOLANG("golang", "https://pkg.go.dev/%s"), + /** + * MS Nuget URL + */ NUGET("nuget", "https://www.nuget.org/packages/%s"); - private final String type; private final String urlPattern; + /** + * Setup PURL type/URL enum + * + * @param type PURL Type + * @param urlPattern URL pattern + */ PurlType(String type, String urlPattern) { this.type = type; this.urlPattern = urlPattern; @@ -38,15 +88,16 @@ public enum PurlType { * Checks if the given PackageURL is supported for conversion. * * @param purl The PackageURL to check - * @return true if the PackageURL can be converted to a browsable URL + * @return true if the PackageURL can be converted to a browsable URL */ - public static boolean isSupported(@NotNull PackageURL purl) { + public static boolean isSupported(@NonNull PackageURL purl) { try { findPurlType(purl.getType()); return true; - } catch (Exception e) { - return false; + } catch (RuntimeException e) { + log.warn("Failed to find PURL type {} from {}: {}", purl.getType(), purl, e.getLocalizedMessage()); } + return false; } /** @@ -54,30 +105,34 @@ public static boolean isSupported(@NotNull PackageURL purl) { * Returns null if the conversion is not possible. * * @param purl The PackageURL to convert - * @return The browsable web URL or null if conversion fails + * @return The browsable web URL or null if conversion fails */ @Nullable - public static String convert(@NotNull PackageURL purl) { + public static String convert(@NonNull PackageURL purl) { try { PurlType purlType = findPurlType(purl.getType()); - String fullName = purl.getNamespace() != null ? - purl.getNamespace() + "/" + purl.getName() : - purl.getName(); + String nameSpace = purl.getNamespace(); + String fullName = nameSpace != null ? nameSpace + "/" + purl.getName() : purl.getName(); return String.format(purlType.getUrlPattern(), fullName); - } catch (Exception e) { - log.debug("Failed to convert purl to URL: {}", purl, e); - return null; + } catch (RuntimeException e) { + log.debug("Failed to convert purl to URL for {}: {}", purl, e.getLocalizedMessage(), e); } + return null; } - private static PurlType findPurlType(String type) { + /** + * Determine if we have a supported PURL Type or not + * + * @param type PURL type string + * @return Supported PURL Type + * @throws IllegalArgumentException if type cannot be found on supported list + */ + private static PurlType findPurlType(@NonNull String type) throws IllegalArgumentException { for (PurlType purlType : PurlType.values()) { if (purlType.getType().equals(type)) { return purlType; } } - throw new IllegalArgumentException( - String.format("Unsupported package type: %s", type) - ); + throw new IllegalArgumentException(String.format("Unsupported package type: %s", type)); } } \ No newline at end of file From fd6dd0a09ef820f99bf2f763d3a72a33614f444b Mon Sep 17 00:00:00 2001 From: eeisegn Date: Mon, 30 Dec 2024 16:06:34 +0000 Subject: [PATCH 19/28] add nonnull check and javadoc --- .../com/scanoss/settings/RuleComparator.java | 26 ++++++++++++++++++- .../java/com/scanoss/settings/Settings.java | 8 +++--- .../com/scanoss/utils/LineRangeUtils.java | 6 ++--- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/scanoss/settings/RuleComparator.java b/src/main/java/com/scanoss/settings/RuleComparator.java index d9a7a60..a45cb9b 100644 --- a/src/main/java/com/scanoss/settings/RuleComparator.java +++ b/src/main/java/com/scanoss/settings/RuleComparator.java @@ -1,5 +1,29 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2024, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package com.scanoss.settings; +import lombok.NonNull; + import java.util.Comparator; @@ -40,7 +64,7 @@ public class RuleComparator implements Comparator { * equal to, or greater than the second rule respectively */ @Override - public int compare(Rule r1, Rule r2) { + public int compare(@NonNull Rule r1, @NonNull Rule r2) { int score1 = calculatePriorityScore(r1); int score2 = calculatePriorityScore(r2); diff --git a/src/main/java/com/scanoss/settings/Settings.java b/src/main/java/com/scanoss/settings/Settings.java index 99ff99a..c8ca386 100644 --- a/src/main/java/com/scanoss/settings/Settings.java +++ b/src/main/java/com/scanoss/settings/Settings.java @@ -25,15 +25,14 @@ import com.google.gson.Gson; import lombok.Builder; import lombok.Data; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; - /** * Represents the SCANOSS scanner settings configuration. * Provides functionality to load and manage scanner settings from JSON files or strings. @@ -43,7 +42,6 @@ @Data @Builder public class Settings { - /** * The Bill of Materials (BOM) configuration containing rules for component handling. * Includes rules for including, ignoring, removing, and replacing components @@ -57,7 +55,7 @@ public class Settings { * @param json The JSON string to parse * @return A new Settings object */ - public static Settings createFromJsonString(@NotNull String json) { + public static Settings createFromJsonString(@NonNull String json) { Gson gson = new Gson(); return gson.fromJson(json, Settings.class); } @@ -68,7 +66,7 @@ public static Settings createFromJsonString(@NotNull String json) { * @param path The path to the JSON file * @return A new Settings object */ - public static Settings createFromPath(@NotNull Path path) { + public static Settings createFromPath(@NonNull Path path) { try { String json = Files.readString(path, StandardCharsets.UTF_8); return createFromJsonString(json); diff --git a/src/main/java/com/scanoss/utils/LineRangeUtils.java b/src/main/java/com/scanoss/utils/LineRangeUtils.java index c0eb5bc..c945dcd 100644 --- a/src/main/java/com/scanoss/utils/LineRangeUtils.java +++ b/src/main/java/com/scanoss/utils/LineRangeUtils.java @@ -23,8 +23,8 @@ package com.scanoss.utils; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; import java.util.ArrayList; import java.util.Collections; @@ -73,7 +73,7 @@ public static List parseLineRanges(String lineRanges) { * @param ranges2 Second set of line ranges * @return true if any intervals overlap */ - public static boolean hasOverlappingRanges(@NotNull List ranges1, @NotNull List ranges2) { + public static boolean hasOverlappingRanges(@NonNull List ranges1, @NonNull List ranges2) { for (LineRange interval1 : ranges1) { for (LineRange interval2 : ranges2) { if (interval1.overlaps(interval2)) { @@ -92,7 +92,7 @@ public static boolean hasOverlappingRanges(@NotNull List ranges1, @No * @return true if any interval from the list overlaps with the given range * @throws NullPointerException if either parameter is null */ - public static boolean hasOverlappingRanges(@NotNull List ranges, @NotNull LineRange range) { + public static boolean hasOverlappingRanges(@NonNull List ranges, @NonNull LineRange range) { for (LineRange interval1 : ranges) { if (interval1.overlaps(range)) { return true; From 9b0b55dd58fcf6d00cc2c940813e8e4a67c3e2b0 Mon Sep 17 00:00:00 2001 From: eeisegn Date: Mon, 30 Dec 2024 16:07:00 +0000 Subject: [PATCH 20/28] added size getters --- src/main/java/com/scanoss/settings/Bom.java | 25 +++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/scanoss/settings/Bom.java b/src/main/java/com/scanoss/settings/Bom.java index d98615c..2b13f6c 100644 --- a/src/main/java/com/scanoss/settings/Bom.java +++ b/src/main/java/com/scanoss/settings/Bom.java @@ -38,27 +38,23 @@ @Data @Builder public class Bom { - /** * List of include rules for adding context when scanning. * These rules are sent to the SCANOSS API and have a higher chance of being * considered part of the resulting scan. */ private final @Singular("include") List include; - /** * List of ignore rules for excluding certain components . * These rules are sent to the SCANOSS API. */ private final @Singular("ignore") List ignore; - /** * List of remove rules for excluding components from results after scanning. * These rules are applied to the results file after scanning and are processed * on the client side. */ private final @Singular("remove") List remove; - /** * List of replace rules for substituting components after scanning. * These rules are applied to the results file after scanning and are processed @@ -67,7 +63,6 @@ public class Bom { */ private final @Singular("replace") List replace; - /** * Cached list of replace rules sorted by priority. * This list is lazily initialized when first accessed through @@ -77,14 +72,13 @@ public class Bom { */ private final List sortedReplace; - /** * Sorts replace rules by priority from highest to lowest: * 1. Rules with both purl/path (most specific) * 3. Rules with only purl * 4. Rules with only path (least specific) * - * @return A new list containing the sorted replace rules + * @return A new list containing the sorted replacement rules */ public List getReplaceRulesByPriority() { if (sortedReplace == null) { @@ -95,5 +89,22 @@ public List getReplaceRulesByPriority() { return sortedReplace; } + /** + * Get the size of the Remove rules + * + * @return size of remove list or zero + */ + public int getRemoveSize() { + return remove != null ? remove.size() : 0; + } + + /** + * Get the size of the Replace rules + * + * @return size of replace rules or zero + */ + public int getReplaceSize() { + return replace != null ? replace.size() : 0; + } } From b701d1323a020802654fa0149fa466632fb323b1 Mon Sep 17 00:00:00 2001 From: eeisegn Date: Mon, 30 Dec 2024 16:07:18 +0000 Subject: [PATCH 21/28] dependency updates --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 625be9c..f042d16 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ 11 11 UTF-8 - 2.0.7 + 2.0.16 0.9.13 com.scanoss.cli.CommandLine @@ -93,7 +93,7 @@ commons-codec commons-codec - 1.16.0 + 1.17.1 org.slf4j From 6a38521d28c388516c7c47381a7f9849e5053a0a Mon Sep 17 00:00:00 2001 From: eeisegn Date: Mon, 30 Dec 2024 16:22:31 +0000 Subject: [PATCH 22/28] formatting and comments --- src/main/java/com/scanoss/dto/ScanFileResult.java | 1 - src/main/java/com/scanoss/utils/LineRangeUtils.java | 7 ++----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/scanoss/dto/ScanFileResult.java b/src/main/java/com/scanoss/dto/ScanFileResult.java index 814eb7f..5783873 100644 --- a/src/main/java/com/scanoss/dto/ScanFileResult.java +++ b/src/main/java/com/scanoss/dto/ScanFileResult.java @@ -34,4 +34,3 @@ public class ScanFileResult { private final String filePath; private final List fileDetails; } - diff --git a/src/main/java/com/scanoss/utils/LineRangeUtils.java b/src/main/java/com/scanoss/utils/LineRangeUtils.java index c945dcd..6ce5e07 100644 --- a/src/main/java/com/scanoss/utils/LineRangeUtils.java +++ b/src/main/java/com/scanoss/utils/LineRangeUtils.java @@ -22,7 +22,6 @@ */ package com.scanoss.utils; - import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -45,10 +44,8 @@ public static List parseLineRanges(String lineRanges) { if (lineRanges == null || lineRanges.trim().isEmpty()) { return Collections.emptyList(); } - String[] ranges = lineRanges.split(","); List intervals = new ArrayList<>(ranges.length); - for (String range : ranges) { String[] bounds = range.trim().split("-"); if (bounds.length == 2) { @@ -58,11 +55,10 @@ public static List parseLineRanges(String lineRanges) { intervals.add(new LineRange(start, end)); } catch (NumberFormatException e) { // Skip invalid intervals - log.debug("Invalid interval format: {} in range {}", range, e.getMessage()); + log.debug("Skipping invalid interval format: {} in range ({}): {}", range, bounds, e.getMessage()); } } } - return intervals; } @@ -74,6 +70,7 @@ public static List parseLineRanges(String lineRanges) { * @return true if any intervals overlap */ public static boolean hasOverlappingRanges(@NonNull List ranges1, @NonNull List ranges2) { + // TODO is this required. It only seems to be used in tests? for (LineRange interval1 : ranges1) { for (LineRange interval2 : ranges2) { if (interval1.overlaps(interval2)) { From 54a3bf721a9a35a15dc52e4ca229a17bf9a0e4d6 Mon Sep 17 00:00:00 2001 From: eeisegn Date: Mon, 30 Dec 2024 17:10:20 +0000 Subject: [PATCH 23/28] adding TODOs --- src/main/java/com/scanoss/Scanner.java | 1 + src/main/java/com/scanoss/utils/LineRangeUtils.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/main/java/com/scanoss/Scanner.java b/src/main/java/com/scanoss/Scanner.java index 3686311..66038e9 100644 --- a/src/main/java/com/scanoss/Scanner.java +++ b/src/main/java/com/scanoss/Scanner.java @@ -411,6 +411,7 @@ public List scanFolder(@NonNull String folder) { * @param files list of files to scan * @return List of scan result strings (in JSON format) */ + //TODO: Include postProcessing stage public List scanFileList(@NonNull String folder, @NonNull List files) { return processFileList(folder, files, scanFileProcessor); } diff --git a/src/main/java/com/scanoss/utils/LineRangeUtils.java b/src/main/java/com/scanoss/utils/LineRangeUtils.java index 6ce5e07..57cdeb4 100644 --- a/src/main/java/com/scanoss/utils/LineRangeUtils.java +++ b/src/main/java/com/scanoss/utils/LineRangeUtils.java @@ -90,6 +90,7 @@ public static boolean hasOverlappingRanges(@NonNull List ranges1, @No * @throws NullPointerException if either parameter is null */ public static boolean hasOverlappingRanges(@NonNull List ranges, @NonNull LineRange range) { + // TODO add test case for (LineRange interval1 : ranges) { if (interval1.overlaps(range)) { return true; From e297bac8a3c5e9b86cdf33c38f580ede9c096eb1 Mon Sep 17 00:00:00 2001 From: eeisegn Date: Mon, 30 Dec 2024 17:12:16 +0000 Subject: [PATCH 24/28] simplification --- .../com/scanoss/ScannerPostProcessor.java | 266 ++++++++---------- 1 file changed, 120 insertions(+), 146 deletions(-) diff --git a/src/main/java/com/scanoss/ScannerPostProcessor.java b/src/main/java/com/scanoss/ScannerPostProcessor.java index ac2ac56..860cf48 100644 --- a/src/main/java/com/scanoss/ScannerPostProcessor.java +++ b/src/main/java/com/scanoss/ScannerPostProcessor.java @@ -33,6 +33,7 @@ import com.scanoss.utils.LineRangeUtils; import com.scanoss.utils.Purl2Url; import lombok.Builder; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -56,7 +57,6 @@ @Slf4j @Builder public class ScannerPostProcessor { - /** * Maps purl to Component (ScanFileDetail) */ @@ -70,28 +70,22 @@ public class ScannerPostProcessor { * @param bom Bom containing BOM rules * @return List of processed scan results */ - public List process(@NotNull List scanFileResults, @NotNull Bom bom) { - List processedResults = new ArrayList<>(scanFileResults); - + public List process(@NonNull List scanFileResults, @NonNull Bom bom) { + int removeSize = bom.getRemoveSize(); + int replaceSize = bom.getReplaceSize(); log.info("Starting scan results processing with {} results", scanFileResults.size()); - log.debug("BOM configuration - Remove rules: {}, Replace rules: {}", - bom.getRemove() != null ? bom.getRemove().size() : 0, - bom.getReplace() != null ? bom.getReplace().size() : 0); - + log.debug("BOM configuration - Remove rules: {}, Replace rules: {}", removeSize, replaceSize); - purl2ComponentDetailsMap = buildPurl2ComponentDetailsMap(scanFileResults); - - - if (bom.getRemove() != null && !bom.getRemove().isEmpty()) { - log.info("Applying {} remove rules to scan results", bom.getRemove().size()); - processedResults = applyRemoveRules(processedResults, bom.getRemove()); + buildPurl2ComponentDetailsMap(scanFileResults); + List processedResults = new ArrayList<>(scanFileResults); + if (removeSize > 0) { + log.info("Applying {} remove rules to scan results", removeSize); + applyRemoveRules(processedResults, bom.getRemove()); } - - if (bom.getReplace() != null && !bom.getReplace().isEmpty()) { - log.info("Applying {} replace rules to scan results", bom.getReplace().size()); - processedResults = applyReplaceRules(processedResults, bom.getReplaceRulesByPriority()); + if (replaceSize > 0) { + log.info("Applying {} replace rules to scan results", replaceSize); + applyReplaceRules(processedResults, bom.getReplaceRulesByPriority()); } - log.info("Scan results processing completed. Original results: {}, Processed results: {}", scanFileResults.size(), processedResults.size()); return processedResults; @@ -102,41 +96,38 @@ public List process(@NotNull List scanFileResult * This map enables efficient component lookup during the replacement process. * * @param scanFileResults List of scan results to process - * @return Map where keys are PURLs and values are their associated component details */ - private Map buildPurl2ComponentDetailsMap(@NotNull List scanFileResults) { + private void buildPurl2ComponentDetailsMap(@NonNull List scanFileResults) { log.debug("Creating Purl Component Map from scan results"); - - Map index = new HashMap<>(); - + purl2ComponentDetailsMap = new HashMap<>(); for (ScanFileResult result : scanFileResults) { - if (result == null || result.getFileDetails() == null) { + List fileDetails = result != null ? result.getFileDetails() : null; + if (fileDetails == null) { + log.warn("Null result or empty scan file result. Skipping: {}", result); continue; } - // Iterate through file details - for (ScanFileDetails details : result.getFileDetails()) { - if (details == null || details.getPurls() == null) { + for (ScanFileDetails details : fileDetails) { + String[] purls = details != null ? details.getPurls() : null; + if (purls == null) { + log.warn("Null details or empty scan file result details. Skipping: {}", details); continue; } - // Iterate through purls for each detail - for (String purl : details.getPurls()) { - if (purl == null || purl.trim().isEmpty()) { + for (String purl : purls) { + String trimmedPurl = purl != null ? purl.trim() : ""; + if (trimmedPurl.isEmpty()) { + log.warn("Empty purl details found. Skipping: {}", details); continue; } - // Only store if purl not already in map - String trimmedPurl = purl.trim(); - if (!index.containsKey(trimmedPurl)) { - index.put(trimmedPurl, details); + if (!purl2ComponentDetailsMap.containsKey(trimmedPurl)) { + purl2ComponentDetailsMap.put(trimmedPurl, details); } } } } - - log.debug("Purl Component Map created with {} entries", index.size()); - return index; + log.debug("Purl Component Map created with {} entries", purl2ComponentDetailsMap.size()); } @@ -148,14 +139,12 @@ private Map buildPurl2ComponentDetailsMap(@NotNull List * @param rules The list of replacement rules to apply * @return The modified input list of scan results with updated PURLs */ - private List applyReplaceRules(@NotNull List results, @NotNull List rules) { + private void applyReplaceRules(@NonNull List results, @NonNull List rules) { log.debug("Starting replace rules application for {} results with {} rules", results.size(), rules.size()); results.forEach(result -> applyReplaceRulesOnResult(result, rules)); - return results; } - /** * Processes a single scan result against all replacement rules. * Applies the first matching rule found to update the result's package information. @@ -163,13 +152,12 @@ private List applyReplaceRules(@NotNull List res * @param result The scan result to process * @param rules List of replacement rules to check against */ - private void applyReplaceRulesOnResult(@NotNull ScanFileResult result, @NotNull List rules) { - if (!isValidScanResult(result)) { - log.warn("Invalid scan result structure for file: {}", result.getFilePath()); + private void applyReplaceRulesOnResult(@NonNull ScanFileResult result, @NonNull List rules) { + // Check if this result is processable + if (isInvalidScanResult(result)) { + log.warn("Invalid scan result structure. Cannot apply replace rules for file: {}", result); return; } - - // Find the first matching rule and apply its replacement // This ensures only one rule is applied per result, maintaining consistency rules.stream() @@ -186,44 +174,42 @@ private void applyReplaceRulesOnResult(@NotNull ScanFileResult result, @NotNull * @param result The scan result to update * @param rule The replacement rule containing the new package URL */ - private void updateResultWithReplaceRule(@NotNull ScanFileResult result, @NotNull ReplaceRule rule) { + private void updateResultWithReplaceRule(@NonNull ScanFileResult result, @NonNull ReplaceRule rule) { PackageURL newPurl = createPackageUrl(rule); - if (newPurl == null) return; - - + if (newPurl == null) + return; List componentDetails = result.getFileDetails(); + if (componentDetails == null) { + log.warn("Null scan file details found. Skipping: {}", result); + return; + } for (ScanFileDetails componentDetail : componentDetails ) { - if (componentDetail == null) { + log.warn("Null scan file component details found. Skipping: {}", result); continue; } - - ScanFileDetails newFileDetails = createUpdatedResultDetails(componentDetail, newPurl, rule); + ScanFileDetails newFileDetails = createUpdatedResultDetails(componentDetail, newPurl); result.getFileDetails().set(0, newFileDetails); - log.debug("Updated package URL from {} to {} for file: {}", componentDetail.getPurls()[0], newPurl, result.getFilePath()); } - - } - /** * Creates a PackageURL object from a replacement rule's target URL. * * @param rule The replacement rule containing the new package URL - * @return A new PackageURL object, or null if the URL is malformed + * @return A new PackageURL object, or null if the URL is malformed */ - private PackageURL createPackageUrl(ReplaceRule rule) { + private PackageURL createPackageUrl(@NonNull ReplaceRule rule) { try { return new PackageURL(rule.getReplaceWith()); } catch (MalformedPackageURLException e) { log.warn("Failed to parse PURL from replace rule: {}. Skipping", rule); - return null; } + return null; } @@ -234,67 +220,40 @@ private PackageURL createPackageUrl(ReplaceRule rule) { * in the replacement rule. * * @param existingComponent The current component details to use as a base - * @param newPackageUrl The new package URL containing updated package information - * @param replacementRule The rule to extract the license, if exist + * @param newPurl The new package URL containing updated package information * @return Updated component details with specific fields overridden */ private ScanFileDetails createUpdatedResultDetails(ScanFileDetails existingComponent, - PackageURL newPackageUrl, - @NotNull ReplaceRule replacementRule) { - - - // Check if we already have processed this package URL - ScanFileDetails cachedComponent = purl2ComponentDetailsMap.get(newPackageUrl.toString()); - - // Extract license information from the replacement rule - String definedLicenseName = replacementRule.getLicense(); - LicenseDetails newLicense = definedLicenseName != null - ? LicenseDetails.builder().name(definedLicenseName).build() - : null; - - if (cachedComponent != null) { - // Update existing component with cached information - return existingComponent.toBuilder() - .copyrightDetails(null) - .licenseDetails(definedLicenseName != null - ? new LicenseDetails[]{ newLicense } - : cachedComponent.getLicenseDetails()) - .version(newPackageUrl.getVersion() != null - ? newPackageUrl.getVersion() - : existingComponent.getVersion()) - .purls(new String[]{ newPackageUrl.toString() }) - .component(newPackageUrl.getName()) - .vendor(newPackageUrl.getNamespace()) + PackageURL newPurl) { + // Check for cached component + ScanFileDetails cached = purl2ComponentDetailsMap.get(newPurl.toString()); + + if (cached != null) { + //TODO: Clarification on copyright, Vulns, etc + //currentComponent.toBuilder().component().vendor().purls().licenseDetails() + //Version if we have a package url with version + //pkg:github/scanoss@1.0.0 + return cached.toBuilder() + .file(existingComponent.getFile()) + .fileHash(existingComponent.getFileHash()) + .fileUrl(existingComponent.getFileUrl()) + .purls(new String[]{newPurl.toString()}) + .component(newPurl.getName()) + .vendor(newPurl.getNamespace()) .build(); } - - // Create new component when no cached version exists - return ScanFileDetails.builder() - .licenseDetails(newLicense != null - ? new LicenseDetails[]{ newLicense } - : null) - .purls(new String[]{ newPackageUrl.toString() }) - .url(Purl2Url.convert(newPackageUrl)) - .version(determineVersion(newPackageUrl, existingComponent)) - .component(newPackageUrl.getName()) - .vendor(newPackageUrl.getNamespace()) + // If no cached info, create minimal version + return existingComponent.toBuilder() + .copyrightDetails(new CopyrightDetails[]{}) //TODO: Check if we need the empty Object + .licenseDetails(new LicenseDetails[]{}) + .vulnerabilityDetails(new VulnerabilityDetails[]{}) + .purls(new String[]{newPurl.toString()}) + .url("") // TODO: Implement purl2Url in PackageURL upstream library + .component(newPurl.getName()) + .vendor(newPurl.getNamespace()) .build(); } - /** - * Determines the version to use by checking if the new PURL has a version specified. - * If the new PURL has no version, falls back to the existing component's version. - * - * @param newPurl The new PURL containing potential version information - * @param existingComponent The existing component with fallback version information - * @return The determined version string - */ - private String determineVersion(PackageURL newPurl, ScanFileDetails existingComponent) { - return newPurl.getVersion() != null - ? newPurl.getVersion() - : existingComponent.getVersion(); - } - /** * Applies remove rules to scan results, filtering out matches based on certain criteria. @@ -308,17 +267,13 @@ private String determineVersion(PackageURL newPurl, ScanFileDetails existingComp * * @param results The list of scan results to process * @param rules The list of remove rules to apply - * @return A filtered list with matching results removed based on the above criteria */ - public List applyRemoveRules(@NotNull List results, @NotNull List rules) { + public void applyRemoveRules(@NonNull List results, @NonNull List rules) { log.debug("Starting remove rules application to {} results", results.size()); - List resultsList = new ArrayList<>(results); - resultsList.removeIf(result -> matchesRemovalCriteria(result, rules)); - log.debug("Remove rules application completed. Results remaining: {}", resultsList.size()); - return resultsList; + results.removeIf(result -> matchesRemovalCriteria(result, rules)); + log.debug("Remove rules application completed. Results remaining: {}", results.size()); } - /** * Determines if a scan result should be excluded based on the removal rules. *

@@ -332,13 +287,13 @@ public List applyRemoveRules(@NotNull List resul * @param rules List of removal rules to check against * @return true if the result should be removed, false otherwise */ - private Boolean matchesRemovalCriteria(@NotNull ScanFileResult result, @NotNull List rules) { - - if (!isValidScanResult(result)) { + private Boolean matchesRemovalCriteria(@NonNull ScanFileResult result, @NonNull List rules) { + // Make sure it's a valid result before processing + if (isInvalidScanResult(result)) { log.warn("Invalid scan result structure for file: {}", result.getFilePath()); return false; } - + // TODO. Should only apply the first matching rule also. No need to iterate over them all return rules.stream() .filter(rule -> isMatchingRule(result, rule)) .anyMatch(rule -> { @@ -347,25 +302,23 @@ private Boolean matchesRemovalCriteria(@NotNull ScanFileResult result, @NotNull // - returns true if rule has line range AND result overlaps with it // - returns false otherwise (continue checking remaining rules) boolean ruleHasLineRange = rule.getStartLine() != null && rule.getEndLine() != null; - return !ruleHasLineRange || isLineRangeMatch(rule, result); + return !ruleHasLineRange || isRemoveLineRangeMatch(rule, result); }); } - /** * Checks if a scan result matches the path and/or PURL patterns defined in a rule. - * * The match is considered successful if any of these conditions are met: - * 1. Rule has both path and PURL defined: Both must match the result - * 2. Rule has only PURL defined: PURL must match the result - * 3. Rule has only path defined: Path must match the result + * 1. Rule has both a path and PURL defined: Both must match the result + * 2. Rule has only a PURL defined: PURL must match the result + * 3. Rule has only a path defined: Path must match the result * * @param result The scan result to check * @param rule The rule containing the patterns to match against * @param Type parameter extending Rule class * @return true if the result matches the rule's patterns according to above conditions */ - private Boolean isMatchingRule(@NotNull ScanFileResult result, @NotNull T rule) { + private Boolean isMatchingRule(@NonNull ScanFileResult result, @NonNull T rule) { // Check if rule has valid path and/or PURL patterns boolean hasPath = rule.getPath() != null && !rule.getPath().isEmpty(); boolean hasPurl = rule.getPurl() != null && !rule.getPurl().isEmpty(); @@ -374,22 +327,26 @@ private Boolean isMatchingRule(@NotNull ScanFileResult result, // 1. Both path and PURL match if (hasPath && hasPurl && isPathAndPurlMatch(rule, result)) { return true; + } // 2. Only PURL match required and matches - } else if (hasPurl && isPurlOnlyMatch(rule, result)) { + if (hasPurl && isPurlOnlyMatch(rule, result)) { return true; + } // 3. Only path match required and matches - } if (hasPath && isPathOnlyMatch(rule, result)) { + if (hasPath && isPathOnlyMatch(rule, result)) { return true; } - return false; } - /** - * Checks if line range of the remove rule match the result + * Checks if the line range of the remove rule match the result + * + * @param rule Remove Rule + * @param result Scan file Result + * @return true if remove rule is in range, false otherwise */ - private boolean isLineRangeMatch(RemoveRule rule, ScanFileResult result) { + private boolean isRemoveLineRangeMatch(@NonNull RemoveRule rule, @NonNull ScanFileResult result) { LineRange ruleLineRange = new LineRange(rule.getStartLine(), rule.getEndLine()); String lines = result.getFileDetails().get(0).getLines(); @@ -400,8 +357,13 @@ private boolean isLineRangeMatch(RemoveRule rule, ScanFileResult result) { /** * Checks if both path and purl of the rule match the result + * + * @param rule BOM Rule + * @param result Scan file Result + * @return true if it matches, false otherwise */ - private boolean isPathAndPurlMatch(Rule rule, ScanFileResult result) { + private boolean isPathAndPurlMatch(@NonNull Rule rule, @NonNull ScanFileResult result) { + // TODO. Path needs to 'startWith', it does not need to be an exact match return Objects.equals(rule.getPath(), result.getFilePath()) && isPurlMatch(rule.getPurl(), result.getFileDetails().get(0).getPurls()); } @@ -409,26 +371,39 @@ private boolean isPathAndPurlMatch(Rule rule, ScanFileResult result) { /** * Checks if the rule's path matches the result (ignoring purl) + * + * @param rule BOM Rule + * @param result Scan file Results + * @return true if it matches, false otherwise */ - private boolean isPathOnlyMatch(Rule rule, ScanFileResult result) { + private boolean isPathOnlyMatch(@NonNull Rule rule, @NonNull ScanFileResult result) { + // TODO. Path needs to 'startWith', it does not need to be an exact match return Objects.equals(rule.getPath(), result.getFilePath()); } /** * Checks if the rule's purl matches the result (ignoring path) + * + * @param rule BOM Rule + * @param result Scan file Result + * @return true if it matches, false otherwise */ - private boolean isPurlOnlyMatch(Rule rule, ScanFileResult result) { + private boolean isPurlOnlyMatch(@NonNull Rule rule, @NonNull ScanFileResult result) { + // TODO what happens if there is no purl in the result tree? return isPurlMatch(rule.getPurl(), result.getFileDetails().get(0).getPurls()); } /** * Checks if a specific purl exists in an array of purls + * + * @param rulePurl PURL from Rule + * @param resultPurls List of Scan file Result PURLs + * @return true if it matches, false otherwise */ private boolean isPurlMatch(String rulePurl, String[] resultPurls) { if (rulePurl == null || resultPurls == null) { return false; } - for (String resultPurl : resultPurls) { if (Objects.equals(rulePurl, resultPurl)) { return true; @@ -441,11 +416,10 @@ private boolean isPurlMatch(String rulePurl, String[] resultPurls) { * Validates if a scan result has the required fields for rule processing. * * @param result The scan result to validate - * @return true if the result has valid file details and PURLs + * @return true if the result has invalid file details or PURLs */ - private boolean isValidScanResult(@NotNull ScanFileResult result) { - return result.getFileDetails() != null - && !result.getFileDetails().isEmpty() - && result.getFileDetails().get(0) != null; + private boolean isInvalidScanResult(@NonNull ScanFileResult result) { + List details = result.getFileDetails(); + return details == null || details.isEmpty() || details.get(0) == null; } } From ef4d095f428fead129530e222cc871c2c6661212 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 3 Jan 2025 19:07:02 +0100 Subject: [PATCH 25/28] chore: adds post processing stage to sdk --- src/main/java/com/scanoss/Scanner.java | 34 ++++++-- .../com/scanoss/ScannerPostProcessor.java | 77 +++++++++++++++---- src/main/java/com/scanoss/Winnowing.java | 27 ------- .../java/com/scanoss/cli/ScanCommandLine.java | 17 +--- .../java/com/scanoss/utils/JsonUtils.java | 23 ++++++ src/test/java/com/scanoss/TestJsonUtils.java | 48 ++++++++++++ .../java/com/scanoss/TestLineRangeUtils.java | 3 + .../com/scanoss/TestScannerPostProcessor.java | 4 +- 8 files changed, 169 insertions(+), 64 deletions(-) diff --git a/src/main/java/com/scanoss/Scanner.java b/src/main/java/com/scanoss/Scanner.java index 66038e9..b879db3 100644 --- a/src/main/java/com/scanoss/Scanner.java +++ b/src/main/java/com/scanoss/Scanner.java @@ -22,12 +22,15 @@ */ package com.scanoss; +import com.scanoss.dto.ScanFileResult; import com.scanoss.exceptions.ScannerException; import com.scanoss.exceptions.WinnowingException; import com.scanoss.processor.FileProcessor; import com.scanoss.processor.ScanFileProcessor; import com.scanoss.processor.WfpFileProcessor; import com.scanoss.rest.ScanApi; +import com.scanoss.settings.Settings; +import com.scanoss.utils.JsonUtils; import lombok.Builder; import lombok.Getter; import lombok.NonNull; @@ -89,6 +92,8 @@ public class Scanner { private ScanApi scanApi; private ScanFileProcessor scanFileProcessor; private WfpFileProcessor wfpFileProcessor; + private Settings settings; + private ScannerPostProcessor postProcessor; @SuppressWarnings("unused") private Scanner(Boolean skipSnippets, Boolean allExtensions, Boolean obfuscate, Boolean hpsm, @@ -96,7 +101,8 @@ private Scanner(Boolean skipSnippets, Boolean allExtensions, Boolean obfuscate, Integer retryLimit, String url, String apiKey, String scanFlags, String sbomType, String sbom, Integer snippetLimit, String customCert, Proxy proxy, Winnowing winnowing, ScanApi scanApi, - ScanFileProcessor scanFileProcessor, WfpFileProcessor wfpFileProcessor + ScanFileProcessor scanFileProcessor, WfpFileProcessor wfpFileProcessor, + Settings settings, ScannerPostProcessor postProcessor ) { this.skipSnippets = skipSnippets; this.allExtensions = allExtensions; @@ -128,7 +134,9 @@ private Scanner(Boolean skipSnippets, Boolean allExtensions, Boolean obfuscate, this.wfpFileProcessor = Objects.requireNonNullElseGet(wfpFileProcessor, () -> WfpFileProcessor.builder() .winnowing(this.winnowing) .build()); - } + this.settings = Objects.requireNonNullElseGet(settings, () -> Settings.builder().build()); + this.postProcessor = Objects.requireNonNullElseGet(postProcessor, () -> + ScannerPostProcessor.builder().build()); } /** * Generate a WFP/Fingerprint for the given file @@ -399,9 +407,9 @@ public String scanFile(@NonNull String filename) throws ScannerException, Winnow * @param folder folder to scan * @return List of scan result strings (in JSON format) */ - //TODO: Include postProcessing stage public List scanFolder(@NonNull String folder) { - return processFolder(folder, scanFileProcessor); + List results = processFolder(folder, scanFileProcessor); + return postProcessResults(results); } /** @@ -411,9 +419,23 @@ public List scanFolder(@NonNull String folder) { * @param files list of files to scan * @return List of scan result strings (in JSON format) */ - //TODO: Include postProcessing stage public List scanFileList(@NonNull String folder, @NonNull List files) { - return processFileList(folder, files, scanFileProcessor); + List results = processFileList(folder, files, scanFileProcessor); + return postProcessResults(results); + } + + /** + * Post-processes scan results based on BOM (Bill of Materials) settings if available. + * @param results List of raw scan results in JSON string format + * @return Processed results, either modified based on BOM or original results if no BOM exists + */ + private List postProcessResults(List results) { + if (settings.getBom() != null) { + List scanFileResults = JsonUtils.toScanFileResults(results); + List newScanFileResults = this.postProcessor.process(scanFileResults, this.settings.getBom()); + return JsonUtils.toRawJsonString(newScanFileResults); + } + return results; } } \ No newline at end of file diff --git a/src/main/java/com/scanoss/ScannerPostProcessor.java b/src/main/java/com/scanoss/ScannerPostProcessor.java index 860cf48..4b672ed 100644 --- a/src/main/java/com/scanoss/ScannerPostProcessor.java +++ b/src/main/java/com/scanoss/ScannerPostProcessor.java @@ -25,6 +25,7 @@ import com.github.packageurl.MalformedPackageURLException; import com.github.packageurl.PackageURL; import com.scanoss.dto.*; +import com.scanoss.dto.enums.MatchType; import com.scanoss.settings.Bom; import com.scanoss.settings.RemoveRule; import com.scanoss.settings.ReplaceRule; @@ -153,9 +154,14 @@ private void applyReplaceRules(@NonNull List results, @NonNull L * @param rules List of replacement rules to check against */ private void applyReplaceRulesOnResult(@NonNull ScanFileResult result, @NonNull List rules) { - // Check if this result is processable - if (isInvalidScanResult(result)) { - log.warn("Invalid scan result structure. Cannot apply replace rules for file: {}", result); + // Make sure it's a valid result before processing + if (hasInvalidStructure(result)) { + log.warn("Scan result has invalid structure - missing required fields for file: {}", result.getFilePath()); + return; + } + + if (hasNoValidMatch(result)) { + log.debug("Scan result has no valid matches for file: {}", result.getFilePath()); return; } // Find the first matching rule and apply its replacement @@ -248,7 +254,7 @@ private ScanFileDetails createUpdatedResultDetails(ScanFileDetails existingCompo .licenseDetails(new LicenseDetails[]{}) .vulnerabilityDetails(new VulnerabilityDetails[]{}) .purls(new String[]{newPurl.toString()}) - .url("") // TODO: Implement purl2Url in PackageURL upstream library + .url(Purl2Url.isSupported(newPurl) ? Purl2Url.convert(newPurl) : "") .component(newPurl.getName()) .vendor(newPurl.getNamespace()) .build(); @@ -289,11 +295,16 @@ public void applyRemoveRules(@NonNull List results, @NonNull Lis */ private Boolean matchesRemovalCriteria(@NonNull ScanFileResult result, @NonNull List rules) { // Make sure it's a valid result before processing - if (isInvalidScanResult(result)) { - log.warn("Invalid scan result structure for file: {}", result.getFilePath()); + if (hasInvalidStructure(result)) { + log.warn("Scan result has invalid structure - missing required fields for file: {}", result.getFilePath()); + return false; + } + + if (hasNoValidMatch(result)) { + log.debug("Scan result has no valid matches for file: {}", result.getFilePath()); return false; } - // TODO. Should only apply the first matching rule also. No need to iterate over them all + return rules.stream() .filter(rule -> isMatchingRule(result, rule)) .anyMatch(rule -> { @@ -363,8 +374,7 @@ private boolean isRemoveLineRangeMatch(@NonNull RemoveRule rule, @NonNull ScanFi * @return true if it matches, false otherwise */ private boolean isPathAndPurlMatch(@NonNull Rule rule, @NonNull ScanFileResult result) { - // TODO. Path needs to 'startWith', it does not need to be an exact match - return Objects.equals(rule.getPath(), result.getFilePath()) && + return result.getFilePath().startsWith(rule.getPath()) && isPurlMatch(rule.getPurl(), result.getFileDetails().get(0).getPurls()); } @@ -377,8 +387,7 @@ private boolean isPathAndPurlMatch(@NonNull Rule rule, @NonNull ScanFileResult r * @return true if it matches, false otherwise */ private boolean isPathOnlyMatch(@NonNull Rule rule, @NonNull ScanFileResult result) { - // TODO. Path needs to 'startWith', it does not need to be an exact match - return Objects.equals(rule.getPath(), result.getFilePath()); + return result.getFilePath().startsWith(rule.getPath()); } /** @@ -389,7 +398,8 @@ private boolean isPathOnlyMatch(@NonNull Rule rule, @NonNull ScanFileResult resu * @return true if it matches, false otherwise */ private boolean isPurlOnlyMatch(@NonNull Rule rule, @NonNull ScanFileResult result) { - // TODO what happens if there is no purl in the result tree? + // TODO what happens if there is no purl in the result tree? (DONE) + // I won't happen since the invalid results are skipped upstream within isInvalidScanResult() return isPurlMatch(rule.getPurl(), result.getFileDetails().get(0).getPurls()); } @@ -413,13 +423,48 @@ private boolean isPurlMatch(String rulePurl, String[] resultPurls) { } /** - * Validates if a scan result has the required fields for rule processing. + * Checks if a scan result contains the minimum required data structure for processing. + * This validation ensures that: + * 1. The result has a valid file path identifier + * 2. The result contains a non-empty list of scan details + * 3. The primary scan detail entry (first in the list) exists + * + * This structural validation is a prerequisite for any further processing of scan results, + * such as match analysis or rule processing. Without these basic elements, the scan result + * cannot be meaningfully processed. * * @param result The scan result to validate - * @return true if the result has invalid file details or PURLs + * @return true if the basic structure is invalid, false if valid */ - private boolean isInvalidScanResult(@NonNull ScanFileResult result) { + private boolean hasInvalidStructure(@NonNull ScanFileResult result) { + String filepath = result.getFilePath(); List details = result.getFileDetails(); - return details == null || details.isEmpty() || details.get(0) == null; + return filepath == null || + details == null || + details.isEmpty() || + details.get(0) == null; } + + /** + * Checks if the scan result has a valid match. + * A result is considered to have no valid match if: + * - Match type is 'none' + * - Lines array is null + * - PURLs array is null or empty + * + * @param result The scan result to validate + * @return true if there's no valid match, false if there is a valid match + */ + private boolean hasNoValidMatch(@NonNull ScanFileResult result) { + if (hasInvalidStructure(result)) { + return true; + } + + ScanFileDetails firstDetail = result.getFileDetails().get(0); + return firstDetail.getMatchType() == MatchType.none || + firstDetail.getLines() == null || + firstDetail.getPurls() == null || + firstDetail.getPurls().length == 0; + } + } diff --git a/src/main/java/com/scanoss/Winnowing.java b/src/main/java/com/scanoss/Winnowing.java index 19e8742..303d8eb 100644 --- a/src/main/java/com/scanoss/Winnowing.java +++ b/src/main/java/com/scanoss/Winnowing.java @@ -209,33 +209,6 @@ private Boolean skipSnippets(@NonNull String filename, char[] contents) { } } } - // TODO do we still want this? - // Check to see if the first newline is very far away. If so, it's another hint this could be a binary/data file -// for (int i = 0; i < contents.length; i++) { -// if (contents[i] == '\n') { -// return false; -// } else if (i > MAX_LONG_LINE_CHARS) { -// log.trace("Skipping snippets due to file line being too long: {} - {}", filename, MAX_LONG_LINE_CHARS); -// return true; -// } -// } - // TODO do we want to skip a whole file is some of it is a large single line? -// StringBuilder outputBuilder = new StringBuilder(); -// for (char c: contents) { -// if (c == '\n') { // New line, check line length -// if (outputBuilder.length() > MAX_LONG_LINE_CHARS) { -// log.trace("Skipping snippets due to file line being too long: {} - {}", filename, MAX_LONG_LINE_CHARS); -// return true; -// } -// outputBuilder.setLength(0); // empty the string again -// } else { -// outputBuilder.append(c); -// } -// } -// if (outputBuilder.length() > MAX_LONG_LINE_CHARS) { // Check the last string length -// log.trace("Skipping snippets due to file line being too long: {} - {}", filename, MAX_LONG_LINE_CHARS); -// return true; -// } return false; } diff --git a/src/main/java/com/scanoss/cli/ScanCommandLine.java b/src/main/java/com/scanoss/cli/ScanCommandLine.java index 06a48b8..3b8ddd1 100644 --- a/src/main/java/com/scanoss/cli/ScanCommandLine.java +++ b/src/main/java/com/scanoss/cli/ScanCommandLine.java @@ -116,8 +116,6 @@ class ScanCommandLine implements Runnable { private Scanner scanner; - private List scanFileResults; - private Settings settings; /** * Run the 'scan' command @@ -181,6 +179,7 @@ public void run() { .hiddenFilesFolders(allHidden).numThreads(numThreads).url(apiUrl).apiKey(apiKey) .retryLimit(retryLimit).timeout(Duration.ofSeconds(timeoutLimit)).scanFlags(scanFlags) .sbomType(sbomType).sbom(sbom).snippetLimit(snippetLimit).customCert(caCertPem).proxy(proxy).hpsm(enableHpsm) + .settings(this.settings) .build(); File f = new File(fileFolder); @@ -194,14 +193,6 @@ public void run() { } else { throw new RuntimeException(String.format("Error: Specified path is not a file or a folder: %s\n", fileFolder)); } - - if (settings != null && settings.getBom() != null) { - ScannerPostProcessor scannerPostProcessor = ScannerPostProcessor.builder().build(); - scanFileResults = scannerPostProcessor.process(scanFileResults, settings.getBom()); - } - - var out = spec.commandLine().getOut(); - JsonUtils.writeJsonPretty(toScanFileResultJsonObject(scanFileResults), null); // Uses System.out } /** @@ -228,12 +219,13 @@ private String loadFileToString(@NonNull String filename) { * @param file file to scan */ private void scanFile(String file) { + var out = spec.commandLine().getOut(); var err = spec.commandLine().getErr(); try { printMsg(err, String.format("Scanning %s...", file)); String result = scanner.scanFile(file); if (result != null && !result.isEmpty()) { - scanFileResults = JsonUtils.toScanFileResultsFromObject(JsonUtils.toJsonObject(result)); + JsonUtils.writeJsonPretty(JsonUtils.toJsonObject(result), out); return; } else { err.println("Warning: No results returned."); @@ -246,7 +238,6 @@ private void scanFile(String file) { } throw new RuntimeException(String.format("Something went wrong while scanning %s", file)); } - /** * Scan the specified folder/directory and return the results * @@ -261,7 +252,7 @@ private void scanFolder(String folder) { if (results != null && !results.isEmpty()) { printMsg(err, String.format("Found %d results.", results.size())); printDebug(err, "Converting to JSON..."); - scanFileResults = JsonUtils.toScanFileResultsFromObject(JsonUtils.joinJsonObjects(JsonUtils.toJsonObjects(results))); + JsonUtils.writeJsonPretty(JsonUtils.joinJsonObjects(JsonUtils.toJsonObjects(results)), out); return; } else { err.println("Error: No results return."); diff --git a/src/main/java/com/scanoss/utils/JsonUtils.java b/src/main/java/com/scanoss/utils/JsonUtils.java index bc8f0da..11eb21e 100644 --- a/src/main/java/com/scanoss/utils/JsonUtils.java +++ b/src/main/java/com/scanoss/utils/JsonUtils.java @@ -179,6 +179,29 @@ public static List toScanFileResults(@NonNull List resul return scanFileResults; } + + /** + * Convert a list of ScanFileResult objects to a list of raw JSON strings + * + * @param results List of ScanFileResult objects to convert + * @return List of raw JSON strings + * @throws JsonParseException JSON Parsing failed + * @throws IllegalStateException JSON field is not of JSON Object type + */ + public static List toRawJsonString(@NonNull List results) throws JsonParseException, IllegalStateException { + List rawJsonStrings = new ArrayList<>(results.size()); + Gson gson = new Gson(); + + results.forEach(result -> { + JsonObject jsonObject = new JsonObject(); + JsonElement detailsJson = gson.toJsonTree(result.getFileDetails()); + jsonObject.add(result.getFilePath(), detailsJson); + rawJsonStrings.add(jsonObject.toString()); + }); + + return rawJsonStrings; + } + /** * Converts a list of ScanFileResult objects into a JSON object where the file paths are keys * and the corresponding file details are the values diff --git a/src/test/java/com/scanoss/TestJsonUtils.java b/src/test/java/com/scanoss/TestJsonUtils.java index 8e2c7fd..8f6e7ba 100644 --- a/src/test/java/com/scanoss/TestJsonUtils.java +++ b/src/test/java/com/scanoss/TestJsonUtils.java @@ -87,4 +87,52 @@ public void TestRawResultsPositive() { log.info("Finished {} -->", methodName); } + @Test + public void testToRawJsonStringPositive() { + String methodName = new Object() {}.getClass().getEnclosingMethod().getName(); + log.info("<-- Starting {}", methodName); + + JsonObject jsonObject = JsonUtils.toJsonObject(jsonResultsString); + List sampleScanResults = JsonUtils.toScanFileResultsFromObject(jsonObject); //TODO: Create sampleScanResults with a helper function + + // Convert to raw JSON strings + List rawJsonStrings = JsonUtils.toRawJsonString(sampleScanResults); + + // Verify results + assertNotNull("Raw JSON strings should not be null", rawJsonStrings); + assertEquals("Should have correct number of results", 4, rawJsonStrings.size()); + + // Verify each JSON string can be parsed back to objects + for (String jsonString : rawJsonStrings) { + JsonObject jObject = JsonUtils.toJsonObject(jsonString); + assertNotNull("Parsed JSON object should not be null", jObject); + assertEquals("Each JSON object should have exactly one key", 1, jObject.keySet().size()); + } + + // Verify first result contains expected file path + JsonObject firstResult = JsonUtils.toJsonObject(rawJsonStrings.get(0)); + assertTrue("First result should contain scanoss/__init__.py", firstResult.has("scanoss/__init__.py")); + + JsonObject secondResult = JsonUtils.toJsonObject(rawJsonStrings.get(1)); + assertTrue("Second result should contain CMSsite/admin/js/npm.js", secondResult.has("CMSsite/admin/js/npm.js")); + + JsonObject thirdResult = JsonUtils.toJsonObject(rawJsonStrings.get(2)); + assertTrue("Second result should contain src/spdx.c", thirdResult.has("src/spdx.c")); + + JsonObject fourthResult = JsonUtils.toJsonObject(rawJsonStrings.get(3)); + assertTrue("Second result should contain scanoss/api/__init__.py", fourthResult.has("scanoss/api/__init__.py")); + + log.info("Finished {} -->", methodName); + } + + + @Test + public void testToRawJsonStringEmptyList() { + List emptyList = new ArrayList<>(); + List result = JsonUtils.toRawJsonString(emptyList); + assertNotNull("Result should not be null for empty input", result); + assertTrue("Result should be empty for empty input", result.isEmpty()); + } + + } diff --git a/src/test/java/com/scanoss/TestLineRangeUtils.java b/src/test/java/com/scanoss/TestLineRangeUtils.java index f863339..0ff1940 100644 --- a/src/test/java/com/scanoss/TestLineRangeUtils.java +++ b/src/test/java/com/scanoss/TestLineRangeUtils.java @@ -107,6 +107,9 @@ public void testParseInvalidFormat() { List ranges = LineRangeUtils.parseLineRanges("11-52-81"); assertTrue("Invalid format should be skipped", ranges.isEmpty()); + ranges = LineRangeUtils.parseLineRanges(",,,"); + assertTrue("Invalid format should be skipped", ranges.isEmpty()); + ranges = LineRangeUtils.parseLineRanges("abc-def"); assertTrue("Non-numeric ranges should be skipped", ranges.isEmpty()); diff --git a/src/test/java/com/scanoss/TestScannerPostProcessor.java b/src/test/java/com/scanoss/TestScannerPostProcessor.java index 3c36ba5..5cd8015 100644 --- a/src/test/java/com/scanoss/TestScannerPostProcessor.java +++ b/src/test/java/com/scanoss/TestScannerPostProcessor.java @@ -186,7 +186,7 @@ public void TestMultipleRemoveRules() { List results = scannerPostProcessor.process(sampleScanResults, bom); // Verify - assertTrue("Should remove all results", results.isEmpty()); + assertFalse("Should keep scanoss/__init__.py since it's a non match", results.isEmpty()); log.info("Finished {} -->", methodName); } @@ -232,7 +232,7 @@ public void testRemoveRuleWithNonOverlappingLineRanges() { List results = scannerPostProcessor.process(sampleScanResults, bom); // Verify - should keep because lines don't overlap - assertEquals("Results should match original", sampleScanResults, results); + assertEquals("Results should match original", sampleScanResults.size(), results.size()); log.info("Finished {} -->", methodName); } From 395cf284a43370139706aeaf6a1d81136aadf6d5 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 3 Jan 2025 19:31:02 +0100 Subject: [PATCH 26/28] chore: add test for hasOverlappingRanges --- .../com/scanoss/ScannerPostProcessor.java | 5 ----- .../com/scanoss/utils/LineRangeUtils.java | 20 ----------------- .../java/com/scanoss/TestLineRangeUtils.java | 22 ++++++++++--------- 3 files changed, 12 insertions(+), 35 deletions(-) diff --git a/src/main/java/com/scanoss/ScannerPostProcessor.java b/src/main/java/com/scanoss/ScannerPostProcessor.java index 4b672ed..c47b0f4 100644 --- a/src/main/java/com/scanoss/ScannerPostProcessor.java +++ b/src/main/java/com/scanoss/ScannerPostProcessor.java @@ -250,9 +250,6 @@ private ScanFileDetails createUpdatedResultDetails(ScanFileDetails existingCompo } // If no cached info, create minimal version return existingComponent.toBuilder() - .copyrightDetails(new CopyrightDetails[]{}) //TODO: Check if we need the empty Object - .licenseDetails(new LicenseDetails[]{}) - .vulnerabilityDetails(new VulnerabilityDetails[]{}) .purls(new String[]{newPurl.toString()}) .url(Purl2Url.isSupported(newPurl) ? Purl2Url.convert(newPurl) : "") .component(newPurl.getName()) @@ -398,8 +395,6 @@ private boolean isPathOnlyMatch(@NonNull Rule rule, @NonNull ScanFileResult resu * @return true if it matches, false otherwise */ private boolean isPurlOnlyMatch(@NonNull Rule rule, @NonNull ScanFileResult result) { - // TODO what happens if there is no purl in the result tree? (DONE) - // I won't happen since the invalid results are skipped upstream within isInvalidScanResult() return isPurlMatch(rule.getPurl(), result.getFileDetails().get(0).getPurls()); } diff --git a/src/main/java/com/scanoss/utils/LineRangeUtils.java b/src/main/java/com/scanoss/utils/LineRangeUtils.java index 57cdeb4..da2d40e 100644 --- a/src/main/java/com/scanoss/utils/LineRangeUtils.java +++ b/src/main/java/com/scanoss/utils/LineRangeUtils.java @@ -62,25 +62,6 @@ public static List parseLineRanges(String lineRanges) { return intervals; } - /** - * Checks if two sets of line ranges overlap - * - * @param ranges1 First set of line ranges - * @param ranges2 Second set of line ranges - * @return true if any intervals overlap - */ - public static boolean hasOverlappingRanges(@NonNull List ranges1, @NonNull List ranges2) { - // TODO is this required. It only seems to be used in tests? - for (LineRange interval1 : ranges1) { - for (LineRange interval2 : ranges2) { - if (interval1.overlaps(interval2)) { - return true; - } - } - } - return false; - } - /** * Checks if a list of line ranges overlaps with a single range * @@ -90,7 +71,6 @@ public static boolean hasOverlappingRanges(@NonNull List ranges1, @No * @throws NullPointerException if either parameter is null */ public static boolean hasOverlappingRanges(@NonNull List ranges, @NonNull LineRange range) { - // TODO add test case for (LineRange interval1 : ranges) { if (interval1.overlaps(range)) { return true; diff --git a/src/test/java/com/scanoss/TestLineRangeUtils.java b/src/test/java/com/scanoss/TestLineRangeUtils.java index 0ff1940..09007fc 100644 --- a/src/test/java/com/scanoss/TestLineRangeUtils.java +++ b/src/test/java/com/scanoss/TestLineRangeUtils.java @@ -125,14 +125,15 @@ public void testHasOverlappingRanges() { }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); - String ranges1Str = "1-10,20-30"; - String ranges2Str = "5-15,25-35"; + String rangesStr = "1-10,20-30"; + int range_start = 8; + int range_end = 15; - List ranges1 = LineRangeUtils.parseLineRanges(ranges1Str); - List ranges2 = LineRangeUtils.parseLineRanges(ranges2Str); + List ranges = LineRangeUtils.parseLineRanges(rangesStr); + LineRange range = new LineRange(range_start, range_end); assertTrue("Should detect overlapping ranges", - LineRangeUtils.hasOverlappingRanges(ranges1, ranges2)); + LineRangeUtils.hasOverlappingRanges(ranges, range)); log.info("Finished {} -->", methodName); } @@ -143,14 +144,15 @@ public void testNoOverlappingRanges() { }.getClass().getEnclosingMethod().getName(); log.info("<-- Starting {}", methodName); - String ranges1Str = "1-10,20-30"; - String ranges2Str = "40-50,60-70"; + String rangesStr = "1-10,20-30"; + int range_start = 11; + int range_end = 15; - List ranges1 = LineRangeUtils.parseLineRanges(ranges1Str); - List ranges2 = LineRangeUtils.parseLineRanges(ranges2Str); + List ranges = LineRangeUtils.parseLineRanges(rangesStr); + LineRange range = new LineRange(range_start, range_end); assertFalse("Should not detect overlapping ranges", - LineRangeUtils.hasOverlappingRanges(ranges1, ranges2)); + LineRangeUtils.hasOverlappingRanges(ranges, range)); log.info("Finished {} -->", methodName); } From 9d6c4ff7729e2900680621eece0e4c813531075d Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 9 Jan 2025 10:02:22 +0100 Subject: [PATCH 27/28] fix: enforce strict path & purl rule matching when both are present --- .../java/com/scanoss/ScannerPostProcessor.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/scanoss/ScannerPostProcessor.java b/src/main/java/com/scanoss/ScannerPostProcessor.java index c47b0f4..3272ef3 100644 --- a/src/main/java/com/scanoss/ScannerPostProcessor.java +++ b/src/main/java/com/scanoss/ScannerPostProcessor.java @@ -236,9 +236,6 @@ private ScanFileDetails createUpdatedResultDetails(ScanFileDetails existingCompo if (cached != null) { //TODO: Clarification on copyright, Vulns, etc - //currentComponent.toBuilder().component().vendor().purls().licenseDetails() - //Version if we have a package url with version - //pkg:github/scanoss@1.0.0 return cached.toBuilder() .file(existingComponent.getFile()) .fileHash(existingComponent.getFileHash()) @@ -332,18 +329,20 @@ private Boolean isMatchingRule(@NonNull ScanFileResult result, boolean hasPurl = rule.getPurl() != null && !rule.getPurl().isEmpty(); // Check three possible matching conditions: + // 1. Both path and PURL match - if (hasPath && hasPurl && isPathAndPurlMatch(rule, result)) { - return true; + if (hasPath && hasPurl) { + return isPathAndPurlMatch(rule, result); } // 2. Only PURL match required and matches - if (hasPurl && isPurlOnlyMatch(rule, result)) { - return true; + if (hasPurl) { + return isPurlOnlyMatch(rule, result); } // 3. Only path match required and matches - if (hasPath && isPathOnlyMatch(rule, result)) { - return true; + if (hasPath) { + return isPathOnlyMatch(rule, result); } + return false; } From 94d42169d2344b49d60e28babeeb05c73e58c84d Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 9 Jan 2025 14:19:29 +0100 Subject: [PATCH 28/28] fix: remove license and vulns when no component is found & change name on target --- pom.xml | 2 +- src/main/java/com/scanoss/ScannerPostProcessor.java | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index f042d16..6b075ff 100644 --- a/pom.xml +++ b/pom.xml @@ -186,7 +186,7 @@ - with-dependencies-slf4j-excluded + with-dependencies-excluded-slf4j jar diff --git a/src/main/java/com/scanoss/ScannerPostProcessor.java b/src/main/java/com/scanoss/ScannerPostProcessor.java index 3272ef3..009fa3a 100644 --- a/src/main/java/com/scanoss/ScannerPostProcessor.java +++ b/src/main/java/com/scanoss/ScannerPostProcessor.java @@ -235,7 +235,6 @@ private ScanFileDetails createUpdatedResultDetails(ScanFileDetails existingCompo ScanFileDetails cached = purl2ComponentDetailsMap.get(newPurl.toString()); if (cached != null) { - //TODO: Clarification on copyright, Vulns, etc return cached.toBuilder() .file(existingComponent.getFile()) .fileHash(existingComponent.getFileHash()) @@ -247,6 +246,10 @@ private ScanFileDetails createUpdatedResultDetails(ScanFileDetails existingCompo } // If no cached info, create minimal version return existingComponent.toBuilder() + .copyrightDetails(new CopyrightDetails[]{}) + .licenseDetails(new LicenseDetails[]{}) + .vulnerabilityDetails(new VulnerabilityDetails[]{}) + .version(null) .purls(new String[]{newPurl.toString()}) .url(Purl2Url.isSupported(newPurl) ? Purl2Url.convert(newPurl) : "") .component(newPurl.getName())