diff --git a/core/src/main/java/com/devonfw/tools/solicitor/Solicitor.java b/core/src/main/java/com/devonfw/tools/solicitor/Solicitor.java index 0be13e4f..6723b670 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/Solicitor.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/Solicitor.java @@ -164,7 +164,7 @@ private void readInventory() { Reader reader = this.readerFactory.readerFor(readerSetup.getType()); try { reader.readInventory(readerSetup.getType(), readerSetup.getSource(), readerSetup.getApplication(), - readerSetup.getUsagePattern(), readerSetup.getRepoType(), readerSetup.getConfiguration()); + readerSetup.getUsagePattern(), readerSetup.getRepoType(), readerSetup.getPackageType(), readerSetup.getConfiguration()); } catch (SolicitorRuntimeException sre) { if (this.tolerateMissingInput && sre.getCause() instanceof FileNotFoundException) { Application app = readerSetup.getApplication(); diff --git a/core/src/main/java/com/devonfw/tools/solicitor/SolicitorSetup.java b/core/src/main/java/com/devonfw/tools/solicitor/SolicitorSetup.java index 1bebaeaa..c561eb66 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/SolicitorSetup.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/SolicitorSetup.java @@ -35,6 +35,8 @@ public static class ReaderSetup { private UsagePattern usagePattern; private String repoType; + + private String packageType; private Map configuration; @@ -88,6 +90,15 @@ public String getRepoType() { return this.repoType; } + /** + * This method gets the field packageType. + * + * @return the field packageType + */ + public String getPackageType() { + + return this.packageType; + } /** * This method gets the field configuration. * @@ -157,8 +168,19 @@ public void setRepoType(String repoType) { this.repoType = repoType; } + + /** + * This method sets the field packageType. + * + * @param packageType the new value of the field packageType + */ + public void setPackageType(String packageType) { + + this.packageType = packageType; + } } + private String engagementName; private List readerSetups = new ArrayList<>(); diff --git a/core/src/main/java/com/devonfw/tools/solicitor/common/LogMessages.java b/core/src/main/java/com/devonfw/tools/solicitor/common/LogMessages.java index 6cd91c39..432fbd86 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/common/LogMessages.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/common/LogMessages.java @@ -107,7 +107,10 @@ public enum LogMessages { FAILED_READING_FILE(71, "Reading file '{}' failed"), // EMPTY_PACKAGE_URL(72, "The package URL is null or empty."), // EMPTY_PACKAGE_PATH(73, "The package path is null or empty."), // - CONTENT_FILE_TOO_LARGE(74, + EMPTY_PACKAGE_TYPE(74, "The package type is null or empty."), // + UNKNOWN_PACKAGE_TYPE(75, + "The CSV file contains packageType '{}' which is not supported and will be ignored. Solicitor reports might be incomplete"), // + CONTENT_FILE_TOO_LARGE(76, "The size of the content file '{}' is '{}' (max. allowed is '{}'). Reading will be skipped."); private final String message; diff --git a/core/src/main/java/com/devonfw/tools/solicitor/config/ConfigFactory.java b/core/src/main/java/com/devonfw/tools/solicitor/config/ConfigFactory.java index e5a60228..37371ba9 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/config/ConfigFactory.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/config/ConfigFactory.java @@ -132,6 +132,7 @@ public ModelRoot createConfig(String url) { + "https://github.com/devonfw/solicitor/issues/263"); } rs.setRepoType(rc.getRepoType()); + rs.setPackageType(rc.getPackageType()); rs.setConfiguration(rc.getConfiguration()); rs = resolvePlaceholdersInReader(rs, placeHolderMap); this.solicitorSetup.getReaderSetups().add(rs); diff --git a/core/src/main/java/com/devonfw/tools/solicitor/config/ReaderConfig.java b/core/src/main/java/com/devonfw/tools/solicitor/config/ReaderConfig.java index 3ce7d926..442b29f0 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/config/ReaderConfig.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/config/ReaderConfig.java @@ -27,6 +27,9 @@ public class ReaderConfig { @JsonProperty private String repoType; + + @JsonProperty + private String packageType; @JsonProperty private Map configuration; @@ -49,6 +52,16 @@ public String getRepoType() { return this.repoType; } + + /** + * This method gets the field packageType. + * + * @return the field packageType + */ + public String getPackageType() { + + return this.packageType; + } /** * This method gets the field configuration. @@ -99,7 +112,16 @@ public void setRepoType(String repoType) { this.repoType = repoType; } + + /** + * This method sets the field packageType. + * + * @param packageType the new value of the field packageType + */ + public void setPackageType(String packageType) { + this.packageType = packageType; + } /** * This method sets the field configuration. * diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/Reader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/Reader.java index 0f343288..84fe4652 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/reader/Reader.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/reader/Reader.java @@ -36,9 +36,10 @@ public interface Reader { * @param application all read {@link ApplicationComponent} need to be linked with this {@link Application} * @param usagePattern the {@link UsagePattern} which applies for all read {@link ApplicationComponent}s * @param repoType the type of Repository to download the sources from + * @param packageType the packageType to create the packageUrl * @param configuration optional configuration settings for readers */ public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern, - String repoType, Map configuration); + String repoType, String packageType, Map configuration); } diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/csv/CsvReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/csv/CsvReader.java index 28f85e7c..4b41aeee 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/reader/csv/CsvReader.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/reader/csv/CsvReader.java @@ -14,13 +14,19 @@ import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVRecord; import org.springframework.stereotype.Component; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.devonfw.tools.solicitor.common.LogMessages; +import com.devonfw.tools.solicitor.common.PackageURLHelper; import com.devonfw.tools.solicitor.common.SolicitorRuntimeException; import com.devonfw.tools.solicitor.model.inventory.ApplicationComponent; import com.devonfw.tools.solicitor.model.masterdata.Application; import com.devonfw.tools.solicitor.model.masterdata.UsagePattern; import com.devonfw.tools.solicitor.reader.AbstractReader; import com.devonfw.tools.solicitor.reader.Reader; +import com.devonfw.tools.solicitor.reader.maven.MavenReader; +import com.github.packageurl.PackageURL; /** * A {@link Reader} for files in CSV format. @@ -46,6 +52,8 @@ @Component public class CsvReader extends AbstractReader implements Reader { + private Logger logger = LoggerFactory.getLogger(CsvReader.class); // not static final for testing + // purposes /** * The supported type of this {@link Reader}. @@ -62,7 +70,7 @@ public Set getSupportedTypes() { /** {@inheritDoc} */ @Override public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern, - String repoType, Map configuration) { + String repoType, String packageType, Map configuration) { int components = 0; int licenses = 0; @@ -214,6 +222,7 @@ public void readInventory(String type, String sourceUrl, Application application appComponent.setVersion(version); appComponent.setUsagePattern(usagePattern); appComponent.setRepoType(repoType); + appComponent.setPackageUrl(getPackageURL(packageType, groupId, artifactId, version)); // merge ApplicationComponentImpl with same key if they appear // on @@ -248,6 +257,8 @@ public void readInventory(String type, String sourceUrl, Application application appComponent.setVersion(record.get(2)); appComponent.setUsagePattern(usagePattern); appComponent.setRepoType(repoType); + appComponent.setPackageUrl(getPackageURL(packageType, record.get(0), record.get(1), record.get(2))); + // merge ApplicationComponentImpl with same key if they appear // on // subsequent lines (multilicensing) @@ -275,4 +286,43 @@ public void readInventory(String type, String sourceUrl, Application application } + /** + * Call the appropriate {@link PackageURLHelper} method to create a packageURL. Returns null if no + * {@link PackageURLHelper} exists for the packageType. + * + * @param packageType the package type + * @param groupId the groupId if available + * @param artifactId the artifactId + * @param version the version + * @return the created PackageURL + */ + public String getPackageURL(String packageType, String groupId, String artifactId, String version) { + + if (packageType == null || packageType.isEmpty()) { + this.logger.warn(LogMessages.EMPTY_PACKAGE_TYPE.msg(), packageType); + return null; + } + switch (packageType) { + case "maven": + return PackageURLHelper.fromMavenCoordinates(groupId, artifactId, version).toString(); + case "npm": + return PackageURLHelper.fromNpmPackageNameAndVersion(artifactId, version).toString(); + case "pypi": + return PackageURLHelper.fromPyPICoordinates(artifactId, version).toString(); + default: + this.logger.warn(LogMessages.UNKNOWN_PACKAGE_TYPE.msg(), packageType); + return null; + } + } + + /** + * Sets the logger. Available for testing purposes only. + * + * @param logger the logger + */ + void setLogger(Logger logger) { + + this.logger = logger; + } + } diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/cyclonedx/CyclonedxReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/cyclonedx/CyclonedxReader.java index a95953a0..e6258cd2 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/reader/cyclonedx/CyclonedxReader.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/reader/cyclonedx/CyclonedxReader.java @@ -55,7 +55,7 @@ public Set getSupportedTypes() { /** {@inheritDoc} */ @Override public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern, - String repoType, Map configuration) { + String repoType, String packageType, Map configuration) { int componentCount = 0; int licenseCount = 0; diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/gradle/GradleReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/gradle/GradleReader.java index e589f6b0..f1d910c4 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/reader/gradle/GradleReader.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/reader/gradle/GradleReader.java @@ -61,7 +61,7 @@ public Set getSupportedTypes() { /** {@inheritDoc} */ @Override public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern, - String repoType, Map configuration) { + String repoType, String packageType, Map configuration) { this.deprecationChecker.check(false, "Use of Reader of type 'gradle' is deprecated, use 'gradle2' instead. See https://github.com/devonfw/solicitor/issues/58"); diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/gradle/GradleReader2.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/gradle/GradleReader2.java index e7ed6cb5..006af338 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/reader/gradle/GradleReader2.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/reader/gradle/GradleReader2.java @@ -48,7 +48,7 @@ public Set getSupportedTypes() { /** {@inheritDoc} */ @Override public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern, - String repoType, Map configuration) { + String repoType, String packageType, Map configuration) { int components = 0; int licenses = 0; diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/maven/MavenReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/maven/MavenReader.java index ee48d2d7..094349f6 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/reader/maven/MavenReader.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/reader/maven/MavenReader.java @@ -59,7 +59,7 @@ public Set getSupportedTypes() { /** {@inheritDoc} */ @Override public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern, - String repoType, Map configuration) { + String repoType, String packageType, Map configuration) { int components = 0; int licenses = 0; diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/npmlicensechecker/NpmLicenseCheckerReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/npmlicensechecker/NpmLicenseCheckerReader.java index df6ad64e..49136528 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/reader/npmlicensechecker/NpmLicenseCheckerReader.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/reader/npmlicensechecker/NpmLicenseCheckerReader.java @@ -48,7 +48,7 @@ public Set getSupportedTypes() { @SuppressWarnings("rawtypes") @Override public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern, - String repoType, Map configuration) { + String repoType, String packageType, Map configuration) { int componentCount = 0; int licenseCount = 0; diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/npmlicensecrawler/NpmLicenseCrawlerReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/npmlicensecrawler/NpmLicenseCrawlerReader.java index 9ddb9be8..ff10ee41 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/reader/npmlicensecrawler/NpmLicenseCrawlerReader.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/reader/npmlicensecrawler/NpmLicenseCrawlerReader.java @@ -67,7 +67,7 @@ public Set getSupportedTypes() { /** {@inheritDoc} */ @Override public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern, - String repoType, Map configuration) { + String repoType, String packageType, Map configuration) { this.deprecationChecker.check(false, "Use of Reader of type '" + SUPPORTED_TYPE diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/ort/OrtReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/ort/OrtReader.java index db15b3a3..cb27e879 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/reader/ort/OrtReader.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/reader/ort/OrtReader.java @@ -44,7 +44,7 @@ public Set getSupportedTypes() { @SuppressWarnings("rawtypes") @Override public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern, - String repoType, Map configuration) { + String repoType, String packageType, Map configuration) { int componentCount = 0; int licenseCount = 0; diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/piplicenses/PipLicensesReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/piplicenses/PipLicensesReader.java index bf903d50..cf335a2d 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/reader/piplicenses/PipLicensesReader.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/reader/piplicenses/PipLicensesReader.java @@ -44,7 +44,7 @@ public Set getSupportedTypes() { @SuppressWarnings("rawtypes") @Override public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern, - String repoType, Map configuration) { + String repoType, String packageType, Map configuration) { int componentCount = 0; int licenseCount = 0; diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/yarn/YarnReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/yarn/YarnReader.java index 625a7165..8a6e7bae 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/reader/yarn/YarnReader.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/reader/yarn/YarnReader.java @@ -48,7 +48,7 @@ public Set getSupportedTypes() { @SuppressWarnings("rawtypes") @Override public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern, - String repoType, Map configuration) { + String repoType, String packageType, Map configuration) { String content = cutSourceJson(sourceUrl); diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReader.java index 1ca903b4..44423011 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReader.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReader.java @@ -55,7 +55,7 @@ public Set getSupportedTypes() { @SuppressWarnings("rawtypes") @Override public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern, - String repoType, Map configuration) { + String repoType, String packageType, Map configuration) { String content = readAndPreprocessJson(sourceUrl); diff --git a/core/src/test/java/com/devonfw/tools/solicitor/reader/csv/CsvReaderTests.java b/core/src/test/java/com/devonfw/tools/solicitor/reader/csv/CsvReaderTests.java index e6567ee7..5c52174d 100644 --- a/core/src/test/java/com/devonfw/tools/solicitor/reader/csv/CsvReaderTests.java +++ b/core/src/test/java/com/devonfw/tools/solicitor/reader/csv/CsvReaderTests.java @@ -6,12 +6,15 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,6 +30,8 @@ public class CsvReaderTests { Application application; + private Logger logger; + public CsvReaderTests() { // configuration settings @@ -59,7 +64,7 @@ public CsvReaderTests() { csvr.setModelFactory(modelFactory); csvr.setInputStreamFactory(new FileInputStreamFactory()); csvr.readInventory("csv", "src/test/resources/csvlicenses.csv", this.application, UsagePattern.DYNAMIC_LINKING, - "maven", configuration); + "maven", "maven", configuration); } @@ -101,4 +106,115 @@ public void testFindLicense() { assertTrue(found); } + /** + * Tests if packageUrl for maven has been created. Reader runs with config. + */ + @Test + public void testFindMavenPackageUrl() { + + List lapc = this.application.getApplicationComponents(); + boolean found = false; + for (ApplicationComponent ap : lapc) { + if (ap.getArtifactId().equals("albireo") && ap.getPackageUrl().equals("pkg:maven/org.eclipse/albireo@0.0.3")) { + found = true; + break; + } + } + assertTrue(found); + + } + + /** + * Tests if packageUrl for npm has been created. Reader runs without config. + */ + @Test + public void testFindNpmPackageUrl() { + + Application application; + ModelFactory modelFactory = new ModelFactoryImpl(); + application = modelFactory.newApplication("testAppNpm", "0.0.0.TEST", "1.1.2111", "http://bla.com", "NPM"); + CsvReader csvr = new CsvReader(); + csvr.setModelFactory(modelFactory); + csvr.setInputStreamFactory(new FileInputStreamFactory()); + csvr.readInventory("csv", "src/test/resources/csvlicenses_npm.csv", application, UsagePattern.DYNAMIC_LINKING, + "npm", "npm", null); + + List lapc = application.getApplicationComponents(); + boolean found = false; + for (ApplicationComponent ap : lapc) { + if (ap.getArtifactId().equals("@babel/code-frame") + && ap.getPackageUrl().equals("pkg:npm/%40babel/code-frame@7.8.3")) { + found = true; + break; + } + } + assertTrue(found); + + } + + /** + * Tests if packageUrl for pypi has been created. Reader runs with config and swapped artifactId and version position. + */ + @Test + public void testFindPypiPackageUrl() { + + Application application; + ModelFactory modelFactory = new ModelFactoryImpl(); + application = modelFactory.newApplication("testAppPypi", "0.0.0.TEST", "1.1.2111", "http://bla.com", "Python"); + + // configuration settings + Map configuration = new HashMap(); + configuration.put("artifactId", "2"); + configuration.put("version", "1"); + configuration.put("delimiter", ";"); + + CsvReader csvr = new CsvReader(); + csvr.setModelFactory(modelFactory); + csvr.setInputStreamFactory(new FileInputStreamFactory()); + csvr.readInventory("csv", "src/test/resources/csvlicenses_pypi.csv", application, UsagePattern.DYNAMIC_LINKING, + "pypi", "pypi", configuration); + + List lapc = application.getApplicationComponents(); + boolean found = false; + for (ApplicationComponent ap : lapc) { + if (ap.getArtifactId().equals("python-dateutil") && ap.getPackageUrl().equals("pkg:pypi/python-dateutil@2.8.1")) { + found = true; + break; + } + } + assertTrue(found); + + } + + /** + * Tests if reader crashes with correct warn message if no packageType is given. Reader runs without config. + */ + @Test + public void testNullPackageUrl() { + + String expectedLogMessage = "The package type is null or empty"; + + Application application; + ModelFactory modelFactory = new ModelFactoryImpl(); + application = modelFactory.newApplication("testAppNull", "0.0.0.TEST", "1.1.2111", "http://bla.com", "Python"); + CsvReader csvr = new CsvReader(); + csvr.setModelFactory(modelFactory); + csvr.setInputStreamFactory(new FileInputStreamFactory()); + + this.logger = mock(Logger.class); + csvr.setLogger(this.logger); + + try { + csvr.readInventory("csv", "src/test/resources/csvlicenses_pypi.csv", application, UsagePattern.DYNAMIC_LINKING, + null, null, null); + } catch (Exception e) { + // Capture the log messages + ArgumentCaptor logEntry1 = ArgumentCaptor.forClass(String.class); + verify(logger).warn(logEntry1.capture()); + + assertEquals(expectedLogMessage, logEntry1.getValue()); + } + + } + } diff --git a/core/src/test/java/com/devonfw/tools/solicitor/reader/cyclonedx/CyclonedxReaderTests.java b/core/src/test/java/com/devonfw/tools/solicitor/reader/cyclonedx/CyclonedxReaderTests.java index 32667739..7e68c505 100644 --- a/core/src/test/java/com/devonfw/tools/solicitor/reader/cyclonedx/CyclonedxReaderTests.java +++ b/core/src/test/java/com/devonfw/tools/solicitor/reader/cyclonedx/CyclonedxReaderTests.java @@ -49,7 +49,7 @@ public void readMavenFileAndCheckSize() { this.cdxr.setInputStreamFactory(new FileInputStreamFactory()); this.cdxr.setDelegatingPackageURLHandler(this.delegatingPurlHandler); this.cdxr.readInventory("maven", "src/test/resources/mavensbom.json", application, UsagePattern.DYNAMIC_LINKING, - "cyclonedx", null); + "cyclonedx", null, null); LOG.info(application.toString()); assertEquals(32, application.getApplicationComponents().size()); @@ -85,7 +85,7 @@ public void readMavenFileAndCheckSizeNegative() { this.cdxr.setInputStreamFactory(new FileInputStreamFactory()); this.cdxr.setDelegatingPackageURLHandler(this.delegatingPurlHandler); this.cdxr.readInventory("maven", "src/test/resources/mavensbom.json", application, UsagePattern.DYNAMIC_LINKING, - "cyclonedx", null); + "cyclonedx", null, null); LOG.info(application.toString()); assertEquals(32, application.getApplicationComponents().size()); @@ -108,7 +108,7 @@ public void readMavenFileAndCheckSingleContentSize() { this.cdxr.setInputStreamFactory(new FileInputStreamFactory()); this.cdxr.setDelegatingPackageURLHandler(this.delegatingPurlHandler); this.cdxr.readInventory("maven", "src/test/resources/mavensbom.json", application, UsagePattern.DYNAMIC_LINKING, - "someRepoType", null); + "someRepoType", null, null); LOG.info(application.toString()); assertEquals(32, application.getApplicationComponents().size()); @@ -170,7 +170,7 @@ public void readNpmFileAndCheckSize() { this.cdxr.setInputStreamFactory(new FileInputStreamFactory()); this.cdxr.setDelegatingPackageURLHandler(this.delegatingPurlHandler); this.cdxr.readInventory("npm", "src/test/resources/npmsbom.json", application, UsagePattern.DYNAMIC_LINKING, - "cyclonedx", null); + "cyclonedx", null, null); LOG.info(application.toString()); assertEquals(74, application.getApplicationComponents().size()); diff --git a/core/src/test/java/com/devonfw/tools/solicitor/reader/gradle/GradleReader2Tests.java b/core/src/test/java/com/devonfw/tools/solicitor/reader/gradle/GradleReader2Tests.java index dee7adcb..dfc04abd 100644 --- a/core/src/test/java/com/devonfw/tools/solicitor/reader/gradle/GradleReader2Tests.java +++ b/core/src/test/java/com/devonfw/tools/solicitor/reader/gradle/GradleReader2Tests.java @@ -34,7 +34,7 @@ public GradleReader2Tests() { gr.setModelFactory(modelFactory); gr.setInputStreamFactory(new FileInputStreamFactory()); gr.readInventory("gradle2", "src/test/resources/licenseReport.json", this.application, UsagePattern.DYNAMIC_LINKING, - "maven", null); + "maven", null, null); } diff --git a/core/src/test/java/com/devonfw/tools/solicitor/reader/gradle/GradleReaderTests.java b/core/src/test/java/com/devonfw/tools/solicitor/reader/gradle/GradleReaderTests.java index 196fd071..6dd8ee81 100644 --- a/core/src/test/java/com/devonfw/tools/solicitor/reader/gradle/GradleReaderTests.java +++ b/core/src/test/java/com/devonfw/tools/solicitor/reader/gradle/GradleReaderTests.java @@ -43,7 +43,7 @@ public void check(boolean warnOnly, String detailsString) { gr.setModelFactory(modelFactory); gr.setInputStreamFactory(new FileInputStreamFactory()); gr.readInventory("gradle", "src/test/resources/licenseReport.json", this.application, UsagePattern.DYNAMIC_LINKING, - "maven", null); + "maven", null, null); } diff --git a/core/src/test/java/com/devonfw/tools/solicitor/reader/maven/MavenReaderTests.java b/core/src/test/java/com/devonfw/tools/solicitor/reader/maven/MavenReaderTests.java index 13f492c1..ac89fed3 100644 --- a/core/src/test/java/com/devonfw/tools/solicitor/reader/maven/MavenReaderTests.java +++ b/core/src/test/java/com/devonfw/tools/solicitor/reader/maven/MavenReaderTests.java @@ -39,7 +39,7 @@ public void readFileAndCheckSize() { mr.setModelFactory(modelFactory); mr.setInputStreamFactory(new FileInputStreamFactory()); mr.readInventory("maven", "src/test/resources/licenses_sample.xml", application, UsagePattern.DYNAMIC_LINKING, - "maven", null); + "maven", null, null); LOG.info(application.toString()); assertEquals(95, application.getApplicationComponents().size()); @@ -74,7 +74,7 @@ public void testProtectionAgainstXxe() { try { mr.readInventory("maven", "src/test/resources/licenses_sample_with_doctype.xml", application, - UsagePattern.DYNAMIC_LINKING, "maven", null); + UsagePattern.DYNAMIC_LINKING, "maven", null, null); fail("Expected exception was not thrown"); } catch (SolicitorRuntimeException e) { // we check detailed message to make sure the exception is not thrown due to other reasons diff --git a/core/src/test/java/com/devonfw/tools/solicitor/reader/npmlicensechecker/NpmLicenseCheckerReaderTests.java b/core/src/test/java/com/devonfw/tools/solicitor/reader/npmlicensechecker/NpmLicenseCheckerReaderTests.java index d4a5fac4..1ea36e16 100644 --- a/core/src/test/java/com/devonfw/tools/solicitor/reader/npmlicensechecker/NpmLicenseCheckerReaderTests.java +++ b/core/src/test/java/com/devonfw/tools/solicitor/reader/npmlicensechecker/NpmLicenseCheckerReaderTests.java @@ -34,7 +34,7 @@ public NpmLicenseCheckerReaderTests() { gr.setModelFactory(modelFactory); gr.setInputStreamFactory(new FileInputStreamFactory()); gr.readInventory("npm-license-checker", "src/test/resources/npmLicenseCheckerReport.json", this.application, - UsagePattern.DYNAMIC_LINKING, "npm", null); + UsagePattern.DYNAMIC_LINKING, "npm", null, null); } diff --git a/core/src/test/java/com/devonfw/tools/solicitor/reader/npmlicensecrawler/NpmLicenseCrawlerReaderTests.java b/core/src/test/java/com/devonfw/tools/solicitor/reader/npmlicensecrawler/NpmLicenseCrawlerReaderTests.java index 086402d9..ee0dcbe4 100644 --- a/core/src/test/java/com/devonfw/tools/solicitor/reader/npmlicensecrawler/NpmLicenseCrawlerReaderTests.java +++ b/core/src/test/java/com/devonfw/tools/solicitor/reader/npmlicensecrawler/NpmLicenseCrawlerReaderTests.java @@ -42,7 +42,7 @@ public void check(boolean warnOnly, String detailsString) { }); nr.setModelFactory(modelFactory); nr.setInputStreamFactory(new FileInputStreamFactory()); - nr.readInventory("npm", "src/test/resources/npmlicenses.csv", this.application, UsagePattern.DYNAMIC_LINKING, "npm", + nr.readInventory("npm", "src/test/resources/npmlicenses.csv", this.application, UsagePattern.DYNAMIC_LINKING, "npm", null, null); } diff --git a/core/src/test/java/com/devonfw/tools/solicitor/reader/ort/OrtReaderTests.java b/core/src/test/java/com/devonfw/tools/solicitor/reader/ort/OrtReaderTests.java index 29179aab..c97dca1d 100644 --- a/core/src/test/java/com/devonfw/tools/solicitor/reader/ort/OrtReaderTests.java +++ b/core/src/test/java/com/devonfw/tools/solicitor/reader/ort/OrtReaderTests.java @@ -33,7 +33,7 @@ public OrtReaderTests() { OrtReader pr = new OrtReader(); pr.setModelFactory(modelFactory); pr.setInputStreamFactory(new FileInputStreamFactory()); - pr.readInventory("ort", "src/test/resources/analyzer-result.json", this.application, UsagePattern.DYNAMIC_LINKING, "ort", + pr.readInventory("ort", "src/test/resources/analyzer-result.json", this.application, UsagePattern.DYNAMIC_LINKING, "ort", null, null); } diff --git a/core/src/test/java/com/devonfw/tools/solicitor/reader/piplicenses/PipReaderTests.java b/core/src/test/java/com/devonfw/tools/solicitor/reader/piplicenses/PipReaderTests.java index 8e0afe46..19e5689e 100644 --- a/core/src/test/java/com/devonfw/tools/solicitor/reader/piplicenses/PipReaderTests.java +++ b/core/src/test/java/com/devonfw/tools/solicitor/reader/piplicenses/PipReaderTests.java @@ -33,7 +33,7 @@ public PipReaderTests() { PipLicensesReader pr = new PipLicensesReader(); pr.setModelFactory(modelFactory); pr.setInputStreamFactory(new FileInputStreamFactory()); - pr.readInventory("pip", "src/test/resources/pipReport.json", this.application, UsagePattern.DYNAMIC_LINKING, "pip", + pr.readInventory("pip", "src/test/resources/pipReport.json", this.application, UsagePattern.DYNAMIC_LINKING, "pip", null, null); } diff --git a/core/src/test/java/com/devonfw/tools/solicitor/reader/yarn/YarnReaderTests.java b/core/src/test/java/com/devonfw/tools/solicitor/reader/yarn/YarnReaderTests.java index 83998d28..2ee47a1a 100644 --- a/core/src/test/java/com/devonfw/tools/solicitor/reader/yarn/YarnReaderTests.java +++ b/core/src/test/java/com/devonfw/tools/solicitor/reader/yarn/YarnReaderTests.java @@ -34,7 +34,7 @@ public YarnReaderTests() { yr.setModelFactory(modelFactory); yr.setInputStreamFactory(new FileInputStreamFactory()); yr.readInventory("yarn", "src/test/resources/yarnReport.json", this.application, UsagePattern.DYNAMIC_LINKING, - "yarn", null); + "yarn", null, null); } diff --git a/core/src/test/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReaderTests.java b/core/src/test/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReaderTests.java index dc02828e..49a96c00 100644 --- a/core/src/test/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReaderTests.java +++ b/core/src/test/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReaderTests.java @@ -40,7 +40,7 @@ public YarnModernReaderTests() { yr.setModelFactory(modelFactory); yr.setInputStreamFactory(new FileInputStreamFactory()); yr.readInventory("yarn-modern", "src/test/resources/yarnModernReport.json", this.application, - UsagePattern.STATIC_LINKING, "npm", null); + UsagePattern.STATIC_LINKING, "npm", null, null); } diff --git a/core/src/test/resources/csvlicenses_npm.csv b/core/src/test/resources/csvlicenses_npm.csv new file mode 100644 index 00000000..86b82603 --- /dev/null +++ b/core/src/test/resources/csvlicenses_npm.csv @@ -0,0 +1,3 @@ +;@babel/code-frame;7.8.3;MIT;https://github.com/babel/babel/tree/master/packages/babel-code-frame +;@babel/compat-data;7.15.0;MIT;https://github.com/babel/babel.git +;accepts;1.3.7;MIT;https://github.com/jshttp/accepts.git \ No newline at end of file diff --git a/core/src/test/resources/csvlicenses_pypi.csv b/core/src/test/resources/csvlicenses_pypi.csv new file mode 100644 index 00000000..122853fc --- /dev/null +++ b/core/src/test/resources/csvlicenses_pypi.csv @@ -0,0 +1,3 @@ +;3.6.3;APScheduler;MIT;https://github.com/agronholm/apscheduler;; +;1.0.8;Keras-Applications;MIT;https://github.com/keras-team/keras-applications;; +;2.8.1;python-dateutil;Dual License;https://dateutil.readthedocs.io;; \ No newline at end of file diff --git a/documentation/master-solicitor.asciidoc b/documentation/master-solicitor.asciidoc index 01a0073d..60cdc923 100644 --- a/documentation/master-solicitor.asciidoc +++ b/documentation/master-solicitor.asciidoc @@ -149,6 +149,7 @@ The internal business data model consists of 6 entities: | artifactId | String | component identifier: maven artifactId | version | String | component identifier: Version | repoType | String | component identifier: RepoType +| packageType | String | component identifier: PackageType | packageUrl | String | the https://github.com/package-url/purl-spec[Package URL] as an technology neutral component identifier | noticeFileUrl | String | URL referencing a NOTICE file to be included in the attributions (optional, see <>) | noticeFileContent | String | resolved content of noticeFileUrl (optional, see <>) @@ -340,6 +341,7 @@ Within this section the different applications (=deliverables) of the engagement "source" : "classpath:samples/licenses_devon4j.xml", <7> <10> "usagePattern" : "DYNAMIC_LINKING", <8> "repoType" : "maven" <9> + "packageType" : "maven" <11> } ] } ], @@ -353,6 +355,7 @@ Within this section the different applications (=deliverables) of the engagement <8> usage pattern; possible values: `DYNAMIC_LINKING`, `STATIC_LINKING`, `STANDALONE_PRODUCT`; see description below <9> repoType: `repoType` to be set in the `ApplicationComponent` . *This parameter is deprecated and should no longer be used*, see <>. The value of `repoType` in `ApplicationComponent` will otherwise be determined from the type info in the PackageURL of the component. <10> _placeholder patterns might be used here_ +<11> packageType: type of the packages in the input data. Must be a valid packageUrl type (see https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst). Relevant when using the CSV reader. ===== Usage Patterns The usage pattern describes how the ``ApplicationComponent``s (libraries, packages) which are read in via the Reader are linked (in)to the ``Application``s executable. The kind of linking might affect the legal evaluation of the license compliance. @@ -550,7 +553,8 @@ In _Solicitor_ the data is read with the following part of the config "readers" : [ { "type" : "csv", "source" : "file:path/to/the/file.csv", - "usagePattern" : "DYNAMIC_LINKING" + "usagePattern" : "DYNAMIC_LINKING", + "packageType": "maven" } ] ---- @@ -621,7 +625,6 @@ These configurations may also be used to overwrite options of a https://commons. Important: In case that a component has multiple licenses attached, there needs to be a separate line in the csv file for each license. -WARNING: The CSV reader currently does not fill the attribute `packageUrl`. Any functionality/reporting based on this attribute will be disfunctional for data read by the CSV reader. === NPM @@ -1307,7 +1310,7 @@ The Generic Excel Writer exists purely for debugging purposes. This writer write "APPLICATIONCOMPONENT" : "classpath:com/devonfw/tools/solicitor/sql/allden_applicationcomponents.sql", "LICENSE" : "classpath:com/devonfw/tools/solicitor/sql/allden_normalizedlicenses.sql", "OSSLICENSES" : "classpath:com/devonfw/tools/solicitor/sql/ossapplicationcomponents.sql", - ... + ... } } ] @@ -1574,7 +1577,7 @@ artifacts: - name: pkg/npm/@somescope/somepackage/1.2.3 <1> url: https://github.com/foo/bar <2> licenseCurations: <3> - - operation: REMOVE + - operation: REMOVE path: "sources/package/readme.md" ruleIdentifier: "proprietary-license_unknown_13.RULE" matchedText: ".* to be paid .*" @@ -1603,8 +1606,8 @@ artifacts: <7> Further packages to follow. ===== Rules for curating licenses -Curating licenses is done by REMOVING (i.e. ignoring) specific license findings from ScanCode, by REPLACING the detected license with another one or by ADDING license findings either to specific files or on top level (not related to specific file of the package sources). -In addition to the conditions/data which is specific for any of the below described operations it is always possible to define a comment which is intended to be included in any audit trail log for documentation purposes (not yet used/implemented). +Curating licenses is done by REMOVING (i.e. ignoring) specific license findings from ScanCode, by REPLACING the detected license with another one or by ADDING license findings either to specific files or on top level (not related to specific file of the package sources). +In addition to the conditions/data which is specific for any of the below described operations it is always possible to define a comment which is intended to be included in any audit trail log for documentation purposes (not yet used/implemented). ====== Licenses: REMOVE @@ -1624,7 +1627,7 @@ Instead of removing licenses (ignoring the finding) they might be replaced with Data: * `newLicense` is the key / id of the license to use instead (replacing `files[].licenses[].spdx_license_key`) -* `url` is the url pointing to the license text +* `url` is the url pointing to the license text At least one of the two parameters has to be set. @@ -1658,14 +1661,14 @@ At least one of the conditions has to be defined. This follows the above principles. It uses the same conditions as REMOVE and uses a parameter to define the copyright to use instead: Data: -* `newCopyright`: The copyright entry to use instead of the originally found copyright +* `newCopyright`: The copyright entry to use instead of the originally found copyright ====== Copyright: ADD Adding new copyrights is done by defining rules which add new copyright info (to the copyrights found in a source file) - or "on top level". Conditions: -* `path` of the file within the sources (defined as a regular expression; if omitted the copyright will be applied on "top level"). +* `path` of the file within the sources (defined as a regular expression; if omitted the copyright will be applied on "top level"). Note that it is again only possible to add copyrigts to paths which are listed in the scancode json Data: @@ -1712,9 +1715,9 @@ artifacts: ===== Hierarchical definition of rules -Different version of a package/component or even different packages/components within the same namespace often require mostly the same curations to be applied. To avoid being forced to redefine curations for every single version it is possible to define curations by just specifying a prefix part in the name attribute. +Different version of a package/component or even different packages/components within the same namespace often require mostly the same curations to be applied. To avoid being forced to redefine curations for every single version it is possible to define curations by just specifying a prefix part in the name attribute. -Example of available levels/prefixes for `pkg:/maven/ch.qos.logback/logback-classic@1.2.3` +Example of available levels/prefixes for `pkg:/maven/ch.qos.logback/logback-classic@1.2.3` * `pkg` * `pkg/maven` @@ -1897,6 +1900,7 @@ Spring beans implementing this interface will be called at certain points in the == Release Notes Changes in 1.25.0:: * https://github.com/devonfw/solicitor/issues/277: When reading content (license texts or notice files) within the scancode adapter files which are greater than 1 million bytes will be skipped. This avoids large memory consumption and resulting instability. +* https://github.com/devonfw/solicitor/issues/274: Fixed issue where no packageURL was created when using the CSV reader. Added attribute 'packageType'. Changes in 1.24.2:: * https://github.com/devonfw/solicitor/pull/271: Fixed an incompatibility with JDK 8.