Bugfix/csvreader missing packageurl (#275)

* Add "packageType" parameter and logic to create purl in CSV Reader. Refactor tests and config readers * Add warn message for unknown packagetypes * Improve log message * Add release note * Add Unit Test and format code * Add documentation * Update CSV reader doc * Move switch-case block to dedicated method * Run tests with packageType=null * Check for null or empty packageType. Make logger non static for testing purposes. * Add mockito dependency for tests. * Add tests for npm, pypi and empty packageType * minor improvement * swap position of artifactId and version in config * formatting * Consistent syntax * Shorten if condition * Set packageType=null in unit tests * Update typo in documentation/master-solicitor.asciidoc * remove mockito dependency from POM as it is already included via spring-boot-starter-test --------- Co-authored-by: chrimih <[email protected]> Co-authored-by: ohecker <[email protected]>
devonfw · Jul 25, 2024 · 6276bfa · 6276bfa
1 parent 3abdf85
commit 6276bfa
Show file tree

Hide file tree

Showing 31 changed files with 265 additions and 40 deletions.
diff --git a/core/src/main/java/com/devonfw/tools/solicitor/Solicitor.java b/core/src/main/java/com/devonfw/tools/solicitor/Solicitor.java
@@ -164,7 +164,7 @@ private void readInventory() {
       Reader reader = this.readerFactory.readerFor(readerSetup.getType());
       try {
         reader.readInventory(readerSetup.getType(), readerSetup.getSource(), readerSetup.getApplication(),
-            readerSetup.getUsagePattern(), readerSetup.getRepoType(), readerSetup.getConfiguration());
+            readerSetup.getUsagePattern(), readerSetup.getRepoType(), readerSetup.getPackageType(), readerSetup.getConfiguration());
       } catch (SolicitorRuntimeException sre) {
         if (this.tolerateMissingInput && sre.getCause() instanceof FileNotFoundException) {
           Application app = readerSetup.getApplication();

diff --git a/core/src/main/java/com/devonfw/tools/solicitor/SolicitorSetup.java b/core/src/main/java/com/devonfw/tools/solicitor/SolicitorSetup.java
@@ -35,6 +35,8 @@ public static class ReaderSetup {
     private UsagePattern usagePattern;
 
     private String repoType;
+
+    private String packageType;
 
     private Map<String, String> configuration;
 
@@ -88,6 +90,15 @@ public String getRepoType() {
       return this.repoType;
     }
 
+    /**
+     * This method gets the field <code>packageType</code>.
+     *
+     * @return the field packageType
+     */
+    public String getPackageType() {
+
+      return this.packageType;
+    }
     /**
      * This method gets the field <code>configuration</code>.
      *
@@ -157,8 +168,19 @@ public void setRepoType(String repoType) {
 
       this.repoType = repoType;
     }
+
+    /**
+     * This method sets the field <code>packageType</code>.
+     *
+     * @param packageType the new value of the field packageType
+     */
+    public void setPackageType(String packageType) {
+
+      this.packageType = packageType;
+    }
   }
 
+
   private String engagementName;
 
   private List<ReaderSetup> readerSetups = new ArrayList<>();

diff --git a/core/src/main/java/com/devonfw/tools/solicitor/common/LogMessages.java b/core/src/main/java/com/devonfw/tools/solicitor/common/LogMessages.java
@@ -107,7 +107,10 @@ public enum LogMessages {
   FAILED_READING_FILE(71, "Reading file '{}' failed"), //
   EMPTY_PACKAGE_URL(72, "The package URL is null or empty."), //
   EMPTY_PACKAGE_PATH(73, "The package path is null or empty."), //
-  CONTENT_FILE_TOO_LARGE(74,
+  EMPTY_PACKAGE_TYPE(74, "The package type is null or empty."), //
+  UNKNOWN_PACKAGE_TYPE(75,
+      "The CSV file contains packageType '{}' which is not supported and will be ignored. Solicitor reports might be incomplete"), //
+  CONTENT_FILE_TOO_LARGE(76,
       "The size of the content file '{}' is '{}' (max. allowed is '{}'). Reading will be skipped.");
 
   private final String message;

diff --git a/core/src/main/java/com/devonfw/tools/solicitor/config/ConfigFactory.java b/core/src/main/java/com/devonfw/tools/solicitor/config/ConfigFactory.java
@@ -132,6 +132,7 @@ public ModelRoot createConfig(String url) {
                   + "https://github.com/devonfw/solicitor/issues/263");
         }
         rs.setRepoType(rc.getRepoType());
+        rs.setPackageType(rc.getPackageType());
         rs.setConfiguration(rc.getConfiguration());
         rs = resolvePlaceholdersInReader(rs, placeHolderMap);
         this.solicitorSetup.getReaderSetups().add(rs);

diff --git a/core/src/main/java/com/devonfw/tools/solicitor/config/ReaderConfig.java b/core/src/main/java/com/devonfw/tools/solicitor/config/ReaderConfig.java
@@ -27,6 +27,9 @@ public class ReaderConfig {
 
   @JsonProperty
   private String repoType;
+
+  @JsonProperty
+  private String packageType;
 
   @JsonProperty
   private Map<String, String> configuration;
@@ -49,6 +52,16 @@ public String getRepoType() {
 
     return this.repoType;
   }
+
+  /**
+   * This method gets the field <code>packageType</code>.
+   *
+   * @return the field packageType
+   */
+  public String getPackageType() {
+
+    return this.packageType;
+  }
 
   /**
    * This method gets the field <code>configuration</code>.
@@ -99,7 +112,16 @@ public void setRepoType(String repoType) {
 
     this.repoType = repoType;
   }
+
+  /**
+   * This method sets the field <code>packageType</code>.
+   *
+   * @param packageType the new value of the field packageType
+   */
+  public void setPackageType(String packageType) {
 
+    this.packageType = packageType;
+  }
   /**
    * This method sets the field <code>configuration</code>.
    *

diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/Reader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/Reader.java
@@ -36,9 +36,10 @@ public interface Reader {
    * @param application all read {@link ApplicationComponent} need to be linked with this {@link Application}
    * @param usagePattern the {@link UsagePattern} which applies for all read {@link ApplicationComponent}s
    * @param repoType the type of Repository to download the sources from
+   * @param packageType the packageType to create the packageUrl
    * @param configuration optional configuration settings for readers
    */
   public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern,
-      String repoType, Map<String, String> configuration);
+      String repoType, String packageType, Map<String, String> configuration);
 
 }
diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/csv/CsvReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/csv/CsvReader.java
@@ -14,13 +14,19 @@
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVRecord;
 import org.springframework.stereotype.Component;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import com.devonfw.tools.solicitor.common.LogMessages;
+import com.devonfw.tools.solicitor.common.PackageURLHelper;
 import com.devonfw.tools.solicitor.common.SolicitorRuntimeException;
 import com.devonfw.tools.solicitor.model.inventory.ApplicationComponent;
 import com.devonfw.tools.solicitor.model.masterdata.Application;
 import com.devonfw.tools.solicitor.model.masterdata.UsagePattern;
 import com.devonfw.tools.solicitor.reader.AbstractReader;
 import com.devonfw.tools.solicitor.reader.Reader;
+import com.devonfw.tools.solicitor.reader.maven.MavenReader;
+import com.github.packageurl.PackageURL;
 
 /**
  * A {@link Reader} for files in CSV format.
@@ -46,6 +52,8 @@
 
 @Component
 public class CsvReader extends AbstractReader implements Reader {
+  private Logger logger = LoggerFactory.getLogger(CsvReader.class); // not static final for testing
+                                                                    // purposes
 
   /**
    * The supported type of this {@link Reader}.
@@ -62,7 +70,7 @@ public Set<String> getSupportedTypes() {
   /** {@inheritDoc} */
   @Override
   public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern,
-      String repoType, Map<String, String> configuration) {
+      String repoType, String packageType, Map<String, String> configuration) {
 
     int components = 0;
     int licenses = 0;
@@ -214,6 +222,7 @@ public void readInventory(String type, String sourceUrl, Application application
           appComponent.setVersion(version);
           appComponent.setUsagePattern(usagePattern);
           appComponent.setRepoType(repoType);
+          appComponent.setPackageUrl(getPackageURL(packageType, groupId, artifactId, version));
 
           // merge ApplicationComponentImpl with same key if they appear
           // on
@@ -248,6 +257,8 @@ public void readInventory(String type, String sourceUrl, Application application
           appComponent.setVersion(record.get(2));
           appComponent.setUsagePattern(usagePattern);
           appComponent.setRepoType(repoType);
+          appComponent.setPackageUrl(getPackageURL(packageType, record.get(0), record.get(1), record.get(2)));
+
           // merge ApplicationComponentImpl with same key if they appear
           // on
           // subsequent lines (multilicensing)
@@ -275,4 +286,43 @@ public void readInventory(String type, String sourceUrl, Application application
 
   }
 
+  /**
+   * Call the appropriate {@link PackageURLHelper} method to create a packageURL. Returns null if no
+   * {@link PackageURLHelper} exists for the packageType.
+   *
+   * @param packageType the package type
+   * @param groupId the groupId if available
+   * @param artifactId the artifactId
+   * @param version the version
+   * @return the created PackageURL
+   */
+  public String getPackageURL(String packageType, String groupId, String artifactId, String version) {
+
+    if (packageType == null || packageType.isEmpty()) {
+      this.logger.warn(LogMessages.EMPTY_PACKAGE_TYPE.msg(), packageType);
+      return null;
+    }
+    switch (packageType) {
+      case "maven":
+        return PackageURLHelper.fromMavenCoordinates(groupId, artifactId, version).toString();
+      case "npm":
+        return PackageURLHelper.fromNpmPackageNameAndVersion(artifactId, version).toString();
+      case "pypi":
+        return PackageURLHelper.fromPyPICoordinates(artifactId, version).toString();
+      default:
+        this.logger.warn(LogMessages.UNKNOWN_PACKAGE_TYPE.msg(), packageType);
+        return null;
+    }
+  }
+
+  /**
+   * Sets the logger. Available for testing purposes only.
+   *
+   * @param logger the logger
+   */
+  void setLogger(Logger logger) {
+
+    this.logger = logger;
+  }
+
 }
diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/cyclonedx/CyclonedxReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/cyclonedx/CyclonedxReader.java
@@ -55,7 +55,7 @@ public Set<String> getSupportedTypes() {
   /** {@inheritDoc} */
   @Override
   public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern,
-      String repoType, Map<String, String> configuration) {
+      String repoType, String packageType, Map<String, String> configuration) {
 
     int componentCount = 0;
     int licenseCount = 0;

diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/gradle/GradleReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/gradle/GradleReader.java
@@ -61,7 +61,7 @@ public Set<String> getSupportedTypes() {
   /** {@inheritDoc} */
   @Override
   public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern,
-      String repoType, Map<String, String> configuration) {
+      String repoType, String packageType, Map<String, String> configuration) {
 
     this.deprecationChecker.check(false,
         "Use of Reader of type 'gradle' is deprecated, use 'gradle2' instead. See https://github.com/devonfw/solicitor/issues/58");

diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/gradle/GradleReader2.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/gradle/GradleReader2.java
@@ -48,7 +48,7 @@ public Set<String> getSupportedTypes() {
   /** {@inheritDoc} */
   @Override
   public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern,
-      String repoType, Map<String, String> configuration) {
+      String repoType, String packageType, Map<String, String> configuration) {
 
     int components = 0;
     int licenses = 0;

diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/maven/MavenReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/maven/MavenReader.java
@@ -59,7 +59,7 @@ public Set<String> getSupportedTypes() {
   /** {@inheritDoc} */
   @Override
   public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern,
-      String repoType, Map<String, String> configuration) {
+      String repoType, String packageType, Map<String, String> configuration) {
 
     int components = 0;
     int licenses = 0;

diff --git a/...in/java/com/devonfw/tools/solicitor/reader/npmlicensechecker/NpmLicenseCheckerReader.java b/...in/java/com/devonfw/tools/solicitor/reader/npmlicensechecker/NpmLicenseCheckerReader.java
@@ -48,7 +48,7 @@ public Set<String> getSupportedTypes() {
   @SuppressWarnings("rawtypes")
   @Override
   public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern,
-      String repoType, Map<String, String> configuration) {
+      String repoType, String packageType, Map<String, String> configuration) {
 
     int componentCount = 0;
     int licenseCount = 0;

diff --git a/...in/java/com/devonfw/tools/solicitor/reader/npmlicensecrawler/NpmLicenseCrawlerReader.java b/...in/java/com/devonfw/tools/solicitor/reader/npmlicensecrawler/NpmLicenseCrawlerReader.java
@@ -67,7 +67,7 @@ public Set<String> getSupportedTypes() {
   /** {@inheritDoc} */
   @Override
   public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern,
-      String repoType, Map<String, String> configuration) {
+      String repoType, String packageType, Map<String, String> configuration) {
 
     this.deprecationChecker.check(false,
         "Use of Reader of type '" + SUPPORTED_TYPE

diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/ort/OrtReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/ort/OrtReader.java
@@ -44,7 +44,7 @@ public Set<String> getSupportedTypes() {
   @SuppressWarnings("rawtypes")
   @Override
   public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern,
-      String repoType, Map<String, String> configuration) {
+      String repoType, String packageType, Map<String, String> configuration) {
 
     int componentCount = 0;
     int licenseCount = 0;

diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/piplicenses/PipLicensesReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/piplicenses/PipLicensesReader.java
@@ -44,7 +44,7 @@ public Set<String> getSupportedTypes() {
   @SuppressWarnings("rawtypes")
   @Override
   public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern,
-      String repoType, Map<String, String> configuration) {
+      String repoType, String packageType, Map<String, String> configuration) {
 
     int componentCount = 0;
     int licenseCount = 0;

diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/yarn/YarnReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/yarn/YarnReader.java
@@ -48,7 +48,7 @@ public Set<String> getSupportedTypes() {
   @SuppressWarnings("rawtypes")
   @Override
   public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern,
-      String repoType, Map<String, String> configuration) {
+      String repoType, String packageType, Map<String, String> configuration) {
 
     String content = cutSourceJson(sourceUrl);
 

diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReader.java
@@ -55,7 +55,7 @@ public Set<String> getSupportedTypes() {
   @SuppressWarnings("rawtypes")
   @Override
   public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern,
-      String repoType, Map<String, String> configuration) {
+      String repoType, String packageType, Map<String, String> configuration) {
 
     String content = readAndPreprocessJson(sourceUrl);