diff --git a/.gitignore b/.gitignore index 506e381e42..d73962f835 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ target/ .idea/ .vscode/ +# windows +~ \ No newline at end of file diff --git a/README.md b/README.md index 0abbb5bae7..113141ff58 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ CI/CD environments. * Robust policy engine with support for global and per-project policies * Security risk and compliance * License risk and compliance - * Operational risk and compliance + * Operational risk and compliance * Ecosystem agnostic with built-in repository support for: * Cargo (Rust) * Composer (PHP) @@ -73,9 +73,10 @@ CI/CD environments. * Hex (Erlang/Elixir) * Maven (Java) * NPM (Javascript) + * CPAN (Perl) * NuGet (.NET) * Pypi (Python) - * More coming soon. + * More coming soon. * Identifies APIs and external service components including: * Service provider * Endpoint URIs @@ -143,7 +144,7 @@ Dependency-Track has three distribution variants. They are: | Package | Package Format | Recommended | Supported | Docker | Download | |:-----------|:------------------------|:-----------:|:---------:|:------:|:--------:| -| API Server | Executable WAR | ✅ | ✅ | ✅ | ✅ | +| API Server | Executable WAR | ✅ | ✅ | ✅ | ✅ | | Frontend | Single Page Application | ✅ | ✅ | ✅ | ✅ | | Bundled | Executable WAR | ❌ | ☑️ | ✅ | ✅ | diff --git a/docs/_docs/datasources/repositories.md b/docs/_docs/datasources/repositories.md index b8644bc5b4..5ecc964c3d 100644 --- a/docs/_docs/datasources/repositories.md +++ b/docs/_docs/datasources/repositories.md @@ -30,11 +30,12 @@ Dependency-Track supports the following default repositories: | npm | NPM | 1 | | nuget | NuGet | 1 | | pypi | PyPi | 1 | +| cpan | CPAN | 1 | Additional repositories can be added for each supported ecosystem. Additionally, repositories can be enabled or disabled -as well as identified as 'internal'. For internal repositories, a username and/or password may be specified. This allows -Dependency-Track to scan packages from private repositories that require basic authentication, like Azure Artifacts or +as well as identified as 'internal'. For internal repositories, a username and/or password may be specified. This allows +Dependency-Track to scan packages from private repositories that require basic authentication, like Azure Artifacts or Artifactory. ![repositories](/images/screenshots/repositories.png) @@ -46,13 +47,13 @@ Artifactory. ### Outdated Version Tracking -One primary use-case for the support of repositories is the identification of outdated components. By leveraging tight -integration with APIs available from various repositories, the platform can identify outdated versions of components -across multiple ecosystems. Dependency-Track relies on Package URL (PURL) to identify the ecosystem a component belongs -to, the metadata about the component, and uses that data to query the various repositories capable of supporting the +One primary use-case for the support of repositories is the identification of outdated components. By leveraging tight +integration with APIs available from various repositories, the platform can identify outdated versions of components +across multiple ecosystems. Dependency-Track relies on Package URL (PURL) to identify the ecosystem a component belongs +to, the metadata about the component, and uses that data to query the various repositories capable of supporting the components ecosystem. -Package URL is natively supported in the [CycloneDX](http://cyclonedx.org/) BOM specification. By using CycloneDX as a +Package URL is natively supported in the [CycloneDX](http://cyclonedx.org/) BOM specification. By using CycloneDX as a means to populate project dependencies, organizations benefit from the many use-cases Package URL provides, including leveraging repositories to identify outdated components. diff --git a/src/main/java/org/dependencytrack/model/RepositoryType.java b/src/main/java/org/dependencytrack/model/RepositoryType.java index 9229f1357a..5c5f90dd44 100644 --- a/src/main/java/org/dependencytrack/model/RepositoryType.java +++ b/src/main/java/org/dependencytrack/model/RepositoryType.java @@ -28,6 +28,7 @@ */ public enum RepositoryType { + CPAN, MAVEN, NPM, GEM, @@ -50,6 +51,8 @@ public static RepositoryType resolve(PackageURL packageURL) { return MAVEN; } else if (PackageURL.StandardTypes.NPM.equals(type)) { return NPM; + } else if ("cpan".equals(type)) { // Not defined in StandardTypes + return CPAN; } else if (PackageURL.StandardTypes.GEM.equals(type)) { return GEM; } else if (PackageURL.StandardTypes.PYPI.equals(type)) { diff --git a/src/main/java/org/dependencytrack/persistence/DefaultObjectGenerator.java b/src/main/java/org/dependencytrack/persistence/DefaultObjectGenerator.java index b05f92b5b4..ea903b73d0 100644 --- a/src/main/java/org/dependencytrack/persistence/DefaultObjectGenerator.java +++ b/src/main/java/org/dependencytrack/persistence/DefaultObjectGenerator.java @@ -18,11 +18,11 @@ */ package org.dependencytrack.persistence; -import alpine.common.logging.Logger; -import alpine.model.ManagedUser; -import alpine.model.Permission; -import alpine.model.Team; -import alpine.server.auth.PasswordService; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; import org.dependencytrack.RequirementsVerifier; import org.dependencytrack.auth.Permissions; import org.dependencytrack.model.ConfigPropertyConstants; @@ -32,12 +32,11 @@ import org.dependencytrack.parser.spdx.json.SpdxLicenseDetailParser; import org.dependencytrack.persistence.defaults.DefaultLicenseGroupImporter; import org.dependencytrack.util.NotificationUtil; - -import javax.servlet.ServletContextEvent; -import javax.servlet.ServletContextListener; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; +import alpine.common.logging.Logger; +import alpine.model.ManagedUser; +import alpine.model.Permission; +import alpine.model.Team; +import alpine.server.auth.PasswordService; /** * Creates default objects on an empty database. @@ -207,6 +206,7 @@ private List getAutomationPermissions(final List fullLis private void loadDefaultRepositories() { try (QueryManager qm = new QueryManager()) { LOGGER.info("Synchronizing default repositories to datastore"); + qm.createRepository(RepositoryType.CPAN, "cpan-public-registry", "https://fastapi.metacpan.org/v1/", true, false); qm.createRepository(RepositoryType.GEM, "rubygems.org", "https://rubygems.org/", true, false); qm.createRepository(RepositoryType.HEX, "hex.pm", "https://hex.pm/", true, false); qm.createRepository(RepositoryType.MAVEN, "central", "https://repo1.maven.org/maven2/", true, false); diff --git a/src/main/java/org/dependencytrack/tasks/GitHubAdvisoryMirrorTask.java b/src/main/java/org/dependencytrack/tasks/GitHubAdvisoryMirrorTask.java index 5177c17f76..3d5e7d2482 100644 --- a/src/main/java/org/dependencytrack/tasks/GitHubAdvisoryMirrorTask.java +++ b/src/main/java/org/dependencytrack/tasks/GitHubAdvisoryMirrorTask.java @@ -18,17 +18,18 @@ */ package org.dependencytrack.tasks; -import alpine.common.logging.Logger; -import alpine.event.framework.Event; -import alpine.event.framework.LoggableSubscriber; -import alpine.model.ConfigProperty; -import alpine.notification.Notification; -import alpine.notification.NotificationLevel; -import com.github.packageurl.MalformedPackageURLException; -import com.github.packageurl.PackageURL; -import com.github.packageurl.PackageURLBuilder; -import io.pebbletemplates.pebble.PebbleEngine; -import io.pebbletemplates.pebble.template.PebbleTemplate; +import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ACCESS_TOKEN; +import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ENABLED; +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.commons.lang3.tuple.Pair; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; @@ -53,20 +54,17 @@ import org.dependencytrack.parser.github.graphql.model.PageableList; import org.dependencytrack.persistence.QueryManager; import org.json.JSONObject; - -import java.io.IOException; -import java.io.StringWriter; -import java.io.Writer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ACCESS_TOKEN; -import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ENABLED; +import com.github.packageurl.MalformedPackageURLException; +import com.github.packageurl.PackageURL; +import com.github.packageurl.PackageURLBuilder; +import alpine.common.logging.Logger; +import alpine.event.framework.Event; +import alpine.event.framework.LoggableSubscriber; +import alpine.model.ConfigProperty; +import alpine.notification.Notification; +import alpine.notification.NotificationLevel; +import io.pebbletemplates.pebble.PebbleEngine; +import io.pebbletemplates.pebble.template.PebbleTemplate; public class GitHubAdvisoryMirrorTask implements LoggableSubscriber { @@ -326,6 +324,13 @@ private VulnerableSoftware mapVulnerabilityToVulnerableSoftware(final QueryManag return null; } + /** + * Map GitHub ecosystem to PackageURL type + * + * @param ecosystem GitHub ecosystem + * @return the PackageURL for the ecosystem + * @see https://github.com/github/advisory-database + */ private String mapGitHubEcosystemToPurlType(final String ecosystem) { switch (ecosystem.toUpperCase()) { case "MAVEN": diff --git a/src/main/java/org/dependencytrack/tasks/repositories/CpanMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/CpanMetaAnalyzer.java new file mode 100644 index 0000000000..f310322a18 --- /dev/null +++ b/src/main/java/org/dependencytrack/tasks/repositories/CpanMetaAnalyzer.java @@ -0,0 +1,104 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Steve Springett. All Rights Reserved. + */ +package org.dependencytrack.tasks.repositories; + +import java.io.IOException; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.util.EntityUtils; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.RepositoryType; +import org.json.JSONObject; +import alpine.common.logging.Logger; + +/** + * An IMetaAnalyzer implementation that supports CPAN. + */ +public class CpanMetaAnalyzer extends AbstractMetaAnalyzer { + + private static final Logger LOGGER = Logger.getLogger(CpanMetaAnalyzer.class); + private static final String DEFAULT_BASE_URL = "https://fastapi.metacpan.org/v1"; + private static final String API_URL = "/module/%s"; + + CpanMetaAnalyzer() { + this.baseUrl = DEFAULT_BASE_URL; + } + + /** + * {@inheritDoc} + */ + public boolean isApplicable(final Component component) { + return component.getPurl() != null && "cpan".equals(component.getPurl().getType()); + } + + /** + * {@inheritDoc} + */ + public RepositoryType supportedRepositoryType() { + return RepositoryType.CPAN; + } + + /** + * {@inheritDoc} + */ + public MetaModel analyze(final Component component) { + final MetaModel meta = new MetaModel(component); + if (component.getPurl() != null) { + + final String packageName; + if (component.getPurl().getNamespace() != null) { + packageName = component.getPurl().getNamespace() + "%2F" + component.getPurl().getName(); + } else { + packageName = component.getPurl().getName(); + } + + final String url = String.format(baseUrl + API_URL, packageName); + try (final CloseableHttpResponse response = processHttpRequest(url)) { + if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + if (response.getEntity()!=null) { + String responseString = EntityUtils.toString(response.getEntity()); + var jsonObject = new JSONObject(responseString); + final String latest = jsonObject.optString("version"); + if (latest != null) { + meta.setLatestVersion(latest); + } + final String published = jsonObject.optString("date"); + if (published != null) { + final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + try { + meta.setPublishedTimestamp(dateFormat.parse(published)); + } catch (ParseException e) { + LOGGER.warn("An error occurred while parsing upload time", e); + } + } + } + } else { + handleUnexpectedHttpResponse(LOGGER, url, response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase(), component); + } + } catch (IOException e) { + handleRequestException(LOGGER, e); + } + } + return meta; + } + +} diff --git a/src/main/java/org/dependencytrack/tasks/repositories/IMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/IMetaAnalyzer.java index d8f43f310d..42a526b8dd 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/IMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/IMetaAnalyzer.java @@ -18,10 +18,10 @@ */ package org.dependencytrack.tasks.repositories; -import com.github.packageurl.PackageURL; import org.dependencytrack.exception.MetaAnalyzerException; import org.dependencytrack.model.Component; import org.dependencytrack.model.RepositoryType; +import com.github.packageurl.PackageURL; /** * Interface that defines Repository Meta Analyzers. @@ -90,6 +90,11 @@ static IMetaAnalyzer build(Component component) { if (analyzer.isApplicable(component)) { return analyzer; } + } else if ("cpan".equals(component.getPurl().getType())) { + IMetaAnalyzer analyzer = new CpanMetaAnalyzer(); + if (analyzer.isApplicable(component)) { + return analyzer; + } } else if (PackageURL.StandardTypes.GEM.equals(component.getPurl().getType())) { IMetaAnalyzer analyzer = new GemMetaAnalyzer(); if (analyzer.isApplicable(component)) { diff --git a/src/main/java/org/dependencytrack/tasks/scanners/SnykAnalysisTask.java b/src/main/java/org/dependencytrack/tasks/scanners/SnykAnalysisTask.java index 7ccb7a99d4..0c2cd82476 100644 --- a/src/main/java/org/dependencytrack/tasks/scanners/SnykAnalysisTask.java +++ b/src/main/java/org/dependencytrack/tasks/scanners/SnykAnalysisTask.java @@ -18,22 +18,20 @@ */ package org.dependencytrack.tasks.scanners; -import alpine.Config; -import alpine.common.logging.Logger; -import alpine.common.metrics.Metrics; -import alpine.common.util.UrlUtil; -import alpine.event.framework.Event; -import alpine.event.framework.LoggableUncaughtExceptionHandler; -import alpine.event.framework.Subscriber; -import alpine.model.ConfigProperty; -import alpine.notification.Notification; -import alpine.notification.NotificationLevel; -import alpine.security.crypto.DataEncryption; -import com.github.packageurl.PackageURL; -import io.github.resilience4j.micrometer.tagged.TaggedRetryMetrics; -import io.github.resilience4j.retry.Retry; -import io.github.resilience4j.retry.RetryConfig; -import io.github.resilience4j.retry.RetryRegistry; +import static io.github.resilience4j.core.IntervalFunction.ofExponentialBackoff; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.apache.http.Header; @@ -62,22 +60,22 @@ import org.dependencytrack.util.RoundRobinAccessor; import org.json.JSONArray; import org.json.JSONObject; - -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static io.github.resilience4j.core.IntervalFunction.ofExponentialBackoff; +import com.github.packageurl.PackageURL; +import alpine.Config; +import alpine.common.logging.Logger; +import alpine.common.metrics.Metrics; +import alpine.common.util.UrlUtil; +import alpine.event.framework.Event; +import alpine.event.framework.LoggableUncaughtExceptionHandler; +import alpine.event.framework.Subscriber; +import alpine.model.ConfigProperty; +import alpine.notification.Notification; +import alpine.notification.NotificationLevel; +import alpine.security.crypto.DataEncryption; +import io.github.resilience4j.micrometer.tagged.TaggedRetryMetrics; +import io.github.resilience4j.retry.Retry; +import io.github.resilience4j.retry.RetryConfig; +import io.github.resilience4j.retry.RetryRegistry; /** * Subscriber task that performs an analysis of component using Snyk vulnerability REST API. @@ -94,6 +92,7 @@ public class SnykAnalysisTask extends BaseComponentAnalyzerTask implements Cache PackageURL.StandardTypes.GEM, PackageURL.StandardTypes.GENERIC, PackageURL.StandardTypes.HEX, + "cpan", // Not defined in StandardTypes PackageURL.StandardTypes.MAVEN, PackageURL.StandardTypes.NPM, PackageURL.StandardTypes.NUGET, diff --git a/src/test/java/org/dependencytrack/model/RepositoryTypeTest.java b/src/test/java/org/dependencytrack/model/RepositoryTypeTest.java index 19bdef2931..d96a870153 100644 --- a/src/test/java/org/dependencytrack/model/RepositoryTypeTest.java +++ b/src/test/java/org/dependencytrack/model/RepositoryTypeTest.java @@ -18,14 +18,15 @@ */ package org.dependencytrack.model; -import com.github.packageurl.PackageURL; import org.junit.Assert; import org.junit.Test; +import com.github.packageurl.PackageURL; public class RepositoryTypeTest { @Test public void testEnums() { + Assert.assertEquals("CPAN", RepositoryType.CPAN.name()); Assert.assertEquals("MAVEN", RepositoryType.MAVEN.name()); Assert.assertEquals("NPM", RepositoryType.NPM.name()); Assert.assertEquals("GEM", RepositoryType.GEM.name()); @@ -41,6 +42,12 @@ public void testResolveMaven() throws Exception { Assert.assertEquals(RepositoryType.MAVEN, RepositoryType.resolve(purl)); } + @Test + public void testResolveCpan() throws Exception { + PackageURL purl = new PackageURL("pkg:cpan/artifact@1.0.0"); + Assert.assertEquals(RepositoryType.CPAN, RepositoryType.resolve(purl)); + } + @Test public void testResolveNpm() throws Exception { PackageURL purl = new PackageURL("pkg:npm/artifact@1.0.0"); @@ -76,4 +83,4 @@ public void testResolveUnsupported() throws Exception { PackageURL purl = new PackageURL("pkg:generic/artifact@1.0.0"); Assert.assertEquals(RepositoryType.UNSUPPORTED, RepositoryType.resolve(purl)); } -} +} diff --git a/src/test/java/org/dependencytrack/persistence/DefaultObjectGeneratorTest.java b/src/test/java/org/dependencytrack/persistence/DefaultObjectGeneratorTest.java index 72fc588653..e72c52cb74 100644 --- a/src/test/java/org/dependencytrack/persistence/DefaultObjectGeneratorTest.java +++ b/src/test/java/org/dependencytrack/persistence/DefaultObjectGeneratorTest.java @@ -18,6 +18,7 @@ */ package org.dependencytrack.persistence; +import java.lang.reflect.Method; import org.dependencytrack.PersistenceCapableTest; import org.dependencytrack.auth.Permissions; import org.dependencytrack.model.ConfigPropertyConstants; @@ -25,8 +26,6 @@ import org.junit.Assert; import org.junit.Test; -import java.lang.reflect.Method; - public class DefaultObjectGeneratorTest extends PersistenceCapableTest { @Test @@ -72,7 +71,7 @@ public void testLoadDefaultRepositories() throws Exception { Method method = generator.getClass().getDeclaredMethod("loadDefaultRepositories"); method.setAccessible(true); method.invoke(generator); - Assert.assertEquals(13, qm.getAllRepositories().size()); + Assert.assertEquals(14, qm.getAllRepositories().size()); } @Test diff --git a/src/test/java/org/dependencytrack/resources/v1/RepositoryResourceTest.java b/src/test/java/org/dependencytrack/resources/v1/RepositoryResourceTest.java index 11f4250ba7..53a25b6c5b 100644 --- a/src/test/java/org/dependencytrack/resources/v1/RepositoryResourceTest.java +++ b/src/test/java/org/dependencytrack/resources/v1/RepositoryResourceTest.java @@ -18,8 +18,10 @@ */ package org.dependencytrack.resources.v1; -import alpine.server.filters.ApiFilter; -import alpine.server.filters.AuthenticationFilter; +import java.util.Date; +import javax.json.JsonArray; +import javax.json.JsonObject; +import javax.ws.rs.core.Response; import org.dependencytrack.ResourceTest; import org.dependencytrack.model.RepositoryMetaComponent; import org.dependencytrack.model.RepositoryType; @@ -31,11 +33,8 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; - -import javax.json.JsonArray; -import javax.json.JsonObject; -import javax.ws.rs.core.Response; -import java.util.Date; +import alpine.server.filters.ApiFilter; +import alpine.server.filters.AuthenticationFilter; public class RepositoryResourceTest extends ResourceTest { @@ -61,10 +60,10 @@ public void getRepositoriesTest() { .header(X_API_KEY, apiKey) .get(Response.class); Assert.assertEquals(200, response.getStatus(), 0); - Assert.assertEquals(String.valueOf(13), response.getHeaderString(TOTAL_COUNT_HEADER)); + Assert.assertEquals(String.valueOf(14), response.getHeaderString(TOTAL_COUNT_HEADER)); JsonArray json = parseJsonArray(response); Assert.assertNotNull(json); - Assert.assertEquals(13, json.size()); + Assert.assertEquals(14, json.size()); for (int i=0; i 0); + } + + @Test + public void testAnalyzerFutureQ() throws Exception { + Component component = new Component(); + component.setPurl(new PackageURL("pkg:cpan/Future::Q@0.27")); + + CpanMetaAnalyzer analyzer = new CpanMetaAnalyzer(); + Assert.assertTrue(analyzer.isApplicable(component)); + Assert.assertEquals(RepositoryType.CPAN, analyzer.supportedRepositoryType()); + MetaModel metaModel = analyzer.analyze(component); + Assert.assertTrue(new ComponentVersion("0.27").compareTo(new ComponentVersion(metaModel.getLatestVersion())) < 0); + Assert.assertTrue(new Date().compareTo(metaModel.getPublishedTimestamp()) > 0); + } +}