Skip to content

Commit

Permalink
Support for CPAN repository
Browse files Browse the repository at this point in the history
Signed-off-by: Walter de Boer <[email protected]>
  • Loading branch information
Walter de Boer committed Mar 11, 2023
1 parent 3a5989a commit d324a67
Show file tree
Hide file tree
Showing 13 changed files with 274 additions and 93 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,5 @@ target/
.idea/
.vscode/

# windows
~
7 changes: 4 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,17 +65,18 @@ CI/CD environments.
* Robust policy engine with support for global and per-project policies
* Security risk and compliance
* License risk and compliance
* Operational risk and compliance
* Operational risk and compliance
* Ecosystem agnostic with built-in repository support for:
* Cargo (Rust)
* Composer (PHP)
* Gems (Ruby)
* Hex (Erlang/Elixir)
* Maven (Java)
* NPM (Javascript)
* CPAN (Perl)
* NuGet (.NET)
* Pypi (Python)
* More coming soon.
* More coming soon.
* Identifies APIs and external service components including:
* Service provider
* Endpoint URIs
Expand Down Expand Up @@ -143,7 +144,7 @@ Dependency-Track has three distribution variants. They are:

| Package | Package Format | Recommended | Supported | Docker | Download |
|:-----------|:------------------------|:-----------:|:---------:|:------:|:--------:|
| API Server | Executable WAR |||||
| API Server | Executable WAR |||||
| Frontend | Single Page Application |||||
| Bundled | Executable WAR || ☑️ |||

Expand Down
15 changes: 8 additions & 7 deletions docs/_docs/datasources/repositories.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,12 @@ Dependency-Track supports the following default repositories:
| npm | NPM | 1 |
| nuget | NuGet | 1 |
| pypi | PyPi | 1 |
| cpan | CPAN | 1 |


Additional repositories can be added for each supported ecosystem. Additionally, repositories can be enabled or disabled
as well as identified as 'internal'. For internal repositories, a username and/or password may be specified. This allows
Dependency-Track to scan packages from private repositories that require basic authentication, like Azure Artifacts or
as well as identified as 'internal'. For internal repositories, a username and/or password may be specified. This allows
Dependency-Track to scan packages from private repositories that require basic authentication, like Azure Artifacts or
Artifactory.

![repositories](/images/screenshots/repositories.png)
Expand All @@ -46,13 +47,13 @@ Artifactory.

### Outdated Version Tracking

One primary use-case for the support of repositories is the identification of outdated components. By leveraging tight
integration with APIs available from various repositories, the platform can identify outdated versions of components
across multiple ecosystems. Dependency-Track relies on Package URL (PURL) to identify the ecosystem a component belongs
to, the metadata about the component, and uses that data to query the various repositories capable of supporting the
One primary use-case for the support of repositories is the identification of outdated components. By leveraging tight
integration with APIs available from various repositories, the platform can identify outdated versions of components
across multiple ecosystems. Dependency-Track relies on Package URL (PURL) to identify the ecosystem a component belongs
to, the metadata about the component, and uses that data to query the various repositories capable of supporting the
components ecosystem.

Package URL is natively supported in the [CycloneDX](http://cyclonedx.org/) BOM specification. By using CycloneDX as a
Package URL is natively supported in the [CycloneDX](http://cyclonedx.org/) BOM specification. By using CycloneDX as a
means to populate project dependencies, organizations benefit from the many use-cases Package URL provides, including
leveraging repositories to identify outdated components.

Expand Down
3 changes: 3 additions & 0 deletions src/main/java/org/dependencytrack/model/RepositoryType.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
*/
public enum RepositoryType {

CPAN,
MAVEN,
NPM,
GEM,
Expand All @@ -50,6 +51,8 @@ public static RepositoryType resolve(PackageURL packageURL) {
return MAVEN;
} else if (PackageURL.StandardTypes.NPM.equals(type)) {
return NPM;
} else if ("cpan".equals(type)) { // Not defined in StandardTypes
return CPAN;
} else if (PackageURL.StandardTypes.GEM.equals(type)) {
return GEM;
} else if (PackageURL.StandardTypes.PYPI.equals(type)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@
*/
package org.dependencytrack.persistence;

import alpine.common.logging.Logger;
import alpine.model.ManagedUser;
import alpine.model.Permission;
import alpine.model.Team;
import alpine.server.auth.PasswordService;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import org.dependencytrack.RequirementsVerifier;
import org.dependencytrack.auth.Permissions;
import org.dependencytrack.model.ConfigPropertyConstants;
Expand All @@ -32,12 +32,11 @@
import org.dependencytrack.parser.spdx.json.SpdxLicenseDetailParser;
import org.dependencytrack.persistence.defaults.DefaultLicenseGroupImporter;
import org.dependencytrack.util.NotificationUtil;

import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import alpine.common.logging.Logger;
import alpine.model.ManagedUser;
import alpine.model.Permission;
import alpine.model.Team;
import alpine.server.auth.PasswordService;

/**
* Creates default objects on an empty database.
Expand Down Expand Up @@ -207,6 +206,7 @@ private List<Permission> getAutomationPermissions(final List<Permission> fullLis
private void loadDefaultRepositories() {
try (QueryManager qm = new QueryManager()) {
LOGGER.info("Synchronizing default repositories to datastore");
qm.createRepository(RepositoryType.CPAN, "cpan-public-registry", "https://fastapi.metacpan.org/v1/", true, false);
qm.createRepository(RepositoryType.GEM, "rubygems.org", "https://rubygems.org/", true, false);
qm.createRepository(RepositoryType.HEX, "hex.pm", "https://hex.pm/", true, false);
qm.createRepository(RepositoryType.MAVEN, "central", "https://repo1.maven.org/maven2/", true, false);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,18 @@
*/
package org.dependencytrack.tasks;

import alpine.common.logging.Logger;
import alpine.event.framework.Event;
import alpine.event.framework.LoggableSubscriber;
import alpine.model.ConfigProperty;
import alpine.notification.Notification;
import alpine.notification.NotificationLevel;
import com.github.packageurl.MalformedPackageURLException;
import com.github.packageurl.PackageURL;
import com.github.packageurl.PackageURLBuilder;
import io.pebbletemplates.pebble.PebbleEngine;
import io.pebbletemplates.pebble.template.PebbleTemplate;
import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ACCESS_TOKEN;
import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ENABLED;
import java.io.IOException;
import java.io.StringWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.CloseableHttpResponse;
Expand All @@ -53,20 +54,17 @@
import org.dependencytrack.parser.github.graphql.model.PageableList;
import org.dependencytrack.persistence.QueryManager;
import org.json.JSONObject;

import java.io.IOException;
import java.io.StringWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ACCESS_TOKEN;
import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ENABLED;
import com.github.packageurl.MalformedPackageURLException;
import com.github.packageurl.PackageURL;
import com.github.packageurl.PackageURLBuilder;
import alpine.common.logging.Logger;
import alpine.event.framework.Event;
import alpine.event.framework.LoggableSubscriber;
import alpine.model.ConfigProperty;
import alpine.notification.Notification;
import alpine.notification.NotificationLevel;
import io.pebbletemplates.pebble.PebbleEngine;
import io.pebbletemplates.pebble.template.PebbleTemplate;

public class GitHubAdvisoryMirrorTask implements LoggableSubscriber {

Expand Down Expand Up @@ -326,6 +324,13 @@ private VulnerableSoftware mapVulnerabilityToVulnerableSoftware(final QueryManag
return null;
}

/**
* Map GitHub ecosystem to PackageURL type
*
* @param ecosystem GitHub ecosystem
* @return the PackageURL for the ecosystem
* @see https://github.com/github/advisory-database
*/
private String mapGitHubEcosystemToPurlType(final String ecosystem) {
switch (ecosystem.toUpperCase()) {
case "MAVEN":
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
/*
* This file is part of Dependency-Track.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* Copyright (c) Steve Springett. All Rights Reserved.
*/
package org.dependencytrack.tasks.repositories;

import java.io.IOException;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.util.EntityUtils;
import org.dependencytrack.model.Component;
import org.dependencytrack.model.RepositoryType;
import org.json.JSONObject;
import alpine.common.logging.Logger;

/**
* An IMetaAnalyzer implementation that supports CPAN.
*/
public class CpanMetaAnalyzer extends AbstractMetaAnalyzer {

private static final Logger LOGGER = Logger.getLogger(CpanMetaAnalyzer.class);
private static final String DEFAULT_BASE_URL = "https://fastapi.metacpan.org/v1";
private static final String API_URL = "/module/%s";

CpanMetaAnalyzer() {
this.baseUrl = DEFAULT_BASE_URL;
}

/**
* {@inheritDoc}
*/
public boolean isApplicable(final Component component) {
return component.getPurl() != null && "cpan".equals(component.getPurl().getType());
}

/**
* {@inheritDoc}
*/
public RepositoryType supportedRepositoryType() {
return RepositoryType.CPAN;
}

/**
* {@inheritDoc}
*/
public MetaModel analyze(final Component component) {
final MetaModel meta = new MetaModel(component);
if (component.getPurl() != null) {

final String packageName;
if (component.getPurl().getNamespace() != null) {
packageName = component.getPurl().getNamespace() + "%2F" + component.getPurl().getName();
} else {
packageName = component.getPurl().getName();
}

final String url = String.format(baseUrl + API_URL, packageName);
try (final CloseableHttpResponse response = processHttpRequest(url)) {
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
if (response.getEntity()!=null) {
String responseString = EntityUtils.toString(response.getEntity());
var jsonObject = new JSONObject(responseString);
final String latest = jsonObject.optString("version");
if (latest != null) {
meta.setLatestVersion(latest);
}
final String published = jsonObject.optString("date");
if (published != null) {
final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
try {
meta.setPublishedTimestamp(dateFormat.parse(published));
} catch (ParseException e) {
LOGGER.warn("An error occurred while parsing upload time", e);
}
}
}
} else {
handleUnexpectedHttpResponse(LOGGER, url, response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase(), component);
}
} catch (IOException e) {
handleRequestException(LOGGER, e);
}
}
return meta;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@
*/
package org.dependencytrack.tasks.repositories;

import com.github.packageurl.PackageURL;
import org.dependencytrack.exception.MetaAnalyzerException;
import org.dependencytrack.model.Component;
import org.dependencytrack.model.RepositoryType;
import com.github.packageurl.PackageURL;

/**
* Interface that defines Repository Meta Analyzers.
Expand Down Expand Up @@ -90,6 +90,11 @@ static IMetaAnalyzer build(Component component) {
if (analyzer.isApplicable(component)) {
return analyzer;
}
} else if ("cpan".equals(component.getPurl().getType())) {
IMetaAnalyzer analyzer = new CpanMetaAnalyzer();
if (analyzer.isApplicable(component)) {
return analyzer;
}
} else if (PackageURL.StandardTypes.GEM.equals(component.getPurl().getType())) {
IMetaAnalyzer analyzer = new GemMetaAnalyzer();
if (analyzer.isApplicable(component)) {
Expand Down
Loading

0 comments on commit d324a67

Please sign in to comment.