Merge branch 'main' into docs

google · Oct 23, 2024 · 4dea789 · 4dea789
2 parents 367abc4 + e054385
commit 4dea789
Show file tree

Hide file tree

Showing 25 changed files with 502 additions and 104 deletions.
diff --git a/.github/workflows/prerelease-check.yml b/.github/workflows/prerelease-check.yml
@@ -105,6 +105,28 @@ jobs:
         uses: ./.github/workflows/test-action
         with:
           codecov_token: ${{ secrets.CODECOV_TOKEN }}
+  generators:
+    permissions:
+      contents: read # to fetch code (actions/checkout)
+    name: (re)generate code
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+        with:
+          persist-credentials: false
+          ref: ${{ inputs.commit }}
+      - name: Set up Go
+        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          go-version: stable
+          check-latest: true
+      - name: Run generators
+        run: go generate ./...
+      - run: |
+          git diff --name-only \
+            | xargs -I '{}' bash -c \
+              'echo "::error file={}::This needs to be regenerated by running \`go generate ./...\`" && false'
   release-helper:
     permissions:
       contents: read # to fetch code (actions/checkout)

diff --git a/cmd/osv-scanner/__snapshots__/main_test.snap b/cmd/osv-scanner/__snapshots__/main_test.snap
@@ -1812,6 +1812,7 @@ Filtered 16 vulnerabilities from output
 | https://osv.dev/CVE-2021-36770      | 7.8  | Debian    | perl                           | 5.24.1-3+deb9u7                    | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
 | https://osv.dev/CVE-2023-31484      | 8.1  | Debian    | perl                           | 5.24.1-3+deb9u7                    | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
 | https://osv.dev/CVE-2023-47038      | 7.8  | Debian    | perl                           | 5.24.1-3+deb9u7                    | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
+| https://osv.dev/DLA-3926-1          |      | Debian    | perl                           | 5.24.1-3+deb9u7                    | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
 | https://osv.dev/DLA-3072-1          |      | Debian    | postgresql-11                  | 11.15-1.pgdg90+1                   | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
 | https://osv.dev/DLA-3189-1          |      | Debian    | postgresql-11                  | 11.15-1.pgdg90+1                   | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
 | https://osv.dev/DLA-3316-1          |      | Debian    | postgresql-11                  | 11.15-1.pgdg90+1                   | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
@@ -1989,6 +1990,7 @@ Filtered 16 vulnerabilities from output
 | https://osv.dev/CVE-2021-36770      | 7.8  | Debian    | perl                           | 5.24.1-3+deb9u7                    | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
 | https://osv.dev/CVE-2023-31484      | 8.1  | Debian    | perl                           | 5.24.1-3+deb9u7                    | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
 | https://osv.dev/CVE-2023-47038      | 7.8  | Debian    | perl                           | 5.24.1-3+deb9u7                    | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
+| https://osv.dev/DLA-3926-1          |      | Debian    | perl                           | 5.24.1-3+deb9u7                    | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
 | https://osv.dev/DLA-3072-1          |      | Debian    | postgresql-11                  | 11.15-1.pgdg90+1                   | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
 | https://osv.dev/DLA-3189-1          |      | Debian    | postgresql-11                  | 11.15-1.pgdg90+1                   | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
 | https://osv.dev/DLA-3316-1          |      | Debian    | postgresql-11                  | 11.15-1.pgdg90+1                   | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
@@ -2382,6 +2384,42 @@ No issues found
 
 ---
 
+[TestRun_MavenTransitive/resolve_transitive_dependencies_with_native_datda_source - 1]
+Scanned <rootdir>/fixtures/maven-transitive/registry.xml file as a pom.xml and found 59 packages
++-------------------------------------+------+-----------+-----------------------------------------------+---------+----------------------------------------+
+| OSV URL                             | CVSS | ECOSYSTEM | PACKAGE                                       | VERSION | SOURCE                                 |
++-------------------------------------+------+-----------+-----------------------------------------------+---------+----------------------------------------+
+| https://osv.dev/GHSA-cm6r-892j-jv2g | 6.1  | Maven     | com.google.android.gms:play-services-basement | 10.0.0  | fixtures/maven-transitive/registry.xml |
+| https://osv.dev/GHSA-7rjr-3q55-vv33 | 9.0  | Maven     | org.apache.logging.log4j:log4j-core           | 2.14.1  | fixtures/maven-transitive/registry.xml |
+| https://osv.dev/GHSA-8489-44mv-ggj8 | 6.6  | Maven     | org.apache.logging.log4j:log4j-core           | 2.14.1  | fixtures/maven-transitive/registry.xml |
+| https://osv.dev/GHSA-jfh8-c2jp-5v3q | 10.0 | Maven     | org.apache.logging.log4j:log4j-core           | 2.14.1  | fixtures/maven-transitive/registry.xml |
+| https://osv.dev/GHSA-p6xc-xr62-6r2g | 8.6  | Maven     | org.apache.logging.log4j:log4j-core           | 2.14.1  | fixtures/maven-transitive/registry.xml |
++-------------------------------------+------+-----------+-----------------------------------------------+---------+----------------------------------------+
+
+---
+
+[TestRun_MavenTransitive/resolve_transitive_dependencies_with_native_datda_source - 2]
+
+---
+
+[TestRun_MavenTransitive/scans_dependencies_from_multiple_registries - 1]
+Scanned <rootdir>/fixtures/maven-transitive/registry.xml file as a pom.xml and found 59 packages
++-------------------------------------+------+-----------+-----------------------------------------------+---------+----------------------------------------+
+| OSV URL                             | CVSS | ECOSYSTEM | PACKAGE                                       | VERSION | SOURCE                                 |
++-------------------------------------+------+-----------+-----------------------------------------------+---------+----------------------------------------+
+| https://osv.dev/GHSA-cm6r-892j-jv2g | 6.1  | Maven     | com.google.android.gms:play-services-basement | 10.0.0  | fixtures/maven-transitive/registry.xml |
+| https://osv.dev/GHSA-7rjr-3q55-vv33 | 9.0  | Maven     | org.apache.logging.log4j:log4j-core           | 2.14.1  | fixtures/maven-transitive/registry.xml |
+| https://osv.dev/GHSA-8489-44mv-ggj8 | 6.6  | Maven     | org.apache.logging.log4j:log4j-core           | 2.14.1  | fixtures/maven-transitive/registry.xml |
+| https://osv.dev/GHSA-jfh8-c2jp-5v3q | 10.0 | Maven     | org.apache.logging.log4j:log4j-core           | 2.14.1  | fixtures/maven-transitive/registry.xml |
+| https://osv.dev/GHSA-p6xc-xr62-6r2g | 8.6  | Maven     | org.apache.logging.log4j:log4j-core           | 2.14.1  | fixtures/maven-transitive/registry.xml |
++-------------------------------------+------+-----------+-----------------------------------------------+---------+----------------------------------------+
+
+---
+
+[TestRun_MavenTransitive/scans_dependencies_from_multiple_registries - 2]
+
+---
+
 [TestRun_MavenTransitive/scans_pom.xml_with_non_UTF-8_encoding - 1]
 Scanned <rootdir>/fixtures/maven-transitive/encoding.xml file as a pom.xml and found 2 packages
 +-------------------------------------+------+-----------+-------------+---------+----------------------------------------+

diff --git a/cmd/osv-scanner/fix/main.go b/cmd/osv-scanner/fix/main.go
@@ -14,7 +14,6 @@ import (
 	"github.com/google/osv-scanner/internal/remediation/upgrade"
 	"github.com/google/osv-scanner/internal/resolution"
 	"github.com/google/osv-scanner/internal/resolution/client"
-	"github.com/google/osv-scanner/internal/resolution/datasource"
 	"github.com/google/osv-scanner/internal/resolution/lockfile"
 	"github.com/google/osv-scanner/internal/resolution/manifest"
 	"github.com/google/osv-scanner/pkg/reporter"
@@ -68,6 +67,10 @@ func Command(stdout, stderr io.Writer, r *reporter.Reporter) *cli.Command {
 					return nil
 				},
 			},
+			&cli.StringFlag{
+				Name:  "maven-registry",
+				Usage: "URL of the default Maven registry to fetch metadata",
+			},
 			&cli.StringFlag{
 				Name:  "relock-cmd",
 				Usage: "command to run to regenerate lockfile on disk after changing the manifest",
@@ -279,7 +282,7 @@ func action(ctx *cli.Context, stdout, stderr io.Writer) (reporter.Reporter, erro
 	}
 
 	if opts.Manifest != "" {
-		rw, err := manifest.GetReadWriter(opts.Manifest)
+		rw, err := manifest.GetReadWriter(opts.Manifest, ctx.String("maven-registry"))
 		if err != nil {
 			return nil, err
 		}
@@ -312,7 +315,7 @@ func action(ctx *cli.Context, stdout, stderr io.Writer) (reporter.Reporter, erro
 			}
 			opts.Client.DependencyClient = cl
 		case resolve.Maven:
-			cl, err := client.NewMavenRegistryClient(datasource.MavenCentral)
+			cl, err := client.NewMavenRegistryClient(ctx.String("maven-registry"))
 			if err != nil {
 				return nil, err
 			}

diff --git a/cmd/osv-scanner/fix/noninteractive.go b/cmd/osv-scanner/fix/noninteractive.go
@@ -295,6 +295,22 @@ func autoOverride(ctx context.Context, r reporter.Reporter, opts osvFixOptions,
 		return err
 	}
 
+	if opts.ManifestRW.System() == resolve.Maven {
+		// Update Maven registries based on the repositories defined in pom.xml,
+		// as well as the repositories merged from parent pom.xml.
+		// TODO: add registries defined in settings.xml
+		// https://github.com/google/osv-scanner/issues/1269
+		specific, ok := manif.EcosystemSpecific.(manifest.MavenManifestSpecific)
+		if ok {
+			registries := make([]client.Registry, len(specific.Repositories))
+			for i, repo := range specific.Repositories {
+				registries[i] = client.Registry{URL: string(repo.URL)}
+			}
+			if err := opts.Client.DependencyClient.AddRegistries(registries); err != nil {
+				return err
+			}
+		}
+	}
 	client.PreFetch(ctx, opts.Client, manif.Requirements, manif.FilePath)
 	res, err := resolution.Resolve(ctx, opts.Client, manif, opts.ResolveOpts)
 	if err != nil {

diff --git a/cmd/osv-scanner/fixtures/maven-transitive/parent.xml b/cmd/osv-scanner/fixtures/maven-transitive/parent.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>com.mycompany.app</groupId>
+  <artifactId>parent</artifactId>
+  <version>1.0.0</version>
+
+  <name>my-app</name>
+
+  <packaging>pom</packaging>
+
+
+  <dependencies>
+    <dependency>
+    <!-- depends on com.google.android.gms:play-services v10.0.0-->
+      <groupId>com.google.android.gms</groupId>
+      <artifactId>play-services</artifactId>
+      <version>10.0.0</version>
+    </dependency>
+  </dependencies>
+
+  <repositories>
+    <repository>
+      <id>google-android</id>
+      <url>https://dl.google.com/dl/android/maven2</url>
+    </repository>
+  </repositories>
+
+</project>
diff --git a/cmd/osv-scanner/fixtures/maven-transitive/registry.xml b/cmd/osv-scanner/fixtures/maven-transitive/registry.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>com.mycompany.app</groupId>
+  <artifactId>my-app</artifactId>
+  <version>1.0.0</version>
+
+  <name>my-app</name>
+
+  <parent>
+    <groupId>com.mycompany.app</groupId>
+    <artifactId>parent</artifactId>
+    <version>1.0.0</version>
+    <relativePath>./parent.xml</relativePath>
+  </parent>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-web</artifactId>
+      <version>2.14.1</version>
+    </dependency>
+  </dependencies>
+
+</project>
diff --git a/cmd/osv-scanner/main_test.go b/cmd/osv-scanner/main_test.go
@@ -911,6 +911,16 @@ func TestRun_MavenTransitive(t *testing.T) {
 			args: []string{"", "--config=./fixtures/osv-scanner-empty-config.toml", "--experimental-offline", "--experimental-download-offline-databases", "./fixtures/maven-transitive/pom.xml"},
 			exit: 0,
 		},
+		{
+			name: "scans dependencies from multiple registries",
+			args: []string{"", "--config=./fixtures/osv-scanner-empty-config.toml", "-L", "pom.xml:./fixtures/maven-transitive/registry.xml"},
+			exit: 1,
+		},
+		{
+			name: "resolve transitive dependencies with native datda source",
+			args: []string{"", "--config=./fixtures/osv-scanner-empty-config.toml", "--experimental-resolution-data-source=native", "-L", "pom.xml:./fixtures/maven-transitive/registry.xml"},
+			exit: 1,
+		},
 	}
 
 	for _, tt := range tests {

diff --git a/cmd/osv-scanner/scan/main.go b/cmd/osv-scanner/scan/main.go
@@ -138,6 +138,22 @@ func Command(stdout, stderr io.Writer, r *reporter.Reporter) *cli.Command {
 				TakesFile: true,
 				Hidden:    true,
 			},
+			&cli.StringFlag{
+				Name:  "experimental-resolution-data-source",
+				Usage: "source to fetch package information from; value can be: deps.dev, native",
+				Value: "deps.dev",
+				Action: func(_ *cli.Context, s string) error {
+					if s != "deps.dev" && s != "native" {
+						return fmt.Errorf("unsupported data-source \"%s\" - must be one of: deps.dev, native", s)
+					}
+
+					return nil
+				},
+			},
+			&cli.StringFlag{
+				Name:  "experimental-maven-registry",
+				Usage: "URL of the default registry to fetch Maven metadata",
+			},
 		},
 		ArgsUsage: "[directory1 directory2...]",
 		Action: func(c *cli.Context) error {
@@ -228,6 +244,10 @@ func action(context *cli.Context, stdout, stderr io.Writer) (reporter.Reporter,
 			ScanLicensesSummary:   context.Bool("experimental-licenses-summary"),
 			ScanLicensesAllowlist: context.StringSlice("experimental-licenses"),
 			ScanOCIImage:          context.String("experimental-oci-image"),
+			TransitiveScanningActions: osvscanner.TransitiveScanningActions{
+				NativeDataSource: context.String("experimental-resolution-data-source") == "native",
+				MavenRegistry:    context.String("experimental-maven-registry"),
+			},
 		},
 	}, r)
 

diff --git a/cmd/osv-scanner/update/main.go b/cmd/osv-scanner/update/main.go
@@ -78,7 +78,7 @@ func action(ctx *cli.Context, stdout, stderr io.Writer) (reporter.Reporter, erro
 	if err != nil {
 		return nil, err
 	}
-	options.ManifestRW, err = manifest.GetReadWriter(options.Manifest)
+	options.ManifestRW, err = manifest.GetReadWriter(options.Manifest, "")
 	if err != nil {
 		return nil, err
 	}

diff --git a/internal/manifest/maven.go b/internal/manifest/maven.go
@@ -31,26 +31,38 @@ func (e MavenResolverExtractor) Extract(f lockfile.DepFile) ([]lockfile.PackageD
 
 	var project maven.Project
 	if err := datasource.NewMavenDecoder(f).Decode(&project); err != nil {
-		return []lockfile.PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err)
+		return nil, fmt.Errorf("could not extract from %s: %w", f.Path(), err)
+	}
+	// Empty JDK and ActivationOS indicates merging the default profiles.
+	if err := project.MergeProfiles("", maven.ActivationOS{}); err != nil {
+		return nil, fmt.Errorf("failed to merge profiles: %w", err)
+	}
+	for _, repo := range project.Repositories {
+		if err := e.MavenRegistryAPIClient.AddRegistry(string(repo.URL)); err != nil {
+			return nil, fmt.Errorf("failed to add registry %s: %w", repo.URL, err)
+		}
 	}
 	// Merging parents data by parsing local parent pom.xml or fetching from upstream.
 	if err := mavenutil.MergeParents(ctx, e.MavenRegistryAPIClient, &project, project.Parent, 1, f.Path(), true); err != nil {
-		return []lockfile.PackageDetails{}, fmt.Errorf("failed to merge parents: %w", err)
+		return nil, fmt.Errorf("failed to merge parents: %w", err)
 	}
 	// Process the dependencies:
 	//  - dedupe dependencies and dependency management
 	//  - import dependency management
 	//  - fill in missing dependency version requirement
 	project.ProcessDependencies(func(groupID, artifactID, version maven.String) (maven.DependencyManagement, error) {
-		root := maven.Parent{ProjectKey: maven.ProjectKey{GroupID: groupID, ArtifactID: artifactID, Version: version}}
-		var result maven.Project
-		if err := mavenutil.MergeParents(ctx, e.MavenRegistryAPIClient, &result, root, 0, f.Path(), false); err != nil {
-			return maven.DependencyManagement{}, err
-		}
-
-		return result.DependencyManagement, nil
+		return mavenutil.GetDependencyManagement(ctx, e.MavenRegistryAPIClient, groupID, artifactID, version)
 	})
 
+	if registries := e.MavenRegistryAPIClient.GetRegistries(); len(registries) > 0 {
+		clientRegs := make([]client.Registry, len(registries))
+		for i, reg := range registries {
+			clientRegs[i] = client.Registry{URL: reg}
+		}
+		if err := e.DependencyClient.AddRegistries(clientRegs); err != nil {
+			return nil, err
+		}
+	}
 	overrideClient := client.NewOverrideClient(e.DependencyClient)
 	resolver := mavenresolve.NewResolver(overrideClient)
 
@@ -93,9 +105,10 @@ func (e MavenResolverExtractor) Extract(f lockfile.DepFile) ([]lockfile.PackageD
 	}
 	overrideClient.AddVersion(root, reqs)
 
+	client.PreFetch(ctx, overrideClient, reqs, f.Path())
 	g, err := resolver.Resolve(ctx, root.VersionKey)
 	if err != nil {
-		return []lockfile.PackageDetails{}, fmt.Errorf("failed resolving %v: %w", root, err)
+		return nil, fmt.Errorf("failed resolving %v: %w", root, err)
 	}
 	for i, e := range g.Edges {
 		e.Type = dep.Type{}
@@ -127,7 +140,7 @@ func (e MavenResolverExtractor) Extract(f lockfile.DepFile) ([]lockfile.PackageD
 func ParseMavenWithResolver(depClient client.DependencyClient, mavenClient *datasource.MavenRegistryAPIClient, pathToLockfile string) ([]lockfile.PackageDetails, error) {
 	f, err := lockfile.OpenLocalDepFile(pathToLockfile)
 	if err != nil {
-		return []lockfile.PackageDetails{}, err
+		return nil, err
 	}
 	defer f.Close()