Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: Update v2 with new changes from main #1344

Merged
merged 3 commits into from
Oct 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ name: Checks

on:
push:
branches: [main]
branches: [main, v2]
pull_request:
# The branches below must be a subset of the branches above
branches: [main]
branches: [main, v2]

concurrency:
# Pushing new changes to a branch will cancel any in-progress CI runs
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/codeql-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ name: "CodeQL"

on:
push:
branches: [main]
branches: [main, v2]
pull_request:
# The branches below must be a subset of the branches above
branches: [main]
branches: [main, v2]

# Restrict jobs in this workflow to have no permissions by default; permissions
# should be granted per job as needed using a dedicated `permissions` block
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/osv-scanner-unified-action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@ name: OSV-Scanner Scheduled Scan

on:
pull_request:
branches: ["main"]
branches: ["main", "v2"]
schedule:
- cron: "12 12 * * 1"
push:
branches: ["main"]
branches: ["main", "v2"]

# Restrict jobs in this workflow to have no permissions by default; permissions
# should be granted per job as needed using a dedicated `permissions` block
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/scorecards.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ on:
schedule:
- cron: "32 22 * * 6"
push:
branches: ["main"]
branches: ["main", "v2"]

# Restrict jobs in this workflow to have no permissions by default; permissions
# should be granted per job as needed using a dedicated `permissions` block
Expand Down
11 changes: 11 additions & 0 deletions cmd/osv-scanner/__snapshots__/main_test.snap
Original file line number Diff line number Diff line change
Expand Up @@ -2372,6 +2372,17 @@ Scanned <rootdir>/fixtures/locks-insecure/composer.lock file and found 1 package

---

[TestRun_MavenTransitive/does_not_scan_transitive_dependencies_for_pom.xml_with_no-resolve - 1]
Scanning dir ./fixtures/maven-transitive/pom.xml
Scanned <rootdir>/fixtures/maven-transitive/pom.xml file and found 1 package
No issues found

---

[TestRun_MavenTransitive/does_not_scan_transitive_dependencies_for_pom.xml_with_no-resolve - 2]

---

[TestRun_MavenTransitive/does_not_scan_transitive_dependencies_for_pom.xml_with_offline_mode - 1]
Scanning dir ./fixtures/maven-transitive/pom.xml
Scanned <rootdir>/fixtures/maven-transitive/pom.xml file and found 1 package
Expand Down
7 changes: 4 additions & 3 deletions cmd/osv-scanner/fix/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -171,8 +171,9 @@ func Command(stdout, stderr io.Writer, r *reporter.Reporter) *cli.Command {
},
// Offline database flags, copied from osv-scanner scan
&cli.BoolFlag{
Name: "experimental-offline",
Usage: "checks for vulnerabilities using local databases that are already cached",
Name: "experimental-offline-vulnerabilities",
Aliases: []string{"experimental-offline"},
Usage: "checks for vulnerabilities using local databases that are already cached",
},
&cli.BoolFlag{
Name: "experimental-download-offline-databases",
Expand Down Expand Up @@ -327,7 +328,7 @@ func action(ctx *cli.Context, stdout, stderr io.Writer) (reporter.Reporter, erro
}
}

if ctx.Bool("experimental-offline") {
if ctx.Bool("experimental-offline-vulnerabilities") {
var err error
opts.Client.VulnerabilityClient, err = client.NewOSVOfflineClient(
r,
Expand Down
6 changes: 6 additions & 0 deletions cmd/osv-scanner/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -911,6 +911,12 @@ func TestRun_MavenTransitive(t *testing.T) {
args: []string{"", "--config=./fixtures/osv-scanner-empty-config.toml", "--experimental-offline", "--experimental-download-offline-databases", "./fixtures/maven-transitive/pom.xml"},
exit: 0,
},
{
// Direct dependencies do not have any vulnerability.
name: "does not scan transitive dependencies for pom.xml with no-resolve",
args: []string{"", "--config=./fixtures/osv-scanner-empty-config.toml", "--experimental-no-resolve", "./fixtures/maven-transitive/pom.xml"},
exit: 0,
},
{
name: "scans dependencies from multiple registries",
args: []string{"", "--config=./fixtures/osv-scanner-empty-config.toml", "-L", "pom.xml:./fixtures/maven-transitive/registry.xml"},
Expand Down
41 changes: 39 additions & 2 deletions cmd/osv-scanner/scan/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,15 @@ import (
"github.com/urfave/cli/v2"
)

// flags that require network access and values to disable them.
var offlineFlags = map[string]string{
"skip-git": "true",
"experimental-offline-vulnerabilities": "true",
"experimental-no-resolve": "true",
"experimental-licenses-summary": "false",
// "experimental-licenses": "", // StringSliceFlag has to be manually cleared.
}

func Command(stdout, stderr io.Writer, r *reporter.Reporter) *cli.Command {
return &cli.Command{
Name: "scan",
Expand Down Expand Up @@ -109,6 +118,24 @@ func Command(stdout, stderr io.Writer, r *reporter.Reporter) *cli.Command {
},
&cli.BoolFlag{
Name: "experimental-offline",
Usage: "run in offline mode, disabling any features requiring network access",
Action: func(ctx *cli.Context, b bool) error {
if !b {
return nil
}
// Disable the features requiring network access.
for flag, value := range offlineFlags {
// TODO(michaelkedar): do something if the flag was already explicitly set.
if err := ctx.Set(flag, value); err != nil {
panic(fmt.Sprintf("failed setting offline flag %s to %s: %v", flag, value, err))
}
}

return nil
},
},
&cli.BoolFlag{
Name: "experimental-offline-vulnerabilities",
Usage: "checks for vulnerabilities using local databases that are already cached",
},
&cli.BoolFlag{
Expand Down Expand Up @@ -138,6 +165,10 @@ func Command(stdout, stderr io.Writer, r *reporter.Reporter) *cli.Command {
TakesFile: true,
Hidden: true,
},
&cli.BoolFlag{
Name: "experimental-no-resolve",
Usage: "disable transitive dependency resolution of manifest files",
},
&cli.StringFlag{
Name: "experimental-resolution-data-source",
Usage: "source to fetch package information from; value can be: deps.dev, native",
Expand Down Expand Up @@ -221,6 +252,11 @@ func action(context *cli.Context, stdout, stderr io.Writer) (reporter.Reporter,
callAnalysisStates = createCallAnalysisStates(context.StringSlice("call-analysis"), context.StringSlice("no-call-analysis"))
}

scanLicensesAllowlist := context.StringSlice("experimental-licenses")
if context.Bool("experimental-offline") {
scanLicensesAllowlist = []string{}
}

vulnResult, err := osvscanner.DoScan(osvscanner.ScannerActions{
LockfilePaths: context.StringSlice("lockfile"),
SBOMPaths: context.StringSlice("sbom"),
Expand All @@ -234,17 +270,18 @@ func action(context *cli.Context, stdout, stderr io.Writer) (reporter.Reporter,
ExperimentalScannerActions: osvscanner.ExperimentalScannerActions{
LocalDBPath: context.String("experimental-local-db-path"),
DownloadDatabases: context.Bool("experimental-download-offline-databases"),
CompareOffline: context.Bool("experimental-offline"),
CompareOffline: context.Bool("experimental-offline-vulnerabilities"),
// License summary mode causes all
// packages to appear in the json as
// every package has a license - even
// if it's just the UNKNOWN license.
ShowAllPackages: context.Bool("experimental-all-packages") ||
context.Bool("experimental-licenses-summary"),
ScanLicensesSummary: context.Bool("experimental-licenses-summary"),
ScanLicensesAllowlist: context.StringSlice("experimental-licenses"),
ScanLicensesAllowlist: scanLicensesAllowlist,
ScanOCIImage: context.String("experimental-oci-image"),
TransitiveScanningActions: osvscanner.TransitiveScanningActions{
Disabled: context.Bool("experimental-no-resolve"),
NativeDataSource: context.String("experimental-resolution-data-source") == "native",
MavenRegistry: context.String("experimental-maven-registry"),
},
Expand Down
4 changes: 2 additions & 2 deletions docs/guided-remediation.md
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ The following flag may be used to limit the patches allowed for your dependencie

### Data source

By default, we use the [deps.dev API](https://docs.deps.dev/api/v3alpha/) to find version and dependency information of packages during remediation.
By default, we use the [deps.dev API](https://docs.deps.dev/api/) to find version and dependency information of packages during remediation.

If instead you'd like to use your ecosystem's native registry API (e.g. `https://registry.npmjs.org`), you can use the `--data-source=native` flag. `osv-scanner fix` will attempt to use the authentication specified by the native tooling (e.g. `npm config`)

Expand All @@ -264,7 +264,7 @@ If your project uses mirrored or private registries, you will need to use `--dat

### Offline Vulnerability Database

The `fix` subcommand supports the `--experimental-offline` and `--experimental-download-offline-databases` flags.
The `fix` subcommand supports the `--experimental-offline-vulnerabilities` and `--experimental-download-offline-databases` flags.

For more information, see [Offline Mode](./offline-mode.md).

Expand Down
2 changes: 2 additions & 0 deletions docs/offline-mode.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ The offline database flag `--experimental-offline` causes OSV-Scanner to scan yo
osv-scanner --experimental-offline ./path/to/your/dir
```

To use offline mode for just the vulnerability database, but allow other features to possibly make network requests (e.g. [transitive dependency scanning](./supported_languages_and_lockfiles.md/#transitive-dependency-scanning)), you can use the `--experimental-offline-vulnerabilities` flag instead.

## Download offline databases option

The download offline databases flag `--experimental-download-offline-databases` allows OSV-Scanner to download or update your local database when running in offline mode, to make it easier to get started. This option only works when you also set the offline flag.
Expand Down
10 changes: 9 additions & 1 deletion docs/supported_languages_and_lockfiles.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ Vendored dependencies have been directly copied into the project folder, but do

## Transitive dependency scanning

OSV-Scanner supports transitive dependency scanning for Maven pom.xml. This feature is enabled by default when scanning, but it is disabled in the [offline mode](./offline-mode.md).
OSV-Scanner supports transitive dependency scanning for Maven pom.xml. This feature is enabled by default when scanning, but it can be disabled using the `--experimental-no-resolve` flag. It is also disabled in the [offline mode](./offline-mode.md).

OSV-Scanner uses [deps.dev’s resolver library](https://pkg.go.dev/deps.dev/util/resolve) to compute the dependency graph of a project. This graph includes all of the direct and transitive dependencies. By default, [deps.dev API](https://docs.deps.dev/api/v3/index.html) is queried for package versions and requirements. The support for private registries is [coming soon](https://github.com/google/osv-scanner/issues/1045).

Expand All @@ -81,6 +81,14 @@ After the dependency resolution, the OSV database is queried for the vulnerabili
{: .note }
Test dependencies are not supported yet in the computed dependency graph for Maven pom.xml.

### Data source

By default, we use the [deps.dev API](https://docs.deps.dev/api/v3/) to find version and dependency information of packages during transitive scanning.

If instead you'd like to fetch data from [Maven Central](https://repo.maven.apache.org/maven2/), you can use the `--experimental-resolution-data-source=native` flag.

If your project uses mirrored or private registries, in addition to setting `--experimental-resolution-data-source=native`, you will need to use the `--experimental-maven-registry=<full-registry-url>` flag to specify the registry (e.g. `--experimental-maven-registry=https://repo.maven.apache.org/maven2/`).

## Custom Lockfiles

If you have a custom lockfile that we do not support or prefer to do your own custom parsing, you can extract the custom lockfile information and create a custom intermediate file containing dependency information so that osv-scanner can still check for vulnerabilities.
Expand Down
9 changes: 5 additions & 4 deletions pkg/osvscanner/osvscanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ type ExperimentalScannerActions struct {
}

type TransitiveScanningActions struct {
Disabled bool
NativeDataSource bool
MavenRegistry string
}
Expand Down Expand Up @@ -171,7 +172,7 @@ func scanDir(r reporter.Reporter, dir string, skipGit bool, recursive bool, useG

if !info.IsDir() {
if extractor, _ := lockfile.FindExtractor(path, ""); extractor != nil {
pkgs, err := scanLockfile(r, path, "", compareOffline, transitiveAct)
pkgs, err := scanLockfile(r, path, "", transitiveAct)
if err != nil {
r.Errorf("Attempted to scan lockfile but failed: %s\n", path)
}
Expand Down Expand Up @@ -353,7 +354,7 @@ func scanImage(r reporter.Reporter, path string) ([]scannedPackage, error) {

// scanLockfile will load, identify, and parse the lockfile path passed in, and add the dependencies specified
// within to `query`
func scanLockfile(r reporter.Reporter, path string, parseAs string, compareOffline bool, transitiveAct TransitiveScanningActions) ([]scannedPackage, error) {
func scanLockfile(r reporter.Reporter, path string, parseAs string, transitiveAct TransitiveScanningActions) ([]scannedPackage, error) {
var err error
var parsedLockfile lockfile.Lockfile

Expand All @@ -371,7 +372,7 @@ func scanLockfile(r reporter.Reporter, path string, parseAs string, compareOffli
case "osv-scanner":
parsedLockfile, err = lockfile.FromOSVScannerResults(path)
default:
if !compareOffline && (parseAs == "pom.xml" || filepath.Base(path) == "pom.xml") {
if !transitiveAct.Disabled && (parseAs == "pom.xml" || filepath.Base(path) == "pom.xml") {
parsedLockfile, err = extractMavenDeps(f, transitiveAct)
} else {
parsedLockfile, err = lockfile.ExtractDeps(f, parseAs)
Expand Down Expand Up @@ -907,7 +908,7 @@ func DoScan(actions ScannerActions, r reporter.Reporter) (models.VulnerabilityRe
r.Errorf("Failed to resolved path with error %s\n", err)
return models.VulnerabilityResults{}, err
}
pkgs, err := scanLockfile(r, lockfilePath, parseAs, actions.CompareOffline, actions.TransitiveScanningActions)
pkgs, err := scanLockfile(r, lockfilePath, parseAs, actions.TransitiveScanningActions)
if err != nil {
return models.VulnerabilityResults{}, err
}
Expand Down