From 64f5c2dc67b227ada221753e591e93e5f5edf3e5 Mon Sep 17 00:00:00 2001 From: Xueqin Cui <72771658+cuixq@users.noreply.github.com> Date: Mon, 11 Dec 2023 15:10:06 +1100 Subject: [PATCH] scan and report dependency groups of vulnerabilities (#655) Issue https://github.com/google/osv-scanner/issues/332 Non-default dependency groups are recorded in strings as per eco-system: - **Composer:** development dependencies in `packages-dev` - **Conan:** dependencies in `build-requires` and `python-requires` - **Maven:** `` in `` - **npm:** `dev` and `optional` dependencies - **pipenv:** development dependencies in `develop` - **pnpm:** development dependencies with `dev` as true - **Poetry:** optional dependencies with `optional = true` - **Pubspec:** development dependencies marked with `dev` - **requirements.txt:** group of a dependency is the file name without the extension Reporters: - **table:** non-default groups are appended to the end of package name, for example: `abc (development)` - **json:** non-default group information in `dependencyGroups` --------- Co-authored-by: josieang <32358891+josieang@users.noreply.github.com> Co-authored-by: Gareth Jones Co-authored-by: Mend Renovate --- internal/output/table.go | 7 +- pkg/lockfile/dpkg-status.go | 4 +- pkg/lockfile/fixtures/maven/with-scope.xml | 10 +++ .../fixtures/npm/optional-package.v1.json | 22 +++++++ .../fixtures/npm/optional-package.v2.json | 30 +++++++++ .../fixtures/poetry/optional-package.lock | 15 +++++ pkg/lockfile/helpers_test.go | 3 +- pkg/lockfile/parse-composer-lock.go | 1 + pkg/lockfile/parse-composer-lock_test.go | 2 + pkg/lockfile/parse-conan-lock-v2_test.go | 10 +++ pkg/lockfile/parse-conan-lock.go | 9 +-- pkg/lockfile/parse-maven-lock.go | 15 ++++- pkg/lockfile/parse-maven-lock_test.go | 20 ++++++ pkg/lockfile/parse-npm-lock-v1_test.go | 36 +++++++++++ pkg/lockfile/parse-npm-lock-v2_test.go | 41 ++++++++++++ pkg/lockfile/parse-npm-lock.go | 37 +++++++++++ pkg/lockfile/parse-pipenv-lock.go | 22 ++++--- pkg/lockfile/parse-pipenv-lock_test.go | 3 + pkg/lockfile/parse-pnpm-lock.go | 7 ++ pkg/lockfile/parse-pnpm-lock_test.go | 1 + pkg/lockfile/parse-poetry-lock.go | 15 +++-- pkg/lockfile/parse-poetry-lock_test.go | 20 ++++++ pkg/lockfile/parse-pubspec-lock.go | 13 +++- pkg/lockfile/parse-pubspec-lock_test.go | 2 + pkg/lockfile/parse-requirements-txt.go | 23 ++++++- pkg/lockfile/parse-requirements-txt_test.go | 64 +++++++++++++++++++ pkg/lockfile/types.go | 27 ++++++++ pkg/models/results.go | 4 +- pkg/osv/osv.go | 6 +- pkg/osvscanner/osvscanner.go | 2 + pkg/osvscanner/vulnerability_result.go | 1 + 31 files changed, 441 insertions(+), 31 deletions(-) create mode 100644 pkg/lockfile/fixtures/maven/with-scope.xml create mode 100644 pkg/lockfile/fixtures/npm/optional-package.v1.json create mode 100644 pkg/lockfile/fixtures/npm/optional-package.v2.json create mode 100644 pkg/lockfile/fixtures/poetry/optional-package.lock diff --git a/internal/output/table.go b/internal/output/table.go index 80d20ce4a92..4230785ed88 100644 --- a/internal/output/table.go +++ b/internal/output/table.go @@ -17,6 +17,7 @@ import ( "golang.org/x/exp/maps" "github.com/google/osv-scanner/internal/utility/results" + "github.com/google/osv-scanner/pkg/lockfile" "github.com/google/osv-scanner/pkg/models" "github.com/jedib0t/go-pretty/v6/table" @@ -130,7 +131,11 @@ func tableBuilderInner(vulnResult *models.VulnerabilityResults, addStyling bool, outputRow = append(outputRow, "GIT", pkgCommitStr, pkgCommitStr) shouldMerge = true } else { - outputRow = append(outputRow, pkg.Package.Ecosystem, pkg.Package.Name, pkg.Package.Version) + name := pkg.Package.Name + if lockfile.Ecosystem(pkg.Package.Ecosystem).IsDevGroup(pkg.DepGroups) { + name += " (dev)" + } + outputRow = append(outputRow, pkg.Package.Ecosystem, name, pkg.Package.Version) } outputRow = append(outputRow, source.Path) diff --git a/pkg/lockfile/dpkg-status.go b/pkg/lockfile/dpkg-status.go index 404ee1b4475..46d09e6b7cb 100644 --- a/pkg/lockfile/dpkg-status.go +++ b/pkg/lockfile/dpkg-status.go @@ -121,8 +121,8 @@ func (e DpkgStatusExtractor) Extract(f DepFile) ([]PackageDetails, error) { pkg := parseDpkgPackageGroup(group) // PackageDetails does not contain any field that represent a "not installed" state - // To manage this state and avoid false positives, empty struct means "not installed" so skip it - if (PackageDetails{}) == pkg { + // To manage this state and avoid false positives, empty ecosystem means "not installed" so skip it + if pkg.Ecosystem == "" { continue } diff --git a/pkg/lockfile/fixtures/maven/with-scope.xml b/pkg/lockfile/fixtures/maven/with-scope.xml new file mode 100644 index 00000000000..179b05a3175 --- /dev/null +++ b/pkg/lockfile/fixtures/maven/with-scope.xml @@ -0,0 +1,10 @@ + + + + junit + junit + 4.12 + test + + + diff --git a/pkg/lockfile/fixtures/npm/optional-package.v1.json b/pkg/lockfile/fixtures/npm/optional-package.v1.json new file mode 100644 index 00000000000..83422ba536c --- /dev/null +++ b/pkg/lockfile/fixtures/npm/optional-package.v1.json @@ -0,0 +1,22 @@ +{ + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true, + "optional": true + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "optional": true, + "requires": { + "has-flag": "^3.0.0" + } + } + } +} diff --git a/pkg/lockfile/fixtures/npm/optional-package.v2.json b/pkg/lockfile/fixtures/npm/optional-package.v2.json new file mode 100644 index 00000000000..728403ca765 --- /dev/null +++ b/pkg/lockfile/fixtures/npm/optional-package.v2.json @@ -0,0 +1,30 @@ +{ + "name": "my-library", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "dependencies": {}, + "devDependencies": { "wrappy": "^1.0.0" } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "optional": true + }, + "node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "devOptional": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + } + }, + "dependencies": {} +} diff --git a/pkg/lockfile/fixtures/poetry/optional-package.lock b/pkg/lockfile/fixtures/poetry/optional-package.lock new file mode 100644 index 00000000000..c3ce31affcb --- /dev/null +++ b/pkg/lockfile/fixtures/poetry/optional-package.lock @@ -0,0 +1,15 @@ +[[package]] +name = "numpy" +version = "1.23.3" +description = "NumPy is the fundamental package for array computing with Python." +category = "main" +optional = true +python-versions = ">=3.8" + +[metadata] +lock-version = "1.1" +python-versions = "^3.8" +content-hash = "399777887f0c3171cbc3fc8a8e350d0fca4d882cf126657f60ec83872572ed44" + +[metadata.files] +numpy = [] diff --git a/pkg/lockfile/helpers_test.go b/pkg/lockfile/helpers_test.go index 90c71587ee3..bc26b62af7b 100644 --- a/pkg/lockfile/helpers_test.go +++ b/pkg/lockfile/helpers_test.go @@ -3,6 +3,7 @@ package lockfile_test import ( "errors" "fmt" + "reflect" "strings" "testing" @@ -48,7 +49,7 @@ func hasPackage(t *testing.T, packages []lockfile.PackageDetails, pkg lockfile.P t.Helper() for _, details := range packages { - if details == pkg { + if reflect.DeepEqual(details, pkg) { return true } } diff --git a/pkg/lockfile/parse-composer-lock.go b/pkg/lockfile/parse-composer-lock.go index a665ad7083e..43054fef989 100644 --- a/pkg/lockfile/parse-composer-lock.go +++ b/pkg/lockfile/parse-composer-lock.go @@ -60,6 +60,7 @@ func (e ComposerLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { Commit: composerPackage.Dist.Reference, Ecosystem: ComposerEcosystem, CompareAs: ComposerEcosystem, + DepGroups: []string{"dev"}, }) } diff --git a/pkg/lockfile/parse-composer-lock_test.go b/pkg/lockfile/parse-composer-lock_test.go index befaf544cae..6b7048b5a04 100644 --- a/pkg/lockfile/parse-composer-lock_test.go +++ b/pkg/lockfile/parse-composer-lock_test.go @@ -125,6 +125,7 @@ func TestParseComposerLock_OnePackageDev(t *testing.T) { Commit: "4c115873c86ad5bd0ac6d962db70ca53bf8fb874", Ecosystem: lockfile.ComposerEcosystem, CompareAs: lockfile.ComposerEcosystem, + DepGroups: []string{"dev"}, }, }) } @@ -152,6 +153,7 @@ func TestParseComposerLock_TwoPackages(t *testing.T) { Commit: "11336f6f84e16a720dae9d8e6ed5019efa85a0f9", Ecosystem: lockfile.ComposerEcosystem, CompareAs: lockfile.ComposerEcosystem, + DepGroups: []string{"dev"}, }, }) } diff --git a/pkg/lockfile/parse-conan-lock-v2_test.go b/pkg/lockfile/parse-conan-lock-v2_test.go index 8e2a8da8cb1..58dbb6e37e6 100644 --- a/pkg/lockfile/parse-conan-lock-v2_test.go +++ b/pkg/lockfile/parse-conan-lock-v2_test.go @@ -52,6 +52,7 @@ func TestParseConanLock_v2_OnePackage(t *testing.T) { Version: "1.2.11", Ecosystem: lockfile.ConanEcosystem, CompareAs: lockfile.ConanEcosystem, + DepGroups: []string{"requires"}, }, }) } @@ -71,6 +72,7 @@ func TestParseConanLock_v2_NoName(t *testing.T) { Version: "1.2.11", Ecosystem: lockfile.ConanEcosystem, CompareAs: lockfile.ConanEcosystem, + DepGroups: []string{"requires"}, }, }) } @@ -90,12 +92,14 @@ func TestParseConanLock_v2_TwoPackages(t *testing.T) { Version: "1.2.11", Ecosystem: lockfile.ConanEcosystem, CompareAs: lockfile.ConanEcosystem, + DepGroups: []string{"requires"}, }, { Name: "bzip2", Version: "1.0.8", Ecosystem: lockfile.ConanEcosystem, CompareAs: lockfile.ConanEcosystem, + DepGroups: []string{"requires"}, }, }) } @@ -115,30 +119,35 @@ func TestParseConanLock_v2_NestedDependencies(t *testing.T) { Version: "1.2.13", Ecosystem: lockfile.ConanEcosystem, CompareAs: lockfile.ConanEcosystem, + DepGroups: []string{"requires"}, }, { Name: "bzip2", Version: "1.0.8", Ecosystem: lockfile.ConanEcosystem, CompareAs: lockfile.ConanEcosystem, + DepGroups: []string{"requires"}, }, { Name: "freetype", Version: "2.12.1", Ecosystem: lockfile.ConanEcosystem, CompareAs: lockfile.ConanEcosystem, + DepGroups: []string{"requires"}, }, { Name: "libpng", Version: "1.6.39", Ecosystem: lockfile.ConanEcosystem, CompareAs: lockfile.ConanEcosystem, + DepGroups: []string{"requires"}, }, { Name: "brotli", Version: "1.0.9", Ecosystem: lockfile.ConanEcosystem, CompareAs: lockfile.ConanEcosystem, + DepGroups: []string{"requires"}, }, }) } @@ -158,6 +167,7 @@ func TestParseConanLock_v2_OnePackageDev(t *testing.T) { Version: "1.11.1", Ecosystem: lockfile.ConanEcosystem, CompareAs: lockfile.ConanEcosystem, + DepGroups: []string{"build-requires"}, }, }) } diff --git a/pkg/lockfile/parse-conan-lock.go b/pkg/lockfile/parse-conan-lock.go index 7620836fbe0..38c40cb04d2 100644 --- a/pkg/lockfile/parse-conan-lock.go +++ b/pkg/lockfile/parse-conan-lock.go @@ -132,7 +132,7 @@ func parseConanV1Lock(lockfile ConanLockFile) []PackageDetails { return packages } -func parseConanRequires(packages *[]PackageDetails, requires []string) { +func parseConanRequires(packages *[]PackageDetails, requires []string, group string) { for _, ref := range requires { reference := parseConanRenference(ref) // skip entries with no name, they are most likely consumer's conanfiles @@ -146,6 +146,7 @@ func parseConanRequires(packages *[]PackageDetails, requires []string) { Version: reference.Version, Ecosystem: ConanEcosystem, CompareAs: ConanEcosystem, + DepGroups: []string{group}, }) } } @@ -157,9 +158,9 @@ func parseConanV2Lock(lockfile ConanLockFile) []PackageDetails { uint64(len(lockfile.Requires))+uint64(len(lockfile.BuildRequires))+uint64(len(lockfile.PythonRequires)), ) - parseConanRequires(&packages, lockfile.Requires) - parseConanRequires(&packages, lockfile.BuildRequires) - parseConanRequires(&packages, lockfile.PythonRequires) + parseConanRequires(&packages, lockfile.Requires, "requires") + parseConanRequires(&packages, lockfile.BuildRequires, "build-requires") + parseConanRequires(&packages, lockfile.PythonRequires, "python-requires") return packages } diff --git a/pkg/lockfile/parse-maven-lock.go b/pkg/lockfile/parse-maven-lock.go index 342181b5b2d..b60091b3897 100644 --- a/pkg/lockfile/parse-maven-lock.go +++ b/pkg/lockfile/parse-maven-lock.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/google/osv-scanner/internal/cachedregexp" ) @@ -14,6 +15,7 @@ type MavenLockDependency struct { GroupID string `xml:"groupId"` ArtifactID string `xml:"artifactId"` Version string `xml:"version"` + Scope string `xml:"scope"` } func (mld MavenLockDependency) parseResolvedVersion(version string) string { @@ -121,24 +123,31 @@ func (e MavenLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { for _, lockPackage := range parsedLockfile.Dependencies { finalName := lockPackage.GroupID + ":" + lockPackage.ArtifactID - details[finalName] = PackageDetails{ + pkgDetails := PackageDetails{ Name: finalName, Version: lockPackage.ResolveVersion(*parsedLockfile), Ecosystem: MavenEcosystem, CompareAs: MavenEcosystem, } + if strings.TrimSpace(lockPackage.Scope) != "" { + pkgDetails.DepGroups = append(pkgDetails.DepGroups, lockPackage.Scope) + } + details[finalName] = pkgDetails } // managed dependencies take precedent over standard dependencies for _, lockPackage := range parsedLockfile.ManagedDependencies { finalName := lockPackage.GroupID + ":" + lockPackage.ArtifactID - - details[finalName] = PackageDetails{ + pkgDetails := PackageDetails{ Name: finalName, Version: lockPackage.ResolveVersion(*parsedLockfile), Ecosystem: MavenEcosystem, CompareAs: MavenEcosystem, } + if strings.TrimSpace(lockPackage.Scope) != "" { + pkgDetails.DepGroups = append(pkgDetails.DepGroups, lockPackage.Scope) + } + details[finalName] = pkgDetails } return pkgDetailsMapToSlice(details), nil diff --git a/pkg/lockfile/parse-maven-lock_test.go b/pkg/lockfile/parse-maven-lock_test.go index 38b2edd6637..ccabd8cf098 100644 --- a/pkg/lockfile/parse-maven-lock_test.go +++ b/pkg/lockfile/parse-maven-lock_test.go @@ -290,3 +290,23 @@ func TestMavenLockDependency_ResolveVersion(t *testing.T) { }) } } + +func TestParseMavenLock_WithScope(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseMavenLock("fixtures/maven/with-scope.xml") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "junit:junit", + Version: "4.12", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + DepGroups: []string{"test"}, + }, + }) +} diff --git a/pkg/lockfile/parse-npm-lock-v1_test.go b/pkg/lockfile/parse-npm-lock-v1_test.go index fa129a89601..81a55e0832a 100644 --- a/pkg/lockfile/parse-npm-lock-v1_test.go +++ b/pkg/lockfile/parse-npm-lock-v1_test.go @@ -71,6 +71,7 @@ func TestParseNpmLock_v1_OnePackageDev(t *testing.T) { Version: "1.0.2", Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, + DepGroups: []string{"dev"}, }, }) } @@ -241,6 +242,7 @@ func TestParseNpmLock_v1_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", + DepGroups: []string{"dev"}, }, { Name: "is-number-1", @@ -248,6 +250,7 @@ func TestParseNpmLock_v1_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "be5935f8d2595bcd97b05718ef1eeae08d812e10", + DepGroups: []string{"dev"}, }, { Name: "is-number-2", @@ -269,6 +272,7 @@ func TestParseNpmLock_v1_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", + DepGroups: []string{"dev"}, }, { Name: "is-number-3", @@ -276,6 +280,7 @@ func TestParseNpmLock_v1_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "82ae8802978da40d7f1be5ad5943c9e550ab2c89", + DepGroups: []string{"dev"}, }, { Name: "is-number-4", @@ -283,6 +288,7 @@ func TestParseNpmLock_v1_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", + DepGroups: []string{"dev"}, }, { Name: "is-number-5", @@ -290,6 +296,7 @@ func TestParseNpmLock_v1_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", + DepGroups: []string{"dev"}, }, { Name: "is-number-6", @@ -297,6 +304,7 @@ func TestParseNpmLock_v1_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", + DepGroups: []string{"dev"}, }, { Name: "postcss-calc", @@ -318,6 +326,7 @@ func TestParseNpmLock_v1_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "280b560161b751ba226d50c7db1e0a14a78c2de0", + DepGroups: []string{"dev"}, }, }) } @@ -379,3 +388,30 @@ func TestParseNpmLock_v1_Alias(t *testing.T) { }, }) } + +func TestParseNpmLock_v1_OptionalPackage(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseNpmLock("fixtures/npm/optional-package.v1.json") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "wrappy", + Version: "1.0.2", + Ecosystem: lockfile.NpmEcosystem, + CompareAs: lockfile.NpmEcosystem, + DepGroups: []string{"dev", "optional"}, + }, + { + Name: "supports-color", + Version: "5.5.0", + Ecosystem: lockfile.NpmEcosystem, + CompareAs: lockfile.NpmEcosystem, + DepGroups: []string{"optional"}, + }, + }) +} diff --git a/pkg/lockfile/parse-npm-lock-v2_test.go b/pkg/lockfile/parse-npm-lock-v2_test.go index 33b7a639661..f9ee0da3e05 100644 --- a/pkg/lockfile/parse-npm-lock-v2_test.go +++ b/pkg/lockfile/parse-npm-lock-v2_test.go @@ -71,6 +71,7 @@ func TestParseNpmLock_v2_OnePackageDev(t *testing.T) { Version: "1.0.2", Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, + DepGroups: []string{"dev"}, }, }) } @@ -223,6 +224,7 @@ func TestParseNpmLock_v2_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "c5a7ba5e0ad98b8db1cb8ce105403dd4b768cced", + DepGroups: []string{"dev"}, }, { Name: "is-number-1", @@ -230,6 +232,7 @@ func TestParseNpmLock_v2_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", + DepGroups: []string{"dev"}, }, { Name: "is-number-1", @@ -237,6 +240,7 @@ func TestParseNpmLock_v2_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "be5935f8d2595bcd97b05718ef1eeae08d812e10", + DepGroups: []string{"dev"}, }, { Name: "is-number-2", @@ -244,6 +248,7 @@ func TestParseNpmLock_v2_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", + DepGroups: []string{"dev"}, }, { Name: "is-number-2", @@ -251,6 +256,7 @@ func TestParseNpmLock_v2_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "82dcc8e914dabd9305ab9ae580709a7825e824f5", + DepGroups: []string{"dev"}, }, { Name: "is-number-3", @@ -258,6 +264,7 @@ func TestParseNpmLock_v2_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", + DepGroups: []string{"dev"}, }, { Name: "is-number-3", @@ -265,6 +272,7 @@ func TestParseNpmLock_v2_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "82ae8802978da40d7f1be5ad5943c9e550ab2c89", + DepGroups: []string{"dev"}, }, { Name: "is-number-4", @@ -272,6 +280,7 @@ func TestParseNpmLock_v2_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", + DepGroups: []string{"dev"}, }, { Name: "is-number-5", @@ -279,6 +288,7 @@ func TestParseNpmLock_v2_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", + DepGroups: []string{"dev"}, }, { Name: "postcss-calc", @@ -300,6 +310,7 @@ func TestParseNpmLock_v2_Commits(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "280b560161b751ba226d50c7db1e0a14a78c2de0", + DepGroups: []string{"dev"}, }, }) } @@ -320,6 +331,7 @@ func TestParseNpmLock_v2_Files(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "", + DepGroups: []string{"dev"}, }, { Name: "abbrev", @@ -327,6 +339,7 @@ func TestParseNpmLock_v2_Files(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "", + DepGroups: []string{"dev"}, }, { Name: "abbrev", @@ -334,6 +347,7 @@ func TestParseNpmLock_v2_Files(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, CompareAs: lockfile.NpmEcosystem, Commit: "", + DepGroups: []string{"dev"}, }, }) } @@ -368,3 +382,30 @@ func TestParseNpmLock_v2_Alias(t *testing.T) { }, }) } + +func TestParseNpmLock_v2_OptionalPackage(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseNpmLock("fixtures/npm/optional-package.v2.json") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "wrappy", + Version: "1.0.2", + Ecosystem: lockfile.NpmEcosystem, + CompareAs: lockfile.NpmEcosystem, + DepGroups: []string{"optional"}, + }, + { + Name: "supports-color", + Version: "5.5.0", + Ecosystem: lockfile.NpmEcosystem, + CompareAs: lockfile.NpmEcosystem, + DepGroups: []string{"dev", "optional"}, + }, + }) +} diff --git a/pkg/lockfile/parse-npm-lock.go b/pkg/lockfile/parse-npm-lock.go index 7d0f1b428ed..36a9e781e02 100644 --- a/pkg/lockfile/parse-npm-lock.go +++ b/pkg/lockfile/parse-npm-lock.go @@ -12,6 +12,9 @@ type NpmLockDependency struct { // For an aliased package, Version is like "npm:[name]@[version]" Version string `json:"version"` Dependencies map[string]NpmLockDependency `json:"dependencies,omitempty"` + + Dev bool `json:"dev,omitempty"` + Optional bool `json:"optional,omitempty"` } type NpmLockPackage struct { @@ -20,6 +23,10 @@ type NpmLockPackage struct { Version string `json:"version"` Resolved string `json:"resolved"` Dependencies map[string]string `json:"dependencies"` + + Dev bool `json:"dev,omitempty"` + DevOptional bool `json:"devOptional,omitempty"` + Optional bool `json:"optional,omitempty"` } type NpmLockfile struct { @@ -56,6 +63,20 @@ func mergePkgDetailsMap(m1 map[string]PackageDetails, m2 map[string]PackageDetai return details } +func (dep NpmLockDependency) depGroups() []string { + if dep.Dev && dep.Optional { + return []string{"dev", "optional"} + } + if dep.Dev { + return []string{"dev"} + } + if dep.Optional { + return []string{"optional"} + } + + return nil +} + func parseNpmLockDependencies(dependencies map[string]NpmLockDependency) map[string]PackageDetails { details := map[string]PackageDetails{} @@ -97,6 +118,7 @@ func parseNpmLockDependencies(dependencies map[string]NpmLockDependency) map[str Ecosystem: NpmEcosystem, CompareAs: NpmEcosystem, Commit: commit, + DepGroups: detail.depGroups(), } } @@ -114,6 +136,20 @@ func extractNpmPackageName(name string) string { return pkgName } +func (pkg NpmLockPackage) depGroups() []string { + if pkg.Dev { + return []string{"dev"} + } + if pkg.Optional { + return []string{"optional"} + } + if pkg.DevOptional { + return []string{"dev", "optional"} + } + + return nil +} + func parseNpmLockPackages(packages map[string]NpmLockPackage) map[string]PackageDetails { details := map[string]PackageDetails{} @@ -143,6 +179,7 @@ func parseNpmLockPackages(packages map[string]NpmLockPackage) map[string]Package Ecosystem: NpmEcosystem, CompareAs: NpmEcosystem, Commit: commit, + DepGroups: detail.depGroups(), } } diff --git a/pkg/lockfile/parse-pipenv-lock.go b/pkg/lockfile/parse-pipenv-lock.go index 1c74e98ad93..3d3ce3e6aaf 100644 --- a/pkg/lockfile/parse-pipenv-lock.go +++ b/pkg/lockfile/parse-pipenv-lock.go @@ -34,13 +34,13 @@ func (e PipenvLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { details := make(map[string]PackageDetails) - addPkgDetails(details, parsedLockfile.Packages) - addPkgDetails(details, parsedLockfile.PackagesDev) + addPkgDetails(details, parsedLockfile.Packages, "") + addPkgDetails(details, parsedLockfile.PackagesDev, "dev") return pkgDetailsMapToSlice(details), nil } -func addPkgDetails(details map[string]PackageDetails, packages map[string]PipenvPackage) { +func addPkgDetails(details map[string]PackageDetails, packages map[string]PipenvPackage, group string) { for name, pipenvPackage := range packages { if pipenvPackage.Version == "" { continue @@ -48,11 +48,17 @@ func addPkgDetails(details map[string]PackageDetails, packages map[string]Pipenv version := pipenvPackage.Version[2:] - details[name+"@"+version] = PackageDetails{ - Name: name, - Version: version, - Ecosystem: PipenvEcosystem, - CompareAs: PipenvEcosystem, + if _, ok := details[name+"@"+version]; !ok { + pkgDetails := PackageDetails{ + Name: name, + Version: version, + Ecosystem: PipenvEcosystem, + CompareAs: PipenvEcosystem, + } + if group != "" { + pkgDetails.DepGroups = append(pkgDetails.DepGroups, group) + } + details[name+"@"+version] = pkgDetails } } } diff --git a/pkg/lockfile/parse-pipenv-lock_test.go b/pkg/lockfile/parse-pipenv-lock_test.go index df672d7dc51..b9abbd0bbfa 100644 --- a/pkg/lockfile/parse-pipenv-lock_test.go +++ b/pkg/lockfile/parse-pipenv-lock_test.go @@ -123,6 +123,7 @@ func TestParsePipenvLock_OnePackageDev(t *testing.T) { Version: "2.1.1", Ecosystem: lockfile.PipenvEcosystem, CompareAs: lockfile.PipenvEcosystem, + DepGroups: []string{"dev"}, }, }) } @@ -148,6 +149,7 @@ func TestParsePipenvLock_TwoPackages(t *testing.T) { Version: "2.1.1", Ecosystem: lockfile.PipenvEcosystem, CompareAs: lockfile.PipenvEcosystem, + DepGroups: []string{"dev"}, }, }) } @@ -204,6 +206,7 @@ func TestParsePipenvLock_MultiplePackages(t *testing.T) { Version: "1.0.0", Ecosystem: lockfile.PipenvEcosystem, CompareAs: lockfile.PipenvEcosystem, + DepGroups: []string{"dev"}, }, { Name: "markupsafe", diff --git a/pkg/lockfile/parse-pnpm-lock.go b/pkg/lockfile/parse-pnpm-lock.go index 10a38f119aa..eb062d2d594 100644 --- a/pkg/lockfile/parse-pnpm-lock.go +++ b/pkg/lockfile/parse-pnpm-lock.go @@ -23,6 +23,7 @@ type PnpmLockPackage struct { Resolution PnpmLockPackageResolution `yaml:"resolution"` Name string `yaml:"name"` Version string `yaml:"version"` + Dev bool `yaml:"dev"` } type PnpmLockfile struct { @@ -152,12 +153,18 @@ func parsePnpmLock(lockfile PnpmLockfile) []PackageDetails { } } + var depGroups []string + if pkg.Dev { + depGroups = append(depGroups, "dev") + } + packages = append(packages, PackageDetails{ Name: name, Version: version, Ecosystem: PnpmEcosystem, CompareAs: PnpmEcosystem, Commit: commit, + DepGroups: depGroups, }) } diff --git a/pkg/lockfile/parse-pnpm-lock_test.go b/pkg/lockfile/parse-pnpm-lock_test.go index bf6aeccbc08..b436fe57f76 100644 --- a/pkg/lockfile/parse-pnpm-lock_test.go +++ b/pkg/lockfile/parse-pnpm-lock_test.go @@ -432,6 +432,7 @@ func TestParsePnpmLock_Tarball(t *testing.T) { Ecosystem: lockfile.PnpmEcosystem, CompareAs: lockfile.PnpmEcosystem, Commit: "", + DepGroups: []string{"dev"}, }, }) } diff --git a/pkg/lockfile/parse-poetry-lock.go b/pkg/lockfile/parse-poetry-lock.go index 25290c940f6..7668daeb720 100644 --- a/pkg/lockfile/parse-poetry-lock.go +++ b/pkg/lockfile/parse-poetry-lock.go @@ -13,9 +13,10 @@ type PoetryLockPackageSource struct { } type PoetryLockPackage struct { - Name string `toml:"name"` - Version string `toml:"version"` - Source PoetryLockPackageSource `toml:"source"` + Name string `toml:"name"` + Version string `toml:"version"` + Optional bool `toml:"optional"` + Source PoetryLockPackageSource `toml:"source"` } type PoetryLockFile struct { @@ -43,13 +44,17 @@ func (e PoetryLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { packages := make([]PackageDetails, 0, len(parsedLockfile.Packages)) for _, lockPackage := range parsedLockfile.Packages { - packages = append(packages, PackageDetails{ + pkgDetails := PackageDetails{ Name: lockPackage.Name, Version: lockPackage.Version, Commit: lockPackage.Source.Commit, Ecosystem: PoetryEcosystem, CompareAs: PoetryEcosystem, - }) + } + if lockPackage.Optional { + pkgDetails.DepGroups = append(pkgDetails.DepGroups, "optional") + } + packages = append(packages, pkgDetails) } return packages, nil diff --git a/pkg/lockfile/parse-poetry-lock_test.go b/pkg/lockfile/parse-poetry-lock_test.go index 4a146b9a980..c02cafb9bae 100644 --- a/pkg/lockfile/parse-poetry-lock_test.go +++ b/pkg/lockfile/parse-poetry-lock_test.go @@ -191,3 +191,23 @@ func TestParsePoetryLock_PackageWithLegacySource(t *testing.T) { }, }) } + +func TestParsePoetryLock_OptionalPackage(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParsePoetryLock("fixtures/poetry/optional-package.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "numpy", + Version: "1.23.3", + Ecosystem: lockfile.PoetryEcosystem, + CompareAs: lockfile.PoetryEcosystem, + DepGroups: []string{"optional"}, + }, + }) +} diff --git a/pkg/lockfile/parse-pubspec-lock.go b/pkg/lockfile/parse-pubspec-lock.go index df84d596418..1981c4a99f9 100644 --- a/pkg/lockfile/parse-pubspec-lock.go +++ b/pkg/lockfile/parse-pubspec-lock.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "path/filepath" + "strings" "gopkg.in/yaml.v3" ) @@ -54,6 +55,7 @@ type PubspecLockPackage struct { Source string `yaml:"source"` Description PubspecLockDescription `yaml:"description"` Version string `yaml:"version"` + Dependency string `yaml:"dependency"` } type PubspecLockfile struct { @@ -84,12 +86,19 @@ func (e PubspecLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { packages := make([]PackageDetails, 0, len(parsedLockfile.Packages)) for name, pkg := range parsedLockfile.Packages { - packages = append(packages, PackageDetails{ + pkgDetails := PackageDetails{ Name: name, Version: pkg.Version, Commit: pkg.Description.Ref, Ecosystem: PubEcosystem, - }) + } + for _, str := range strings.Split(pkg.Dependency, " ") { + if str == "dev" { + pkgDetails.DepGroups = append(pkgDetails.DepGroups, "dev") + break + } + } + packages = append(packages, pkgDetails) } return packages, nil diff --git a/pkg/lockfile/parse-pubspec-lock_test.go b/pkg/lockfile/parse-pubspec-lock_test.go index c08e5355e6c..37209148567 100644 --- a/pkg/lockfile/parse-pubspec-lock_test.go +++ b/pkg/lockfile/parse-pubspec-lock_test.go @@ -133,6 +133,7 @@ func TestParsePubspecLock_OnePackageDev(t *testing.T) { Name: "build_runner", Version: "2.2.1", Ecosystem: lockfile.PubEcosystem, + DepGroups: []string{"dev"}, }, }) } @@ -179,6 +180,7 @@ func TestParsePubspecLock_MixedPackages(t *testing.T) { Name: "build_runner", Version: "2.2.1", Ecosystem: lockfile.PubEcosystem, + DepGroups: []string{"dev"}, }, { Name: "shelf", diff --git a/pkg/lockfile/parse-requirements-txt.go b/pkg/lockfile/parse-requirements-txt.go index 038df33b1a8..890cd1aeb4e 100644 --- a/pkg/lockfile/parse-requirements-txt.go +++ b/pkg/lockfile/parse-requirements-txt.go @@ -112,6 +112,17 @@ func (e RequirementsTxtExtractor) Extract(f DepFile) ([]PackageDetails, error) { func parseRequirementsTxt(f DepFile, requiredAlready map[string]struct{}) ([]PackageDetails, error) { packages := map[string]PackageDetails{} + group := strings.TrimSuffix(filepath.Base(f.Path()), filepath.Ext(f.Path())) + hasGroup := func(groups []string) bool { + for _, g := range groups { + if g == group { + return true + } + } + + return false + } + scanner := bufio.NewScanner(f) for scanner.Scan() { line := scanner.Text() @@ -125,7 +136,6 @@ func parseRequirementsTxt(f DepFile, requiredAlready map[string]struct{}) ([]Pac } line = removeComments(line) - if ar := strings.TrimPrefix(line, "-r "); ar != line { err := func() error { af, err := f.Open(ar) @@ -167,8 +177,15 @@ func parseRequirementsTxt(f DepFile, requiredAlready map[string]struct{}) ([]Pac } detail := parseLine(line) - - packages[detail.Name+"@"+detail.Version] = detail + key := detail.Name + "@" + detail.Version + if _, ok := packages[key]; !ok { + packages[key] = detail + } + d := packages[key] + if !hasGroup(d.DepGroups) { + d.DepGroups = append(d.DepGroups, group) + packages[key] = d + } } if err := scanner.Err(); err != nil { diff --git a/pkg/lockfile/parse-requirements-txt_test.go b/pkg/lockfile/parse-requirements-txt_test.go index 2d201bb82df..98e4e337fb6 100644 --- a/pkg/lockfile/parse-requirements-txt_test.go +++ b/pkg/lockfile/parse-requirements-txt_test.go @@ -107,6 +107,7 @@ func TestParseRequirementsTxt_OneRequirementUnconstrained(t *testing.T) { Version: "0.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"one-package-unconstrained"}, }, }) } @@ -126,6 +127,7 @@ func TestParseRequirementsTxt_OneRequirementConstrained(t *testing.T) { Version: "2.2.24", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"one-package-constrained"}, }, }) } @@ -145,78 +147,91 @@ func TestParseRequirementsTxt_MultipleRequirementsConstrained(t *testing.T) { Version: "2.5.1", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-constrained"}, }, { Name: "beautifulsoup4", Version: "4.9.3", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-constrained"}, }, { Name: "boto3", Version: "1.17.19", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-constrained"}, }, { Name: "botocore", Version: "1.20.19", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-constrained"}, }, { Name: "certifi", Version: "2020.12.5", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-constrained"}, }, { Name: "chardet", Version: "4.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-constrained"}, }, { Name: "circus", Version: "0.17.1", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-constrained"}, }, { Name: "click", Version: "7.1.2", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-constrained"}, }, { Name: "django-debug-toolbar", Version: "3.2.1", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-constrained"}, }, { Name: "django-filter", Version: "2.4.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-constrained"}, }, { Name: "django-nose", Version: "1.4.7", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-constrained"}, }, { Name: "django-storages", Version: "1.11.1", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-constrained"}, }, { Name: "django", Version: "2.2.24", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-constrained"}, }, }) } @@ -236,48 +251,56 @@ func TestParseRequirementsTxt_MultipleRequirementsMixed(t *testing.T) { Version: "0.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-mixed"}, }, { Name: "flask-cors", Version: "0.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-mixed"}, }, { Name: "pandas", Version: "0.23.4", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-mixed"}, }, { Name: "numpy", Version: "1.16.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-mixed"}, }, { Name: "scikit-learn", Version: "0.20.1", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-mixed"}, }, { Name: "sklearn", Version: "0.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-mixed"}, }, { Name: "requests", Version: "0.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-mixed"}, }, { Name: "gevent", Version: "0.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-mixed"}, }, }) } @@ -297,60 +320,70 @@ func TestParseRequirementsTxt_FileFormatExample(t *testing.T) { Version: "0.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"file-format-example"}, }, { Name: "pytest-cov", Version: "0.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"file-format-example"}, }, { Name: "beautifulsoup4", Version: "0.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"file-format-example"}, }, { Name: "docopt", Version: "0.6.1", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"file-format-example"}, }, { Name: "keyring", Version: "4.1.1", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"file-format-example"}, }, { Name: "coverage", Version: "0.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"file-format-example"}, }, { Name: "mopidy-dirble", Version: "1.1", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"file-format-example"}, }, { Name: "rejected", Version: "0.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"file-format-example"}, }, { Name: "green", Version: "0.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"file-format-example"}, }, { Name: "django", Version: "2.2.24", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"other-file"}, }, }) } @@ -370,6 +403,7 @@ func TestParseRequirementsTxt_WithAddedSupport(t *testing.T) { Version: "20.3.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"with-added-support"}, }, }) } @@ -389,18 +423,21 @@ func TestParseRequirementsTxt_NonNormalizedNames(t *testing.T) { Version: "5.4.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"non-normalized-names"}, }, { Name: "pillow", Version: "1.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"non-normalized-names"}, }, { Name: "twisted", Version: "20.3.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"non-normalized-names"}, }, }) } @@ -420,60 +457,70 @@ func TestParseRequirementsTxt_WithMultipleROptions(t *testing.T) { Version: "0.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-mixed"}, }, { Name: "flask-cors", Version: "0.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-mixed"}, }, { Name: "pandas", Version: "0.23.4", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-mixed", "with-multiple-r-options"}, }, { Name: "numpy", Version: "1.16.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-mixed"}, }, { Name: "scikit-learn", Version: "0.20.1", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-mixed"}, }, { Name: "sklearn", Version: "0.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-mixed"}, }, { Name: "requests", Version: "0.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-mixed"}, }, { Name: "gevent", Version: "0.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"multiple-packages-mixed"}, }, { Name: "requests", Version: "1.2.3", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"with-multiple-r-options"}, }, { Name: "django", Version: "2.2.24", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"one-package-constrained"}, }, }) } @@ -502,24 +549,28 @@ func TestParseRequirementsTxt_DuplicateROptions(t *testing.T) { Version: "0.1.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"duplicate-r-base"}, }, { Name: "pandas", Version: "0.23.4", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"duplicate-r-dev"}, }, { Name: "requests", Version: "1.2.3", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"duplicate-r-test", "duplicate-r-dev"}, }, { Name: "unittest", Version: "1.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"duplicate-r-test"}, }, }) } @@ -539,12 +590,14 @@ func TestParseRequirementsTxt_CyclicRSelf(t *testing.T) { Version: "0.23.4", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"cyclic-r-self"}, }, { Name: "requests", Version: "1.2.3", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"cyclic-r-self"}, }, }) } @@ -564,18 +617,21 @@ func TestParseRequirementsTxt_CyclicRComplex(t *testing.T) { Version: "1", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"cyclic-r-complex-1"}, }, { Name: "cyclic-r-complex", Version: "2", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"cyclic-r-complex-2"}, }, { Name: "cyclic-r-complex", Version: "3", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"cyclic-r-complex-3"}, }, }) } @@ -595,24 +651,28 @@ func TestParseRequirementsTxt_WithPerRequirementOptions(t *testing.T) { Version: "1.26.121", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"with-per-requirement-options"}, }, { Name: "foo", Version: "1.0.0", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"with-per-requirement-options"}, }, { Name: "fooproject", Version: "1.2", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"with-per-requirement-options"}, }, { Name: "barproject", Version: "1.2", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"with-per-requirement-options"}, }, }) } @@ -632,24 +692,28 @@ func TestParseRequirementsTxt_LineContinuation(t *testing.T) { Version: "1.2.3", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"line-continuation"}, }, { Name: "bar", Version: "4.5\\\\", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"line-continuation"}, }, { Name: "baz", Version: "7.8.9", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"line-continuation"}, }, { Name: "qux", Version: "10.11.12", Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, + DepGroups: []string{"line-continuation"}, }, }) } diff --git a/pkg/lockfile/types.go b/pkg/lockfile/types.go index 3c6037804ad..870b9621c52 100644 --- a/pkg/lockfile/types.go +++ b/pkg/lockfile/types.go @@ -6,8 +6,35 @@ type PackageDetails struct { Commit string `json:"commit,omitempty"` Ecosystem Ecosystem `json:"ecosystem,omitempty"` CompareAs Ecosystem `json:"compareAs,omitempty"` + DepGroups []string `json:"-"` } type Ecosystem string type PackageDetailsParser = func(pathToLockfile string) ([]PackageDetails, error) + +// IsDevGroup returns if any string in groups indicates the development dependency group for the specified ecosystem. +func (sys Ecosystem) IsDevGroup(groups []string) bool { + dev := "" + switch sys { + case ComposerEcosystem, NpmEcosystem, PipEcosystem, PubEcosystem: + // Also PnpmEcosystem(=NpmEcosystem) and PipenvEcosystem(=PipEcosystem). + dev = "dev" + case ConanEcosystem: + dev = "build-requires" + case MavenEcosystem: + dev = "test" + case AlpineEcosystem, BundlerEcosystem, CargoEcosystem, CRANEcosystem, + DebianEcosystem, GoEcosystem, MixEcosystem, NuGetEcosystem: + // We are not able to report development dependencies for these ecosystems. + return false + } + + for _, g := range groups { + if g == dev { + return true + } + } + + return false +} diff --git a/pkg/models/results.go b/pkg/models/results.go index 422fa873ba8..7a24fef65bb 100644 --- a/pkg/models/results.go +++ b/pkg/models/results.go @@ -73,7 +73,8 @@ type SourceInfo struct { } type Metadata struct { - RepoURL string `json:"repo_url"` + RepoURL string `json:"repo_url"` + DepGroups []string `json:"-"` } func (s SourceInfo) String() string { @@ -93,6 +94,7 @@ type License string // TODO: rename this to be Package as it now includes license information too. type PackageVulns struct { Package PackageInfo `json:"package"` + DepGroups []string `json:"dependency_groups,omitempty"` Vulnerabilities []Vulnerability `json:"vulnerabilities,omitempty"` Groups []GroupInfo `json:"groups,omitempty"` Licenses []License `json:"licenses,omitempty"` diff --git a/pkg/osv/osv.go b/pkg/osv/osv.go index 6fe7443363c..313de343c95 100644 --- a/pkg/osv/osv.go +++ b/pkg/osv/osv.go @@ -123,7 +123,8 @@ func MakePkgRequest(pkgDetails lockfile.PackageDetails) *Query { if pkgDetails.Ecosystem == "" && pkgDetails.Commit != "" { return &Query{ Metadata: models.Metadata{ - RepoURL: pkgDetails.Name, + RepoURL: pkgDetails.Name, + DepGroups: pkgDetails.DepGroups, }, Commit: pkgDetails.Commit, } @@ -134,6 +135,9 @@ func MakePkgRequest(pkgDetails lockfile.PackageDetails) *Query { Name: pkgDetails.Name, Ecosystem: string(pkgDetails.Ecosystem), }, + Metadata: models.Metadata{ + DepGroups: pkgDetails.DepGroups, + }, } } } diff --git a/pkg/osvscanner/osvscanner.go b/pkg/osvscanner/osvscanner.go index 03a7aeddb9c..57d562e7d2d 100644 --- a/pkg/osvscanner/osvscanner.go +++ b/pkg/osvscanner/osvscanner.go @@ -380,6 +380,7 @@ func scanLockfile(r reporter.Reporter, path string, parseAs string) ([]scannedPa Version: pkgDetail.Version, Commit: pkgDetail.Commit, Ecosystem: pkgDetail.Ecosystem, + DepGroups: pkgDetail.DepGroups, Source: models.SourceInfo{ Path: path, Type: "lockfile", @@ -692,6 +693,7 @@ type scannedPackage struct { Commit string Version string Source models.SourceInfo + DepGroups []string } // Perform osv scanner action, with optional reporter to output information diff --git a/pkg/osvscanner/vulnerability_result.go b/pkg/osvscanner/vulnerability_result.go index 2d20e4e756c..b085682fc2e 100644 --- a/pkg/osvscanner/vulnerability_result.go +++ b/pkg/osvscanner/vulnerability_result.go @@ -50,6 +50,7 @@ func buildVulnerabilityResults( Ecosystem: string(rawPkg.Ecosystem), } } + pkg.DepGroups = rawPkg.DepGroups if len(vulnsResp.Results[i].Vulns) > 0 { includePackage = true