diff --git a/cmd/osv-scanner/__snapshots__/main_test.snap b/cmd/osv-scanner/__snapshots__/main_test.snap index 4a68608220..3743845355 100755 --- a/cmd/osv-scanner/__snapshots__/main_test.snap +++ b/cmd/osv-scanner/__snapshots__/main_test.snap @@ -168,7 +168,8 @@ Scanned /fixtures/go-project/nested/go.mod file and found 1 package [TestRun/PURL_SBOM_case_sensitivity_(api) - 1] Scanning dir ./fixtures/sbom-insecure/alpine.cdx.xml -Scanned /fixtures/sbom-insecure/alpine.cdx.xml as CycloneDX SBOM and found 14 packages +Scanned /fixtures/sbom-insecure/alpine.cdx.xml file and found 15 packages +Filtered 1 local/unscannable package/s from the scan. +--------------------------------+------+-----------+---------+-----------+---------------------------------------+ | OSV URL | CVSS | ECOSYSTEM | PACKAGE | VERSION | SOURCE | +--------------------------------+------+-----------+---------+-----------+---------------------------------------+ @@ -184,7 +185,8 @@ Scanned /fixtures/sbom-insecure/alpine.cdx.xml as CycloneDX SBOM and fo [TestRun/PURL_SBOM_case_sensitivity_(local) - 1] Scanning dir ./fixtures/sbom-insecure/alpine.cdx.xml -Scanned /fixtures/sbom-insecure/alpine.cdx.xml as CycloneDX SBOM and found 14 packages +Scanned /fixtures/sbom-insecure/alpine.cdx.xml file and found 15 packages +Filtered 1 local/unscannable package/s from the scan. Loaded Alpine local db from /osv-scanner/Alpine/all.zip +--------------------------------+------+-----------+---------+-----------+---------------------------------------+ | OSV URL | CVSS | ECOSYSTEM | PACKAGE | VERSION | SOURCE | @@ -279,10 +281,11 @@ Scanned /fixtures/locks-many/package-lock.json file and found 1 package [TestRun/Scan_locks-many - 1] Scanning dir ./fixtures/locks-many Scanned /fixtures/locks-many/Gemfile.lock file and found 1 package -Scanned /fixtures/locks-many/alpine.cdx.xml as CycloneDX SBOM and found 14 packages +Scanned /fixtures/locks-many/alpine.cdx.xml file and found 15 packages Scanned /fixtures/locks-many/composer.lock file and found 1 package Scanned /fixtures/locks-many/package-lock.json file and found 1 package Scanned /fixtures/locks-many/yarn.lock file and found 1 package +Filtered 1 local/unscannable package/s from the scan. Loaded filter from: /fixtures/locks-many/osv-scanner.toml GHSA-whgm-jr23-g3j9 and 1 alias have been filtered out because: Test manifest file Filtered 1 vulnerability from output @@ -302,7 +305,7 @@ Scanned /fixtures/locks-many-with-invalid/yarn.lock file and found 1 pa --- [TestRun/all_supported_lockfiles_in_the_directory_should_be_checked - 2] -Attempted to scan lockfile but failed: /fixtures/locks-many-with-invalid/composer.lock +Error during extraction: (extracting as php/composerlock) could not extract from /fixtures/locks-many-with-invalid/composer.lock: invalid character ',' looking for beginning of object key string --- @@ -310,7 +313,7 @@ Attempted to scan lockfile but failed: /fixtures/locks-many-with-invali Scanned /fixtures/locks-insecure/osv-scanner-flutter-deps.json file as a osv-scanner and found 3 packages Scanning dir ./fixtures/locks-many Scanned /fixtures/locks-many/Gemfile.lock file and found 1 package -Scanned /fixtures/locks-many/alpine.cdx.xml as CycloneDX SBOM and found 14 packages +Scanned /fixtures/locks-many/alpine.cdx.xml file and found 15 packages Scanned /fixtures/locks-many/composer.lock file and found 1 package Scanned /fixtures/locks-many/package-lock.json file and found 1 package Scanned /fixtures/locks-many/yarn.lock file and found 1 package @@ -318,14 +321,15 @@ Scanning dir ./fixtures/locks-insecure Scanned /fixtures/locks-insecure/composer.lock file and found 1 package Scanning dir ./fixtures/maven-transitive Scanned /fixtures/maven-transitive/pom.xml file and found 3 packages +Filtered 1 local/unscannable package/s from the scan. Package npm/ansi-html/0.0.1 has been filtered out because: (no reason given) Package npm/balanced-match/1.0.2 has been filtered out because: (no reason given) Package Maven/org.apache.logging.log4j:log4j-api/2.14.1 has been filtered out because: it makes the table output really really long Package Maven/org.apache.logging.log4j:log4j-core/2.14.1 has been filtered out because: it makes the table output really really long Package Maven/org.apache.logging.log4j:log4j-web/2.14.1 has been filtered out because: it makes the table output really really long Filtered 5 ignored package/s from the scan. -overriding license for package Alpine/alpine-baselayout-data/3.4.0-r0 with MIT overriding license for package Alpine/alpine-baselayout/3.4.0-r0 with MIT +overriding license for package Alpine/alpine-baselayout-data/3.4.0-r0 with MIT overriding license for package Alpine/alpine-keys/2.4-r1 with MIT overriding license for package Alpine/apk-tools/2.12.10-r1 with MIT overriding license for package Alpine/busybox-binsh/1.36.1-r27 with MIT @@ -333,8 +337,8 @@ overriding license for package Alpine/ca-certificates-bundle/20220614-r4 with MI overriding license for package Alpine/libc-utils/0.7.2-r3 with MIT overriding license for package Alpine/libcrypto3/3.0.8-r0 with MIT overriding license for package Alpine/libssl3/3.0.8-r0 with MIT -overriding license for package Alpine/musl-utils/1.2.3-r4 with MIT overriding license for package Alpine/musl/1.2.3-r4 with MIT +overriding license for package Alpine/musl-utils/1.2.3-r4 with MIT overriding license for package Alpine/scanelf/1.3.5-r1 with MIT overriding license for package Alpine/ssl_client/1.36.1-r27 with MIT overriding license for package Alpine/zlib/1.2.13-r0 with MIT @@ -399,10 +403,11 @@ unknown keys in config file: RustVersionOverride, PackageOverrides.skip, Package warning: ./fixtures/osv-scanner-duplicate-config.toml has multiple ignores for GO-2022-0274 - only the first will be used! Scanning dir ./fixtures/locks-many Scanned /fixtures/locks-many/Gemfile.lock file and found 1 package -Scanned /fixtures/locks-many/alpine.cdx.xml as CycloneDX SBOM and found 14 packages +Scanned /fixtures/locks-many/alpine.cdx.xml file and found 15 packages Scanned /fixtures/locks-many/composer.lock file and found 1 package Scanned /fixtures/locks-many/package-lock.json file and found 1 package Scanned /fixtures/locks-many/yarn.lock file and found 1 package +Filtered 1 local/unscannable package/s from the scan. GHSA-whgm-jr23-g3j9 and 1 alias have been filtered out because: (no reason given) Filtered 1 vulnerability from output No issues found @@ -541,26 +546,17 @@ Scanned /fixtures/locks-insecure/composer.lock file and found 1 package [TestRun/folder_of_supported_sbom_with_vulns - 1] Scanning dir ./fixtures/sbom-insecure/ -Scanned /fixtures/sbom-insecure/alpine.cdx.xml as CycloneDX SBOM and found 14 packages -Scanned /fixtures/sbom-insecure/bad-purls.cdx.xml as CycloneDX SBOM and found 8 packages -Ignored 6 packages with invalid PURLs -Ignored invalid PURL "/" -Ignored invalid PURL "pkg:///" -Ignored invalid PURL "pkg:apk/alpine/@1.36.1-r27?arch=x86_64&upstream=busybox&distro=alpine-3.17.2" -Ignored invalid PURL "pkg:pypi/" -Scanned /fixtures/sbom-insecure/postgres-stretch.cdx.xml as CycloneDX SBOM and found 136 packages -Warning, duplicate PURL found in SBOM: pkg:apk/alpine/libcrypto3@3.0.8-r0?arch=x86_64&upstream=openssl&distro=alpine-3.17.2 -Warning, duplicate PURL found in SBOM: pkg:apk/alpine/zlib@1.2.10-r0?arch=x86_64&upstream=zlib&distro=alpine-3.17.2 -Scanned /fixtures/sbom-insecure/with-duplicates.cdx.xml as CycloneDX SBOM and found 14 packages +Scanned /fixtures/sbom-insecure/alpine.cdx.xml file and found 15 packages +Scanned /fixtures/sbom-insecure/bad-purls.cdx.xml file and found 15 packages +Scanned /fixtures/sbom-insecure/postgres-stretch.cdx.xml file and found 136 packages +Scanned /fixtures/sbom-insecure/with-duplicates.cdx.xml file and found 15 packages +Filtered 9 local/unscannable package/s from the scan. +-------------------------------------+------+-----------+--------------------------------+------------------------------------+-------------------------------------------------+ | OSV URL | CVSS | ECOSYSTEM | PACKAGE | VERSION | SOURCE | +-------------------------------------+------+-----------+--------------------------------+------------------------------------+-------------------------------------------------+ | https://osv.dev/CVE-2018-25032 | 7.5 | Alpine | zlib | 1.2.10-r0 | fixtures/sbom-insecure/alpine.cdx.xml | | https://osv.dev/CVE-2022-37434 | 9.8 | Alpine | zlib | 1.2.10-r0 | fixtures/sbom-insecure/alpine.cdx.xml | | https://osv.dev/DLA-3022-1 | | Debian | dpkg | 1.18.25 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/DLA-3012-1 | | Debian | libxml2 | 2.9.4+dfsg1-2.2+deb9u6 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/DLA-3008-1 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/DLA-3051-1 | | Debian | tzdata | 2021a-0+deb9u3 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/GO-2022-0274 | 6.0 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/GHSA-v95c-p5hm-xq8f | | | | | | | https://osv.dev/GO-2022-0452 | 5.9 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | @@ -577,6 +573,9 @@ Scanned /fixtures/sbom-insecure/with-duplicates.cdx.xml as CycloneDX SB | https://osv.dev/GHSA-jfvp-7x6p-h2pv | | | | | | | https://osv.dev/GO-2022-0493 | 5.3 | Go | golang.org/x/sys | v0.0.0-20210817142637-7d9622a276b7 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/GHSA-p782-xgp4-8hr8 | | | | | | +| https://osv.dev/DLA-3012-1 | | Debian | libxml2 | 2.9.4+dfsg1-2.2+deb9u6 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/DLA-3008-1 | | Debian | openssl | 1.1.0l-1~deb9u5 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/DLA-3051-1 | | Debian | tzdata | 2021a-0+deb9u3 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2018-25032 | 7.5 | Alpine | zlib | 1.2.10-r0 | fixtures/sbom-insecure/with-duplicates.cdx.xml | | https://osv.dev/CVE-2022-37434 | 9.8 | Alpine | zlib | 1.2.10-r0 | fixtures/sbom-insecure/with-duplicates.cdx.xml | +-------------------------------------+------+-----------+--------------------------------+------------------------------------+-------------------------------------------------+ @@ -720,9 +719,8 @@ No issues found --- [TestRun/one_specific_supported_sbom_with_duplicate_PURLs - 1] -Warning, duplicate PURL found in SBOM: pkg:apk/alpine/libcrypto3@3.0.8-r0?arch=x86_64&upstream=openssl&distro=alpine-3.17.2 -Warning, duplicate PURL found in SBOM: pkg:apk/alpine/zlib@1.2.10-r0?arch=x86_64&upstream=zlib&distro=alpine-3.17.2 -Scanned /fixtures/sbom-insecure/with-duplicates.cdx.xml as CycloneDX SBOM and found 14 packages +Scanned /fixtures/sbom-insecure/with-duplicates.cdx.xml file and found 15 packages +Filtered 1 local/unscannable package/s from the scan. +--------------------------------+------+-----------+---------+-----------+------------------------------------------------+ | OSV URL | CVSS | ECOSYSTEM | PACKAGE | VERSION | SOURCE | +--------------------------------+------+-----------+---------+-----------+------------------------------------------------+ @@ -737,12 +735,8 @@ Scanned /fixtures/sbom-insecure/with-duplicates.cdx.xml as CycloneDX SB --- [TestRun/one_specific_supported_sbom_with_invalid_PURLs - 1] -Scanned /fixtures/sbom-insecure/bad-purls.cdx.xml as CycloneDX SBOM and found 8 packages -Ignored 6 packages with invalid PURLs -Ignored invalid PURL "/" -Ignored invalid PURL "pkg:///" -Ignored invalid PURL "pkg:apk/alpine/@1.36.1-r27?arch=x86_64&upstream=busybox&distro=alpine-3.17.2" -Ignored invalid PURL "pkg:pypi/" +Scanned /fixtures/sbom-insecure/bad-purls.cdx.xml file and found 15 packages +Filtered 7 local/unscannable package/s from the scan. No issues found --- @@ -752,7 +746,8 @@ No issues found --- [TestRun/one_specific_supported_sbom_with_vulns - 1] -Scanned /fixtures/sbom-insecure/alpine.cdx.xml as CycloneDX SBOM and found 14 packages +Scanned /fixtures/sbom-insecure/alpine.cdx.xml file and found 15 packages +Filtered 1 local/unscannable package/s from the scan. +--------------------------------+------+-----------+---------+-----------+---------------------------------------+ | OSV URL | CVSS | ECOSYSTEM | PACKAGE | VERSION | SOURCE | +--------------------------------+------+-----------+---------+-----------+---------------------------------------+ @@ -1349,10 +1344,11 @@ Scanned /fixtures/locks-licenses/package-lock.json file and found 4 pac [TestRun_Licenses/No_vulnerabilities_with_license_summary - 1] Scanning dir ./fixtures/locks-many Scanned /fixtures/locks-many/Gemfile.lock file and found 1 package -Scanned /fixtures/locks-many/alpine.cdx.xml as CycloneDX SBOM and found 14 packages +Scanned /fixtures/locks-many/alpine.cdx.xml file and found 15 packages Scanned /fixtures/locks-many/composer.lock file and found 1 package Scanned /fixtures/locks-many/package-lock.json file and found 1 package Scanned /fixtures/locks-many/yarn.lock file and found 1 package +Filtered 1 local/unscannable package/s from the scan. Loaded filter from: /fixtures/locks-many/osv-scanner.toml GHSA-whgm-jr23-g3j9 and 1 alias have been filtered out because: Test manifest file Filtered 1 vulnerability from output @@ -1373,10 +1369,11 @@ Filtered 1 vulnerability from output [TestRun_Licenses/No_vulnerabilities_with_license_summary_in_markdown - 1] Scanning dir ./fixtures/locks-many Scanned /fixtures/locks-many/Gemfile.lock file and found 1 package -Scanned /fixtures/locks-many/alpine.cdx.xml as CycloneDX SBOM and found 14 packages +Scanned /fixtures/locks-many/alpine.cdx.xml file and found 15 packages Scanned /fixtures/locks-many/composer.lock file and found 1 package Scanned /fixtures/locks-many/package-lock.json file and found 1 package Scanned /fixtures/locks-many/yarn.lock file and found 1 package +Filtered 1 local/unscannable package/s from the scan. Loaded filter from: /fixtures/locks-many/osv-scanner.toml GHSA-whgm-jr23-g3j9 and 1 alias have been filtered out because: Test manifest file Filtered 1 vulnerability from output @@ -1395,17 +1392,18 @@ Filtered 1 vulnerability from output [TestRun_Licenses/Some_packages_with_ignored_licenses - 1] Scanning dir ./fixtures/locks-many Scanned /fixtures/locks-many/Gemfile.lock file and found 1 package -Scanned /fixtures/locks-many/alpine.cdx.xml as CycloneDX SBOM and found 14 packages +Scanned /fixtures/locks-many/alpine.cdx.xml file and found 15 packages Scanned /fixtures/locks-many/composer.lock file and found 1 package Scanned /fixtures/locks-many/package-lock.json file and found 1 package Scanned /fixtures/locks-many/yarn.lock file and found 1 package Scanning dir ./fixtures/locks-insecure Scanned /fixtures/locks-insecure/composer.lock file and found 1 package +Filtered 1 local/unscannable package/s from the scan. Package npm/ansi-html/0.0.1 has been filtered out because: (no reason given) Package npm/balanced-match/1.0.2 has been filtered out because: (no reason given) Filtered 2 ignored package/s from the scan. -ignoring license for package Alpine/alpine-baselayout-data/3.4.0-r0 ignoring license for package Alpine/alpine-baselayout/3.4.0-r0 +ignoring license for package Alpine/alpine-baselayout-data/3.4.0-r0 ignoring license for package Alpine/alpine-keys/2.4-r1 ignoring license for package Alpine/apk-tools/2.12.10-r1 ignoring license for package Alpine/busybox-binsh/1.36.1-r27 @@ -1413,8 +1411,8 @@ ignoring license for package Alpine/ca-certificates-bundle/20220614-r4 ignoring license for package Alpine/libc-utils/0.7.2-r3 ignoring license for package Alpine/libcrypto3/3.0.8-r0 ignoring license for package Alpine/libssl3/3.0.8-r0 -ignoring license for package Alpine/musl-utils/1.2.3-r4 overriding license for package Alpine/musl/1.2.3-r4 with UNKNOWN +ignoring license for package Alpine/musl-utils/1.2.3-r4 ignoring license for package Alpine/scanelf/1.3.5-r1 ignoring license for package Alpine/ssl_client/1.36.1-r27 ignoring license for package Alpine/zlib/1.2.13-r0 @@ -1642,10 +1640,11 @@ No issues found [TestRun_LocalDatabases/all_supported_lockfiles_in_the_directory_should_be_checked - 1] Scanning dir ./fixtures/locks-many Scanned /fixtures/locks-many/Gemfile.lock file and found 1 package -Scanned /fixtures/locks-many/alpine.cdx.xml as CycloneDX SBOM and found 14 packages +Scanned /fixtures/locks-many/alpine.cdx.xml file and found 15 packages Scanned /fixtures/locks-many/composer.lock file and found 1 package Scanned /fixtures/locks-many/package-lock.json file and found 1 package Scanned /fixtures/locks-many/yarn.lock file and found 1 package +Filtered 1 local/unscannable package/s from the scan. Loaded filter from: /fixtures/locks-many/osv-scanner.toml Loaded RubyGems local db from /osv-scanner/RubyGems/all.zip Loaded Alpine local db from /osv-scanner/Alpine/all.zip @@ -1664,10 +1663,11 @@ No issues found [TestRun_LocalDatabases/all_supported_lockfiles_in_the_directory_should_be_checked - 3] Scanning dir ./fixtures/locks-many Scanned /fixtures/locks-many/Gemfile.lock file and found 1 package -Scanned /fixtures/locks-many/alpine.cdx.xml as CycloneDX SBOM and found 14 packages +Scanned /fixtures/locks-many/alpine.cdx.xml file and found 15 packages Scanned /fixtures/locks-many/composer.lock file and found 1 package Scanned /fixtures/locks-many/package-lock.json file and found 1 package Scanned /fixtures/locks-many/yarn.lock file and found 1 package +Filtered 1 local/unscannable package/s from the scan. Loaded filter from: /fixtures/locks-many/osv-scanner.toml Loaded RubyGems local db from /osv-scanner/RubyGems/all.zip Loaded Alpine local db from /osv-scanner/Alpine/all.zip @@ -1693,7 +1693,7 @@ Loaded npm local db from /osv-scanner/npm/all.zip --- [TestRun_LocalDatabases/all_supported_lockfiles_in_the_directory_should_be_checked#01 - 2] -Attempted to scan lockfile but failed: /fixtures/locks-many-with-invalid/composer.lock +Error during extraction: (extracting as php/composerlock) could not extract from /fixtures/locks-many-with-invalid/composer.lock: invalid character ',' looking for beginning of object key string --- @@ -1707,7 +1707,7 @@ Loaded npm local db from /osv-scanner/npm/all.zip --- [TestRun_LocalDatabases/all_supported_lockfiles_in_the_directory_should_be_checked#01 - 4] -Attempted to scan lockfile but failed: /fixtures/locks-many-with-invalid/composer.lock +Error during extraction: (extracting as php/composerlock) could not extract from /fixtures/locks-many-with-invalid/composer.lock: invalid character ',' looking for beginning of object key string --- @@ -1827,10 +1827,10 @@ No issues found [TestRun_LocalDatabases/one_specific_supported_sbom_with_vulns - 1] Scanning dir ./fixtures/sbom-insecure/postgres-stretch.cdx.xml -Scanned /fixtures/sbom-insecure/postgres-stretch.cdx.xml as CycloneDX SBOM and found 136 packages +Scanned /fixtures/sbom-insecure/postgres-stretch.cdx.xml file and found 136 packages Loaded Debian local db from /osv-scanner/Debian/all.zip -Loaded OSS-Fuzz local db from /osv-scanner/OSS-Fuzz/all.zip Loaded Go local db from /osv-scanner/Go/all.zip +Loaded OSS-Fuzz local db from /osv-scanner/OSS-Fuzz/all.zip +-------------------------------------+------+-----------+--------------------------------+------------------------------------+-------------------------------------------------+ | OSV URL | CVSS | ECOSYSTEM | PACKAGE | VERSION | SOURCE | +-------------------------------------+------+-----------+--------------------------------+------------------------------------+-------------------------------------------------+ @@ -1848,6 +1848,14 @@ Loaded Go local db from /osv-scanner/Go/all.zip | https://osv.dev/CVE-2019-5188 | 6.7 | Debian | e2fsprogs | 1.43.4-2+deb9u2 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2022-1304 | 7.8 | Debian | e2fsprogs | 1.43.4-2+deb9u2 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/DLA-3910-1 | | Debian | e2fsprogs | 1.43.4-2+deb9u2 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/GHSA-f3fp-gc8g-vw66 | 5.9 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/GHSA-g2j6-57v7-gm8c | 6.1 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/GHSA-jfvp-7x6p-h2pv | 4.8 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/GHSA-m8cg-xc2p-r3fc | 2.5 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/GHSA-v95c-p5hm-xq8f | 6.0 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/GHSA-vpvm-3wq2-2wvm | 7.0 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/GHSA-xr7r-f8xq-vfvv | 8.6 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/GHSA-p782-xgp4-8hr8 | 5.3 | Go | golang.org/x/sys | v0.0.0-20210817142637-7d9622a276b7 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/DSA-5122-1 | 8.8 | Debian | gzip | 1.6-5+deb9u1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2017-0379 | 7.5 | Debian | libgcrypt20 | 1.7.6-2+deb9u4 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2017-7526 | 6.8 | Debian | libgcrypt20 | 1.7.6-2+deb9u4 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | @@ -1988,14 +1996,6 @@ Loaded Go local db from /osv-scanner/Go/all.zip | https://osv.dev/DLA-3782-1 | | Debian | util-linux | 2.29.2-1+deb9u1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/DSA-5123-1 | 8.8 | Debian | xz-utils | 5.2.2-1.2+deb9u1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2024-3094 | 10.0 | Debian | xz-utils | 5.2.2-1.2+deb9u1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/GHSA-f3fp-gc8g-vw66 | 5.9 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/GHSA-g2j6-57v7-gm8c | 6.1 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/GHSA-jfvp-7x6p-h2pv | 4.8 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/GHSA-m8cg-xc2p-r3fc | 2.5 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/GHSA-v95c-p5hm-xq8f | 6.0 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/GHSA-vpvm-3wq2-2wvm | 7.0 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/GHSA-xr7r-f8xq-vfvv | 8.6 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/GHSA-p782-xgp4-8hr8 | 5.3 | Go | golang.org/x/sys | v0.0.0-20210817142637-7d9622a276b7 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +-------------------------------------+------+-----------+--------------------------------+------------------------------------+-------------------------------------------------+ | Unimportant vulnerabilities | | | | | | +-------------------------------------+------+-----------+--------------------------------+------------------------------------+-------------------------------------------------+ @@ -2025,10 +2025,10 @@ Loaded Go local db from /osv-scanner/Go/all.zip [TestRun_LocalDatabases/one_specific_supported_sbom_with_vulns - 3] Scanning dir ./fixtures/sbom-insecure/postgres-stretch.cdx.xml -Scanned /fixtures/sbom-insecure/postgres-stretch.cdx.xml as CycloneDX SBOM and found 136 packages +Scanned /fixtures/sbom-insecure/postgres-stretch.cdx.xml file and found 136 packages Loaded Debian local db from /osv-scanner/Debian/all.zip -Loaded OSS-Fuzz local db from /osv-scanner/OSS-Fuzz/all.zip Loaded Go local db from /osv-scanner/Go/all.zip +Loaded OSS-Fuzz local db from /osv-scanner/OSS-Fuzz/all.zip +-------------------------------------+------+-----------+--------------------------------+------------------------------------+-------------------------------------------------+ | OSV URL | CVSS | ECOSYSTEM | PACKAGE | VERSION | SOURCE | +-------------------------------------+------+-----------+--------------------------------+------------------------------------+-------------------------------------------------+ @@ -2046,6 +2046,14 @@ Loaded Go local db from /osv-scanner/Go/all.zip | https://osv.dev/CVE-2019-5188 | 6.7 | Debian | e2fsprogs | 1.43.4-2+deb9u2 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2022-1304 | 7.8 | Debian | e2fsprogs | 1.43.4-2+deb9u2 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/DLA-3910-1 | | Debian | e2fsprogs | 1.43.4-2+deb9u2 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/GHSA-f3fp-gc8g-vw66 | 5.9 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/GHSA-g2j6-57v7-gm8c | 6.1 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/GHSA-jfvp-7x6p-h2pv | 4.8 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/GHSA-m8cg-xc2p-r3fc | 2.5 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/GHSA-v95c-p5hm-xq8f | 6.0 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/GHSA-vpvm-3wq2-2wvm | 7.0 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/GHSA-xr7r-f8xq-vfvv | 8.6 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +| https://osv.dev/GHSA-p782-xgp4-8hr8 | 5.3 | Go | golang.org/x/sys | v0.0.0-20210817142637-7d9622a276b7 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/DSA-5122-1 | 8.8 | Debian | gzip | 1.6-5+deb9u1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2017-0379 | 7.5 | Debian | libgcrypt20 | 1.7.6-2+deb9u4 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2017-7526 | 6.8 | Debian | libgcrypt20 | 1.7.6-2+deb9u4 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | @@ -2186,14 +2194,6 @@ Loaded Go local db from /osv-scanner/Go/all.zip | https://osv.dev/DLA-3782-1 | | Debian | util-linux | 2.29.2-1+deb9u1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/DSA-5123-1 | 8.8 | Debian | xz-utils | 5.2.2-1.2+deb9u1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | | https://osv.dev/CVE-2024-3094 | 10.0 | Debian | xz-utils | 5.2.2-1.2+deb9u1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/GHSA-f3fp-gc8g-vw66 | 5.9 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/GHSA-g2j6-57v7-gm8c | 6.1 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/GHSA-jfvp-7x6p-h2pv | 4.8 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/GHSA-m8cg-xc2p-r3fc | 2.5 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/GHSA-v95c-p5hm-xq8f | 6.0 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/GHSA-vpvm-3wq2-2wvm | 7.0 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/GHSA-xr7r-f8xq-vfvv | 8.6 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | -| https://osv.dev/GHSA-p782-xgp4-8hr8 | 5.3 | Go | golang.org/x/sys | v0.0.0-20210817142637-7d9622a276b7 | fixtures/sbom-insecure/postgres-stretch.cdx.xml | +-------------------------------------+------+-----------+--------------------------------+------------------------------------+-------------------------------------------------+ | Unimportant vulnerabilities | | | | | | +-------------------------------------+------+-----------+--------------------------------+------------------------------------+-------------------------------------------------+ @@ -2384,10 +2384,11 @@ Scanned /fixtures/locks-requirements/requirements.txt file and found 3 Scanned /fixtures/locks-requirements/the_requirements_for_test.txt file and found 1 package Scanning dir ./fixtures/locks-many Scanned /fixtures/locks-many/Gemfile.lock file and found 1 package -Scanned /fixtures/locks-many/alpine.cdx.xml as CycloneDX SBOM and found 14 packages +Scanned /fixtures/locks-many/alpine.cdx.xml file and found 15 packages Scanned /fixtures/locks-many/composer.lock file and found 1 package Scanned /fixtures/locks-many/package-lock.json file and found 1 package Scanned /fixtures/locks-many/yarn.lock file and found 1 package +Filtered 1 local/unscannable package/s from the scan. --- @@ -2405,10 +2406,11 @@ Scanned /fixtures/locks-requirements/requirements.txt file and found 3 Scanned /fixtures/locks-requirements/the_requirements_for_test.txt file and found 1 package Scanning dir ./fixtures/locks-many Scanned /fixtures/locks-many/Gemfile.lock file and found 1 package -Scanned /fixtures/locks-many/alpine.cdx.xml as CycloneDX SBOM and found 14 packages +Scanned /fixtures/locks-many/alpine.cdx.xml file and found 15 packages Scanned /fixtures/locks-many/composer.lock file and found 1 package Scanned /fixtures/locks-many/package-lock.json file and found 1 package Scanned /fixtures/locks-many/yarn.lock file and found 1 package +Filtered 1 local/unscannable package/s from the scan. --- @@ -2517,7 +2519,7 @@ Scanned /fixtures/locks-insecure/composer.lock file and found 1 package [TestRun_LockfileWithExplicitParseAs/one_lockfile_with_local_path - 1] Scanned /fixtures/locks-many/replace-local.mod file as a go.mod and found 1 package -Filtered 1 local package/s from the scan. +Filtered 1 local/unscannable package/s from the scan. No issues found --- diff --git a/cmd/osv-scanner/main_test.go b/cmd/osv-scanner/main_test.go index a1ea382a99..a31ba2cf28 100644 --- a/cmd/osv-scanner/main_test.go +++ b/cmd/osv-scanner/main_test.go @@ -750,8 +750,11 @@ func TestRun_Licenses(t *testing.T) { } } +// TODO(v2): Image scanning is not temporarily disabled + func TestRun_Docker(t *testing.T) { t.Parallel() + t.Skip("Skipping until image scanning is reenabled") testutility.SkipIfNotAcceptanceTesting(t, "Takes a long time to pull down images") @@ -797,6 +800,7 @@ func TestRun_Docker(t *testing.T) { func TestRun_OCIImage(t *testing.T) { t.Parallel() + t.Skip("Skipping until image scanning is reenabled") testutility.SkipIfNotAcceptanceTesting(t, "Not consistent on MacOS/Windows") diff --git a/go.mod b/go.mod index 190c3fc96e..eb95636231 100644 --- a/go.mod +++ b/go.mod @@ -22,10 +22,10 @@ require ( github.com/ianlancetaylor/demangle v0.0.0-20240912202439-0a2b6291aafd github.com/jedib0t/go-pretty/v6 v6.6.0 github.com/muesli/reflow v0.3.0 + github.com/ossf/osv-schema/bindings/go v0.0.0-20241127234932-c44c7842979f github.com/owenrumney/go-sarif/v2 v2.3.3 github.com/package-url/packageurl-go v0.1.3 github.com/pandatix/go-cvss v0.6.2 - github.com/spdx/tools-golang v0.5.5 github.com/tidwall/gjson v1.18.0 github.com/tidwall/pretty v1.2.1 github.com/tidwall/sjson v1.2.5 @@ -60,6 +60,7 @@ require ( github.com/dlclark/regexp2 v1.11.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/erikvarga/go-rpmdb v0.0.0-20240208180226-b97e041ef9af // indirect github.com/gkampitakis/ciinfo v0.3.0 // indirect github.com/gkampitakis/go-diff v1.3.2 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect @@ -75,6 +76,7 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect @@ -89,6 +91,7 @@ require ( github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect github.com/spdx/gordf v0.0.0-20221230105357-b735bd5aac89 // indirect + github.com/spdx/tools-golang v0.5.5 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/vbatts/tar-split v0.11.5 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect @@ -100,7 +103,9 @@ require ( golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.19.0 // indirect golang.org/x/tools v0.26.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 36bfaa2ada..a1a97e8918 100644 --- a/go.sum +++ b/go.sum @@ -77,18 +77,24 @@ github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBi github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker-credential-helpers v0.8.1 h1:j/eKUktUltBtMzKqmfLB0PAgqYyMHOp5vfsD1807oKo= github.com/docker/docker-credential-helpers v0.8.1/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/erikvarga/go-rpmdb v0.0.0-20240208180226-b97e041ef9af h1:JXdZ7gz1cike1HMJJiP57Ll3/wb7zEjFOBKVDMEFi4M= +github.com/erikvarga/go-rpmdb v0.0.0-20240208180226-b97e041ef9af/go.mod h1:MiEorPk0IChAoCwpg2FXyqVgbNvOlPWZAYHqqIoDNoY= github.com/gkampitakis/ciinfo v0.3.0 h1:gWZlOC2+RYYttL0hBqcoQhM7h1qNkVqvRCV1fOvpAv8= github.com/gkampitakis/ciinfo v0.3.0/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= github.com/gkampitakis/go-snaps v0.5.7 h1:uVGjHR4t4pPHU944udMx7VKHpwepZXmvDMF+yDmI0rg= github.com/gkampitakis/go-snaps v0.5.7/go.mod h1:ZABkO14uCuVxBHAXAfKG+bqNz+aa1bGPAg8jkI0Nk8Y= +github.com/glebarez/go-sqlite v1.20.3 h1:89BkqGOXR9oRmG58ZrzgoY/Fhy5x0M+/WV48U5zVrZ4= +github.com/glebarez/go-sqlite v1.20.3/go.mod h1:u3N6D/wftiAzIOJtZl6BmedqxmmkDfH3q+ihjqxC9u0= github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= @@ -111,6 +117,8 @@ github.com/google/go-containerregistry v0.20.2 h1:B1wPJ1SN/S7pB+ZAimcciVD+r+yV/l github.com/google/go-containerregistry v0.20.2/go.mod h1:z38EKdKh4h7IP2gSfUUqEvalZBqs6AoLeWfUy34nQC8= github.com/google/osv-scalibr v0.1.4-0.20241031120023-761ca671aacb h1:A7IvUJk8r3wMuuAMWxwbkE3WBp+oF/v7CcEt3nCy+lI= github.com/google/osv-scalibr v0.1.4-0.20241031120023-761ca671aacb/go.mod h1:MbEYB+PKqEGjwMdpcoO5DWpi0+57jYgYcw2jlRy8O9Q= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -145,6 +153,8 @@ github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+Ei github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -163,6 +173,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/ossf/osv-schema/bindings/go v0.0.0-20241127234932-c44c7842979f h1:F7CMsEIwWsbYgt9tNLMOnVrqqz1WmxmwpRCLqNeJ1N0= +github.com/ossf/osv-schema/bindings/go v0.0.0-20241127234932-c44c7842979f/go.mod h1:lILztSxHU7VsdlYqCnwgxSDBhbXMf7iEQWtldJCDXPo= github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= github.com/owenrumney/go-sarif/v2 v2.3.3 h1:ubWDJcF5i3L/EIOER+ZyQ03IfplbSU1BLOE26uKQIIU= github.com/owenrumney/go-sarif/v2 v2.3.3/go.mod h1:MSqMMx9WqlBSY7pXoOZWgEsVB4FDNfhcaXDA1j6Sr+w= @@ -177,6 +189,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 h1:VstopitMQi3hZP0fzvnsLmzXZdQGc4bEcgu24cp+d4M= +github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= @@ -319,6 +333,8 @@ golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0 golang.org/x/vuln v1.0.4 h1:SP0mPeg2PmGCu03V+61EcQiOjmpri2XijexKdzv8Z1I= golang.org/x/vuln v1.0.4/go.mod h1:NbJdUQhX8jY++FtuhrXs2Eyx0yePo9pF7nPlIjo9aaQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 h1:T6rh4haD3GVYsgEfWExoCZA2o2FmbNyKpTuAxbEFPTg= google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:wp2WsuBYj6j8wUdo3ToZsdxxixbvQNAHqVJrTgi5E5M= @@ -342,4 +358,13 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/libc v1.22.2 h1:4U7v51GyhlWqQmwCHj28Rdq2Yzwk55ovjFrdPjs8Hb0= +modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= +modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/sqlite v1.20.3 h1:SqGJMMxjj1PHusLxdYxeQSodg7Jxn9WWkaAQjKrntZs= +modernc.org/sqlite v1.20.3/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/internal/config/config.go b/internal/config/config.go index 43af5279c4..13db57ee01 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -10,7 +10,7 @@ import ( "time" "github.com/BurntSushi/toml" - "github.com/google/osv-scanner/pkg/models" + "github.com/google/osv-scanner/internal/imodels" "github.com/google/osv-scanner/pkg/reporter" ) @@ -53,14 +53,17 @@ type PackageOverrideEntry struct { Reason string `toml:"reason"` } -func (e PackageOverrideEntry) matches(pkg models.PackageVulns) bool { - if e.Name != "" && e.Name != pkg.Package.Name { +func (e PackageOverrideEntry) matches(pkg imodels.PackageInfo) bool { + if e.Name != "" && e.Name != pkg.Name { return false } - if e.Version != "" && e.Version != pkg.Package.Version { + if e.Version != "" && e.Version != pkg.Version { return false } - if e.Ecosystem != "" && e.Ecosystem != pkg.Package.Ecosystem { + // If there is an ecosystem filter, the filter must not match both the: + // - Full ecosystem + suffix + // - The base ecosystem + if e.Ecosystem != "" && (e.Ecosystem != pkg.Ecosystem.String() && e.Ecosystem != string(pkg.Ecosystem.Ecosystem)) { return false } if e.Group != "" && !slices.Contains(pkg.DepGroups, e.Group) { @@ -89,7 +92,7 @@ func (c *Config) ShouldIgnore(vulnID string) (bool, IgnoreEntry) { return shouldIgnoreTimestamp(ignoredLine.IgnoreUntil), ignoredLine } -func (c *Config) filterPackageVersionEntries(pkg models.PackageVulns, condition func(PackageOverrideEntry) bool) (bool, PackageOverrideEntry) { +func (c *Config) filterPackageVersionEntries(pkg imodels.PackageInfo, condition func(PackageOverrideEntry) bool) (bool, PackageOverrideEntry) { index := slices.IndexFunc(c.PackageOverrides, func(e PackageOverrideEntry) bool { return e.matches(pkg) && condition(e) }) @@ -102,14 +105,14 @@ func (c *Config) filterPackageVersionEntries(pkg models.PackageVulns, condition } // ShouldIgnorePackage determines if the given package should be ignored based on override entries in the config -func (c *Config) ShouldIgnorePackage(pkg models.PackageVulns) (bool, PackageOverrideEntry) { +func (c *Config) ShouldIgnorePackage(pkg imodels.PackageInfo) (bool, PackageOverrideEntry) { return c.filterPackageVersionEntries(pkg, func(e PackageOverrideEntry) bool { return e.Ignore }) } // ShouldIgnorePackageVulnerabilities determines if the given package should have its vulnerabilities ignored based on override entries in the config -func (c *Config) ShouldIgnorePackageVulnerabilities(pkg models.PackageVulns) bool { +func (c *Config) ShouldIgnorePackageVulnerabilities(pkg imodels.PackageInfo) bool { overrides, _ := c.filterPackageVersionEntries(pkg, func(e PackageOverrideEntry) bool { return e.Vulnerability.Ignore }) @@ -118,7 +121,7 @@ func (c *Config) ShouldIgnorePackageVulnerabilities(pkg models.PackageVulns) boo } // ShouldOverridePackageLicense determines if the given package should have its license ignored or changed based on override entries in the config -func (c *Config) ShouldOverridePackageLicense(pkg models.PackageVulns) (bool, PackageOverrideEntry) { +func (c *Config) ShouldOverridePackageLicense(pkg imodels.PackageInfo) (bool, PackageOverrideEntry) { return c.filterPackageVersionEntries(pkg, func(e PackageOverrideEntry) bool { return e.License.Ignore || len(e.License.Override) > 0 }) diff --git a/internal/config/config_internal_test.go b/internal/config/config_internal_test.go index 79aac2e6cf..d62cf1cdde 100644 --- a/internal/config/config_internal_test.go +++ b/internal/config/config_internal_test.go @@ -8,7 +8,8 @@ import ( "time" "github.com/google/go-cmp/cmp" - "github.com/google/osv-scanner/pkg/models" + "github.com/google/osv-scanner/internal/imodels" + "github.com/google/osv-scanner/internal/imodels/ecosystem" "github.com/google/osv-scanner/pkg/reporter" ) @@ -361,7 +362,7 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { tests := []struct { name string config Config - args models.PackageVulns + args imodels.PackageInfo wantOk bool wantEntry PackageOverrideEntry }{ @@ -376,12 +377,10 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("Go"), DepGroups: []string{"dev"}, }, wantOk: true, @@ -404,12 +403,10 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("Go"), DepGroups: []string{"dev"}, }, wantOk: true, @@ -432,18 +429,87 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib2", - Version: "1.0.0", - Ecosystem: "npm", - }, + args: imodels.PackageInfo{ + Name: "lib2", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("npm"), DepGroups: []string{"dev"}, }, wantOk: false, wantEntry: PackageOverrideEntry{}, }, // ------------------------------------------------------------------------- + { + name: "Ecosystem-level entry with suffix exists and does match", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Ecosystem: "Alpine:3.20", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: imodels.PackageInfo{ + Name: "bin1", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("Alpine:3.20"), + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Ecosystem: "Alpine:3.20", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + { + name: "Ecosystem-level entry with suffix exists and does not match", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Ecosystem: "Alpine:3.20", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: imodels.PackageInfo{ + Name: "bin2", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("Alpine:3.19"), + }, + wantOk: false, + wantEntry: PackageOverrideEntry{}, + }, + { + name: "Ecosystem-level entry without suffix exists and does match", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Ecosystem: "Alpine", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: imodels.PackageInfo{ + Name: "bin1", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("Alpine:3.20"), + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Ecosystem: "Alpine", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + // ------------------------------------------------------------------------- { name: "Group-level entry exists and does match", config: Config{ @@ -456,12 +522,10 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("Go"), DepGroups: []string{"dev"}, }, wantOk: true, @@ -484,12 +548,10 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib2", - Version: "1.0.0", - Ecosystem: "npm", - }, + args: imodels.PackageInfo{ + Name: "lib2", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("npm"), DepGroups: []string{"optional"}, }, wantOk: false, @@ -507,12 +569,10 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib2", - Version: "1.0.0", - Ecosystem: "npm", - }, + args: imodels.PackageInfo{ + Name: "lib2", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("npm"), }, wantOk: false, wantEntry: PackageOverrideEntry{}, @@ -530,12 +590,10 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("Go"), DepGroups: []string{"dev"}, }, wantOk: true, @@ -558,12 +616,10 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.1", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.1", + Ecosystem: ecosystem.MustParse("Go"), DepGroups: []string{"dev"}, }, wantOk: false, @@ -582,12 +638,10 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("Go"), DepGroups: []string{"dev"}, }, wantOk: true, @@ -610,12 +664,10 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib2", - Version: "1.0.0", - Ecosystem: "npm", - }, + args: imodels.PackageInfo{ + Name: "lib2", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("npm"), DepGroups: []string{"dev"}, }, wantOk: false, @@ -636,12 +688,10 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("Go"), }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -666,12 +716,10 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("Go"), }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -696,12 +744,10 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("Go"), DepGroups: []string{"dev"}, }, wantOk: true, @@ -728,12 +774,10 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("Go"), DepGroups: []string{"prod"}, }, wantOk: false, @@ -761,12 +805,10 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "2.0.0", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "2.0.0", + Ecosystem: ecosystem.MustParse("Go"), }, wantOk: false, wantEntry: PackageOverrideEntry{}, @@ -794,7 +836,7 @@ func TestConfig_ShouldIgnorePackageVulnerabilities(t *testing.T) { tests := []struct { name string config Config - args models.PackageVulns + args imodels.PackageInfo wantOk bool }{ { @@ -812,12 +854,10 @@ func TestConfig_ShouldIgnorePackageVulnerabilities(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("Go"), }, wantOk: true, }, @@ -836,12 +876,10 @@ func TestConfig_ShouldIgnorePackageVulnerabilities(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.1", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.1", + Ecosystem: ecosystem.MustParse("Go"), }, wantOk: false, }, @@ -859,12 +897,10 @@ func TestConfig_ShouldIgnorePackageVulnerabilities(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.1", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.1", + Ecosystem: ecosystem.MustParse("Go"), }, wantOk: true, }, @@ -888,7 +924,7 @@ func TestConfig_ShouldOverridePackageLicense(t *testing.T) { tests := []struct { name string config Config - args models.PackageVulns + args imodels.PackageInfo wantOk bool wantEntry PackageOverrideEntry }{ @@ -907,12 +943,10 @@ func TestConfig_ShouldOverridePackageLicense(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("Go"), }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -940,12 +974,10 @@ func TestConfig_ShouldOverridePackageLicense(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: ecosystem.MustParse("Go"), }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -973,12 +1005,10 @@ func TestConfig_ShouldOverridePackageLicense(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.1", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.1", + Ecosystem: ecosystem.MustParse("Go"), }, wantOk: false, wantEntry: PackageOverrideEntry{}, @@ -998,12 +1028,10 @@ func TestConfig_ShouldOverridePackageLicense(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.1", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.1", + Ecosystem: ecosystem.MustParse("Go"), }, wantOk: false, wantEntry: PackageOverrideEntry{}, @@ -1022,12 +1050,10 @@ func TestConfig_ShouldOverridePackageLicense(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.1", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.1", + Ecosystem: ecosystem.MustParse("Go"), }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -1053,12 +1079,10 @@ func TestConfig_ShouldOverridePackageLicense(t *testing.T) { }, }, }, - args: models.PackageVulns{ - Package: models.PackageInfo{ - Name: "lib1", - Version: "1.0.1", - Ecosystem: "Go", - }, + args: imodels.PackageInfo{ + Name: "lib1", + Version: "1.0.1", + Ecosystem: ecosystem.MustParse("Go"), }, wantOk: true, wantEntry: PackageOverrideEntry{ diff --git a/internal/depsdev/license.go b/internal/depsdev/license.go index aca27bacc0..2e3de4b4d7 100644 --- a/internal/depsdev/license.go +++ b/internal/depsdev/license.go @@ -5,9 +5,9 @@ import ( "crypto/x509" "fmt" - "github.com/google/osv-scanner/pkg/lockfile" "github.com/google/osv-scanner/pkg/models" "github.com/google/osv-scanner/pkg/osv" + "github.com/ossf/osv-schema/bindings/go/osvschema" depsdevpb "deps.dev/api/v3" "golang.org/x/sync/errgroup" @@ -22,13 +22,13 @@ import ( const DepsdevAPI = "api.deps.dev:443" // System maps from a lockfile system to the depsdev API system. -var System = map[lockfile.Ecosystem]depsdevpb.System{ - lockfile.NpmEcosystem: depsdevpb.System_NPM, - lockfile.NuGetEcosystem: depsdevpb.System_NUGET, - lockfile.CargoEcosystem: depsdevpb.System_CARGO, - lockfile.GoEcosystem: depsdevpb.System_GO, - lockfile.MavenEcosystem: depsdevpb.System_MAVEN, - lockfile.PipEcosystem: depsdevpb.System_PYPI, +var System = map[osvschema.Ecosystem]depsdevpb.System{ + osvschema.EcosystemNPM: depsdevpb.System_NPM, + osvschema.EcosystemNuGet: depsdevpb.System_NUGET, + osvschema.EcosystemCratesIO: depsdevpb.System_CARGO, + osvschema.EcosystemGo: depsdevpb.System_GO, + osvschema.EcosystemMaven: depsdevpb.System_MAVEN, + osvschema.EcosystemPyPI: depsdevpb.System_PYPI, } // VersionQuery constructs a GetVersion request from the arguments. diff --git a/internal/image/extractor.go b/internal/image/extractor.go index 18dad0ed63..562162109a 100644 --- a/internal/image/extractor.go +++ b/internal/image/extractor.go @@ -12,8 +12,8 @@ import ( "github.com/google/osv-scalibr/extractor/filesystem/language/golang/gobinary" "github.com/google/osv-scalibr/extractor/filesystem/os/apk" "github.com/google/osv-scalibr/extractor/filesystem/os/dpkg" - "github.com/google/osv-scanner/internal/lockfilescalibr" - "github.com/google/osv-scanner/internal/lockfilescalibr/language/javascript/nodemodules" + "github.com/google/osv-scanner/internal/scalibrextract" + "github.com/google/osv-scanner/internal/scalibrextract/language/javascript/nodemodules" "github.com/google/osv-scanner/pkg/lockfile" ) @@ -53,7 +53,7 @@ func extractArtifactDeps(extractPath string, layer *Layer) ([]*extractor.Invento scalibrPath := strings.TrimPrefix(extractPath, "/") foundExtractors := findArtifactExtractor(scalibrPath, pathFileInfo) if len(foundExtractors) == 0 { - return nil, fmt.Errorf("%w for %s", lockfilescalibr.ErrExtractorNotFound, extractPath) + return nil, fmt.Errorf("%w for %s", scalibrextract.ErrExtractorNotFound, extractPath) } inventories := []*extractor.Inventory{} @@ -96,7 +96,7 @@ func extractArtifactDeps(extractPath string, layer *Layer) ([]*extractor.Invento } if extractedAs == "" { - return nil, fmt.Errorf("%w for %s", lockfilescalibr.ErrExtractorNotFound, extractPath) + return nil, fmt.Errorf("%w for %s", scalibrextract.ErrExtractorNotFound, extractPath) } // Perform any one-off translations here diff --git a/internal/image/fixtures/test-node_modules-npm-empty.Dockerfile b/internal/image/fixtures/test-node_modules-npm-empty.Dockerfile index 67ff3b79f7..2dbae77b8f 100644 --- a/internal/image/fixtures/test-node_modules-npm-empty.Dockerfile +++ b/internal/image/fixtures/test-node_modules-npm-empty.Dockerfile @@ -1,6 +1,6 @@ ARG MANAGER_VERSION="10.2.4" -FROM node:20-alpine@sha256:c0a3badbd8a0a760de903e00cedbca94588e609299820557e72cba2a53dbaa2c +FROM node:20-alpine@sha256:426f843809ae05f324883afceebaa2b9cab9cb697097dbb1a2a7a41c5701de72 WORKDIR /prod/app diff --git a/internal/image/fixtures/test-node_modules-npm-full.Dockerfile b/internal/image/fixtures/test-node_modules-npm-full.Dockerfile index 96e136b5f7..1043f54004 100644 --- a/internal/image/fixtures/test-node_modules-npm-full.Dockerfile +++ b/internal/image/fixtures/test-node_modules-npm-full.Dockerfile @@ -1,6 +1,6 @@ ARG MANAGER_VERSION="10.2.4" -FROM node:20-alpine@sha256:c0a3badbd8a0a760de903e00cedbca94588e609299820557e72cba2a53dbaa2c +FROM node:20-alpine@sha256:426f843809ae05f324883afceebaa2b9cab9cb697097dbb1a2a7a41c5701de72 WORKDIR /prod/app diff --git a/internal/image/fixtures/test-node_modules-pnpm-empty.Dockerfile b/internal/image/fixtures/test-node_modules-pnpm-empty.Dockerfile index 7a221ca7ea..fd97227527 100644 --- a/internal/image/fixtures/test-node_modules-pnpm-empty.Dockerfile +++ b/internal/image/fixtures/test-node_modules-pnpm-empty.Dockerfile @@ -1,6 +1,6 @@ ARG MANAGER_VERSION="8.15.4" -FROM node:20-alpine@sha256:c0a3badbd8a0a760de903e00cedbca94588e609299820557e72cba2a53dbaa2c +FROM node:20-alpine@sha256:426f843809ae05f324883afceebaa2b9cab9cb697097dbb1a2a7a41c5701de72 WORKDIR /prod/app diff --git a/internal/image/fixtures/test-node_modules-pnpm-full.Dockerfile b/internal/image/fixtures/test-node_modules-pnpm-full.Dockerfile index 80e1ee6519..666de1ef50 100644 --- a/internal/image/fixtures/test-node_modules-pnpm-full.Dockerfile +++ b/internal/image/fixtures/test-node_modules-pnpm-full.Dockerfile @@ -1,6 +1,6 @@ ARG MANAGER_VERSION="8.15.4" -FROM node:20-alpine@sha256:c0a3badbd8a0a760de903e00cedbca94588e609299820557e72cba2a53dbaa2c +FROM node:20-alpine@sha256:426f843809ae05f324883afceebaa2b9cab9cb697097dbb1a2a7a41c5701de72 WORKDIR /prod/app diff --git a/internal/image/fixtures/test-node_modules-yarn-empty.Dockerfile b/internal/image/fixtures/test-node_modules-yarn-empty.Dockerfile index 41f4c2f423..b9743644d1 100644 --- a/internal/image/fixtures/test-node_modules-yarn-empty.Dockerfile +++ b/internal/image/fixtures/test-node_modules-yarn-empty.Dockerfile @@ -1,6 +1,6 @@ ARG MANAGER_VERSION="1.22.22" -FROM node:20-alpine@sha256:c0a3badbd8a0a760de903e00cedbca94588e609299820557e72cba2a53dbaa2c +FROM node:20-alpine@sha256:426f843809ae05f324883afceebaa2b9cab9cb697097dbb1a2a7a41c5701de72 WORKDIR /prod/app diff --git a/internal/image/fixtures/test-node_modules-yarn-full.Dockerfile b/internal/image/fixtures/test-node_modules-yarn-full.Dockerfile index 99e9653f01..c675c21ad3 100644 --- a/internal/image/fixtures/test-node_modules-yarn-full.Dockerfile +++ b/internal/image/fixtures/test-node_modules-yarn-full.Dockerfile @@ -1,6 +1,6 @@ ARG MANAGER_VERSION="1.22.22" -FROM node:20-alpine@sha256:c0a3badbd8a0a760de903e00cedbca94588e609299820557e72cba2a53dbaa2c +FROM node:20-alpine@sha256:426f843809ae05f324883afceebaa2b9cab9cb697097dbb1a2a7a41c5701de72 WORKDIR /prod/app diff --git a/internal/image/image_test.go b/internal/image/image_test.go deleted file mode 100644 index bc4397ab4e..0000000000 --- a/internal/image/image_test.go +++ /dev/null @@ -1,99 +0,0 @@ -package image_test - -import ( - "errors" - "os" - "testing" - - "github.com/google/osv-scanner/internal/image" - "github.com/google/osv-scanner/internal/testutility" - "github.com/google/osv-scanner/pkg/reporter" -) - -func TestScanImage(t *testing.T) { - t.Parallel() - testutility.SkipIfNotAcceptanceTesting(t, "Not consistent on MacOS/Windows") - - type args struct { - imagePath string - } - tests := []struct { - name string - args args - want testutility.Snapshot - wantErr bool - }{ - { - name: "Alpine 3.10 image tar with 3.18 version file", - args: args{imagePath: "fixtures/test-alpine.tar"}, - want: testutility.NewSnapshot(), - wantErr: false, - }, - { - name: "scanning node_modules using npm with no packages", - args: args{imagePath: "fixtures/test-node_modules-npm-empty.tar"}, - want: testutility.NewSnapshot(), - wantErr: false, - }, - { - name: "scanning node_modules using npm with some packages", - args: args{imagePath: "fixtures/test-node_modules-npm-full.tar"}, - want: testutility.NewSnapshot(), - wantErr: false, - }, - { - name: "scanning node_modules using yarn with no packages", - args: args{imagePath: "fixtures/test-node_modules-yarn-empty.tar"}, - want: testutility.NewSnapshot(), - wantErr: false, - }, - { - name: "scanning node_modules using yarn with some packages", - args: args{imagePath: "fixtures/test-node_modules-yarn-full.tar"}, - want: testutility.NewSnapshot(), - wantErr: false, - }, - { - name: "scanning node_modules using pnpm with no packages", - args: args{imagePath: "fixtures/test-node_modules-pnpm-empty.tar"}, - want: testutility.NewSnapshot(), - wantErr: false, - }, - { - name: "scanning node_modules using pnpm with some packages", - args: args{imagePath: "fixtures/test-node_modules-pnpm-full.tar"}, - want: testutility.NewSnapshot(), - wantErr: false, - }, - { - name: "scanning go binaries that's been overwritten for package tracing", - args: args{imagePath: "fixtures/test-package-tracing.tar"}, - want: testutility.NewSnapshot(), - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - // point out that we need the images to be built and saved separately - if _, err := os.Stat(tt.args.imagePath); errors.Is(err, os.ErrNotExist) { - t.Fatalf("%s does not exist - have you run scripts/build_test_images.sh?", tt.args.imagePath) - } - - got, err := image.ScanImage(&reporter.VoidReporter{}, tt.args.imagePath) - if (err != nil) != tt.wantErr { - t.Errorf("ScanImage() error = %v, wantErr %v", err, tt.wantErr) - return - } - - for _, lockfile := range got.Lockfiles { - for _, pkg := range lockfile.Packages { - pkg.ImageOrigin.LayerID = "" - } - } - - tt.want.MatchJSON(t, got) - }) - } -} diff --git a/internal/image/image_test.go.disabled b/internal/image/image_test.go.disabled new file mode 100644 index 0000000000..787f258cc7 --- /dev/null +++ b/internal/image/image_test.go.disabled @@ -0,0 +1,99 @@ +// package image_test + +// import ( +// "errors" +// "os" +// "testing" + +// "github.com/google/osv-scanner/internal/image" +// "github.com/google/osv-scanner/internal/testutility" +// "github.com/google/osv-scanner/pkg/reporter" +// ) + +// func TestScanImage(t *testing.T) { +// t.Parallel() +// testutility.SkipIfNotAcceptanceTesting(t, "Not consistent on MacOS/Windows") + +// type args struct { +// imagePath string +// } +// tests := []struct { +// name string +// args args +// want testutility.Snapshot +// wantErr bool +// }{ +// { +// name: "Alpine 3.10 image tar with 3.18 version file", +// args: args{imagePath: "fixtures/test-alpine.tar"}, +// want: testutility.NewSnapshot(), +// wantErr: false, +// }, +// { +// name: "scanning node_modules using npm with no packages", +// args: args{imagePath: "fixtures/test-node_modules-npm-empty.tar"}, +// want: testutility.NewSnapshot(), +// wantErr: false, +// }, +// { +// name: "scanning node_modules using npm with some packages", +// args: args{imagePath: "fixtures/test-node_modules-npm-full.tar"}, +// want: testutility.NewSnapshot(), +// wantErr: false, +// }, +// { +// name: "scanning node_modules using yarn with no packages", +// args: args{imagePath: "fixtures/test-node_modules-yarn-empty.tar"}, +// want: testutility.NewSnapshot(), +// wantErr: false, +// }, +// { +// name: "scanning node_modules using yarn with some packages", +// args: args{imagePath: "fixtures/test-node_modules-yarn-full.tar"}, +// want: testutility.NewSnapshot(), +// wantErr: false, +// }, +// { +// name: "scanning node_modules using pnpm with no packages", +// args: args{imagePath: "fixtures/test-node_modules-pnpm-empty.tar"}, +// want: testutility.NewSnapshot(), +// wantErr: false, +// }, +// { +// name: "scanning node_modules using pnpm with some packages", +// args: args{imagePath: "fixtures/test-node_modules-pnpm-full.tar"}, +// want: testutility.NewSnapshot(), +// wantErr: false, +// }, +// { +// name: "scanning go binaries that's been overwritten for package tracing", +// args: args{imagePath: "fixtures/test-package-tracing.tar"}, +// want: testutility.NewSnapshot(), +// wantErr: false, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// t.Parallel() + +// // point out that we need the images to be built and saved separately +// if _, err := os.Stat(tt.args.imagePath); errors.Is(err, os.ErrNotExist) { +// t.Fatalf("%s does not exist - have you run scripts/build_test_images.sh?", tt.args.imagePath) +// } + +// got, err := image.ScanImage(&reporter.VoidReporter{}, tt.args.imagePath) +// if (err != nil) != tt.wantErr { +// t.Errorf("ScanImage() error = %v, wantErr %v", err, tt.wantErr) +// return +// } + +// for _, lockfile := range got.Lockfiles { +// for _, pkg := range lockfile.Packages { +// pkg.ImageOrigin.LayerID = "" +// } +// } + +// tt.want.MatchJSON(t, got) +// }) +// } +// } diff --git a/internal/image/scan.go b/internal/image/scan.go index 689e47d481..cc98f5bbc7 100644 --- a/internal/image/scan.go +++ b/internal/image/scan.go @@ -12,7 +12,7 @@ import ( "github.com/google/osv-scalibr/extractor" "github.com/google/osv-scalibr/extractor/filesystem/os/dpkg" - "github.com/google/osv-scanner/internal/lockfilescalibr" + "github.com/google/osv-scanner/internal/scalibrextract" "github.com/google/osv-scanner/pkg/lockfile" "github.com/google/osv-scanner/pkg/models" "github.com/google/osv-scanner/pkg/reporter" @@ -56,7 +56,7 @@ func ScanImage(r reporter.Reporter, imagePath string) (ScanResults, error) { extractedInventories, err := extractArtifactDeps(file.virtualPath, img.LastLayer()) if err != nil { - if !errors.Is(err, lockfilescalibr.ErrExtractorNotFound) { + if !errors.Is(err, scalibrextract.ErrExtractorNotFound) { r.Errorf("Attempted to extract lockfile but failed: %s - %v\n", file.virtualPath, err) } diff --git a/internal/imodels/ecosystem/ecosystem.go b/internal/imodels/ecosystem/ecosystem.go new file mode 100644 index 0000000000..8fe7d86e49 --- /dev/null +++ b/internal/imodels/ecosystem/ecosystem.go @@ -0,0 +1,107 @@ +package ecosystem + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/ossf/osv-schema/bindings/go/osvschema" +) + +// ecosystemsWithSuffix documents all the ecosystems that can have a suffix +var ecosystemsWithSuffix = map[osvschema.Ecosystem]struct{}{ + osvschema.EcosystemAlpine: {}, + osvschema.EcosystemAlmaLinux: {}, + osvschema.EcosystemAndroid: {}, + osvschema.EcosystemDebian: {}, + osvschema.EcosystemMageia: {}, + osvschema.EcosystemMaven: {}, + osvschema.EcosystemOpenSUSE: {}, + osvschema.EcosystemPhotonOS: {}, + osvschema.EcosystemRedHat: {}, + osvschema.EcosystemRockyLinux: {}, + osvschema.EcosystemSUSE: {}, + osvschema.EcosystemUbuntu: {}, +} + +// Parsed represents an ecosystem-with-suffix string as defined by the [spec], parsed into +// a structured format. +// +// The suffix is optional and is separated from the ecosystem by a colon. +// +// For example, "Debian:7" would be parsed into Parsed{Ecosystem: constants.EcosystemDebian, Suffix: "7"} +// +// [spec]: https://ossf.github.io/osv-schema/ +// +//nolint:recvcheck +type Parsed struct { + Ecosystem osvschema.Ecosystem + Suffix string +} + +func (p *Parsed) IsEmpty() bool { + return p.Ecosystem == "" +} + +// UnmarshalJSON handles unmarshalls a JSON string into a Parsed struct. +// +// This method implements the json.Unmarshaler interface. +// +//goland:noinspection GoMixedReceiverTypes +func (p *Parsed) UnmarshalJSON(data []byte) error { + var str string + err := json.Unmarshal(data, &str) + + if err != nil { + return err + } + + *p = MustParse(str) + + return nil +} + +// MarshalJSON handles marshals a Parsed struct into a JSON string. +// +// This method implements the json.Marshaler interface. +// +//goland:noinspection GoMixedReceiverTypes +func (p Parsed) MarshalJSON() ([]byte, error) { + return []byte(`"` + p.String() + `"`), nil +} + +//goland:noinspection GoMixedReceiverTypes +func (p *Parsed) String() string { + str := string(p.Ecosystem) + + if p.Suffix != "" { + str += ":" + p.Suffix + } + + return str +} + +// MustParse parses a string into a constants.Ecosystem and an optional suffix specified with a ":" +// Panics if there is an invalid ecosystem +func MustParse(str string) Parsed { + parsed, err := Parse(str) + if err != nil { + panic("Failed MustParse: " + err.Error()) + } + + return parsed +} + +// Parse parses a string into a constants.Ecosystem and an optional suffix specified with a ":" +func Parse(str string) (Parsed, error) { + ecosystem, suffix, _ := strings.Cut(str, ":") + + // Always return the full parsed value even if it might be invalid + // Let the caller decide how to handle the error + var err error + if _, ok := ecosystemsWithSuffix[osvschema.Ecosystem(ecosystem)]; !ok && suffix != "" { + err = fmt.Errorf("found ecosystem %q has a suffix %q, but it should not", ecosystem, suffix) + } + + return Parsed{osvschema.Ecosystem(ecosystem), suffix}, err +} diff --git a/internal/imodels/ecosystem/ecosystem_test.go b/internal/imodels/ecosystem/ecosystem_test.go new file mode 100644 index 0000000000..e39505c07d --- /dev/null +++ b/internal/imodels/ecosystem/ecosystem_test.go @@ -0,0 +1,266 @@ +package ecosystem_test + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/google/osv-scanner/internal/imodels/ecosystem" + "github.com/ossf/osv-schema/bindings/go/osvschema" +) + +type testCase struct { + string string + parsed ecosystem.Parsed +} + +func buildCases(t *testing.T) []testCase { + t.Helper() + + return []testCase{ + { + string: "crates.io", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemCratesIO, + Suffix: "", + }, + }, + { + string: "npm", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemNPM, + Suffix: "", + }, + }, + { + string: "Debian: ", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemDebian, + Suffix: " ", + }, + }, + { + string: "Debian::", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemDebian, + Suffix: ":", + }, + }, + { + string: "Alpine", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemAlpine, + Suffix: "", + }, + }, + { + string: "Alpine:v", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemAlpine, + Suffix: "v", + }, + }, + { + string: "Alpine:v3.16", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemAlpine, + Suffix: "v3.16", + }, + }, + { + string: "Alpine:3.16", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemAlpine, + Suffix: "3.16", + }, + }, + { + string: "Maven", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemMaven, + Suffix: "", + }, + }, + { + string: "Maven:https://maven.google.com", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemMaven, + Suffix: "https://maven.google.com", + }, + }, + { + string: "Photon OS", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemPhotonOS, + Suffix: "", + }, + }, + { + string: "Photon OS:abc", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemPhotonOS, + Suffix: "abc", + }, + }, + { + string: "Photon OS:3.0", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemPhotonOS, + Suffix: "3.0", + }, + }, + { + string: "Red Hat", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemRedHat, + Suffix: "", + }, + }, + { + string: "Red Hat:abc", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemRedHat, + Suffix: "abc", + }, + }, + { + string: "Red Hat:rhel_aus:8.4::appstream", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemRedHat, + Suffix: "rhel_aus:8.4::appstream", + }, + }, + { + string: "Ubuntu", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemUbuntu, + Suffix: "", + }, + }, + { + string: "Ubuntu:Pro", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemUbuntu, + Suffix: "Pro", + }, + }, + { + string: "Ubuntu:Pro:18.04:LTS", + parsed: ecosystem.Parsed{ + Ecosystem: osvschema.EcosystemUbuntu, + Suffix: "Pro:18.04:LTS", + }, + }, + } +} + +func TestParsed_UnmarshalJSON(t *testing.T) { + t.Parallel() + + tests := buildCases(t) + for _, tt := range tests { + t.Run(tt.string, func(t *testing.T) { + t.Parallel() + + var got ecosystem.Parsed + + if err := json.Unmarshal([]byte(`"`+tt.string+`"`), &got); err != nil { + t.Fatalf("Unmarshal() = %v; want no error", err) + } + + // ensure that the string is unmarshalled into a struct + if !reflect.DeepEqual(got, tt.parsed) { + t.Errorf("Unmarshal() = %v; want %v", got, tt.parsed) + } + }) + } +} + +func TestParsed_UnmarshalJSON_Errors(t *testing.T) { + t.Parallel() + + tests := []struct { + input string + err string + }{ + {"1", "json: cannot unmarshal number into Go value of type string"}, + {"{}", "json: cannot unmarshal object into Go value of type string"}, + {"{\"ecosystem\": \"npm\"}", "json: cannot unmarshal object into Go value of type string"}, + {"{\"ecosystem\": \"npm\", \"suffix\": \"\"}", "json: cannot unmarshal object into Go value of type string"}, + {"{\"Ecosystem\": \"npm\", \"Suffix\": \"\"}", "json: cannot unmarshal object into Go value of type string"}, + {"[]", "json: cannot unmarshal array into Go value of type string"}, + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + t.Parallel() + + var got ecosystem.Parsed + err := json.Unmarshal([]byte(tt.input), &got) + + if err == nil { + t.Fatalf("Unmarshal() = %v; want an error", err) + } + + if err.Error() != tt.err { + t.Fatalf("Unmarshal() = %v; want %v", err.Error(), tt.err) + } + + if got != (ecosystem.Parsed{}) { + t.Fatalf("Unmarshal() = %v; want %v", got, ecosystem.Parsed{}) + } + }) + } +} + +func TestParsed_MarshalJSON(t *testing.T) { + t.Parallel() + + tests := buildCases(t) + for _, tt := range tests { + t.Run(tt.string, func(t *testing.T) { + t.Parallel() + + got, err := json.Marshal(tt.parsed) + + if err != nil { + t.Fatalf("Marshal() = %v; want no error", err) + } + + // ensure that the struct is marshaled as a string + want := `"` + tt.string + `"` + if string(got) != want { + t.Errorf("Marshal() = %v; want %v", string(got), want) + } + }) + } +} + +func TestParsed_String(t *testing.T) { + t.Parallel() + + tests := buildCases(t) + for _, tt := range tests { + t.Run(tt.string, func(t *testing.T) { + t.Parallel() + + if got := tt.parsed.String(); got != tt.string { + t.Errorf("String() = %v, want %v", got, tt.string) + } + }) + } +} + +func TestParse(t *testing.T) { + t.Parallel() + + tests := buildCases(t) + for _, tt := range tests { + t.Run(tt.string, func(t *testing.T) { + t.Parallel() + + if got := ecosystem.MustParse(tt.string); !reflect.DeepEqual(got, tt.parsed) { + t.Errorf("Parse() = %v, want %v", got, tt.parsed) + } + }) + } +} diff --git a/internal/imodels/imodels.go b/internal/imodels/imodels.go new file mode 100644 index 0000000000..2453005a59 --- /dev/null +++ b/internal/imodels/imodels.go @@ -0,0 +1,165 @@ +package imodels + +import ( + "log" + + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem/os/apk" + "github.com/google/osv-scalibr/extractor/filesystem/os/dpkg" + "github.com/google/osv-scalibr/extractor/filesystem/os/rpm" + "github.com/google/osv-scalibr/extractor/filesystem/sbom/cdx" + "github.com/google/osv-scalibr/extractor/filesystem/sbom/spdx" + "github.com/google/osv-scanner/internal/imodels/ecosystem" + "github.com/google/osv-scanner/internal/scalibrextract/vcs/gitrepo" + "github.com/google/osv-scanner/pkg/models" + + scalibrosv "github.com/google/osv-scalibr/extractor/filesystem/osv" +) + +var sbomExtractors = map[string]struct{}{ + spdx.Extractor{}.Name(): {}, + cdx.Extractor{}.Name(): {}, +} + +var gitExtractors = map[string]struct{}{ + gitrepo.Extractor{}.Name(): {}, +} + +var osExtractors = map[string]struct{}{ + dpkg.Extractor{}.Name(): {}, + apk.Extractor{}.Name(): {}, + rpm.Extractor{}.Name(): {}, +} + +// PackageInfo represents a package found during a scan. This is generally +// converted directly from the extractor.Inventory type, with some restructuring +// for easier use within osv-scanner itself. +type PackageInfo struct { + Name string // Name will be SourceName matching the osv-schema + Version string + Ecosystem ecosystem.Parsed + + Location string // Contains Inventory.Locations[0] + SourceType SourceType + + Commit string + Repository string + + // For package sources + DepGroups []string + + // For OS packages + // This is usually the BinaryName, while Name is the SourceName + OSPackageName string + + AdditionalLocations []string // Contains Inventory.Locations[1..] +} + +// FromInventory converts an extractor.Inventory into a PackageInfo. +// +// For ease of use, this function does not return an error, but will log +// warnings when encountering unexpected inventory entries +func FromInventory(inventory *extractor.Inventory) PackageInfo { + pkgInfo := PackageInfo{ + Name: inventory.Name, + Version: inventory.Version, + Location: inventory.Locations[0], + AdditionalLocations: inventory.Locations[1:], + // TODO: SourceType + } + + // Ignore this error for now as we can't do too much about an unknown ecosystem + eco, err := ecosystem.Parse(inventory.Ecosystem()) + if err != nil { + // TODO(v2): Replace with slog + log.Printf("Warning: %s\n", err.Error()) + } + + pkgInfo.Ecosystem = eco + + if inventory.SourceCode != nil { + pkgInfo.Commit = inventory.SourceCode.Commit + pkgInfo.Repository = inventory.SourceCode.Repo + } + + if dg, ok := inventory.Metadata.(scalibrosv.DepGroups); ok { + pkgInfo.DepGroups = dg.DepGroups() + } + if inventory.Extractor != nil { + extractorName := inventory.Extractor.Name() + if _, ok := osExtractors[extractorName]; ok { + pkgInfo.SourceType = SourceTypeOSPackage + } else if _, ok := sbomExtractors[extractorName]; ok { + pkgInfo.SourceType = SourceTypeSBOM + + // TODO (V2): SBOMs have a special case where we manually convert the PURL here + // instead while PURL to ESI conversion is not complete + purl := inventory.Extractor.ToPURL(inventory) + + if purl != nil { + // Error should never happen here since the PURL is from an already parsed purl + pi, _ := models.PURLToPackage(purl.String()) + pkgInfo.Name = pi.Name + pkgInfo.Version = pi.Version + parsed, err := ecosystem.Parse(pi.Ecosystem) + if err != nil { + // TODO: Replace with slog + log.Printf("Warning, found unexpected ecosystem in purl %q, likely will not return any results for this package.\n", purl.String()) + } + pkgInfo.Ecosystem = parsed + } + } else if _, ok := gitExtractors[extractorName]; ok { + pkgInfo.SourceType = SourceTypeGit + } else { + pkgInfo.SourceType = SourceTypeProjectPackage + } + } + + if metadata, ok := inventory.Metadata.(*apk.Metadata); ok { + pkgInfo.OSPackageName = metadata.PackageName + } else if metadata, ok := inventory.Metadata.(*dpkg.Metadata); ok { + pkgInfo.OSPackageName = metadata.PackageName + // Debian uses source name on osv.dev + // (fallback to using the normal name if source name is empty) + if metadata.SourceName != "" { + pkgInfo.Name = metadata.SourceName + } + } else if metadata, ok := inventory.Metadata.(*rpm.Metadata); ok { + pkgInfo.OSPackageName = metadata.PackageName + } + + return pkgInfo +} + +// PackageScanResult represents a package and its associated vulnerabilities and licenses. +// This struct is used to store the results of a scan at a per package level. +type PackageScanResult struct { + PackageInfo PackageInfo + // TODO: Use osvschema.Vulnerability instead + Vulnerabilities []models.Vulnerability + Licenses []models.License + ImageOriginLayerID string + + // TODO(v2): + // SourceAnalysis *SourceAnalysis + // Any additional scan enrichment steps +} + +type ImageMetadata struct { + // TODO: + // OS + // BaseImage + // LayerMetadata []LayerMetadata +} + +// SourceType categorizes packages based on the extractor that extracted +// the "source", for use in the output. +type SourceType int + +const ( + SourceTypeUnknown SourceType = iota + SourceTypeOSPackage + SourceTypeProjectPackage + SourceTypeSBOM + SourceTypeGit +) diff --git a/internal/imodels/results/scanresults.go b/internal/imodels/results/scanresults.go new file mode 100644 index 0000000000..eee2711e34 --- /dev/null +++ b/internal/imodels/results/scanresults.go @@ -0,0 +1,23 @@ +package results + +import ( + "github.com/google/osv-scanner/internal/config" + "github.com/google/osv-scanner/internal/imodels" +) + +// ScanResults represents the complete results of a scan. +// This includes information that affect multiple packages. +type ScanResults struct { + PackageScanResults []imodels.PackageScanResult + + // TODO(v2): Temporarily commented out until ScanParameters is moved + // to a shared package to avoid cyclic dependencies + // The user parameters for the scan + // ScanParameters + + // Scan config + ConfigManager config.Manager + + // For container scanning, metadata including layer information + ImageMetadata *imodels.ImageMetadata +} diff --git a/internal/lockfilescalibr/translation.go b/internal/lockfilescalibr/translation.go deleted file mode 100644 index 5cebcbf6a9..0000000000 --- a/internal/lockfilescalibr/translation.go +++ /dev/null @@ -1,188 +0,0 @@ -package lockfilescalibr - -import ( - "context" - "fmt" - "io/fs" - "os" - "sort" - - "github.com/google/osv-scalibr/extractor" - "github.com/google/osv-scalibr/extractor/filesystem" - "github.com/google/osv-scalibr/extractor/filesystem/language/dart/pubspec" - "github.com/google/osv-scalibr/extractor/filesystem/language/dotnet/packageslockjson" - "github.com/google/osv-scalibr/extractor/filesystem/language/erlang/mixlock" - "github.com/google/osv-scalibr/extractor/filesystem/language/golang/gomod" - "github.com/google/osv-scalibr/extractor/filesystem/language/java/gradlelockfile" - "github.com/google/osv-scalibr/extractor/filesystem/language/java/gradleverificationmetadataxml" - "github.com/google/osv-scalibr/extractor/filesystem/language/java/pomxml" - "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/packagelockjson" - "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/pnpmlock" - "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/yarnlock" - "github.com/google/osv-scalibr/extractor/filesystem/language/php/composerlock" - "github.com/google/osv-scalibr/extractor/filesystem/language/python/pdmlock" - "github.com/google/osv-scalibr/extractor/filesystem/language/python/pipfilelock" - "github.com/google/osv-scalibr/extractor/filesystem/language/python/poetrylock" - "github.com/google/osv-scalibr/extractor/filesystem/language/python/requirements" - "github.com/google/osv-scalibr/extractor/filesystem/language/r/renvlock" - "github.com/google/osv-scalibr/extractor/filesystem/language/ruby/gemfilelock" - "github.com/google/osv-scalibr/extractor/filesystem/language/rust/cargolock" - - scalibrfs "github.com/google/osv-scalibr/fs" -) - -var lockfileExtractors = []filesystem.Extractor{ - // conanlock.Extractor{}, - packageslockjson.Extractor{}, - mixlock.Extractor{}, - pubspec.Extractor{}, - gomod.Extractor{}, - pomxml.Extractor{}, - gradlelockfile.Extractor{}, - gradleverificationmetadataxml.Extractor{}, - packagelockjson.Extractor{}, - pnpmlock.Extractor{}, - yarnlock.Extractor{}, - composerlock.Extractor{}, - pipfilelock.Extractor{}, - pdmlock.Extractor{}, - poetrylock.Extractor{}, - requirements.Extractor{}, - renvlock.Extractor{}, - gemfilelock.Extractor{}, - cargolock.Extractor{}, -} - -var lockfileExtractorMapping = map[string]string{ - "pubspec.lock": "dart/pubspec", - "pnpm-lock.yaml": "javascript/pnpmlock", - "yarn.lock": "javascript/yarnlock", - "package-lock.json": "javascript/packagelockjson", - "pom.xml": "java/pomxml", - "buildscript-gradle.lockfile": "java/gradlelockfile", - "gradle.lockfile": "java/gradlelockfile", - "verification-metadata.xml": "java/gradleverificationmetadataxml", - "poetry.lock": "python/poetrylock", - "Pipfile.lock": "python/Pipfilelock", - "pdm.lock": "python/pdmlock", - "requirements.txt": "python/requirements", - "Cargo.lock": "rust/Cargolock", - "composer.lock": "php/composerlock", - "mix.lock": "erlang/mixlock", - "renv.lock": "r/renvlock", - "packages.lock.json": "dotnet/packageslockjson", - // "conan.lock": "cpp/conanlock", - "go.mod": "go/gomod", - "Gemfile.lock": "ruby/gemfilelock", -} - -// ExtractWithExtractor attempts to extract the file at the given path with the extractor passed in -func ExtractWithExtractor(ctx context.Context, localPath string, ext filesystem.Extractor) ([]*extractor.Inventory, error) { - info, err := os.Stat(localPath) - if err != nil { - return nil, err - } - - return extractWithExtractor(ctx, localPath, info, ext) -} - -// Extract attempts to extract the file at the given path -// -// Args: -// - localPath: the path to the lockfile -// - extractAs: the name of the lockfile format to extract as (Using OSV-Scanner V1 extractor names) -// -// Returns: -// - []*extractor.Inventory: the extracted lockfile data -// - error: any errors encountered during extraction -// -// If extractAs is not specified, then the function will attempt to -// identify the lockfile format based on the file name. -// -// If no extractors are found, then ErrNoExtractorsFound is returned. -func Extract(ctx context.Context, localPath string, extractAs string) ([]*extractor.Inventory, error) { - info, err := os.Stat(localPath) - if err != nil { - return nil, err - } - - if extractAs != "" { - return extractAsSpecific(ctx, extractAs, localPath, info) - } - - output := []*extractor.Inventory{} - extractorFound := false - - for _, ext := range lockfileExtractors { - if ext.FileRequired(localPath, info) { - extractorFound = true - - inv, err := extractWithExtractor(ctx, localPath, info, ext) - if err != nil { - return nil, err - } - - output = append(output, inv...) - } - } - - if !extractorFound { - return nil, ErrNoExtractorsFound - } - - sort.Slice(output, func(i, j int) bool { - if output[i].Name == output[j].Name { - return output[i].Version < output[j].Version - } - - return output[i].Name < output[j].Name - }) - - return output, nil -} - -// Use the extractor specified by extractAs string key -func extractAsSpecific(ctx context.Context, extractAs string, localPath string, info fs.FileInfo) ([]*extractor.Inventory, error) { - for _, ext := range lockfileExtractors { - if lockfileExtractorMapping[extractAs] == ext.Name() { - return extractWithExtractor(ctx, localPath, info, ext) - } - } - - return nil, fmt.Errorf("%w, requested %s", ErrExtractorNotFound, extractAs) -} - -func extractWithExtractor(ctx context.Context, localPath string, info fs.FileInfo, ext filesystem.Extractor) ([]*extractor.Inventory, error) { - si, err := createScanInput(localPath, info) - if err != nil { - return nil, err - } - - inv, err := ext.Extract(ctx, si) - if err != nil { - return nil, fmt.Errorf("(extracting as %s) %w", ext.Name(), err) - } - - for i := range inv { - inv[i].Extractor = ext - } - - return inv, nil -} - -func createScanInput(path string, fileInfo fs.FileInfo) (*filesystem.ScanInput, error) { - reader, err := os.Open(path) - if err != nil { - return nil, err - } - - si := filesystem.ScanInput{ - FS: os.DirFS("/").(scalibrfs.FS), - Path: path, - Root: "/", - Reader: reader, - Info: fileInfo, - } - - return &si, nil -} diff --git a/internal/lockfilescalibr/translation_test.go b/internal/lockfilescalibr/translation_test.go deleted file mode 100644 index 14c5f72e1d..0000000000 --- a/internal/lockfilescalibr/translation_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package lockfilescalibr - -import ( - "testing" -) - -func TestLockfileScalibrMappingExists(t *testing.T) { - t.Parallel() - - for _, target := range lockfileExtractorMapping { - found := false - for _, ext := range lockfileExtractors { - if target == ext.Name() { - found = true - break - } - } - - if !found { - t.Errorf("Extractor %v not found.", target) - } - } -} diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go deleted file mode 100644 index 2fbc09b24b..0000000000 --- a/internal/sbom/cyclonedx.go +++ /dev/null @@ -1,118 +0,0 @@ -package sbom - -import ( - "errors" - "fmt" - "io" - "path/filepath" - "strings" - - "github.com/CycloneDX/cyclonedx-go" -) - -type CycloneDX struct{} - -type cyclonedxType struct { - name string - bomType cyclonedx.BOMFileFormat -} - -var ( - cycloneDXTypes = []cyclonedxType{ - { - name: "json", - bomType: cyclonedx.BOMFileFormatJSON, - }, - { - name: "xml", - bomType: cyclonedx.BOMFileFormatXML, - }, - } -) - -func (c *CycloneDX) Name() string { - return "CycloneDX" -} - -func (c *CycloneDX) MatchesRecognizedFileNames(path string) bool { - // See https://cyclonedx.org/specification/overview/#recognized-file-patterns - expectedGlobs := []string{ - "bom.xml", - "bom.json", - "*.cdx.json", - "*.cdx.xml", - } - filename := filepath.Base(path) - for _, v := range expectedGlobs { - matched, err := filepath.Match(v, filename) - if err != nil { - // Just panic since the only error is invalid glob pattern - panic("Glob pattern is invalid: " + err.Error()) - } - - if matched { - return true - } - } - - return false -} - -func (c *CycloneDX) enumerateComponents(components []cyclonedx.Component, callback func(Identifier) error) error { - for _, component := range components { - if component.PackageURL != "" { - err := callback(Identifier{ - PURL: component.PackageURL, - }) - if err != nil { - return err - } - } - // Components can have components, so enumerate them recursively. - if component.Components != nil { - err := c.enumerateComponents(*component.Components, callback) - if err != nil { - return err - } - } - } - - return nil -} - -func (c *CycloneDX) enumeratePackages(bom *cyclonedx.BOM, callback func(Identifier) error) error { - if bom.Components == nil { - return nil - } - - return c.enumerateComponents(*bom.Components, callback) -} - -func (c *CycloneDX) GetPackages(r io.ReadSeeker, callback func(Identifier) error) error { - //nolint:prealloc // Not sure how many there will be in advance. - var errs []error - var bom cyclonedx.BOM - - for _, formatType := range cycloneDXTypes { - _, err := r.Seek(0, io.SeekStart) - if err != nil { - return fmt.Errorf("failed to seek to start of file: %w", err) - } - decoder := cyclonedx.NewBOMDecoder(r, formatType.bomType) - err = decoder.Decode(&bom) - if err == nil { - if bom.BOMFormat == "CycloneDX" || strings.HasPrefix(bom.XMLNS, "http://cyclonedx.org/schema/bom") { - return c.enumeratePackages(&bom, callback) - } - - err = errors.New("invalid BOMFormat") - } - - errs = append(errs, fmt.Errorf("failed trying %s: %w", formatType.name, err)) - } - - return InvalidFormatError{ - Msg: "failed to parse CycloneDX", - Errs: errs, - } -} diff --git a/internal/sbom/cyclonedx_test.go b/internal/sbom/cyclonedx_test.go deleted file mode 100644 index fe0f34bcd9..0000000000 --- a/internal/sbom/cyclonedx_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package sbom_test - -import ( - "os" - "path/filepath" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/google/osv-scanner/internal/sbom" -) - -func runCycloneGetPackages(t *testing.T, bomFile string, want []sbom.Identifier) { - t.Helper() - - f, err := os.Open(filepath.Join("fixtures", bomFile)) - if err != nil { - t.Fatalf("Failed to read fixture file: %v", err) - } - defer f.Close() - - got := []sbom.Identifier{} - callback := func(id sbom.Identifier) error { - got = append(got, id) - return nil - } - - cdx := &sbom.CycloneDX{} - err = cdx.GetPackages(f, callback) - if err != nil { - t.Errorf("GetPackages returned an error: %v", err) - } - - if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("GetPackages() returned an unexpected result (-want +got):\n%s", diff) - } -} - -func TestCycloneDXGetPackages(t *testing.T) { - t.Parallel() - tests := []struct { - bomFile string - identifiers []sbom.Identifier - }{ - { - bomFile: "cyclonedx.json", - identifiers: []sbom.Identifier{ - {PURL: "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.12"}, - {PURL: "pkg:maven/org.apache.logging.log4j/log4j-core@2.16.0"}, - }, - }, - { - bomFile: "cyclonedx-empty.json", - identifiers: []sbom.Identifier{}, - }, - } - - for _, tt := range tests { - runCycloneGetPackages(t, tt.bomFile, tt.identifiers) - } -} diff --git a/internal/sbom/fixtures/cyclonedx-empty.json b/internal/sbom/fixtures/cyclonedx-empty.json deleted file mode 100644 index 19516067d3..0000000000 --- a/internal/sbom/fixtures/cyclonedx-empty.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "bomFormat": "CycloneDX", - "specVersion": "1.4", - "version": 1 -} diff --git a/internal/sbom/fixtures/cyclonedx.json b/internal/sbom/fixtures/cyclonedx.json deleted file mode 100644 index d9421e66c3..0000000000 --- a/internal/sbom/fixtures/cyclonedx.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "bomFormat": "CycloneDX", - "specVersion": "1.4", - "version": 1, - "components": [ - { - "type": "container", - "name": "/target.tar", - "components": [ - { - "type": "library", - "name": "HdrHistogram", - "purl": "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.12" - } - ] - }, - { - "type": "library", - "name": "Apache Log4j Core", - "purl": "pkg:maven/org.apache.logging.log4j/log4j-core@2.16.0" - } - ] -} diff --git a/internal/sbom/fixtures/spdx-empty.json b/internal/sbom/fixtures/spdx-empty.json deleted file mode 100644 index b91d22857d..0000000000 --- a/internal/sbom/fixtures/spdx-empty.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "spdxVersion": "SPDX-2.2" -} \ No newline at end of file diff --git a/internal/sbom/fixtures/spdx.json b/internal/sbom/fixtures/spdx.json deleted file mode 100644 index af08dbe266..0000000000 --- a/internal/sbom/fixtures/spdx.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "spdxVersion": "SPDX-2.2", - "packages": [ - { - "name": "HdrHistogram", - "externalRefs": [ - { - "referenceType": "purl", - "referenceLocator": "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.12" - } - ] - }, - { - "name": "Apache Log4j Core", - "externalRefs": [ - { - "referenceType": "purl", - "referenceLocator": "pkg:maven/org.apache.logging.log4j/log4j-core@2.16.0" - } - ] - } - ] -} \ No newline at end of file diff --git a/internal/sbom/sbom.go b/internal/sbom/sbom.go deleted file mode 100644 index f6b5b81086..0000000000 --- a/internal/sbom/sbom.go +++ /dev/null @@ -1,41 +0,0 @@ -package sbom - -import ( - "fmt" - "io" - "strings" -) - -// Identifier is the identifier extracted from the SBOM. -type Identifier struct { - PURL string -} - -// Reader is an interface for all SBOM providers. -type Reader interface { - Name() string - // MatchesRecognizedFileNames checks if the file path is a standard recognized file name - MatchesRecognizedFileNames(path string) bool - GetPackages(r io.ReadSeeker, callback func(Identifier) error) error -} - -var ( - Providers = []Reader{ - &SPDX{}, - &CycloneDX{}, - } -) - -type InvalidFormatError struct { - Msg string - Errs []error -} - -func (e InvalidFormatError) Error() string { - errStrings := make([]string, 0, len(e.Errs)) - for _, e := range e.Errs { - errStrings = append(errStrings, "\t"+e.Error()) - } - - return fmt.Sprintf("%s:\n%s", e.Msg, strings.Join(errStrings, "\n")) -} diff --git a/internal/sbom/spdx.go b/internal/sbom/spdx.go deleted file mode 100644 index 53c4a4b2b8..0000000000 --- a/internal/sbom/spdx.go +++ /dev/null @@ -1,87 +0,0 @@ -//nolint:nosnakecase -package sbom - -import ( - "fmt" - "io" - "path/filepath" - "strings" - - spdx_json "github.com/spdx/tools-golang/json" - "github.com/spdx/tools-golang/rdf" - "github.com/spdx/tools-golang/spdx/v2/v2_3" - "github.com/spdx/tools-golang/tagvalue" -) - -type SPDX struct{} -type spdxLoader func(io.Reader) (*v2_3.Document, error) - -type loader struct { - name string - loader spdxLoader -} - -var ( - spdxLoaders = []loader{ - { - name: "json", - loader: spdx_json.Read, - }, - { - name: "rdf", - loader: rdf.Read, - }, - { - name: "tv", - loader: tagvalue.Read, - }, - } -) - -func (s *SPDX) Name() string { - return "SPDX" -} - -func (s *SPDX) MatchesRecognizedFileNames(path string) bool { - // All spdx files should have the .spdx in the filename, even if - // it's not the extension: https://spdx.github.io/spdx-spec/v2.3/conformance/ - return strings.Contains(strings.ToLower(filepath.Base(path)), ".spdx") -} - -func (s *SPDX) enumeratePackages(doc *v2_3.Document, callback func(Identifier) error) error { - for _, p := range doc.Packages { - for _, r := range p.PackageExternalReferences { - if r.RefType == "purl" { - err := callback(Identifier{ - PURL: r.Locator, - }) - if err != nil { - return err - } - } - } - } - - return nil -} - -func (s *SPDX) GetPackages(r io.ReadSeeker, callback func(Identifier) error) error { - //nolint:prealloc // Not sure how many there will be in advance. - var errs []error - for _, loader := range spdxLoaders { - _, err := r.Seek(0, io.SeekStart) - if err != nil { - return fmt.Errorf("failed to seek to start of file: %w", err) - } - doc, err := loader.loader(r) - if err == nil { - return s.enumeratePackages(doc, callback) - } - errs = append(errs, fmt.Errorf("failed trying %s: %w", loader.name, err)) - } - - return InvalidFormatError{ - Msg: "failed to parse SPDX", - Errs: errs, - } -} diff --git a/internal/sbom/spdx_test.go b/internal/sbom/spdx_test.go deleted file mode 100644 index 820eb24eb8..0000000000 --- a/internal/sbom/spdx_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package sbom_test - -import ( - "os" - "path/filepath" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/google/osv-scanner/internal/sbom" -) - -func runSPDXGetPackages(t *testing.T, bomFile string, want []sbom.Identifier) { - t.Helper() - - f, err := os.Open(filepath.Join("fixtures", bomFile)) - if err != nil { - t.Fatalf("Failed to read fixture file: %v", err) - } - defer f.Close() - - got := []sbom.Identifier{} - callback := func(id sbom.Identifier) error { - got = append(got, id) - return nil - } - - spdx := &sbom.SPDX{} - err = spdx.GetPackages(f, callback) - if err != nil { - t.Errorf("GetPackages returned an error: %v", err) - } - - if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("GetPackages() returned an unexpected result (-want +got):\n%s", diff) - } -} - -func TestSPDXGetPackages(t *testing.T) { - t.Parallel() - tests := []struct { - spdxFile string - identifiers []sbom.Identifier - }{ - { - spdxFile: "spdx.json", - identifiers: []sbom.Identifier{ - {PURL: "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.12"}, - {PURL: "pkg:maven/org.apache.logging.log4j/log4j-core@2.16.0"}, - }, - }, - { - spdxFile: "spdx-empty.json", - identifiers: []sbom.Identifier{}, - }, - } - - for _, tt := range tests { - runSPDXGetPackages(t, tt.spdxFile, tt.identifiers) - } -} diff --git a/internal/lockfilescalibr/errors.go b/internal/scalibrextract/errors.go similarity index 72% rename from internal/lockfilescalibr/errors.go rename to internal/scalibrextract/errors.go index 005ee0012b..0a410500cf 100644 --- a/internal/lockfilescalibr/errors.go +++ b/internal/scalibrextract/errors.go @@ -1,9 +1,8 @@ -package lockfilescalibr +package scalibrextract import "errors" var ErrIncompatibleFileFormat = errors.New("file format is incompatible, but this is expected") var ErrNotImplemented = errors.New("not implemented") var ErrWrongExtractor = errors.New("this extractor did not create this inventory") -var ErrExtractorNotFound = errors.New("could not determine extractor") -var ErrNoExtractorsFound = errors.New("no extractors found to be suitable to this file") +var ErrExtractorNotFound = errors.New("could not determine extractor suitable to this file") diff --git a/internal/scalibrextract/extract.go b/internal/scalibrextract/extract.go new file mode 100644 index 0000000000..80b800c8d2 --- /dev/null +++ b/internal/scalibrextract/extract.go @@ -0,0 +1,120 @@ +package scalibrextract + +import ( + "context" + "fmt" + "io/fs" + "os" + "slices" + + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem" + + scalibrfs "github.com/google/osv-scalibr/fs" +) + +// ExtractWithExtractor attempts to extract the file at the given path with the extractor passed in +// +// # Extract attempts to extract the file at the given path +// +// Args: +// - ctx: the context to use for extraction +// - localPath: the path to the lockfile +// - ext: the extractor to use +// +// Returns: +// - []*extractor.Inventory: the extracted lockfile data +// - error: any errors encountered during extraction +func ExtractWithExtractor(ctx context.Context, localPath string, ext filesystem.Extractor) ([]*extractor.Inventory, error) { + info, err := os.Stat(localPath) + if err != nil { + return nil, err + } + + return extractWithExtractor(ctx, localPath, info, ext) +} + +// ExtractWithExtractors attempts to extract the file at the given path +// by choosing the extractor which passes the FileRequired test +// TODO: Optimise to pass in FileInfo here. +// TODO: Remove reporter +// +// Args: +// - ctx: the context to use for extraction +// - localPath: the path to the lockfile +// - extractors: a slice of extractors to try +// - r: reporter to output logs to +// +// Returns: +// - []*extractor.Inventory: the extracted lockfile data +// - error: any errors encountered during extraction +// +// If no extractors are found, then ErrExtractorNotFound is returned. +func ExtractWithExtractors(ctx context.Context, localPath string, extractors []filesystem.Extractor) ([]*extractor.Inventory, error) { + info, err := os.Stat(localPath) + if err != nil { + return nil, err + } + + result := []*extractor.Inventory{} + extractorFound := false + for _, ext := range extractors { + if !ext.FileRequired(localPath, info) { + continue + } + extractorFound = true + + invs, err := extractWithExtractor(ctx, localPath, info, ext) + if err != nil { + return nil, err + } + + result = append(result, invs...) + } + + if !extractorFound { + return nil, ErrExtractorNotFound + } + + return result, nil +} + +func extractWithExtractor(ctx context.Context, localPath string, info fs.FileInfo, ext filesystem.Extractor) ([]*extractor.Inventory, error) { + si, err := createScanInput(localPath, info) + if err != nil { + return nil, err + } + + invs, err := ext.Extract(ctx, si) + if err != nil { + return nil, fmt.Errorf("(extracting as %s) %w", ext.Name(), err) + } + + for i := range invs { + invs[i].Extractor = ext + } + + slices.SortFunc(invs, inventorySort) + invsCompact := slices.CompactFunc(invs, func(a, b *extractor.Inventory) bool { + return inventorySort(a, b) == 0 + }) + + return invsCompact, nil +} + +func createScanInput(path string, fileInfo fs.FileInfo) (*filesystem.ScanInput, error) { + reader, err := os.Open(path) + if err != nil { + return nil, err + } + + si := filesystem.ScanInput{ + FS: os.DirFS("/").(scalibrfs.FS), + Path: path, + Root: "/", + Reader: reader, + Info: fileInfo, + } + + return &si, nil +} diff --git a/internal/scalibrextract/invsort.go b/internal/scalibrextract/invsort.go new file mode 100644 index 0000000000..6db3211972 --- /dev/null +++ b/internal/scalibrextract/invsort.go @@ -0,0 +1,35 @@ +package scalibrextract + +import ( + "cmp" + "fmt" + + "github.com/google/osv-scalibr/extractor" +) + +// InventorySort is a comparator function for Inventories, to be used in +// tests with cmp.Diff to disregard the order in which the Inventories +// are reported. +func inventorySort(a, b *extractor.Inventory) int { + aLoc := fmt.Sprintf("%v", a.Locations) + bLoc := fmt.Sprintf("%v", b.Locations) + + var aExtr, bExtr string + if a.Extractor != nil { + aExtr = a.Extractor.Name() + } + if b.Extractor != nil { + bExtr = b.Extractor.Name() + } + + aSourceCode := fmt.Sprintf("%v", a.SourceCode) + bSourceCode := fmt.Sprintf("%v", b.SourceCode) + + return cmp.Or( + cmp.Compare(aLoc, bLoc), + cmp.Compare(a.Name, b.Name), + cmp.Compare(a.Version, b.Version), + cmp.Compare(aSourceCode, bSourceCode), + cmp.Compare(aExtr, bExtr), + ) +} diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/extractor.go b/internal/scalibrextract/language/java/pomxmlnet/extractor.go similarity index 99% rename from internal/lockfilescalibr/language/java/pomxmlnet/extractor.go rename to internal/scalibrextract/language/java/pomxmlnet/extractor.go index 3a1a5f51c0..1d9bc03295 100644 --- a/internal/lockfilescalibr/language/java/pomxmlnet/extractor.go +++ b/internal/scalibrextract/language/java/pomxmlnet/extractor.go @@ -31,7 +31,7 @@ type Extractor struct { } // Name of the extractor. -func (e Extractor) Name() string { return "osv/pomxmlnet" } +func (e Extractor) Name() string { return "java/pomxml" } // Version of the extractor. func (e Extractor) Version() int { return 0 } diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/extractor_test.go b/internal/scalibrextract/language/java/pomxmlnet/extractor_test.go similarity index 99% rename from internal/lockfilescalibr/language/java/pomxmlnet/extractor_test.go rename to internal/scalibrextract/language/java/pomxmlnet/extractor_test.go index 556663be75..a61710472c 100644 --- a/internal/lockfilescalibr/language/java/pomxmlnet/extractor_test.go +++ b/internal/scalibrextract/language/java/pomxmlnet/extractor_test.go @@ -9,9 +9,9 @@ import ( "github.com/google/osv-scalibr/extractor" "github.com/google/osv-scalibr/extractor/filesystem/osv" "github.com/google/osv-scalibr/testing/extracttest" - "github.com/google/osv-scanner/internal/lockfilescalibr/language/java/pomxmlnet" "github.com/google/osv-scanner/internal/resolution/clienttest" "github.com/google/osv-scanner/internal/resolution/datasource" + "github.com/google/osv-scanner/internal/scalibrextract/language/java/pomxmlnet" "github.com/google/osv-scanner/internal/testutility" ) diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/empty.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/empty.xml similarity index 100% rename from internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/empty.xml rename to internal/scalibrextract/language/java/pomxmlnet/testdata/maven/empty.xml diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/interpolation.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/interpolation.xml similarity index 100% rename from internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/interpolation.xml rename to internal/scalibrextract/language/java/pomxmlnet/testdata/maven/interpolation.xml diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/invalid-syntax.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/invalid-syntax.xml similarity index 100% rename from internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/invalid-syntax.xml rename to internal/scalibrextract/language/java/pomxmlnet/testdata/maven/invalid-syntax.xml diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/not-pom.txt b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/not-pom.txt similarity index 100% rename from internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/not-pom.txt rename to internal/scalibrextract/language/java/pomxmlnet/testdata/maven/not-pom.txt diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/one-package.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/one-package.xml similarity index 100% rename from internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/one-package.xml rename to internal/scalibrextract/language/java/pomxmlnet/testdata/maven/one-package.xml diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/parent/pom.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/parent/pom.xml similarity index 100% rename from internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/parent/pom.xml rename to internal/scalibrextract/language/java/pomxmlnet/testdata/maven/parent/pom.xml diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/transitive.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/transitive.xml similarity index 100% rename from internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/transitive.xml rename to internal/scalibrextract/language/java/pomxmlnet/testdata/maven/transitive.xml diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/two-packages.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/two-packages.xml similarity index 100% rename from internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/two-packages.xml rename to internal/scalibrextract/language/java/pomxmlnet/testdata/maven/two-packages.xml diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/with-dependency-management.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-dependency-management.xml similarity index 100% rename from internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/with-dependency-management.xml rename to internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-dependency-management.xml diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/with-parent.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-parent.xml similarity index 100% rename from internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/with-parent.xml rename to internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-parent.xml diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/with-scope.xml b/internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-scope.xml similarity index 100% rename from internal/lockfilescalibr/language/java/pomxmlnet/testdata/maven/with-scope.xml rename to internal/scalibrextract/language/java/pomxmlnet/testdata/maven/with-scope.xml diff --git a/internal/lockfilescalibr/language/java/pomxmlnet/testdata/universe/basic-universe.yaml b/internal/scalibrextract/language/java/pomxmlnet/testdata/universe/basic-universe.yaml similarity index 100% rename from internal/lockfilescalibr/language/java/pomxmlnet/testdata/universe/basic-universe.yaml rename to internal/scalibrextract/language/java/pomxmlnet/testdata/universe/basic-universe.yaml diff --git a/internal/lockfilescalibr/language/javascript/nodemodules/extractor.go b/internal/scalibrextract/language/javascript/nodemodules/extractor.go similarity index 100% rename from internal/lockfilescalibr/language/javascript/nodemodules/extractor.go rename to internal/scalibrextract/language/javascript/nodemodules/extractor.go diff --git a/internal/lockfilescalibr/language/osv/osvscannerjson/extractor.go b/internal/scalibrextract/language/osv/osvscannerjson/extractor.go similarity index 100% rename from internal/lockfilescalibr/language/osv/osvscannerjson/extractor.go rename to internal/scalibrextract/language/osv/osvscannerjson/extractor.go diff --git a/internal/lockfilescalibr/language/osv/osvscannerjson/extractor_test.go b/internal/scalibrextract/language/osv/osvscannerjson/extractor_test.go similarity index 97% rename from internal/lockfilescalibr/language/osv/osvscannerjson/extractor_test.go rename to internal/scalibrextract/language/osv/osvscannerjson/extractor_test.go index 65289c4d4c..9868ae6f56 100644 --- a/internal/lockfilescalibr/language/osv/osvscannerjson/extractor_test.go +++ b/internal/scalibrextract/language/osv/osvscannerjson/extractor_test.go @@ -8,7 +8,7 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/osv-scalibr/extractor" "github.com/google/osv-scalibr/testing/extracttest" - "github.com/google/osv-scanner/internal/lockfilescalibr/language/osv/osvscannerjson" + "github.com/google/osv-scanner/internal/scalibrextract/language/osv/osvscannerjson" "github.com/google/osv-scanner/pkg/models" ) diff --git a/internal/lockfilescalibr/language/osv/osvscannerjson/metadata.go b/internal/scalibrextract/language/osv/osvscannerjson/metadata.go similarity index 100% rename from internal/lockfilescalibr/language/osv/osvscannerjson/metadata.go rename to internal/scalibrextract/language/osv/osvscannerjson/metadata.go diff --git a/internal/lockfilescalibr/language/osv/osvscannerjson/testdata/empty.json b/internal/scalibrextract/language/osv/osvscannerjson/testdata/empty.json similarity index 100% rename from internal/lockfilescalibr/language/osv/osvscannerjson/testdata/empty.json rename to internal/scalibrextract/language/osv/osvscannerjson/testdata/empty.json diff --git a/internal/lockfilescalibr/language/osv/osvscannerjson/testdata/multiple-packages-with-vulns.json b/internal/scalibrextract/language/osv/osvscannerjson/testdata/multiple-packages-with-vulns.json similarity index 100% rename from internal/lockfilescalibr/language/osv/osvscannerjson/testdata/multiple-packages-with-vulns.json rename to internal/scalibrextract/language/osv/osvscannerjson/testdata/multiple-packages-with-vulns.json diff --git a/internal/lockfilescalibr/language/osv/osvscannerjson/testdata/not-json.txt b/internal/scalibrextract/language/osv/osvscannerjson/testdata/not-json.txt similarity index 100% rename from internal/lockfilescalibr/language/osv/osvscannerjson/testdata/not-json.txt rename to internal/scalibrextract/language/osv/osvscannerjson/testdata/not-json.txt diff --git a/internal/lockfilescalibr/language/osv/osvscannerjson/testdata/one-package-commit.json b/internal/scalibrextract/language/osv/osvscannerjson/testdata/one-package-commit.json similarity index 100% rename from internal/lockfilescalibr/language/osv/osvscannerjson/testdata/one-package-commit.json rename to internal/scalibrextract/language/osv/osvscannerjson/testdata/one-package-commit.json diff --git a/internal/lockfilescalibr/language/osv/osvscannerjson/testdata/one-package.json b/internal/scalibrextract/language/osv/osvscannerjson/testdata/one-package.json similarity index 100% rename from internal/lockfilescalibr/language/osv/osvscannerjson/testdata/one-package.json rename to internal/scalibrextract/language/osv/osvscannerjson/testdata/one-package.json diff --git a/internal/scalibrextract/vcs/gitrepo/extractor.go b/internal/scalibrextract/vcs/gitrepo/extractor.go new file mode 100644 index 0000000000..db2127e34c --- /dev/null +++ b/internal/scalibrextract/vcs/gitrepo/extractor.go @@ -0,0 +1,120 @@ +package gitrepo + +import ( + "context" + "io/fs" + "path" + "path/filepath" + + "github.com/go-git/go-git/v5" + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scalibr/plugin" + "github.com/google/osv-scalibr/purl" +) + +type Extractor struct{} + +var _ filesystem.Extractor = Extractor{} + +func getCommitSHA(repo *git.Repository) (string, error) { + head, err := repo.Head() + if err != nil { + return "", err + } + + return head.Hash().String(), nil +} + +func getSubmodules(repo *git.Repository) (submodules []*git.SubmoduleStatus, err error) { + worktree, err := repo.Worktree() + if err != nil { + return nil, err + } + ss, err := worktree.Submodules() + if err != nil { + return nil, err + } + for _, s := range ss { + status, err := s.Status() + if err != nil { + continue + } + submodules = append(submodules, status) + } + + return submodules, nil +} + +func createCommitQueryInventory(commit string, path string) *extractor.Inventory { + return &extractor.Inventory{ + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: commit, + }, + Locations: []string{path}, + } +} + +// Name of the extractor. +func (e Extractor) Name() string { return "vcs/gitrepo" } + +// Version of the extractor. +func (e Extractor) Version() int { return 0 } + +// Requirements of the extractor. +func (e Extractor) Requirements() *plugin.Capabilities { + return &plugin.Capabilities{} +} + +// FileRequired returns true for .package-lock.json files under node_modules +func (e Extractor) FileRequired(path string, fi fs.FileInfo) bool { + return fi.IsDir() && filepath.Base(path) == ".git" +} + +// Extract extracts packages from yarn.lock files passed through the scan input. +func (e Extractor) Extract(_ context.Context, input *filesystem.ScanInput) ([]*extractor.Inventory, error) { + // Assume this is fully on a real filesystem + // TODO: Make this support virtual filesystems + repo, err := git.PlainOpen(path.Join(input.Root, input.Path)) + if err != nil { + return nil, err + } + + commitSHA, err := getCommitSHA(repo) + if err != nil { + return nil, err + } + + //nolint:prealloc // Not sure how many there will be in advance. + var packages []*extractor.Inventory + packages = append(packages, createCommitQueryInventory(commitSHA, input.Path)) + + submodules, err := getSubmodules(repo) + if err != nil { + return nil, err + } + + for _, s := range submodules { + // r.Infof("Scanning submodule %s at commit %s\n", s.Path, s.Expected.String()) + packages = append(packages, createCommitQueryInventory(s.Expected.String(), path.Join(input.Path, s.Path))) + } + + return packages, nil +} + +// ToPURL converts an inventory created by this extractor into a PURL. +func (e Extractor) ToPURL(_ *extractor.Inventory) *purl.PackageURL { + return nil +} + +// ToCPEs is not applicable as this extractor does not infer CPEs from the Inventory. +func (e Extractor) ToCPEs(_ *extractor.Inventory) []string { + return nil +} + +// Ecosystem returns the OSV ecosystem ('npm') of the software extracted by this extractor. +func (e Extractor) Ecosystem(_ *extractor.Inventory) string { + return "" +} + +var _ filesystem.Extractor = Extractor{} diff --git a/internal/scalibrextract/vcs/gitrepo/extractor_test.go b/internal/scalibrextract/vcs/gitrepo/extractor_test.go new file mode 100644 index 0000000000..42c05a6465 --- /dev/null +++ b/internal/scalibrextract/vcs/gitrepo/extractor_test.go @@ -0,0 +1,74 @@ +package gitrepo_test + +import ( + "context" + "os" + "path" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/testing/extracttest" + "github.com/google/osv-scanner/internal/scalibrextract/vcs/gitrepo" +) + +func TestExtractor_Extract(t *testing.T) { + t.Parallel() + + tests := []extracttest.TestTableEntry{ + { + Name: "Not a git dir", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/example-not-git", + }, + WantErr: extracttest.ContainsErrStr{Str: "repository does not exist"}, + }, + { + Name: "example git", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/example-git", + }, + WantInventory: []*extractor.Inventory{ + { + Locations: []string{"testdata/example-git"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "862ac4bd2703b622e85f29f55a2fd8cd6caf8182", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + t.Parallel() + extr := gitrepo.Extractor{} + + err := os.Rename(path.Join(tt.InputConfig.Path, "git-hidden"), path.Join(tt.InputConfig.Path, ".git")) + if err != nil { + t.Errorf("can't find git-hidden folder") + } + + defer func() { + err = os.Rename(path.Join(tt.InputConfig.Path, ".git"), path.Join(tt.InputConfig.Path, "git-hidden")) + if err != nil { + t.Fatalf("failed to restore .git to original git-hidden: %v", err) + } + }() + + scanInput := extracttest.GenerateScanInputMock(t, tt.InputConfig) + defer extracttest.CloseTestScanInput(t, scanInput) + + got, err := extr.Extract(context.Background(), &scanInput) + + if diff := cmp.Diff(tt.WantErr, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("%s.Extract(%q) error diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) + return + } + + if diff := cmp.Diff(tt.WantInventory, got, cmpopts.SortSlices(extracttest.InventoryCmpLess)); diff != "" { + t.Errorf("%s.Extract(%q) diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) + } + }) + } +} diff --git a/pkg/osvscanner/fixtures/example-git/a.txt b/internal/scalibrextract/vcs/gitrepo/testdata/example-git/a.txt similarity index 100% rename from pkg/osvscanner/fixtures/example-git/a.txt rename to internal/scalibrextract/vcs/gitrepo/testdata/example-git/a.txt diff --git a/pkg/osvscanner/fixtures/example-git/git-hidden/COMMIT_EDITMSG b/internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/COMMIT_EDITMSG similarity index 100% rename from pkg/osvscanner/fixtures/example-git/git-hidden/COMMIT_EDITMSG rename to internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/COMMIT_EDITMSG diff --git a/pkg/osvscanner/fixtures/example-git/git-hidden/HEAD b/internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/HEAD similarity index 100% rename from pkg/osvscanner/fixtures/example-git/git-hidden/HEAD rename to internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/HEAD diff --git a/pkg/osvscanner/fixtures/example-git/git-hidden/config b/internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/config similarity index 100% rename from pkg/osvscanner/fixtures/example-git/git-hidden/config rename to internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/config diff --git a/pkg/osvscanner/fixtures/example-git/git-hidden/description b/internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/description similarity index 100% rename from pkg/osvscanner/fixtures/example-git/git-hidden/description rename to internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/description diff --git a/pkg/osvscanner/fixtures/example-git/git-hidden/index b/internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/index similarity index 100% rename from pkg/osvscanner/fixtures/example-git/git-hidden/index rename to internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/index diff --git a/pkg/osvscanner/fixtures/example-git/git-hidden/info/exclude b/internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/info/exclude similarity index 100% rename from pkg/osvscanner/fixtures/example-git/git-hidden/info/exclude rename to internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/info/exclude diff --git a/pkg/osvscanner/fixtures/example-git/git-hidden/logs/HEAD b/internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/logs/HEAD similarity index 100% rename from pkg/osvscanner/fixtures/example-git/git-hidden/logs/HEAD rename to internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/logs/HEAD diff --git a/pkg/osvscanner/fixtures/example-git/git-hidden/logs/refs/heads/main b/internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/logs/refs/heads/main similarity index 100% rename from pkg/osvscanner/fixtures/example-git/git-hidden/logs/refs/heads/main rename to internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/logs/refs/heads/main diff --git a/pkg/osvscanner/fixtures/example-git/git-hidden/objects/16/b14f5da9e2fcd6f3f38cc9e584cef2f3c90ebe b/internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/objects/16/b14f5da9e2fcd6f3f38cc9e584cef2f3c90ebe similarity index 100% rename from pkg/osvscanner/fixtures/example-git/git-hidden/objects/16/b14f5da9e2fcd6f3f38cc9e584cef2f3c90ebe rename to internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/objects/16/b14f5da9e2fcd6f3f38cc9e584cef2f3c90ebe diff --git a/pkg/osvscanner/fixtures/example-git/git-hidden/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904 b/internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904 similarity index 100% rename from pkg/osvscanner/fixtures/example-git/git-hidden/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904 rename to internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904 diff --git a/pkg/osvscanner/fixtures/example-git/git-hidden/objects/86/2ac4bd2703b622e85f29f55a2fd8cd6caf8182 b/internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/objects/86/2ac4bd2703b622e85f29f55a2fd8cd6caf8182 similarity index 100% rename from pkg/osvscanner/fixtures/example-git/git-hidden/objects/86/2ac4bd2703b622e85f29f55a2fd8cd6caf8182 rename to internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/objects/86/2ac4bd2703b622e85f29f55a2fd8cd6caf8182 diff --git a/pkg/osvscanner/fixtures/example-git/git-hidden/objects/bf/8fbfe5a434c007b640c12d920683cb19a7b2b9 b/internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/objects/bf/8fbfe5a434c007b640c12d920683cb19a7b2b9 similarity index 100% rename from pkg/osvscanner/fixtures/example-git/git-hidden/objects/bf/8fbfe5a434c007b640c12d920683cb19a7b2b9 rename to internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/objects/bf/8fbfe5a434c007b640c12d920683cb19a7b2b9 diff --git a/pkg/osvscanner/fixtures/example-git/git-hidden/refs/heads/main b/internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/refs/heads/main similarity index 100% rename from pkg/osvscanner/fixtures/example-git/git-hidden/refs/heads/main rename to internal/scalibrextract/vcs/gitrepo/testdata/example-git/git-hidden/refs/heads/main diff --git a/internal/scalibrextract/vcs/gitrepo/testdata/example-not-git/a.txt b/internal/scalibrextract/vcs/gitrepo/testdata/example-not-git/a.txt new file mode 100644 index 0000000000..16b14f5da9 --- /dev/null +++ b/internal/scalibrextract/vcs/gitrepo/testdata/example-not-git/a.txt @@ -0,0 +1 @@ +test file diff --git a/internal/scalibrextract/vcs/gitrepo/testdata/example-not-git/git-hidden/b.txt b/internal/scalibrextract/vcs/gitrepo/testdata/example-not-git/git-hidden/b.txt new file mode 100644 index 0000000000..8c95218fe4 --- /dev/null +++ b/internal/scalibrextract/vcs/gitrepo/testdata/example-not-git/git-hidden/b.txt @@ -0,0 +1 @@ +test file 2 diff --git a/pkg/lockfile/types.go b/pkg/lockfile/types.go index 97a803a816..89122f006b 100644 --- a/pkg/lockfile/types.go +++ b/pkg/lockfile/types.go @@ -2,6 +2,7 @@ package lockfile import "github.com/google/osv-scanner/pkg/models" +// TODO(v2): Remove completely // TODO(v2): These fields do not need JSON tags I believe type PackageDetails struct { Name string `json:"name"` diff --git a/pkg/osv/osv.go b/pkg/osv/osv.go index ad5fb25b89..d07286b987 100644 --- a/pkg/osv/osv.go +++ b/pkg/osv/osv.go @@ -10,7 +10,7 @@ import ( "net/http" "time" - "github.com/google/osv-scanner/pkg/lockfile" + "github.com/google/osv-scanner/internal/imodels" "github.com/google/osv-scanner/pkg/models" "golang.org/x/sync/errgroup" @@ -123,26 +123,15 @@ func MakePURLRequest(purl string) *Query { } } -func MakePkgRequest(pkgDetails lockfile.PackageDetails) *Query { - // API has trouble parsing requests with both commit and Package details filled in - if pkgDetails.Ecosystem == "" && pkgDetails.Commit != "" { - return &Query{ - Metadata: models.Metadata{ - RepoURL: pkgDetails.Name, - DepGroups: pkgDetails.DepGroups, - }, - Commit: pkgDetails.Commit, - } - } - +func MakePkgRequest(pkgInfo imodels.PackageInfo) *Query { return &Query{ - Version: pkgDetails.Version, + Version: pkgInfo.Version, Package: Package{ - Name: pkgDetails.Name, - Ecosystem: string(pkgDetails.Ecosystem), + Name: pkgInfo.Name, + Ecosystem: pkgInfo.Ecosystem.String(), }, Metadata: models.Metadata{ - DepGroups: pkgDetails.DepGroups, + DepGroups: pkgInfo.DepGroups, }, } } diff --git a/pkg/osvscanner/__snapshots__/osvscanner_internal_test.snap b/pkg/osvscanner/__snapshots__/filter_internal_test.snap similarity index 100% rename from pkg/osvscanner/__snapshots__/osvscanner_internal_test.snap rename to pkg/osvscanner/__snapshots__/filter_internal_test.snap diff --git a/pkg/osvscanner/filter.go b/pkg/osvscanner/filter.go new file mode 100644 index 0000000000..b9ff05b1b1 --- /dev/null +++ b/pkg/osvscanner/filter.go @@ -0,0 +1,146 @@ +package osvscanner + +import ( + "fmt" + + "github.com/google/osv-scanner/internal/config" + "github.com/google/osv-scanner/internal/imodels" + "github.com/google/osv-scanner/internal/imodels/results" + "github.com/google/osv-scanner/pkg/models" + "github.com/google/osv-scanner/pkg/reporter" +) + +// filterUnscannablePackages removes packages that don't have enough information to be scanned +// e,g, local packages that specified by path +func filterUnscannablePackages(r reporter.Reporter, scanResults *results.ScanResults) { + packageResults := make([]imodels.PackageScanResult, 0, len(scanResults.PackageScanResults)) + for _, psr := range scanResults.PackageScanResults { + p := psr.PackageInfo + + switch { + // If none of the cases match, skip this package since it's not scannable + case !p.Ecosystem.IsEmpty() && p.Name != "" && p.Version != "": + case p.Commit != "": + default: + continue + } + + packageResults = append(packageResults, psr) + } + + if len(packageResults) != len(scanResults.PackageScanResults) { + r.Infof("Filtered %d local/unscannable package/s from the scan.\n", len(scanResults.PackageScanResults)-len(packageResults)) + } + + scanResults.PackageScanResults = packageResults +} + +// filterIgnoredPackages removes ignore scanned packages according to config. Returns filtered scanned packages. +func filterIgnoredPackages(r reporter.Reporter, scanResults *results.ScanResults) { + configManager := &scanResults.ConfigManager + + out := make([]imodels.PackageScanResult, 0, len(scanResults.PackageScanResults)) + for _, psr := range scanResults.PackageScanResults { + p := psr.PackageInfo + configToUse := configManager.Get(r, p.Location) + + if ignore, ignoreLine := configToUse.ShouldIgnorePackage(p); ignore { + pkgString := fmt.Sprintf("%s/%s/%s", p.Ecosystem.String(), p.Name, p.Version) + + reason := ignoreLine.Reason + if reason == "" { + reason = "(no reason given)" + } + r.Infof("Package %s has been filtered out because: %s\n", pkgString, reason) + + continue + } + out = append(out, psr) + } + + if len(out) != len(scanResults.PackageScanResults) { + r.Infof("Filtered %d ignored package/s from the scan.\n", len(scanResults.PackageScanResults)-len(out)) + } + + scanResults.PackageScanResults = out +} + +// Filters results according to config, preserving order. Returns total number of vulnerabilities removed. +func filterResults(r reporter.Reporter, results *models.VulnerabilityResults, configManager *config.Manager, allPackages bool) int { + removedCount := 0 + newResults := []models.PackageSource{} // Want 0 vulnerabilities to show in JSON as an empty list, not null. + for _, pkgSrc := range results.Results { + configToUse := configManager.Get(r, pkgSrc.Source.Path) + var newPackages []models.PackageVulns + for _, pkgVulns := range pkgSrc.Packages { + newVulns := filterPackageVulns(r, pkgVulns, configToUse) + removedCount += len(pkgVulns.Vulnerabilities) - len(newVulns.Vulnerabilities) + if allPackages || len(newVulns.Vulnerabilities) > 0 || len(pkgVulns.LicenseViolations) > 0 { + newPackages = append(newPackages, newVulns) + } + } + // Don't want to include the package source at all if there are no vulns. + if len(newPackages) > 0 { + pkgSrc.Packages = newPackages + newResults = append(newResults, pkgSrc) + } + } + results.Results = newResults + + return removedCount +} + +// Filters package-grouped vulnerabilities according to config, preserving ordering. Returns filtered package vulnerabilities. +func filterPackageVulns(r reporter.Reporter, pkgVulns models.PackageVulns, configToUse config.Config) models.PackageVulns { + ignoredVulns := map[string]struct{}{} + + // Iterate over groups first to remove all aliases of ignored vulnerabilities. + var newGroups []models.GroupInfo + for _, group := range pkgVulns.Groups { + ignore := false + for _, id := range group.Aliases { + var ignoreLine config.IgnoreEntry + if ignore, ignoreLine = configToUse.ShouldIgnore(id); ignore { + for _, id := range group.Aliases { + ignoredVulns[id] = struct{}{} + } + + reason := ignoreLine.Reason + + if reason == "" { + reason = "(no reason given)" + } + + // NB: This only prints the first reason encountered in all the aliases. + switch len(group.Aliases) { + case 1: + r.Infof("%s has been filtered out because: %s\n", ignoreLine.ID, reason) + case 2: + r.Infof("%s and 1 alias have been filtered out because: %s\n", ignoreLine.ID, reason) + default: + r.Infof("%s and %d aliases have been filtered out because: %s\n", ignoreLine.ID, len(group.Aliases)-1, reason) + } + + break + } + } + if !ignore { + newGroups = append(newGroups, group) + } + } + + var newVulns []models.Vulnerability + if len(newGroups) > 0 { // If there are no groups left then there would be no vulnerabilities. + for _, vuln := range pkgVulns.Vulnerabilities { + if _, filtered := ignoredVulns[vuln.ID]; !filtered { + newVulns = append(newVulns, vuln) + } + } + } + + // Passed by value. We don't want to alter the original PackageVulns. + pkgVulns.Groups = newGroups + pkgVulns.Vulnerabilities = newVulns + + return pkgVulns +} diff --git a/pkg/osvscanner/osvscanner_internal_test.go b/pkg/osvscanner/filter_internal_test.go similarity index 53% rename from pkg/osvscanner/osvscanner_internal_test.go rename to pkg/osvscanner/filter_internal_test.go index d2c72c927c..9d705fa830 100644 --- a/pkg/osvscanner/osvscanner_internal_test.go +++ b/pkg/osvscanner/filter_internal_test.go @@ -1,11 +1,9 @@ package osvscanner import ( - "os" "path/filepath" "testing" - "github.com/google/go-cmp/cmp" "github.com/google/osv-scanner/internal/config" "github.com/google/osv-scanner/internal/testutility" "github.com/google/osv-scanner/pkg/models" @@ -58,56 +56,3 @@ func Test_filterResults(t *testing.T) { }) } } - -func Test_scanGit(t *testing.T) { - t.Parallel() - - type args struct { - r reporter.Reporter - repoDir string - } - tests := []struct { - name string - args args - wantErr bool - wantPkg []scannedPackage - }{ - { - name: "Example Git repo", - args: args{ - r: &reporter.VoidReporter{}, - repoDir: "fixtures/example-git", - }, - wantErr: false, - wantPkg: []scannedPackage{ - { - Commit: "862ac4bd2703b622e85f29f55a2fd8cd6caf8182", - Source: models.SourceInfo{ - Path: "fixtures/example-git", - Type: "git", - }, - }, - }, - }, - } - - err := os.Rename("fixtures/example-git/git-hidden", "fixtures/example-git/.git") - if err != nil { - t.Errorf("can't find git-hidden folder") - } - - for _, tt := range tests { - pkg, err := scanGit(tt.args.r, tt.args.repoDir) - if (err != nil) != tt.wantErr { - t.Errorf("scanGit() error = %v, wantErr %v", err, tt.wantErr) - } - if !cmp.Equal(tt.wantPkg, pkg) { - t.Errorf("scanGit() package = %v, wantPackage %v", pkg, tt.wantPkg) - } - } - - err = os.Rename("fixtures/example-git/.git", "fixtures/example-git/git-hidden") - if err != nil { - t.Errorf("can't find .git folder") - } -} diff --git a/pkg/osvscanner/internal/scanners/lockfile.go b/pkg/osvscanner/internal/scanners/lockfile.go new file mode 100644 index 0000000000..4e2bec44fb --- /dev/null +++ b/pkg/osvscanner/internal/scanners/lockfile.go @@ -0,0 +1,164 @@ +package scanners + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scalibr/extractor/filesystem/language/cpp/conanlock" + "github.com/google/osv-scalibr/extractor/filesystem/language/dart/pubspec" + "github.com/google/osv-scalibr/extractor/filesystem/language/dotnet/packageslockjson" + "github.com/google/osv-scalibr/extractor/filesystem/language/erlang/mixlock" + "github.com/google/osv-scalibr/extractor/filesystem/language/golang/gomod" + "github.com/google/osv-scalibr/extractor/filesystem/language/java/gradlelockfile" + "github.com/google/osv-scalibr/extractor/filesystem/language/java/gradleverificationmetadataxml" + "github.com/google/osv-scalibr/extractor/filesystem/language/java/pomxml" + "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/packagelockjson" + "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/pnpmlock" + "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/yarnlock" + "github.com/google/osv-scalibr/extractor/filesystem/language/php/composerlock" + "github.com/google/osv-scalibr/extractor/filesystem/language/python/pdmlock" + "github.com/google/osv-scalibr/extractor/filesystem/language/python/pipfilelock" + "github.com/google/osv-scalibr/extractor/filesystem/language/python/poetrylock" + "github.com/google/osv-scalibr/extractor/filesystem/language/python/requirements" + "github.com/google/osv-scalibr/extractor/filesystem/language/r/renvlock" + "github.com/google/osv-scalibr/extractor/filesystem/language/ruby/gemfilelock" + "github.com/google/osv-scalibr/extractor/filesystem/language/rust/cargolock" + "github.com/google/osv-scalibr/extractor/filesystem/os/apk" + "github.com/google/osv-scalibr/extractor/filesystem/os/dpkg" + "github.com/google/osv-scanner/internal/output" + "github.com/google/osv-scanner/internal/scalibrextract" + "github.com/google/osv-scanner/internal/scalibrextract/language/osv/osvscannerjson" + "github.com/google/osv-scanner/pkg/reporter" +) + +var lockfileExtractors = []filesystem.Extractor{ + conanlock.Extractor{}, + packageslockjson.Extractor{}, + mixlock.Extractor{}, + pubspec.Extractor{}, + gomod.Extractor{}, + gradlelockfile.Extractor{}, + gradleverificationmetadataxml.Extractor{}, + packagelockjson.Extractor{}, + pnpmlock.Extractor{}, + yarnlock.Extractor{}, + composerlock.Extractor{}, + pipfilelock.Extractor{}, + pdmlock.Extractor{}, + poetrylock.Extractor{}, + requirements.Extractor{}, + renvlock.Extractor{}, + gemfilelock.Extractor{}, + cargolock.Extractor{}, +} + +var lockfileExtractorMapping = map[string]string{ + "pubspec.lock": "dart/pubspec", + "pnpm-lock.yaml": "javascript/pnpmlock", + "yarn.lock": "javascript/yarnlock", + "package-lock.json": "javascript/packagelockjson", + // This translation works for both the transitive scanning and non transitive scanning + // As both extractors have the same name + "pom.xml": "java/pomxml", + "buildscript-gradle.lockfile": "java/gradlelockfile", + "gradle.lockfile": "java/gradlelockfile", + "verification-metadata.xml": "java/gradleverificationmetadataxml", + "poetry.lock": "python/poetrylock", + "Pipfile.lock": "python/Pipfilelock", + "pdm.lock": "python/pdmlock", + "requirements.txt": "python/requirements", + "Cargo.lock": "rust/Cargolock", + "composer.lock": "php/composerlock", + "mix.lock": "erlang/mixlock", + "renv.lock": "r/renvlock", + "packages.lock.json": "dotnet/packageslockjson", + "conan.lock": "cpp/conanlock", + "go.mod": "go/gomod", + "Gemfile.lock": "ruby/gemfilelock", +} + +// ScanLockfile will load, identify, and parse the lockfile path passed in, and add the dependencies specified +// within to `query` +// +// TODO(V2 Models): pomExtractor is temporary until V2 Models +func ScanLockfile(r reporter.Reporter, scanArg string, pomExtractor filesystem.Extractor) ([]*extractor.Inventory, error) { + var err error + var inventories []*extractor.Inventory + + parseAs, path := parseLockfilePath(scanArg) + + path, err = filepath.Abs(path) + if err != nil { + r.Errorf("Failed to resolved path %q with error: %s\n", path, err) + return nil, err + } + extractorsToUse := lockfileExtractors + + if pomExtractor != nil { + extractorsToUse = append(extractorsToUse, pomExtractor) + } else { + extractorsToUse = append(extractorsToUse, pomxml.Extractor{}) + } + + // special case for the APK and DPKG parsers because they have a very generic name while + // living at a specific location, so they are not included in the map of parsers + // used by lockfile.Parse to avoid false-positives when scanning projects + switch parseAs { + case "apk-installed": + inventories, err = scalibrextract.ExtractWithExtractor(context.Background(), path, apk.New(apk.DefaultConfig())) + case "dpkg-status": + inventories, err = scalibrextract.ExtractWithExtractor(context.Background(), path, dpkg.New(dpkg.DefaultConfig())) + case "osv-scanner": + inventories, err = scalibrextract.ExtractWithExtractor(context.Background(), path, osvscannerjson.Extractor{}) + case "": // No specific parseAs specified + inventories, err = scalibrextract.ExtractWithExtractors(context.Background(), path, extractorsToUse) + default: // A specific parseAs without a special case is selected + // Find and extract with the extractor of parseAs + if name, ok := lockfileExtractorMapping[parseAs]; ok { + for _, ext := range extractorsToUse { + if name == ext.Name() { + inventories, err = scalibrextract.ExtractWithExtractor(context.Background(), path, ext) + break + } + } + } else { + return nil, fmt.Errorf("could not determine extractor, requested %s", parseAs) + } + } + + if err != nil { + return nil, err + } + + parsedAsComment := "" + + if parseAs != "" { + parsedAsComment = fmt.Sprintf("as a %s ", parseAs) + } + + pkgCount := len(inventories) + + r.Infof( + "Scanned %s file %sand found %d %s\n", + path, + parsedAsComment, + pkgCount, + output.Form(pkgCount, "package", "packages"), + ) + + return inventories, nil +} + +func parseLockfilePath(scanArg string) (string, string) { + if !strings.Contains(scanArg, ":") { + scanArg = ":" + scanArg + } + + splits := strings.SplitN(scanArg, ":", 2) + + return splits[0], splits[1] +} diff --git a/pkg/osvscanner/internal/scanners/lockfile_test.go b/pkg/osvscanner/internal/scanners/lockfile_test.go new file mode 100644 index 0000000000..c80769ecbf --- /dev/null +++ b/pkg/osvscanner/internal/scanners/lockfile_test.go @@ -0,0 +1,24 @@ +package scanners + +import "testing" + +func TestLockfileScalibrMappingExists(t *testing.T) { + t.Parallel() + + // Every lockfileExtractor should have a mapping, + // this might not be true the other way around as some extractors are dynamically set, + // and not present in lockfileExtractors + for _, target := range lockfileExtractors { + found := false + for _, val := range lockfileExtractorMapping { + if target.Name() == val { + found = true + break + } + } + + if !found { + t.Errorf("Extractor %v not found.", target) + } + } +} diff --git a/pkg/osvscanner/internal/scanners/sbom.go b/pkg/osvscanner/internal/scanners/sbom.go new file mode 100644 index 0000000000..8af2b3eb95 --- /dev/null +++ b/pkg/osvscanner/internal/scanners/sbom.go @@ -0,0 +1,45 @@ +package scanners + +import ( + "context" + "path/filepath" + + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scalibr/extractor/filesystem/sbom/cdx" + "github.com/google/osv-scalibr/extractor/filesystem/sbom/spdx" + "github.com/google/osv-scanner/internal/output" + "github.com/google/osv-scanner/internal/scalibrextract" + "github.com/google/osv-scanner/pkg/reporter" +) + +var SBOMExtractors = []filesystem.Extractor{ + spdx.Extractor{}, + cdx.Extractor{}, +} + +func ScanSBOM(r reporter.Reporter, path string) ([]*extractor.Inventory, error) { + path, err := filepath.Abs(path) + if err != nil { + r.Errorf("Failed to resolved path %q with error: %s\n", path, err) + return nil, err + } + + invs, err := scalibrextract.ExtractWithExtractors(context.Background(), path, SBOMExtractors) + if err != nil { + r.Infof("Failed to parse SBOM %q with error: %s\n", path, err) + return nil, err + } + + pkgCount := len(invs) + if pkgCount > 0 { + r.Infof( + "Scanned %s file and found %d %s\n", + path, + pkgCount, + output.Form(pkgCount, "package", "packages"), + ) + } + + return invs, nil +} diff --git a/pkg/osvscanner/internal/scanners/vendored.go b/pkg/osvscanner/internal/scanners/vendored.go new file mode 100644 index 0000000000..66d585ee3e --- /dev/null +++ b/pkg/osvscanner/internal/scanners/vendored.go @@ -0,0 +1,123 @@ +package scanners + +// const ( +// // This value may need to be tweaked, or be provided as a configurable flag. +// determineVersionThreshold = 0.15 +// maxDetermineVersionFiles = 10000 +// ) + +// var ( +// vendoredLibNames = map[string]struct{}{ +// "3rdparty": {}, +// "dep": {}, +// "deps": {}, +// "thirdparty": {}, +// "third-party": {}, +// "third_party": {}, +// "libs": {}, +// "external": {}, +// "externals": {}, +// "vendor": {}, +// "vendored": {}, +// } +// ) + +// TODO(V2): Migrate to extractor + use determineversions client + +// func ScanDirWithVendoredLibs(r reporter.Reporter, path string) ([]imodels.ScannedPackage, error) { +// r.Infof("Scanning directory for vendored libs: %s\n", path) +// entries, err := os.ReadDir(path) +// if err != nil { +// return nil, err +// } + +// var packages []imodels.ScannedPackage +// for _, entry := range entries { +// if !entry.IsDir() { +// continue +// } + +// libPath := filepath.Join(path, entry.Name()) + +// r.Infof("Scanning potential vendored dir: %s\n", libPath) +// // TODO: make this a goroutine to parallelise this operation +// results, err := queryDetermineVersions(libPath) +// if err != nil { +// r.Infof("Error scanning sub-directory '%s' with error: %v", libPath, err) +// continue +// } + +// if len(results.Matches) > 0 && results.Matches[0].Score > determineVersionThreshold { +// match := results.Matches[0] +// r.Infof("Identified %s as %s at %s.\n", libPath, match.RepoInfo.Address, match.RepoInfo.Commit) +// packages = append(packages, createCommitQueryPackage(match.RepoInfo.Commit, libPath)) +// } +// } + +// return packages, nil +// } + +// func createCommitQueryPackage(commit string, source string) imodels.ScannedPackage { +// return imodels.ScannedPackage{ +// Commit: commit, +// Source: models.SourceInfo{ +// Path: source, +// Type: "git", +// }, +// } +// } + +// func queryDetermineVersions(repoDir string) (*osv.DetermineVersionResponse, error) { +// fileExts := []string{ +// ".hpp", +// ".h", +// ".hh", +// ".cc", +// ".c", +// ".cpp", +// } + +// var hashes []osv.DetermineVersionHash +// if err := filepath.Walk(repoDir, func(p string, info fs.FileInfo, _ error) error { +// if info.IsDir() { +// if _, err := os.Stat(filepath.Join(p, ".git")); err == nil { +// // Found a git repo, stop here as otherwise we may get duplicated +// // results with our regular git commit scanning. +// return filepath.SkipDir +// } +// if _, ok := vendoredLibNames[strings.ToLower(info.Name())]; ok { +// // Ignore nested vendored libraries, as they can cause bad matches. +// return filepath.SkipDir +// } + +// return nil +// } +// for _, ext := range fileExts { +// if filepath.Ext(p) == ext { +// buf, err := os.ReadFile(p) +// if err != nil { +// return err +// } +// hash := md5.Sum(buf) //nolint:gosec +// hashes = append(hashes, osv.DetermineVersionHash{ +// Path: strings.ReplaceAll(p, repoDir, ""), +// Hash: hash[:], +// }) +// if len(hashes) > maxDetermineVersionFiles { +// return errors.New("too many files to hash") +// } +// } +// } + +// return nil +// }); err != nil { +// return nil, fmt.Errorf("failed during hashing: %w", err) +// } + +// result, err := osv.MakeDetermineVersionRequest(filepath.Base(repoDir), hashes) +// if err != nil { +// return nil, fmt.Errorf("failed to determine versions: %w", err) +// } + +// return result, nil +// } diff --git a/pkg/osvscanner/internal/scanners/walker.go b/pkg/osvscanner/internal/scanners/walker.go new file mode 100644 index 0000000000..fc77613f1a --- /dev/null +++ b/pkg/osvscanner/internal/scanners/walker.go @@ -0,0 +1,163 @@ +package scanners + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + + "github.com/go-git/go-git/v5/plumbing/format/gitignore" + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scalibr/extractor/filesystem/language/java/pomxml" + "github.com/google/osv-scanner/internal/customgitignore" + "github.com/google/osv-scanner/internal/output" + "github.com/google/osv-scanner/internal/scalibrextract" + "github.com/google/osv-scanner/internal/scalibrextract/vcs/gitrepo" + "github.com/google/osv-scanner/pkg/reporter" +) + +// ScanDir walks through the given directory to try to find any relevant files +// These include: +// - Any lockfiles with scanLockfile +// - Any SBOM files with scanSBOMFile +// - Any git repositories with scanGit +// +// TODO(V2 Models): pomExtractor is temporary until V2 Models +func ScanDir(r reporter.Reporter, dir string, skipGit bool, recursive bool, useGitIgnore bool, pomExtractor filesystem.Extractor) ([]*extractor.Inventory, error) { + var ignoreMatcher *gitIgnoreMatcher + if useGitIgnore { + var err error + ignoreMatcher, err = parseGitIgnores(dir, recursive) + if err != nil { + r.Errorf("Unable to parse git ignores: %v\n", err) + useGitIgnore = false + } + } + + root := true + + // Setup scan config + relevantExtractors := []filesystem.Extractor{} + if !skipGit { + relevantExtractors = append(relevantExtractors, gitrepo.Extractor{}) + } + relevantExtractors = append(relevantExtractors, lockfileExtractors...) + relevantExtractors = append(relevantExtractors, SBOMExtractors...) + if pomExtractor != nil { + relevantExtractors = append(relevantExtractors, pomExtractor) + } else { + // Use the offline pomxml extractor if networking is unavailable + relevantExtractors = append(relevantExtractors, pomxml.Extractor{}) + } + + var scannedInventories []*extractor.Inventory + + err := filepath.WalkDir(dir, func(path string, info os.DirEntry, err error) error { + if err != nil { + r.Infof("Failed to walk %s: %v\n", path, err) + return err + } + + path, err = filepath.Abs(path) + if err != nil { + r.Errorf("Failed to walk path %s\n", err) + return err + } + + if useGitIgnore { + match, err := ignoreMatcher.match(path, info.IsDir()) + if err != nil { + r.Infof("Failed to resolve gitignore for %s: %v\n", path, err) + // Don't skip if we can't parse now - potentially noisy for directories with lots of items + } else if match { + if root { // Don't silently skip if the argument file was ignored. + r.Errorf("%s was not scanned because it is excluded by a .gitignore file. Use --no-ignore to scan it.\n", path) + } + if info.IsDir() { + return filepath.SkipDir + } + + return nil + } + } + + // -------- Perform scanning -------- + inventories, err := scalibrextract.ExtractWithExtractors(context.Background(), path, relevantExtractors) + if err != nil && !errors.Is(err, scalibrextract.ErrExtractorNotFound) { + r.Errorf("Error during extraction: %s\n", err) + } + + pkgCount := len(inventories) + if pkgCount > 0 { + // TODO(v2): Display the name of the extractor used here + r.Infof( + "Scanned %s file and found %d %s\n", + path, + pkgCount, + output.Form(pkgCount, "package", "packages"), + ) + } + + scannedInventories = append(scannedInventories, inventories...) + + // Optimisation to skip git repository .git dirs + if info.IsDir() && info.Name() == ".git" { + // Always skip git repository directories + return filepath.SkipDir + } + + // TODO(V2): Reenable vendored libs scanning + // if info.IsDir() && !compareOffline { + // if _, ok := vendoredLibNames[strings.ToLower(filepath.Base(path))]; ok { + // pkgs, err := ScanDirWithVendoredLibs(r, path) + // if err != nil { + // r.Infof("scan failed for dir containing vendored libs %s: %v\n", path, err) + // } + // scannedPackages = append(scannedPackages, pkgs...) + // } + // } + + if !root && !recursive && info.IsDir() { + return filepath.SkipDir + } + root = false + + return nil + }) + + return scannedInventories, err +} + +type gitIgnoreMatcher struct { + matcher gitignore.Matcher + repoPath string +} + +func parseGitIgnores(path string, recursive bool) (*gitIgnoreMatcher, error) { + patterns, repoRootPath, err := customgitignore.ParseGitIgnores(path, recursive) + if err != nil { + return nil, err + } + + matcher := gitignore.NewMatcher(patterns) + + return &gitIgnoreMatcher{matcher: matcher, repoPath: repoRootPath}, nil +} + +// gitIgnoreMatcher.match will return true if the file/directory matches a gitignore entry +// i.e. true if it should be ignored +func (m *gitIgnoreMatcher) match(absPath string, isDir bool) (bool, error) { + pathInGit, err := filepath.Rel(m.repoPath, absPath) + if err != nil { + return false, err + } + // must prepend "." to paths because of how gitignore.ReadPatterns interprets paths + pathInGitSep := []string{"."} + if pathInGit != "." { // don't make the path "./." + pathInGitSep = append(pathInGitSep, strings.Split(pathInGit, string(filepath.Separator))...) + } + + return m.matcher.Match(pathInGitSep, isDir), nil +} diff --git a/pkg/osvscanner/osvscanner.go b/pkg/osvscanner/osvscanner.go index f94a5e6844..811e63272e 100644 --- a/pkg/osvscanner/osvscanner.go +++ b/pkg/osvscanner/osvscanner.go @@ -1,47 +1,23 @@ package osvscanner import ( - "bufio" - "cmp" - "context" - "crypto/md5" //nolint:gosec "errors" "fmt" - "io/fs" - "os" - "os/exec" - "path" - "path/filepath" - "slices" - "strings" - - "github.com/google/osv-scalibr/extractor" - "github.com/google/osv-scalibr/extractor/filesystem/os/apk" - "github.com/google/osv-scalibr/extractor/filesystem/os/dpkg" - scalibrosv "github.com/google/osv-scalibr/extractor/filesystem/osv" "github.com/google/osv-scanner/internal/config" - "github.com/google/osv-scanner/internal/customgitignore" "github.com/google/osv-scanner/internal/depsdev" - "github.com/google/osv-scanner/internal/image" + "github.com/google/osv-scanner/internal/imodels" + "github.com/google/osv-scanner/internal/imodels/results" "github.com/google/osv-scanner/internal/local" - "github.com/google/osv-scanner/internal/lockfilescalibr" - "github.com/google/osv-scanner/internal/lockfilescalibr/language/java/pomxmlnet" - "github.com/google/osv-scanner/internal/lockfilescalibr/language/osv/osvscannerjson" "github.com/google/osv-scanner/internal/output" - "github.com/google/osv-scanner/internal/resolution/client" - "github.com/google/osv-scanner/internal/resolution/datasource" - "github.com/google/osv-scanner/internal/sbom" "github.com/google/osv-scanner/internal/semantic" "github.com/google/osv-scanner/internal/version" - "github.com/google/osv-scanner/pkg/lockfile" "github.com/google/osv-scanner/pkg/models" "github.com/google/osv-scanner/pkg/osv" "github.com/google/osv-scanner/pkg/reporter" + "github.com/ossf/osv-schema/bindings/go/osvschema" depsdevpb "deps.dev/api/v3" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing/format/gitignore" ) type ScannerActions struct { @@ -96,755 +72,13 @@ var OnlyUncalledVulnerabilitiesFoundErr = errors.New("only uncalled vulnerabilit // ErrAPIFailed describes errors related to querying API endpoints. var ErrAPIFailed = errors.New("API query failed") -var ( - vendoredLibNames = map[string]struct{}{ - "3rdparty": {}, - "dep": {}, - "deps": {}, - "thirdparty": {}, - "third-party": {}, - "third_party": {}, - "libs": {}, - "external": {}, - "externals": {}, - "vendor": {}, - "vendored": {}, - } -) - -const ( - // This value may need to be tweaked, or be provided as a configurable flag. - determineVersionThreshold = 0.15 - maxDetermineVersionFiles = 10000 -) - -// scanDir walks through the given directory to try to find any relevant files -// These include: -// - Any lockfiles with scanLockfile -// - Any SBOM files with scanSBOMFile -// - Any git repositories with scanGit -func scanDir(r reporter.Reporter, dir string, skipGit bool, recursive bool, useGitIgnore bool, compareOffline bool, transitiveAct TransitiveScanningActions) ([]scannedPackage, error) { - var ignoreMatcher *gitIgnoreMatcher - if useGitIgnore { - var err error - ignoreMatcher, err = parseGitIgnores(dir, recursive) - if err != nil { - r.Errorf("Unable to parse git ignores: %v\n", err) - useGitIgnore = false - } - } - - root := true - - var scannedPackages []scannedPackage - - return scannedPackages, filepath.WalkDir(dir, func(path string, info os.DirEntry, err error) error { - if err != nil { - r.Infof("Failed to walk %s: %v\n", path, err) - return err - } - - path, err = filepath.Abs(path) - if err != nil { - r.Errorf("Failed to walk path %s\n", err) - return err - } - - if useGitIgnore { - match, err := ignoreMatcher.match(path, info.IsDir()) - if err != nil { - r.Infof("Failed to resolve gitignore for %s: %v\n", path, err) - // Don't skip if we can't parse now - potentially noisy for directories with lots of items - } else if match { - if root { // Don't silently skip if the argument file was ignored. - r.Errorf("%s was not scanned because it is excluded by a .gitignore file. Use --no-ignore to scan it.\n", path) - } - if info.IsDir() { - return filepath.SkipDir - } - - return nil - } - } - - if !skipGit && info.IsDir() && info.Name() == ".git" { - pkgs, err := scanGit(r, filepath.Dir(path)+"/") - if err != nil { - r.Infof("scan failed for git repository, %s: %v\n", path, err) - // Not fatal, so don't return and continue scanning other files - } - scannedPackages = append(scannedPackages, pkgs...) - - return filepath.SkipDir - } - - if !info.IsDir() { - pkgs, err := scanLockfile(r, path, "", transitiveAct) - if err != nil { - // If no extractors found then just continue - if !errors.Is(err, lockfilescalibr.ErrNoExtractorsFound) { - r.Errorf("Attempted to scan lockfile but failed: %s\n", path) - } - } - scannedPackages = append(scannedPackages, pkgs...) - - // No need to check for error - // If scan fails, it means it isn't a valid SBOM file, - // so just move onto the next file - pkgs, _ = scanSBOMFile(r, path, true) - scannedPackages = append(scannedPackages, pkgs...) - } - - if info.IsDir() && !compareOffline { - if _, ok := vendoredLibNames[strings.ToLower(filepath.Base(path))]; ok { - pkgs, err := scanDirWithVendoredLibs(r, path) - if err != nil { - r.Infof("scan failed for dir containing vendored libs %s: %v\n", path, err) - } - scannedPackages = append(scannedPackages, pkgs...) - } - } - - if !root && !recursive && info.IsDir() { - return filepath.SkipDir - } - root = false - - return nil - }) -} - -type gitIgnoreMatcher struct { - matcher gitignore.Matcher - repoPath string -} - -func parseGitIgnores(path string, recursive bool) (*gitIgnoreMatcher, error) { - patterns, repoRootPath, err := customgitignore.ParseGitIgnores(path, recursive) - if err != nil { - return nil, err - } - - matcher := gitignore.NewMatcher(patterns) - - return &gitIgnoreMatcher{matcher: matcher, repoPath: repoRootPath}, nil -} - -func queryDetermineVersions(repoDir string) (*osv.DetermineVersionResponse, error) { - fileExts := []string{ - ".hpp", - ".h", - ".hh", - ".cc", - ".c", - ".cpp", - } - - var hashes []osv.DetermineVersionHash - if err := filepath.Walk(repoDir, func(p string, info fs.FileInfo, _ error) error { - if info.IsDir() { - if _, err := os.Stat(filepath.Join(p, ".git")); err == nil { - // Found a git repo, stop here as otherwise we may get duplicated - // results with our regular git commit scanning. - return filepath.SkipDir - } - if _, ok := vendoredLibNames[strings.ToLower(info.Name())]; ok { - // Ignore nested vendored libraries, as they can cause bad matches. - return filepath.SkipDir - } - - return nil - } - for _, ext := range fileExts { - if filepath.Ext(p) == ext { - buf, err := os.ReadFile(p) - if err != nil { - return err - } - hash := md5.Sum(buf) //nolint:gosec - hashes = append(hashes, osv.DetermineVersionHash{ - Path: strings.ReplaceAll(p, repoDir, ""), - Hash: hash[:], - }) - if len(hashes) > maxDetermineVersionFiles { - return errors.New("too many files to hash") - } - } - } - - return nil - }); err != nil { - return nil, fmt.Errorf("failed during hashing: %w", err) - } - - result, err := osv.MakeDetermineVersionRequest(filepath.Base(repoDir), hashes) - if err != nil { - return nil, fmt.Errorf("failed to determine versions: %w", err) - } - - return result, nil -} - -func scanDirWithVendoredLibs(r reporter.Reporter, path string) ([]scannedPackage, error) { - r.Infof("Scanning directory for vendored libs: %s\n", path) - entries, err := os.ReadDir(path) - if err != nil { - return nil, err - } - - var packages []scannedPackage - for _, entry := range entries { - if !entry.IsDir() { - continue - } - - libPath := filepath.Join(path, entry.Name()) - - r.Infof("Scanning potential vendored dir: %s\n", libPath) - // TODO: make this a goroutine to parallelise this operation - results, err := queryDetermineVersions(libPath) - if err != nil { - r.Infof("Error scanning sub-directory '%s' with error: %v", libPath, err) - continue - } - - if len(results.Matches) > 0 && results.Matches[0].Score > determineVersionThreshold { - match := results.Matches[0] - r.Infof("Identified %s as %s at %s.\n", libPath, match.RepoInfo.Address, match.RepoInfo.Commit) - packages = append(packages, createCommitQueryPackage(match.RepoInfo.Commit, libPath)) - } - } - - return packages, nil -} - -// gitIgnoreMatcher.match will return true if the file/directory matches a gitignore entry -// i.e. true if it should be ignored -func (m *gitIgnoreMatcher) match(absPath string, isDir bool) (bool, error) { - pathInGit, err := filepath.Rel(m.repoPath, absPath) - if err != nil { - return false, err - } - // must prepend "." to paths because of how gitignore.ReadPatterns interprets paths - pathInGitSep := []string{"."} - if pathInGit != "." { // don't make the path "./." - pathInGitSep = append(pathInGitSep, strings.Split(pathInGit, string(filepath.Separator))...) - } - - return m.matcher.Match(pathInGitSep, isDir), nil -} - -func scanImage(r reporter.Reporter, path string) ([]scannedPackage, error) { - scanResults, err := image.ScanImage(r, path) - if err != nil { - return []scannedPackage{}, err - } - - packages := make([]scannedPackage, 0) - - for _, l := range scanResults.Lockfiles { - for _, pkgDetail := range l.Packages { - packages = append(packages, scannedPackage{ - Name: pkgDetail.Name, - Version: pkgDetail.Version, - Commit: pkgDetail.Commit, - Ecosystem: pkgDetail.Ecosystem, - DepGroups: pkgDetail.DepGroups, - ImageOrigin: pkgDetail.ImageOrigin, - Source: models.SourceInfo{ - Path: path + ":" + l.FilePath, - Type: "docker", - }, - }) - } - } - - return packages, nil -} - -// scanLockfile will load, identify, and parse the lockfile path passed in, and add the dependencies specified -// within to `query` -func scanLockfile(r reporter.Reporter, path string, parseAs string, transitiveAct TransitiveScanningActions) ([]scannedPackage, error) { - var err error - - var inventories []*extractor.Inventory - - // special case for the APK and DPKG parsers because they have a very generic name while - // living at a specific location, so they are not included in the map of parsers - // used by lockfile.Parse to avoid false-positives when scanning projects - switch parseAs { - case "apk-installed": - inventories, err = lockfilescalibr.ExtractWithExtractor(context.Background(), path, apk.New(apk.DefaultConfig())) - case "dpkg-status": - inventories, err = lockfilescalibr.ExtractWithExtractor(context.Background(), path, dpkg.New(dpkg.DefaultConfig())) - case "osv-scanner": - inventories, err = lockfilescalibr.ExtractWithExtractor(context.Background(), path, osvscannerjson.Extractor{}) - default: - if !transitiveAct.Disabled && (parseAs == "pom.xml" || filepath.Base(path) == "pom.xml") { - ext, extErr := createMavenExtractor(transitiveAct) - if extErr != nil { - return nil, extErr - } - - inventories, err = lockfilescalibr.ExtractWithExtractor(context.Background(), path, ext) - } else { - inventories, err = lockfilescalibr.Extract(context.Background(), path, parseAs) - } - } - - if err != nil { - return nil, err - } - - parsedAsComment := "" - - if parseAs != "" { - parsedAsComment = fmt.Sprintf("as a %s ", parseAs) - } - - slices.SortFunc(inventories, func(i, j *extractor.Inventory) int { - return cmp.Or( - strings.Compare(i.Name, j.Name), - strings.Compare(i.Version, j.Version), - ) - }) - - pkgCount := len(inventories) - - r.Infof( - "Scanned %s file %sand found %d %s\n", - path, - parsedAsComment, - pkgCount, - output.Form(pkgCount, "package", "packages"), - ) - - packages := make([]scannedPackage, 0, pkgCount) - - for _, inv := range inventories { - scannedPackage := scannedPackage{ - Name: inv.Name, - Version: inv.Version, - Source: models.SourceInfo{ - Path: path, - Type: "lockfile", - }, - } - if inv.SourceCode != nil { - scannedPackage.Commit = inv.SourceCode.Commit - } - eco := inv.Ecosystem() - // TODO(rexpan): Refactor these minor patches to individual items - // TODO: Ecosystem should be pared with Enum : Suffix - if eco == "Alpine" { - eco = "Alpine:v3.20" - } - - scannedPackage.Ecosystem = lockfile.Ecosystem(eco) - - if dg, ok := inv.Metadata.(scalibrosv.DepGroups); ok { - scannedPackage.DepGroups = dg.DepGroups() - } - - packages = append(packages, scannedPackage) - } - - return packages, nil -} - -func createMavenExtractor(actions TransitiveScanningActions) (*pomxmlnet.Extractor, error) { - var depClient client.DependencyClient - var err error - if actions.NativeDataSource { - depClient, err = client.NewMavenRegistryClient(actions.MavenRegistry) - } else { - depClient, err = client.NewDepsDevClient(depsdev.DepsdevAPI) - } - if err != nil { - return nil, err - } - - mavenClient, err := datasource.NewMavenRegistryAPIClient(actions.MavenRegistry) - if err != nil { - return nil, err - } - - extractor := pomxmlnet.Extractor{ - DependencyClient: depClient, - MavenRegistryAPIClient: mavenClient, - } - - return &extractor, nil -} - -// scanSBOMFile will load, identify, and parse the SBOM path passed in, and add the dependencies specified -// within to `query` -func scanSBOMFile(r reporter.Reporter, path string, fromFSScan bool) ([]scannedPackage, error) { - var errs []error - packages := map[string]scannedPackage{} - for _, provider := range sbom.Providers { - if fromFSScan && !provider.MatchesRecognizedFileNames(path) { - // Skip if filename is not usually a sbom file of this format. - // Only do this if this is being done in a filesystem scanning context, where we need to be - // careful about spending too much time attempting to parse unrelated files. - // If this is coming from an explicit scan argument, be more relaxed here since it's common for - // filenames to not conform to expected filename standards. - continue - } - - // Opening file inside loop is OK, since providers is not very long, - // and it is unlikely that multiple providers accept the same file name - file, err := os.Open(path) - if err != nil { - return nil, err - } - defer file.Close() - - var ignoredPURLs []string - err = provider.GetPackages(file, func(id sbom.Identifier) error { - _, err := models.PURLToPackage(id.PURL) - if err != nil { - ignoredPURLs = append(ignoredPURLs, id.PURL) - //nolint:nilerr - return nil - } - - if _, ok := packages[id.PURL]; ok { - r.Warnf("Warning, duplicate PURL found in SBOM: %s\n", id.PURL) - } - - packages[id.PURL] = scannedPackage{ - PURL: id.PURL, - Source: models.SourceInfo{ - Path: path, - Type: "sbom", - }, - } - - return nil - }) - if err == nil { - // Found a parsable format. - if len(packages) == 0 { - // But no entries found, so maybe not the correct format - errs = append(errs, sbom.InvalidFormatError{ - Msg: "no Package URLs found", - Errs: []error{ - fmt.Errorf("scanned %s as %s SBOM, but failed to find any package URLs, this is required to scan SBOMs", path, provider.Name()), - }, - }) - - continue - } - r.Infof( - "Scanned %s as %s SBOM and found %d %s\n", - path, - provider.Name(), - len(packages), - output.Form(len(packages), "package", "packages"), - ) - if len(ignoredPURLs) > 0 { - r.Warnf( - "Ignored %d %s with invalid PURLs\n", - len(ignoredPURLs), - output.Form(len(ignoredPURLs), "package", "packages"), - ) - slices.Sort(ignoredPURLs) - for _, purl := range slices.Compact(ignoredPURLs) { - r.Warnf( - "Ignored invalid PURL \"%s\"\n", - purl, - ) - } - } - - sliceOfPackages := make([]scannedPackage, 0, len(packages)) - - for _, pkg := range packages { - sliceOfPackages = append(sliceOfPackages, pkg) - } - - slices.SortFunc(sliceOfPackages, func(i, j scannedPackage) int { - return strings.Compare(i.PURL, j.PURL) - }) - - return sliceOfPackages, nil - } - - var formatErr sbom.InvalidFormatError - if errors.As(err, &formatErr) { - errs = append(errs, err) - continue - } - - return nil, err - } - - // Don't log these errors if we're coming from an FS scan, since it can get very noisy. - if !fromFSScan { - r.Infof("Failed to parse SBOM using all supported formats:\n") - for _, err := range errs { - r.Infof("%s\n", err.Error()) - } - } - - return nil, nil -} - -func getCommitSHA(repoDir string) (string, error) { - repo, err := git.PlainOpen(repoDir) - if err != nil { - return "", err - } - head, err := repo.Head() - if err != nil { - return "", err - } - - return head.Hash().String(), nil -} - -func getSubmodules(repoDir string) (submodules []*git.SubmoduleStatus, err error) { - repo, err := git.PlainOpen(repoDir) - if err != nil { - return nil, err - } - worktree, err := repo.Worktree() - if err != nil { - return nil, err - } - ss, err := worktree.Submodules() - if err != nil { - return nil, err - } - for _, s := range ss { - status, err := s.Status() - if err != nil { - continue - } - submodules = append(submodules, status) - } - - return submodules, nil -} - -// Scan git repository. Expects repoDir to end with / -func scanGit(r reporter.Reporter, repoDir string) ([]scannedPackage, error) { - commit, err := getCommitSHA(repoDir) - if err != nil { - return nil, err - } - r.Infof("Scanning %s at commit %s\n", repoDir, commit) - - //nolint:prealloc // Not sure how many there will be in advance. - var packages []scannedPackage - packages = append(packages, createCommitQueryPackage(commit, repoDir)) - - submodules, err := getSubmodules(repoDir) - if err != nil { - return nil, err - } - - for _, s := range submodules { - r.Infof("Scanning submodule %s at commit %s\n", s.Path, s.Expected.String()) - packages = append(packages, createCommitQueryPackage(s.Expected.String(), path.Join(repoDir, s.Path))) - } - - return packages, nil -} - -func createCommitQueryPackage(commit string, source string) scannedPackage { - return scannedPackage{ - Commit: commit, - Source: models.SourceInfo{ - Path: source, - Type: "git", - }, - } -} - -func runCommandLogError(r reporter.Reporter, name string, args ...string) error { - cmd := exec.Command(name, args...) - - // Get stderr for debugging when docker fails - stderr, err := cmd.StderrPipe() - if err != nil { - r.Errorf("Failed to get stderr: %s\n", err) - return err - } - - err = cmd.Start() - if err != nil { - r.Errorf("Failed to run docker command (%q): %s\n", cmd.String(), err) - return err - } - // This has to be captured before cmd.Wait() is called, as cmd.Wait() closes the stderr pipe. - var stderrLines []string - scanner := bufio.NewScanner(stderr) - for scanner.Scan() { - stderrLines = append(stderrLines, scanner.Text()) - } - - err = cmd.Wait() - if err != nil { - r.Errorf("Docker command exited with code (%q): %d\nSTDERR:\n", cmd.String(), cmd.ProcessState.ExitCode()) - for _, line := range stderrLines { - r.Errorf("> %s\n", line) - } - - return errors.New("failed to run docker command") - } - - return nil -} - -func scanDockerImage(r reporter.Reporter, dockerImageName string) ([]scannedPackage, error) { - tempImageFile, err := os.CreateTemp("", "docker-image-*.tar") - if err != nil { - r.Errorf("Failed to create temporary file: %s\n", err) - return nil, err - } - - err = tempImageFile.Close() - if err != nil { - return nil, err - } - defer os.Remove(tempImageFile.Name()) - - r.Infof("Pulling docker image (%q)...\n", dockerImageName) - err = runCommandLogError(r, "docker", "pull", "-q", dockerImageName) - if err != nil { - return nil, err - } - - r.Infof("Saving docker image (%q) to temporary file...\n", dockerImageName) - err = runCommandLogError(r, "docker", "save", "-o", tempImageFile.Name(), dockerImageName) - if err != nil { - return nil, err - } - - r.Infof("Scanning image...\n") - packages, err := scanImage(r, tempImageFile.Name()) - if err != nil { - return nil, err - } - - // Modify the image path to be the image name, rather than the temporary file name - for i := range packages { - _, internalPath, _ := strings.Cut(packages[i].Source.Path, ":") - packages[i].Source.Path = dockerImageName + ":" + internalPath - } - - return packages, nil -} - -// Filters results according to config, preserving order. Returns total number of vulnerabilities removed. -func filterResults(r reporter.Reporter, results *models.VulnerabilityResults, configManager *config.Manager, allPackages bool) int { - removedCount := 0 - newResults := []models.PackageSource{} // Want 0 vulnerabilities to show in JSON as an empty list, not null. - for _, pkgSrc := range results.Results { - configToUse := configManager.Get(r, pkgSrc.Source.Path) - var newPackages []models.PackageVulns - for _, pkgVulns := range pkgSrc.Packages { - newVulns := filterPackageVulns(r, pkgVulns, configToUse) - removedCount += len(pkgVulns.Vulnerabilities) - len(newVulns.Vulnerabilities) - if allPackages || len(newVulns.Vulnerabilities) > 0 || len(pkgVulns.LicenseViolations) > 0 { - newPackages = append(newPackages, newVulns) - } - } - // Don't want to include the package source at all if there are no vulns. - if len(newPackages) > 0 { - pkgSrc.Packages = newPackages - newResults = append(newResults, pkgSrc) - } - } - results.Results = newResults - - return removedCount -} - -// Filters package-grouped vulnerabilities according to config, preserving ordering. Returns filtered package vulnerabilities. -func filterPackageVulns(r reporter.Reporter, pkgVulns models.PackageVulns, configToUse config.Config) models.PackageVulns { - ignoredVulns := map[string]struct{}{} - - // Iterate over groups first to remove all aliases of ignored vulnerabilities. - var newGroups []models.GroupInfo - for _, group := range pkgVulns.Groups { - ignore := false - for _, id := range group.Aliases { - var ignoreLine config.IgnoreEntry - if ignore, ignoreLine = configToUse.ShouldIgnore(id); ignore { - for _, id := range group.Aliases { - ignoredVulns[id] = struct{}{} - } - - reason := ignoreLine.Reason - - if reason == "" { - reason = "(no reason given)" - } - - // NB: This only prints the first reason encountered in all the aliases. - switch len(group.Aliases) { - case 1: - r.Infof("%s has been filtered out because: %s\n", ignoreLine.ID, reason) - case 2: - r.Infof("%s and 1 alias have been filtered out because: %s\n", ignoreLine.ID, reason) - default: - r.Infof("%s and %d aliases have been filtered out because: %s\n", ignoreLine.ID, len(group.Aliases)-1, reason) - } - - break - } - } - if !ignore { - newGroups = append(newGroups, group) - } - } - - var newVulns []models.Vulnerability - if len(newGroups) > 0 { // If there are no groups left then there would be no vulnerabilities. - for _, vuln := range pkgVulns.Vulnerabilities { - if _, filtered := ignoredVulns[vuln.ID]; !filtered { - newVulns = append(newVulns, vuln) - } - } - } - - // Passed by value. We don't want to alter the original PackageVulns. - pkgVulns.Groups = newGroups - pkgVulns.Vulnerabilities = newVulns - - return pkgVulns -} - -func parseLockfilePath(lockfileElem string) (string, string) { - if !strings.Contains(lockfileElem, ":") { - lockfileElem = ":" + lockfileElem - } - - splits := strings.SplitN(lockfileElem, ":", 2) - - return splits[0], splits[1] -} - -type scannedPackage struct { - PURL string - Name string - Ecosystem lockfile.Ecosystem - Commit string - Version string - Source models.SourceInfo - ImageOrigin *models.ImageOriginDetails - DepGroups []string -} - // Perform osv scanner action, with optional reporter to output information func DoScan(actions ScannerActions, r reporter.Reporter) (models.VulnerabilityResults, error) { if r == nil { r = &reporter.VoidReporter{} } + // TODO(v2): Move the logic of the offline flag moving other flags into here. if actions.CompareOffline { actions.SkipGit = true @@ -857,112 +91,50 @@ func DoScan(actions ScannerActions, r reporter.Reporter) (models.VulnerabilityRe return models.VulnerabilityResults{}, errors.New("databases can only be downloaded when running in offline mode") } - configManager := config.Manager{ - DefaultConfig: config.Config{}, - ConfigMap: make(map[string]config.Config), + scanResult := results.ScanResults{ + ConfigManager: config.Manager{ + DefaultConfig: config.Config{}, + ConfigMap: make(map[string]config.Config), + }, } - //nolint:prealloc // Not sure how many there will be in advance. - var scannedPackages []scannedPackage - if actions.ConfigOverridePath != "" { - err := configManager.UseOverride(r, actions.ConfigOverridePath) + err := scanResult.ConfigManager.UseOverride(r, actions.ConfigOverridePath) if err != nil { r.Errorf("Failed to read config file: %s\n", err) return models.VulnerabilityResults{}, err } } - if actions.ExperimentalScannerActions.ScanOCIImage != "" { - r.Infof("Scanning image %s\n", actions.ExperimentalScannerActions.ScanOCIImage) - pkgs, err := scanImage(r, actions.ExperimentalScannerActions.ScanOCIImage) - if err != nil { - return models.VulnerabilityResults{}, err - } - - scannedPackages = append(scannedPackages, pkgs...) - } - - if actions.DockerImageName != "" { - pkgs, err := scanDockerImage(r, actions.DockerImageName) - if err != nil { - return models.VulnerabilityResults{}, err - } - scannedPackages = append(scannedPackages, pkgs...) - } - - for _, lockfileElem := range actions.LockfilePaths { - parseAs, lockfilePath := parseLockfilePath(lockfileElem) - lockfilePath, err := filepath.Abs(lockfilePath) - if err != nil { - r.Errorf("Failed to resolved path with error %s\n", err) - return models.VulnerabilityResults{}, err - } - pkgs, err := scanLockfile(r, lockfilePath, parseAs, actions.TransitiveScanningActions) - if err != nil { - return models.VulnerabilityResults{}, err - } - scannedPackages = append(scannedPackages, pkgs...) - } - - for _, sbomElem := range actions.SBOMPaths { - sbomElem, err := filepath.Abs(sbomElem) - if err != nil { - return models.VulnerabilityResults{}, fmt.Errorf("failed to resolved path with error %w", err) - } - pkgs, err := scanSBOMFile(r, sbomElem, false) - if err != nil { - return models.VulnerabilityResults{}, err - } - scannedPackages = append(scannedPackages, pkgs...) - } - - for _, commit := range actions.GitCommits { - scannedPackages = append(scannedPackages, createCommitQueryPackage(commit, "HASH")) - } - - for _, dir := range actions.DirectoryPaths { - r.Infof("Scanning dir %s\n", dir) - pkgs, err := scanDir(r, dir, actions.SkipGit, actions.Recursive, !actions.NoIgnore, actions.CompareOffline, actions.TransitiveScanningActions) - if err != nil { - return models.VulnerabilityResults{}, err - } - scannedPackages = append(scannedPackages, pkgs...) - } - - if len(scannedPackages) == 0 { - return models.VulnerabilityResults{}, NoPackagesFoundErr + // ----- Perform Scanning ----- + packages, err := scan(r, actions) + if err != nil { + return models.VulnerabilityResults{}, err } - filteredScannedPackagesWithoutUnscannable := filterUnscannablePackages(scannedPackages) - - if len(filteredScannedPackagesWithoutUnscannable) != len(scannedPackages) { - r.Infof("Filtered %d local package/s from the scan.\n", len(scannedPackages)-len(filteredScannedPackagesWithoutUnscannable)) - } + scanResult.PackageScanResults = packages - filteredScannedPackages := filterIgnoredPackages(r, filteredScannedPackagesWithoutUnscannable, &configManager) + filterUnscannablePackages(r, &scanResult) - if len(filteredScannedPackages) != len(filteredScannedPackagesWithoutUnscannable) { - r.Infof("Filtered %d ignored package/s from the scan.\n", len(filteredScannedPackagesWithoutUnscannable)-len(filteredScannedPackages)) - } + filterIgnoredPackages(r, &scanResult) - overrideGoVersion(r, filteredScannedPackages, &configManager) + overrideGoVersion(r, &scanResult) - vulnsResp, err := makeRequest(r, filteredScannedPackages, actions.CompareOffline, actions.DownloadDatabases, actions.LocalDBPath) + err = makeRequest(r, scanResult.PackageScanResults, actions.CompareOffline, actions.DownloadDatabases, actions.LocalDBPath) if err != nil { return models.VulnerabilityResults{}, err } - var licensesResp [][]models.License if len(actions.ScanLicensesAllowlist) > 0 || actions.ScanLicensesSummary { - licensesResp, err = makeLicensesRequests(filteredScannedPackages) + err = makeLicensesRequests(scanResult.PackageScanResults) if err != nil { return models.VulnerabilityResults{}, err } } - results := buildVulnerabilityResults(r, filteredScannedPackages, vulnsResp, licensesResp, actions, &configManager) - filtered := filterResults(r, &results, &configManager, actions.ShowAllPackages) + results := buildVulnerabilityResults(r, actions, &scanResult) + + filtered := filterResults(r, &results, &scanResult.ConfigManager, actions.ShowAllPackages) if filtered > 0 { r.Infof( "Filtered %d %s from output\n", @@ -973,7 +145,8 @@ func DoScan(actions ScannerActions, r reporter.Reporter) (models.VulnerabilityRe if len(results.Results) > 0 { // Determine the correct error to return. - // TODO: in the next breaking release of osv-scanner, consider + + // TODO(v2): in the next breaking release of osv-scanner, consider // returning a ScanError instead of an error. var vuln bool onlyUncalledVuln := true @@ -1003,65 +176,9 @@ func DoScan(actions ScannerActions, r reporter.Reporter) (models.VulnerabilityRe return results, nil } -// filterUnscannablePackages removes packages that don't have enough information to be scanned -// e,g, local packages that specified by path -func filterUnscannablePackages(packages []scannedPackage) []scannedPackage { - out := make([]scannedPackage, 0, len(packages)) - for _, p := range packages { - switch { - // If none of the cases match, skip this package since it's not scannable - case p.Ecosystem != "" && p.Name != "" && p.Version != "": - case p.Commit != "": - case p.PURL != "": - default: - continue - } - out = append(out, p) - } - - return out -} - -// filterIgnoredPackages removes ignore scanned packages according to config. Returns filtered scanned packages. -func filterIgnoredPackages(r reporter.Reporter, packages []scannedPackage, configManager *config.Manager) []scannedPackage { - out := make([]scannedPackage, 0, len(packages)) - for _, p := range packages { - configToUse := configManager.Get(r, p.Source.Path) - pkg := models.PackageVulns{ - Package: models.PackageInfo{ - Name: p.Name, - Version: p.Version, - Ecosystem: string(p.Ecosystem), - Commit: p.Commit, - }, - DepGroups: p.DepGroups, - } - - if ignore, ignoreLine := configToUse.ShouldIgnorePackage(pkg); ignore { - var pkgString string - if p.PURL != "" { - pkgString = p.PURL - } else { - pkgString = fmt.Sprintf("%s/%s/%s", p.Ecosystem, p.Name, p.Version) - } - reason := ignoreLine.Reason - - if reason == "" { - reason = "(no reason given)" - } - r.Infof("Package %s has been filtered out because: %s\n", pkgString, reason) - - continue - } - out = append(out, p) - } - - return out -} - // patchPackageForRequest modifies packages before they are sent to osv.dev to // account for edge cases. -func patchPackageForRequest(pkg scannedPackage) scannedPackage { +func patchPackageForRequest(pkg imodels.PackageInfo) imodels.PackageInfo { // Assume Go stdlib patch version as the latest version // // This is done because go1.20 and earlier do not support patch @@ -1070,7 +187,7 @@ func patchPackageForRequest(pkg scannedPackage) scannedPackage { // However, if we assume patch version as .0, this will cause a lot of // false positives. This compromise still allows osv-scanner to pick up // when the user is using a minor version that is out-of-support. - if pkg.Name == "stdlib" && pkg.Ecosystem == "Go" { + if pkg.Name == "stdlib" && pkg.Ecosystem.Ecosystem == osvschema.EcosystemGo { v := semantic.ParseSemverLikeVersion(pkg.Version, 3) if len(v.Components) == 2 { pkg.Version = fmt.Sprintf( @@ -1085,64 +202,67 @@ func patchPackageForRequest(pkg scannedPackage) scannedPackage { return pkg } +// TODO(V2): This will be replaced by the new client interface func makeRequest( r reporter.Reporter, - packages []scannedPackage, + packages []imodels.PackageScanResult, compareOffline bool, downloadDBs bool, - localDBPath string) (*osv.HydratedBatchedResponse, error) { + localDBPath string) error { // Make OSV queries from the packages. var query osv.BatchedQuery - for _, p := range packages { + for _, psr := range packages { + p := psr.PackageInfo p = patchPackageForRequest(p) switch { // Prefer making package requests where possible. - case p.Ecosystem != "" && p.Name != "" && p.Version != "": - query.Queries = append(query.Queries, osv.MakePkgRequest(lockfile.PackageDetails{ - Name: p.Name, - Version: p.Version, - Ecosystem: p.Ecosystem, - })) + case !p.Ecosystem.IsEmpty() && p.Name != "" && p.Version != "": + query.Queries = append(query.Queries, osv.MakePkgRequest(p)) case p.Commit != "": query.Queries = append(query.Queries, osv.MakeCommitRequest(p.Commit)) - case p.PURL != "": - query.Queries = append(query.Queries, osv.MakePURLRequest(p.PURL)) default: - return nil, fmt.Errorf("package %v does not have a commit, PURL or ecosystem/name/version identifier", p) + return fmt.Errorf("package %v does not have a commit, PURL or ecosystem/name/version identifier", p) } } + var err error + var hydratedResp *osv.HydratedBatchedResponse if compareOffline { + // TODO(v2): Stop depending on lockfile.PackageDetails and use imodels.PackageInfo // Downloading databases requires network access. - hydratedResp, err := local.MakeRequest(r, query, !downloadDBs, localDBPath) + hydratedResp, err = local.MakeRequest(r, query, !downloadDBs, localDBPath) if err != nil { - return &osv.HydratedBatchedResponse{}, fmt.Errorf("local comparison failed %w", err) + return fmt.Errorf("local comparison failed %w", err) + } + } else { + if osv.RequestUserAgent == "" { + osv.RequestUserAgent = "osv-scanner-api/v" + version.OSVVersion } - return hydratedResp, nil - } - - if osv.RequestUserAgent == "" { - osv.RequestUserAgent = "osv-scanner-api_v" + version.OSVVersion - } + resp, err := osv.MakeRequest(query) + if err != nil { + return fmt.Errorf("%w: osv.dev query failed: %w", ErrAPIFailed, err) + } - resp, err := osv.MakeRequest(query) - if err != nil { - return &osv.HydratedBatchedResponse{}, fmt.Errorf("%w: osv.dev query failed: %w", ErrAPIFailed, err) + hydratedResp, err = osv.Hydrate(resp) + if err != nil { + return fmt.Errorf("%w: failed to hydrate OSV response: %w", ErrAPIFailed, err) + } } - hydratedResp, err := osv.Hydrate(resp) - if err != nil { - return &osv.HydratedBatchedResponse{}, fmt.Errorf("%w: failed to hydrate OSV response: %w", ErrAPIFailed, err) + for i, result := range hydratedResp.Results { + packages[i].Vulnerabilities = result.Vulns } - return hydratedResp, nil + return nil } -func makeLicensesRequests(packages []scannedPackage) ([][]models.License, error) { +// TODO(V2): Replace with client +func makeLicensesRequests(packages []imodels.PackageScanResult) error { queries := make([]*depsdevpb.GetVersionRequest, len(packages)) - for i, pkg := range packages { - system, ok := depsdev.System[pkg.Ecosystem] + for i, psr := range packages { + pkg := psr.PackageInfo + system, ok := depsdev.System[psr.PackageInfo.Ecosystem.Ecosystem] if !ok || pkg.Name == "" || pkg.Version == "" { continue } @@ -1150,19 +270,24 @@ func makeLicensesRequests(packages []scannedPackage) ([][]models.License, error) } licenses, err := depsdev.MakeVersionRequests(queries) if err != nil { - return nil, fmt.Errorf("%w: deps.dev query failed: %w", ErrAPIFailed, err) + return fmt.Errorf("%w: deps.dev query failed: %w", ErrAPIFailed, err) } - return licenses, nil + for i, license := range licenses { + packages[i].Licenses = license + } + + return nil } // Overrides Go version using osv-scanner.toml -func overrideGoVersion(r reporter.Reporter, packages []scannedPackage, configManager *config.Manager) { - for i, pkg := range packages { - if pkg.Name == "stdlib" && pkg.Ecosystem == "Go" { - configToUse := configManager.Get(r, pkg.Source.Path) +func overrideGoVersion(r reporter.Reporter, scanResults *results.ScanResults) { + for i, psr := range scanResults.PackageScanResults { + pkg := psr.PackageInfo + if pkg.Name == "stdlib" && pkg.Ecosystem.Ecosystem == osvschema.EcosystemGo { + configToUse := scanResults.ConfigManager.Get(r, pkg.Location) if configToUse.GoVersionOverride != "" { - packages[i].Version = configToUse.GoVersionOverride + scanResults.PackageScanResults[i].PackageInfo.Version = configToUse.GoVersionOverride } continue diff --git a/pkg/osvscanner/scan.go b/pkg/osvscanner/scan.go new file mode 100644 index 0000000000..1d46922f84 --- /dev/null +++ b/pkg/osvscanner/scan.go @@ -0,0 +1,112 @@ +package osvscanner + +import ( + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scanner/internal/depsdev" + "github.com/google/osv-scanner/internal/imodels" + "github.com/google/osv-scanner/internal/resolution/client" + "github.com/google/osv-scanner/internal/resolution/datasource" + "github.com/google/osv-scanner/internal/scalibrextract/language/java/pomxmlnet" + "github.com/google/osv-scanner/pkg/osvscanner/internal/scanners" + "github.com/google/osv-scanner/pkg/reporter" +) + +// scan essentially converts ScannerActions into PackageScanResult by performing the extractions +func scan(r reporter.Reporter, actions ScannerActions) ([]imodels.PackageScanResult, error) { + var scannedInventories []*extractor.Inventory + + // TODO(V2 Models): Temporarily initialize pom here to reduce PR size + // Eventually, we want to move TransitiveScanningActions into its own models package to avoid + // cyclic imports + var pomExtractor filesystem.Extractor + if !actions.TransitiveScanningActions.Disabled { + var err error + pomExtractor, err = createMavenExtractor(actions.TransitiveScanningActions) + if err != nil { + return nil, err + } + } + + // --- Lockfiles --- + for _, lockfileElem := range actions.LockfilePaths { + invs, err := scanners.ScanLockfile(r, lockfileElem, pomExtractor) + if err != nil { + return nil, err + } + + scannedInventories = append(scannedInventories, invs...) + } + + // --- SBOMs --- + for _, sbomPath := range actions.SBOMPaths { + invs, err := scanners.ScanSBOM(r, sbomPath) + if err != nil { + return nil, err + } + + scannedInventories = append(scannedInventories, invs...) + } + + // --- Directories --- + for _, dir := range actions.DirectoryPaths { + r.Infof("Scanning dir %s\n", dir) + pkgs, err := scanners.ScanDir(r, dir, actions.SkipGit, actions.Recursive, !actions.NoIgnore, pomExtractor) + if err != nil { + return nil, err + } + scannedInventories = append(scannedInventories, pkgs...) + } + + if len(scannedInventories) == 0 { + return nil, NoPackagesFoundErr + } + + // Convert to imodels.PackageScanResult for use in the rest of osv-scanner + packages := []imodels.PackageScanResult{} + for _, inv := range scannedInventories { + pi := imodels.FromInventory(inv) + + packages = append(packages, imodels.PackageScanResult{ + PackageInfo: pi, + }) + } + + // Add on additional direct dependencies passed straight from ScannerActions: + for _, commit := range actions.GitCommits { + pi := imodels.PackageInfo{ + Commit: commit, + } + + packages = append(packages, imodels.PackageScanResult{ + PackageInfo: pi, + }) + } + + return packages, nil +} + +func createMavenExtractor(actions TransitiveScanningActions) (*pomxmlnet.Extractor, error) { + var depClient client.DependencyClient + var err error + if actions.NativeDataSource { + depClient, err = client.NewMavenRegistryClient(actions.MavenRegistry) + } else { + depClient, err = client.NewDepsDevClient(depsdev.DepsdevAPI) + } + if err != nil { + return nil, err + } + + mavenClient, err := datasource.NewMavenRegistryAPIClient(actions.MavenRegistry) + if err != nil { + return nil, err + } + + extractor := pomxmlnet.Extractor{ + DependencyClient: depClient, + MavenRegistryAPIClient: mavenClient, + } + + return &extractor, nil +} diff --git a/pkg/osvscanner/vulnerability_result.go b/pkg/osvscanner/vulnerability_result.go index f28c9a260d..1963159cf5 100644 --- a/pkg/osvscanner/vulnerability_result.go +++ b/pkg/osvscanner/vulnerability_result.go @@ -5,13 +5,13 @@ import ( "sort" "strings" - "github.com/google/osv-scanner/internal/config" "github.com/google/osv-scanner/internal/grouper" + "github.com/google/osv-scanner/internal/imodels" + "github.com/google/osv-scanner/internal/imodels/results" "github.com/google/osv-scanner/internal/output" "github.com/google/osv-scanner/internal/sourceanalysis" "github.com/google/osv-scanner/internal/spdx" "github.com/google/osv-scanner/pkg/models" - "github.com/google/osv-scanner/pkg/osv" "github.com/google/osv-scanner/pkg/reporter" ) @@ -21,49 +21,39 @@ import ( // TODO: This function is getting long, we should refactor it func buildVulnerabilityResults( r reporter.Reporter, - packages []scannedPackage, - vulnsResp *osv.HydratedBatchedResponse, - licensesResp [][]models.License, actions ScannerActions, - configManager *config.Manager, + scanResults *results.ScanResults, ) models.VulnerabilityResults { results := models.VulnerabilityResults{ Results: []models.PackageSource{}, } groupedBySource := map[models.SourceInfo][]models.PackageVulns{} - for i, rawPkg := range packages { + for _, psr := range scanResults.PackageScanResults { + p := psr.PackageInfo includePackage := actions.ShowAllPackages var pkg models.PackageVulns - if rawPkg.Commit != "" { - pkg.Package.Commit = rawPkg.Commit - pkg.Package.Name = rawPkg.Name - } else if rawPkg.PURL != "" { - var err error - pkg.Package, err = models.PURLToPackage(rawPkg.PURL) - - if err != nil { - r.Errorf("Failed to parse purl: %s, with error: %s", rawPkg.PURL, err) - continue - } + if p.Commit != "" { + pkg.Package.Commit = p.Commit + pkg.Package.Name = p.Name } - if rawPkg.Version != "" && rawPkg.Ecosystem != "" { + if p.Version != "" && !p.Ecosystem.IsEmpty() { pkg.Package = models.PackageInfo{ - Name: rawPkg.Name, - Version: rawPkg.Version, - Ecosystem: string(rawPkg.Ecosystem), - ImageOrigin: rawPkg.ImageOrigin, + Name: p.Name, + Version: p.Version, + Ecosystem: p.Ecosystem.String(), + // ImageOrigin: p.ImageOrigin, } } - pkg.DepGroups = rawPkg.DepGroups + pkg.DepGroups = p.DepGroups + configToUse := scanResults.ConfigManager.Get(r, p.Location) - if len(vulnsResp.Results[i].Vulns) > 0 { - configToUse := configManager.Get(r, rawPkg.Source.Path) - if !configToUse.ShouldIgnorePackageVulnerabilities(pkg) { + if len(psr.Vulnerabilities) > 0 { + if !configToUse.ShouldIgnorePackageVulnerabilities(p) { includePackage = true - pkg.Vulnerabilities = vulnsResp.Results[i].Vulns + pkg.Vulnerabilities = psr.Vulnerabilities pkg.Groups = grouper.Group(grouper.ConvertVulnerabilityToIDAliases(pkg.Vulnerabilities)) for i, group := range pkg.Groups { pkg.Groups[i].MaxSeverity = output.MaxSeverity(group, pkg) @@ -78,22 +68,21 @@ func buildVulnerabilityResults( } if actions.ScanLicensesSummary || len(actions.ScanLicensesAllowlist) > 0 { - configToUse := configManager.Get(r, rawPkg.Source.Path) - if override, entry := configToUse.ShouldOverridePackageLicense(pkg); override { + if override, entry := configToUse.ShouldOverridePackageLicense(p); override { if entry.License.Ignore { r.Infof("ignoring license for package %s/%s/%s\n", pkg.Package.Ecosystem, pkg.Package.Name, pkg.Package.Version) - licensesResp[i] = nil + psr.Licenses = []models.License{} } else { overrideLicenses := make([]models.License, len(entry.License.Override)) for j, license := range entry.License.Override { overrideLicenses[j] = models.License(license) } r.Infof("overriding license for package %s/%s/%s with %s\n", pkg.Package.Ecosystem, pkg.Package.Name, pkg.Package.Version, strings.Join(entry.License.Override, ",")) - licensesResp[i] = overrideLicenses + psr.Licenses = overrideLicenses } } if len(actions.ScanLicensesAllowlist) > 0 { - pkg.Licenses = licensesResp[i] + pkg.Licenses = psr.Licenses for _, license := range pkg.Licenses { satisfies, err := spdx.Satisfies(license, actions.ScanLicensesAllowlist) @@ -109,12 +98,32 @@ func buildVulnerabilityResults( includePackage = true } } + if actions.ScanLicensesSummary { - pkg.Licenses = licensesResp[i] + pkg.Licenses = psr.Licenses } } if includePackage { - groupedBySource[rawPkg.Source] = append(groupedBySource[rawPkg.Source], pkg) + var sourceType string + switch p.SourceType { + case imodels.SourceTypeOSPackage: + sourceType = "os" + case imodels.SourceTypeProjectPackage: + sourceType = "lockfile" + case imodels.SourceTypeSBOM: + sourceType = "sbom" + case imodels.SourceTypeGit: + sourceType = "git" + case imodels.SourceTypeUnknown: + sourceType = "unknown" + default: + sourceType = "unknown" + } + source := models.SourceInfo{ + Path: p.Location, + Type: sourceType, + } + groupedBySource[source] = append(groupedBySource[source], pkg) } } diff --git a/pkg/osvscanner/vulnerability_result_internal_test.go b/pkg/osvscanner/vulnerability_result_internal_test.go index c60d8503a6..2ef604bfd5 100644 --- a/pkg/osvscanner/vulnerability_result_internal_test.go +++ b/pkg/osvscanner/vulnerability_result_internal_test.go @@ -4,8 +4,10 @@ import ( "testing" "github.com/google/osv-scanner/internal/config" + "github.com/google/osv-scanner/internal/imodels" + "github.com/google/osv-scanner/internal/imodels/ecosystem" + "github.com/google/osv-scanner/internal/imodels/results" "github.com/google/osv-scanner/internal/testutility" - "github.com/google/osv-scanner/pkg/lockfile" "github.com/google/osv-scanner/pkg/models" "github.com/google/osv-scanner/pkg/osv" "github.com/google/osv-scanner/pkg/reporter" @@ -14,40 +16,30 @@ import ( func Test_assembleResult(t *testing.T) { t.Parallel() type args struct { - r reporter.Reporter - packages []scannedPackage - vulnsResp *osv.HydratedBatchedResponse - licensesResp [][]models.License - actions ScannerActions - config *config.Manager + r reporter.Reporter + actions ScannerActions + scanResults *results.ScanResults + config config.Manager } - packages := []scannedPackage{ + packages := []imodels.PackageInfo{ { - Name: "pkg-1", - Ecosystem: lockfile.Ecosystem("npm"), - Version: "1.0.0", - Source: models.SourceInfo{ - Path: "dir/package-lock.json", - Type: "lockfile", - }, - }, - { - Name: "pkg-2", - Ecosystem: lockfile.Ecosystem("npm"), - Version: "1.0.0", - Source: models.SourceInfo{ - Path: "dir/package-lock.json", - Type: "lockfile", - }, - }, - { - Name: "pkg-3", - Ecosystem: lockfile.Ecosystem("npm"), - Version: "1.0.0", - Source: models.SourceInfo{ - Path: "other-dir/package-lock.json", - Type: "lockfile", - }, + Name: "pkg-1", + Ecosystem: ecosystem.MustParse("npm"), + Version: "1.0.0", + Location: "dir/package-lock.json", + SourceType: imodels.SourceTypeProjectPackage, + }, { + Name: "pkg-2", + Ecosystem: ecosystem.MustParse("npm"), + Version: "1.0.0", + Location: "dir/package-lock.json", + SourceType: imodels.SourceTypeProjectPackage, + }, { + Name: "pkg-3", + Ecosystem: ecosystem.MustParse("npm"), + Version: "1.0.0", + Location: "other-dir/package-lock.json", + SourceType: imodels.SourceTypeProjectPackage, }, } vulnsResp := &osv.HydratedBatchedResponse{ @@ -68,11 +60,13 @@ func Test_assembleResult(t *testing.T) { })}, }, } + licensesResp := [][]models.License{ {models.License("MIT"), models.License("0BSD")}, {models.License("MIT")}, {models.License("UNKNOWN")}, } + makeLicensesResp := func() [][]models.License { cpy := make([][]models.License, len(licensesResp)) copy(cpy, licensesResp) @@ -80,126 +74,130 @@ func Test_assembleResult(t *testing.T) { return cpy } + // makeScanResults make a separate instance of ScanResults to avoid mutations changing other tests + makeScanResults := func() *results.ScanResults { + scanResults := results.ScanResults{ + ConfigManager: config.Manager{}, + } + licensesResp = makeLicensesResp() + for i := range packages { + scanResults.PackageScanResults = append(scanResults.PackageScanResults, imodels.PackageScanResult{ + PackageInfo: packages[i], + Vulnerabilities: vulnsResp.Results[i].Vulns, + Licenses: licensesResp[i], + }) + } + + return &scanResults + } + callAnalysisStates := make(map[string]bool) tests := []struct { name string args args - }{{ - name: "group_vulnerabilities", - args: args{ - r: &reporter.VoidReporter{}, - packages: packages, - vulnsResp: vulnsResp, - licensesResp: makeLicensesResp(), - actions: ScannerActions{ - ExperimentalScannerActions: ExperimentalScannerActions{ - ShowAllPackages: false, - ScanLicensesAllowlist: nil, - }, - CallAnalysisStates: callAnalysisStates, - }, - config: &config.Manager{}, - }, - }, { - name: "group_vulnerabilities_with_all_packages_included", - args: args{ - r: &reporter.VoidReporter{}, - packages: packages, - vulnsResp: vulnsResp, - licensesResp: makeLicensesResp(), - actions: ScannerActions{ - ExperimentalScannerActions: ExperimentalScannerActions{ - ShowAllPackages: true, - ScanLicensesAllowlist: nil, + }{ + { + name: "group_vulnerabilities", + args: args{ + r: &reporter.VoidReporter{}, + scanResults: makeScanResults(), + actions: ScannerActions{ + ExperimentalScannerActions: ExperimentalScannerActions{ + ShowAllPackages: false, + ScanLicensesAllowlist: nil, + }, + CallAnalysisStates: callAnalysisStates, }, - CallAnalysisStates: callAnalysisStates, }, - config: &config.Manager{}, }, - }, { - name: "group_vulnerabilities_with_licenses", - args: args{ - r: &reporter.VoidReporter{}, - packages: packages, - vulnsResp: vulnsResp, - licensesResp: makeLicensesResp(), - actions: ScannerActions{ - ExperimentalScannerActions: ExperimentalScannerActions{ - ShowAllPackages: true, - ScanLicensesSummary: true, - ScanLicensesAllowlist: nil, + { + name: "group_vulnerabilities_with_all_packages_included", + args: args{ + r: &reporter.VoidReporter{}, + scanResults: makeScanResults(), + actions: ScannerActions{ + ExperimentalScannerActions: ExperimentalScannerActions{ + ShowAllPackages: true, + ScanLicensesAllowlist: nil, + }, + CallAnalysisStates: callAnalysisStates, }, - CallAnalysisStates: callAnalysisStates, }, - config: &config.Manager{}, - }, - }, { - name: "group_vulnerabilities_with_license_allowlist", - args: args{ - r: &reporter.VoidReporter{}, - packages: packages, - vulnsResp: vulnsResp, - licensesResp: makeLicensesResp(), - actions: ScannerActions{ - ExperimentalScannerActions: ExperimentalScannerActions{ - ShowAllPackages: false, - ScanLicensesAllowlist: []string{"MIT", "0BSD"}, + }, { + name: "group_vulnerabilities_with_licenses", + args: args{ + r: &reporter.VoidReporter{}, + scanResults: makeScanResults(), + actions: ScannerActions{ + ExperimentalScannerActions: ExperimentalScannerActions{ + ShowAllPackages: true, + ScanLicensesSummary: true, + ScanLicensesAllowlist: nil, + }, + CallAnalysisStates: callAnalysisStates, }, - CallAnalysisStates: callAnalysisStates, }, + }, { + name: "group_vulnerabilities_with_license_allowlist", + args: args{ + r: &reporter.VoidReporter{}, + scanResults: makeScanResults(), - config: &config.Manager{}, - }, - }, { - name: "group_vulnerabilities_with_license_allowlist_and_license_override", - args: args{ - r: &reporter.VoidReporter{}, - packages: packages, - vulnsResp: vulnsResp, - licensesResp: makeLicensesResp(), - actions: ScannerActions{ - ExperimentalScannerActions: ExperimentalScannerActions{ - ShowAllPackages: false, - ScanLicensesAllowlist: []string{"MIT", "0BSD"}, + actions: ScannerActions{ + ExperimentalScannerActions: ExperimentalScannerActions{ + ShowAllPackages: false, + ScanLicensesAllowlist: []string{"MIT", "0BSD"}, + }, + CallAnalysisStates: callAnalysisStates, }, - CallAnalysisStates: callAnalysisStates, }, - config: &config.Manager{ - OverrideConfig: &config.Config{ - PackageOverrides: []config.PackageOverrideEntry{ - { - Name: "pkg-3", - Ecosystem: "npm", - License: config.License{ - Override: []string{"MIT"}, + }, { + name: "group_vulnerabilities_with_license_allowlist_and_license_override", + args: args{ + r: &reporter.VoidReporter{}, + scanResults: makeScanResults(), + actions: ScannerActions{ + ExperimentalScannerActions: ExperimentalScannerActions{ + ShowAllPackages: false, + ScanLicensesAllowlist: []string{"MIT", "0BSD"}, + }, + CallAnalysisStates: callAnalysisStates, + }, + config: config.Manager{ + OverrideConfig: &config.Config{ + PackageOverrides: []config.PackageOverrideEntry{ + { + Name: "pkg-3", + Ecosystem: "npm", + License: config.License{ + Override: []string{"MIT"}, + }, }, }, }, }, }, - }, - }, { - name: "group_vulnerabilities_with_license_allowlist_and_all_packages", - args: args{ - r: &reporter.VoidReporter{}, - packages: packages, - vulnsResp: vulnsResp, - licensesResp: makeLicensesResp(), - actions: ScannerActions{ - ExperimentalScannerActions: ExperimentalScannerActions{ - ShowAllPackages: true, - ScanLicensesAllowlist: []string{"MIT", "0BSD"}, + }, { + name: "group_vulnerabilities_with_license_allowlist_and_all_packages", + args: args{ + r: &reporter.VoidReporter{}, + scanResults: makeScanResults(), + actions: ScannerActions{ + ExperimentalScannerActions: ExperimentalScannerActions{ + ShowAllPackages: true, + ScanLicensesAllowlist: []string{"MIT", "0BSD"}, + }, + CallAnalysisStates: callAnalysisStates, }, - CallAnalysisStates: callAnalysisStates, }, - config: &config.Manager{}, }, - }} + } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - got := buildVulnerabilityResults(tt.args.r, tt.args.packages, tt.args.vulnsResp, tt.args.licensesResp, tt.args.actions, tt.args.config) + tt.args.scanResults.ConfigManager = tt.args.config + got := buildVulnerabilityResults(tt.args.r, tt.args.actions, tt.args.scanResults) testutility.NewSnapshot().MatchJSON(t, got) }) }