ROX-16446: scanner v4 vlun updater (stackrox#6423)

Co-authored-by: Ross Tannenbaum <[email protected]>
Co-authored-by: J. Victor Martins <[email protected]>

.github/workflows/update-vulnerabilties.yaml

@@ -0,0 +1,74 @@
+name: Update Vulnerabilities
+on:
+  schedule:
+    - cron: "0 */3 * * *"
+
+jobs:
+  read-versions:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.output-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Generate matrix JSON
+        id: output-matrix
+        run: |
+          EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
+          echo "matrix<<$EOF" >> $GITHUB_OUTPUT
+          sed -n '/^\s*\(#.*\)\?$/!p' scanner/updater/version/RELEASE_VERSION | jq -Rs '{ versions: ( sub("\n$"; "")|split("\n") ) }' | tee -a "$GITHUB_OUTPUT"
+          echo $EOF >> $GITHUB_OUTPUT
+
+  upload-vulnerabilities:
+    needs: read-versions
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false #if one of the versions fails to get updated vulns, we still want to update for the other versions
+      max-parallel: 2 #limit the number of concurrent jobs to avoid job interruption by OOM as creating json blob has been memory-intensive
+      matrix:
+        version: ${{ fromJson(needs.read-versions.outputs.matrix).versions }}
+    env:
+      ROX_PRODUCT_VERSION: ${{ matrix.version }}
+    steps:
+      - name: Authenticate with Google Cloud
+        uses: google-github-actions/auth@v1
+        with:
+          credentials_json: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }}
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v1
+
+      - name: Update vulnerabilities
+        continue-on-error: true
+        run: |
+          DOWNLOAD_URL="https://github.com/stackrox/stackrox/archive/refs/tags/${{ env.ROX_PRODUCT_VERSION }}.zip"
+          FILE_NAME=$(basename "$DOWNLOAD_URL")
+          wget "$DOWNLOAD_URL" -O "$FILE_NAME"
+          # Check if wget succeeded
+          if [ $? -ne 0 ]; then
+            echo "Download failed. Terminating current matrix step."
+            exit 1
+          fi
+
+          unzip "$FILE_NAME" -d "${FILE_NAME}-dir"
+          cd "${FILE_NAME}-dir/stackrox-"*
+          if [ ! -d "scanner" ]; then
+            echo "Scanner directory not found. Terminating current matrix step."
+            exit 1
+          fi
+
+          cd scanner
+          go run cmd/updater/main.go -output-dir=${{ env.ROX_PRODUCT_VERSION }}
+          gsutil cp -r "${{ env.ROX_PRODUCT_VERSION }}" "gs://scanner-v4-test/vulnerability-bundles"
+
+  send-notification:
+    needs:
+      - read-versions
+      - upload-vulnerabilities
+    runs-on: ubuntu-latest
+    if: failure()
+    steps:
+    - name: Send Slack notification on workflow failure
+      run: |
+        curl -X POST -H 'Content-type: application/json' --data '{"text":"Workflow failed in workflow ${{ github.workflow }} in repository ${{ github.repository }}: Failed to update vulnerabilities"}' ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }}

go.mod

@@ -149,6 +149,8 @@ require (
 	sigs.k8s.io/yaml v1.3.0
 )
 
+require github.com/klauspost/compress v1.16.7
+
 require (
 	cloud.google.com/go v0.110.6 // indirect
 	cloud.google.com/go/compute v1.23.0 // indirect
@@ -273,7 +275,6 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 // indirect
 	github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f // indirect
 	github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d // indirect

scanner/cmd/updater/main.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"log"
+
+	"github.com/quay/zlog"
+	"github.com/stackrox/rox/scanner/updater"
+)
+
+func main() {
+	// Parse command-line flags
+	outputDir := flag.String("output-dir", "", "Output directory")
+	flag.Parse()
+
+	// Check if outputDir flag is provided
+	if *outputDir == "" {
+		log.Fatal("Missing argument for the output directory.")
+	}
+
+	ctx := context.Background()
+	if err := updater.Export(ctx, *outputDir); err != nil {
+		zlog.Error(ctx).Err(err).Send()
+	}
+}

scanner/updater/export.go

@@ -0,0 +1,104 @@
+package updater
+
+import (
+	"context"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/klauspost/compress/zstd"
+	"github.com/quay/claircore/libvuln/driver"
+	"github.com/quay/claircore/libvuln/jsonblob"
+	"github.com/quay/claircore/libvuln/updates"
+	"github.com/quay/zlog"
+	"github.com/stackrox/rox/scanner/updater/manual"
+	"golang.org/x/time/rate"
+
+	// default updaters
+	_ "github.com/quay/claircore/updater/defaults"
+)
+
+// Export is responsible for triggering the updaters to download Common Vulnerabilities and Exposures (CVEs) data
+// and then outputting the result as a zstd-compressed file with .ztd extension
+func Export(ctx context.Context, outputDir string) error {
+
+	err := os.MkdirAll(outputDir, 0700)
+	if err != nil {
+		return err
+	}
+	// create output json file
+	outputFile, err := os.Create(filepath.Join(outputDir, "output.json.ztd"))
+	if err != nil {
+		return err
+	}
+
+	limiter := rate.NewLimiter(rate.Every(time.Second), 5)
+	httpClient := &http.Client{
+		Transport: &rateLimitedTransport{
+			limiter:   limiter,
+			transport: http.DefaultTransport,
+		},
+	}
+
+	zstdWriter, err := zstd.NewWriter(outputFile)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		closeErr := zstdWriter.Close()
+		if closeErr != nil {
+			zlog.Error(ctx).Err(closeErr).Msg("Failed to closing zstdWriter")
+		}
+	}()
+
+	updaterSet, err := manual.UpdaterSet(ctx, nil)
+	if err != nil {
+		return err
+	}
+	outOfTree := [][]driver.Updater{
+		make([]driver.Updater, 0),
+	}
+	outOfTree = append(outOfTree, updaterSet.Updaters())
+
+	for i, uSet := range [][]string{
+		{"oracle", "photon", "suse", "aws", "rhcc"},
+		{"alpine", "rhel", "ubuntu", "osv", "debian"},
+	} {
+		jsonStore, err := jsonblob.New()
+		if err != nil {
+			return err
+		}
+
+		updateMgr, err := updates.NewManager(ctx, jsonStore, updates.NewLocalLockSource(), httpClient,
+			updates.WithEnabled(uSet),
+			updates.WithOutOfTree(outOfTree[i]),
+		)
+		if err != nil {
+			return err
+		}
+
+		if err := updateMgr.Run(ctx); err != nil {
+			return err
+		}
+
+		err = jsonStore.Store(zstdWriter)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+type rateLimitedTransport struct {
+	limiter   *rate.Limiter
+	transport http.RoundTripper
+}
+
+func (t *rateLimitedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	if err := t.limiter.Wait(req.Context()); err != nil {
+		return nil, err
+	}
+	return t.transport.RoundTrip(req)
+}

scanner/updater/manual/data.go

@@ -0,0 +1,9 @@
+package manual
+
+import (
+	"github.com/quay/claircore"
+)
+
+// manuallyEnrichedVulns lists vulnerabilities not tracked by other means.
+// An example entry is in manual_example_test.go.
+var manuallyEnrichedVulns = []*claircore.Vulnerability{}

scanner/updater/manual/manual.go

@@ -0,0 +1,46 @@
+// Package manual provides a custom updater for vulnerability scanner.
+// This updater allows manual input of vulnerability data.
+package manual
+
+import (
+	"context"
+	"io"
+
+	"github.com/quay/claircore"
+	"github.com/quay/claircore/libvuln/driver"
+)
+
+// Factory is the UpdaterSetFactory exposed by this package.
+// All configuration is done on the returned updaters.
+type Factory struct {
+}
+
+// UpdaterSet creates a new updater set with the provided vulnerability data.
+func UpdaterSet(_ context.Context, vulns []*claircore.Vulnerability) (driver.UpdaterSet, error) {
+	res := driver.NewUpdaterSet()
+	err := res.Add(&updater{data: vulns})
+	if err != nil {
+		return res, err
+	}
+	return res, nil
+}
+
+type updater struct {
+	data []*claircore.Vulnerability
+}
+
+// Name provides a name for the updater.
+func (u *updater) Name() string { return `ManualUpdater` }
+
+// Fetch returns nil values as the manual updater does not fetch data.
+func (u *updater) Fetch(_ context.Context, _ driver.Fingerprint) (io.ReadCloser, driver.Fingerprint, error) {
+	return nil, "", nil
+}
+
+// Parse returns the provided vulnerability data or defaults to manuallyEnrichedVulns.
+func (u *updater) Parse(_ context.Context, _ io.ReadCloser) ([]*claircore.Vulnerability, error) {
+	if u.data == nil || len(u.data) == 0 {
+		return manuallyEnrichedVulns, nil
+	}
+	return u.data, nil
+}

scanner/updater/manual/manual_example_test.go

@@ -0,0 +1,65 @@
+package manual
+
+import (
+	"context"
+	"net/http"
+	"testing"
+
+	"github.com/quay/claircore"
+	"github.com/quay/claircore/datastore/postgres"
+	"github.com/quay/claircore/libvuln/driver"
+	"github.com/quay/claircore/libvuln/updates"
+	"github.com/quay/claircore/test/integration"
+	pgtest "github.com/quay/claircore/test/postgres"
+	"github.com/quay/zlog"
+)
+
+var manuallyTestVulns = []*claircore.Vulnerability{
+	{
+		Updater:            "manual",
+		Name:               "GHSA-cj7v-27pg-wf7q",
+		Description:        "URI use within Jetty's HttpURI class can parse invalid URIs such as http://localhost;/path as having an authority with a host of localhost;A URIs of the type http://localhost;/path should be interpreted to be either invalid or as localhost; to be the userinfo and no host. However, HttpURI.host returns localhost; which is definitely wrong.",
+		Links:              "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2022/07/GHSA-cj7v-27pg-wf7q/GHSA-cj7v-27pg-wf7q.json",
+		Severity:           "CVSS:3.1/AV:N/AC:L/PR:H/UI:N/S:U/C:N/I:L/A:N",
+		NormalizedSeverity: claircore.Low,
+		Package: &claircore.Package{
+			Name: "org.eclipse.jetty:jetty-http",
+			Kind: claircore.BINARY,
+		},
+		FixedInVersion: "fixed=9.4.47&introduced=0",
+		Repo: &claircore.Repository{
+			Name: "maven",
+		},
+	}}
+
+func ManualUpdater(t *testing.T) {
+	ctx := context.Background()
+	updaters := make([]driver.Updater, 0, 1)
+
+	// Append updater sets directly to the updaters.
+	appendUpdaterSet := func(updaterSet driver.UpdaterSet, err error) {
+		if err != nil {
+			zlog.Error(ctx).Msg(err.Error())
+			return
+		}
+		updaters = append(updaters, updaterSet.Updaters()...)
+	}
+
+	integration.NeedDB(t)
+	pool := pgtest.TestMatcherDB(ctx, t)
+	store := postgres.NewMatcherStore(pool)
+	appendUpdaterSet(UpdaterSet(ctx, manuallyTestVulns))
+
+	updaterSetMgr, err := updates.NewManager(ctx, store, updates.NewLocalLockSource(), http.DefaultClient,
+		updates.WithOutOfTree(updaters),
+	)
+	if err != nil {
+		zlog.Error(ctx).Msg(err.Error())
+		return
+	}
+
+	if err = updaterSetMgr.Run(ctx); err != nil {
+		zlog.Error(ctx).Msg(err.Error())
+		return
+	}
+}

scanner/updater/version/RELEASE_VERSION

@@ -0,0 +1,9 @@
+# This file serves as a list of Stackrox versions.
+# As Scanner and Stackrox versions are aligned, the scanner updater is responsible for updating vulnerability data across all versions listed in this file.
+# Subsequently, this updated data is uploaded to Google Storage.
+# For each individual version of Scanner's vulnerability updater, corresponding vulnerability data is collected and then seamlessly uploaded to Google Storage.
+# This process ensures that each version's vulnerability data is accurately captured and maintained in accordance with its respective version.
+4.2.0
+4.1.x-476-g1aa6557244
+4.1.x-820-g3a14dfce97
+4.1.x-914-ga9d57ecef0