Rpm module vulnerability matching (#156)

* Parse rpm modules from oval files

Module name and stream is parsed from oval files and associated with
appropriate rpm packages. Module information is stored in vuln database
table.

* Add package module to MatchConstraint

New PackageModule is added to MatchConstraint. This constraint queries
vulnerability with matching module name.

When package is not part of a module it can be vulnerable only to
non-modular vulnerabilities.

* Add PackageModule constrain to rhel matcher

Rhel matcher now supports rpm module vulnerability. Only package vulnerabilities
within same module will be reported.

Other non-modular rpms will not be affected by this change because
non-modular rpms contains empty module.

* Create module regex only once

Module regex is created only once at a beginning.

* Add package_module to vuln index

package_module is used to query vulnerability.

* Use Package cache in oval parser

Cache key contains package and module name. The cache reduce size of
oval data.

internal/vulnstore/postgres/get.go

@@ -75,6 +75,7 @@ func get(ctx context.Context, pool *pgxpool.Pool, records []*claircore.IndexReco
 				&v.NormalizedSeverity,
 				&v.Package.Name,
 				&v.Package.Version,
+				&v.Package.Module,
 				&v.Package.Kind,
 				&v.Dist.DID,
 				&v.Dist.Name,

internal/vulnstore/postgres/getupdateoperationdiff.go

@@ -29,6 +29,7 @@ func getUpdateDiff(ctx context.Context, pool *pgxpool.Pool, prev, cur uuid.UUID)
 		normalized_severity,
 		package_name,
 		package_version,
+		package_module,
 		package_kind,
 		dist_id,
 		dist_name,

internal/vulnstore/postgres/querybuilder.go

@@ -43,6 +43,8 @@ func buildGetQuery(record *claircore.IndexRecord, opts *vulnstore.GetOpts) (stri
 		}
 		var ex goqu.Ex
 		switch m {
+		case driver.PackageModule:
+			ex = goqu.Ex{"package_module": record.Package.Module}
 		case driver.DistributionDID:
 			ex = goqu.Ex{"dist_id": record.Distribution.DID}
 		case driver.DistributionName:
@@ -92,6 +94,7 @@ func buildGetQuery(record *claircore.IndexRecord, opts *vulnstore.GetOpts) (stri
 		"normalized_severity",
 		"package_name",
 		"package_version",
+		"package_module",
 		"package_kind",
 		"dist_id",
 		"dist_name",

internal/vulnstore/postgres/querybuilder_test.go

@@ -18,7 +18,7 @@ func Test_GetQueryBuilder_Deterministic_Args(t *testing.T) {
 	const (
 		preamble = `SELECT
 		"id", "name", "description", "links", "severity", "normalized_severity", "package_name", "package_version",
-		"package_kind", "dist_id", "dist_name", "dist_version", "dist_version_code_name",
+		"package_module", "package_kind", "dist_id", "dist_name", "dist_version", "dist_version_code_name",
 		"dist_version_id", "dist_arch", "dist_cpe", "dist_pretty_name", "repo_name", "repo_key",
 		"repo_uri", "fixed_in_version", "updater"
 		FROM "vuln"
@@ -164,6 +164,21 @@ func Test_GetQueryBuilder_Deterministic_Args(t *testing.T) {
 				}
 			},
 		},
+		{
+			name: "module-filter",
+			expectedQuery: preamble + noSource +
+				`("package_module" = 'module:0'))`,
+			matchExps: []driver.MatchConstraint{driver.PackageModule},
+			indexRecord: func() *claircore.IndexRecord {
+				pkgs := test.GenUniquePackages(1)
+				pkgs[0].Source = &claircore.Package{} // clear source field
+				dists := test.GenUniqueDistributions(1)
+				return &claircore.IndexRecord{
+					Package:      pkgs[0],
+					Distribution: dists[0],
+				}
+			},
+		},
 	}
 
 	// This is safe to do because SQL doesn't care about what whitespace is

internal/vulnstore/postgres/scan_vulnerability.go

@@ -22,6 +22,7 @@ func scanVulnerability(v *claircore.Vulnerability, row scanner) error {
 		&v.NormalizedSeverity,
 		&v.Package.Name,
 		&v.Package.Version,
+		&v.Package.Module,
 		&v.Package.Kind,
 		&v.Dist.DID,
 		&v.Dist.Name,

internal/vulnstore/postgres/update_e2e_test.go

@@ -306,6 +306,7 @@ func checkInsertedVulns(ctx context.Context, t *testing.T, db *sqlx.DB, id uuid.
 	vuln.severity,
 	vuln.package_name,
 	vuln.package_version,
+	vuln.package_module,
 	vuln.package_kind,
 	vuln.dist_id,
 	vuln.dist_name,
@@ -354,6 +355,7 @@ WHERE uo.ref = $1::uuid;`
 			&vuln.Severity,
 			&vuln.Package.Name,
 			&vuln.Package.Version,
+			&vuln.Package.Module,
 			&vuln.Package.Kind,
 			&vuln.Dist.DID,
 			&vuln.Dist.Name,

internal/vulnstore/postgres/updatevulnerabilities.go

@@ -36,17 +36,17 @@ func updateVulnerabilites(ctx context.Context, pool *pgxpool.Pool, updater strin
 		attempt AS (
 			INSERT INTO vuln (
 				hash_kind, hash, name, updater, description, links, severity, normalized_severity, package_name, package_version,
-				package_kind, dist_id, dist_name, dist_version, dist_version_code_name, dist_version_id, dist_arch, dist_cpe,
-				dist_pretty_name, repo_name, repo_key, repo_uri, fixed_in_version, version_kind, vulnerable_range
+				package_module, package_kind, dist_id, dist_name, dist_version, dist_version_code_name, dist_version_id, dist_arch,
+				dist_cpe, dist_pretty_name, repo_name, repo_key, repo_uri, fixed_in_version, version_kind, vulnerable_range
 			) VALUES (
 			  $1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
 			  $11, $12, $13, $14, $15, $16, $17, $18,
-			  $19, $20, $21, $22, $23, $24, VersionRange($25, $26)
+			  $19, $20, $21, $22, $23, $24, $25, VersionRange($26, $27)
 			)
 			ON CONFLICT (hash_kind, hash) DO NOTHING
 			RETURNING id)
 		INSERT INTO uo_vuln (uo, vuln) VALUES (
-			$27,
+			$28,
 			COALESCE (
 				(SELECT id FROM attempt),
 				(SELECT id FROM vuln WHERE hash_kind = $1 AND hash = $2)))
@@ -95,7 +95,7 @@ func updateVulnerabilites(ctx context.Context, pool *pgxpool.Pool, updater strin
 		err := mBatcher.Queue(ctx, insert,
 			hashKind, hash,
 			vuln.Name, vuln.Updater, vuln.Description, vuln.Links, vuln.Severity, vuln.NormalizedSeverity,
-			pkg.Name, pkg.Version, pkg.Kind,
+			pkg.Name, pkg.Version, pkg.Module, pkg.Kind,
 			dist.DID, dist.Name, dist.Version, dist.VersionCodeName, dist.VersionID, dist.Arch, dist.CPE, dist.PrettyName,
 			repo.Name, repo.Key, repo.URI,
 			vuln.FixedInVersion, vKind, vrLower, vrUpper,
@@ -132,6 +132,7 @@ func md5Vuln(v *claircore.Vulnerability) (string, []byte) {
 	if v.Package != nil {
 		b.WriteString(v.Package.Name)
 		b.WriteString(v.Package.Version)
+		b.WriteString(v.Package.Module)
 		b.WriteString(v.Package.Kind)
 	}
 	if v.Dist != nil {

libvuln/driver/matcher.go

@@ -15,6 +15,8 @@ const (
 	PackageSourceName
 	// should match claircore.Package.Name => claircore.Vulnerability.Package.Name
 	PackageName
+	// should match claircore.Package.Module => claircore.Vulnerability.Package.Module
+	PackageModule
 	// should match claircore.Package.Distribution.DID => claircore.Vulnerability.Package.Distribution.DID
 	DistributionDID
 	// should match claircore.Package.Distribution.Name => claircore.Vulnerability.Package.Distribution.Name

libvuln/migrations/migration1.go

@@ -54,6 +54,7 @@ const (
 		normalized_severity    TEXT,
 		package_name           TEXT,
 		package_version        TEXT,
+		package_module         TEXT,
 		package_kind           TEXT,
 		dist_id                TEXT,
 		dist_name              TEXT,
@@ -75,7 +76,8 @@ const (
 	CREATE INDEX IF NOT EXISTS vuln_package_idx on vuln (
 		package_name,
 		package_kind,
-		package_version
+		package_version,
+		package_module
 	);
 	CREATE INDEX IF NOT EXISTS vuln_dist_idx on vuln (
 		dist_id,

pkg/ovalutil/rpm.go

@@ -2,12 +2,19 @@ package ovalutil
 
 import (
 	"context"
+	"regexp"
 
 	"github.com/quay/claircore"
 	"github.com/quay/goval-parser/oval"
 	"github.com/rs/zerolog"
 )
 
+var moduleComentRegex *regexp.Regexp
+
+func init() {
+	moduleComentRegex = regexp.MustCompile(`(Module )(.*)( is enabled)`)
+}
+
 // ProtoVulnFunc allows a caller to create a prototype vulnerability that will be used
 // copied and further defined for every applicable oval.Criterion discovered.
 //
@@ -37,6 +44,11 @@ func RPMDefsToVulns(ctx context.Context, root oval.Root, protoVuln ProtoVulnFunc
 		cris := cris[:0]
 		walkCriterion(ctx, &def.Criteria, &cris)
 		// unpack criterions into vulnerabilities
+		enabledModules := getEnabledModules(cris)
+		if len(enabledModules) == 0 {
+			// add default empty module
+			enabledModules = append(enabledModules, "")
+		}
 		for _, criterion := range cris {
 			// lookup test
 			_, index, err := root.Tests.Lookup(criterion.TestRef)
@@ -71,19 +83,24 @@ func RPMDefsToVulns(ctx context.Context, root oval.Root, protoVuln ProtoVulnFunc
 				if state.EVR == nil {
 					continue
 				}
-				// copy prototype
-				vuln := *protoVuln
-				if pkg, ok := pkgcache[object.Name]; !ok {
-					p := &claircore.Package{
-						Name: object.Name,
+
+				for _, module := range enabledModules {
+					vuln := *protoVuln
+					vuln.FixedInVersion = state.EVR.Body
+
+					pkgCacheKey := object.Name + module
+					if pkg, ok := pkgcache[pkgCacheKey]; !ok {
+						p := &claircore.Package{
+							Name:   object.Name,
+							Module: module,
+						}
+						pkgcache[pkgCacheKey] = p
+						vuln.Package = p
+					} else {
+						vuln.Package = pkg
 					}
-					pkgcache[object.Name] = p
-					vuln.Package = p
-				} else {
-					vuln.Package = pkg
+					vulns = append(vulns, &vuln)
 				}
-				vuln.FixedInVersion = state.EVR.Body
-				vulns = append(vulns, &vuln)
 			}
 		}
 	}
@@ -105,3 +122,15 @@ func walkCriterion(ctx context.Context, node *oval.Criteria, cris *[]*oval.Crite
 		*cris = append(*cris, &c)
 	}
 }
+
+func getEnabledModules(cris []*oval.Criterion) []string {
+	enabledModules := []string{}
+	for _, criterion := range cris {
+		matches := moduleComentRegex.FindStringSubmatch(criterion.Comment)
+		if matches != nil && len(matches) > 2 && matches[2] != "" {
+			moduleNameStream := matches[2]
+			enabledModules = append(enabledModules, moduleNameStream)
+		}
+	}
+	return enabledModules
+}

rhel/matcher.go

@@ -32,6 +32,7 @@ func (*Matcher) Query() []driver.MatchConstraint {
 		//driver.PackageDistributionCPE,
 		driver.DistributionName,
 		driver.DistributionPrettyName,
+		driver.PackageModule,
 	}
 }