From 66c3fbaa8f55c83b39994dfe41adb0577112835a Mon Sep 17 00:00:00 2001 From: Louis DeLosSantos Date: Mon, 10 Feb 2020 14:10:47 -0500 Subject: [PATCH] Louis/remove db coalescer --- .gitignore | 1 + alpine/ecosystem.go | 4 +- digest.go | 5 ++ dpkg/ecosystem.go | 4 +- internal/indexer/coalescer.go | 10 ++- internal/indexer/controller/coalesce.go | 59 ++++++++++++---- internal/indexer/ecosystem.go | 2 +- internal/indexer/linux/coalescer.go | 69 +++++-------------- internal/indexer/linux/coalescer_test.go | 63 ++++++++--------- .../indexer/postgres/distributionsbylayer.go | 3 + .../postgres/distributionsbylayer_test.go | 12 ++-- .../postgres/indexdistributions_test.go | 16 ++--- .../postgres/indexpackage_benchmark_test.go | 40 +++++------ .../indexer/postgres/indexpackage_test.go | 32 ++++----- internal/indexer/postgres/indexreport_test.go | 5 +- .../indexer/postgres/layerscanned_test.go | 36 +++++----- .../indexer/postgres/manifestscanned_test.go | 16 ++--- internal/indexer/postgres/packagesbylayer.go | 3 + .../packagesbylayer_benchmark_test.go | 22 +++--- .../indexer/postgres/packagesbylayer_test.go | 12 ++-- .../indexer/postgres/repositoriesbylayer.go | 3 + .../postgres/repositoriesbylayer_test.go | 12 ++-- .../indexer/postgres/setindexfinished_test.go | 3 +- .../indexer/postgres/setindexreport_test.go | 7 +- internal/indexer/versionedscanner.go | 38 ++++++---- rpm/ecosystem.go | 4 +- .../postgres_test.go => test/digest.go | 4 +- 27 files changed, 256 insertions(+), 229 deletions(-) rename internal/indexer/postgres/postgres_test.go => test/digest.go (82%) diff --git a/.gitignore b/.gitignore index 5d848c77d..feb46f9fd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ vendor +*.swp book diff --git a/alpine/ecosystem.go b/alpine/ecosystem.go index eb5e718c6..b09558929 100644 --- a/alpine/ecosystem.go +++ b/alpine/ecosystem.go @@ -20,8 +20,8 @@ func NewEcosystem(ctx context.Context) *indexer.Ecosystem { RepositoryScanners: func(ctx context.Context) ([]indexer.RepositoryScanner, error) { return []indexer.RepositoryScanner{}, nil }, - Coalescer: func(ctx context.Context, store indexer.Store) (indexer.Coalescer, error) { - return linux.NewCoalescer(store, &Scanner{}), nil + Coalescer: func(ctx context.Context) (indexer.Coalescer, error) { + return linux.NewCoalescer(), nil }, } } diff --git a/digest.go b/digest.go index ecc573352..709172b31 100644 --- a/digest.go +++ b/digest.go @@ -10,6 +10,11 @@ import ( "hash" ) +const ( + SHA256 = "sha256" + SHA512 = "sha512" +) + // Digest is a type representing the hash of some data. // // It's used throughout claircore packages as an attempt to remain independent diff --git a/dpkg/ecosystem.go b/dpkg/ecosystem.go index 1e25c9778..6eae128b2 100644 --- a/dpkg/ecosystem.go +++ b/dpkg/ecosystem.go @@ -24,8 +24,8 @@ func NewEcosystem(ctx context.Context) *indexer.Ecosystem { RepositoryScanners: func(ctx context.Context) ([]indexer.RepositoryScanner, error) { return []indexer.RepositoryScanner{}, nil }, - Coalescer: func(ctx context.Context, store indexer.Store) (indexer.Coalescer, error) { - return linux.NewCoalescer(store, &Scanner{}), nil + Coalescer: func(ctx context.Context) (indexer.Coalescer, error) { + return linux.NewCoalescer(), nil }, } } diff --git a/internal/indexer/coalescer.go b/internal/indexer/coalescer.go index 1bab68f46..17bc168b2 100644 --- a/internal/indexer/coalescer.go +++ b/internal/indexer/coalescer.go @@ -6,10 +6,18 @@ import ( "github.com/quay/claircore" ) +// layerArifact aggregates the any artifacts found within a layer +type LayerArtifacts struct { + Hash claircore.Digest + Pkgs []*claircore.Package + Dist []*claircore.Distribution // each layer can only have a single distribution + Repos []*claircore.Repository +} + // Coalescer takes a set of layers and creates coalesced IndexReport. // // A coalesced IndexReport should provide only the packages present in the // final container image once all layers were applied. type Coalescer interface { - Coalesce(ctx context.Context, layers []*claircore.Layer) (*claircore.IndexReport, error) + Coalesce(ctx context.Context, artifacts []*LayerArtifacts) (*claircore.IndexReport, error) } diff --git a/internal/indexer/controller/coalesce.go b/internal/indexer/controller/coalesce.go index 1a84e117d..40ee12666 100644 --- a/internal/indexer/controller/coalesce.go +++ b/internal/indexer/controller/coalesce.go @@ -18,22 +18,55 @@ func coalesce(ctx context.Context, s *Controller) (State, error) { Str("state", s.getState().String()). Logger() ctx = log.WithContext(ctx) - coalescers := []indexer.Coalescer{} + cctx, cancel := context.WithCancel(ctx) + defer cancel() + mu := sync.Mutex{} + reports := []*claircore.IndexReport{} + g := errgroup.Group{} + // dispatch a coalescer go routine for each ecosystem for _, ecosystem := range s.Ecosystems { - c, err := ecosystem.Coalescer(ctx, s.Store) + artifacts := []*indexer.LayerArtifacts{} + pkgScanners, _ := ecosystem.PackageScanners(cctx) + distScanners, _ := ecosystem.DistributionScanners(cctx) + repoScanners, _ := ecosystem.RepositoryScanners(cctx) + // pack artifacts var + for _, layer := range s.manifest.Layers { + la := &indexer.LayerArtifacts{ + Hash: layer.Hash, + } + var vscnrs indexer.VersionedScanners + vscnrs.PStoVS(pkgScanners) + // get packages from layer + pkgs, err := s.Store.PackagesByLayer(cctx, layer.Hash, vscnrs) + if err != nil { + // on an early return cctx is canceled, and all inflight coalescers are canceled as well + return Terminal, fmt.Errorf("failed to retrieve packages for %v: %v", layer.Hash, err) + } + la.Pkgs = append(la.Pkgs, pkgs...) + // get distributions from layer + vscnrs.DStoVS(distScanners) // method allocates new vscnr underlying array, clearing old contents + dists, err := s.Store.DistributionsByLayer(cctx, layer.Hash, vscnrs) + if err != nil { + return Terminal, fmt.Errorf("failed to retrieve distributions for %v: %v", layer.Hash, err) + } + la.Dist = append(la.Dist, dists...) + // get repositories from layer + vscnrs.RStoVS(repoScanners) + repos, err := s.Store.RepositoriesByLayer(cctx, layer.Hash, vscnrs) + if err != nil { + return Terminal, fmt.Errorf("failed to retrieve repositories for %v: %v", layer.Hash, err) + } + la.Repos = append(la.Repos, repos...) + // pack artifacts array in layer order + artifacts = append(artifacts, la) + } + coalescer, err := ecosystem.Coalescer(cctx) if err != nil { - return Terminal, fmt.Errorf("failed to create coalescer: %v", err) + return Terminal, fmt.Errorf("failed to get coalescer from ecosystem: %v", err) } - coalescers = append(coalescers, c) - } - - mu := sync.Mutex{} - reports := []*claircore.IndexReport{} - g, gctx := errgroup.WithContext(ctx) - for _, c := range coalescers { - cc := c + // dispatch coalescer g.Go(func() error { - sr, err := cc.Coalesce(gctx, s.manifest.Layers) + sr, err := coalescer.Coalesce(cctx, artifacts) if err != nil { return err } @@ -47,9 +80,7 @@ func coalesce(ctx context.Context, s *Controller) (State, error) { if err := g.Wait(); err != nil { return Terminal, err } - s.report = MergeSR(s.report, reports) - return IndexFinished, nil } diff --git a/internal/indexer/ecosystem.go b/internal/indexer/ecosystem.go index 48e329c44..817c0e7e0 100644 --- a/internal/indexer/ecosystem.go +++ b/internal/indexer/ecosystem.go @@ -13,7 +13,7 @@ type Ecosystem struct { PackageScanners func(ctx context.Context) ([]PackageScanner, error) DistributionScanners func(ctx context.Context) ([]DistributionScanner, error) RepositoryScanners func(ctx context.Context) ([]RepositoryScanner, error) - Coalescer func(ctx context.Context, store Store) (Coalescer, error) + Coalescer func(ctx context.Context) (Coalescer, error) } // EcosystemsToScanners extracts and dedupes multiple ecosystems and returns their discrete scanners diff --git a/internal/indexer/linux/coalescer.go b/internal/indexer/linux/coalescer.go index a763797f6..1e3b375d1 100644 --- a/internal/indexer/linux/coalescer.go +++ b/internal/indexer/linux/coalescer.go @@ -5,7 +5,6 @@ import ( "github.com/quay/claircore" "github.com/quay/claircore/internal/indexer" - "github.com/quay/claircore/osrelease" ) // layerArifact aggregates the any artifacts found within a layer @@ -22,19 +21,13 @@ type layerArtifacts struct { // It is expected to run a coalescer per "ecosystem". For example it would make sense to coalesce results // for dpkg, os-release, and apt scanners type Coalescer struct { - // a store to access scanartifacts - store indexer.Store - ps indexer.PackageScanner - ds indexer.DistributionScanner - ir *claircore.IndexReport + // the IndexReport this Coalescer is working on + ir *claircore.IndexReport } // NewCoalescer is a constructor for a Coalescer -func NewCoalescer(store indexer.Store, ps indexer.PackageScanner) *Coalescer { +func NewCoalescer() *Coalescer { return &Coalescer{ - store: store, - ps: ps, - ds: &osrelease.Scanner{}, ir: &claircore.IndexReport{ // we will only fill these fields Environments: map[string][]*claircore.Environment{}, @@ -48,37 +41,9 @@ func NewCoalescer(store indexer.Store, ps indexer.PackageScanner) *Coalescer { // Coalesce coalesces artifacts found in layers and creates a final IndexReport with // the final package details found in the image. This method blocks and when its finished // the c.ir field will hold the final IndexReport -func (c *Coalescer) Coalesce(ctx context.Context, layers []*claircore.Layer) (*claircore.IndexReport, error) { - var err error - // populate layer artifacts - artifacts := []layerArtifacts{} - for _, layer := range layers { - a := layerArtifacts{ - hash: layer.Hash, - } - - a.pkgs, err = c.store.PackagesByLayer(ctx, layer.Hash, indexer.VersionedScanners{c.ps}) - if err != nil { - return nil, err - } - - a.dist, err = c.store.DistributionsByLayer(ctx, layer.Hash, indexer.VersionedScanners{c.ds}) - if err != nil { - return nil, err - } - - artifacts = append(artifacts, a) - } - err = c.coalesce(ctx, artifacts) - return c.ir, err -} - -// coalesce performs the business logic of coalescing context free scanned artifacts -// into a penultimate IndexReport. this method is heavily commented to express -// the reasoning and assumptions. -func (c *Coalescer) coalesce(ctx context.Context, artifacts []layerArtifacts) error { +func (c *Coalescer) Coalesce(ctx context.Context, artifacts []*indexer.LayerArtifacts) (*claircore.IndexReport, error) { if ctx.Err() != nil { - return ctx.Err() + return nil, ctx.Err() } // In our coalescing logic if a Distribution is found in layer (n) all packages found // in layers 0-(n) will be associated with this layer. This is a heuristic. @@ -89,8 +54,8 @@ func (c *Coalescer) coalesce(ctx context.Context, artifacts []layerArtifacts) er // This is a requirement for handling dist upgrades where a layer may have it's operating system updated var currDist *claircore.Distribution for _, a := range artifacts { - if len(a.dist) != 0 { - currDist = a.dist[0] + if len(a.Dist) != 0 { + currDist = a.Dist[0] c.ir.Distributions[currDist.ID] = currDist break } @@ -108,13 +73,13 @@ func (c *Coalescer) coalesce(ctx context.Context, artifacts []layerArtifacts) er // creating the environments we discover packages in. for _, layerArtifacts := range artifacts { // check if we need to update our currDist - if len(layerArtifacts.dist) != 0 { - currDist = layerArtifacts.dist[0] + if len(layerArtifacts.Dist) != 0 { + currDist = layerArtifacts.Dist[0] c.ir.Distributions[currDist.ID] = currDist } // associate packages with their environments - if len(layerArtifacts.pkgs) != 0 { - for _, pkg := range layerArtifacts.pkgs { + if len(layerArtifacts.Pkgs) != 0 { + for _, pkg := range layerArtifacts.Pkgs { // if we encounter a package where we haven't recorded a package database, // initialize the package database var distID string @@ -125,7 +90,7 @@ func (c *Coalescer) coalesce(ctx context.Context, artifacts []layerArtifacts) er packages := map[string]*claircore.Package{pkg.ID: pkg} environment := &claircore.Environment{ PackageDB: pkg.PackageDB, - IntroducedIn: layerArtifacts.hash, + IntroducedIn: layerArtifacts.Hash, DistributionID: distID, } environments := map[string]*claircore.Environment{pkg.ID: environment} @@ -135,7 +100,7 @@ func (c *Coalescer) coalesce(ctx context.Context, artifacts []layerArtifacts) er if _, ok := dbs[pkg.PackageDB].packages[pkg.ID]; !ok { environment := &claircore.Environment{ PackageDB: pkg.PackageDB, - IntroducedIn: layerArtifacts.hash, + IntroducedIn: layerArtifacts.Hash, DistributionID: distID, } dbs[pkg.PackageDB].packages[pkg.ID] = pkg @@ -145,7 +110,7 @@ func (c *Coalescer) coalesce(ctx context.Context, artifacts []layerArtifacts) er } } if ctx.Err() != nil { - return ctx.Err() + return nil, ctx.Err() } // we now have all the packages associated with their introduced in layers and environments. // we must now prune any packages removed between layers. this coalescer works on the assumption @@ -163,12 +128,12 @@ func (c *Coalescer) coalesce(ctx context.Context, artifacts []layerArtifacts) er var packagesToKeep = map[string][]string{} for i := len(artifacts) - 1; i >= 0; i-- { layerArtifacts := artifacts[i] - if len(layerArtifacts.pkgs) == 0 { + if len(layerArtifacts.Pkgs) == 0 { continue } // used as a temporary accumulator of package ids in this layer var tmpPackagesToKeep = map[string][]string{} - for _, pkg := range layerArtifacts.pkgs { + for _, pkg := range layerArtifacts.Pkgs { // have we already inventoried packages from this database ? if _, ok := packagesToKeep[pkg.PackageDB]; !ok { // ... we haven't so add to our temporary accumlator @@ -203,5 +168,5 @@ func (c *Coalescer) coalesce(ctx context.Context, artifacts []layerArtifacts) er c.ir.Environments[pkg.ID] = append(c.ir.Environments[pkg.ID], db.environments[pkg.ID]) } } - return nil + return c.ir, nil } diff --git a/internal/indexer/linux/coalescer_test.go b/internal/indexer/linux/coalescer_test.go index 98c66a6e9..3d8bbb498 100644 --- a/internal/indexer/linux/coalescer_test.go +++ b/internal/indexer/linux/coalescer_test.go @@ -1,13 +1,12 @@ package linux import ( - "bytes" "context" - "crypto/sha256" "strconv" "testing" "github.com/quay/claircore" + "github.com/quay/claircore/internal/indexer" "github.com/quay/claircore/test" "github.com/quay/claircore/test/log" ) @@ -21,9 +20,6 @@ func Test_Coalescer(t *testing.T) { defer done() ctx = log.TestLogger(ctx, t) coalescer := &Coalescer{ - store: nil, - ps: nil, - ds: nil, ir: &claircore.IndexReport{ Environments: map[string][]*claircore.Environment{}, Packages: map[string]*claircore.Package{}, @@ -38,59 +34,58 @@ func Test_Coalescer(t *testing.T) { // are tagged wih this distribution pkgs := test.GenUniquePackages(6) dists := test.GenUniqueDistributions(3) // we will discard dist 0 due to zero value ambiguity - layerArtifacts := []layerArtifacts{ + layerArtifacts := []*indexer.LayerArtifacts{ { - pkgs: pkgs[0:1], - dist: nil, - repos: nil, + Hash: test.RandomSHA256Digest(t), + Pkgs: pkgs[0:1], + Dist: nil, + Repos: nil, }, { - pkgs: pkgs[1:2], - dist: nil, - repos: nil, + Hash: test.RandomSHA256Digest(t), + Pkgs: pkgs[1:2], + Dist: nil, + Repos: nil, }, { - pkgs: pkgs[2:3], - dist: dists[1:2], - repos: nil, + Hash: test.RandomSHA256Digest(t), + Pkgs: pkgs[2:3], + Dist: dists[1:2], + Repos: nil, }, { - pkgs: pkgs[3:4], - dist: nil, - repos: nil, + Hash: test.RandomSHA256Digest(t), + Pkgs: pkgs[3:4], + Dist: nil, + Repos: nil, }, { - pkgs: pkgs[4:5], - dist: dists[2:], - repos: nil, + Hash: test.RandomSHA256Digest(t), + Pkgs: pkgs[4:5], + Dist: dists[2:], + Repos: nil, }, { - pkgs: pkgs[5:], - dist: nil, - repos: nil, + Hash: test.RandomSHA256Digest(t), + Pkgs: pkgs[5:], + Dist: nil, + Repos: nil, }, } - var err error - for i := range layerArtifacts { - layerArtifacts[i].hash, err = claircore.NewDigest("sha256", bytes.Repeat([]byte{uint8(i)}, sha256.Size)) - if err != nil { - t.Fatal(err) - } - } - err = coalescer.coalesce(ctx, layerArtifacts) + ir, err := coalescer.Coalesce(ctx, layerArtifacts) if err != nil { t.Fatalf("received error from coalesce method: %v", err) } // we expect packages 1-4 to be tagged with dist id 1 // and packages 5-6 to be tagged with dist id 2 for i := 0; i < 4; i++ { - environment := coalescer.ir.Environments[strconv.Itoa(i)][0] + environment := ir.Environments[strconv.Itoa(i)][0] if environment.DistributionID != "1" { t.Fatalf("expected distribution id %d but got %s", 1, environment.DistributionID) } } for i := 4; i < 6; i++ { - environment := coalescer.ir.Environments[strconv.Itoa(i)][0] + environment := ir.Environments[strconv.Itoa(i)][0] if environment.DistributionID != "2" { t.Fatalf("expected distribution id %d but got %s", 2, environment.DistributionID) } diff --git a/internal/indexer/postgres/distributionsbylayer.go b/internal/indexer/postgres/distributionsbylayer.go index dc1141e0c..9c66b09b7 100644 --- a/internal/indexer/postgres/distributionsbylayer.go +++ b/internal/indexer/postgres/distributionsbylayer.go @@ -32,6 +32,9 @@ WHERE func distributionsByLayer(ctx context.Context, db *sqlx.DB, hash claircore.Digest, scnrs indexer.VersionedScanners) ([]*claircore.Distribution, error) { // TODO Use passed-in Context. + if len(scnrs) == 0 { + return []*claircore.Distribution{}, nil + } // get scanner ids scannerIDs := []int64{} for _, scnr := range scnrs { diff --git a/internal/indexer/postgres/distributionsbylayer_test.go b/internal/indexer/postgres/distributionsbylayer_test.go index 334ef0c1c..173f4a742 100644 --- a/internal/indexer/postgres/distributionsbylayer_test.go +++ b/internal/indexer/postgres/distributionsbylayer_test.go @@ -30,37 +30,37 @@ func Test_DistributionsByLayer_Success(t *testing.T) { }{ { name: "10 dists, 5 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), dists: 10, scnrs: 5, }, { name: "50 distss, 25 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), dists: 50, scnrs: 25, }, { name: "100 distss, 50 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), dists: 100, scnrs: 50, }, { name: "500 distss, 250 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), dists: 500, scnrs: 250, }, { name: "1000 distss, 500 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), dists: 1000, scnrs: 500, }, { name: "2000 distss, 1000 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), dists: 2000, scnrs: 1000, }, diff --git a/internal/indexer/postgres/indexdistributions_test.go b/internal/indexer/postgres/indexdistributions_test.go index 29c54b1f8..9dcb484df 100644 --- a/internal/indexer/postgres/indexdistributions_test.go +++ b/internal/indexer/postgres/indexdistributions_test.go @@ -31,56 +31,56 @@ func Test_IndexDistributions_Success(t *testing.T) { name: "10 packages", dists: 10, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "50 packages", dists: 50, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "100 packages", dists: 100, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "250 packages", dists: 250, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "500 packages", dists: 500, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "1000 packages", dists: 1000, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "2000 packages", dists: 2000, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "3000 packages", dists: 3000, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, } diff --git a/internal/indexer/postgres/indexpackage_benchmark_test.go b/internal/indexer/postgres/indexpackage_benchmark_test.go index 8debdc369..52b7387ca 100644 --- a/internal/indexer/postgres/indexpackage_benchmark_test.go +++ b/internal/indexer/postgres/indexpackage_benchmark_test.go @@ -29,14 +29,14 @@ func Benchmark_IndexPackages(b *testing.B) { name: "10 packages", pkgs: 10, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, }, { name: "10 packages with duplicates", pkgs: 10, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, duplicates: true, }, @@ -44,14 +44,14 @@ func Benchmark_IndexPackages(b *testing.B) { name: "50 packages", pkgs: 50, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, }, { name: "50 packages with duplicates", pkgs: 50, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, duplicates: true, }, @@ -59,14 +59,14 @@ func Benchmark_IndexPackages(b *testing.B) { name: "100 packages", pkgs: 100, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, }, { name: "100 packages with duplicates", pkgs: 100, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, duplicates: true, }, @@ -74,14 +74,14 @@ func Benchmark_IndexPackages(b *testing.B) { name: "250 packages", pkgs: 250, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, }, { name: "250 packages", pkgs: 250, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, duplicates: true, }, @@ -89,14 +89,14 @@ func Benchmark_IndexPackages(b *testing.B) { name: "500 packages", pkgs: 500, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, }, { name: "500 packages with duplicates", pkgs: 500, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, duplicates: true, }, @@ -104,14 +104,14 @@ func Benchmark_IndexPackages(b *testing.B) { name: "1000 packages", pkgs: 1000, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, }, { name: "1000 packages with duplicates", pkgs: 1000, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, duplicates: true, }, @@ -119,14 +119,14 @@ func Benchmark_IndexPackages(b *testing.B) { name: "2000 packages", pkgs: 2000, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, }, { name: "2000 packages with duplicates", pkgs: 2000, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, duplicates: true, }, @@ -134,14 +134,14 @@ func Benchmark_IndexPackages(b *testing.B) { name: "3000 packages", pkgs: 3000, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, }, { name: "3000 packages with duplicates", pkgs: 3000, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, duplicates: true, }, @@ -149,14 +149,14 @@ func Benchmark_IndexPackages(b *testing.B) { name: "4000 packages", pkgs: 4000, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, }, { name: "4000 packages with duplicates", pkgs: 4000, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, duplicates: true, }, @@ -164,14 +164,14 @@ func Benchmark_IndexPackages(b *testing.B) { name: "5000 packages", pkgs: 5000, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, }, { name: "5000 packages with duplicates", pkgs: 5000, layer: &claircore.Layer{ - Hash: randomHash(b), + Hash: test.RandomSHA256Digest(b), }, duplicates: true, }, diff --git a/internal/indexer/postgres/indexpackage_test.go b/internal/indexer/postgres/indexpackage_test.go index 9a718c37b..3631ab380 100644 --- a/internal/indexer/postgres/indexpackage_test.go +++ b/internal/indexer/postgres/indexpackage_test.go @@ -40,56 +40,56 @@ func Test_IndexPackages_Success_Parallel(t *testing.T) { name: "10 packages", pkgs: 10, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "50 packages", pkgs: 50, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "100 packages", pkgs: 100, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "250 packages", pkgs: 250, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "500 packages", pkgs: 500, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "1000 packages", pkgs: 1000, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "2000 packages", pkgs: 2000, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "3000 packages", pkgs: 3000, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, } @@ -142,56 +142,56 @@ func Test_IndexPackages_Success(t *testing.T) { name: "10 packages", pkgs: 10, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "50 packages", pkgs: 50, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "100 packages", pkgs: 100, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "250 packages", pkgs: 250, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "500 packages", pkgs: 500, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "1000 packages", pkgs: 1000, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "2000 packages", pkgs: 2000, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, { name: "3000 packages", pkgs: 3000, layer: &claircore.Layer{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), }, }, } diff --git a/internal/indexer/postgres/indexreport_test.go b/internal/indexer/postgres/indexreport_test.go index 8e1a9957b..e060382b4 100644 --- a/internal/indexer/postgres/indexreport_test.go +++ b/internal/indexer/postgres/indexreport_test.go @@ -8,6 +8,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/quay/claircore" + "github.com/quay/claircore/test" "github.com/quay/claircore/test/integration" "github.com/quay/claircore/test/log" ) @@ -29,9 +30,9 @@ func Test_IndexReport_Success(t *testing.T) { }{ { name: "full scan result", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), expectedSR: &claircore.IndexReport{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), State: "test-state", Success: true, Err: "", diff --git a/internal/indexer/postgres/layerscanned_test.go b/internal/indexer/postgres/layerscanned_test.go index 838adead0..f12ac75e1 100644 --- a/internal/indexer/postgres/layerscanned_test.go +++ b/internal/indexer/postgres/layerscanned_test.go @@ -27,19 +27,19 @@ func Test_LayerScanned_Packages_False(t *testing.T) { }{ { name: "single scanner, single package", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 1, pkgs: 1, }, { name: "4 scanners, 4 packages", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 4, pkgs: 4, }, { name: "4 scanners, 8 packages", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 4, pkgs: 8, }, @@ -91,19 +91,19 @@ func Test_LayerScanned_Distributions_False(t *testing.T) { }{ { name: "single scanner, single distribution", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 1, dists: 1, }, { name: "4 scanners, 4 distributions", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 4, dists: 4, }, { name: "4 scanners, 8 distributions", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 4, dists: 8, }, @@ -156,19 +156,19 @@ func Test_LayerScanned_Repository_False(t *testing.T) { }{ { name: "single scanner, single repositories", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 1, repos: 1, }, { name: "4 scanners, 4 repositories", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 4, repos: 4, }, { name: "4 scanners, 8 repositories", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 4, repos: 8, }, @@ -220,19 +220,19 @@ func Test_LayerScanned_Packages_True(t *testing.T) { }{ { name: "single scanner, single package", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 1, pkgs: 1, }, { name: "4 scanners, 4 packages", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 4, pkgs: 4, }, { name: "4 scanners, 8 packages", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 4, pkgs: 8, }, @@ -289,19 +289,19 @@ func Test_LayerScanned_Distribution_True(t *testing.T) { }{ { name: "single scanner, single package", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 1, dists: 1, }, { name: "4 scanners, 4 distributions", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 4, dists: 4, }, { name: "4 scanners, 8 distributions", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 4, dists: 8, }, @@ -358,19 +358,19 @@ func Test_LayerScanned_Repository_True(t *testing.T) { }{ { name: "single scanner, single repository", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 1, dists: 1, }, { name: "4 scanners, 4 repository", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 4, dists: 4, }, { name: "4 scanners, 8 repository", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scnrs: 4, dists: 8, }, diff --git a/internal/indexer/postgres/manifestscanned_test.go b/internal/indexer/postgres/manifestscanned_test.go index 140e528e4..2bbf44774 100644 --- a/internal/indexer/postgres/manifestscanned_test.go +++ b/internal/indexer/postgres/manifestscanned_test.go @@ -28,22 +28,22 @@ func Test_ManifestScanned_Failure(t *testing.T) { var tt = []scannerTestcase{ { name: "one scanner", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scanners: 1, }, { name: "two scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scanners: 2, }, { name: "five scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scanners: 5, }, { name: "ten scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scanners: 10, }, } @@ -81,22 +81,22 @@ func Test_ManifestScanned_Success(t *testing.T) { var tt = []scannerTestcase{ { name: "one scanner", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scanners: 1, }, { name: "two scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scanners: 2, }, { name: "five scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scanners: 5, }, { name: "ten scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), scanners: 10, }, } diff --git a/internal/indexer/postgres/packagesbylayer.go b/internal/indexer/postgres/packagesbylayer.go index 019b8b59b..75ac396d7 100644 --- a/internal/indexer/postgres/packagesbylayer.go +++ b/internal/indexer/postgres/packagesbylayer.go @@ -37,6 +37,9 @@ WHERE func packagesByLayer(ctx context.Context, db *sqlx.DB, hash claircore.Digest, scnrs indexer.VersionedScanners) ([]*claircore.Package, error) { // TODO Use passed-in Context. + if len(scnrs) == 0 { + return []*claircore.Package{}, nil + } // get scanner ids scannerIDs := []int64{} for _, scnr := range scnrs { diff --git a/internal/indexer/postgres/packagesbylayer_benchmark_test.go b/internal/indexer/postgres/packagesbylayer_benchmark_test.go index 4e3dfd714..cf1dc8783 100644 --- a/internal/indexer/postgres/packagesbylayer_benchmark_test.go +++ b/internal/indexer/postgres/packagesbylayer_benchmark_test.go @@ -23,67 +23,67 @@ func Benchmark_PackagesByLayer(b *testing.B) { }{ { name: "10 package, 5 scanners", - hash: randomHash(b), + hash: test.RandomSHA256Digest(b), pkgs: 10, scnrs: 5, }, { name: "50 packages, 25 scanners", - hash: randomHash(b), + hash: test.RandomSHA256Digest(b), pkgs: 50, scnrs: 25, }, { name: "100 packages, 50 scanners", - hash: randomHash(b), + hash: test.RandomSHA256Digest(b), pkgs: 100, scnrs: 50, }, { name: "500 packages, 250 scanners", - hash: randomHash(b), + hash: test.RandomSHA256Digest(b), pkgs: 500, scnrs: 250, }, { name: "1000 packages, 500 scanners", - hash: randomHash(b), + hash: test.RandomSHA256Digest(b), pkgs: 1000, scnrs: 500, }, { name: "2000 packages, 1000 scanners", - hash: randomHash(b), + hash: test.RandomSHA256Digest(b), pkgs: 2000, scnrs: 1000, }, { name: "3000 packages, 2000 scanners", - hash: randomHash(b), + hash: test.RandomSHA256Digest(b), pkgs: 3000, scnrs: 1000, }, { name: "3000 packages, 500 scanners", - hash: randomHash(b), + hash: test.RandomSHA256Digest(b), pkgs: 3000, scnrs: 500, }, { name: "3000 packages, 250 scanners", - hash: randomHash(b), + hash: test.RandomSHA256Digest(b), pkgs: 3000, scnrs: 250, }, { name: "3000 packages, 50 scanners", - hash: randomHash(b), + hash: test.RandomSHA256Digest(b), pkgs: 2000, scnrs: 50, }, { name: "3000 packages, 10 scanners", - hash: randomHash(b), + hash: test.RandomSHA256Digest(b), pkgs: 2000, scnrs: 10, }, diff --git a/internal/indexer/postgres/packagesbylayer_test.go b/internal/indexer/postgres/packagesbylayer_test.go index ab17d90c1..7843104b8 100644 --- a/internal/indexer/postgres/packagesbylayer_test.go +++ b/internal/indexer/postgres/packagesbylayer_test.go @@ -30,37 +30,37 @@ func Test_PackagesByLayer_Success(t *testing.T) { }{ { name: "10 package, 5 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), pkgs: 10, scnrs: 5, }, { name: "50 packages, 25 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), pkgs: 50, scnrs: 25, }, { name: "100 packages, 50 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), pkgs: 100, scnrs: 50, }, { name: "500 packages, 250 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), pkgs: 500, scnrs: 250, }, { name: "1000 packages, 500 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), pkgs: 1000, scnrs: 500, }, { name: "2000 packages, 1000 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), pkgs: 2000, scnrs: 1000, }, diff --git a/internal/indexer/postgres/repositoriesbylayer.go b/internal/indexer/postgres/repositoriesbylayer.go index a7195e1cb..2014d4131 100644 --- a/internal/indexer/postgres/repositoriesbylayer.go +++ b/internal/indexer/postgres/repositoriesbylayer.go @@ -27,6 +27,9 @@ WHERE func repositoriesByLayer(ctx context.Context, db *sqlx.DB, hash claircore.Digest, scnrs indexer.VersionedScanners) ([]*claircore.Repository, error) { // TODO Use passed-in Context. + if len(scnrs) == 0 { + return []*claircore.Repository{}, nil + } // get scanner ids scannerIDs := []int64{} for _, scnr := range scnrs { diff --git a/internal/indexer/postgres/repositoriesbylayer_test.go b/internal/indexer/postgres/repositoriesbylayer_test.go index 91fa259e8..39a6c4d29 100644 --- a/internal/indexer/postgres/repositoriesbylayer_test.go +++ b/internal/indexer/postgres/repositoriesbylayer_test.go @@ -30,37 +30,37 @@ func Test_RepositoriesByLayer_Success(t *testing.T) { }{ { name: "10 repos, 5 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), repos: 10, scnrs: 5, }, { name: "50 repos, 25 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), repos: 50, scnrs: 25, }, { name: "100 repos, 50 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), repos: 100, scnrs: 50, }, { name: "500 repos, 250 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), repos: 500, scnrs: 250, }, { name: "1000 repos, 500 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), repos: 1000, scnrs: 500, }, { name: "2000 repos, 1000 scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), repos: 2000, scnrs: 1000, }, diff --git a/internal/indexer/postgres/setindexfinished_test.go b/internal/indexer/postgres/setindexfinished_test.go index 1e43c31d3..997f3e076 100644 --- a/internal/indexer/postgres/setindexfinished_test.go +++ b/internal/indexer/postgres/setindexfinished_test.go @@ -9,6 +9,7 @@ import ( "github.com/quay/claircore" "github.com/quay/claircore/internal/indexer" + "github.com/quay/claircore/test" "github.com/quay/claircore/test/integration" "github.com/quay/claircore/test/log" ) @@ -56,7 +57,7 @@ func Test_SetScanFinished_Success(t *testing.T) { }{ { name: "no previous scanners", - hash: randomHash(t), + hash: test.RandomSHA256Digest(t), previousScnrs: []scnrInfo{}, updatedScnrs: []scnrInfo{ scnrInfo{ diff --git a/internal/indexer/postgres/setindexreport_test.go b/internal/indexer/postgres/setindexreport_test.go index 4f9052d97..e31a2e42f 100644 --- a/internal/indexer/postgres/setindexreport_test.go +++ b/internal/indexer/postgres/setindexreport_test.go @@ -7,6 +7,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/quay/claircore" + "github.com/quay/claircore/test" "github.com/quay/claircore/test/integration" "github.com/quay/claircore/test/log" ) @@ -16,7 +17,7 @@ func Test_SetIndexReport_StateUpdate(t *testing.T) { ctx, done := context.WithCancel(context.Background()) defer done() - hash := randomHash(t) + hash := test.RandomSHA256Digest(t) var tt = []struct { // the name of the test name string @@ -90,7 +91,7 @@ func Test_SetIndexReport_Success(t *testing.T) { { name: "single package. no nested source", sr: &claircore.IndexReport{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), State: "test-state", Success: true, Err: "", @@ -99,7 +100,7 @@ func Test_SetIndexReport_Success(t *testing.T) { { name: "single package nested source", sr: &claircore.IndexReport{ - Hash: randomHash(t), + Hash: test.RandomSHA256Digest(t), State: "test-state", Success: true, Err: "", diff --git a/internal/indexer/versionedscanner.go b/internal/indexer/versionedscanner.go index a89a8a18e..bf1fe0cd8 100644 --- a/internal/indexer/versionedscanner.go +++ b/internal/indexer/versionedscanner.go @@ -19,14 +19,16 @@ type VersionedScanner interface { // not concurrency safe type VersionedScanners []VersionedScanner -// PStoVS takes an array of PackageScanners and appends VersionedScanners with -// VersionScanner types. func (vs *VersionedScanners) PStoVS(scnrs []PackageScanner) { - temp := make([]VersionedScanner, 0) - for _, scnr := range scnrs { - temp = append(temp, scnr) + n := len(scnrs) + if cap(*vs) < n { + *vs = make([]VersionedScanner, n) + } else { + *vs = (*vs)[:n] + } + for i := 0; i < n; i++ { + (*vs)[i] = scnrs[i] } - *vs = temp } // VStoPS returns an array of PackageScanners @@ -41,11 +43,15 @@ func (vs VersionedScanners) VStoPS() []PackageScanner { // DStoVS takes an array of DistributionScanners and appends VersionedScanners with // VersionScanner types. func (vs *VersionedScanners) DStoVS(scnrs []DistributionScanner) { - temp := make([]VersionedScanner, 0) - for _, scnr := range scnrs { - temp = append(temp, scnr) + n := len(scnrs) + if cap(*vs) < n { + *vs = make([]VersionedScanner, n) + } else { + *vs = (*vs)[:n] + } + for i := 0; i < n; i++ { + (*vs)[i] = scnrs[i] } - *vs = temp } // VStoDS returns an array of DistributionScanners @@ -60,11 +66,15 @@ func (vs VersionedScanners) VStoDS() []DistributionScanner { // RStoVS takes an array of RepositoryScanners and appends VersionedScanners with // VersionScanner types. func (vs *VersionedScanners) RStoVS(scnrs []RepositoryScanner) { - temp := make([]VersionedScanner, 0) - for _, scnr := range scnrs { - temp = append(temp, scnr) + n := len(scnrs) + if cap(*vs) < n { + *vs = make([]VersionedScanner, n) + } else { + *vs = (*vs)[:n] + } + for i := 0; i < n; i++ { + (*vs)[i] = scnrs[i] } - *vs = temp } // VStoRS returns an array of RepositoryScanners diff --git a/rpm/ecosystem.go b/rpm/ecosystem.go index 916826608..03b295727 100644 --- a/rpm/ecosystem.go +++ b/rpm/ecosystem.go @@ -28,8 +28,8 @@ func NewEcosystem(ctx context.Context) *indexer.Ecosystem { RepositoryScanners: func(ctx context.Context) ([]indexer.RepositoryScanner, error) { return []indexer.RepositoryScanner{}, nil }, - Coalescer: func(ctx context.Context, store indexer.Store) (indexer.Coalescer, error) { - return linux.NewCoalescer(store, &Scanner{}), nil + Coalescer: func(ctx context.Context) (indexer.Coalescer, error) { + return linux.NewCoalescer(), nil }, } } diff --git a/internal/indexer/postgres/postgres_test.go b/test/digest.go similarity index 82% rename from internal/indexer/postgres/postgres_test.go rename to test/digest.go index 58619c7ee..919f84622 100644 --- a/internal/indexer/postgres/postgres_test.go +++ b/test/digest.go @@ -1,4 +1,4 @@ -package postgres +package test import ( "crypto/rand" @@ -10,7 +10,7 @@ import ( ) // RandomHash returns a random Digest. -func randomHash(t testing.TB) claircore.Digest { +func RandomSHA256Digest(t testing.TB) claircore.Digest { b := make([]byte, sha256.Size) if _, err := io.ReadFull(rand.Reader, b); err != nil { t.Fatal(err)