From c809930205eaad77f13427c7b965a6194ea671ac Mon Sep 17 00:00:00 2001 From: ldelossa Date: Mon, 17 May 2021 10:58:27 -0400 Subject: [PATCH] libindex: fix manifest_index unique constraint This commit adds a migration to fix the missing unique constraint on the `manifest_index` table. Without this constraint, the `manifest_index` table can bloat to absurd size. In order to deal with tables which have already bloated to absurd size, we simply truncate the `manifest_index` table. We also truncate the `manifest_scanned` table which allows all subsequent manifests submitted to clair to be re-indexed. This re-index will rebuild the `manifest_index` for the manifest being submitted, but perform no other work since all layer data still exists for a previously seen manifest. Signed-off-by: ldelossa --- libindex/libindex.go | 2 +- libindex/migrations/migration3.go | 19 +++++++++++++++++++ libindex/migrations/migrations.go | 7 +++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 libindex/migrations/migration3.go diff --git a/libindex/libindex.go b/libindex/libindex.go index ff46fc0ba..f359f535f 100644 --- a/libindex/libindex.go +++ b/libindex/libindex.go @@ -21,7 +21,7 @@ import ( "github.com/quay/claircore/internal/indexer/controller" ) -const versionMagic = "libindex number: 1\n" +const versionMagic = "libindex number: 2\n" // Libindex implements the method set for scanning and indexing a Manifest. type Libindex struct { diff --git a/libindex/migrations/migration3.go b/libindex/migrations/migration3.go new file mode 100644 index 000000000..44843c5cc --- /dev/null +++ b/libindex/migrations/migration3.go @@ -0,0 +1,19 @@ +package migrations + +const ( + // This migration truncates the manifest_index and scanned_manifest tables + // and adds a unique index to manifest_index. This is required since the + // manifest_index table currently bloats with duplicate records. + // + // After this migration is complete manifests will need to be re-indexed + // for notifications on these manifests to work correctly. + // + // Index reports will still be served without a re-index being necessary. + migration3 = ` +LOCK manifest_index; +LOCK scanned_manifest; +TRUNCATE manifest_index; +TRUNCATE scanned_manifest; +CREATE UNIQUE INDEX manifest_index_unique ON manifest_index (package_id, COALESCE(dist_id, 0), COALESCE(repo_id, 0), manifest_id); +` +) diff --git a/libindex/migrations/migrations.go b/libindex/migrations/migrations.go index df74ad352..72b3fe829 100644 --- a/libindex/migrations/migrations.go +++ b/libindex/migrations/migrations.go @@ -25,4 +25,11 @@ var Migrations = []migrate.Migration{ return err }, }, + { + ID: 3, + Up: func(tx *sql.Tx) error { + _, err := tx.Exec(migration3) + return err + }, + }, }