libindex: fix manifest_index unique constraint

This commit adds a migration to fix the missing unique constraint on
the `manifest_index` table. Without this constraint, the
`manifest_index` table can bloat to absurd size.

In order to deal with tables which have already bloated to absurd size,
we simply truncate the `manifest_index` table. We also truncate the
`manifest_scanned` table which allows all subsequent manifests submitted
to clair to be re-indexed.

This re-index will rebuild the `manifest_index` for the manifest being
submitted, but perform no other work since all layer data still exists
for a previously seen manifest.

Signed-off-by: ldelossa <[email protected]>

libindex/libindex.go

@@ -21,7 +21,7 @@ import (
 	"github.com/quay/claircore/internal/indexer/controller"
 )
 
-const versionMagic = "libindex number: 1\n"
+const versionMagic = "libindex number: 2\n"
 
 // Libindex implements the method set for scanning and indexing a Manifest.
 type Libindex struct {

libindex/migrations/migration3.go

@@ -0,0 +1,19 @@
+package migrations
+
+const (
+	// This migration truncates the manifest_index and scanned_manifest tables
+	// and adds a unique index to manifest_index. This is required since the
+	// manifest_index table currently bloats with duplicate records.
+	//
+	// After this migration is complete manifests will need to be re-indexed
+	// for notifications on these manifests to work correctly.
+	//
+	// Index reports will still be served without a re-index being necessary.
+	migration3 = `
+LOCK manifest_index;
+LOCK scanned_manifest;
+TRUNCATE manifest_index;
+TRUNCATE scanned_manifest;
+CREATE UNIQUE INDEX manifest_index_unique ON manifest_index (package_id, COALESCE(dist_id, 0), COALESCE(repo_id, 0), manifest_id);
+`
+)

libindex/migrations/migrations.go

@@ -25,4 +25,11 @@ var Migrations = []migrate.Migration{
 			return err
 		},
 	},
+	{
+		ID: 3,
+		Up: func(tx *sql.Tx) error {
+			_, err := tx.Exec(migration3)
+			return err
+		},
+	},
 }