Migrate scans to have their own mongo collection

backend/src/migrations/009_update_avscan_type.ts

@@ -3,8 +3,10 @@ import FileModel from '../models/File.js'
 export async function up() {
   const results = await FileModel.find({ avScan: { $type: 'object' } }, null, { strict: false, lean: true })
   results.forEach(async (result) => {
-    if (!Array.isArray(result.avScan)) {
-      await FileModel.findOneAndUpdate({ _id: result._id }, { $set: { avScan: [result.avScan] } })
+    if (result.get('avScan') !== undefined) {
+      if (!Array.isArray(result.get('avScan'))) {
+        await FileModel.findOneAndUpdate({ _id: result._id }, { $set: { avScan: [result.get('avScan')] } })
+      }
     }
   })
 }

backend/src/migrations/012_add_avscan_lastRanAt_property.ts

@@ -3,9 +3,11 @@ import FileModel from '../models/File.js'
 export async function up() {
   const files = await FileModel.find({})
   for (const file of files) {
-    for (const avResult of file.avScan) {
-      if (avResult.lastRunAt === undefined) {
-        avResult.lastRunAt = file.createdAt
+    if (file.get('avScan') !== undefined) {
+      for (const avResult of file.get('avScan')) {
+        if (avResult.lastRunAt === undefined) {
+          avResult.lastRunAt = file.createdAt
+        }
       }
     }
     await file.save()

backend/src/migrations/015_migrate_avscan_to_own_model.ts

@@ -0,0 +1,28 @@
+import FileModel from '../models/File.js'
+import ScanModel, { ArtefactKind } from '../models/Scan.js'
+
+export async function up() {
+  // convert avScan from being stored in File to a new Scan Document
+  const files = await FileModel.find({})
+  for (const file of files) {
+    if (file.get('avScan') !== undefined) {
+      for (const avResult of file.get('avScan')) {
+        // create new Scan Document
+        const newScan = new ScanModel({
+          artefactKind: ArtefactKind.File,
+          fileId: file._id,
+          ...avResult,
+          createdAt: file.createdAt,
+          updatedAt: file.updatedAt,
+        })
+        await newScan.save()
+      }
+    }
+  }
+  // remove all old avScan fields
+  await FileModel.updateMany({ avScan: { $exists: true } }, { $unset: { avScan: 1 } })
+}
+
+export async function down() {
+  /* NOOP */
+}

backend/src/models/File.ts

@@ -1,8 +1,6 @@
 import { model, ObjectId, Schema } from 'mongoose'
 import MongooseDelete, { SoftDeleteDocument } from 'mongoose-delete'
 
-import { FileScanResult, ScanState } from '../connectors/fileScanning/Base.js'
-
 // This interface stores information about the properties on the base object.
 // It should be used for plain object representations, e.g. for sending to the
 // client.
@@ -19,8 +17,6 @@ export interface FileInterface {
 
   complete: boolean
 
-  avScan: Array<FileScanResult>
-
   createdAt: Date
   updatedAt: Date
 }
@@ -41,17 +37,6 @@ const FileSchema = new Schema<FileInterfaceDoc>(
     bucket: { type: String, required: true },
     path: { type: String, required: true },
 
-    avScan: [
-      {
-        toolName: { type: String },
-        scannerVersion: { type: String },
-        state: { type: String, enum: Object.values(ScanState) },
-        isInfected: { type: Boolean },
-        viruses: [{ type: String }],
-        lastRunAt: { type: Schema.Types.Date },
-      },
-    ],
-
     complete: { type: Boolean, default: false },
   },
   {

backend/src/models/Scan.ts

@@ -0,0 +1,70 @@
+import { model, ObjectId, Schema } from 'mongoose'
+import MongooseDelete, { SoftDeleteDocument } from 'mongoose-delete'
+
+import { FileScanResult, ScanState } from '../connectors/fileScanning/Base.js'
+
+// This interface stores information about the properties on the base object.
+// It should be used for plain object representations, e.g. for sending to the
+// client.
+export interface ScanInterface extends FileScanResult {
+  _id: ObjectId
+
+  createdAt: Date
+  updatedAt: Date
+}
+
+export const ArtefactKind = {
+  File: 'file',
+  Image: 'image',
+} as const
+export type ArtefactKindKeys = (typeof ArtefactKind)[keyof typeof ArtefactKind]
+
+export type ArtefactDetails =
+  | {
+      artefactKind: 'file'
+      fileId: string
+    }
+  | {
+      artefactKind: 'image'
+      repositoryName: string
+      // use Digest as image Tags can be overwritten but digests are immutable
+      imageDigest: string
+      // TODO: ultimately use backend/src/models/Release.ts:ImageRef, but ImageRef needs converting to use Digest rather than Tag first
+    }
+
+// The doc type includes all values in the plain interface, as well as all the
+// properties and functions that Mongoose provides.  If a function takes in an
+// object from Mongoose it should use this interface
+export type ScanInterfaceDoc = ScanInterface & ArtefactDetails & SoftDeleteDocument
+
+const ScanSchema = new Schema<ScanInterfaceDoc>(
+  {
+    artefactKind: { type: String, enum: Object.values(ArtefactKind), required: true },
+    fileId: { type: String },
+    repositoryName: { type: String },
+    imageDigest: { type: String },
+
+    toolName: { type: String, required: true },
+    scannerVersion: { type: String },
+    state: { type: String, enum: Object.values(ScanState), required: true },
+    isInfected: { type: Boolean },
+    viruses: [{ type: String }],
+    lastRunAt: { type: Schema.Types.Date, required: true },
+  },
+  {
+    timestamps: true,
+    collection: 'v2_scans',
+    toJSON: { getters: true },
+  },
+)
+
+ScanSchema.plugin(MongooseDelete, {
+  overrideMethods: 'all',
+  deletedBy: true,
+  deletedByType: Schema.Types.ObjectId,
+  deletedAt: true,
+})
+
+const ScanModel = model<ScanInterfaceDoc>('v2_Scan', ScanSchema)
+
+export default ScanModel

backend/src/scripts/listRegistryDigests.ts

@@ -0,0 +1,64 @@
+import fetch from 'node-fetch'
+
+import { getAccessToken } from '../routes/v1/registryAuth.js'
+import { getHttpsAgent } from '../services/http.js'
+import log from '../services/log.js'
+import config from '../utils/config.js'
+import { connectToMongoose, disconnectFromMongoose } from '../utils/database.js'
+
+const httpsAgent = getHttpsAgent({
+  rejectUnauthorized: !config.registry.insecure,
+})
+
+async function script() {
+  await connectToMongoose()
+
+  const registry = `https://localhost:5000/v2`
+
+  const token = await getAccessToken({ dn: 'user' }, [{ type: 'registry', class: '', name: 'catalog', actions: ['*'] }])
+
+  const authorisation = `Bearer ${token}`
+
+  const catalog = (await fetch(`${registry}/_catalog`, {
+    headers: {
+      Authorization: authorisation,
+    },
+    agent: httpsAgent,
+  }).then((res) => res.json())) as object
+
+  await Promise.all(
+    catalog['repositories'].map(async (repositoryName) => {
+      const repositoryToken = await getAccessToken({ dn: 'user' }, [
+        { type: 'repository', class: '', name: repositoryName, actions: ['*'] },
+      ])
+      const repositoryAuthorisation = `Bearer ${repositoryToken}`
+
+      const repositoryTags = (await fetch(`${registry}/${repositoryName}/tags/list`, {
+        headers: {
+          Authorization: repositoryAuthorisation,
+        },
+        agent: httpsAgent,
+      }).then((res) => res.json())) as object
+
+      await Promise.all(
+        repositoryTags['tags'].map(async (tag) => {
+          const repositoryDigest = await fetch(`${registry}/${repositoryName}/manifests/${tag}`, {
+            headers: {
+              Authorization: repositoryAuthorisation,
+              Accept: 'application/vnd.docker.distribution.manifest.v2+json',
+            },
+            agent: httpsAgent,
+          }).then((res) => {
+            return res.headers.get('docker-content-digest')
+          })
+
+          log.info({ repositoryName: repositoryName, tag: tag, digest: repositoryDigest }, 'Digest')
+        }),
+      )
+    }),
+  )
+
+  setTimeout(disconnectFromMongoose, 50)
+}
+
+script()

backend/src/services/file.ts

@@ -7,6 +7,7 @@ import authorisation from '../connectors/authorisation/index.js'
 import { FileScanResult, ScanState } from '../connectors/fileScanning/Base.js'
 import scanners from '../connectors/fileScanning/index.js'
 import FileModel, { FileInterface, FileInterfaceDoc } from '../models/File.js'
+import ScanModel, { ArtefactKind } from '../models/Scan.js'
 import { UserInterface } from '../models/User.js'
 import config from '../utils/config.js'
 import { BadReq, Forbidden, NotFound } from '../utils/error.js'
@@ -29,7 +30,6 @@ export function isFileInterfaceDoc(data: unknown): data is FileInterfaceDoc {
     !('bucket' in data) ||
     !('path' in data) ||
     !('complete' in data) ||
-    !('avScan' in data) ||
     !('deleted' in data) ||
     !('createdAt' in data) ||
     !('updatedAt' in data) ||
@@ -81,19 +81,18 @@ export async function uploadFile(user: UserInterface, modelId: string, name: str
 
 async function updateFileWithResults(_id: Schema.Types.ObjectId, results: FileScanResult[]) {
   for (const result of results) {
-    const updateExistingResult = await FileModel.updateOne(
-      { _id, 'avScan.toolName': result.toolName },
+    const updateExistingResult = await ScanModel.updateOne(
+      { fileId: _id, toolName: result.toolName },
       {
-        $set: { 'avScan.$': { ...result } },
+        $set: { ...result },
       },
     )
     if (updateExistingResult.modifiedCount === 0) {
-      await FileModel.updateOne(
-        { _id, avScan: { $exists: true } },
-        {
-          $push: { avScan: { toolName: result.toolName, state: result.state, lastRunAt: new Date() } },
-        },
-      )
+      await ScanModel.create({
+        artefactKind: ArtefactKind.File,
+        fileId: _id,
+        ...result,
+      })
     }
   }
 }
@@ -141,7 +140,17 @@ export async function getFilesByIds(user: UserInterface, modelId: string, fileId
   if (fileIds.length === 0) {
     return []
   }
-  const files = await FileModel.find({ _id: { $in: fileIds } })
+  const files = await FileModel.aggregate([
+    { $match: { fileId: { $in: fileIds } } },
+    {
+      $lookup: {
+        from: 'v2_scans',
+        localField: 'id',
+        foreignField: 'fileId',
+        as: 'avScan',
+      },
+    },
+  ])
 
   if (files.length !== fileIds.length) {
     const notFoundFileIds = fileIds.filter((id) => files.some((file) => file._id.toString() === id))
@@ -196,13 +205,14 @@ export async function markFileAsCompleteAfterImport(path: string) {
   }
 }
 
-function fileScanDelay(file: FileInterface): number {
+async function fileScanDelay(file: FileInterface): Promise<number> {
   const delay = config.connectors.fileScanners.retryDelayInMinutes
   if (delay === undefined) {
     return 0
   }
   let minutesBeforeRetrying = 0
-  for (const scanResult of file.avScan) {
+  const fileAvScans = await ScanModel.find({ fileId: file._id })
+  for (const scanResult of fileAvScans) {
     const delayInMilliseconds = delay * 60000
     const scanTimeAtLimit = scanResult.lastRunAt.getTime() + delayInMilliseconds
     if (scanTimeAtLimit > new Date().getTime()) {
@@ -229,7 +239,7 @@ export async function rerunFileScan(user: UserInterface, modelId, fileId: string
   if (!file.size || file.size === 0) {
     throw BadReq('Cannot run scan on an empty file')
   }
-  const minutesBeforeRescanning = fileScanDelay(file)
+  const minutesBeforeRescanning = await fileScanDelay(file)
   if (minutesBeforeRescanning > 0) {
     throw BadReq(`Please wait ${plural(minutesBeforeRescanning, 'minute')} before attempting a rescan ${file.name}`)
   }

backend/src/services/mirroredModel.ts

@@ -17,6 +17,7 @@ import { FileInterfaceDoc } from '../models/File.js'
 import { ModelDoc, ModelInterface } from '../models/Model.js'
 import { ModelCardRevisionDoc } from '../models/ModelCardRevision.js'
 import { ReleaseDoc } from '../models/Release.js'
+import { ScanInterfaceDoc } from '../models/Scan.js'
 import { UserInterface } from '../models/User.js'
 import config from '../utils/config.js'
 import { BadReq, Forbidden, InternalError } from '../utils/error.js'
@@ -598,14 +599,14 @@ async function checkReleaseFiles(user: UserInterface, modelId: string, semvers:
   }
 
   if (scanners.info()) {
-    const files: FileInterfaceDoc[] = await getFilesByIds(user, modelId, fileIds)
+    const files: (FileInterfaceDoc & { avScan: ScanInterfaceDoc[] })[] = await getFilesByIds(user, modelId, fileIds)
     const scanErrors: {
       missingScan: Array<{ name: string; id: string }>
       incompleteScan: Array<{ name: string; id: string }>
       failedScan: Array<{ name: string; id: string }>
     } = { missingScan: [], incompleteScan: [], failedScan: [] }
     for (const file of files) {
-      if (!file.avScan) {
+      if (!file.avScan || file.avScan.length === 0) {
         scanErrors.missingScan.push({ name: file.name, id: file.id })
       } else if (file.avScan.some((scanResult) => scanResult.state !== ScanState.Complete)) {
         scanErrors.incompleteScan.push({ name: file.name, id: file.id })

backend/test/services/__snapshots__/file.spec.ts.snap

@@ -11,6 +11,33 @@ exports[`services > file > downloadFile > success 1`] = `
 exports[`services > file > getFilesByIds > success 1`] = `
 [
   {
+    "avScan": [],
+    "example": "file",
+  },
+]
+`;
+
+exports[`services > file > getFilesByIds > success with scans mapped 1`] = `
+[
+  {
+    "_id": "123",
+    "avScan": [
+      {
+        "fileId": "123",
+      },
+      {
+        "fileId": "123",
+      },
+    ],
+    "example": "file",
+  },
+  {
+    "_id": "321",
+    "avScan": [
+      {
+        "fileId": "321",
+      },
+    ],
     "example": "file",
   },
 ]