Migrate scans to have their own mongo collection

backend/src/migrations/015_migrate_avscan_to_own_model.ts

@@ -0,0 +1,35 @@
+import FileModel from '../models/File.js'
+import ScanModel, { ArtefactType } from '../models/Scan.js'
+
+export async function up() {
+  // convert avScan from being stored in File to a new Scan Document
+  const files = await FileModel.find({ avScan: { $exists: true } })
+  for (const file of files) {
+    // TS linting fix - we already check for existing so this should never happen
+    if (file.avScan === undefined) {
+      continue
+    }
+    for (const avResult of file.avScan) {
+      // create new Scan Document
+      const newScan = new ScanModel({
+        artefactType: ArtefactType.File,
+        fileId: file._id,
+        toolName: avResult.toolName,
+        scannerVersion: avResult.scannerVersion,
+        state: avResult.state,
+        isInfected: avResult.isInfected,
+        viruses: avResult.viruses,
+        lastRunAt: avResult.lastRunAt,
+        createdAt: file.createdAt,
+        updatedAt: file.updatedAt,
+      })
+      await newScan.save()
+    }
+  }
+  // remove all old avScan fields
+  await FileModel.updateMany({ avScan: { $exists: true } }, { $unset: { avScan: 1 } })
+}
+
+export async function down() {
+  /* NOOP */
+}

backend/src/models/File.ts

@@ -41,16 +41,23 @@ const FileSchema = new Schema<FileInterfaceDoc>(
     bucket: { type: String, required: true },
     path: { type: String, required: true },
 
-    avScan: [
-      {
-        toolName: { type: String },
-        scannerVersion: { type: String },
-        state: { type: String, enum: Object.values(ScanState) },
-        isInfected: { type: Boolean },
-        viruses: [{ type: String }],
-        lastRunAt: { type: Schema.Types.Date },
+    avScan: {
+      type: [
+        {
+          toolName: { type: String },
+          scannerVersion: { type: String },
+          state: { type: String, enum: Object.values(ScanState) },
+          isInfected: { type: Boolean },
+          viruses: [{ type: String }],
+          lastRunAt: { type: Schema.Types.Date },
+        },
+      ],
+      required: false,
+      // prevent legacy field from being used
+      validate: function (val: any): boolean {
+        return val === undefined || val.length === 0
       },
-    ],
+    },
 
     complete: { type: Boolean, default: false },
   },

backend/src/models/Scan.ts

@@ -0,0 +1,110 @@
+import { model, ObjectId, Schema } from 'mongoose'
+import MongooseDelete, { SoftDeleteDocument } from 'mongoose-delete'
+
+import { ScanState, ScanStateKeys } from '../connectors/fileScanning/Base.js'
+
+// This interface stores information about the properties on the base object.
+// It should be used for plain object representations, e.g. for sending to the
+// client.
+export interface ScanInterface {
+  _id: ObjectId
+
+  // file or image
+  artefactType: ArtefactTypeKeys
+  // for files
+  fileId?: string
+  // for images
+  // map this to registry GET /v2/<name>/manifests/<reference>
+  repositoryName?: string
+  imageDigest?: string
+
+  toolName: string
+  scannerVersion?: string
+  state: ScanStateKeys
+  isInfected?: boolean
+  viruses?: string[]
+  lastRunAt: Date
+
+  createdAt: Date
+  updatedAt: Date
+}
+
+export const ArtefactType = {
+  File: 'file',
+  Image: 'image',
+} as const
+export type ArtefactTypeKeys = (typeof ArtefactType)[keyof typeof ArtefactType]
+
+// The doc type includes all values in the plain interface, as well as all the
+// properties and functions that Mongoose provides.  If a function takes in an
+// object from Mongoose it should use this interface
+export type ScanInterfaceDoc = ScanInterface & SoftDeleteDocument
+
+const ScanSchema = new Schema<ScanInterfaceDoc>(
+  {
+    artefactType: { type: String, enum: Object.values(ArtefactType), required: true },
+    fileId: {
+      type: String,
+      required: function (): boolean {
+        return this['artefactType'] === ArtefactType.File
+      },
+      validate: function (val: any): boolean {
+        if (this['artefactType'] === ArtefactType.File && val) {
+          return true
+        }
+        throw new Error(`Cannot provide a 'fileId' with '${JSON.stringify({ artefactType: this['artefactType'] })}'`)
+      },
+    },
+    imageDigest: {
+      type: String,
+      required: function (): boolean {
+        return this['artefactType'] === ArtefactType.Image
+      },
+      validate: function (val: any): boolean {
+        if (this['artefactType'] === ArtefactType.Image && val) {
+          return true
+        }
+        throw new Error(
+          `Cannot provide an 'imageDigest' with '${JSON.stringify({ artefactType: this['artefactType'] })}'`,
+        )
+      },
+    },
+    repositoryName: {
+      type: String,
+      required: function (): boolean {
+        return this['artefactType'] === ArtefactType.Image
+      },
+      validate: function (val: any): boolean {
+        if (this['artefactType'] === ArtefactType.Image && val) {
+          return true
+        }
+        throw new Error(
+          `Cannot provide a 'repositoryName' with '${JSON.stringify({ artefactType: this['artefactType'] })}'`,
+        )
+      },
+    },
+
+    toolName: { type: String, required: true },
+    scannerVersion: { type: String },
+    state: { type: String, enum: Object.values(ScanState), required: true },
+    isInfected: { type: Boolean },
+    viruses: [{ type: String }],
+    lastRunAt: { type: Schema.Types.Date, required: true },
+  },
+  {
+    timestamps: true,
+    collection: 'v2_scans',
+    toJSON: { getters: true },
+  },
+)
+
+ScanSchema.plugin(MongooseDelete, {
+  overrideMethods: 'all',
+  deletedBy: true,
+  deletedByType: Schema.Types.ObjectId,
+  deletedAt: true,
+})
+
+const ScanModel = model<ScanInterfaceDoc>('v2_Scan', ScanSchema)
+
+export default ScanModel

backend/src/scripts/listRegistryDigests.ts

@@ -0,0 +1,64 @@
+import fetch from 'node-fetch'
+
+import { getAccessToken } from '../routes/v1/registryAuth.js'
+import { getHttpsAgent } from '../services/http.js'
+import log from '../services/log.js'
+import config from '../utils/config.js'
+import { connectToMongoose, disconnectFromMongoose } from '../utils/database.js'
+
+const httpsAgent = getHttpsAgent({
+  rejectUnauthorized: !config.registry.insecure,
+})
+
+async function script() {
+  await connectToMongoose()
+
+  const registry = `https://localhost:5000/v2`
+
+  const token = await getAccessToken({ dn: 'user' }, [{ type: 'registry', class: '', name: 'catalog', actions: ['*'] }])
+
+  const authorisation = `Bearer ${token}`
+
+  const catalog = (await fetch(`${registry}/_catalog`, {
+    headers: {
+      Authorization: authorisation,
+    },
+    agent: httpsAgent,
+  }).then((res) => res.json())) as object
+
+  await Promise.all(
+    catalog['repositories'].map(async (repositoryName) => {
+      const repositoryToken = await getAccessToken({ dn: 'user' }, [
+        { type: 'repository', class: '', name: repositoryName, actions: ['*'] },
+      ])
+      const repositoryAuthorisation = `Bearer ${repositoryToken}`
+
+      const repositoryTags = (await fetch(`${registry}/${repositoryName}/tags/list`, {
+        headers: {
+          Authorization: repositoryAuthorisation,
+        },
+        agent: httpsAgent,
+      }).then((res) => res.json())) as object
+
+      await Promise.all(
+        repositoryTags['tags'].map(async (tag) => {
+          const repositoryDigest = await fetch(`${registry}/${repositoryName}/manifests/${tag}`, {
+            headers: {
+              Authorization: repositoryAuthorisation,
+              Accept: 'application/vnd.docker.distribution.manifest.v2+json',
+            },
+            agent: httpsAgent,
+          }).then((res) => {
+            return res.headers.get('docker-content-digest')
+          })
+
+          log.info({ repositoryName: repositoryName, tag: tag, digest: repositoryDigest }, 'Digest')
+        }),
+      )
+    }),
+  )
+
+  setTimeout(disconnectFromMongoose, 50)
+}
+
+script()

backend/src/services/file.ts

@@ -7,6 +7,7 @@ import authorisation from '../connectors/authorisation/index.js'
 import { FileScanResult, ScanState } from '../connectors/fileScanning/Base.js'
 import scanners from '../connectors/fileScanning/index.js'
 import FileModel, { FileInterface, FileInterfaceDoc } from '../models/File.js'
+import ScanModel, { ArtefactType } from '../models/Scan.js'
 import { UserInterface } from '../models/User.js'
 import config from '../utils/config.js'
 import { BadReq, Forbidden, NotFound } from '../utils/error.js'
@@ -29,7 +30,6 @@ export function isFileInterfaceDoc(data: unknown): data is FileInterfaceDoc {
     !('bucket' in data) ||
     !('path' in data) ||
     !('complete' in data) ||
-    !('avScan' in data) ||
     !('deleted' in data) ||
     !('createdAt' in data) ||
     !('updatedAt' in data) ||
@@ -81,19 +81,18 @@ export async function uploadFile(user: UserInterface, modelId: string, name: str
 
 async function updateFileWithResults(_id: Schema.Types.ObjectId, results: FileScanResult[]) {
   for (const result of results) {
-    const updateExistingResult = await FileModel.updateOne(
-      { _id, 'avScan.toolName': result.toolName },
+    const updateExistingResult = await ScanModel.updateOne(
+      { fileId: _id, toolName: result.toolName },
       {
-        $set: { 'avScan.$': { ...result } },
+        $set: { ...result },
       },
     )
     if (updateExistingResult.modifiedCount === 0) {
-      await FileModel.updateOne(
-        { _id, avScan: { $exists: true } },
-        {
-          $push: { avScan: { toolName: result.toolName, state: result.state, lastRunAt: new Date() } },
-        },
-      )
+      await ScanModel.create({
+        artefactType: ArtefactType.File,
+        fileId: _id,
+        ...result,
+      })
     }
   }
 }
@@ -148,8 +147,15 @@ export async function getFilesByIds(user: UserInterface, modelId: string, fileId
     throw NotFound(`The requested files were not found.`, { fileIds: notFoundFileIds })
   }
 
+  const fileAvScans = await ScanModel.find({ fileId: { $in: fileIds } })
+  const filesWithAvScans = files.map((file) => {
+    const relevantAvScans = fileAvScans.filter((scan, _) => scan.fileId === file._id.toString())
+    file.avScan = (file.avScan || []).concat(relevantAvScans)
+    return file
+  })
+
   const auths = await authorisation.files(user, model, files, FileAction.View)
-  return files.filter((_, i) => auths[i].success)
+  return filesWithAvScans.filter((_, i) => auths[i].success)
 }
 
 export async function removeFile(user: UserInterface, modelId: string, fileId: string) {
@@ -196,13 +202,14 @@ export async function markFileAsCompleteAfterImport(path: string) {
   }
 }
 
-function fileScanDelay(file: FileInterface): number {
+async function fileScanDelay(file: FileInterface): Promise<number> {
   const delay = config.connectors.fileScanners.retryDelayInMinutes
   if (delay === undefined) {
     return 0
   }
   let minutesBeforeRetrying = 0
-  for (const scanResult of file.avScan) {
+  const fileAvScans = await ScanModel.find({ fileId: file._id })
+  for (const scanResult of fileAvScans) {
     const delayInMilliseconds = delay * 60000
     const scanTimeAtLimit = scanResult.lastRunAt.getTime() + delayInMilliseconds
     if (scanTimeAtLimit > new Date().getTime()) {
@@ -229,7 +236,7 @@ export async function rerunFileScan(user: UserInterface, modelId, fileId: string
   if (!file.size || file.size === 0) {
     throw BadReq('Cannot run scan on an empty file')
   }
-  const minutesBeforeRescanning = fileScanDelay(file)
+  const minutesBeforeRescanning = await fileScanDelay(file)
   if (minutesBeforeRescanning > 0) {
     throw BadReq(`Please wait ${plural(minutesBeforeRescanning, 'minute')} before attempting a rescan ${file.name}`)
   }

backend/src/services/mirroredModel.ts

@@ -605,7 +605,7 @@ async function checkReleaseFiles(user: UserInterface, modelId: string, semvers:
       failedScan: Array<{ name: string; id: string }>
     } = { missingScan: [], incompleteScan: [], failedScan: [] }
     for (const file of files) {
-      if (!file.avScan) {
+      if (!file.avScan || file.avScan.length === 0) {
         scanErrors.missingScan.push({ name: file.name, id: file.id })
       } else if (file.avScan.some((scanResult) => scanResult.state !== ScanState.Complete)) {
         scanErrors.incompleteScan.push({ name: file.name, id: file.id })

backend/test/services/__snapshots__/file.spec.ts.snap

@@ -11,6 +11,33 @@ exports[`services > file > downloadFile > success 1`] = `
 exports[`services > file > getFilesByIds > success 1`] = `
 [
   {
+    "avScan": [],
+    "example": "file",
+  },
+]
+`;
+
+exports[`services > file > getFilesByIds > success with scans mapped 1`] = `
+[
+  {
+    "_id": "123",
+    "avScan": [
+      {
+        "fileId": "123",
+      },
+      {
+        "fileId": "123",
+      },
+    ],
+    "example": "file",
+  },
+  {
+    "_id": "321",
+    "avScan": [
+      {
+        "fileId": "321",
+      },
+    ],
     "example": "file",
   },
 ]