Migrate scans to have their own mongo collection

backend/src/connectors/audit/Base.ts

@@ -1,7 +1,7 @@
 import { Request } from 'express'
 
 import { AccessRequestDoc } from '../../models/AccessRequest.js'
-import { FileInterface, FileInterfaceDoc } from '../../models/File.js'
+import { FileInterface } from '../../models/File.js'
 import { InferenceDoc } from '../../models/Inference.js'
 import { ModelCardInterface, ModelDoc, ModelInterface } from '../../models/Model.js'
 import { ReleaseDoc } from '../../models/Release.js'
@@ -136,8 +136,8 @@ export abstract class BaseAuditConnector {
   abstract onUpdateModelCard(req: Request, modelId: string, modelCard: ModelCardInterface)
   abstract onViewModelCardRevisions(req: Request, modelId: string, modelCards: ModelCardInterface[])
 
-  abstract onCreateFile(req: Request, file: FileInterfaceDoc)
-  abstract onViewFile(req: Request, file: FileInterfaceDoc)
+  abstract onCreateFile(req: Request, file: FileInterface)
+  abstract onViewFile(req: Request, file: FileInterface)
   abstract onViewFiles(req: Request, modelId: string, files: FileInterface[])
   abstract onDeleteFile(req: Request, modelId: string, fileId: string)
   abstract onUpdateFile(req: Request, modelId: string, fileId: string)

backend/src/connectors/authorisation/base.ts

@@ -1,5 +1,5 @@
 import { AccessRequestDoc } from '../../models/AccessRequest.js'
-import { FileInterfaceDoc } from '../../models/File.js'
+import { FileInterface } from '../../models/File.js'
 import { EntryVisibility, ModelDoc } from '../../models/Model.js'
 import { ReleaseDoc } from '../../models/Release.js'
 import { ResponseDoc } from '../../models/Response.js'
@@ -77,7 +77,7 @@ export class BasicAuthorisationConnector {
     return (await this.responses(user, [response], action))[0]
   }
 
-  async file(user: UserInterface, model: ModelDoc, file: FileInterfaceDoc, action: FileActionKeys) {
+  async file(user: UserInterface, model: ModelDoc, file: FileInterface, action: FileActionKeys) {
     return (await this.files(user, model, [file], action))[0]
   }
 
@@ -243,7 +243,7 @@ export class BasicAuthorisationConnector {
   async files(
     user: UserInterface,
     model: ModelDoc,
-    files: Array<FileInterfaceDoc>,
+    files: Array<FileInterface>,
     action: FileActionKeys,
   ): Promise<Array<Response>> {
     // Does the user have a valid access request for this model?
@@ -265,7 +265,7 @@ export class BasicAuthorisationConnector {
           return {
             success: false,
             info: 'You do not have permission to upload a file.',
-            id: file.id,
+            id: file._id.toString(),
           }
         }
 
@@ -278,11 +278,11 @@ export class BasicAuthorisationConnector {
           return {
             success: false,
             info: 'You need to have an approved access request or have permission to download a file.',
-            id: file.id,
+            id: file._id.toString(),
           }
         }
 
-        return { success: true, id: file.id }
+        return { success: true, id: file._id.toString() }
       }),
     )
   }

backend/src/connectors/fileScanning/Base.ts

@@ -1,12 +1,9 @@
 import { FileInterface } from '../../models/File.js'
-export interface FileScanResult {
-  toolName: string
-  scannerVersion?: string
-  state: ScanStateKeys
-  isInfected?: boolean
-  viruses?: string[]
-  lastRunAt: Date
-}
+import { ScanInterface } from '../../models/Scan.js'
+export type FileScanResult = Pick<
+  ScanInterface,
+  'toolName' | 'scannerVersion' | 'state' | 'isInfected' | 'viruses' | 'lastRunAt'
+>
 
 export const ScanState = {
   NotScanned: 'notScanned',

backend/src/migrations/009_update_avscan_type.ts

@@ -3,8 +3,10 @@ import FileModel from '../models/File.js'
 export async function up() {
   const results = await FileModel.find({ avScan: { $type: 'object' } }, null, { strict: false, lean: true })
   results.forEach(async (result) => {
-    if (!Array.isArray(result.avScan)) {
-      await FileModel.findOneAndUpdate({ _id: result._id }, { $set: { avScan: [result.avScan] } })
+    if (result.get('avScan') !== undefined) {
+      if (!Array.isArray(result.get('avScan'))) {
+        await FileModel.findOneAndUpdate({ _id: result._id }, { $set: { avScan: [result.get('avScan')] } })
+      }
     }
   })
 }

backend/src/migrations/012_add_avscan_lastRanAt_property.ts

@@ -3,9 +3,11 @@ import FileModel from '../models/File.js'
 export async function up() {
   const files = await FileModel.find({})
   for (const file of files) {
-    for (const avResult of file.avScan) {
-      if (avResult.lastRunAt === undefined) {
-        avResult.lastRunAt = file.createdAt
+    if (file.get('avScan') !== undefined) {
+      for (const avResult of file.get('avScan')) {
+        if (avResult.lastRunAt === undefined) {
+          avResult.lastRunAt = file.createdAt
+        }
       }
     }
     await file.save()

backend/src/migrations/015_migrate_avscan_to_own_model.ts

@@ -0,0 +1,28 @@
+import FileModel from '../models/File.js'
+import ScanModel, { ArtefactKind } from '../models/Scan.js'
+
+export async function up() {
+  // convert avScan from being stored in File to a new Scan Document
+  const files = await FileModel.find({})
+  for (const file of files) {
+    if (file.get('avScan') !== undefined) {
+      for (const avResult of file.get('avScan')) {
+        // create new Scan Document
+        const newScan = new ScanModel({
+          artefactKind: ArtefactKind.File,
+          fileId: file._id,
+          ...avResult,
+          createdAt: file.createdAt,
+          updatedAt: file.updatedAt,
+        })
+        await newScan.save()
+      }
+    }
+  }
+  // remove all old avScan fields
+  await FileModel.updateMany({ avScan: { $exists: true } }, { $unset: { avScan: 1 } })
+}
+
+export async function down() {
+  /* NOOP */
+}

backend/src/models/File.ts

@@ -1,7 +1,7 @@
 import { model, ObjectId, Schema } from 'mongoose'
 import MongooseDelete, { SoftDeleteDocument } from 'mongoose-delete'
 
-import { FileScanResult, ScanState } from '../connectors/fileScanning/Base.js'
+import { ScanInterface } from './Scan.js'
 
 // This interface stores information about the properties on the base object.
 // It should be used for plain object representations, e.g. for sending to the
@@ -19,8 +19,6 @@ export interface FileInterface {
 
   complete: boolean
 
-  avScan: Array<FileScanResult>
-
   createdAt: Date
   updatedAt: Date
 }
@@ -29,6 +27,8 @@ export interface FileInterface {
 // properties and functions that Mongoose provides.  If a function takes in an
 // object from Mongoose it should use this interface
 export type FileInterfaceDoc = FileInterface & SoftDeleteDocument
+// `id` is used by the python API so we need to keep this to prevent a breaking change
+export type FileWithScanResultsInterface = FileInterface & { avScan: ScanInterface[]; id: string }
 
 const FileSchema = new Schema<FileInterfaceDoc>(
   {
@@ -41,17 +41,6 @@ const FileSchema = new Schema<FileInterfaceDoc>(
     bucket: { type: String, required: true },
     path: { type: String, required: true },
 
-    avScan: [
-      {
-        toolName: { type: String },
-        scannerVersion: { type: String },
-        state: { type: String, enum: Object.values(ScanState) },
-        isInfected: { type: Boolean },
-        viruses: [{ type: String }],
-        lastRunAt: { type: Schema.Types.Date },
-      },
-    ],
-
     complete: { type: Boolean, default: false },
   },
   {

backend/src/models/Scan.ts

@@ -0,0 +1,70 @@
+import { model, ObjectId, Schema } from 'mongoose'
+import MongooseDelete, { SoftDeleteDocument } from 'mongoose-delete'
+
+import { ScanState, ScanStateKeys } from '../connectors/fileScanning/Base.js'
+
+export type ScanInterface = {
+  _id: ObjectId
+
+  toolName: string
+  scannerVersion?: string
+  state: ScanStateKeys
+  isInfected?: boolean
+  viruses?: string[]
+  lastRunAt: Date
+
+  createdAt: Date
+  updatedAt: Date
+} & (
+  | {
+      artefactKind: typeof ArtefactKind.File
+      fileId: string
+    }
+  | {
+      artefactKind: typeof ArtefactKind.Image
+      repositoryName: string
+      // use Digest as image Tags can be overwritten but digests are immutable
+      imageDigest: string
+      // TODO: ultimately use backend/src/models/Release.ts:ImageRef, but ImageRef needs converting to use Digest rather than Tag first
+    }
+)
+
+export const ArtefactKind = {
+  File: 'file',
+  Image: 'image',
+} as const
+export type ArtefactKindKeys = (typeof ArtefactKind)[keyof typeof ArtefactKind]
+
+export type ScanInterfaceDoc = ScanInterface & SoftDeleteDocument
+
+const ScanSchema = new Schema<ScanInterfaceDoc>(
+  {
+    artefactKind: { type: String, enum: Object.values(ArtefactKind), required: true },
+    fileId: { type: String },
+    repositoryName: { type: String },
+    imageDigest: { type: String },
+
+    toolName: { type: String, required: true },
+    scannerVersion: { type: String },
+    state: { type: String, enum: Object.values(ScanState), required: true },
+    isInfected: { type: Boolean },
+    viruses: [{ type: String }],
+    lastRunAt: { type: Schema.Types.Date, required: true },
+  },
+  {
+    timestamps: true,
+    collection: 'v2_scans',
+    toJSON: { getters: true },
+  },
+)
+
+ScanSchema.plugin(MongooseDelete, {
+  overrideMethods: 'all',
+  deletedBy: true,
+  deletedByType: Schema.Types.ObjectId,
+  deletedAt: true,
+})
+
+const ScanModel = model<ScanInterfaceDoc>('v2_Scan', ScanSchema)
+
+export default ScanModel

backend/src/routes/v2/model/file/getDownloadFile.ts

@@ -6,7 +6,7 @@ import { z } from 'zod'
 
 import { AuditInfo } from '../../../../connectors/audit/Base.js'
 import audit from '../../../../connectors/audit/index.js'
-import { FileInterface, FileInterfaceDoc } from '../../../../models/File.js'
+import { FileInterface, FileWithScanResultsInterface } from '../../../../models/File.js'
 import { downloadFile, getFileById } from '../../../../services/file.js'
 import { getFileByReleaseFileName } from '../../../../services/release.js'
 import { registerPath } from '../../../../services/specification.js'
@@ -92,7 +92,7 @@ export const getDownloadFile = [
   async (req: Request, res: Response<GetDownloadFileResponse>) => {
     req.audit = AuditInfo.ViewFile
     const { params } = parse(req, getDownloadFileSchema)
-    let file: FileInterfaceDoc
+    let file: FileWithScanResultsInterface
     if ('semver' in params) {
       file = await getFileByReleaseFileName(req.user, params.modelId, params.semver, params.fileName)
     } else {
@@ -107,7 +107,7 @@ export const getDownloadFile = [
     res.set('Content-Length', String(file.size))
     // TODO: support ranges
     // res.set('Accept-Ranges', 'bytes')
-    const stream = await downloadFile(req.user, file._id)
+    const stream = await downloadFile(req.user, file.id)
 
     if (!stream.Body) {
       throw InternalError('We were not able to retrieve the body of this file', { fileId: file._id })

backend/src/routes/v2/release/getRelease.ts

@@ -4,7 +4,7 @@ import { z } from 'zod'
 
 import { AuditInfo } from '../../../connectors/audit/Base.js'
 import audit from '../../../connectors/audit/index.js'
-import { FileInterface } from '../../../models/File.js'
+import { FileWithScanResultsInterface } from '../../../models/File.js'
 import { ReleaseInterface } from '../../../models/Release.js'
 import { ResponseKind } from '../../../models/Response.js'
 import { getFilesByIds } from '../../../services/file.js'
@@ -40,7 +40,7 @@ registerPath({
 })
 
 interface getReleaseResponse {
-  release: ReleaseInterface & { files: FileInterface[] }
+  release: ReleaseInterface & { files: FileWithScanResultsInterface[] }
 }
 
 export const getRelease = [

backend/src/scripts/listRegistryDigests.ts

@@ -0,0 +1,64 @@
+import fetch from 'node-fetch'
+
+import { getAccessToken } from '../routes/v1/registryAuth.js'
+import { getHttpsAgent } from '../services/http.js'
+import log from '../services/log.js'
+import config from '../utils/config.js'
+import { connectToMongoose, disconnectFromMongoose } from '../utils/database.js'
+
+const httpsAgent = getHttpsAgent({
+  rejectUnauthorized: !config.registry.insecure,
+})
+
+async function script() {
+  await connectToMongoose()
+
+  const registry = `https://localhost:5000/v2`
+
+  const token = await getAccessToken({ dn: 'user' }, [{ type: 'registry', class: '', name: 'catalog', actions: ['*'] }])
+
+  const authorisation = `Bearer ${token}`
+
+  const catalog = (await fetch(`${registry}/_catalog`, {
+    headers: {
+      Authorization: authorisation,
+    },
+    agent: httpsAgent,
+  }).then((res) => res.json())) as object
+
+  await Promise.all(
+    catalog['repositories'].map(async (repositoryName) => {
+      const repositoryToken = await getAccessToken({ dn: 'user' }, [
+        { type: 'repository', class: '', name: repositoryName, actions: ['*'] },
+      ])
+      const repositoryAuthorisation = `Bearer ${repositoryToken}`
+
+      const repositoryTags = (await fetch(`${registry}/${repositoryName}/tags/list`, {
+        headers: {
+          Authorization: repositoryAuthorisation,
+        },
+        agent: httpsAgent,
+      }).then((res) => res.json())) as object
+
+      await Promise.all(
+        repositoryTags['tags'].map(async (tag) => {
+          const repositoryDigest = await fetch(`${registry}/${repositoryName}/manifests/${tag}`, {
+            headers: {
+              Authorization: repositoryAuthorisation,
+              Accept: 'application/vnd.docker.distribution.manifest.v2+json',
+            },
+            agent: httpsAgent,
+          }).then((res) => {
+            return res.headers.get('docker-content-digest')
+          })
+
+          log.info({ repositoryName: repositoryName, tag: tag, digest: repositoryDigest }, 'Digest')
+        }),
+      )
+    }),
+  )
+
+  setTimeout(disconnectFromMongoose, 50)
+}
+
+script()