From 7759a0fa88360d34e3892cc92ce066cd021d75e9 Mon Sep 17 00:00:00 2001 From: Jonathan Fallon Date: Fri, 20 Dec 2024 12:04:42 +0100 Subject: [PATCH] add tests --- .../etl/providers/FileManager.unit.spec.ts | 127 +++++++++++------- 1 file changed, 78 insertions(+), 49 deletions(-) diff --git a/api/src/etl/providers/FileManager.unit.spec.ts b/api/src/etl/providers/FileManager.unit.spec.ts index afe56f2662..ec8f22e316 100644 --- a/api/src/etl/providers/FileManager.unit.spec.ts +++ b/api/src/etl/providers/FileManager.unit.spec.ts @@ -1,6 +1,7 @@ import { mkdir, readFile, rm } from "@/deps.ts"; import { afterAll, + assert, assertEquals, assertSpyCall, assertSpyCalls, @@ -10,16 +11,20 @@ import { stub, } from "@/dev_deps.ts"; import { createHash } from "@/lib/crypto/index.ts"; +import { exists } from "@/lib/file/index.ts"; import { join } from "@/lib/path/index.ts"; -import fetcher from "../../lib/fetcher/index.ts"; +import { S3StorageProvider } from "@/pdc/providers/storage/index.ts"; import { writeFile } from "../helpers/index.ts"; import { FileManager } from "./FileManager.ts"; describe("File Manager", () => { const GEO_PERIMETER_TMP_DIR = "/tmp/perimeter-geo-test"; - const RESSOURCE_URL = - "http://www.get.domaine.fr/system/files/documents/2022/09/file"; + const RESSOURCE_URL = "http://www.get.domaine.fr/system/files/documents/2022/09/file"; + const MIRROR_URL = "https://s3.domain.fr/bucket/0524479cc58cffd3d5db6b96ea893853744fe33f15ce76123f8a772a546c4252"; + const FETCH_ARGS: RequestInit = { method: "GET", redirect: "follow" }; const FILE_CONTENT_STRING = "{}"; + const FILE_SHA256 = "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"; + let READABLE_STREAM: ReadableStream; let fileManager: FileManager; @@ -47,79 +52,103 @@ describe("File Manager", () => { }); it("should return ressource file if available", async () => { - const getStub = stub(fetcher, "get"); + const fetchFunctionStub = stub(window, "fetch"); try { // Arrange - const existingFilepath = join( - fileManager.downloadPath, - await createHash(RESSOURCE_URL), - ); + const existingFilepath = join(fileManager.downloadPath, await createHash(RESSOURCE_URL)); await mkdir(fileManager.downloadPath, { recursive: true }); - await writeFile(READABLE_STREAM, existingFilepath); + await writeFile(existingFilepath, READABLE_STREAM); // Act - const filepath = await fileManager.download( - RESSOURCE_URL, - ); + const filepath = await fileManager.download({ url: RESSOURCE_URL }); assertEquals(await readFile(filepath, "utf8"), FILE_CONTENT_STRING); } finally { - getStub.restore(); + fetchFunctionStub.restore(); } // Assert - assertSpyCalls(getStub, 0); + assertSpyCalls(fetchFunctionStub, 0); + }); + + it("should discard the file if the hash is invalid", async () => { + const fetchFunctionStub = stub(window, "fetch", async () => new Response(READABLE_STREAM)); + + try { + // Arrange + const existingFilepath = join(fileManager.downloadPath, await createHash(RESSOURCE_URL)); + await mkdir(fileManager.downloadPath, { recursive: true }); + await writeFile( + existingFilepath, + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("invalid content")); + controller.close(); + }, + }), + ); + + // make sure the file exists with the invalid content + assert(true === await exists(existingFilepath)); + assert(await readFile(existingFilepath, "utf8"), "invalid content"); + + // Act + await fileManager.download({ url: RESSOURCE_URL, sha256: FILE_SHA256 }); + + // make sure the file has been removed and re-downloaded + // with the right content + assert(true === await exists(existingFilepath)); + assert(await readFile(existingFilepath, "utf8"), FILE_CONTENT_STRING); + } finally { + fetchFunctionStub.restore(); + } }); - it("should download ressource url if not on fs", async () => { + it("should download from mirror if missing locally", async () => { // Arrange - const getStub = stub( - fetcher, - "get", - async () => new Response(READABLE_STREAM), - ); + const fetchFunctionStub = stub(window, "fetch", async () => new Response(READABLE_STREAM)); + try { // Act - const filepath = await fileManager.download( - RESSOURCE_URL, - ); - assertEquals(await readFile(filepath, "utf8"), FILE_CONTENT_STRING); + const filepath = await fileManager.download({ url: RESSOURCE_URL }); + assertEquals(await readFile(filepath, { encoding: "utf8" }), FILE_CONTENT_STRING); } finally { - getStub.restore(); + fetchFunctionStub.restore(); } // Assert - assertSpyCall(getStub, 0, { args: [RESSOURCE_URL] }); + assertSpyCall(fetchFunctionStub, 0, { args: [MIRROR_URL, FETCH_ARGS] }); }); - it("should fallback to miror if any error code with download ressource", async () => { + it("should fallback to source on cache miss", async () => { // Arrange - let nb = 0; - const getStub = stub( - fetcher, - "get", - async () => { - if (nb === 0) { - nb += 1; - throw new Error("Invalid URL"); - } else { - return new Response(READABLE_STREAM); - } - }, - ); + const s3ProviderStub = stub(S3StorageProvider.prototype, "upload", async () => { + return MIRROR_URL; + }); + + let fetchFunctionCalls = 0; + const fetchFunctionStub = stub(window, "fetch", async () => { + if (fetchFunctionCalls === 0) { + fetchFunctionCalls = 1; + // first call to the mirror throws an error + throw new Error("Invalid URL"); + } + + // subsequent calls to the source return the file + // call 2: download from source + // call 3: upload to cache + return new Response(READABLE_STREAM); + }); try { // Act - const filepath = await fileManager.download( - RESSOURCE_URL, - ); + const filepath = await fileManager.download({ url: RESSOURCE_URL }); assertEquals(await readFile(filepath, "utf8"), FILE_CONTENT_STRING); } finally { - getStub.restore(); + fetchFunctionStub.restore(); + s3ProviderStub.restore(); } // Assert - assertSpyCalls(getStub, 2); - assertSpyCall(getStub, 0, { args: [RESSOURCE_URL] }); - assertSpyCall(getStub, 1, { - args: [`${fileManager.mirrorUrl}/${await createHash(RESSOURCE_URL)}`], - }); + assertSpyCalls(fetchFunctionStub, 2); + assertSpyCall(fetchFunctionStub, 0, { args: [MIRROR_URL, FETCH_ARGS] }); + assertSpyCall(fetchFunctionStub, 1, { args: [RESSOURCE_URL, FETCH_ARGS] }); }); });