add tests

api/src/etl/providers/FileManager.unit.spec.ts

@@ -1,6 +1,7 @@
 import { mkdir, readFile, rm } from "@/deps.ts";
 import {
   afterAll,
+  assert,
   assertEquals,
   assertSpyCall,
   assertSpyCalls,
@@ -10,16 +11,20 @@ import {
   stub,
 } from "@/dev_deps.ts";
 import { createHash } from "@/lib/crypto/index.ts";
+import { exists } from "@/lib/file/index.ts";
 import { join } from "@/lib/path/index.ts";
-import fetcher from "../../lib/fetcher/index.ts";
+import { S3StorageProvider } from "@/pdc/providers/storage/index.ts";
 import { writeFile } from "../helpers/index.ts";
 import { FileManager } from "./FileManager.ts";
 
 describe("File Manager", () => {
   const GEO_PERIMETER_TMP_DIR = "/tmp/perimeter-geo-test";
-  const RESSOURCE_URL =
-    "http://www.get.domaine.fr/system/files/documents/2022/09/file";
+  const RESSOURCE_URL = "http://www.get.domaine.fr/system/files/documents/2022/09/file";
+  const MIRROR_URL = "https://s3.domain.fr/bucket/0524479cc58cffd3d5db6b96ea893853744fe33f15ce76123f8a772a546c4252";
+  const FETCH_ARGS: RequestInit = { method: "GET", redirect: "follow" };
   const FILE_CONTENT_STRING = "{}";
+  const FILE_SHA256 = "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a";
+
   let READABLE_STREAM: ReadableStream;
   let fileManager: FileManager;
 
@@ -47,79 +52,103 @@ describe("File Manager", () => {
   });
 
   it("should return ressource file if available", async () => {
-    const getStub = stub(fetcher, "get");
+    const fetchFunctionStub = stub(window, "fetch");
     try {
       // Arrange
-      const existingFilepath = join(
-        fileManager.downloadPath,
-        await createHash(RESSOURCE_URL),
-      );
+      const existingFilepath = join(fileManager.downloadPath, await createHash(RESSOURCE_URL));
       await mkdir(fileManager.downloadPath, { recursive: true });
-      await writeFile(READABLE_STREAM, existingFilepath);
+      await writeFile(existingFilepath, READABLE_STREAM);
 
       // Act
-      const filepath = await fileManager.download(
-        RESSOURCE_URL,
-      );
+      const filepath = await fileManager.download({ url: RESSOURCE_URL });
       assertEquals(await readFile(filepath, "utf8"), FILE_CONTENT_STRING);
     } finally {
-      getStub.restore();
+      fetchFunctionStub.restore();
     }
     // Assert
-    assertSpyCalls(getStub, 0);
+    assertSpyCalls(fetchFunctionStub, 0);
+  });
+
+  it("should discard the file if the hash is invalid", async () => {
+    const fetchFunctionStub = stub(window, "fetch", async () => new Response(READABLE_STREAM));
+
+    try {
+      // Arrange
+      const existingFilepath = join(fileManager.downloadPath, await createHash(RESSOURCE_URL));
+      await mkdir(fileManager.downloadPath, { recursive: true });
+      await writeFile(
+        existingFilepath,
+        new ReadableStream({
+          start(controller) {
+            controller.enqueue(new TextEncoder().encode("invalid content"));
+            controller.close();
+          },
+        }),
+      );
+
+      // make sure the file exists with the invalid content
+      assert(true === await exists(existingFilepath));
+      assert(await readFile(existingFilepath, "utf8"), "invalid content");
+
+      // Act
+      await fileManager.download({ url: RESSOURCE_URL, sha256: FILE_SHA256 });
+
+      // make sure the file has been removed and re-downloaded
+      // with the right content
+      assert(true === await exists(existingFilepath));
+      assert(await readFile(existingFilepath, "utf8"), FILE_CONTENT_STRING);
+    } finally {
+      fetchFunctionStub.restore();
+    }
   });
 
-  it("should download ressource url if not on fs", async () => {
+  it("should download from mirror if missing locally", async () => {
     // Arrange
-    const getStub = stub(
-      fetcher,
-      "get",
-      async () => new Response(READABLE_STREAM),
-    );
+    const fetchFunctionStub = stub(window, "fetch", async () => new Response(READABLE_STREAM));
+
     try {
       // Act
-      const filepath = await fileManager.download(
-        RESSOURCE_URL,
-      );
-      assertEquals(await readFile(filepath, "utf8"), FILE_CONTENT_STRING);
+      const filepath = await fileManager.download({ url: RESSOURCE_URL });
+      assertEquals(await readFile(filepath, { encoding: "utf8" }), FILE_CONTENT_STRING);
     } finally {
-      getStub.restore();
+      fetchFunctionStub.restore();
     }
     // Assert
-    assertSpyCall(getStub, 0, { args: [RESSOURCE_URL] });
+    assertSpyCall(fetchFunctionStub, 0, { args: [MIRROR_URL, FETCH_ARGS] });
   });
 
-  it("should fallback to miror if any error code with download ressource", async () => {
+  it("should fallback to source on cache miss", async () => {
     // Arrange
-    let nb = 0;
-    const getStub = stub(
-      fetcher,
-      "get",
-      async () => {
-        if (nb === 0) {
-          nb += 1;
-          throw new Error("Invalid URL");
-        } else {
-          return new Response(READABLE_STREAM);
-        }
-      },
-    );
+    const s3ProviderStub = stub(S3StorageProvider.prototype, "upload", async () => {
+      return MIRROR_URL;
+    });
+
+    let fetchFunctionCalls = 0;
+    const fetchFunctionStub = stub(window, "fetch", async () => {
+      if (fetchFunctionCalls === 0) {
+        fetchFunctionCalls = 1;
+        // first call to the mirror throws an error
+        throw new Error("Invalid URL");
+      }
+
+      // subsequent calls to the source return the file
+      // call 2: download from source
+      // call 3: upload to cache
+      return new Response(READABLE_STREAM);
+    });
 
     try {
       // Act
-      const filepath = await fileManager.download(
-        RESSOURCE_URL,
-      );
+      const filepath = await fileManager.download({ url: RESSOURCE_URL });
       assertEquals(await readFile(filepath, "utf8"), FILE_CONTENT_STRING);
     } finally {
-      getStub.restore();
+      fetchFunctionStub.restore();
+      s3ProviderStub.restore();
     }
 
     // Assert
-    assertSpyCalls(getStub, 2);
-    assertSpyCall(getStub, 0, { args: [RESSOURCE_URL] });
-    assertSpyCall(getStub, 1, {
-      args: [`${fileManager.mirrorUrl}/${await createHash(RESSOURCE_URL)}`],
-    });
+    assertSpyCalls(fetchFunctionStub, 2);
+    assertSpyCall(fetchFunctionStub, 0, { args: [MIRROR_URL, FETCH_ARGS] });
+    assertSpyCall(fetchFunctionStub, 1, { args: [RESSOURCE_URL, FETCH_ARGS] });
   });
 });