dumili: Allow paddleocr to have a URL as input instead of base64 data…

…, wip

apps/dumili/api/docker-compose-dev.yml

@@ -12,6 +12,7 @@ services:
       - "8006:8081"
     volumes:
       - /root/.paddleocr
+      - ./paddleocr/server.py:/server.py
     build:
       context: ../../..
       dockerfile: apps/dumili/api/paddleocr/Dockerfile

apps/dumili/api/package.json

@@ -11,9 +11,7 @@
     "prisma-pull-generate": "prisma db pull && prisma generate",
     "prisma-generate": "prisma generate",
     "prisma-migrate": "prisma migrate deploy",
-    "dev:setup": "docker compose -f docker-compose-dev.yml up --force-recreate -d",
-    "dev:blocking": "bash ../../../packages/prisma-schemas/wait-until-db-ready.bash && prisma migrate deploy && prisma generate",
-    "dev": "concurrently --kill-others-on-fail -n typecheck,bun \"tsc --noEmit\" \"bun --inspect run --hot index.ts\"",
+    "dev": "docker compose -f docker-compose-dev.yml up --force-recreate -d && bash ../../../packages/prisma-schemas/wait-until-db-ready.bash && prisma migrate deploy && prisma generate && concurrently --kill-others-on-fail -n docker-compose,typecheck,bun \"docker compose -f docker-compose-dev.yml logs -f\" \"tsc --noEmit\" \"bun --inspect run --hot index.ts\"",
     "prod:deploy": "DIR=apps/dumili/api pnpm -F '~ci' prod:docker-compose-up",
     "prod-build-docker-api": "REPO_NAME=ghcr.io/bperel/dumili-api pnpm -F '~ci' prod:build-docker -f apps/dumili/api/Dockerfile",
     "prod-build-docker-kumiko": "REPO_NAME=ghcr.io/bperel/kumiko pnpm -F '~ci' prod:build-docker -f apps/dumili/kumiko/Dockerfile",

apps/dumili/api/paddleocr/Dockerfile

@@ -8,7 +8,7 @@ RUN cd openssl-1.1.1v && ./config && make -j$(nproc) && make test
 
 FROM python:3.9-slim
 
-RUN apt update && apt install -y --no-install-recommends libgl1-mesa-glx libgomp1 libglib2.0-0 gcc python3-dev && apt clean
+RUN apt update && apt install -y --no-install-recommends libgl1-mesa-glx libgomp1 libglib2.0-0 gcc python3-dev patch && apt clean
 
 COPY --from=libs /home/openssl-1.1.1v/libcrypto.so.1.1 /lib/
 COPY --from=libs /home/openssl-1.1.1v/libssl.so.1.1 /lib/

apps/dumili/api/paddleocr/requirements.txt

@@ -1,2 +1,2 @@
-paddleocr>=2.7.0.2
-paddlepaddle>=2.6.0
+paddleocr>=2.8.1
+paddlepaddle>=3.0.0b1

apps/dumili/api/paddleocr/server.py

@@ -4,10 +4,7 @@
 utils.run_check()
 from paddleocr import PaddleOCR
 from http.server import BaseHTTPRequestHandler, HTTPServer
-import random
 import json
-import os
-import base64
 
 # select countrycode, GROUP_CONCAT(languagecode ORDER BY entries_with_language DESC) AS languages
 # from (select countrycode, coalesce(inducks_entry.languagecode, ip.languagecode) as languagecode, count(*) as entries_with_language
@@ -332,42 +329,27 @@
 
 class PaddleOCRRequestHandler(BaseHTTPRequestHandler):
     def do_POST(self):
-        content_length = int(self.headers['Content-Length']) 
-        base64Text = self.rfile.read(content_length)
-
-        file_name = ''.join((random.choice('abcdefghi') for i in range(5))) + '.png'
-        try:
-            file_content = base64.b64decode(base64Text)
-            with open(file_name,"wb") as f:
-                f.write(file_content)
-        except Exception as e:
-            print(str(e))
-
-        result = None
-        # result = ocr_languages['french'].ocr(file_name, cls=True)
-        # result = result[0]
-        if result is None:
-            os.remove(file_name)
-            self.send_response(200)
-            self.send_header('Content-type', 'application/json')
-            self.end_headers()
-            self.wfile.write(json.dumps([]).encode())
-            return
-
-        boxes = [line[0] for line in result]
-        texts = [line[1][0] for line in result]
-        scores = [line[1][1] for line in result]
+        content_length = int(self.headers['Content-Length'])
+        post_data = json.loads(self.rfile.read(content_length))
+        url = post_data['url']
+        language = post_data['language']
+        result = ocr_languages[language].ocr(url, cls=True)
+        result = result[0]
 
         converted_data = []
-        for i in range(len(boxes)):
-            converted_item = {
-                "box": boxes[i],
-                "text": texts[i],
-                "confidence": scores[i]
-            }
-            converted_data.append(converted_item)
+        if result is not None:
+          boxes = [line[0] for line in result]
+          texts = [line[1][0] for line in result]
+          scores = [line[1][1] for line in result]
+
+          for i in range(len(boxes)):
+              converted_item = {
+                  "box": boxes[i],
+                  "text": texts[i],
+                  "confidence": scores[i]
+              }
+              converted_data.append(converted_item)
 
-        os.remove(file_name)
         self.send_response(200)
         self.send_header('Content-type', 'application/json')
         self.end_headers()

apps/dumili/api/prisma/schema.prisma

@@ -75,8 +75,9 @@ model storySuggestion {
   storycode         String              @db.VarChar(19)
   entryId           Int                 @map("entry_id")
   ocrDetailsId      Int?                @map("ocr_details_id")
+  ocrDetails        aiOcrPossibleStory?
+  isChosenByAi      Boolean             @default(false) @map("is_chosen_by_ai")
   acceptedOnEntries entry[]             @relation("entry_accepted_story_suggested_idTostory_suggestion")
-  ocrDetails        aiOcrPossibleStory? @relation(fields: [ocrDetailsId], references: [id], onDelete: Restrict, onUpdate: Restrict, map: "story_suggestion_ai_ocr_possible_story_id_fk")
   entry             entry               @relation(fields: [entryId], references: [id], onDelete: Cascade, onUpdate: Restrict, map: "story_suggestion_entry_id_fk")
 
   @@index([entryId], map: "story_suggestion_entry_id_fk")
@@ -99,34 +100,31 @@ model issueSuggestion {
 }
 
 model aiOcrPossibleStory {
-  id               Int               @id @default(autoincrement())
-  pageId           Int               @map("page_id")
-  ocrResultId      Int?              @map("ocr_result_id")
-  confidence       Int               @db.SmallInt
-  ocrResult        aiOcrResult?      @relation(fields: [ocrResultId], references: [id], onDelete: Restrict, onUpdate: Restrict, map: "ai_ocr_possible_story_ai_ocr_result_id_fk")
-  page             page              @relation(fields: [pageId], references: [id], onUpdate: Restrict, map: "ai_ocr_possible_story_page_id_fk")
-  storySuggestions storySuggestion[]
+  id                Int             @id @default(autoincrement())
+  pageId            Int             @map("page_id")
+  page              page            @relation(fields: [pageId], references: [id], onUpdate: Restrict, map: "ai_ocr_possible_story_page_id_fk")
+  score             Int             @db.SmallInt
+  storySuggestionId Int             @unique @map("story_suggestion_id")
+  storySuggestion   storySuggestion @relation(fields: [storySuggestionId], references: [id], onDelete: Restrict, onUpdate: Restrict, map: "ai_ocr_possible_story_story_suggestion_id_fk")
 
   @@index([pageId], map: "ai_ocr_possible_story_page_id_fk")
-  @@index([ocrResultId], map: "ai_ocr_possible_story_ai_ocr_result_id_fk")
   @@map("ai_ocr_possible_story")
 }
 
 model aiOcrResult {
-  id                 Int                  @id @default(autoincrement())
-  pageId             Int                  @map("page_id")
-  x1                 Int
-  x2                 Int
-  x3                 Int
-  x4                 Int
-  y1                 Int
-  y2                 Int
-  y3                 Int
-  y4                 Int
-  text               String               @db.Text
-  confidence         Float                @db.Float
-  page               page                 @relation(fields: [pageId], references: [id], onUpdate: Restrict, map: "ai_ocr_result_page_id_fk")
-  aiOcrPossibleStory aiOcrPossibleStory[]
+  id         Int    @id @default(autoincrement())
+  pageId     Int    @map("page_id")
+  x1         Int
+  x2         Int
+  x3         Int
+  x4         Int
+  y1         Int
+  y2         Int
+  y3         Int
+  y4         Int
+  text       String @db.Text
+  confidence Float  @db.Float
+  page       page   @relation(fields: [pageId], references: [id], onUpdate: Restrict, map: "ai_ocr_result_page_id_fk")
 
   @@index([pageId], map: "ai_ocr_result_page_id_fk")
   @@map("ai_ocr_result")

apps/dumili/api/services/indexation/index.ts

@@ -1,10 +1,9 @@
-import axios from "axios";
 import type { Server, Socket } from "socket.io";
 
 import type { NamespaceWithData, SessionDataWithIndexation } from "~/index";
 import { prisma } from "~/index";
 import CoaServices from "~dm-services/coa/types";
-import { storyKinds } from "~dumili-types/storyKinds";
+import { COVER, ILLUSTRATION, STORY, storyKinds } from "~dumili-types/storyKinds";
 import { getEntryFromPage, getEntryPages } from "~dumili-utils/entryPages";
 import type {
   entry,
@@ -159,7 +158,7 @@ export default (io: Server) => {
                   await prisma.storyKindSuggestion.findFirstOrThrow({
                     where: {
                       entryId: newEntry.id,
-                      kind: 'c',
+                      kind: COVER,
                     },
                     select: {
                       id: true,
@@ -331,89 +330,69 @@ export default (io: Server) => {
         const { indexation } = indexationSocket.data;
 
         const entry = indexation.entries.find(({ id }) => id === entryId);
-        if (entry?.acceptedStoryKind?.kind === "n") {
-          const entryPages = getEntryPages(indexation, entryId);
-          const results = await Promise.all(
-            entryPages
-              .map(({ aiKumikoResultPanels, url }) => ({
-                pageUrl: url,
-                panel: aiKumikoResultPanels[0],
-              }))
-              .map(({ panel, pageUrl }, idx) =>
-                axios<Buffer>({
-                  url: pageUrl.replace(
-                    "/pg_",
-                    `/c_crop,h_${panel.height},w_${panel.width},x_${panel.x},y_${panel.y},pg_`
-                  ),
-                  responseType: "arraybuffer",
-                }).then(({ data }) =>
-                  runOcr(data.toString("base64")).then((ocrResults) => ({
-                    pageId: entryPages[idx].id,
-                    ocrResults,
-                  })),
-                ),
-              ),
+        if (entry?.acceptedStoryKind?.kind === STORY) {
+          const { url, aiKumikoResultPanels, id: pageId } = getEntryPages(indexation, entryId)[0]!;
+          const firstPanel = aiKumikoResultPanels[0];
+          const firstPanelUrl = url.replace(
+            "/pg_",
+            `/c_crop,h_${firstPanel.height},w_${firstPanel.width},x_${firstPanel.x},y_${firstPanel.y},pg_`
           );
 
-          prisma.aiOcrResult.createMany({
-            data: results
-              .map(({ pageId, ocrResults }) =>
-                ocrResults.map(
-                  ({
-                    confidence,
-                    text,
-                    box: [[x1, y1], [x2, y2], [x3, y3], [x4, y4]],
-                  }) => ({
-                    pageId,
+          const ocrResults = await runOcr(firstPanelUrl);
+
+          const { results: searchResults } = await coaServices.searchStory(
+            ocrResults.map(({ text }) => text),
+            false,
+          );
+
+          const { stories: storyDetails } = await coaServices.getStoryDetails(
+            searchResults.map(({ storycode }) => storycode),
+          );
+
+          const { storyversions: storyversionDetails } = await coaServices.getStoryversionsDetails(
+            searchResults.map(({ storycode }) => storyDetails![storycode].originalstoryversioncode!),
+          );
+
+          const storyResults = searchResults.filter(({ storycode }) =>
+            storyversionDetails![storyDetails![storycode].originalstoryversioncode!].kind === STORY
+          );
+
+          await
+            prisma.page.update({
+              where: {
+                id: pageId,
+              },
+              data: {
+                aiOcrResults: {
+                  deleteMany: {},
+                  create: ocrResults.map(({ confidence, text, box: [[x1, y1], [x2, y2], [x3, y3], [x4, y4]] }) => ({
                     confidence,
                     text,
                     x1,
-                    x2,
                     y1,
+                    x2,
                     y2,
                     x3,
-                    x4,
                     y3,
-                    y4,
-                  }),
-                ),
-              )
-              .flat(),
-          });
-
-          await prisma.$transaction(
-            (
-              await Promise.all(
-                results.map(({ ocrResults }) =>
-                  coaServices.searchStory(
-                    ocrResults.map(({ text }) => text),
-                    false,
-                  ),
-                ),
-              )
-            )
-              .map(({ results: searchResults }, idx) => [
-                prisma.storySuggestion.deleteMany({
-                  where: {
-                    entryId,
-                  },
-                }),
-                prisma.storySuggestion.createMany({
-                  data: searchResults.map(({ storycode }) => ({
-                    entryId,
-                    storycode,
-                  })),
-                }),
-                prisma.aiOcrPossibleStory.createMany({
-                  data: searchResults.map(({ storycode, score }) => ({
-                    pageId: entryPages[idx]!.id,
-                    storycode,
-                    confidence: score,
-                  })),
-                }),
-              ])
-              .flat(),
-          );
+                    x4,
+                    y4
+                  }))
+                },
+                aiOcrPossibleStories: {
+                  deleteMany: {},
+                  create: storyResults.map(({ storycode, score }) => ({
+                    score,
+                    storySuggestion: {
+                      create: {
+                        storycode,
+                        entryId,
+                        isChosenByAi: true
+                      }
+                    }
+                  }))
+                }
+              }
+            });
 
           callback({ status: "OK" });
         } else {
@@ -436,9 +415,9 @@ const inferStoryKindFromAiResults = (
 ) =>
 (panelsOfPage.length === 1
   ? pageNumber === 1
-    ? "c" // cover
-    : "i" // illustration
-  : "n" // story
+    ? COVER
+    : ILLUSTRATION
+  : STORY
 )
 
 const acceptStorySuggestion = async (

apps/dumili/api/services/indexation/ocr.ts

@@ -19,5 +19,7 @@ export const extendBoundaries = (
   height: height + extendBy,
 });
 
-export const runOcr = async (base64: string): Promise<OcrResult[]> =>
-  axios.post(process.env.OCR_HOST!, base64).then(({ data }) => data);
+export const runOcr = async (url: string): Promise<OcrResult[]> => {
+  console.log("Running OCR on", url);
+  return axios.post(process.env.OCR_HOST!, { url, language: 'french' }).then(({ data }) => data);
+};

apps/dumili/api/services/indexation/types.ts

@@ -14,7 +14,7 @@ export const indexationPayloadInclude = {
       aiKumikoResultPanels: true,
       aiOcrPossibleStories: {
         include: {
-          storySuggestions: true,
+          storySuggestion: true,
         },
       },
       aiOcrResults: true,
@@ -24,10 +24,18 @@ export const indexationPayloadInclude = {
   issueSuggestions: true,
   entries: {
     include: {
-      acceptedStory: true,
+      acceptedStory: {
+        include: {
+          ocrDetails: true,
+        },
+      },
       acceptedStoryKind: true,
       storyKindSuggestions: true,
-      storySuggestions: true,
+      storySuggestions: {
+        include: {
+          ocrDetails: true,
+        }
+      },
     },
   },
 } as const;
@@ -78,7 +86,7 @@ export default abstract class {
     suggestion: Prisma.storySuggestionUncheckedCreateInput,
     callback: (
       data: Errorable<
-        { createdStorySuggestion: storySuggestion },
+        { createdStorySuggestion: Pick<storySuggestion, 'id' | 'storycode'> },
         "You are not allowed to update this resource"
       >,
     ) => void,