From 08fb04c3898146fc6332bd309162272d8656db1d Mon Sep 17 00:00:00 2001 From: Bruno Perel Date: Mon, 11 Nov 2024 23:38:59 +0100 Subject: [PATCH] dumili: Allow paddleocr to have a URL as input instead of base64 data, wip --- apps/dumili/api/docker-compose-dev.yml | 1 + apps/dumili/api/package.json | 4 +- apps/dumili/api/paddleocr/Dockerfile | 2 +- apps/dumili/api/paddleocr/requirements.txt | 4 +- apps/dumili/api/paddleocr/server.py | 54 +++---- apps/dumili/api/prisma/schema.prisma | 44 +++--- apps/dumili/api/services/indexation/index.ts | 139 ++++++++---------- apps/dumili/api/services/indexation/ocr.ts | 6 +- apps/dumili/api/services/indexation/types.ts | 16 +- apps/dumili/src/components/Entry.vue | 22 ++- .../src/components/IssueSuggestionList.vue | 2 +- .../src/components/StorySuggestionList.vue | 26 ++-- apps/dumili/src/components/SuggestionList.vue | 90 +++++++----- apps/dumili/src/composables/useAi.ts | 5 +- apps/dumili/src/style.scss | 3 +- apps/dumili/types/storyKinds.ts | 31 ++-- .../prisma-schemas/wait-until-db-ready.bash | 2 +- 17 files changed, 232 insertions(+), 219 deletions(-) mode change 100644 => 100755 apps/dumili/api/paddleocr/server.py diff --git a/apps/dumili/api/docker-compose-dev.yml b/apps/dumili/api/docker-compose-dev.yml index e95b9544d..d09c3430c 100644 --- a/apps/dumili/api/docker-compose-dev.yml +++ b/apps/dumili/api/docker-compose-dev.yml @@ -12,6 +12,7 @@ services: - "8006:8081" volumes: - /root/.paddleocr + - ./paddleocr/server.py:/server.py build: context: ../../.. dockerfile: apps/dumili/api/paddleocr/Dockerfile diff --git a/apps/dumili/api/package.json b/apps/dumili/api/package.json index f99215bf9..80cb39d8b 100644 --- a/apps/dumili/api/package.json +++ b/apps/dumili/api/package.json @@ -11,9 +11,7 @@ "prisma-pull-generate": "prisma db pull && prisma generate", "prisma-generate": "prisma generate", "prisma-migrate": "prisma migrate deploy", - "dev:setup": "docker compose -f docker-compose-dev.yml up --force-recreate -d", - "dev:blocking": "bash ../../../packages/prisma-schemas/wait-until-db-ready.bash && prisma migrate deploy && prisma generate", - "dev": "concurrently --kill-others-on-fail -n typecheck,bun \"tsc --noEmit\" \"bun --inspect run --hot index.ts\"", + "dev": "docker compose -f docker-compose-dev.yml up --force-recreate -d && bash ../../../packages/prisma-schemas/wait-until-db-ready.bash && prisma migrate deploy && prisma generate && concurrently --kill-others-on-fail -n docker-compose,typecheck,bun \"docker compose -f docker-compose-dev.yml logs -f\" \"tsc --noEmit\" \"bun --inspect run --hot index.ts\"", "prod:deploy": "DIR=apps/dumili/api pnpm -F '~ci' prod:docker-compose-up", "prod-build-docker-api": "REPO_NAME=ghcr.io/bperel/dumili-api pnpm -F '~ci' prod:build-docker -f apps/dumili/api/Dockerfile", "prod-build-docker-kumiko": "REPO_NAME=ghcr.io/bperel/kumiko pnpm -F '~ci' prod:build-docker -f apps/dumili/kumiko/Dockerfile", diff --git a/apps/dumili/api/paddleocr/Dockerfile b/apps/dumili/api/paddleocr/Dockerfile index 0b6beb5a5..819007bac 100644 --- a/apps/dumili/api/paddleocr/Dockerfile +++ b/apps/dumili/api/paddleocr/Dockerfile @@ -8,7 +8,7 @@ RUN cd openssl-1.1.1v && ./config && make -j$(nproc) && make test FROM python:3.9-slim -RUN apt update && apt install -y --no-install-recommends libgl1-mesa-glx libgomp1 libglib2.0-0 gcc python3-dev && apt clean +RUN apt update && apt install -y --no-install-recommends libgl1-mesa-glx libgomp1 libglib2.0-0 gcc python3-dev patch && apt clean COPY --from=libs /home/openssl-1.1.1v/libcrypto.so.1.1 /lib/ COPY --from=libs /home/openssl-1.1.1v/libssl.so.1.1 /lib/ diff --git a/apps/dumili/api/paddleocr/requirements.txt b/apps/dumili/api/paddleocr/requirements.txt index 5633d375d..2a23365e5 100644 --- a/apps/dumili/api/paddleocr/requirements.txt +++ b/apps/dumili/api/paddleocr/requirements.txt @@ -1,2 +1,2 @@ -paddleocr>=2.7.0.2 -paddlepaddle>=2.6.0 +paddleocr>=2.8.1 +paddlepaddle>=3.0.0b1 diff --git a/apps/dumili/api/paddleocr/server.py b/apps/dumili/api/paddleocr/server.py old mode 100644 new mode 100755 index 6dc7471f8..a01367bf5 --- a/apps/dumili/api/paddleocr/server.py +++ b/apps/dumili/api/paddleocr/server.py @@ -4,10 +4,7 @@ utils.run_check() from paddleocr import PaddleOCR from http.server import BaseHTTPRequestHandler, HTTPServer -import random import json -import os -import base64 # select countrycode, GROUP_CONCAT(languagecode ORDER BY entries_with_language DESC) AS languages # from (select countrycode, coalesce(inducks_entry.languagecode, ip.languagecode) as languagecode, count(*) as entries_with_language @@ -332,42 +329,27 @@ class PaddleOCRRequestHandler(BaseHTTPRequestHandler): def do_POST(self): - content_length = int(self.headers['Content-Length']) - base64Text = self.rfile.read(content_length) - - file_name = ''.join((random.choice('abcdefghi') for i in range(5))) + '.png' - try: - file_content = base64.b64decode(base64Text) - with open(file_name,"wb") as f: - f.write(file_content) - except Exception as e: - print(str(e)) - - result = None - # result = ocr_languages['french'].ocr(file_name, cls=True) - # result = result[0] - if result is None: - os.remove(file_name) - self.send_response(200) - self.send_header('Content-type', 'application/json') - self.end_headers() - self.wfile.write(json.dumps([]).encode()) - return - - boxes = [line[0] for line in result] - texts = [line[1][0] for line in result] - scores = [line[1][1] for line in result] + content_length = int(self.headers['Content-Length']) + post_data = json.loads(self.rfile.read(content_length)) + url = post_data['url'] + language = post_data['language'] + result = ocr_languages[language].ocr(url, cls=True) + result = result[0] converted_data = [] - for i in range(len(boxes)): - converted_item = { - "box": boxes[i], - "text": texts[i], - "confidence": scores[i] - } - converted_data.append(converted_item) + if result is not None: + boxes = [line[0] for line in result] + texts = [line[1][0] for line in result] + scores = [line[1][1] for line in result] + + for i in range(len(boxes)): + converted_item = { + "box": boxes[i], + "text": texts[i], + "confidence": scores[i] + } + converted_data.append(converted_item) - os.remove(file_name) self.send_response(200) self.send_header('Content-type', 'application/json') self.end_headers() diff --git a/apps/dumili/api/prisma/schema.prisma b/apps/dumili/api/prisma/schema.prisma index 0e0bcbaa6..7db403621 100644 --- a/apps/dumili/api/prisma/schema.prisma +++ b/apps/dumili/api/prisma/schema.prisma @@ -75,8 +75,9 @@ model storySuggestion { storycode String @db.VarChar(19) entryId Int @map("entry_id") ocrDetailsId Int? @map("ocr_details_id") + ocrDetails aiOcrPossibleStory? + isChosenByAi Boolean @default(false) @map("is_chosen_by_ai") acceptedOnEntries entry[] @relation("entry_accepted_story_suggested_idTostory_suggestion") - ocrDetails aiOcrPossibleStory? @relation(fields: [ocrDetailsId], references: [id], onDelete: Restrict, onUpdate: Restrict, map: "story_suggestion_ai_ocr_possible_story_id_fk") entry entry @relation(fields: [entryId], references: [id], onDelete: Cascade, onUpdate: Restrict, map: "story_suggestion_entry_id_fk") @@index([entryId], map: "story_suggestion_entry_id_fk") @@ -99,34 +100,31 @@ model issueSuggestion { } model aiOcrPossibleStory { - id Int @id @default(autoincrement()) - pageId Int @map("page_id") - ocrResultId Int? @map("ocr_result_id") - confidence Int @db.SmallInt - ocrResult aiOcrResult? @relation(fields: [ocrResultId], references: [id], onDelete: Restrict, onUpdate: Restrict, map: "ai_ocr_possible_story_ai_ocr_result_id_fk") - page page @relation(fields: [pageId], references: [id], onUpdate: Restrict, map: "ai_ocr_possible_story_page_id_fk") - storySuggestions storySuggestion[] + id Int @id @default(autoincrement()) + pageId Int @map("page_id") + page page @relation(fields: [pageId], references: [id], onUpdate: Restrict, map: "ai_ocr_possible_story_page_id_fk") + score Int @db.SmallInt + storySuggestionId Int @unique @map("story_suggestion_id") + storySuggestion storySuggestion @relation(fields: [storySuggestionId], references: [id], onDelete: Restrict, onUpdate: Restrict, map: "ai_ocr_possible_story_story_suggestion_id_fk") @@index([pageId], map: "ai_ocr_possible_story_page_id_fk") - @@index([ocrResultId], map: "ai_ocr_possible_story_ai_ocr_result_id_fk") @@map("ai_ocr_possible_story") } model aiOcrResult { - id Int @id @default(autoincrement()) - pageId Int @map("page_id") - x1 Int - x2 Int - x3 Int - x4 Int - y1 Int - y2 Int - y3 Int - y4 Int - text String @db.Text - confidence Float @db.Float - page page @relation(fields: [pageId], references: [id], onUpdate: Restrict, map: "ai_ocr_result_page_id_fk") - aiOcrPossibleStory aiOcrPossibleStory[] + id Int @id @default(autoincrement()) + pageId Int @map("page_id") + x1 Int + x2 Int + x3 Int + x4 Int + y1 Int + y2 Int + y3 Int + y4 Int + text String @db.Text + confidence Float @db.Float + page page @relation(fields: [pageId], references: [id], onUpdate: Restrict, map: "ai_ocr_result_page_id_fk") @@index([pageId], map: "ai_ocr_result_page_id_fk") @@map("ai_ocr_result") diff --git a/apps/dumili/api/services/indexation/index.ts b/apps/dumili/api/services/indexation/index.ts index b81f0e4d1..669a2dcd9 100644 --- a/apps/dumili/api/services/indexation/index.ts +++ b/apps/dumili/api/services/indexation/index.ts @@ -1,10 +1,9 @@ -import axios from "axios"; import type { Server, Socket } from "socket.io"; import type { NamespaceWithData, SessionDataWithIndexation } from "~/index"; import { prisma } from "~/index"; import CoaServices from "~dm-services/coa/types"; -import { storyKinds } from "~dumili-types/storyKinds"; +import { COVER, ILLUSTRATION, STORY, storyKinds } from "~dumili-types/storyKinds"; import { getEntryFromPage, getEntryPages } from "~dumili-utils/entryPages"; import type { entry, @@ -159,7 +158,7 @@ export default (io: Server) => { await prisma.storyKindSuggestion.findFirstOrThrow({ where: { entryId: newEntry.id, - kind: 'c', + kind: COVER, }, select: { id: true, @@ -331,89 +330,69 @@ export default (io: Server) => { const { indexation } = indexationSocket.data; const entry = indexation.entries.find(({ id }) => id === entryId); - if (entry?.acceptedStoryKind?.kind === "n") { - const entryPages = getEntryPages(indexation, entryId); - const results = await Promise.all( - entryPages - .map(({ aiKumikoResultPanels, url }) => ({ - pageUrl: url, - panel: aiKumikoResultPanels[0], - })) - .map(({ panel, pageUrl }, idx) => - axios({ - url: pageUrl.replace( - "/pg_", - `/c_crop,h_${panel.height},w_${panel.width},x_${panel.x},y_${panel.y},pg_` - ), - responseType: "arraybuffer", - }).then(({ data }) => - runOcr(data.toString("base64")).then((ocrResults) => ({ - pageId: entryPages[idx].id, - ocrResults, - })), - ), - ), + if (entry?.acceptedStoryKind?.kind === STORY) { + const { url, aiKumikoResultPanels, id: pageId } = getEntryPages(indexation, entryId)[0]!; + const firstPanel = aiKumikoResultPanels[0]; + const firstPanelUrl = url.replace( + "/pg_", + `/c_crop,h_${firstPanel.height},w_${firstPanel.width},x_${firstPanel.x},y_${firstPanel.y},pg_` ); - prisma.aiOcrResult.createMany({ - data: results - .map(({ pageId, ocrResults }) => - ocrResults.map( - ({ - confidence, - text, - box: [[x1, y1], [x2, y2], [x3, y3], [x4, y4]], - }) => ({ - pageId, + const ocrResults = await runOcr(firstPanelUrl); + + const { results: searchResults } = await coaServices.searchStory( + ocrResults.map(({ text }) => text), + false, + ); + + const { stories: storyDetails } = await coaServices.getStoryDetails( + searchResults.map(({ storycode }) => storycode), + ); + + const { storyversions: storyversionDetails } = await coaServices.getStoryversionsDetails( + searchResults.map(({ storycode }) => storyDetails![storycode].originalstoryversioncode!), + ); + + const storyResults = searchResults.filter(({ storycode }) => + storyversionDetails![storyDetails![storycode].originalstoryversioncode!].kind === STORY + ); + + await + prisma.page.update({ + where: { + id: pageId, + }, + data: { + aiOcrResults: { + deleteMany: {}, + create: ocrResults.map(({ confidence, text, box: [[x1, y1], [x2, y2], [x3, y3], [x4, y4]] }) => ({ confidence, text, x1, - x2, y1, + x2, y2, x3, - x4, y3, - y4, - }), - ), - ) - .flat(), - }); - - await prisma.$transaction( - ( - await Promise.all( - results.map(({ ocrResults }) => - coaServices.searchStory( - ocrResults.map(({ text }) => text), - false, - ), - ), - ) - ) - .map(({ results: searchResults }, idx) => [ - prisma.storySuggestion.deleteMany({ - where: { - entryId, - }, - }), - prisma.storySuggestion.createMany({ - data: searchResults.map(({ storycode }) => ({ - entryId, - storycode, - })), - }), - prisma.aiOcrPossibleStory.createMany({ - data: searchResults.map(({ storycode, score }) => ({ - pageId: entryPages[idx]!.id, - storycode, - confidence: score, - })), - }), - ]) - .flat(), - ); + x4, + y4 + })) + }, + aiOcrPossibleStories: { + deleteMany: {}, + create: storyResults.map(({ storycode, score }) => ({ + score, + storySuggestion: { + create: { + storycode, + entryId, + isChosenByAi: true + } + } + })) + } + } + }); callback({ status: "OK" }); } else { @@ -436,9 +415,9 @@ const inferStoryKindFromAiResults = ( ) => (panelsOfPage.length === 1 ? pageNumber === 1 - ? "c" // cover - : "i" // illustration - : "n" // story + ? COVER + : ILLUSTRATION + : STORY ) const acceptStorySuggestion = async ( diff --git a/apps/dumili/api/services/indexation/ocr.ts b/apps/dumili/api/services/indexation/ocr.ts index 4e74228b4..cf806b8b4 100644 --- a/apps/dumili/api/services/indexation/ocr.ts +++ b/apps/dumili/api/services/indexation/ocr.ts @@ -19,5 +19,7 @@ export const extendBoundaries = ( height: height + extendBy, }); -export const runOcr = async (base64: string): Promise => - axios.post(process.env.OCR_HOST!, base64).then(({ data }) => data); +export const runOcr = async (url: string): Promise => { + console.log("Running OCR on", url); + return axios.post(process.env.OCR_HOST!, { url, language: 'french' }).then(({ data }) => data); +}; diff --git a/apps/dumili/api/services/indexation/types.ts b/apps/dumili/api/services/indexation/types.ts index 492ebeb96..f0efdfe2a 100644 --- a/apps/dumili/api/services/indexation/types.ts +++ b/apps/dumili/api/services/indexation/types.ts @@ -14,7 +14,7 @@ export const indexationPayloadInclude = { aiKumikoResultPanels: true, aiOcrPossibleStories: { include: { - storySuggestions: true, + storySuggestion: true, }, }, aiOcrResults: true, @@ -24,10 +24,18 @@ export const indexationPayloadInclude = { issueSuggestions: true, entries: { include: { - acceptedStory: true, + acceptedStory: { + include: { + ocrDetails: true, + }, + }, acceptedStoryKind: true, storyKindSuggestions: true, - storySuggestions: true, + storySuggestions: { + include: { + ocrDetails: true, + } + }, }, }, } as const; @@ -78,7 +86,7 @@ export default abstract class { suggestion: Prisma.storySuggestionUncheckedCreateInput, callback: ( data: Errorable< - { createdStorySuggestion: storySuggestion }, + { createdStorySuggestion: Pick }, "You are not allowed to update this resource" >, ) => void, diff --git a/apps/dumili/src/components/Entry.vue b/apps/dumili/src/components/Entry.vue index 85200d58b..5a7902db6 100644 --- a/apps/dumili/src/components/Entry.vue +++ b/apps/dumili/src/components/Entry.vue @@ -12,8 +12,19 @@ :is-ai-source="(suggestion) => suggestion.isChosenByAi" :item-class="(suggestion) => [`kind-${suggestion.kind}`]" > -