Skip to content

Commit

Permalink
dumili: WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@bperel
bperel committed Dec 16, 2024
1 parent 0fd13fe commit dea8026
Show file tree
Hide file tree
Showing 18 changed files with 211 additions and 146 deletions.
61 changes: 42 additions & 19 deletions apps/dumili/api/prisma/schema.prisma
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,21 +33,21 @@ model entry {

model image {
id Int @id @default(autoincrement())
url String @unique(map: "page_image_url_uindex") @db.VarChar(255)
url String @unique(map: "image_url_uindex") @db.VarChar(255)
aiKumikoInferredStoryKind pageImageAiKumikoInferredStoryKind? @map("ai_kumiko_inferred_story_kind")
aiKumikoResultPanels aiKumikoResultPanel[]
aiOcrResults aiOcrResult[]
page page[]
@@map("page_image")
@@map("image")
}

model page {
id Int @id @default(autoincrement())
pageNumber Int @map("page_number") @db.SmallInt
indexationId String @map("indexation_id") @db.VarChar(20)
imageId Int? @map("image_id")
image image? @relation(fields: [imageId], references: [id], onDelete: Cascade, onUpdate: Restrict, map: "page_image_id_fk")
image image? @relation(fields: [imageId], references: [id], onDelete: Restrict, onUpdate: Restrict, map: "page_image_id_fk")
indexation indexation @relation(fields: [indexationId], references: [id], onDelete: Cascade, onUpdate: Restrict, map: "page_indexation_id_fk")
@@unique([indexationId, pageNumber], map: "page_indexation_id_page_number_uindex")
Expand All @@ -70,15 +70,14 @@ model indexation {
}

model storyKindSuggestion {
id Int @id @default(autoincrement())
id Int @id @default(autoincrement())
kind storyKind
isChosenByAi Boolean @default(false) @map("is_chosen_by_ai")
entryId Int @map("entry_id")
acceptedOnEntries entry[] @relation("entry_accepted_story_kind_suggested_idTostory_kind_suggestion")
entry entry @relation("story_kind_suggestion_entry_idToentry", fields: [entryId], references: [id], onDelete: Cascade, onUpdate: Restrict, map: "story_kind_suggestion_entry_id_fk")
entryId Int @map("entry_id")
acceptedOnEntries entry[] @relation("entry_accepted_story_kind_suggested_idTostory_kind_suggestion")
entry entry @relation("story_kind_suggestion_entry_idToentry", fields: [entryId], references: [id], onDelete: Cascade, onUpdate: Restrict, map: "story_kind_suggestion_entry_id_fk")
ai storyKindSuggestionAi?
@@index([entryId], map: "story_kind_suggestion_entry_id_fk")
@@index([isChosenByAi], map: "story_kind_suggestion_ai_source_page_id_fk")
@@map("story_kind_suggestion")
}

Expand All @@ -87,25 +86,25 @@ model storySuggestion {
storycode String @db.VarChar(19)
entryId Int @map("entry_id")
ocrDetailsId Int? @map("ocr_details_id")
isChosenByAi Boolean @map("is_chosen_by_ai")
ocrDetails aiOcrPossibleStory[]
acceptedOnEntries entry[] @relation("entry_accepted_story_suggested_idTostory_suggestion")
entry entry @relation(fields: [entryId], references: [id], onDelete: Cascade, onUpdate: Restrict, map: "story_suggestion_entry_id_fk")
ai storySuggestionAi?
@@index([entryId], map: "story_suggestion_entry_id_fk")
@@index([ocrDetailsId], map: "story_suggestion_ai_ocr_possible_story_id_fk")
@@map("story_suggestion")
}

model issueSuggestion {
id Int @id @default(autoincrement())
indexationId String @map("indexation_id") @db.VarChar(20)
isChosenByAi Boolean @map("is_chosen_by_ai")
publicationcode String @db.VarChar(12)
issuenumber String @db.VarChar(13)
issuecode String @db.VarChar(25)
acceptedOnEntries indexation[] @relation("indexation_accepted_issue_suggestion_idToissue_suggestion")
indexation indexation @relation(fields: [indexationId], references: [id], onDelete: Cascade, onUpdate: Restrict, map: "issue_suggestion_indexation_id_fk")
id Int @id @default(autoincrement())
indexationId String @map("indexation_id") @db.VarChar(20)
publicationcode String @db.VarChar(12)
issuenumber String @db.VarChar(13)
issuecode String @db.VarChar(25)
acceptedOnEntries indexation[] @relation("indexation_accepted_issue_suggestion_idToissue_suggestion")
indexation indexation @relation(fields: [indexationId], references: [id], onDelete: Cascade, onUpdate: Restrict, map: "issue_suggestion_indexation_id_fk")
ai issueSuggestionAi?
@@index([indexationId], map: "issue_suggestion_indexation_id_fk")
@@map("issue_suggestion")
Expand Down Expand Up @@ -134,7 +133,7 @@ model aiOcrResult {
y4 Int
text String @db.Text
confidence Float @db.Float
image image @relation(fields: [imageId], references: [id], onUpdate: Restrict, map: "ai_ocr_result_page_image_id_fk")
image image @relation(fields: [imageId], references: [id], onUpdate: Restrict, map: "ai_ocr_result_image_id_fk")
@@index([imageId], map: "ai_ocr_result_image_id_index")
@@map("ai_ocr_result")
Expand All @@ -153,6 +152,30 @@ model aiKumikoResultPanel {
@@map("ai_kumiko_result_panel")
}

model issueSuggestionAi {
id Int @id @default(autoincrement())
suggestionId Int @unique(map: "issue_suggestion_ai_issue_suggestion_id_unique_fk") @map("suggestion_id")
issueSuggestion issueSuggestion @relation(fields: [suggestionId], references: [id], onUpdate: Restrict, map: "issue_suggestion_ai_issue_suggestion_id_unique_fk")
@@map("issue_suggestion_ai")
}

model storyKindSuggestionAi {
id Int @id @default(autoincrement())
suggestionId Int @unique(map: "story_kind_suggestion_ai_story_kind_suggestion_id_unique_fk") @map("suggestion_id")
storyKindSuggestion storyKindSuggestion @relation(fields: [suggestionId], references: [id], onUpdate: Restrict, map: "story_kind_suggestion_ai_story_kind_suggestion_id_unique_fk")
@@map("story_kind_suggestion_ai")
}

model storySuggestionAi {
id Int @id @default(autoincrement())
suggestionId Int @unique(map: "story_suggestion_ai_story_suggestion_id_unique_fk") @map("suggestion_id")
storySuggestion storySuggestion @relation(fields: [suggestionId], references: [id], onUpdate: Restrict, map: "story_suggestion_ai_story_suggestion_id_fk")
@@map("story_suggestion_ai")
}

enum storyKind {
a
c
Expand Down
89 changes: 50 additions & 39 deletions apps/dumili/api/services/indexation/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,9 +233,11 @@ export default (io: Server) => {
},
data: {
storySuggestions: {
create: storyResults.map(({ storycode, score }) => ({
create: storyResults.map(({ storycode, score }): Prisma.storySuggestionCreateWithoutEntryInput => ({
storycode,
isChosenByAi: true,
ai: {
create: {},
},
ocrDetails: {
create: {
score,
Expand Down Expand Up @@ -269,7 +271,7 @@ export default (io: Server) => {

const setInferredEntryStoryKind = async (entryId: entry["id"]) => {
indexationSocket.emit("setInferredEntryStoryKind", entryId);
const indexation = indexationSocket.data.indexation;
const {indexation} = indexationSocket.data;
const mostInferredStoryKind = (
await prisma.image.groupBy({
by: ["aiKumikoInferredStoryKind"],
Expand All @@ -286,41 +288,36 @@ export default (io: Server) => {
},
},
})
).pop()!.aiKumikoInferredStoryKind!;
).pop();

const entryIdx = indexationSocket.data.indexation.entries.findIndex(
({ id }) => id === entryId
);
console.log(
`Kumiko: entry #${entryIdx}: inferred story kind is ${mostInferredStoryKind}`
`Kumiko: entry #${entryIdx}: inferred story kind is ${mostInferredStoryKind?.aiKumikoInferredStoryKind}`
);

const newEntry = await prisma.entry.update({
include: {
storyKindSuggestions: true,
},
await prisma.storyKindSuggestionAi.deleteMany({
where: {
id: entryId,
},
data: {
storyKindSuggestions: {
deleteMany: {
isChosenByAi: true,
},
updateMany: {
data: {
isChosenByAi: true,
},
where: {
kind: mostInferredStoryKind,
},
},
suggestionId: {
in: indexation.entries[entryIdx].storyKindSuggestions.map(
({ id }) => id
)
},
},
});
}});

let createdSuggestionAi = null
if (mostInferredStoryKind?.aiKumikoInferredStoryKind) {
createdSuggestionAi = await prisma.storyKindSuggestionAi.create({
data: {
suggestionId: indexation.entries[entryIdx].storyKindSuggestions.find(({ kind }) => kind === mostInferredStoryKind.aiKumikoInferredStoryKind)!.id
}
});
}

indexationSocket.emit("setInferredEntryStoryKindEnd", entryId);

return newEntry;
return createdSuggestionAi
};

indexationSocket.on("setPageUrl", async (id, url, callback) => {
Expand Down Expand Up @@ -679,10 +676,11 @@ export default (io: Server) => {
return;
}

const entry = getEntryFromPage(indexation, pageId)!;
await setInferredEntryStoryKind(entry.id);

callback({ status: "OK" });
setKumikoInferredPageStoryKinds(page, true).finally(async () => {
const entry = getEntryFromPage(indexation, pageId)!;
await setInferredEntryStoryKind(entry.id);
callback({ status: "OK" });
})
});

indexationSocket.on(
Expand All @@ -707,6 +705,18 @@ export default (io: Server) => {
}
);

indexationSocket.on("inferEntryStoryKind", async (entryId, callback) => {
if (indexationSocket.data.indexation.entries.some(({ id }) => id === entryId)) {
return setInferredEntryStoryKind(entryId).then(() => callback({ status: "OK" }));
}
else {
callback({
error: `This indexation does not have any entry with this ID`,
errorDetails: JSON.stringify({ entryId }),
});
}
});

indexationSocket.on("createEntry", async (callback) =>
createEntry(indexationSocket.data.indexation.id)
.then(async ({ id }) => {
Expand All @@ -716,13 +726,14 @@ export default (io: Server) => {
return setKumikoInferredPageStoryKinds(
getEntryPages(indexationSocket.data.indexation, id)[0]
).finally(() =>
setInferredEntryStoryKind(id).then((entry) =>
acceptStoryKindSuggestion(
entry.storyKindSuggestions.find(
({ isChosenByAi }) => isChosenByAi
)!.id,
entry.id
)
setInferredEntryStoryKind(id).then((inferredSuggestion) =>
{
if (inferredSuggestion) {
return acceptStoryKindSuggestion(inferredSuggestion?.suggestionId,
id
);
}
}
)
);
})
Expand All @@ -735,7 +746,7 @@ const inferStoryKindFromAiResults = (
panelsOfPage: KumikoProcessedResult[],
pageNumber: number
) =>
panelsOfPage.length === 1 ? (pageNumber === 1 ? COVER : ILLUSTRATION) : STORY;
pageNumber === 1 ? COVER : (panelsOfPage.length === 1 ? ILLUSTRATION : STORY);

export const acceptStoryKindSuggestion = (
suggestionId: storyKindSuggestion["id"] | null,
Expand Down
31 changes: 25 additions & 6 deletions apps/dumili/api/services/indexation/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,28 @@ export const indexationPayloadInclude = {
},
},
},
acceptedIssueSuggestion: true,
issueSuggestions: true,
acceptedIssueSuggestion: {
include: {
ai: true,
},
},
issueSuggestions: {
include: {
ai: true,
},
},
entries: {
include: {
acceptedStory: {
include: {
ocrDetails: true,
},
},
acceptedStoryKind: true,
storyKindSuggestions: true,
acceptedStoryKind: {include: {ai: true}},
storyKindSuggestions: {include: {ai: true}},
storySuggestions: {
include: {
ai: true,
ocrDetails: true,
},
},
Expand Down Expand Up @@ -69,7 +78,7 @@ export default abstract class {
) => void;

abstract createStorySuggestion: (
suggestion: Prisma.storySuggestionUncheckedCreateInput,
suggestion: Prisma.storySuggestionUncheckedCreateInput & {ai: boolean},
callback: (
data: Errorable<
{ createdStorySuggestion: Pick<storySuggestion, "id" | "storycode"> },
Expand All @@ -93,7 +102,7 @@ export default abstract class {
suggestion: Omit<
Prisma.issueSuggestionUncheckedCreateInput,
"indexationId"
>,
> & {ai: boolean},
callback: (data: { suggestionId: storySuggestion["id"] }) => void,
) => void;

Expand Down Expand Up @@ -138,6 +147,16 @@ export default abstract class {
) => void,
) => void;

abstract inferEntryStoryKind: (
entryId: entry["id"],
callback: (
data: Errorable<
{ status: "OK" },
| "This indexation does not have any entry with this ID"
>,
) => void,
) => void;

abstract runKumikoOnPage: (
pageId: page["id"],
callback: (
Expand Down
9 changes: 5 additions & 4 deletions apps/dumili/src/components/AiSuggestionIcon.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,15 @@ const component = computed(() =>
svg {
width: 20px;
height: 20px;
min-width: 20px;
padding: 4px;
&.button {
cursor: help;
background: black;
background: lightgrey;
border-radius: 10px;
padding: 4px;
height: 20px;
opacity: 0.5;
cursor: help;
}
&.idle {
Expand Down
Loading

0 comments on commit dea8026

Please sign in to comment.