From b43ba82962526818a82f142dba71f75ce1e1e339 Mon Sep 17 00:00:00 2001 From: MrOrz Date: Fri, 8 Sep 2023 00:04:35 +0800 Subject: [PATCH 1/8] feat(CreateMediaArticle): add stub code for fill in ydoc --- package-lock.json | 226 +++++++++++++++++++- package.json | 5 +- src/graphql/mutations/CreateMediaArticle.js | 66 ++++++ 3 files changed, 295 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 467eea66..29e182d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,12 +40,15 @@ "passport-instagram-graph": "^1.0.1", "passport-twitter": "^1.0.4", "pm2": "^4.0.0", + "prosemirror-schema-basic": "^1.2.2", + "prosemirror-state": "^1.4.3", "pug": "^2.0.3", "rollbar": "^2.3.7", "sharp": "^0.30.7", "url-regex": "^5.0.0", "xlsx": "^0.17.3", - "xxhashjs": "^0.2.2" + "xxhashjs": "^0.2.2", + "y-prosemirror": "^1.2.1" }, "devDependencies": { "@babel/cli": "^7.17.0", @@ -9880,6 +9883,15 @@ "node": ">=0.10.0" } }, + "node_modules/isomorphic.js": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.2.5.tgz", + "integrity": "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==", + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/isstream": { "version": "0.1.2", "license": "MIT" @@ -12218,6 +12230,25 @@ "node": ">= 0.8.0" } }, + "node_modules/lib0": { + "version": "0.2.86", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.86.tgz", + "integrity": "sha512-kxigQTM4Q7NwJkEgdqQvU21qiR37twcqqLmh+/SbiGbRLfPlLVbHyY9sWp7PwXh0Xus9ELDSjsUOwcrdt5yZ4w==", + "dependencies": { + "isomorphic.js": "^0.2.4" + }, + "bin": { + "0gentesthtml": "bin/gentesthtml.js", + "0serve": "bin/0serve.js" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", @@ -12940,6 +12971,11 @@ "node": ">= 0.8.0" } }, + "node_modules/orderedmap": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/orderedmap/-/orderedmap-2.1.1.tgz", + "integrity": "sha512-TvAWxi0nDe1j/rtMcWcIj94+Ffe6n7zhow33h40SKxmsmozs6dz/e+EajymfoFcHd7sxNn8yHM8839uixMOV6g==" + }, "node_modules/p-event": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/p-event/-/p-event-4.2.0.tgz", @@ -13776,6 +13812,50 @@ "node": ">= 6" } }, + "node_modules/prosemirror-model": { + "version": "1.19.3", + "resolved": "https://registry.npmjs.org/prosemirror-model/-/prosemirror-model-1.19.3.tgz", + "integrity": "sha512-tgSnwN7BS7/UM0sSARcW+IQryx2vODKX4MI7xpqY2X+iaepJdKBPc7I4aACIsDV/LTaTjt12Z56MhDr9LsyuZQ==", + "dependencies": { + "orderedmap": "^2.0.0" + } + }, + "node_modules/prosemirror-schema-basic": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/prosemirror-schema-basic/-/prosemirror-schema-basic-1.2.2.tgz", + "integrity": "sha512-/dT4JFEGyO7QnNTe9UaKUhjDXbTNkiWTq/N4VpKaF79bBjSExVV2NXmJpcM7z/gD7mbqNjxbmWW5nf1iNSSGnw==", + "dependencies": { + "prosemirror-model": "^1.19.0" + } + }, + "node_modules/prosemirror-state": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/prosemirror-state/-/prosemirror-state-1.4.3.tgz", + "integrity": "sha512-goFKORVbvPuAQaXhpbemJFRKJ2aixr+AZMGiquiqKxaucC6hlpHNZHWgz5R7dS4roHiwq9vDctE//CZ++o0W1Q==", + "dependencies": { + "prosemirror-model": "^1.0.0", + "prosemirror-transform": "^1.0.0", + "prosemirror-view": "^1.27.0" + } + }, + "node_modules/prosemirror-transform": { + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/prosemirror-transform/-/prosemirror-transform-1.7.5.tgz", + "integrity": "sha512-U/fWB6frEzY7dzwJUo+ir8dU1JEanaI/RwL12Imy9js/527N0v/IRUKewocP1kTq998JNT18IGtThaDLwLOBxQ==", + "dependencies": { + "prosemirror-model": "^1.0.0" + } + }, + "node_modules/prosemirror-view": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/prosemirror-view/-/prosemirror-view-1.32.0.tgz", + "integrity": "sha512-HwW7IWgca6ehiW2PA48H/8yl0TakA0Ms5LgN5Krc97oar7GfjIKE/NocUsLe74Jq4mwyWKUNoBljE8WkXKZwng==", + "dependencies": { + "prosemirror-model": "^1.16.0", + "prosemirror-state": "^1.0.0", + "prosemirror-transform": "^1.1.0" + } + }, "node_modules/proto3-json-serializer": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-1.1.1.tgz", @@ -16196,6 +16276,45 @@ "cuint": "^0.2.2" } }, + "node_modules/y-prosemirror": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/y-prosemirror/-/y-prosemirror-1.2.1.tgz", + "integrity": "sha512-czMBfB1eL2awqmOSxQM8cS/fsUOGE6fjvyPLInrh4crPxFiw67wDpwIW+EGBYKRa04sYbS0ScGj7ZgvWuDrmBQ==", + "dependencies": { + "lib0": "^0.2.42" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "prosemirror-model": "^1.7.1", + "prosemirror-state": "^1.2.3", + "prosemirror-view": "^1.9.10", + "y-protocols": "^1.0.1", + "yjs": "^13.5.38" + } + }, + "node_modules/y-protocols": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/y-protocols/-/y-protocols-1.0.6.tgz", + "integrity": "sha512-vHRF2L6iT3rwj1jub/K5tYcTT/mEYDUppgNPXwp8fmLpui9f7Yeq3OEtTLVF012j39QnV+KEQpNqoN7CWU7Y9Q==", + "peer": true, + "dependencies": { + "lib0": "^0.2.85" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "yjs": "^13.0.0" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", @@ -16238,6 +16357,23 @@ "node": ">=12" } }, + "node_modules/yjs": { + "version": "13.6.8", + "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.8.tgz", + "integrity": "sha512-ZPq0hpJQb6f59B++Ngg4cKexDJTvfOgeiv0sBc4sUm8CaBWH7OQC4kcCgrqbjJ/B2+6vO49exvTmYfdlPtcjbg==", + "peer": true, + "dependencies": { + "lib0": "^0.2.74" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/ylru": { "version": "1.2.1", "license": "MIT", @@ -23290,6 +23426,11 @@ "version": "3.0.1", "dev": true }, + "isomorphic.js": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.2.5.tgz", + "integrity": "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==" + }, "isstream": { "version": "0.1.2" }, @@ -25030,6 +25171,14 @@ "type-check": "~0.3.2" } }, + "lib0": { + "version": "0.2.86", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.86.tgz", + "integrity": "sha512-kxigQTM4Q7NwJkEgdqQvU21qiR37twcqqLmh+/SbiGbRLfPlLVbHyY9sWp7PwXh0Xus9ELDSjsUOwcrdt5yZ4w==", + "requires": { + "isomorphic.js": "^0.2.4" + } + }, "lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", @@ -25528,6 +25677,11 @@ "word-wrap": "~1.2.3" } }, + "orderedmap": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/orderedmap/-/orderedmap-2.1.1.tgz", + "integrity": "sha512-TvAWxi0nDe1j/rtMcWcIj94+Ffe6n7zhow33h40SKxmsmozs6dz/e+EajymfoFcHd7sxNn8yHM8839uixMOV6g==" + }, "p-event": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/p-event/-/p-event-4.2.0.tgz", @@ -26111,6 +26265,50 @@ "sisteransi": "^1.0.5" } }, + "prosemirror-model": { + "version": "1.19.3", + "resolved": "https://registry.npmjs.org/prosemirror-model/-/prosemirror-model-1.19.3.tgz", + "integrity": "sha512-tgSnwN7BS7/UM0sSARcW+IQryx2vODKX4MI7xpqY2X+iaepJdKBPc7I4aACIsDV/LTaTjt12Z56MhDr9LsyuZQ==", + "requires": { + "orderedmap": "^2.0.0" + } + }, + "prosemirror-schema-basic": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/prosemirror-schema-basic/-/prosemirror-schema-basic-1.2.2.tgz", + "integrity": "sha512-/dT4JFEGyO7QnNTe9UaKUhjDXbTNkiWTq/N4VpKaF79bBjSExVV2NXmJpcM7z/gD7mbqNjxbmWW5nf1iNSSGnw==", + "requires": { + "prosemirror-model": "^1.19.0" + } + }, + "prosemirror-state": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/prosemirror-state/-/prosemirror-state-1.4.3.tgz", + "integrity": "sha512-goFKORVbvPuAQaXhpbemJFRKJ2aixr+AZMGiquiqKxaucC6hlpHNZHWgz5R7dS4roHiwq9vDctE//CZ++o0W1Q==", + "requires": { + "prosemirror-model": "^1.0.0", + "prosemirror-transform": "^1.0.0", + "prosemirror-view": "^1.27.0" + } + }, + "prosemirror-transform": { + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/prosemirror-transform/-/prosemirror-transform-1.7.5.tgz", + "integrity": "sha512-U/fWB6frEzY7dzwJUo+ir8dU1JEanaI/RwL12Imy9js/527N0v/IRUKewocP1kTq998JNT18IGtThaDLwLOBxQ==", + "requires": { + "prosemirror-model": "^1.0.0" + } + }, + "prosemirror-view": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/prosemirror-view/-/prosemirror-view-1.32.0.tgz", + "integrity": "sha512-HwW7IWgca6ehiW2PA48H/8yl0TakA0Ms5LgN5Krc97oar7GfjIKE/NocUsLe74Jq4mwyWKUNoBljE8WkXKZwng==", + "requires": { + "prosemirror-model": "^1.16.0", + "prosemirror-state": "^1.0.0", + "prosemirror-transform": "^1.1.0" + } + }, "proto3-json-serializer": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-1.1.1.tgz", @@ -27846,6 +28044,23 @@ "cuint": "^0.2.2" } }, + "y-prosemirror": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/y-prosemirror/-/y-prosemirror-1.2.1.tgz", + "integrity": "sha512-czMBfB1eL2awqmOSxQM8cS/fsUOGE6fjvyPLInrh4crPxFiw67wDpwIW+EGBYKRa04sYbS0ScGj7ZgvWuDrmBQ==", + "requires": { + "lib0": "^0.2.42" + } + }, + "y-protocols": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/y-protocols/-/y-protocols-1.0.6.tgz", + "integrity": "sha512-vHRF2L6iT3rwj1jub/K5tYcTT/mEYDUppgNPXwp8fmLpui9f7Yeq3OEtTLVF012j39QnV+KEQpNqoN7CWU7Y9Q==", + "peer": true, + "requires": { + "lib0": "^0.2.85" + } + }, "y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", @@ -27875,6 +28090,15 @@ "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==" }, + "yjs": { + "version": "13.6.8", + "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.8.tgz", + "integrity": "sha512-ZPq0hpJQb6f59B++Ngg4cKexDJTvfOgeiv0sBc4sUm8CaBWH7OQC4kcCgrqbjJ/B2+6vO49exvTmYfdlPtcjbg==", + "peer": true, + "requires": { + "lib0": "^0.2.74" + } + }, "ylru": { "version": "1.2.1" }, diff --git a/package.json b/package.json index a7b2b541..87a136c2 100644 --- a/package.json +++ b/package.json @@ -55,12 +55,15 @@ "passport-instagram-graph": "^1.0.1", "passport-twitter": "^1.0.4", "pm2": "^4.0.0", + "prosemirror-schema-basic": "^1.2.2", + "prosemirror-state": "^1.4.3", "pug": "^2.0.3", "rollbar": "^2.3.7", "sharp": "^0.30.7", "url-regex": "^5.0.0", "xlsx": "^0.17.3", - "xxhashjs": "^0.2.2" + "xxhashjs": "^0.2.2", + "y-prosemirror": "^1.2.1" }, "devDependencies": { "@babel/cli": "^7.17.0", diff --git a/src/graphql/mutations/CreateMediaArticle.js b/src/graphql/mutations/CreateMediaArticle.js index 8c1a281d..0a9116e6 100644 --- a/src/graphql/mutations/CreateMediaArticle.js +++ b/src/graphql/mutations/CreateMediaArticle.js @@ -7,6 +7,9 @@ import mediaManager, { } from 'util/mediaManager'; import { assertUser, getContentDefaultStatus } from 'util/user'; import client from 'util/client'; +import { getAIResponse } from 'graphql/util'; +import { schema } from 'prosemirror-schema-basic'; +import { EditorState } from 'prosemirror-state'; import { ArticleReferenceInput } from 'graphql/models/ArticleReference'; import MutationResult from 'graphql/models/MutationResult'; @@ -157,6 +160,67 @@ async function createNewMediaArticle({ return articleId; } +/** + * @param {string} articleId + * @returns {boolean} if the transcript is written successfully + */ +async function writeAITranscriptIfExists(articleId) { + const { + body: { _source: article }, + } = await client.get({ + index: 'articles', + type: 'doc', + id: articleId, + }); + + const aiResponse = await getAIResponse({ + type: 'TRANSCRIPT', + docId: article.attachmentHash, + }); + + if (!aiResponse) return false; + + // Write aiResponse to articles + const writeToArticleTextPromise = client.update({ + index: 'articles', + type: 'doc', + id: articleId, + body: { + doc: { + text: aiResponse.text, + }, + }, + }); + + // Prosemirror editor state with AI response text + const tempState = EditorState.create({ schema }); + const proseMirrorState = tempState.apply( + tempState.tr.insertText(aiResponse.text) + ); + + console.log('[proseMirrorState]', proseMirrorState.toJSON()); + + // Create Y.doc and write to ydoc collection + const createYdocPromise = client.index({ + index: 'articles', + type: 'doc', + id: article.attachmentHash, + body: { + doc: { + ydoc: '', + }, + }, + }); + + try { + await Promise.all([writeToArticleTextPromise, createYdocPromise]); + } catch (e) { + console.error('[writeAITranscriptIfExists]', e); + return false; + } + return true; +} + export default { type: MutationResult, description: 'Create a media article and/or a replyRequest', @@ -191,6 +255,8 @@ export default { user, }); + writeAITranscriptIfExists(); + await createOrUpdateReplyRequest({ articleId, user, From ec376a7d5c6e2344fdf8fed6e56276c8a806aa3b Mon Sep 17 00:00:00 2001 From: MrOrz Date: Tue, 26 Sep 2023 03:09:35 +0800 Subject: [PATCH 2/8] refactor(CreateMediaArticle): refactor writeAITranscript so that it can be used easily without worring about transcript source --- src/graphql/mutations/CreateMediaArticle.js | 48 ++++++++------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/src/graphql/mutations/CreateMediaArticle.js b/src/graphql/mutations/CreateMediaArticle.js index 0a9116e6..ac9b6e50 100644 --- a/src/graphql/mutations/CreateMediaArticle.js +++ b/src/graphql/mutations/CreateMediaArticle.js @@ -161,42 +161,23 @@ async function createNewMediaArticle({ } /** - * @param {string} articleId + * @param {string} articleId - For target article + * @param {string} attachmentHash - For + * @param {string} text * @returns {boolean} if the transcript is written successfully */ -async function writeAITranscriptIfExists(articleId) { - const { - body: { _source: article }, - } = await client.get({ - index: 'articles', - type: 'doc', - id: articleId, - }); - - const aiResponse = await getAIResponse({ - type: 'TRANSCRIPT', - docId: article.attachmentHash, - }); - - if (!aiResponse) return false; - +export async function writeAITranscript(articleId, attachmentHash, text) { // Write aiResponse to articles const writeToArticleTextPromise = client.update({ index: 'articles', type: 'doc', id: articleId, - body: { - doc: { - text: aiResponse.text, - }, - }, + body: { doc: { text } }, }); // Prosemirror editor state with AI response text const tempState = EditorState.create({ schema }); - const proseMirrorState = tempState.apply( - tempState.tr.insertText(aiResponse.text) - ); + const proseMirrorState = tempState.apply(tempState.tr.insertText(text)); console.log('[proseMirrorState]', proseMirrorState.toJSON()); @@ -204,10 +185,10 @@ async function writeAITranscriptIfExists(articleId) { const createYdocPromise = client.index({ index: 'articles', type: 'doc', - id: article.attachmentHash, + id: attachmentHash, body: { doc: { - ydoc: '', + ydoc: proseMirrorState.toJSON().toString('base64'), }, }, }); @@ -215,7 +196,7 @@ async function writeAITranscriptIfExists(articleId) { try { await Promise.all([writeToArticleTextPromise, createYdocPromise]); } catch (e) { - console.error('[writeAITranscriptIfExists]', e); + console.error('[writeAITranscript]', e); return false; } return true; @@ -255,7 +236,16 @@ export default { user, }); - writeAITranscriptIfExists(); + (async function() { + const aiResponse = await getAIResponse({ + type: 'TRANSCRIPT', + docId: mediaEntry.id, + }); + + if (!aiResponse) return; + + await writeAITranscript(articleId, mediaEntry.id, aiResponse.text); + })(); await createOrUpdateReplyRequest({ articleId, From 2b84038b94ffb8188811dd212a75af823c411554 Mon Sep 17 00:00:00 2001 From: MrOrz Date: Wed, 27 Sep 2023 02:53:43 +0800 Subject: [PATCH 3/8] feat(CreateMediaArticle): attempt to fix writeAITranscript() - Can write to article.text and load in article detail - Can write to ydoc.ydoc but cannot load in UI --- package-lock.json | 5 ++-- package.json | 3 ++- src/graphql/mutations/CreateMediaArticle.js | 26 ++++++++++----------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/package-lock.json b/package-lock.json index 29e182d8..d8141dd0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,7 +48,8 @@ "url-regex": "^5.0.0", "xlsx": "^0.17.3", "xxhashjs": "^0.2.2", - "y-prosemirror": "^1.2.1" + "y-prosemirror": "^1.2.1", + "yjs": "^13.6.8" }, "devDependencies": { "@babel/cli": "^7.17.0", @@ -16361,7 +16362,6 @@ "version": "13.6.8", "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.8.tgz", "integrity": "sha512-ZPq0hpJQb6f59B++Ngg4cKexDJTvfOgeiv0sBc4sUm8CaBWH7OQC4kcCgrqbjJ/B2+6vO49exvTmYfdlPtcjbg==", - "peer": true, "dependencies": { "lib0": "^0.2.74" }, @@ -28094,7 +28094,6 @@ "version": "13.6.8", "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.8.tgz", "integrity": "sha512-ZPq0hpJQb6f59B++Ngg4cKexDJTvfOgeiv0sBc4sUm8CaBWH7OQC4kcCgrqbjJ/B2+6vO49exvTmYfdlPtcjbg==", - "peer": true, "requires": { "lib0": "^0.2.74" } diff --git a/package.json b/package.json index 87a136c2..83d5ab37 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,8 @@ "url-regex": "^5.0.0", "xlsx": "^0.17.3", "xxhashjs": "^0.2.2", - "y-prosemirror": "^1.2.1" + "y-prosemirror": "^1.2.1", + "yjs": "^13.6.8" }, "devDependencies": { "@babel/cli": "^7.17.0", diff --git a/src/graphql/mutations/CreateMediaArticle.js b/src/graphql/mutations/CreateMediaArticle.js index ac9b6e50..1515a412 100644 --- a/src/graphql/mutations/CreateMediaArticle.js +++ b/src/graphql/mutations/CreateMediaArticle.js @@ -9,7 +9,9 @@ import { assertUser, getContentDefaultStatus } from 'util/user'; import client from 'util/client'; import { getAIResponse } from 'graphql/util'; import { schema } from 'prosemirror-schema-basic'; +import { encodeStateAsUpdate } from 'yjs'; import { EditorState } from 'prosemirror-state'; +import { prosemirrorToYDoc } from 'y-prosemirror'; import { ArticleReferenceInput } from 'graphql/models/ArticleReference'; import MutationResult from 'graphql/models/MutationResult'; @@ -162,11 +164,10 @@ async function createNewMediaArticle({ /** * @param {string} articleId - For target article - * @param {string} attachmentHash - For * @param {string} text - * @returns {boolean} if the transcript is written successfully + * @returns result of article & ydoc operation */ -export async function writeAITranscript(articleId, attachmentHash, text) { +export function writeAITranscript(articleId, text) { // Write aiResponse to articles const writeToArticleTextPromise = client.update({ index: 'articles', @@ -179,27 +180,26 @@ export async function writeAITranscript(articleId, attachmentHash, text) { const tempState = EditorState.create({ schema }); const proseMirrorState = tempState.apply(tempState.tr.insertText(text)); - console.log('[proseMirrorState]', proseMirrorState.toJSON()); + // Encode ProseMirror doc node into binary in the same way as Hocuspocus + // @ref https://tiptap.dev/hocuspocus/guides/persistence#faq-in-what-format-should-i-save-my-document + const ydoc = prosemirrorToYDoc(proseMirrorState.doc); + const buffer = encodeStateAsUpdate(ydoc); // Create Y.doc and write to ydoc collection const createYdocPromise = client.index({ - index: 'articles', + index: 'ydocs', type: 'doc', - id: attachmentHash, + id: articleId, body: { - doc: { - ydoc: proseMirrorState.toJSON().toString('base64'), - }, + ydoc: buffer.toString('base64'), }, }); try { - await Promise.all([writeToArticleTextPromise, createYdocPromise]); + return Promise.all([writeToArticleTextPromise, createYdocPromise]); } catch (e) { console.error('[writeAITranscript]', e); - return false; } - return true; } export default { @@ -244,7 +244,7 @@ export default { if (!aiResponse) return; - await writeAITranscript(articleId, mediaEntry.id, aiResponse.text); + await writeAITranscript(articleId, aiResponse.text); })(); await createOrUpdateReplyRequest({ From 0b10b9eb248dc9741931effd9223c55ffaaf53d2 Mon Sep 17 00:00:00 2001 From: MrOrz Date: Wed, 27 Sep 2023 13:38:16 +0800 Subject: [PATCH 4/8] fix(CreateMediaArticle): convert UInt8Array to buffer before encoding as base64 --- src/graphql/mutations/CreateMediaArticle.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/graphql/mutations/CreateMediaArticle.js b/src/graphql/mutations/CreateMediaArticle.js index 1515a412..3818c6c6 100644 --- a/src/graphql/mutations/CreateMediaArticle.js +++ b/src/graphql/mutations/CreateMediaArticle.js @@ -183,7 +183,7 @@ export function writeAITranscript(articleId, text) { // Encode ProseMirror doc node into binary in the same way as Hocuspocus // @ref https://tiptap.dev/hocuspocus/guides/persistence#faq-in-what-format-should-i-save-my-document const ydoc = prosemirrorToYDoc(proseMirrorState.doc); - const buffer = encodeStateAsUpdate(ydoc); + const buffer = Buffer.from(encodeStateAsUpdate(ydoc)); // Create Y.doc and write to ydoc collection const createYdocPromise = client.index({ From 016e063e17b5c2afe8e0aa7f46ef9d4bab14c6c9 Mon Sep 17 00:00:00 2001 From: MrOrz Date: Wed, 27 Sep 2023 16:15:18 +0800 Subject: [PATCH 5/8] refactor(CreateMediaArticle): adjust error messages --- src/graphql/mutations/CreateMediaArticle.js | 40 +++++++++++++-------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/graphql/mutations/CreateMediaArticle.js b/src/graphql/mutations/CreateMediaArticle.js index 3818c6c6..b0be847c 100644 --- a/src/graphql/mutations/CreateMediaArticle.js +++ b/src/graphql/mutations/CreateMediaArticle.js @@ -195,11 +195,7 @@ export function writeAITranscript(articleId, text) { }, }); - try { - return Promise.all([writeToArticleTextPromise, createYdocPromise]); - } catch (e) { - console.error('[writeAITranscript]', e); - } + return Promise.all([writeToArticleTextPromise, createYdocPromise]); } export default { @@ -236,16 +232,30 @@ export default { user, }); - (async function() { - const aiResponse = await getAIResponse({ - type: 'TRANSCRIPT', - docId: mediaEntry.id, - }); - - if (!aiResponse) return; - - await writeAITranscript(articleId, aiResponse.text); - })(); + // Write AI transcript to article & ydoc without blocking + getAIResponse({ + type: 'TRANSCRIPT', + docId: mediaEntry.id, + }) + .then(aiResponse => { + if (!aiResponse) { + throw new Error('AI transcript not found'); + } + return writeAITranscript(articleId, aiResponse.text); + }) + .then(() => { + console.log( + `[CreateMediaArticle] AI transcript for ${ + mediaEntry.id + } applied to article ${articleId}` + ); + }) + .catch(e => + console.warn( + `[CreateMediaArticle] ${mediaEntry.id} (article ${articleId})`, + e + ) + ); await createOrUpdateReplyRequest({ articleId, From 6ee0c01026fa9606ba763300e7ca1d52f6d797a0 Mon Sep 17 00:00:00 2001 From: MrOrz Date: Fri, 29 Sep 2023 00:47:08 +0800 Subject: [PATCH 6/8] refactor(CreateMediaArticle): adjust async operations - Each operation just await its dependencies - CraeteMediaArticle returns when all operations settles --- src/graphql/mutations/CreateMediaArticle.js | 66 ++++++++++----------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/src/graphql/mutations/CreateMediaArticle.js b/src/graphql/mutations/CreateMediaArticle.js index b0be847c..c24bdd41 100644 --- a/src/graphql/mutations/CreateMediaArticle.js +++ b/src/graphql/mutations/CreateMediaArticle.js @@ -1,4 +1,4 @@ -import { GraphQLString, GraphQLNonNull } from 'graphql'; +import { GraphQLString, GraphQLNonNull, GraphQLBoolean } from 'graphql'; import sharp from 'sharp'; import { MediaType, variants } from '@cofacts/media-manager'; import mediaManager, { @@ -206,11 +206,8 @@ export default { articleType: { type: new GraphQLNonNull(ArticleTypeEnum) }, reference: { type: new GraphQLNonNull(ArticleReferenceInput) }, reason: { - // FIXME: Change to required field after LINE bot is implemented - // type: new GraphQLNonNull(GraphQLString), type: GraphQLString, - description: - 'The reason why the user want to submit this article. Mandatory for 1st sender', + description: 'The reason why the user want to submit this article', }, }, async resolve( @@ -225,44 +222,45 @@ export default { articleType, }); - const articleId = await createNewMediaArticle({ + const aritcleIdPromise = createNewMediaArticle({ mediaEntry, articleType, reference, user, }); - // Write AI transcript to article & ydoc without blocking - getAIResponse({ + const aiResponsePromise = getAIResponse({ type: 'TRANSCRIPT', docId: mediaEntry.id, - }) - .then(aiResponse => { - if (!aiResponse) { - throw new Error('AI transcript not found'); - } - return writeAITranscript(articleId, aiResponse.text); - }) - .then(() => { - console.log( - `[CreateMediaArticle] AI transcript for ${ - mediaEntry.id - } applied to article ${articleId}` - ); - }) - .catch(e => - console.warn( - `[CreateMediaArticle] ${mediaEntry.id} (article ${articleId})`, - e - ) - ); - - await createOrUpdateReplyRequest({ - articleId, - user, - reason, }); - return { id: articleId }; + await Promise.all([ + // Update reply request + aritcleIdPromise.then(articleId => + createOrUpdateReplyRequest({ + articleId, + user, + reason, + }) + ), + + // Write AI transcript to article & ydoc + Promise.all([aritcleIdPromise, aiResponsePromise]) + .then(([articleId, aiResponse]) => { + if (!aiResponse) { + throw new Error('AI transcript not found'); + } + return writeAITranscript(articleId, aiResponse.text); + }) + .then(() => { + console.log( + `[CreateMediaArticle] AI transcript for ${mediaEntry.id} applied` + ); + }) + // It's OK to fail this promise, just log as warning + .catch(e => console.warn(`[CreateMediaArticle] ${mediaEntry.id}:`, e)), + ]); + + return { id: await aritcleIdPromise }; }, }; From 21e4d6f0a3fa3d2692ed312f748866edee941a7f Mon Sep 17 00:00:00 2001 From: MrOrz Date: Fri, 29 Sep 2023 00:47:25 +0800 Subject: [PATCH 7/8] test(CreateMediaArticle): add unit test for transcript --- .../__fixtures__/CreateMediaArticle.js | 9 +++++- .../mutations/__tests__/CreateMediaArticle.js | 32 ++++++++++++++++--- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/graphql/mutations/__fixtures__/CreateMediaArticle.js b/src/graphql/mutations/__fixtures__/CreateMediaArticle.js index a1cc3359..5c55dde3 100644 --- a/src/graphql/mutations/__fixtures__/CreateMediaArticle.js +++ b/src/graphql/mutations/__fixtures__/CreateMediaArticle.js @@ -1,9 +1,16 @@ export default { - [`/articles/doc/image1`]: { + '/articles/doc/image1': { text: '', attachmentUrl: 'http://foo/image.jpeg', attachmentHash: 'ffff8000', replyRequestCount: 1, references: [{ type: 'LINE' }], }, + '/airesponses/doc/ocr': { + docId: 'mock_image_hash', + type: 'TRANSCRIPT', + text: 'OCR result of output image', + status: 'SUCCESS', + createdAt: '2020-01-01T00:00:00.000Z', + }, }; diff --git a/src/graphql/mutations/__tests__/CreateMediaArticle.js b/src/graphql/mutations/__tests__/CreateMediaArticle.js index eba91269..6431f925 100644 --- a/src/graphql/mutations/__tests__/CreateMediaArticle.js +++ b/src/graphql/mutations/__tests__/CreateMediaArticle.js @@ -1,7 +1,9 @@ +import Y from 'yjs'; +import MockDate from 'mockdate'; + import gql from 'util/GraphQL'; import { loadFixtures, unloadFixtures } from 'util/fixtures'; import client from 'util/client'; -import MockDate from 'mockdate'; import fixtures from '../__fixtures__/CreateMediaArticle'; import { getReplyRequestId } from '../CreateOrUpdateReplyRequest'; import mediaManager from 'util/mediaManager'; @@ -15,7 +17,7 @@ describe('creation', () => { }); afterAll(() => unloadFixtures(fixtures)); - it('creates a media article and a reply request', async () => { + it('creates a media article, a reply request, a ydoc and fills in OCR result', async () => { MockDate.set(1485593157011); const userId = 'test'; const appId = 'foo'; @@ -98,7 +100,7 @@ describe('creation', () => { "replyRequestCount": 1, "status": "NORMAL", "tags": Array [], - "text": "", + "text": "OCR result of output image", "updatedAt": "2017-01-28T08:45:57.011Z", "userId": "test", } @@ -133,7 +135,23 @@ describe('creation', () => { } `); - // // Cleanup + const { + body: { + _source: { ydoc: encodedYdoc }, + }, + } = await client.get({ + index: 'ydocs', + type: 'doc', + id: data.CreateMediaArticle.id, + }); + + const ydoc = new Y.Doc(); + Y.applyUpdate(ydoc, Buffer.from(encodedYdoc, 'base64')); + expect(ydoc.getXmlFragment('prosemirror')).toMatchInlineSnapshot( + `"OCR result of output image"` + ); + + // Cleanup await client.delete({ index: 'articles', type: 'doc', @@ -145,6 +163,12 @@ describe('creation', () => { type: 'doc', id: replyRequestId, }); + + await client.delete({ + index: 'ydocs', + type: 'doc', + id: data.CreateMediaArticle.id, + }); }); it('avoids creating duplicated media articles and adds replyRequests automatically', async () => { From 9a0068072b2d6568b7d9ba8411aa945ffcf99504 Mon Sep 17 00:00:00 2001 From: MrOrz Date: Fri, 29 Sep 2023 01:20:00 +0800 Subject: [PATCH 8/8] feat(CreateMediaArticle): add user and snapshot version to AI transcript --- src/graphql/mutations/CreateMediaArticle.js | 30 ++++++++++++++++--- .../mutations/__tests__/CreateMediaArticle.js | 11 ++++++- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/graphql/mutations/CreateMediaArticle.js b/src/graphql/mutations/CreateMediaArticle.js index c24bdd41..f3454356 100644 --- a/src/graphql/mutations/CreateMediaArticle.js +++ b/src/graphql/mutations/CreateMediaArticle.js @@ -1,4 +1,4 @@ -import { GraphQLString, GraphQLNonNull, GraphQLBoolean } from 'graphql'; +import { GraphQLString, GraphQLNonNull } from 'graphql'; import sharp from 'sharp'; import { MediaType, variants } from '@cofacts/media-manager'; import mediaManager, { @@ -9,7 +9,7 @@ import { assertUser, getContentDefaultStatus } from 'util/user'; import client from 'util/client'; import { getAIResponse } from 'graphql/util'; import { schema } from 'prosemirror-schema-basic'; -import { encodeStateAsUpdate } from 'yjs'; +import Y from 'yjs'; import { EditorState } from 'prosemirror-state'; import { prosemirrorToYDoc } from 'y-prosemirror'; @@ -22,6 +22,12 @@ const METADATA = { cacheControl: 'public, max-age=31536000, immutable', }; +const AI_TRANSCRIBER_DESCRIPTION = JSON.stringify({ + id: 'ai-transcript', + appId: 'RUMORS_AI', + name: 'AI Transcript', +}); + const VALID_ARTICLE_TYPE_TO_MEDIA_TYPE = { IMAGE: MediaType.image, VIDEO: MediaType.video, @@ -183,7 +189,17 @@ export function writeAITranscript(articleId, text) { // Encode ProseMirror doc node into binary in the same way as Hocuspocus // @ref https://tiptap.dev/hocuspocus/guides/persistence#faq-in-what-format-should-i-save-my-document const ydoc = prosemirrorToYDoc(proseMirrorState.doc); - const buffer = Buffer.from(encodeStateAsUpdate(ydoc)); + + // Setup user mapping + const permanentUserData = new Y.PermanentUserData(ydoc); + permanentUserData.setUserMapping( + ydoc, + ydoc.clientID, + AI_TRANSCRIBER_DESCRIPTION + ); + + // Create initial version snapshot + const snapshot = Y.snapshot(ydoc); // Create Y.doc and write to ydoc collection const createYdocPromise = client.index({ @@ -191,7 +207,13 @@ export function writeAITranscript(articleId, text) { type: 'doc', id: articleId, body: { - ydoc: buffer.toString('base64'), + ydoc: Buffer.from(Y.encodeStateAsUpdate(ydoc)).toString('base64'), + versions: [ + { + createdAt: new Date().toISOString(), + snapshot: Buffer.from(Y.encodeSnapshot(snapshot)).toString('base64'), + }, + ], }, }); diff --git a/src/graphql/mutations/__tests__/CreateMediaArticle.js b/src/graphql/mutations/__tests__/CreateMediaArticle.js index 6431f925..2031ede9 100644 --- a/src/graphql/mutations/__tests__/CreateMediaArticle.js +++ b/src/graphql/mutations/__tests__/CreateMediaArticle.js @@ -137,7 +137,7 @@ describe('creation', () => { const { body: { - _source: { ydoc: encodedYdoc }, + _source: { ydoc: encodedYdoc, versions }, }, } = await client.get({ index: 'ydocs', @@ -145,11 +145,20 @@ describe('creation', () => { id: data.CreateMediaArticle.id, }); + // Expect ydoc in Elasticsearch to contain prosemirror, user and snapshot versions const ydoc = new Y.Doc(); Y.applyUpdate(ydoc, Buffer.from(encodedYdoc, 'base64')); expect(ydoc.getXmlFragment('prosemirror')).toMatchInlineSnapshot( `"OCR result of output image"` ); + expect(Object.keys(ydoc.getMap('users').toJSON())).toMatchInlineSnapshot(` + Array [ + "{\\"id\\":\\"ai-transcript\\",\\"appId\\":\\"RUMORS_AI\\",\\"name\\":\\"AI Transcript\\"}", + ] + `); + expect(versions[0].createdAt).toMatchInlineSnapshot( + `"2017-01-28T08:45:57.011Z"` + ); // Cleanup await client.delete({