From e7bd281eff0574ffdfbd5f6875be07f676140a97 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 22 Mar 2023 18:47:11 +0800 Subject: [PATCH 1/7] feat: fetch note from articles api --- src/util.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/util.ts b/src/util.ts index 4ff8367..9ddbcf9 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1,7 +1,7 @@ +import { format } from 'date-fns' import { diff_match_patch } from 'diff-match-patch' import { DateTime } from 'luxon' import escape from 'markdown-escape' -import { format } from 'date-fns' export const DATE_FORMAT_W_OUT_SECONDS = "yyyy-MM-dd'T'HH:mm" export const DATE_FORMAT = `${DATE_FORMAT_W_OUT_SECONDS}:ss` @@ -72,6 +72,12 @@ export interface Label { name: string } +export enum HighlightType { + Highlight = 'HIGHLIGHT', + Note = 'NOTE', + Redaction = 'REDACTION', +} + export interface Highlight { id: string quote: string @@ -79,6 +85,7 @@ export interface Highlight { patch: string updatedAt: string labels?: Label[] + type: HighlightType } export interface HighlightPoint { @@ -146,6 +153,7 @@ export const loadArticles = async ( labels { name } + type } labels { name From 55405b4aad076c7d9fa7f5e860ac52bd7a82652c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 22 Mar 2023 18:49:18 +0800 Subject: [PATCH 2/7] feat: add note variable to the article and highlight template --- src/index.ts | 65 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/src/index.ts b/src/index.ts index 499fb8a..84bbfd6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,6 +14,7 @@ import { escapeQuotationMarks, formatDate, getHighlightLocation, + HighlightType, loadArticles, loadDeletedArticleSlugs, PageType, @@ -205,9 +206,8 @@ const fetchOmnivore = async (inBackground = false) => { dateSaved, content: article.content, datePublished, + note: '', } - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - const content = render(articleTemplate, articleView) // sort highlights by location if selected in options highlightOrder === HighlightOrder.LOCATION && article.highlights?.sort((a, b) => { @@ -225,30 +225,41 @@ const fetchOmnivore = async (inBackground = false) => { return compareHighlightsInFile(a, b) } }) - const highlightBatch: (IBatchBlock & { id: string })[] = - article.highlights?.map((it) => { - // Build content string based on template - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - const content = render(highlightTemplate, { - ...articleView, - text: it.quote, - labels: it.labels, - highlightUrl: `https://omnivore.app/me/${article.slug}#${it.id}`, - dateHighlighted: formatDate( - new Date(it.updatedAt), - preferredDateFormat - ), + const highlightBatch: IBatchBlock[] = + article.highlights + ?.map((it) => { + // append note variable to article template + if (it.type == HighlightType.Note) { + articleView.note = it.annotation + } + // Build content string based on template + // eslint-disable-next-line @typescript-eslint/no-unsafe-call + const content = render(highlightTemplate, { + ...articleView, + text: it.quote, + labels: it.labels, + highlightUrl: `https://omnivore.app/me/${article.slug}#${it.id}`, + dateHighlighted: formatDate( + new Date(it.updatedAt), + preferredDateFormat + ), + }) + const noteChild = it.annotation + ? { content: it.annotation } + : undefined + return { + content, + children: noteChild ? [noteChild] : undefined, + properties: { + id: it.id, + type: it.type, + }, + } }) - const noteChild = it.annotation - ? { content: it.annotation } - : undefined - return { - content, - children: noteChild ? [noteChild] : undefined, - id: it.id, - } - }) || [] - + .filter((it) => it.properties.type === HighlightType.Highlight) || // filter out notes and redactions + [] + // eslint-disable-next-line @typescript-eslint/no-unsafe-call + const content = render(articleTemplate, articleView) let isNewArticle = true // update existing block if article is already in the page const existingBlocks = ( @@ -286,7 +297,9 @@ const fetchOmnivore = async (inBackground = false) => { [(str ?u) ?s] [(= ?s "${existingBlock.uuid}")] [?b :block/content ?c] - [(clojure.string/includes? ?c "${highlight.id}")]]` + [(clojure.string/includes? ?c "${ + highlight.properties?.id as string + }")]]` ) ).flat() if (existingHighlights.length > 0) { From e95909c85bf04ac293aac650e252da57ad2dd227 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 22 Mar 2023 19:08:15 +0800 Subject: [PATCH 3/7] add endpoint to the api request --- src/util.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/util.ts b/src/util.ts index 9ddbcf9..cbcaa73 100644 --- a/src/util.ts +++ b/src/util.ts @@ -102,9 +102,10 @@ const requestHeaders = (apiKey: string) => ({ export const loadArticle = async ( slug: string, - apiKey: string + apiKey: string, + endpoint = ENDPOINT ): Promise
=> { - const res = await fetch(ENDPOINT, { + const res = await fetch(endpoint, { headers: requestHeaders(apiKey), body: `{"query":"\\n query GetArticle(\\n $username: String!\\n $slug: String!\\n ) {\\n article(username: $username, slug: $slug) {\\n ... on ArticleSuccess {\\n article {\\n ...ArticleFields\\n highlights {\\n ...HighlightFields\\n }\\n labels {\\n ...LabelFields\\n }\\n }\\n }\\n ... on ArticleError {\\n errorCodes\\n }\\n }\\n }\\n \\n fragment ArticleFields on Article {\\n savedAt\\n }\\n\\n \\n fragment HighlightFields on Highlight {\\n id\\n quote\\n annotation\\n }\\n\\n \\n fragment LabelFields on Label {\\n name\\n }\\n\\n","variables":{"username":"me","slug":"${slug}"}}`, method: 'POST', @@ -121,9 +122,10 @@ export const loadArticles = async ( updatedAt = '', query = '', includeContent = false, - format = 'html' + format = 'html', + endpoint = ENDPOINT ): Promise<[Article[], boolean]> => { - const res = await fetch(ENDPOINT, { + const res = await fetch(endpoint, { headers: requestHeaders(apiKey), body: JSON.stringify({ query: ` @@ -192,9 +194,10 @@ export const loadDeletedArticleSlugs = async ( apiKey: string, after = 0, first = 10, - updatedAt = '' + updatedAt = '', + endpoint = ENDPOINT ): Promise<[string[], boolean]> => { - const res = await fetch(ENDPOINT, { + const res = await fetch(endpoint, { headers: requestHeaders(apiKey), body: `{"query":"\\n query UpdatesSince($after: String, $first: Int, $since: Date!) {\\n updatesSince(first: $first, after: $after, since: $since) {\\n ... on UpdatesSinceSuccess {\\n edges {\\n updateReason\\n node {\\n slug\\n }\\n }\\n pageInfo {\\n hasNextPage\\n }\\n }\\n ... on UpdatesSinceError {\\n errorCodes\\n }\\n }\\n }\\n ","variables":{"after":"${after}","first":${first}, "since":"${ updatedAt || '2021-01-01' From 3bb633c0296b29524f0da129f79635b7c5f30453 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 22 Mar 2023 19:08:31 +0800 Subject: [PATCH 4/7] feat: add server endpoint to the settings --- src/index.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index 84bbfd6..7eae6cb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -46,6 +46,7 @@ interface Settings { highlightTemplate: string loading: boolean syncJobId: number + endpoint: string } const siteNameFromUrl = (originalArticleUrl: string): string => { @@ -96,6 +97,7 @@ const fetchOmnivore = async (inBackground = false) => { highlightTemplate, graph, loading, + endpoint, } = logseq.settings as Settings // prevent multiple fetches if (loading) { @@ -181,7 +183,8 @@ const fetchOmnivore = async (inBackground = false) => { parseDateTime(syncAt).toISO(), getQueryFromFilter(filter, customQuery), true, - 'markdown' + 'markdown', + endpoint ) const articleBatch: IBatchBlock[] = [] @@ -373,7 +376,8 @@ const fetchOmnivore = async (inBackground = false) => { apiKey, after, size, - parseDateTime(syncAt).toISO() + parseDateTime(syncAt).toISO(), + endpoint ) for (const slug of deletedArticleSlugs) { @@ -555,6 +559,13 @@ date-published:: {{{datePublished}}} default: `> {{{text}}} [⤴️]({{{highlightUrl}}}) {{#labels}} #[[{{{name}}}]] {{/labels}}`, inputAs: 'textarea', }, + { + key: 'endpoint', + type: 'string', + title: 'API Endpoint', + description: "Enter the Omnivore server's API endpoint", + default: 'https://api-prod.omnivore.app/api/graphql', + }, ] logseq.useSettingsSchema(settingsSchema) From 173fc9b6d17bb36763f8ba6eb3c8163a2b1e27a1 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 23 Mar 2023 10:15:16 +0800 Subject: [PATCH 5/7] if either highlight is not a highlight, put it at the end --- src/index.ts | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/index.ts b/src/index.ts index 7eae6cb..de68e1f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -215,6 +215,13 @@ const fetchOmnivore = async (inBackground = false) => { highlightOrder === HighlightOrder.LOCATION && article.highlights?.sort((a, b) => { try { + // if either highlight is not a highlight, put it at the end + if ( + a.type !== HighlightType.Highlight || + b.type !== HighlightType.Highlight + ) { + return 1 + } if (article.pageType === PageType.File) { // sort by location in file return compareHighlightsInFile(a, b) @@ -229,11 +236,16 @@ const fetchOmnivore = async (inBackground = false) => { } }) const highlightBatch: IBatchBlock[] = - article.highlights + (article.highlights ?.map((it) => { - // append note variable to article template - if (it.type == HighlightType.Note) { - articleView.note = it.annotation + const highlightType = it.type + // filter out notes and redactions + if (highlightType !== HighlightType.Highlight) { + // add note variable to article template + if (highlightType === HighlightType.Note) { + articleView.note = it.annotation + } + return undefined } // Build content string based on template // eslint-disable-next-line @typescript-eslint/no-unsafe-call @@ -255,12 +267,10 @@ const fetchOmnivore = async (inBackground = false) => { children: noteChild ? [noteChild] : undefined, properties: { id: it.id, - type: it.type, }, } }) - .filter((it) => it.properties.type === HighlightType.Highlight) || // filter out notes and redactions - [] + .filter((it) => it !== undefined) as IBatchBlock[]) || [] // eslint-disable-next-line @typescript-eslint/no-unsafe-call const content = render(articleTemplate, articleView) let isNewArticle = true From df1ebea0e0458f9a6bf9ddc26e516e1c9a340ef2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 23 Mar 2023 10:31:19 +0800 Subject: [PATCH 6/7] update settings --- src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index de68e1f..f1c2f84 100644 --- a/src/index.ts +++ b/src/index.ts @@ -544,7 +544,7 @@ const main = async (baseInfo: LSPluginBaseInfo) => { type: 'string', title: 'Enter the template to use for new articles', description: - 'Enter the template to use for new articles. Required variables are: {{{title}}}, {{{omnivoreUrl}}}. Optional variables are: {{{siteName}}}, {{{originalUrl}}}, {{{author}}}, {{{labels}}}, {{{dateSaved}}}, {{{datePublished}}}', + 'We use {{ mustache }} template: http://mustache.github.io/mustache.5.html. Required variables are: title, omnivoreUrl. Optional variables are: siteName, originalUrl, author, labels, dateSaved, datePublished, note', default: `[{{{title}}}]({{{omnivoreUrl}}}) collapsed:: true site:: {{#siteName}}[{{{siteName}}}]{{/siteName}}({{{originalUrl}}}) @@ -565,7 +565,7 @@ date-published:: {{{datePublished}}} type: 'string', title: 'Enter the template to use for new highlights', description: - 'Enter the template to use for new highlights. Required variables are: {{{text}}}, {{{highlightUrl}}}. Optional variables are {{{dateHighlighted}}}. You can also use the variables in the article template.', + 'We use {{ mustache }} template: http://mustache.github.io/mustache.5.html. Required variables are: text, highlightUrl. Optional variables are dateHighlighted. You can also use the variables in the article template.', default: `> {{{text}}} [⤴️]({{{highlightUrl}}}) {{#labels}} #[[{{{name}}}]] {{/labels}}`, inputAs: 'textarea', }, From 866528f4521a09990f07565d27423b96423871cd Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 23 Mar 2023 11:27:07 +0800 Subject: [PATCH 7/7] Remove note and redactions from highlights --- src/index.ts | 169 ++++++++++++++++++++++++--------------------------- 1 file changed, 79 insertions(+), 90 deletions(-) diff --git a/src/index.ts b/src/index.ts index f1c2f84..8dbebb6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -198,8 +198,11 @@ const fetchOmnivore = async (inBackground = false) => { const datePublished = article.publishedAt ? formatDate(new Date(article.publishedAt), preferredDateFormat) : undefined + const note = article.highlights?.find( + (h) => h.type === HighlightType.Note + ) // Build content string based on template - const articleView = { + const articleVariables = { title: article.title, omnivoreUrl: `https://omnivore.app/me/${article.slug}`, siteName, @@ -209,19 +212,16 @@ const fetchOmnivore = async (inBackground = false) => { dateSaved, content: article.content, datePublished, - note: '', + note: note?.annotation, } + // filter out notes and redactions + const highlights = article.highlights?.filter( + (h) => h.type === HighlightType.Highlight + ) // sort highlights by location if selected in options - highlightOrder === HighlightOrder.LOCATION && - article.highlights?.sort((a, b) => { + if (highlightOrder === HighlightOrder.LOCATION) { + highlights?.sort((a, b) => { try { - // if either highlight is not a highlight, put it at the end - if ( - a.type !== HighlightType.Highlight || - b.type !== HighlightType.Highlight - ) { - return 1 - } if (article.pageType === PageType.File) { // sort by location in file return compareHighlightsInFile(a, b) @@ -235,44 +235,33 @@ const fetchOmnivore = async (inBackground = false) => { return compareHighlightsInFile(a, b) } }) + } const highlightBatch: IBatchBlock[] = - (article.highlights - ?.map((it) => { - const highlightType = it.type - // filter out notes and redactions - if (highlightType !== HighlightType.Highlight) { - // add note variable to article template - if (highlightType === HighlightType.Note) { - articleView.note = it.annotation - } - return undefined - } - // Build content string based on template - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - const content = render(highlightTemplate, { - ...articleView, - text: it.quote, - labels: it.labels, - highlightUrl: `https://omnivore.app/me/${article.slug}#${it.id}`, - dateHighlighted: formatDate( - new Date(it.updatedAt), - preferredDateFormat - ), - }) - const noteChild = it.annotation - ? { content: it.annotation } - : undefined - return { - content, - children: noteChild ? [noteChild] : undefined, - properties: { - id: it.id, - }, - } + highlights?.map((it) => { + // Build content string based on template + // eslint-disable-next-line @typescript-eslint/no-unsafe-call + const content = render(highlightTemplate, { + ...articleVariables, + text: it.quote, + labels: it.labels, + highlightUrl: `https://omnivore.app/me/${article.slug}#${it.id}`, + dateHighlighted: formatDate( + new Date(it.updatedAt), + preferredDateFormat + ), }) - .filter((it) => it !== undefined) as IBatchBlock[]) || [] - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - const content = render(articleTemplate, articleView) + const noteChild = it.annotation + ? { content: it.annotation } + : undefined + return { + content, + children: noteChild ? [noteChild] : undefined, + properties: { + id: it.id, + }, + } + }) || [] + let isNewArticle = true // update existing block if article is already in the page const existingBlocks = ( @@ -289,21 +278,22 @@ const fetchOmnivore = async (inBackground = false) => { [(clojure.string/includes? ?c "${article.slug}")]]` ) ).flat() + // eslint-disable-next-line @typescript-eslint/no-unsafe-call + const articleContent = render(articleTemplate, articleVariables) if (existingBlocks.length > 0) { isNewArticle = false const existingBlock = existingBlocks[0] // update the first existing block - if (existingBlock.content !== content) { - await logseq.Editor.updateBlock(existingBlock.uuid, content) + if (existingBlock.content !== articleContent) { + await logseq.Editor.updateBlock(existingBlock.uuid, articleContent) } // delete the rest of the existing blocks await deleteBlocks(existingBlocks.slice(1)) - if (highlightBatch.length > 0) { - // append highlights to existing block - for (const highlight of highlightBatch) { - const existingHighlights = ( - await logseq.DB.datascriptQuery( - `[:find (pull ?b [*]) + // append highlights to existing block + for (const highlight of highlightBatch) { + const existingHighlights = ( + await logseq.DB.datascriptQuery( + `[:find (pull ?b [*]) :where [?b :block/parent ?p] [?p :block/uuid ?u] @@ -313,23 +303,23 @@ const fetchOmnivore = async (inBackground = false) => { [(clojure.string/includes? ?c "${ highlight.properties?.id as string }")]]` - ) - ).flat() - if (existingHighlights.length > 0) { - const existingHighlight = existingHighlights[0] - // update existing highlight if content is different - existingHighlight.content !== highlight.content && - (await logseq.Editor.updateBlock( - existingHighlight.uuid, - highlight.content - )) - - // checking notes - const noteChild = highlight.children?.[0] - if (noteChild) { - const existingNotes = ( - await logseq.DB.datascriptQuery( - `[:find (pull ?b [*]) + ) + ).flat() + if (existingHighlights.length > 0) { + const existingHighlight = existingHighlights[0] + // update existing highlight if content is different + existingHighlight.content !== highlight.content && + (await logseq.Editor.updateBlock( + existingHighlight.uuid, + highlight.content + )) + + // checking notes + const noteChild = highlight.children?.[0] + if (noteChild) { + const existingNotes = ( + await logseq.DB.datascriptQuery( + `[:find (pull ?b [*]) :where [?b :block/parent ?p] [?p :block/uuid ?u] @@ -339,32 +329,31 @@ const fetchOmnivore = async (inBackground = false) => { [(= ?c "${escapeQuotationMarks( noteChild.content )}")]]` - ) - ).flat() - if (existingNotes.length == 0) { - // append new note - await logseq.Editor.insertBlock( - existingHighlight.uuid, - noteChild.content, - { sibling: false } - ) - } + ) + ).flat() + if (existingNotes.length == 0) { + // append new note + await logseq.Editor.insertBlock( + existingHighlight.uuid, + noteChild.content, + { sibling: false } + ) } - } else { - // append new highlight - await logseq.Editor.insertBatchBlock( - existingBlock.uuid, - highlight, - { sibling: false } - ) } + } else { + // append new highlight + await logseq.Editor.insertBatchBlock( + existingBlock.uuid, + highlight, + { sibling: false } + ) } } } isNewArticle && articleBatch.unshift({ - content, + content: articleContent, children: highlightBatch, }) }