From 0d6712df6e133a1d75e5a3dde32ce20ba0590561 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Sat, 11 May 2024 11:32:16 +0800 Subject: [PATCH 1/3] polling new content api to get highlighted content --- src/api.ts | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/src/api.ts b/src/api.ts index 1300187..541104b 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,4 +1,5 @@ import { Item, ItemFormat, Omnivore } from '@omnivore-app/api' +import { requestUrl } from 'obsidian' export enum HighlightColors { Yellow = 'yellow', @@ -7,6 +8,50 @@ export enum HighlightColors { Blue = 'blue', } +interface GetContentResponse { + data: { + libraryItemId: string + downloadUrl: string + error?: string + }[] +} + +const getContent = async ( + endpoint: string, + apiKey: string, + libraryItemIds: string[], +): Promise => { + const response = await requestUrl({ + url: `${endpoint}/api/content`, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: apiKey, + }, + body: JSON.stringify({ libraryItemIds, format: 'highlightedMarkdown' }), + }) + + return response.json +} + +const downloadFromUrl = async (url: string): Promise => { + try { + // polling until download is ready or failed + const response = await requestUrl({ + url, + }) + return response.text + } catch (error) { + // retry after 1 second if download returns 404 + if (error.status === 404) { + await sleep(1000) + return downloadFromUrl(url) + } + + throw error + } +} + export const getItems = async ( endpoint: string, apiKey: string, @@ -27,11 +72,40 @@ export const getItems = async ( after, first, query: `${updatedAt ? 'updated:' + updatedAt : ''} sort:saved-asc ${query}`, - includeContent, + includeContent: false, format, }) const articles = response.edges.map((e) => e.node) + if (includeContent && articles.length > 0) { + try { + const content = await getContent( + endpoint, + apiKey, + articles.map((a) => a.id), + ) + + await Promise.allSettled( + content.data.map(async (c, index) => { + if (c.error) { + console.error('Error fetching content', c.error) + return + } + + // timeout if download takes too long + articles[index].content = await Promise.race([ + downloadFromUrl(c.downloadUrl), + new Promise( + (_, reject) => setTimeout(() => reject('Timeout'), 10000), // 10 seconds + ), + ]) + }), + ) + } catch (error) { + console.error('Error fetching content', error) + } + } + return [articles, response.pageInfo.hasNextPage] } From 09a270b8405c5bd4a37557d8f8dc16cbfdd13f73 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 13 May 2024 20:49:28 +0800 Subject: [PATCH 2/3] timeout if downloading content takes more than 10 minutes --- src/api.ts | 64 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/src/api.ts b/src/api.ts index 541104b..6797c90 100644 --- a/src/api.ts +++ b/src/api.ts @@ -52,6 +52,41 @@ const downloadFromUrl = async (url: string): Promise => { } } +const fetchContentForItems = async ( + endpoint: string, + apiKey: string, + items: Item[], +) => { + const content = await getContent( + endpoint, + apiKey, + items.map((a) => a.id), + ) + + await Promise.allSettled( + content.data.map(async (c) => { + if (c.error) { + console.error('Error fetching content', c.error) + return + } + + const item = items.find((i) => i.id === c.libraryItemId) + if (!item) { + console.error('Item not found', c.libraryItemId) + return + } + + // timeout if download takes too long + item.content = await Promise.race([ + downloadFromUrl(c.downloadUrl), + new Promise( + (_, reject) => setTimeout(() => reject('Timeout'), 600_000), // 10 minutes + ), + ]) + }), + ) +} + export const getItems = async ( endpoint: string, apiKey: string, @@ -76,37 +111,16 @@ export const getItems = async ( format, }) - const articles = response.edges.map((e) => e.node) - if (includeContent && articles.length > 0) { + const items = response.edges.map((e) => e.node) + if (includeContent && items.length > 0) { try { - const content = await getContent( - endpoint, - apiKey, - articles.map((a) => a.id), - ) - - await Promise.allSettled( - content.data.map(async (c, index) => { - if (c.error) { - console.error('Error fetching content', c.error) - return - } - - // timeout if download takes too long - articles[index].content = await Promise.race([ - downloadFromUrl(c.downloadUrl), - new Promise( - (_, reject) => setTimeout(() => reject('Timeout'), 10000), // 10 seconds - ), - ]) - }), - ) + await fetchContentForItems(endpoint, apiKey, items) } catch (error) { console.error('Error fetching content', error) } } - return [articles, response.pageInfo.hasNextPage] + return [items, response.pageInfo.hasNextPage] } export const deleteItem = async ( From 432a40497b95d418c0c65c12bf4e55ff0fc671e1 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 16 May 2024 15:46:58 +0800 Subject: [PATCH 3/3] fix: wrong base url in the api request --- src/api.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/api.ts b/src/api.ts index 6797c90..8f1f1b0 100644 --- a/src/api.ts +++ b/src/api.ts @@ -16,13 +16,15 @@ interface GetContentResponse { }[] } +const baseUrl = (endpoint: string) => endpoint.replace(/\/api\/graphql$/, '') + const getContent = async ( endpoint: string, apiKey: string, libraryItemIds: string[], ): Promise => { const response = await requestUrl({ - url: `${endpoint}/api/content`, + url: `${baseUrl(endpoint)}/api/content`, method: 'POST', headers: { 'Content-Type': 'application/json', @@ -80,7 +82,7 @@ const fetchContentForItems = async ( item.content = await Promise.race([ downloadFromUrl(c.downloadUrl), new Promise( - (_, reject) => setTimeout(() => reject('Timeout'), 600_000), // 10 minutes + (_, reject) => setTimeout(() => reject('Timeout'), 60_000), // 60 seconds ), ]) }), @@ -99,7 +101,7 @@ export const getItems = async ( ): Promise<[Item[], boolean]> => { const omnivore = new Omnivore({ authToken: apiKey, - baseUrl: endpoint, + baseUrl: baseUrl(endpoint), timeoutMs: 10000, // 10 seconds }) @@ -130,7 +132,7 @@ export const deleteItem = async ( ) => { const omnivore = new Omnivore({ authToken: apiKey, - baseUrl: endpoint, + baseUrl: baseUrl(endpoint), timeoutMs: 10000, // 10 seconds })