Merge pull request #236 from omnivore-app/feature/new-content-api

polling new content api to get highlighted content

src/api.ts

@@ -1,4 +1,5 @@
 import { Item, ItemFormat, Omnivore } from '@omnivore-app/api'
+import { requestUrl } from 'obsidian'
 
 export enum HighlightColors {
   Yellow = 'yellow',
@@ -7,6 +8,87 @@ export enum HighlightColors {
   Blue = 'blue',
 }
 
+interface GetContentResponse {
+  data: {
+    libraryItemId: string
+    downloadUrl: string
+    error?: string
+  }[]
+}
+
+const baseUrl = (endpoint: string) => endpoint.replace(/\/api\/graphql$/, '')
+
+const getContent = async (
+  endpoint: string,
+  apiKey: string,
+  libraryItemIds: string[],
+): Promise<GetContentResponse> => {
+  const response = await requestUrl({
+    url: `${baseUrl(endpoint)}/api/content`,
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: apiKey,
+    },
+    body: JSON.stringify({ libraryItemIds, format: 'highlightedMarkdown' }),
+  })
+
+  return response.json
+}
+
+const downloadFromUrl = async (url: string): Promise<string> => {
+  try {
+    // polling until download is ready or failed
+    const response = await requestUrl({
+      url,
+    })
+    return response.text
+  } catch (error) {
+    // retry after 1 second if download returns 404
+    if (error.status === 404) {
+      await sleep(1000)
+      return downloadFromUrl(url)
+    }
+
+    throw error
+  }
+}
+
+const fetchContentForItems = async (
+  endpoint: string,
+  apiKey: string,
+  items: Item[],
+) => {
+  const content = await getContent(
+    endpoint,
+    apiKey,
+    items.map((a) => a.id),
+  )
+
+  await Promise.allSettled(
+    content.data.map(async (c) => {
+      if (c.error) {
+        console.error('Error fetching content', c.error)
+        return
+      }
+
+      const item = items.find((i) => i.id === c.libraryItemId)
+      if (!item) {
+        console.error('Item not found', c.libraryItemId)
+        return
+      }
+
+      // timeout if download takes too long
+      item.content = await Promise.race([
+        downloadFromUrl(c.downloadUrl),
+        new Promise<string>(
+          (_, reject) => setTimeout(() => reject('Timeout'), 60_000), // 60 seconds
+        ),
+      ])
+    }),
+  )
+}
+
 export const getItems = async (
   endpoint: string,
   apiKey: string,
@@ -19,20 +101,28 @@ export const getItems = async (
 ): Promise<[Item[], boolean]> => {
   const omnivore = new Omnivore({
     authToken: apiKey,
-    baseUrl: endpoint,
+    baseUrl: baseUrl(endpoint),
     timeoutMs: 10000, // 10 seconds
   })
 
   const response = await omnivore.items.search({
     after,
     first,
     query: `${updatedAt ? 'updated:' + updatedAt : ''} sort:saved-asc ${query}`,
-    includeContent,
+    includeContent: false,
     format,
   })
 
-  const articles = response.edges.map((e) => e.node)
-  return [articles, response.pageInfo.hasNextPage]
+  const items = response.edges.map((e) => e.node)
+  if (includeContent && items.length > 0) {
+    try {
+      await fetchContentForItems(endpoint, apiKey, items)
+    } catch (error) {
+      console.error('Error fetching content', error)
+    }
+  }
+
+  return [items, response.pageInfo.hasNextPage]
 }
 
 export const deleteItem = async (
@@ -42,7 +132,7 @@ export const deleteItem = async (
 ) => {
   const omnivore = new Omnivore({
     authToken: apiKey,
-    baseUrl: endpoint,
+    baseUrl: baseUrl(endpoint),
     timeoutMs: 10000, // 10 seconds
   })