Merge pull request #3563 from omnivore-app/fix/export-job

export at most 1000 most recent items
omnivore-app · Feb 23, 2024 · 2d45aa1 · 2d45aa1
2 parents 802ffbb + b947100
commit 2d45aa1
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 21 deletions.
diff --git a/packages/api/src/jobs/integration/export_all_items.ts b/packages/api/src/jobs/integration/export_all_items.ts
@@ -1,6 +1,6 @@
 import { IntegrationType } from '../../entity/integration'
 import { findIntegration } from '../../services/integrations'
-import { searchLibraryItems } from '../../services/library_item'
+import { findRecentLibraryItems } from '../../services/library_item'
 import { findActiveUser } from '../../services/user'
 import { enqueueExportItem } from '../../utils/createTask'
 import { logger } from '../../utils/logger'
@@ -39,31 +39,37 @@ export const exportAllItems = async (jobData: ExportAllItemsJobData) => {
     return
   }
 
-  // get paginated items from the database
-  const first = 50
-  let after = 0
-  for (;;) {
-    console.log('searching for items...', {
+  const maxItems = 1000
+  const limit = 100
+  let offset = 0
+  // get max 1000 most recent items from the database
+  while (offset < maxItems) {
+    const libraryItems = await findRecentLibraryItems(userId, limit, offset)
+    if (libraryItems.length === 0) {
+      logger.info('no library items found', {
+        userId,
+      })
+      return
+    }
+
+    logger.info('enqueuing export item...', {
       userId,
-      first,
-      after,
+      offset,
+      integrationId,
     })
-    const searchResult = await searchLibraryItems(
-      { from: after, size: first },
-      userId
-    )
-    const libraryItems = searchResult.libraryItems
-    const size = libraryItems.length
-    if (size === 0) {
-      break
-    }
 
     await enqueueExportItem({
       userId,
       libraryItemIds: libraryItems.map((item) => item.id),
       integrationId,
     })
 
-    after += size
+    offset += libraryItems.length
+
+    logger.info('exported items', {
+      userId,
+      offset,
+      integrationId,
+    })
   }
 }
diff --git a/packages/api/src/jobs/integration/export_item.ts b/packages/api/src/jobs/integration/export_item.ts
@@ -21,7 +21,6 @@ export const exportItem = async (jobData: ExportItemJobData) => {
   if (libraryItems.length === 0) {
     logger.error('library items not found', {
       userId,
-      libraryItemIds,
     })
     return
   }
@@ -40,7 +39,6 @@ export const exportItem = async (jobData: ExportItemJobData) => {
     integrations.map(async (integration) => {
       const logObject = {
         userId,
-        libraryItemIds,
         integrationId: integration.id,
       }
       logger.info('exporting item...', logObject)

diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts
@@ -652,14 +652,35 @@ export const searchLibraryItems = async (
   )
 }
 
+export const findRecentLibraryItems = async (
+  userId: string,
+  limit = 1000,
+  offset?: number
+) => {
+  return authTrx(
+    async (tx) =>
+      tx
+        .createQueryBuilder(LibraryItem, 'library_item')
+        .where('library_item.user_id = :userId', { userId })
+        .andWhere('library_item.state = :state', {
+          state: LibraryItemState.Succeeded,
+        })
+        .orderBy('library_item.saved_at', 'DESC', 'NULLS LAST')
+        .take(limit)
+        .skip(offset)
+        .getMany(),
+    undefined,
+    userId
+  )
+}
+
 export const findLibraryItemsByIds = async (ids: string[], userId: string) => {
   return authTrx(
     async (tx) =>
       tx
         .createQueryBuilder(LibraryItem, 'library_item')
         .leftJoinAndSelect('library_item.labels', 'labels')
         .leftJoinAndSelect('library_item.highlights', 'highlights')
-        .leftJoinAndSelect('highlights.user', 'user')
         .where('library_item.id IN (:...ids)', { ids })
         .getMany(),
     undefined,