diff --git a/packages/api/src/jobs/integration/export_all_items.ts b/packages/api/src/jobs/integration/export_all_items.ts index fb29b20fb5..930a4c8df4 100644 --- a/packages/api/src/jobs/integration/export_all_items.ts +++ b/packages/api/src/jobs/integration/export_all_items.ts @@ -1,6 +1,6 @@ import { IntegrationType } from '../../entity/integration' import { findIntegration } from '../../services/integrations' -import { searchLibraryItems } from '../../services/library_item' +import { findRecentLibraryItems } from '../../services/library_item' import { findActiveUser } from '../../services/user' import { enqueueExportItem } from '../../utils/createTask' import { logger } from '../../utils/logger' @@ -39,24 +39,24 @@ export const exportAllItems = async (jobData: ExportAllItemsJobData) => { return } - // get paginated items from the database - const first = 50 - let after = 0 - for (;;) { - console.log('searching for items...', { + const maxItems = 1000 + const limit = 100 + let offset = 0 + // get max 1000 most recent items from the database + while (offset < maxItems) { + const libraryItems = await findRecentLibraryItems(userId, limit, offset) + if (libraryItems.length === 0) { + logger.info('no library items found', { + userId, + }) + return + } + + logger.info('enqueuing export item...', { userId, - first, - after, + offset, + integrationId, }) - const searchResult = await searchLibraryItems( - { from: after, size: first }, - userId - ) - const libraryItems = searchResult.libraryItems - const size = libraryItems.length - if (size === 0) { - break - } await enqueueExportItem({ userId, @@ -64,6 +64,12 @@ export const exportAllItems = async (jobData: ExportAllItemsJobData) => { integrationId, }) - after += size + offset += libraryItems.length + + logger.info('exported items', { + userId, + offset, + integrationId, + }) } } diff --git a/packages/api/src/jobs/integration/export_item.ts b/packages/api/src/jobs/integration/export_item.ts index 896ca45b4e..cd99bba5ef 100644 --- a/packages/api/src/jobs/integration/export_item.ts +++ b/packages/api/src/jobs/integration/export_item.ts @@ -21,7 +21,6 @@ export const exportItem = async (jobData: ExportItemJobData) => { if (libraryItems.length === 0) { logger.error('library items not found', { userId, - libraryItemIds, }) return } @@ -40,7 +39,6 @@ export const exportItem = async (jobData: ExportItemJobData) => { integrations.map(async (integration) => { const logObject = { userId, - libraryItemIds, integrationId: integration.id, } logger.info('exporting item...', logObject) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 4cb5817842..e2f73cc0ea 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -652,6 +652,28 @@ export const searchLibraryItems = async ( ) } +export const findRecentLibraryItems = async ( + userId: string, + limit = 1000, + offset?: number +) => { + return authTrx( + async (tx) => + tx + .createQueryBuilder(LibraryItem, 'library_item') + .where('library_item.user_id = :userId', { userId }) + .andWhere('library_item.state = :state', { + state: LibraryItemState.Succeeded, + }) + .orderBy('library_item.saved_at', 'DESC', 'NULLS LAST') + .take(limit) + .skip(offset) + .getMany(), + undefined, + userId + ) +} + export const findLibraryItemsByIds = async (ids: string[], userId: string) => { return authTrx( async (tx) => @@ -659,7 +681,6 @@ export const findLibraryItemsByIds = async (ids: string[], userId: string) => { .createQueryBuilder(LibraryItem, 'library_item') .leftJoinAndSelect('library_item.labels', 'labels') .leftJoinAndSelect('library_item.highlights', 'highlights') - .leftJoinAndSelect('highlights.user', 'user') .where('library_item.id IN (:...ids)', { ids }) .getMany(), undefined,