From dd4eb2dbd8418002e019a132f4579f4ff4e1ebbe Mon Sep 17 00:00:00 2001 From: Animesh Tarodia <61413897+anitarodia@users.noreply.github.com> Date: Tue, 27 Aug 2024 13:22:31 +0530 Subject: [PATCH 1/2] Update paginator.ts Introduced paginateEach and paginateEachSearchApi: These generator functions allow you to iterate over each page individually, improving memory efficiency. --- sdk-output/paginator.ts | 85 ++++++++++++++++++++++++++++++++--------- 1 file changed, 66 insertions(+), 19 deletions(-) diff --git a/sdk-output/paginator.ts b/sdk-output/paginator.ts index 9c94fec..32a0cb3 100644 --- a/sdk-output/paginator.ts +++ b/sdk-output/paginator.ts @@ -38,7 +38,11 @@ export interface ExtraParams { } export class Paginator { - public static async paginate< + /** + * Generator function that yields each page of results as it paginates through the API responses. + * Use this function for memory-efficient pagination. + */ + public static async *paginateEach< T, TResult, A extends PaginationParams & ExtraParams @@ -47,9 +51,9 @@ export class Paginator { callbackFn: (this: T, args: A) => Promise>, args?: A, increment?: number - ): Promise> { + ): AsyncGenerator { let params: PaginationParams = args ? args : { limit: 0, offset: 0 }; - const maxLimit = params && params.limit ? params.limit : 0; + const maxLimit = params.limit ? params.limit : 0; if (!params.offset) { params.offset = 0; } @@ -58,28 +62,71 @@ export class Paginator { } params.limit = increment; - let modified: TResult[] = []; while (true) { console.log(`Paginating call, offset = ${params.offset}`); let results = await callbackFn.call(thisArg, params); - modified.push.apply(modified, results.data); + yield results.data; + + // Break if the response contains fewer results than the increment or if the max limit is reached if ( results.data.length < increment || - (modified.length >= maxLimit && maxLimit > 0) + (params.offset! + results.data.length >= maxLimit && maxLimit > 0) ) { - results.data = modified; - return results; + break; } + params.offset += increment; } } + /** + * Collects all paginated results and returns them as a single array. + * This function is compatible with existing implementations that expect all data at once. + */ + public static async paginate< + T, + TResult, + A extends PaginationParams & ExtraParams + >( + thisArg: T, + callbackFn: (this: T, args: A) => Promise>, + args?: A, + increment?: number + ): Promise { + const allData: TResult[] = []; + for await (const page of this.paginateEach(thisArg, callbackFn, args, increment)) { + allData.push(...page); + } + return allData; + } + + /** + * Paginates through search results and collects them into a single array. + * Compatible with existing implementations. + */ public static async paginateSearchApi( searchAPI: SearchApi, search: Search, increment?: number, limit?: number - ): Promise> { + ): Promise { + const allData: SearchDocument[] = []; + for await (const page of this.paginateEachSearchApi(searchAPI, search, increment, limit)) { + allData.push(...page); + } + return allData; + } + + /** + * Generator function that yields each page of search results as it paginates through the API responses. + * Use this function for memory-efficient pagination. + */ + public static async *paginateEachSearchApi( + searchAPI: SearchApi, + search: Search, + increment?: number, + limit?: number + ): AsyncGenerator { increment = increment ? increment : 250; const searchParams: SearchApiSearchPostRequest = { search: search, @@ -87,30 +134,30 @@ export class Paginator { }; let offset = 0; const maxLimit = limit ? limit : 0; - let modified: SearchDocument[] = []; - if (!search.sort || search.sort.length != 1) { - throw "search must include exactly one sort parameter to paginate properly"; + if (!search.sort || search.sort.length !== 1) { + throw new Error("Search must include exactly one sort parameter to paginate properly"); } while (true) { console.log(`Paginating call, offset = ${offset}`); let results = await searchAPI.searchPost(searchParams); - modified.push.apply(modified, results.data); + yield results.data; + + // Break if the response contains fewer results than the increment or if the max limit is reached if ( results.data.length < increment || - (modified.length >= maxLimit && maxLimit > 0) + (offset + results.data.length >= maxLimit && maxLimit > 0) ) { - results.data = modified; - return results; + break; } else { - const result = results.data[results.data.length - 1]; + const lastResult = results.data[results.data.length - 1]; if (searchParams.search.sort) { searchParams.search.searchAfter = [ - result[searchParams.search.sort[0].replace("-", "")], + (lastResult as any)[searchParams.search.sort[0].replace("-", "")] ]; } else { - throw "search unexpectedly did not return a result we can search after!"; + throw new Error("Search unexpectedly did not return a result we can search after!"); } } offset += increment; From ebb3cf4eeff14bf6d37c5df5d893d9e594e85bf2 Mon Sep 17 00:00:00 2001 From: Animesh Tarodia <61413897+anitarodia@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:50:48 +0530 Subject: [PATCH 2/2] Update tsconfig.json Added for AsyncGenerator support in Paginator --- sdk-output/tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk-output/tsconfig.json b/sdk-output/tsconfig.json index bcb0315..a336d92 100644 --- a/sdk-output/tsconfig.json +++ b/sdk-output/tsconfig.json @@ -9,7 +9,8 @@ "rootDir": ".", "lib": [ "es6", - "dom" + "dom", + "ES2018" ], "typeRoots": [ "node_modules/@types"