Improved Paginator Functionality

sdk-output/paginator.ts

@@ -38,7 +38,11 @@ export interface ExtraParams {
 }
 
 export class Paginator {
-  public static async paginate<
+  /**
+   * Generator function that yields each page of results as it paginates through the API responses.
+   * Use this function for memory-efficient pagination.
+   */
+  public static async *paginateEach<
     T,
     TResult,
     A extends PaginationParams & ExtraParams
@@ -47,9 +51,9 @@ export class Paginator {
     callbackFn: (this: T, args: A) => Promise<AxiosResponse<TResult[], any>>,
     args?: A,
     increment?: number
-  ): Promise<AxiosResponse<TResult[], any>> {
+  ): AsyncGenerator<TResult[], void, unknown> {
     let params: PaginationParams = args ? args : { limit: 0, offset: 0 };
-    const maxLimit = params && params.limit ? params.limit : 0;
+    const maxLimit = params.limit ? params.limit : 0;
     if (!params.offset) {
       params.offset = 0;
     }
@@ -58,59 +62,102 @@ export class Paginator {
     }
     params.limit = increment;
 
-    let modified: TResult[] = [];
     while (true) {
       console.log(`Paginating call, offset = ${params.offset}`);
       let results = await callbackFn.call(thisArg, params);
-      modified.push.apply(modified, results.data);
+      yield results.data;
+
+      // Break if the response contains fewer results than the increment or if the max limit is reached
       if (
         results.data.length < increment ||
-        (modified.length >= maxLimit && maxLimit > 0)
+        (params.offset! + results.data.length >= maxLimit && maxLimit > 0)
       ) {
-        results.data = modified;
-        return results;
+        break;
       }
+
       params.offset += increment;
     }
   }
 
+  /**
+   * Collects all paginated results and returns them as a single array.
+   * This function is compatible with existing implementations that expect all data at once.
+   */
+  public static async paginate<
+    T,
+    TResult,
+    A extends PaginationParams & ExtraParams
+  >(
+    thisArg: T,
+    callbackFn: (this: T, args: A) => Promise<AxiosResponse<TResult[], any>>,
+    args?: A,
+    increment?: number
+  ): Promise<TResult[]> {
+    const allData: TResult[] = [];
+    for await (const page of this.paginateEach(thisArg, callbackFn, args, increment)) {
+      allData.push(...page);
+    }
+    return allData;
+  }
+
+  /**
+   * Paginates through search results and collects them into a single array.
+   * Compatible with existing implementations.
+   */
   public static async paginateSearchApi(
     searchAPI: SearchApi,
     search: Search,
     increment?: number,
     limit?: number
-  ): Promise<AxiosResponse<SearchDocument[], any>> {
+  ): Promise<SearchDocument[]> {
+    const allData: SearchDocument[] = [];
+    for await (const page of this.paginateEachSearchApi(searchAPI, search, increment, limit)) {
+      allData.push(...page);
+    }
+    return allData;
+  }
+
+  /**
+   * Generator function that yields each page of search results as it paginates through the API responses.
+   * Use this function for memory-efficient pagination.
+   */
+  public static async *paginateEachSearchApi(
+    searchAPI: SearchApi,
+    search: Search,
+    increment?: number,
+    limit?: number
+  ): AsyncGenerator<SearchDocument[], void, unknown> {
     increment = increment ? increment : 250;
     const searchParams: SearchApiSearchPostRequest = {
       search: search,
       limit: increment,
     };
     let offset = 0;
     const maxLimit = limit ? limit : 0;
-    let modified: SearchDocument[] = [];
 
-    if (!search.sort || search.sort.length != 1) {
-      throw "search must include exactly one sort parameter to paginate properly";
+    if (!search.sort || search.sort.length !== 1) {
+      throw new Error("Search must include exactly one sort parameter to paginate properly");
     }
 
     while (true) {
       console.log(`Paginating call, offset = ${offset}`);
       let results = await searchAPI.searchPost(searchParams);
-      modified.push.apply(modified, results.data);
+      yield results.data;
+
+      // Break if the response contains fewer results than the increment or if the max limit is reached
       if (
         results.data.length < increment ||
-        (modified.length >= maxLimit && maxLimit > 0)
+        (offset + results.data.length >= maxLimit && maxLimit > 0)
       ) {
-        results.data = modified;
-        return results;
+        break;
       } else {
-        const result = <any>results.data[results.data.length - 1];
+        const lastResult = results.data[results.data.length - 1];
         if (searchParams.search.sort) {
           searchParams.search.searchAfter = [
-            result[searchParams.search.sort[0].replace("-", "")],
+            (lastResult as any)[searchParams.search.sort[0].replace("-", "")]
           ];
         } else {
-          throw "search unexpectedly did not return a result we can search after!";
+          throw new Error("Search unexpectedly did not return a result we can search after!");
         }
       }
       offset += increment;

sdk-output/tsconfig.json

@@ -9,7 +9,8 @@
     "rootDir": ".",
     "lib": [
       "es6",
-      "dom"
+      "dom",
+      "ES2018"
     ],
     "typeRoots": [
       "node_modules/@types"