Refactor getDocumentStream w.r.t. the cursor based pagination in getP…

…ageByType

shared-libs/cht-datasource/src/libs/data-context.ts

@@ -1,4 +1,4 @@
-import { hasField, isRecord } from './core';
+import { hasField, isRecord, Page } from './core';
 import { isLocalDataContext, LocalDataContext } from '../local/libs/data-context';
 import { assertRemoteDataContext, isRemoteDataContext, RemoteDataContext } from '../remote/libs/data-context';
 
@@ -42,24 +42,20 @@ export const adapt = <T>(
 
 /** @internal */
 export const getDocumentStream = async function* <S, T>(
-  fetchFunction: (args: S, l: number, s: number) => Promise<T[]>,
+  fetchFunction: (args: S, l: number, s: number) => Promise<Page<T>>,
   fetchFunctionArgs: S
 ): AsyncGenerator<T, void> {
   const limit = 100;
   let skip = 0;
-  // TODO: found a bug in this function where if limit is 1
-  // then it will always return true but right now since limit
-  // is hardcoded to 100, it won't but if limit is made to be
-  // dynamic then, change this implementation
-  const hasMoreResults = () => skip && skip % limit === 0;
+  const hasMoreResults = () => skip !== -1;
 
   do {
     const docs = await fetchFunction(fetchFunctionArgs, limit, skip);
 
-    for (const doc of docs) {
+    for (const doc of docs.data) {
       yield doc;
     }
 
-    skip += docs.length;
+    skip = Number(docs.cursor);
   } while (hasMoreResults());
 };

shared-libs/cht-datasource/test/libs/data-context.spec.ts

@@ -130,8 +130,9 @@ describe('context lib', () => {
 
     it('yields document one by one', async () => {
       const mockDocs = [{ id: 1 }, { id: 2 }, { id: 3 }];
+      const mockPage = { data: mockDocs, cursor: '-1' };
       const extraArg = 'value';
-      fetchFunctionStub.resolves(mockDocs);
+      fetchFunctionStub.resolves(mockPage);
 
       const generator = getDocumentStream(fetchFunctionStub, extraArg);
 
@@ -148,11 +149,13 @@ describe('context lib', () => {
     it('should handle multiple pages',  async () => {
       const mockDoc = { id: 1 };
       const mockDocs1 = Array.from({ length: 100 }, () => ({ ...mockDoc }));
+      const mockPage1 = { data: mockDocs1, cursor: '100' };
       const mockDocs2 = [{ id: 101 }];
+      const mockPage2 = { data: mockDocs2, cursor: '-1' };
       const extraArg = 'value';
 
-      fetchFunctionStub.onFirstCall().resolves(mockDocs1);
-      fetchFunctionStub.onSecondCall().resolves(mockDocs2);
+      fetchFunctionStub.onFirstCall().resolves(mockPage1);
+      fetchFunctionStub.onSecondCall().resolves(mockPage2);
 
       const generator = getDocumentStream(fetchFunctionStub, extraArg);
 
@@ -168,7 +171,7 @@ describe('context lib', () => {
     });
 
     it('should handle empty result', async () => {
-      fetchFunctionStub.resolves([]);
+      fetchFunctionStub.resolves({ data: [], cursor: '-1' });
 
       const generator = getDocumentStream(fetchFunctionStub, { limit: 10, skip: 0 });