feat(#9193): add functionality of getting places as pages or async iterables in cht-datasource

api/src/controllers/place.js

@@ -9,19 +9,35 @@ const getPlace = ({ with_lineage }) => ctx.bind(
     : Place.v1.get
 );
 
+const getPageByType = () => ctx.bind(Place.v1.getPage);
+
+const checkUserPermissions = async (req) => {
+  const userCtx = await auth.getUserCtx(req);
+  if (!auth.isOnlineOnly(userCtx) || !auth.hasAllPermissions(userCtx, 'can_view_contacts')) {
+    return Promise.reject({ code: 403, message: 'Insufficient privileges' });
+  }
+};
+
 module.exports = {
   v1: {
     get: serverUtils.doOrError(async (req, res) => {
-      const userCtx = await auth.getUserCtx(req);
-      if (!auth.isOnlineOnly(userCtx) || !auth.hasAllPermissions(userCtx, 'can_view_contacts')) {
-        return Promise.reject({ code: 403, message: 'Insufficient privileges' });
-      }
+      await checkUserPermissions(req);
       const { uuid } = req.params;
       const place = await getPlace(req.query)(Qualifier.byUuid(uuid));
       if (!place) {
         return serverUtils.error({ status: 404, message: 'Place not found' }, req, res);
       }
       return res.json(place);
+    }),
+    getAll: serverUtils.doOrError(async (req, res) => {
+      await checkUserPermissions(req);
+
+      const placeType = Qualifier.byContactType(req.query.type);
+      const limit = req.query.limit ? Number(req.query.limit) : req.query.limit;
+
+      const docs = await getPageByType()( placeType, req.query.cursor, limit );
+
+      return res.json(docs);
     })
   }
 };

api/src/routing.js

@@ -467,6 +467,7 @@ app.postJson('/api/v1/places/:id', function(req, res) {
     .catch(err => serverUtils.error(err, req, res));
 });
 
+app.get('/api/v1/place', place.v1.getAll);
 app.get('/api/v1/place/:uuid', place.v1.get);
 
 app.postJson('/api/v1/people', function(req, res) {

api/tests/mocha/controllers/person.spec.js

@@ -229,7 +229,7 @@ describe('Person Controller', () => {
         expect(serverUtilsError.calledOnceWithExactly(error, req, res)).to.be.true;
       });
 
-      it('returns 400 error when argument is invalid', async () => {
+      it('returns 400 error when personType is invalid', async () => {
         const err = new InvalidArgumentError(`Invalid contact type: [${invalidPersonType}]`);
         isOnlineOnly.returns(true);
         hasAllPermissions.returns(true);

api/tests/mocha/controllers/place.spec.js

@@ -1,6 +1,6 @@
 const sinon = require('sinon');
 const { expect } = require('chai');
-const { Place, Qualifier } = require('@medic/cht-datasource');
+const { Place, Qualifier, InvalidArgumentError} = require('@medic/cht-datasource');
 const auth = require('../../../src/auth');
 const controller = require('../../../src/controllers/place');
 const dataContext = require('../../../src/services/data-context');
@@ -154,5 +154,112 @@ describe('Place Controller', () => {
         expect(serverUtilsError.calledOnceWithExactly(error, req, res)).to.be.true;
       });
     });
+
+    describe('getAll', () => {
+      let placeGetPageByType;
+      let qualifierByContactType;
+      const placeType = 'place';
+      const invalidPlaceType = 'invalidPlace';
+      const placeTypeQualifier = { contactType: placeType };
+      const place = { name: 'Clinic' };
+      const limit = 100;
+      const cursor = null;
+      const places = Array.from({ length: 3 }, () => ({ ...place }));
+
+      beforeEach(() => {
+        req = {
+          query: {
+            type: placeType,
+            cursor,
+            limit,
+          }
+        };
+        placeGetPageByType = sinon.stub();
+        qualifierByContactType = sinon.stub(Qualifier, 'byContactType');
+        dataContextBind.withArgs(Place.v1.getPage).returns(placeGetPageByType);
+        qualifierByContactType.returns(placeTypeQualifier);
+      });
+
+      afterEach(() => {
+        expect(getUserCtx.calledOnceWithExactly(req)).to.be.true;
+        expect(isOnlineOnly.calledOnceWithExactly(userCtx)).to.be.true;
+      });
+
+      it('returns a page of places with correct query params', async () => {
+        isOnlineOnly.returns(true);
+        hasAllPermissions.returns(true);
+        placeGetPageByType.resolves(places);
+
+        await controller.v1.getAll(req, res);
+
+        expect(hasAllPermissions.calledOnceWithExactly(userCtx, 'can_view_contacts')).to.be.true;
+        expect(qualifierByContactType.calledOnceWithExactly(req.query.type)).to.be.true;
+        expect(dataContextBind.calledOnceWithExactly(Place.v1.getPage)).to.be.true;
+        expect(placeGetPageByType.calledOnceWithExactly(placeTypeQualifier, cursor, limit)).to.be.true;
+        expect(res.json.calledOnceWithExactly(places)).to.be.true;
+        expect(serverUtilsError.notCalled).to.be.true;
+      });
+
+      it('returns error if user does not have can_view_contacts permission', async () => {
+        const error = { code: 403, message: 'Insufficient privileges' };
+        isOnlineOnly.returns(true);
+        hasAllPermissions.returns(false);
+
+        await controller.v1.getAll(req, res);
+
+        expect(hasAllPermissions.calledOnceWithExactly(userCtx, 'can_view_contacts')).to.be.true;
+        expect(dataContextBind.notCalled).to.be.true;
+        expect(qualifierByContactType.notCalled).to.be.true;
+        expect(placeGetPageByType.notCalled).to.be.true;
+        expect(res.json.notCalled).to.be.true;
+        expect(serverUtilsError.calledOnceWithExactly(error, req, res)).to.be.true;
+      });
+
+      it('returns error if not an online user', async () => {
+        const error = { code: 403, message: 'Insufficient privileges' };
+        isOnlineOnly.returns(false);
+
+        await controller.v1.getAll(req, res);
+
+        expect(hasAllPermissions.notCalled).to.be.true;
+        expect(dataContextBind.notCalled).to.be.true;
+        expect(qualifierByContactType.notCalled).to.be.true;
+        expect(placeGetPageByType.notCalled).to.be.true;
+        expect(res.json.notCalled).to.be.true;
+        expect(serverUtilsError.calledOnceWithExactly(error, req, res)).to.be.true;
+      });
+
+      it('returns 400 error when placeType is invalid', async () => {
+        const err = new InvalidArgumentError(`Invalid contact type: [${invalidPlaceType}].`);
+        isOnlineOnly.returns(true);
+        hasAllPermissions.returns(true);
+        placeGetPageByType.throws(err);
+
+        await controller.v1.getAll(req, res);
+
+        expect(hasAllPermissions.calledOnceWithExactly(userCtx, 'can_view_contacts')).to.be.true;
+        expect(qualifierByContactType.calledOnceWithExactly(req.query.type)).to.be.true;
+        expect(dataContextBind.calledOnceWithExactly(Place.v1.getPage)).to.be.true;
+        expect(placeGetPageByType.calledOnceWithExactly(placeTypeQualifier, cursor, limit)).to.be.true;
+        expect(res.json.notCalled).to.be.true;
+        expect(serverUtilsError.calledOnceWithExactly(err, req, res)).to.be.true;
+      });
+
+      it('rethrows error in case of other errors', async () => {
+        const err = new Error('error');
+        isOnlineOnly.returns(true);
+        hasAllPermissions.returns(true);
+        placeGetPageByType.throws(err);
+
+        await controller.v1.getAll(req, res);
+
+        expect(hasAllPermissions.calledOnceWithExactly(userCtx, 'can_view_contacts')).to.be.true;
+        expect(qualifierByContactType.calledOnceWithExactly(req.query.type)).to.be.true;
+        expect(dataContextBind.calledOnceWithExactly(Place.v1.getPage)).to.be.true;
+        expect(placeGetPageByType.calledOnceWithExactly(placeTypeQualifier, cursor, limit)).to.be.true;
+        expect(res.json.notCalled).to.be.true;
+        expect(serverUtilsError.calledOnceWithExactly(err, req, res)).to.be.true;
+      });
+    });
   });
 });

shared-libs/cht-datasource/src/index.ts

@@ -70,6 +70,34 @@ export const getDatasource = (ctx: DataContext) => {
          * @throws Error if no UUID is provided
          */
         getByUuidWithLineage: (uuid: string) => ctx.bind(Place.v1.getWithLineage)(Qualifier.byUuid(uuid)),
+
+        /**
+         * Returns an array of places for the provided page specifications.
+         * @param placeType the type of place to return
+         * @param cursor the token identifying which page to retrieve. A `null` value indicates the first page should be
+         * returned. Subsequent pages can be retrieved by providing the cursor returned with the previous page.
+         * @param limit the maximum number of place to return. Default is 100.
+         * @returns a page of places for the provided specifications
+         * @throws InvalidArgumentError if no type is provided or if the type is not for a place
+         * @throws InvalidArgumentError if the provided limit is `<= 0`
+         * @throws InvalidArgumentError if the provided cursor is not a valid page token or `null`
+         * @see {@link getByType} which provides the same data, but without having to manually account for paging
+         */
+        getPageByType: (
+          placeType: string,
+          cursor: Nullable<string> = null,
+          limit = 100
+        ) => ctx.bind(Place.v1.getPage)(
+          Qualifier.byContactType(placeType), cursor, limit
+        ),
+
+        /**
+         * Returns a generator for fetching all places with the given type.
+         * @param placeType the type of place to return
+         * @returns a generator for fetching all places with the given type
+         * @throws InvalidArgumentError if no type if provided or if the type is not for a place
+         */
+        getByType: (placeType: string) => ctx.bind(Place.v1.getAll)(Qualifier.byContactType(placeType))
       },
       person: {
         /**
@@ -95,9 +123,9 @@ export const getDatasource = (ctx: DataContext) => {
          * returned. Subsequent pages can be retrieved by providing the cursor returned with the previous page.
          * @param limit the maximum number of people to return. Default is 100.
          * @returns a page of people for the provided specifications
-         * @throws Error if no type is provided or if the type is not for a person
-         * @throws Error if the provided limit is `<= 0`
-         * @throws Error if the provided cursor is not a valid page token or `null`
+         * @throws InvalidArgumentError if no type is provided or if the type is not for a person
+         * @throws InvalidArgumentError if the provided limit is `<= 0`
+         * @throws InvalidArgumentError if the provided cursor is not a valid page token or `null`
          * @see {@link getByType} which provides the same data, but without having to manually account for paging
          */
         getPageByType: (
@@ -112,7 +140,7 @@ export const getDatasource = (ctx: DataContext) => {
          * Returns a generator for fetching all people with the given type.
          * @param personType the type of people to return
          * @returns a generator for fetching all people with the given type
-         * @throws Error if no type is provided or if the type is not for a person
+         * @throws InvalidArgumentError if no type is provided or if the type is not for a person
          */
         getByType: (personType: string) => ctx.bind(Person.v1.getAll)(Qualifier.byContactType(personType)),
       }

shared-libs/cht-datasource/src/libs/core.ts

@@ -1,4 +1,6 @@
 import { DataContext } from './data-context';
+import { ContactTypeQualifier, isContactTypeQualifier } from '../qualifier';
+import { InvalidArgumentError } from './error';
 
 /**
  * A value that could be `null`.
@@ -144,3 +146,26 @@ export const getPagedGenerator = async function* <S, T>(
 
   return null;
 };
+
+/** @internal */
+export const assertTypeQualifier: (qualifier: unknown) => asserts qualifier is ContactTypeQualifier = (
+  qualifier: unknown
+) => {
+  if (!isContactTypeQualifier(qualifier)) {
+    throw new InvalidArgumentError(`Invalid contact type [${JSON.stringify(qualifier)}].`);
+  }
+};
+
+/** @internal */
+export const assertLimit: (limit: unknown) => asserts limit is number = (limit: unknown) => {
+  if (typeof limit !== 'number' || !Number.isInteger(limit) || limit <= 0) {
+    throw new InvalidArgumentError(`The limit must be a positive number: [${String(limit)}].`);
+  }
+};
+
+/** @internal */
+export const assertCursor: (cursor: unknown) => asserts cursor is Nullable<string> = (cursor: unknown) => {
+  if (cursor !== null && (typeof cursor !== 'string' || !cursor.length)) {
+    throw new InvalidArgumentError(`Invalid cursor token: [${String(cursor)}].`);
+  }
+};

shared-libs/cht-datasource/src/local/libs/doc.ts

@@ -1,5 +1,5 @@
 import logger from '@medic/logger';
-import { Nullable } from '../../libs/core';
+import { Nullable, Page } from '../../libs/core';
 import { Doc, isDoc } from '../../libs/doc';
 
 /** @internal */
@@ -53,3 +53,52 @@ export const queryDocsByKey = (
   limit: number,
   skip: number
 ): Promise<Nullable<Doc>[]> => queryDocs(db, view, { include_docs: true, key, limit, skip });
+
+/**
+ * Resolves a page containing an array of T using the getFunction to retrieve documents from the database
+ * and the filterFunction to validate the returned documents are all of type T.
+ * The length of the page's data array is guaranteed to equal limit unless there is no more data to retrieve
+ * from the database. This function will try to minimize the number of getFunction calls required to find
+ * the necessary data by over-fetching during followup calls if some retrieved docs are rejected by the filterFunction.
+ * @internal
+ */
+export const fetchAndFilter = <T extends Doc>(
+  getFunction: (limit: number, skip: number) => Promise<Nullable<T>[]>,
+  filterFunction: (doc: Nullable<T>, uuid?: string) => boolean,
+  limit: number,
+): typeof recursionInner => {
+  const recursionInner = async (
+    currentLimit: number,
+    currentSkip: number,
+    currentDocs: T[] = [],
+  ): Promise<Page<T>> => {
+    const docs = await getFunction(currentLimit, currentSkip);
+    const noMoreResults = docs.length < currentLimit;
+    const newDocs = docs.filter((doc): doc is T => filterFunction(doc));
+    const overFetchCount = currentDocs.length + newDocs.length - limit || 0;
+    const totalDocs = [...currentDocs, ...newDocs].slice(0, limit);
+
+    if (noMoreResults) {
+      return {data: totalDocs, cursor: null};
+    }
+
+    if (totalDocs.length === limit) {
+      const nextSkip = currentSkip + currentLimit - overFetchCount;
+
+      return {data: totalDocs, cursor: nextSkip.toString()};
+    }
+
+    // Re-fetch twice as many docs as we need to limit number of recursions
+    const missingCount = currentLimit - newDocs.length;
+    logger.debug(`Found [${missingCount.toString()}] invalid docs. Re-fetching additional records.`);
+    const nextLimit = missingCount * 2;
+    const nextSkip = currentSkip + currentLimit;
+
+    return recursionInner(
+      nextLimit,
+      nextSkip,
+      totalDocs,
+    );
+  };
+  return recursionInner;
+};