Improvements to retry logic (#58)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cht-user-management",
-  "version": "1.0.13",
+  "version": "1.0.14",
   "main": "dist/index.js",
   "dependencies": {
     "@fastify/autoload": "^5.8.0",
@@ -15,6 +15,7 @@
     "@types/node": "^20.8.8",
     "@types/uuid": "^9.0.6",
     "axios": "^1.5.1",
+    "axios-retry": "^4.0.0",
     "csv": "^6.3.5",
     "dotenv": "^16.3.1",
     "fastify": "^4.23.2",

scripts/retry-logic-server.js

@@ -0,0 +1,49 @@
+const express = require('express');
+
+const app = express();
+
+// Middleware function to set timeout for all POST requests
+const timeoutMiddleware = (req, res, next) => {
+  const TIMEOUT_DURATION = 1000;
+
+  req.setTimeout(TIMEOUT_DURATION, () => {
+    // Handle timeout
+    res.status(408).send('Request Timeout');
+  });
+
+  next();
+};
+
+app.post('/_session', (req, res) => {
+  res.set('Set-Cookie', 'AuthSession=abc123');
+  res.status(200).send('OK');
+});
+
+app.get('/medic/_design/medic-client/_view/contacts_by_type_freetext', (req, res) => {
+  const startkey = JSON.parse(req.query.startkey);
+  console.log('contacts_by_type_freetext', startkey);
+  const DATA = [
+    // eslint-disable-next-line max-len
+    { id: 'e847f6e2-6dba-46dd-8128-5b153d0cd75f', key: ['b_sub_county', 'name:malava'], value: 'false false b_sub_county malava', doc: { _id: 'e847f6e2-6dba-46dd-8128-5b153d0cd75f', _rev: '1-cd20b7095c20172237867233b0375eda', parent: { _id: '95d9abd1-7c17-41b1-af98-595509f96631' }, type: 'contact', is_name_generated: 'false', name: 'Malava', external_id: '', contact: { _id: '1e3d8375-6ab4-4409-be3f-3324db7658e9' }, contact_type: 'b_sub_county', reported_date: 1702573623984 } },
+    // eslint-disable-next-line max-len
+    { id: '2926bf4c-63eb-433d-a2b4-274fd05d2f1c', key: ['c_community_health_unit', 'name:chu'], value: 'false false c_community_health_unit chu', doc: { _id: '2926bf4c-63eb-433d-a2b4-274fd05d2f1c', _rev: '1-c15f26fe064f8357c19d1124286bf4c4', name: 'Chu', PARENT: 'Chepalungu', code: '123456', type: 'contact', contact_type: 'c_community_health_unit', parent: { _id: 'e847f6e2-6dba-46dd-8128-5b153d0cd75f', parent: { _id: '95d9abd1-7c17-41b1-af98-595509f96631' } }, contact: { _id: 'bb9ebc4c6af161ee0f53b42339001fb1' }, reported_date: 1701631255451 } }, 
+  ];
+  res.json({
+    total_rows: 2,
+    offset: 0,
+    rows: DATA.filter(r => r.key[0] === startkey[0])
+  });
+});
+
+app.use(timeoutMiddleware);
+
+app.all('*', (req, res) => {
+  setTimeout(() => {
+    res.status(200).send('OK');
+  }, 2000);
+});
+
+// Start the server
+app.listen(3556, () => {
+  console.log(`Server is listening on port ${3556}`);
+});

src/lib/cht-api.ts

@@ -1,8 +1,12 @@
 import _ from 'lodash';
 import axios, { AxiosHeaders } from 'axios';
+import axiosRetry from 'axios-retry';
+import { axiosRetryConfig } from './retry-logic';
 import { UserPayload } from '../services/user-payload';
 import { AuthenticationInfo, Config, ContactType } from '../config';
 
+axiosRetry(axios, axiosRetryConfig);
+
 const {
   NODE_ENV
 } = process.env;
@@ -164,7 +168,11 @@ export class ChtApi {
   createUser = async (user: UserPayload): Promise<void> => {
     const url = `${this.protocolAndHost}/api/v1/users`;
     console.log('axios.post', url);
-    await axios.post(url, user, this.authorizationOptions());
+    const axiosRequestionConfig = {
+      ...this.authorizationOptions(),
+      'axios-retry': { retries: 0 }, // upload-manager handles retries for this
+    };
+    await axios.post(url, user, axiosRequestionConfig);
   };
 
   getParentAndSibling = async (parentId: string, contactType: ContactType): Promise<{ parent: any; sibling: any }> => {

src/lib/retry-logic.ts

@@ -0,0 +1,72 @@
+import { AxiosError, AxiosRequestConfig } from 'axios';
+import isRetryAllowed from 'is-retry-allowed';
+import { UserPayload } from '../services/user-payload';
+import { ChtApi } from './cht-api';
+
+const RETRY_COUNT = 4;
+const RETRYABLE_STATUS_CODES = [500, 502, 503, 504, 511];
+
+export const axiosRetryConfig = {
+  retries: RETRY_COUNT,
+  retryDelay: () => 1000,
+  retryCondition: (err: AxiosError) => {
+    const status = err.response?.status;
+    return (!status || RETRYABLE_STATUS_CODES.includes(status)) && isRetryAllowed(err);
+  },
+  onRetry: (retryCount: number, error: AxiosError, requestConfig: AxiosRequestConfig) => {
+    console.log(`${requestConfig.url} failure. Retrying (${retryCount})`);
+  },
+};
+
+export async function retryOnUpdateConflict<T>(funcWithGetAndPut: () => Promise<T>): Promise<T> {
+  for (let retryCount = 0; retryCount < RETRY_COUNT; retryCount++) {
+    try {
+      return await funcWithGetAndPut();
+    } catch (err : any) {
+      const statusCode = err.response?.status;
+      if (statusCode === 409) {
+        console.log(`Retrying on update-conflict (${retryCount})`);
+        continue;
+      }
+
+      throw err;
+    }
+  }
+
+  throw Error('update-conflict 409 persisted');
+}
+
+export async function createUserWithRetries(userPayload: UserPayload, chtApi: ChtApi): Promise<{ username: string; password: string }> {
+  for (let retryCount = 0; retryCount < RETRY_COUNT; ++retryCount) {
+    try {
+      await chtApi.createUser(userPayload);
+      return userPayload;
+    } catch (err: any) {
+      if (axiosRetryConfig.retryCondition(err)) {
+        continue;
+      }
+
+      if (err.response?.status !== 400) {
+        throw err;
+      }
+
+      // no idea when/why some instances yield "response.data" as JSON vs some as string
+      const errorMessage = err.response?.data?.error?.message || err.response?.data;
+      console.error('createUser retry because', errorMessage);
+      if (errorMessage.includes('already taken.')) {
+        userPayload.makeUsernameMoreComplex();
+        continue;
+      }
+
+      const RETRY_PASSWORD_STRINGS = ['The password must be at least', 'The password is too easy to guess.'];
+      if (RETRY_PASSWORD_STRINGS.find(str => errorMessage.includes(str))) {
+        userPayload.regeneratePassword();
+        continue;
+      }
+
+      throw err;
+    }
+  }
+
+  throw new Error('could not create user ' + userPayload.contact);
+}

src/services/upload-manager.ts

@@ -1,13 +1,14 @@
 import EventEmitter from 'events';
-import { ChtApi, PlacePayload } from '../lib/cht-api';
 
+import * as RetryLogic from '../lib/retry-logic';
+import { ChtApi, PlacePayload } from '../lib/cht-api';
+import { Config } from '../config';
 import Place, { PlaceUploadState } from './place';
-import { UserPayload } from './user-payload';
+import RemotePlaceCache from '../lib/remote-place-cache';
 import SessionCache, { SessionCacheUploadState } from './session-cache';
-import { UploadReplacementPlace } from './upload.replacement';
 import { UploadNewPlace } from './upload.new';
-import { Config } from '../config';
-import RemotePlaceCache from '../lib/remote-place-cache';
+import { UploadReplacementPlace } from './upload.replacement';
+import { UserPayload } from './user-payload';
 
 const UPLOAD_BATCH_SIZE = 10;
 
@@ -54,15 +55,16 @@ export class UploadManager extends EventEmitter {
         place.creationDetails.placeId = placeId;
       }
 
-      await uploader.linkContactAndPlace(place, place.creationDetails?.placeId);
+      const createdPlaceId = place.creationDetails.placeId; // closure required for typescript
+      await RetryLogic.retryOnUpdateConflict<void>(() => uploader.linkContactAndPlace(place, createdPlaceId));
 
       if (!place.creationDetails.contactId) {
         throw Error('creationDetails.contactId not set');
       }
 
       if (!place.creationDetails.username) {
         const userPayload = new UserPayload(place, place.creationDetails.placeId, place.creationDetails.contactId);
-        const { username, password } = await tryCreateUser(userPayload, chtApi);
+        const { username, password } = await RetryLogic.createUserWithRetries(userPayload, chtApi);
         place.creationDetails.username = username;
         place.creationDetails.password = password;
       }
@@ -73,7 +75,7 @@ export class UploadManager extends EventEmitter {
       console.log(`successfully created ${JSON.stringify(place.creationDetails)}`);
       this.eventedPlaceStateChange(place, PlaceUploadState.SUCCESS);
     } catch (err: any) {
-      const errorDetails = err.response?.data.error || err.toString();
+      const errorDetails = err.response?.data?.error ? JSON.stringify(err.response.data.error) : err.toString();
       console.log('error when creating user', errorDetails);
       place.uploadError = errorDetails;
       this.eventedPlaceStateChange(place, PlaceUploadState.FAILURE);
@@ -100,32 +102,3 @@ export class UploadManager extends EventEmitter {
     this.emit('places_state_change', state);
   };
 }
-
-async function tryCreateUser (userPayload: UserPayload, chtApi: ChtApi): Promise<{ username: string; password: string }> {
-  for (let retryCount = 0; retryCount < 5; ++retryCount) {
-    try {
-      await chtApi.createUser(userPayload);
-      return userPayload;
-    } catch (err: any) {      
-      if (err?.response?.status !== 400) {
-        throw err;
-      }
-
-      const msg = err.response?.data?.error?.message || err.response?.data;
-      console.error('createUser retry because', msg);
-      if (msg?.includes('already taken')) {
-        userPayload.makeUsernameMoreComplex();
-        continue;
-      }
-
-      if (msg?.includes('password')) { // password too easy to guess
-        userPayload.regeneratePassword();
-        continue;
-      }
-
-      throw err;
-    }
-  }
-
-  throw new Error('could not create user ' + userPayload.contact);
-}

src/services/upload.replacement.ts

@@ -1,5 +1,6 @@
 import { ChtApi, PlacePayload } from '../lib/cht-api';
 import Place from './place';
+import { retryOnUpdateConflict } from '../lib/retry-logic';
 import { Uploader } from './upload-manager';
 
 export class UploadReplacementPlace implements Uploader {
@@ -21,7 +22,7 @@ export class UploadReplacementPlace implements Uploader {
       throw Error('contactId and placeId are required');
     }
 
-    const updatedPlaceId = await this.chtApi.updatePlace(payload, contactId);
+    const updatedPlaceId = await retryOnUpdateConflict<string>(() => this.chtApi.updatePlace(payload, contactId));
     const disabledUsers = await this.chtApi.disableUsersWithPlace(placeId);
     place.creationDetails.disabledUsers = disabledUsers;