Skip to content

Commit

Permalink
feat(calling): update failover logic for cc (webex#3805)
Browse files Browse the repository at this point in the history
  • Loading branch information
@rarajes2
rarajes2 authored Sep 18, 2024
1 parent 11f97f5 commit c04a29f
Show file tree
Hide file tree
Showing 3 changed files with 118 additions and 9 deletions.
1 change: 1 addition & 0 deletions packages/calling/src/CallingClient/constants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ export const SEC_TO_MSEC_MFACTOR = 1000;
export const MINUTES_TO_SEC_MFACTOR = 60;
export const REG_RANDOM_T_FACTOR_UPPER_LIMIT = 10000;
export const REG_TRY_BACKUP_TIMER_VAL_IN_SEC = 1200;
export const REG_TRY_BACKUP_TIMER_VAL_FOR_CC_IN_SEC = 114;
export const REG_FAILBACK_429_MAX_RETRIES = 5;
export const REGISTER_UTIL = 'registerDevice';
export const GET_MOBIUS_SERVERS_UTIL = 'getMobiusServers';
Expand Down
107 changes: 102 additions & 5 deletions packages/calling/src/CallingClient/registration/register.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import {
KEEPALIVE_UTIL,
MINUTES_TO_SEC_MFACTOR,
REGISTRATION_FILE,
REG_TRY_BACKUP_TIMER_VAL_FOR_CC_IN_SEC,
REG_TRY_BACKUP_TIMER_VAL_IN_SEC,
SEC_TO_MSEC_MFACTOR,
} from '../constants';
Expand Down Expand Up @@ -64,6 +65,17 @@ describe('Registration Tests', () => {
},
};

const ccMockResponse = {
...mockResponse,
body: {
...mockResponse.body,
serviceData: {
domain: '',
indicator: 'contactcenter',
},
},
};

const failurePayload = <WebexRequestPayload>(<unknown>{
statusCode: 500,
body: mockPostResponse,
Expand All @@ -85,15 +97,19 @@ describe('Registration Tests', () => {
let restoreSpy;
let postRegistrationSpy;

beforeEach(() => {
const setupRegistration = (mockServiceData) => {
const mutex = new Mutex();
reg = createRegistration(webex, MockServiceData, mutex, lineEmitter, LOGGER.INFO);
reg = createRegistration(webex, mockServiceData, mutex, lineEmitter, LOGGER.INFO);
reg.setMobiusServers(mobiusUris.primary, mobiusUris.backup);
jest.clearAllMocks();
restartSpy = jest.spyOn(reg, 'restartRegistration');
failbackRetry429Spy = jest.spyOn(reg, FAILBACK_429_RETRY_UTIL);
restoreSpy = jest.spyOn(reg, 'restorePreviousRegistration');
postRegistrationSpy = jest.spyOn(reg, 'postRegistration');
};

beforeEach(() => {
setupRegistration(MockServiceData);
});

afterEach(() => {
Expand Down Expand Up @@ -218,6 +234,36 @@ describe('Registration Tests', () => {
expect(reg.getActiveMobiusUrl()).toEqual(mobiusUris.backup[0]);
});

it('cc: verify unreachable primary with reachable backup server', async () => {
setupRegistration({...MockServiceData, indicator: ServiceIndicator.CONTACT_CENTER});

jest.useFakeTimers();
webex.request
.mockRejectedValueOnce(failurePayload)
.mockRejectedValueOnce(failurePayload)
.mockResolvedValueOnce(successPayload);

expect(reg.getStatus()).toEqual(RegistrationStatus.IDLE);
await reg.triggerRegistration();
jest.advanceTimersByTime(REG_TRY_BACKUP_TIMER_VAL_FOR_CC_IN_SEC * SEC_TO_MSEC_MFACTOR);
await flushPromises();

expect(webex.request).toBeCalledTimes(3);
expect(webex.request).toBeCalledWith({
...ccMockResponse,
method: 'POST',
uri: `${mobiusUris.primary[0]}device`,
});
expect(webex.request).toBeCalledWith({
...ccMockResponse,
method: 'POST',
uri: `${mobiusUris.backup[0]}device`,
});
expect(reg.getStatus()).toEqual(RegistrationStatus.ACTIVE);
/* Active Url must match with the backup url as per the test */
expect(reg.getActiveMobiusUrl()).toEqual(mobiusUris.backup[0]);
});

it('verify unreachable primary and backup servers', async () => {
jest.useFakeTimers();
// try the primary twice and register successfully with backup servers
Expand Down Expand Up @@ -444,15 +490,14 @@ describe('Registration Tests', () => {
file: REGISTRATION_FILE,
method: 'startKeepaliveTimer',
};

const mockKeepAliveBody = {device: mockPostResponse.device};

beforeEach(async () => {
const beforeEachSetupForKeepalive = async () => {
postRegistrationSpy.mockResolvedValueOnce(successPayload);
jest.useFakeTimers();
await reg.triggerRegistration();
expect(reg.getStatus()).toBe(RegistrationStatus.ACTIVE);
});
};

afterEach(() => {
jest.clearAllTimers();
Expand All @@ -471,6 +516,7 @@ describe('Registration Tests', () => {
});

it('verify successful keep-alive cases', async () => {
await beforeEachSetupForKeepalive();
const keepAlivePayload = <WebexRequestPayload>(<unknown>{
statusCode: 200,
body: mockKeepAliveBody,
Expand All @@ -487,6 +533,7 @@ describe('Registration Tests', () => {
});

it('verify failure keep-alive cases: Retry Success', async () => {
await beforeEachSetupForKeepalive();
const failurePayload = <WebexRequestPayload>(<unknown>{
statusCode: 503,
body: mockKeepAliveBody,
Expand Down Expand Up @@ -517,6 +564,7 @@ describe('Registration Tests', () => {
});

it('verify failure keep-alive cases: Restore failure', async () => {
await beforeEachSetupForKeepalive();
const restoreSpy = jest.spyOn(reg, 'restorePreviousRegistration');
const restartRegSpy = jest.spyOn(reg, 'restartRegistration');
const reconnectSpy = jest.spyOn(reg, 'reconnectOnFailure');
Expand Down Expand Up @@ -565,6 +613,7 @@ describe('Registration Tests', () => {
});

it('verify failure keep-alive cases: Restore Success', async () => {
await beforeEachSetupForKeepalive();
const restoreSpy = jest.spyOn(reg, 'restorePreviousRegistration');
const restartRegSpy = jest.spyOn(reg, 'restartRegistration');
const reconnectSpy = jest.spyOn(reg, 'reconnectOnFailure');
Expand Down Expand Up @@ -616,6 +665,7 @@ describe('Registration Tests', () => {
});

it('verify failure followed by recovery of keepalive', async () => {
await beforeEachSetupForKeepalive();
const failurePayload = <WebexRequestPayload>(<unknown>{
statusCode: 503,
body: mockKeepAliveBody,
Expand Down Expand Up @@ -647,7 +697,53 @@ describe('Registration Tests', () => {
expect(reg.keepaliveTimer).toBe(timer);
});

it('cc: verify failover to backup server after 4 keep alive failure with primary server', async () => {
// Register with contact center service
setupRegistration({...MockServiceData, indicator: ServiceIndicator.CONTACT_CENTER});
await beforeEachSetupForKeepalive();

const failurePayload = <WebexRequestPayload>(<unknown>{
statusCode: 503,
body: mockKeepAliveBody,
});
const successPayload = <WebexRequestPayload>(<unknown>{
statusCode: 200,
body: mockKeepAliveBody,
});

const clearIntervalSpy = jest.spyOn(global, 'clearInterval');

jest
.spyOn(reg, 'postKeepAlive')
.mockRejectedValueOnce(failurePayload)
.mockRejectedValueOnce(failurePayload)
.mockRejectedValueOnce(failurePayload)
.mockRejectedValueOnce(failurePayload)
.mockResolvedValue(successPayload);

expect(reg.getStatus()).toBe(RegistrationStatus.ACTIVE);

const timer = reg.keepaliveTimer;

jest.advanceTimersByTime(5 * mockPostResponse.keepaliveInterval * SEC_TO_MSEC_MFACTOR);
await flushPromises();

expect(clearIntervalSpy).toBeCalledOnceWith(timer);
expect(reg.getStatus()).toBe(RegistrationStatus.INACTIVE);
expect(reg.keepaliveTimer).not.toBe(timer);

webex.request.mockRejectedValueOnce(failurePayload).mockResolvedValue(successPayload);

jest.advanceTimersByTime(REG_TRY_BACKUP_TIMER_VAL_FOR_CC_IN_SEC * SEC_TO_MSEC_MFACTOR);
await flushPromises();

/* Active Url must match with the backup url as per the test */
expect(reg.getActiveMobiusUrl()).toEqual(mobiusUris.backup[0]);
expect(reg.getStatus()).toBe(RegistrationStatus.ACTIVE);
});

it('verify final error for keep-alive', async () => {
await beforeEachSetupForKeepalive();
const restoreSpy = jest.spyOn(reg, 'restorePreviousRegistration');
const restartRegSpy = jest.spyOn(reg, 'restartRegistration');
const reconnectSpy = jest.spyOn(reg, 'reconnectOnFailure');
Expand Down Expand Up @@ -686,6 +782,7 @@ describe('Registration Tests', () => {
});

it('verify failure keep-alive case with active call present: Restore Success after call ends', async () => {
await beforeEachSetupForKeepalive();
const restoreSpy = jest.spyOn(reg, 'restorePreviousRegistration');
const restartRegSpy = jest.spyOn(reg, 'restartRegistration');
const reconnectSpy = jest.spyOn(reg, 'reconnectOnFailure');
Expand Down
19 changes: 15 additions & 4 deletions packages/calling/src/CallingClient/registration/register.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import {
IDeviceInfo,
RegistrationStatus,
ServiceData,
ServiceIndicator,
WebexRequestPayload,
} from '../../common/types';
import {ISDKConnector, WebexSDK} from '../../SDKConnector/types';
Expand All @@ -39,6 +40,7 @@ import {
DEFAULT_REHOMING_INTERVAL_MIN,
DEFAULT_REHOMING_INTERVAL_MAX,
DEFAULT_KEEPALIVE_INTERVAL,
REG_TRY_BACKUP_TIMER_VAL_FOR_CC_IN_SEC,
} from '../constants';
import {LINE_EVENTS, LineEmitterCallback} from '../line/types';
import {LineError} from '../../Errors/catalog/LineError';
Expand Down Expand Up @@ -73,6 +75,7 @@ export class Registration implements IRegistration {
private backupMobiusUris: string[];
private registerRetry = false;
private reconnectPending = false;
private isCCFlow = false;

/**
*/
Expand All @@ -85,6 +88,8 @@ export class Registration implements IRegistration {
) {
this.sdkConnector = SDKConnector;
this.serviceData = serviceData;
this.isCCFlow = serviceData.indicator === ServiceIndicator.CONTACT_CENTER;

if (!this.sdkConnector.getWebex()) {
SDKConnector.setWebex(webex);
}
Expand Down Expand Up @@ -257,8 +262,12 @@ export class Registration implements IRegistration {

let interval = this.getRegRetryInterval(attempt);

if (timeElapsed + interval > REG_TRY_BACKUP_TIMER_VAL_IN_SEC) {
const excessVal = timeElapsed + interval - REG_TRY_BACKUP_TIMER_VAL_IN_SEC;
const TIMER_THRESHOLD = this.isCCFlow
? REG_TRY_BACKUP_TIMER_VAL_FOR_CC_IN_SEC
: REG_TRY_BACKUP_TIMER_VAL_IN_SEC;

if (timeElapsed + interval > TIMER_THRESHOLD) {
const excessVal = timeElapsed + interval - TIMER_THRESHOLD;

interval -= excessVal;
}
Expand Down Expand Up @@ -681,13 +690,15 @@ export class Registration implements IRegistration {
private startKeepaliveTimer(url: string, interval: number) {
let keepAliveRetryCount = 0;
this.clearKeepaliveTimer();
const RETRY_COUNT_THRESHOLD = this.isCCFlow ? 4 : 5;

this.keepaliveTimer = setInterval(async () => {
const logContext = {
file: REGISTRATION_FILE,
method: this.startKeepaliveTimer.name,
};
await this.mutex.runExclusive(async () => {
if (this.isDeviceRegistered() && keepAliveRetryCount < 5) {
if (this.isDeviceRegistered() && keepAliveRetryCount < RETRY_COUNT_THRESHOLD) {
try {
const res = await this.postKeepAlive(url);
log.info(`Sent Keepalive, status: ${res.statusCode}`, logContext);
Expand Down Expand Up @@ -720,7 +731,7 @@ export class Registration implements IRegistration {
{method: this.startKeepaliveTimer.name, file: REGISTRATION_FILE}
);

if (abort || keepAliveRetryCount >= 5) {
if (abort || keepAliveRetryCount >= RETRY_COUNT_THRESHOLD) {
this.setStatus(RegistrationStatus.INACTIVE);
this.clearKeepaliveTimer();
this.clearFailbackTimer();
Expand Down

0 comments on commit c04a29f

Please sign in to comment.