From 21a407c0b6e4e93e785a9423bb06657fc6e7f462 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Mon, 22 Jan 2024 15:24:20 -0800 Subject: [PATCH 1/7] kill-switch --- src/dlob-subscriber/DLOBSubscriberIO.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index 41491e7..88e7bb3 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -15,6 +15,8 @@ import { l2WithBNToStrings, } from '../utils/utils'; +const SLOT_DIFF_KILLSWITCH_THRESHOLD = 50; + type wsMarketL2Args = { marketIndex: number; marketType: MarketType; @@ -139,6 +141,20 @@ export class DLOBSubscriberIO extends DLOBSubscriber { l2Args.marketIndex ); + if ( + Math.abs(slot - l2Formatted['marketSlot']) > + SLOT_DIFF_KILLSWITCH_THRESHOLD || + Math.abs(slot - l2Formatted['oracleData']['slot']) > + SLOT_DIFF_KILLSWITCH_THRESHOLD + ) { + console.log(`Killing process due to slot diffs: + dlobProvider slot: ${slot} + oracle slot: ${l2Formatted['oracleData']['slot']} + market slot: ${l2Formatted['marketSlot']} + `); + process.exit(1); + } + const l2Formatted_depth100 = Object.assign({}, l2Formatted, { bids: l2Formatted.bids.slice(0, 100), asks: l2Formatted.asks.slice(0, 100), From 06ab7ef6724192c22731b0c349f3dbcc2ca73285 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Mon, 22 Jan 2024 15:46:32 -0800 Subject: [PATCH 2/7] parseInt --- src/dlob-subscriber/DLOBSubscriberIO.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index 88e7bb3..4cf1d96 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -144,7 +144,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { if ( Math.abs(slot - l2Formatted['marketSlot']) > SLOT_DIFF_KILLSWITCH_THRESHOLD || - Math.abs(slot - l2Formatted['oracleData']['slot']) > + Math.abs(slot - parseInt(l2Formatted['oracleData']['slot'])) > SLOT_DIFF_KILLSWITCH_THRESHOLD ) { console.log(`Killing process due to slot diffs: From 5fd1f7172dc3dbfb6757f3b1008405350c10b4d8 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Mon, 22 Jan 2024 18:03:28 -0800 Subject: [PATCH 3/7] remove market slot kill switch --- src/dlob-subscriber/DLOBSubscriberIO.ts | 11 ++++++----- src/publishers/dlobPublisher.ts | 4 ++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index 4cf1d96..9d7fa0c 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -15,7 +15,7 @@ import { l2WithBNToStrings, } from '../utils/utils'; -const SLOT_DIFF_KILLSWITCH_THRESHOLD = 50; +const SLOT_DIFF_KILLSWITCH_THRESHOLD = 100; type wsMarketL2Args = { marketIndex: number; @@ -38,6 +38,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { public marketL2Args: wsMarketL2Args[] = []; public lastSeenL2Formatted: Map>; redisClient: RedisClient; + public marketKillSwitchSlotDiffThreshold: number; constructor( config: DLOBSubscriptionConfig & { @@ -45,10 +46,13 @@ export class DLOBSubscriberIO extends DLOBSubscriber { perpMarketInfos: wsMarketInfo[]; spotMarketInfos: wsMarketInfo[]; spotMarketSubscribers: SubscriberLookup; + marketKillSwitchSlotDiffThreshold?: number; } ) { super(config); this.redisClient = config.redisClient; + this.marketKillSwitchSlotDiffThreshold = + config.marketKillSwitchSlotDiffThreshold || 200; // Set up appropriate maps this.lastSeenL2Formatted = new Map(); @@ -142,15 +146,12 @@ export class DLOBSubscriberIO extends DLOBSubscriber { ); if ( - Math.abs(slot - l2Formatted['marketSlot']) > - SLOT_DIFF_KILLSWITCH_THRESHOLD || Math.abs(slot - parseInt(l2Formatted['oracleData']['slot'])) > - SLOT_DIFF_KILLSWITCH_THRESHOLD + SLOT_DIFF_KILLSWITCH_THRESHOLD ) { console.log(`Killing process due to slot diffs: dlobProvider slot: ${slot} oracle slot: ${l2Formatted['oracleData']['slot']} - market slot: ${l2Formatted['marketSlot']} `); process.exit(1); } diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index 10d9504..caa45ef 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -71,6 +71,9 @@ const ORDERBOOK_UPDATE_INTERVAL = parseInt(process.env.ORDERBOOK_UPDATE_INTERVAL) || 1000; const WS_FALLBACK_FETCH_INTERVAL = 10_000; +const MARKET_KILLSWITCH_SLOT_DIFF_THRESHOLD = + parseInt(process.env.MARKET_KILLSWITCH_SLOT_DIFF_THRESHOLD) || 300; + // comma separated list of perp market indexes to load: i.e. 0,1,2,3 const PERP_MARKETS_TO_LOAD = process.env.PERP_MARKETS_TO_LOAD !== undefined @@ -395,6 +398,7 @@ const main = async () => { spotMarketSubscribers: MARKET_SUBSCRIBERS, perpMarketInfos, spotMarketInfos, + marketKillSwitchSlotDiffThreshold: MARKET_KILLSWITCH_SLOT_DIFF_THRESHOLD, }); await dlobSubscriber.subscribe(); if (useWebsocket && !FEATURE_FLAGS.DISABLE_GPA_REFRESH) { From 8f8635ab312d9281f6362e983894d07beaeb1926 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Mon, 22 Jan 2024 18:04:30 -0800 Subject: [PATCH 4/7] change slot diff killswitch threshold --- src/dlob-subscriber/DLOBSubscriberIO.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index 9d7fa0c..c4039cb 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -15,7 +15,7 @@ import { l2WithBNToStrings, } from '../utils/utils'; -const SLOT_DIFF_KILLSWITCH_THRESHOLD = 100; +const SLOT_DIFF_KILLSWITCH_THRESHOLD = 200; type wsMarketL2Args = { marketIndex: number; From e4ed4814223ed3c9b9c2903a40c6fc58c0fee855 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Tue, 23 Jan 2024 13:25:57 -0800 Subject: [PATCH 5/7] consolidate conditions for killswitch --- src/dlob-subscriber/DLOBSubscriberIO.ts | 14 +++++++------- src/publishers/dlobPublisher.ts | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index c4039cb..6fad2a0 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -15,8 +15,6 @@ import { l2WithBNToStrings, } from '../utils/utils'; -const SLOT_DIFF_KILLSWITCH_THRESHOLD = 200; - type wsMarketL2Args = { marketIndex: number; marketType: MarketType; @@ -38,7 +36,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { public marketL2Args: wsMarketL2Args[] = []; public lastSeenL2Formatted: Map>; redisClient: RedisClient; - public marketKillSwitchSlotDiffThreshold: number; + public killSwitchSlotDiffThreshold: number; constructor( config: DLOBSubscriptionConfig & { @@ -46,13 +44,13 @@ export class DLOBSubscriberIO extends DLOBSubscriber { perpMarketInfos: wsMarketInfo[]; spotMarketInfos: wsMarketInfo[]; spotMarketSubscribers: SubscriberLookup; - marketKillSwitchSlotDiffThreshold?: number; + killSwitchSlotDiffThreshold?: number; } ) { super(config); this.redisClient = config.redisClient; - this.marketKillSwitchSlotDiffThreshold = - config.marketKillSwitchSlotDiffThreshold || 200; + this.killSwitchSlotDiffThreshold = + config.killSwitchSlotDiffThreshold || 200; // Set up appropriate maps this.lastSeenL2Formatted = new Map(); @@ -147,7 +145,9 @@ export class DLOBSubscriberIO extends DLOBSubscriber { if ( Math.abs(slot - parseInt(l2Formatted['oracleData']['slot'])) > - SLOT_DIFF_KILLSWITCH_THRESHOLD + this.killSwitchSlotDiffThreshold || + Math.abs(slot - l2Formatted['marketSlot']) > + this.killSwitchSlotDiffThreshold ) { console.log(`Killing process due to slot diffs: dlobProvider slot: ${slot} diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index caa45ef..4181a9b 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -71,8 +71,8 @@ const ORDERBOOK_UPDATE_INTERVAL = parseInt(process.env.ORDERBOOK_UPDATE_INTERVAL) || 1000; const WS_FALLBACK_FETCH_INTERVAL = 10_000; -const MARKET_KILLSWITCH_SLOT_DIFF_THRESHOLD = - parseInt(process.env.MARKET_KILLSWITCH_SLOT_DIFF_THRESHOLD) || 300; +const KILLSWITCH_SLOT_DIFF_THRESHOLD = + parseInt(process.env.KILLSWITCH_SLOT_DIFF_THRESHOLD) || 200; // comma separated list of perp market indexes to load: i.e. 0,1,2,3 const PERP_MARKETS_TO_LOAD = @@ -398,7 +398,7 @@ const main = async () => { spotMarketSubscribers: MARKET_SUBSCRIBERS, perpMarketInfos, spotMarketInfos, - marketKillSwitchSlotDiffThreshold: MARKET_KILLSWITCH_SLOT_DIFF_THRESHOLD, + killSwitchSlotDiffThreshold: KILLSWITCH_SLOT_DIFF_THRESHOLD, }); await dlobSubscriber.subscribe(); if (useWebsocket && !FEATURE_FLAGS.DISABLE_GPA_REFRESH) { From 1726f9ea5a8f301ce9382a17ba2cae8e5054f716 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Tue, 23 Jan 2024 13:35:49 -0800 Subject: [PATCH 6/7] logging change --- src/dlob-subscriber/DLOBSubscriberIO.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index 6fad2a0..87aa4a9 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -149,9 +149,10 @@ export class DLOBSubscriberIO extends DLOBSubscriber { Math.abs(slot - l2Formatted['marketSlot']) > this.killSwitchSlotDiffThreshold ) { - console.log(`Killing process due to slot diffs: + console.log(`Killing process due to slot diffs for market ${marketName}: dlobProvider slot: ${slot} oracle slot: ${l2Formatted['oracleData']['slot']} + market slot: ${l2Formatted['marketSlot']} `); process.exit(1); } From 6eb95323fe10932c796afdb87e4cb0edb184d70f Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Tue, 23 Jan 2024 14:04:46 -0800 Subject: [PATCH 7/7] remove market check --- src/dlob-subscriber/DLOBSubscriberIO.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index 87aa4a9..97b063f 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -145,9 +145,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { if ( Math.abs(slot - parseInt(l2Formatted['oracleData']['slot'])) > - this.killSwitchSlotDiffThreshold || - Math.abs(slot - l2Formatted['marketSlot']) > - this.killSwitchSlotDiffThreshold + this.killSwitchSlotDiffThreshold ) { console.log(`Killing process due to slot diffs for market ${marketName}: dlobProvider slot: ${slot}