feat: improve osmosis probes

shapeshift · Jul 27, 2023 · e9d2be2 · e9d2be2
1 parent 077d9c3
commit e9d2be2
Show file tree

Hide file tree

Showing 5 changed files with 153 additions and 66 deletions.
diff --git a/go/coinstacks/osmosis/daemon/liveness.sh b/go/coinstacks/osmosis/daemon/liveness.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+FILE=/root/.osmosisd/.latest_block_height
+
+STATUS=$(curl -sf http://localhost:26657/status) || exit 1
+
+LATEST_BLOCK_HEIGHT=$(echo $STATUS | jq -r '.result.sync_info.latest_block_height')
+
+if [[ ! -f "$FILE" ]]; then
+  echo $LATEST_BLOCK_HEIGHT > $FILE
+  exit 1
+fi
+
+PREV_LATEST_BLOCK_HEIGHT=$(cat $FILE)
+echo $LATEST_BLOCK_HEIGHT > $FILE
+
+if [[ $LATEST_BLOCK_HEIGHT -gt $PREV_LATEST_BLOCK_HEIGHT ]]; then
+  exit 0
+fi
+
+echo "node is stalled... (PREV_LATEST_BLOCK_HEIGHT: $PREV_LATEST_BLOCK_HEIGHT, LATEST_BLOCK_HEIGHT: $LATEST_BLOCK_HEIGHT)"
+
+exit 1
diff --git a/go/coinstacks/osmosis/daemon/readiness.sh b/go/coinstacks/osmosis/daemon/readiness.sh
@@ -1,9 +1,60 @@
 #!/bin/bash
 
-SYNCING=$(curl -sf http://localhost:1317/syncing | jq -r .syncing)
+BLOCK_HEIGHT_BUFFER=5
 
-if [[ $SYNCING == "false" ]]; then
-  echo "node is synced"
+SYNCING=$(curl -sf http://localhost:1317/syncing) || exit 1
+NET_INFO=$(curl -sf http://localhost:26657/net_info) || exit 1
+STATUS=$(curl -sf http://localhost:26657/status) || exit 1
+
+IS_SYNCING=$(echo $SYNCING | jq -r .syncing)
+CATCHING_UP=$(echo $STATUS | jq -r '.result.sync_info.catching_up')
+NUM_PEERS=$(echo $NET_INFO | jq -r '.result.n_peers')
+
+get_best_block_height() {
+  local best_block_height=0
+
+  for reference_url in "$@"; do
+    local status=$(curl -sf $reference_url)
+
+    if [[ $status != "" ]]; then
+      local latest_block_height=$(echo $status | jq -r '.result.sync_info.latest_block_height')
+
+      if (( latest_block_height > best_block_height )); then
+        best_block_height=$latest_block_height
+      fi
+    fi
+  done
+
+  echo $best_block_height
+}
+
+reference_validation() {
+  BEST_BLOCK_HEIGHT=$(get_best_block_height https://rpc.osmosis.zone/status https://rpc-osmosis.keplr.app/status)
+  LATEST_BLOCK_HEIGHT=$(echo $STATUS | jq -r '.result.sync_info.latest_block_height')
+
+  if (( BEST_BLOCK_HEIGHT > 0 )); then
+    BEST_BLOCK_HEIGHT_WITH_BUFFER=$(( BEST_BLOCK_HEIGHT - BLOCK_HEIGHT_BUFFER ))
+
+    if (( LATEST_BLOCK_HEIGHT >= BEST_BLOCK_HEIGHT_WITH_BUFFER )); then
+      echo "node is synced with $NUM_PEERS and within safe buffer of reference node"
+      exit 0
+    fi
+
+    echo "node is synced with $NUM_PEERS peers, but not within safe buffer of reference node"
+    exit 1
+  fi 
+}
+
+if [[ $IS_SYNCING == false && $CATCHING_UP == false ]]; then
+  if (( $NUM_PEERS > 0 )); then
+    # if node is reporting synced, double check against reference nodes
+    reference_validation
+
+    echo "node is synced with $NUM_PEERS peers"
+    exit 0
+  fi
+
+  echo "node is synced, but has no peers"
   exit 0
 fi
 

diff --git a/go/coinstacks/osmosis/pulumi/index.ts b/go/coinstacks/osmosis/pulumi/index.ts
@@ -19,6 +19,21 @@ export = async (): Promise<Outputs> => {
             'daemon-api': { port: 1317, pathPrefix: '/lcd', stripPathPrefix: true },
             'daemon-rpc': { port: 26657, pathPrefix: '/rpc', stripPathPrefix: true },
           },
+          startupProbe: {
+            httpGet: { path: '/status', port: 26657 },
+            periodSeconds: 30,
+            failureThreshold: 60,
+            timeoutSeconds: 10,
+          },
+          livenessProbe: {
+            periodSeconds: 30,
+            timeoutSeconds: 10,
+          },
+          readinessProbe: {
+            periodSeconds: 30,
+            timeoutSeconds: 10,
+            failureThreshold: 10,
+          },
         }
       default:
         throw new Error(`no support for coin service: ${service.name}`)

diff --git a/pulumi/src/index.ts b/pulumi/src/index.ts
@@ -72,8 +72,9 @@ export interface CoinServiceArgs extends ServiceConfig {
   dataDir?: string
   configMapData?: Record<string, string>
   volumeMounts?: Array<k8s.types.input.core.v1.VolumeMount>
-  readinessProbe?: k8s.types.input.core.v1.Probe
+  startupProbe?: k8s.types.input.core.v1.Probe
   livenessProbe?: k8s.types.input.core.v1.Probe
+  readinessProbe?: k8s.types.input.core.v1.Probe
 }
 
 export interface Service {

diff --git a/pulumi/src/statefulService.ts b/pulumi/src/statefulService.ts
@@ -17,15 +17,23 @@ export function createCoinService(args: CoinServiceArgs, assetName: string): Ser
     }
   })()
 
-  const livenessScript = (() => {
+  const startupProbe = (() => {
+    try {
+      return readFileSync(`../${args.name}/startup.sh`).toString()
+    } catch (err) {
+      return ''
+    }
+  })()
+
+  const livenessProbe = (() => {
     try {
       return readFileSync(`../${args.name}/liveness.sh`).toString()
     } catch (err) {
       return ''
     }
   })()
 
-  const readinessScript = (() => {
+  const readinessProbe = (() => {
     try {
       return readFileSync(`../${args.name}/readiness.sh`).toString()
     } catch (err) {
@@ -35,14 +43,13 @@ export function createCoinService(args: CoinServiceArgs, assetName: string): Ser
 
   const configMapData = {
     ...(Boolean(initScript) && { [`${args.name}-init.sh`]: initScript }),
-    ...(Boolean(livenessScript) && { [`${args.name}-liveness.sh`]: livenessScript }),
-    ...(Boolean(readinessScript) && { [`${args.name}-readiness.sh`]: readinessScript }),
+    ...(Boolean(startupProbe) && { [`${args.name}-startup.sh`]: startupProbe }),
+    ...(Boolean(livenessProbe) && { [`${args.name}-liveness.sh`]: livenessProbe }),
+    ...(Boolean(readinessProbe) && { [`${args.name}-readiness.sh`]: readinessProbe }),
     ...(args.configMapData ?? {}),
   }
 
-  const containers: Array<k8s.types.input.core.v1.Container> = []
-
-  const serviceContainer: k8s.types.input.core.v1.Container = {
+  const container: k8s.types.input.core.v1.Container = {
     name,
     image: args.image,
     command: initScript && !args.command ? ['/init.sh'] : args.command,
@@ -58,15 +65,28 @@ export function createCoinService(args: CoinServiceArgs, assetName: string): Ser
         ...(args.memoryRequest && { memory: args.memoryRequest }),
       },
     },
-    ...(!readinessScript &&
+    ports: ports.map(({ port: containerPort, name }) => ({ containerPort, name })),
+    securityContext: { runAsUser: 0 },
+    ...((startupProbe || args.startupProbe) && {
+      startupProbe: {
+        ...(startupProbe && { exec: { command: ['/startup.sh'] }, initialDelaySeconds: 30 }),
+        ...args.startupProbe,
+      },
+    }),
+    ...((livenessProbe || args.livenessProbe) &&
+      args.livenessProbe && {
+        livenessProbe: {
+          ...(livenessProbe && { exec: { command: ['/liveness.sh'] }, initialDelaySeconds: 30 }),
+          ...args.livenessProbe,
+        },
+      }),
+    ...((readinessProbe || args.readinessProbe) &&
       args.readinessProbe && {
         readinessProbe: {
-          initialDelaySeconds: 30,
+          ...(readinessProbe && { exec: { command: ['/readiness.sh'] }, initialDelaySeconds: 30 }),
           ...args.readinessProbe,
         },
       }),
-    ports: ports.map(({ port: containerPort, name }) => ({ containerPort, name })),
-    securityContext: { runAsUser: 0 },
     volumeMounts: [
       {
         name: `data-${args.name}`,
@@ -81,60 +101,37 @@ export function createCoinService(args: CoinServiceArgs, assetName: string): Ser
             },
           ]
         : []),
+      ...(startupProbe
+        ? [
+            {
+              name: 'config-map',
+              mountPath: '/startup.sh',
+              subPath: `${args.name}-startup.sh`,
+            },
+          ]
+        : []),
+      ...(livenessProbe
+        ? [
+            {
+              name: 'config-map',
+              mountPath: '/liveness.sh',
+              subPath: `${args.name}-liveness.sh`,
+            },
+          ]
+        : []),
+      ...(readinessProbe
+        ? [
+            {
+              name: 'config-map',
+              mountPath: '/readiness.sh',
+              subPath: `${args.name}-readiness.sh`,
+            },
+          ]
+        : []),
       ...(args.volumeMounts ?? []),
     ],
   }
 
-  containers.push(serviceContainer)
-
-  if (readinessScript || livenessScript) {
-    const monitorContainer: k8s.types.input.core.v1.Container = {
-      name: `${name}-monitor`,
-      image: 'shapeshiftdao/unchained-probe:1.0.0',
-      env,
-      ...(readinessScript && {
-        readinessProbe: {
-          exec: {
-            command: ['/readiness.sh'],
-          },
-          initialDelaySeconds: 30,
-          ...args.readinessProbe,
-        },
-      }),
-      ...(livenessScript && {
-        livenessProbe: {
-          exec: {
-            command: ['/liveness.sh'],
-          },
-          initialDelaySeconds: 30,
-          ...args.livenessProbe,
-        },
-      }),
-      volumeMounts: [
-        ...(readinessScript
-          ? [
-              {
-                name: 'config-map',
-                mountPath: '/readiness.sh',
-                subPath: `${args.name}-readiness.sh`,
-              },
-            ]
-          : []),
-        ...(livenessScript
-          ? [
-              {
-                name: 'config-map',
-                mountPath: '/liveness.sh',
-                subPath: `${args.name}-liveness.sh`,
-              },
-            ]
-          : []),
-      ],
-    }
-
-    containers.push(monitorContainer)
-  }
-
   const volumeClaimTemplates = [
     {
       metadata: {
@@ -155,7 +152,7 @@ export function createCoinService(args: CoinServiceArgs, assetName: string): Ser
   return {
     name: args.name,
     configMapData,
-    containers,
+    containers: [container],
     ports,
     volumeClaimTemplates,
   }