handle node failure more better

vshn · Sep 16, 2024 · d84ea0f · d84ea0f
1 parent 441de0b
commit d84ea0f
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 1 deletion.
diff --git a/deployment/apps/http-echo/statefulset.yaml b/deployment/apps/http-echo/statefulset.yaml
@@ -14,6 +14,15 @@ spec:
         app: http-echo
     spec:
       terminationGracePeriodSeconds: 1
+      tolerations:
+        - key: "node.kubernetes.io/unreachable"
+          operator: "Exists"
+          effect: "NoExecute"
+          tolerationSeconds: 30
+        - key: "node.kubernetes.io/not-ready"
+          operator: "Exists"
+          effect: "NoExecute"
+          tolerationSeconds: 30
       containers:
         - name: echo
           image: docker.io/hashicorp/http-echo:1.0

diff --git a/podstatus/app.py b/podstatus/app.py
@@ -115,12 +115,15 @@ def watch_pods():
                 ):
                     pod = event["object"]
                     pod_name = pod.metadata.name
-                    pod_status = pod.status.phase
                     pod_index = pod.metadata.labels.get(
                         "statefulset.kubernetes.io/pod-name", "unknown"
                     )
                     pod_node = pod.spec.node_name if pod.spec.node_name else "unknown"
 
+                    pod_status = pod.status.phase
+                    if pod.metadata.deletion_timestamp is not None:
+                        pod_status = "Terminating"
+
                     yield f'data: {{"name": "{pod_name}", "status": "{pod_status}", "index": "{pod_index}", "node": "{pod_node}"}}\n\n'
             except Exception as e:
                 logging.error(f"Error watching pods: {e}")