From 832ddb34a4ba974253444e85c9a34c6510241e63 Mon Sep 17 00:00:00 2001 From: j4ckstraw Date: Fri, 16 Jun 2023 19:54:25 +0800 Subject: [PATCH] check if pod is ready before pinger when node crash, kubelet will not update pod status, goldpinger on crashed node will not work, but other goldpinger instance will continue ping the bad one. this patch remove the not ready goldpinger instance from pinger list. ``` NAME READY STATUS RESTARTS AGE goldpinger-8749c476d-5n7lt 1/1 Running 0 78d goldpinger-8749c476d-bj752 0/1 Terminating 0 78d goldpinger-8749c476d-bzf6t 1/1 Running 0 78d goldpinger-8749c476d-dv28g 1/1 Running 0 4h25m goldpinger-8749c476d-gwhw4 1/1 Running 0 4h25m goldpinger-8749c476d-prgwd 1/1 Running 0 78d { "conditions": [ { "lastProbeTime": null, "lastTransitionTime": "2023-03-29T12:38:40Z", "status": "True", "type": "Initialized" }, { "lastProbeTime": null, "lastTransitionTime": "2023-06-16T02:24:10Z", "message": "containers with unready status: [goldpinger]", "reason": "ContainersNotReady", "status": "False", "type": "Ready" }, { "lastProbeTime": null, "lastTransitionTime": "2023-06-16T02:24:10Z", "message": "containers with unready status: [goldpinger]", "reason": "ContainersNotReady", "status": "False", "type": "ContainersReady" }, { "lastProbeTime": null, "lastTransitionTime": "2023-03-29T12:38:39Z", "status": "True", "type": "PodScheduled" } ], "containerStatuses": [ { "containerID": "docker://xxx", "image": "xxx/goldpinger:v3.5.0", "imageID": "docker-pullable://xxx/goldpinger@sha256:xxx", "lastState": {}, "name": "goldpinger", "ready": false, "restartCount": 0, "started": true, "state": { "running": { "startedAt": "2023-03-29T12:38:56Z" } } } ], "hostIP": "1.2.3.3", "phase": "Running", "podIP": "1.2.3.4", "podIPs": [ { "ip": "1.2.3.4" } ], "qosClass": "Burstable", "startTime": "2023-03-29T12:38:40Z" } ``` Signed-off-by: j4ckstraw --- pkg/goldpinger/k8s.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/goldpinger/k8s.go b/pkg/goldpinger/k8s.go index 7227933..d3c4801 100644 --- a/pkg/goldpinger/k8s.go +++ b/pkg/goldpinger/k8s.go @@ -21,6 +21,7 @@ import ( "go.uber.org/zap" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubectl/pkg/util/podutils" k8snet "k8s.io/utils/net" ) @@ -120,6 +121,9 @@ func GetAllPods() map[string]*GoldpingerPod { podMap := make(map[string]*GoldpingerPod) for _, pod := range pods.Items { + if !podutils.IsPodReady(pod) { + continue + } podMap[pod.Name] = &GoldpingerPod{ Name: getPodNodeName(pod), PodIP: getPodIP(pod),