fix: kube-ovn-controller cannot be ready when ENABLE_METRICS is false (…

…#4886) * fix: kube-ovn-controller cannot be ready when ENABLE_METRICS is false * ovn-monitor: refactor pprof to metrics --------- Signed-off-by: zhaocongqi <[email protected]>
kubeovn · Jan 21, 2025 · cb39c63 · cb39c63
1 parent 5abd85c
commit cb39c63
Show file tree

Hide file tree

Showing 11 changed files with 128 additions and 45 deletions.
diff --git a/charts/kube-ovn/templates/controller-deploy.yaml b/charts/kube-ovn/templates/controller-deploy.yaml
@@ -173,6 +173,7 @@ spec:
                 - /kube-ovn/kube-ovn-healthcheck
                 - --port=10660
                 - --tls={{- .Values.func.SECURE_SERVING }}
+                - --enable-metrics={{- .Values.networking.ENABLE_METRICS }}
             periodSeconds: 3
             timeoutSeconds: 5
           livenessProbe:
@@ -181,6 +182,7 @@ spec:
                 - /kube-ovn/kube-ovn-healthcheck
                 - --port=10660
                 - --tls={{- .Values.func.SECURE_SERVING }}
+                - --enable-metrics={{- .Values.networking.ENABLE_METRICS }}
             initialDelaySeconds: 300
             periodSeconds: 7
             failureThreshold: 5
@@ -208,4 +210,3 @@ spec:
           secret:
             optional: true
             secretName: kube-ovn-tls
-
diff --git a/charts/kube-ovn/templates/monitor-deploy.yaml b/charts/kube-ovn/templates/monitor-deploy.yaml
@@ -49,6 +49,7 @@ spec:
           - --logtostderr=false
           - --alsologtostderr=true
           - --log_file_max_size=200
+          - --enable-metrics={{- .Values.networking.ENABLE_METRICS }}
           securityContext:
             runAsUser: 0
             privileged: false
@@ -113,6 +114,7 @@ spec:
                 - /kube-ovn/kube-ovn-healthcheck
                 - --port=10661
                 - --tls={{- .Values.func.SECURE_SERVING }}
+                - --enable-metrics={{- .Values.networking.ENABLE_METRICS }}
             timeoutSeconds: 5
           readinessProbe:
             failureThreshold: 3
@@ -124,6 +126,7 @@ spec:
                 - /kube-ovn/kube-ovn-healthcheck
                 - --port=10661
                 - --tls={{- .Values.func.SECURE_SERVING }}
+                - --enable-metrics={{- .Values.networking.ENABLE_METRICS }}
             timeoutSeconds: 5
       nodeSelector:
         kubernetes.io/os: "linux"

diff --git a/charts/kube-ovn/templates/ovncni-ds.yaml b/charts/kube-ovn/templates/ovncni-ds.yaml
@@ -154,6 +154,7 @@ spec:
               - /kube-ovn/kube-ovn-healthcheck
               - --port=10665
               - --tls={{- .Values.func.SECURE_SERVING }}
+              - --enable-metrics={{- .Values.networking.ENABLE_METRICS }}
           timeoutSeconds: 5
         livenessProbe:
           failureThreshold: 3
@@ -165,6 +166,7 @@ spec:
               - /kube-ovn/kube-ovn-healthcheck
               - --port=10665
               - --tls={{- .Values.func.SECURE_SERVING }}
+              - --enable-metrics={{- .Values.networking.ENABLE_METRICS }}
           timeoutSeconds: 5
         resources:
           requests:

diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go
@@ -80,15 +80,36 @@ func CmdMain() {
 			}()
 		}
 
-		if !config.EnableMetrics {
-			return
-		}
-		metrics.InitKlogMetrics()
-		metrics.InitClientGoMetrics()
-		addr := util.JoinHostPort(metricsAddr, config.PprofPort)
-		if err := metrics.Run(ctx, config.KubeRestConfig, addr, config.SecureServing, servePprofInMetricsServer); err != nil {
-			util.LogFatalAndExit(err, "failed to run metrics server")
+		if config.EnableMetrics {
+			metrics.InitKlogMetrics()
+			metrics.InitClientGoMetrics()
+			addr := util.JoinHostPort(metricsAddr, config.PprofPort)
+			if err := metrics.Run(ctx, config.KubeRestConfig, addr, config.SecureServing, servePprofInMetricsServer); err != nil {
+				util.LogFatalAndExit(err, "failed to run metrics server")
+			}
+		} else {
+			klog.Info("metrics server is disabled")
+			listerner, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP(metricsAddr), Port: int(config.PprofPort)})
+			if err != nil {
+				util.LogFatalAndExit(err, "failed to listen on %s", util.JoinHostPort(metricsAddr, config.PprofPort))
+			}
+			svr := manager.Server{
+				Name: "health-check",
+				Server: &http.Server{
+					Handler:           http.NewServeMux(),
+					MaxHeaderBytes:    1 << 20,
+					IdleTimeout:       90 * time.Second,
+					ReadHeaderTimeout: 32 * time.Second,
+				},
+				Listener: listerner,
+			}
+			go func() {
+				if err = svr.Start(ctx); err != nil {
+					util.LogFatalAndExit(err, "failed to run health check server")
+				}
+			}()
 		}
+
 		<-ctx.Done()
 	}()
 

diff --git a/cmd/daemon/cniserver.go b/cmd/daemon/cniserver.go
@@ -29,9 +29,6 @@ import (
 func CmdMain() {
 	defer klog.Flush()
 
-	daemon.InitMetrics()
-	metrics.InitKlogMetrics()
-
 	config := daemon.ParseFlags()
 	klog.Info(versions.String())
 
@@ -143,10 +140,36 @@ func CmdMain() {
 		}()
 	}
 
-	listenAddr := util.JoinHostPort(addr, config.PprofPort)
-	if err = metrics.Run(ctx, nil, listenAddr, config.SecureServing, servePprofInMetricsServer); err != nil {
-		util.LogFatalAndExit(err, "failed to run metrics server")
+	if config.EnableMetrics {
+		daemon.InitMetrics()
+		metrics.InitKlogMetrics()
+		listenAddr := util.JoinHostPort(addr, config.PprofPort)
+		if err = metrics.Run(ctx, nil, listenAddr, config.SecureServing, servePprofInMetricsServer); err != nil {
+			util.LogFatalAndExit(err, "failed to run metrics server")
+		}
+	} else {
+		klog.Info("metrics server is disabled")
+		listerner, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP(addr), Port: int(config.PprofPort)})
+		if err != nil {
+			util.LogFatalAndExit(err, "failed to listen on %s", util.JoinHostPort(addr, config.PprofPort))
+		}
+		svr := manager.Server{
+			Name: "health-check",
+			Server: &http.Server{
+				Handler:           http.NewServeMux(),
+				MaxHeaderBytes:    1 << 20,
+				IdleTimeout:       90 * time.Second,
+				ReadHeaderTimeout: 32 * time.Second,
+			},
+			Listener: listerner,
+		}
+		go func() {
+			if err = svr.Start(ctx); err != nil {
+				util.LogFatalAndExit(err, "failed to run health check server")
+			}
+		}()
 	}
+
 	<-stopCh
 }
 

diff --git a/cmd/health_check/health_check.go b/cmd/health_check/health_check.go
@@ -15,6 +15,7 @@ import (
 func CmdMain() {
 	port := pflag.Int32("port", 0, "Target port")
 	tls := pflag.Bool("tls", false, "Dial the server with TLS")
+	enableMetrics := pflag.Bool("enable-metrics", true, "Whether to support metrics query")
 
 	klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
 	klog.InitFlags(klogFlags)
@@ -46,7 +47,7 @@ func CmdMain() {
 	}
 
 	addr := util.JoinHostPort(ip, *port)
-	if *tls {
+	if *enableMetrics && *tls {
 		addr = "tls://" + addr
 	} else {
 		addr = "tcp://" + addr

diff --git a/cmd/ovn_monitor/ovn_monitor.go b/cmd/ovn_monitor/ovn_monitor.go
@@ -1,11 +1,13 @@
 package ovn_monitor
 
 import (
-	"os"
-	"strings"
+	"net"
+	"net/http"
+	"time"
 
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 
 	"github.com/kubeovn/kube-ovn/pkg/metrics"
@@ -14,8 +16,6 @@ import (
 	"github.com/kubeovn/kube-ovn/versions"
 )
 
-const port = 10661
-
 func CmdMain() {
 	defer klog.Flush()
 
@@ -26,24 +26,43 @@ func CmdMain() {
 		util.LogFatalAndExit(err, "failed to parse config")
 	}
 
-	addr := config.ListenAddress
-	if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" {
-		if ips := strings.Split(os.Getenv("POD_IPS"), ","); len(ips) == 1 {
-			addr = util.JoinHostPort(ips[0], port)
-		}
-	}
-
-	exporter := ovn.NewExporter(config)
-	if err = exporter.StartConnection(); err != nil {
-		klog.Errorf("%s failed to connect db socket properly: %s", ovn.GetExporterName(), err)
-		go exporter.TryClientConnection()
-	}
-	exporter.StartOvnMetrics()
-
 	ctrl.SetLogger(klog.NewKlogr())
 	ctx := signals.SetupSignalHandler()
-	if err = metrics.Run(ctx, nil, addr, config.SecureServing, false); err != nil {
-		util.LogFatalAndExit(err, "failed to run metrics server")
+
+	metricsAddr := util.GetDefaultListenAddr()
+	if config.EnableMetrics {
+		exporter := ovn.NewExporter(config)
+		if err = exporter.StartConnection(); err != nil {
+			klog.Errorf("%s failed to connect db socket properly: %s", ovn.GetExporterName(), err)
+			go exporter.TryClientConnection()
+		}
+		exporter.StartOvnMetrics()
+		addr := util.JoinHostPort(metricsAddr, config.MetricsPort)
+		if err = metrics.Run(ctx, nil, addr, config.SecureServing, false); err != nil {
+			util.LogFatalAndExit(err, "failed to run metrics server")
+		}
+	} else {
+		klog.Info("metrics server is disabled")
+		listerner, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP(util.GetDefaultListenAddr()), Port: int(config.MetricsPort)})
+		if err != nil {
+			util.LogFatalAndExit(err, "failed to listen on %s", util.JoinHostPort(metricsAddr, config.MetricsPort))
+		}
+		svr := manager.Server{
+			Name: "health-check",
+			Server: &http.Server{
+				Handler:           http.NewServeMux(),
+				MaxHeaderBytes:    1 << 20,
+				IdleTimeout:       90 * time.Second,
+				ReadHeaderTimeout: 32 * time.Second,
+			},
+			Listener: listerner,
+		}
+		go func() {
+			if err = svr.Start(ctx); err != nil {
+				util.LogFatalAndExit(err, "failed to run health check server")
+			}
+		}()
 	}
+
 	<-ctx.Done()
 }
diff --git a/cmd/speaker/speaker.go b/cmd/speaker/speaker.go
@@ -24,9 +24,11 @@ func CmdMain() {
 	ctrl.SetLogger(klog.NewKlogr())
 	ctx := signals.SetupSignalHandler()
 	go func() {
-		metrics.InitKlogMetrics()
-		if err = metrics.Run(ctx, nil, util.JoinHostPort("0.0.0.0", config.PprofPort), false, false); err != nil {
-			util.LogFatalAndExit(err, "failed to run metrics server")
+		if config.EnableMetrics {
+			metrics.InitKlogMetrics()
+			if err = metrics.Run(ctx, nil, util.JoinHostPort("0.0.0.0", config.PprofPort), false, false); err != nil {
+				util.LogFatalAndExit(err, "failed to run metrics server")
+			}
 		}
 		<-ctx.Done()
 	}()

diff --git a/dist/images/install.sh b/dist/images/install.sh
@@ -22,6 +22,7 @@ ENABLE_LB_SVC=${ENABLE_LB_SVC:-false}
 ENABLE_NAT_GW=${ENABLE_NAT_GW:-true}
 ENABLE_KEEP_VM_IP=${ENABLE_KEEP_VM_IP:-true}
 ENABLE_ARP_DETECT_IP_CONFLICT=${ENABLE_ARP_DETECT_IP_CONFLICT:-true}
+ENABLE_METRICS=${ENABLE_METRICS:-true}
 # comma-separated string of nodelocal DNS ip addresses
 NODE_LOCAL_DNS_IP=${NODE_LOCAL_DNS_IP:-}
 ENABLE_IC=${ENABLE_IC:-$(kubectl get node --show-labels | grep -qw "ovn.kubernetes.io/ic-gw" && echo true || echo false)}
@@ -4182,6 +4183,7 @@ spec:
           - --log_file_max_size=200
           - --enable-lb-svc=$ENABLE_LB_SVC
           - --keep-vm-ip=$ENABLE_KEEP_VM_IP
+          - --enable-metrics=$ENABLE_METRICS
           - --node-local-dns-ip=$NODE_LOCAL_DNS_IP
           - --secure-serving=${SECURE_SERVING}
           - --ovsdb-con-timeout=$OVSDB_CON_TIMEOUT
@@ -4240,6 +4242,7 @@ spec:
                 - /kube-ovn/kube-ovn-healthcheck
                 - --port=10660
                 - --tls=${SECURE_SERVING}
+                - --enable-metrics=$ENABLE_METRICS
             periodSeconds: 3
             timeoutSeconds: 5
           livenessProbe:
@@ -4248,6 +4251,7 @@ spec:
                 - /kube-ovn/kube-ovn-healthcheck
                 - --port=10660
                 - --tls=${SECURE_SERVING}
+                - --enable-metrics=$ENABLE_METRICS
             initialDelaySeconds: 300
             periodSeconds: 7
             failureThreshold: 5
@@ -4342,6 +4346,7 @@ spec:
           - --alsologtostderr=true
           - --log_file=/var/log/kube-ovn/kube-ovn-cni.log
           - --log_file_max_size=200
+          - --enable-metrics=$ENABLE_METRICS
           - --kubelet-dir=$KUBELET_DIR
           - --enable-tproxy=$ENABLE_TPROXY
           - --ovs-vsctl-concurrency=$OVS_VSCTL_CONCURRENCY
@@ -4422,6 +4427,7 @@ spec:
               - /kube-ovn/kube-ovn-healthcheck
               - --port=10665
               - --tls=${SECURE_SERVING}
+              - --enable-metrics=$ENABLE_METRICS
           timeoutSeconds: 5
         readinessProbe:
           failureThreshold: 3
@@ -4432,6 +4438,7 @@ spec:
               - /kube-ovn/kube-ovn-healthcheck
               - --port=10665
               - --tls=${SECURE_SERVING}
+              - --enable-metrics=$ENABLE_METRICS
           timeoutSeconds: 5
         resources:
           requests:
@@ -4523,6 +4530,7 @@ spec:
           - --alsologtostderr=true
           - --log_file=/var/log/kube-ovn/kube-ovn-pinger.log
           - --log_file_max_size=200
+          - --enable-metrics=$ENABLE_METRICS
           imagePullPolicy: $IMAGE_PULL_POLICY
           securityContext:
             runAsUser: 0
@@ -4653,6 +4661,7 @@ spec:
           - --logtostderr=false
           - --alsologtostderr=true
           - --log_file_max_size=200
+          - --enable-metrics=$ENABLE_METRICS
           securityContext:
             runAsUser: 0
             privileged: false
@@ -4717,6 +4726,7 @@ spec:
                 - /kube-ovn/kube-ovn-healthcheck
                 - --port=10661
                 - --tls=${SECURE_SERVING}
+                - --enable-metrics=$ENABLE_METRICS
             timeoutSeconds: 5
           readinessProbe:
             failureThreshold: 3
@@ -4728,6 +4738,7 @@ spec:
                 - /kube-ovn/kube-ovn-healthcheck
                 - --port=10661
                 - --tls=${SECURE_SERVING}
+                - --enable-metrics=$ENABLE_METRICS
             timeoutSeconds: 5
       nodeSelector:
         kubernetes.io/os: "linux"