Merge branch 'master' into health

Makefile

@@ -40,7 +40,7 @@ KUBEVIRT_TEST_YAML = https://kubevirt.io/labs/manifests/vm.yaml
 CILIUM_VERSION = 1.14.1
 CILIUM_IMAGE_REPO = quay.io/cilium/cilium
 
-CERT_MANAGER_VERSION = v1.12.3
+CERT_MANAGER_VERSION = v1.12.5
 CERT_MANAGER_CONTROLLER = quay.io/jetstack/cert-manager-controller:$(CERT_MANAGER_VERSION)
 CERT_MANAGER_CAINJECTOR = quay.io/jetstack/cert-manager-cainjector:$(CERT_MANAGER_VERSION)
 CERT_MANAGER_WEBHOOK = quay.io/jetstack/cert-manager-webhook:$(CERT_MANAGER_VERSION)
@@ -769,7 +769,7 @@ kind-install-webhook: kind-install
 	kubectl rollout status deployment/cert-manager-cainjector -n cert-manager --timeout 120s
 	kubectl rollout status deployment/cert-manager-webhook -n cert-manager --timeout 120s
 
-	kubectl apply -f yamls/webhook.yaml
+	sed 's#image: .*#image: $(REGISTRY)/kube-ovn:$(VERSION)#' yamls/webhook.yaml | kubectl apply -f -
 	kubectl rollout status deployment/kube-ovn-webhook -n kube-system --timeout 120s
 
 .PHONY: kind-install-cilium-chaining

charts/templates/kube-ovn-crd.yaml

@@ -39,6 +39,10 @@ spec:
                   type: string
                 subnet:
                   type: string
+                replicas:
+                  type: integer
+                  minimum: 1
+                  maximum: 3
             status:
               type: object
               properties:

charts/templates/ovncni-ds.yaml

@@ -79,6 +79,7 @@ spec:
           - --enable-metrics={{- .Values.networking.ENABLE_METRICS }}
           - --kubelet-dir={{ .Values.kubelet_conf.KUBELET_DIR }}
           - --enable-tproxy={{ .Values.func.ENABLE_TPROXY }}
+          - --ovs-vsctl-concurrency={{ .Values.performance.OVS_VSCTL_CONCURRENCY }}
         securityContext:
           runAsUser: 0
           privileged: true

charts/values.yaml

@@ -97,6 +97,7 @@ performance:
   RPMS: "openvswitch-kmod"
   GC_INTERVAL: 360
   INSPECT_INTERVAL: 20
+  OVS_VSCTL_CONCURRENCY: 100
 
 debug:
   ENABLE_MIRROR: false

cmd/daemon/cniserver.go

@@ -20,6 +20,7 @@ import (
 
 	kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions"
 	"github.com/kubeovn/kube-ovn/pkg/daemon"
+	"github.com/kubeovn/kube-ovn/pkg/ovs"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 	"github.com/kubeovn/kube-ovn/versions"
 )
@@ -37,6 +38,8 @@ func CmdMain() {
 		util.LogFatalAndExit(err, "failed to do the OS initialization")
 	}
 
+	ovs.UpdateOVSVsctlLimiter(config.OVSVsctlConcurrency)
+
 	nicBridgeMappings, err := daemon.InitOVSBridges()
 	if err != nil {
 		util.LogFatalAndExit(err, "failed to initialize OVS bridges")

cmd/webhook/server.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"flag"
+	"os"
 
 	"github.com/spf13/pflag"
 	appsv1 "k8s.io/api/apps/v1"
@@ -10,6 +11,7 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	ctrlwebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
 
@@ -39,6 +41,7 @@ func main() {
 	klog.Infof(versions.String())
 
 	port := pflag.Int("port", 8443, "The port webhook listen on.")
+	healthProbePort := pflag.Int32("health-probe-port", 8080, "The port health probes listen on.")
 
 	klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
 	klog.InitFlags(klogFlags)
@@ -73,6 +76,7 @@ func main() {
 		Metrics: metricsserver.Options{
 			BindAddress: "0",
 		},
+		HealthProbeBindAddress: util.JoinHostPort(os.Getenv("POD_IP"), *healthProbePort),
 	})
 	if err != nil {
 		panic(err)
@@ -91,6 +95,13 @@ func main() {
 		panic(err)
 	}
 
+	if err = mgr.AddHealthzCheck("liveness probe", healthz.Ping); err != nil {
+		panic(err)
+	}
+	if err = mgr.AddReadyzCheck("readiness probe", healthz.Ping); err != nil {
+		panic(err)
+	}
+
 	// Start the server by starting a previously-set-up manager
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		panic(err)

dist/images/Dockerfile.base

@@ -1,5 +1,5 @@
 # syntax = docker/dockerfile:experimental
-FROM ubuntu:22.04 as ovs-builder
+FROM ubuntu:23.10 as ovs-builder
 
 ARG ARCH
 ARG DEBIAN_FRONTEND=noninteractive
@@ -73,7 +73,7 @@ RUN mkdir /packages/ && \
     cp /usr/src/ovn-*deb /packages && \
     cd /packages && rm -f *source* *doc* *datapath* *docker* *vtep* *test* *dev*
 
-FROM ubuntu:22.04
+FROM ubuntu:23.10
 
 ARG DEBIAN_FRONTEND=noninteractive
 RUN apt update && apt upgrade -y && apt install ca-certificates python3 hostname libunwind8 netbase \

dist/images/install.sh

@@ -33,6 +33,7 @@ IFACE=${IFACE:-}
 DPDK_TUNNEL_IFACE=${DPDK_TUNNEL_IFACE:-br-phy}
 ENABLE_BIND_LOCAL_IP=${ENABLE_BIND_LOCAL_IP:-true}
 ENABLE_TPROXY=${ENABLE_TPROXY:-false}
+OVS_VSCTL_CONCURRENCY=${OVS_VSCTL_CONCURRENCY:-100}
 
 # debug
 DEBUG_WRAPPER=${DEBUG_WRAPPER:-}
@@ -264,6 +265,10 @@ spec:
                   type: string
                 subnet:
                   type: string
+                replicas:
+                  type: integer
+                  minimum: 1
+                  maximum: 3
             status:
               type: object
               properties:
@@ -4065,6 +4070,7 @@ spec:
           - --log_file_max_size=0
           - --kubelet-dir=$KUBELET_DIR
           - --enable-tproxy=$ENABLE_TPROXY
+          - --ovs-vsctl-concurrency=$OVS_VSCTL_CONCURRENCY
         securityContext:
           runAsUser: 0
           privileged: true

go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230905082151-e28c4d73a589
 	github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875
 	github.com/moby/sys/mountinfo v0.6.2
-	github.com/onsi/ginkgo/v2 v2.12.1
+	github.com/onsi/ginkgo/v2 v2.13.0
 	github.com/onsi/gomega v1.28.0
 	github.com/osrg/gobgp/v3 v3.19.0
 	github.com/ovn-org/libovsdb v0.0.0-20230711201130-6785b52d4020
@@ -39,7 +39,7 @@ require (
 	golang.org/x/mod v0.13.0
 	golang.org/x/sys v0.13.0
 	golang.org/x/time v0.3.0
-	google.golang.org/grpc v1.58.2
+	google.golang.org/grpc v1.58.3
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/k8snetworkplumbingwg/multus-cni.v4 v4.0.2
 	k8s.io/api v0.28.2
@@ -231,7 +231,8 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.25.0 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
-	golang.org/x/net v0.16.0 // indirect
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
+	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/oauth2 v0.13.0 // indirect
 	golang.org/x/sync v0.4.0 // indirect
 	golang.org/x/term v0.13.0 // indirect

go.sum

@@ -1662,8 +1662,8 @@ github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxm
 github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo=
 github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts=
 github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM=
-github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA=
-github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
+github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
+github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
@@ -2299,8 +2299,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
-golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos=
-golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2923,8 +2923,8 @@ google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD
 google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
 google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
 google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g=
-google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I=
-google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
+google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
+google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

pkg/apis/kubeovn/v1/types.go

@@ -853,8 +853,9 @@ type VpcDns struct {
 }
 
 type VpcDNSSpec struct {
-	Vpc    string `json:"vpc"`
-	Subnet string `json:"subnet"`
+	Replicas int32  `json:"replicas"`
+	Vpc      string `json:"vpc"`
+	Subnet   string `json:"subnet"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/controller/vpc_dns.go

@@ -193,12 +193,6 @@ func (c *Controller) handleAddOrUpdateVPCDNS(key string) error {
 		return err
 	}
 
-	if err := c.checkOvnDefaultSpecProvider(); err != nil {
-		err := fmt.Errorf("failed to check %s spec provider, %v", util.DefaultSubnet, err)
-		klog.Error(err)
-		return err
-	}
-
 	if err := c.checkVpcDNSDuplicated(vpcDNS); err != nil {
 		err = fmt.Errorf("failed to deploy %s, %v", vpcDNS.Name, err)
 		klog.Error(err)
@@ -277,6 +271,10 @@ func (c *Controller) createOrUpdateVpcDNSDep(vpcDNS *kubeovnv1.VpcDns) error {
 		return err
 	}
 
+	if vpcDNS.Spec.Replicas != 0 {
+		newDp.Spec.Replicas = &vpcDNS.Spec.Replicas
+	}
+
 	if needToCreateDp {
 		_, err := c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace).
 			Create(context.Background(), newDp, metav1.CreateOptions{})
@@ -491,20 +489,6 @@ func (c *Controller) checkOvnNad() error {
 	return nil
 }
 
-func (c *Controller) checkOvnDefaultSpecProvider() error {
-	cachedSubnet, err := c.subnetsLister.Get(util.DefaultSubnet)
-	if err != nil {
-		klog.Error(err)
-		return fmt.Errorf("failed to get default subnet %v", err)
-	}
-
-	if cachedSubnet.Spec.Provider != nadProvider {
-		return fmt.Errorf("the %s provider does not exist", nadProvider)
-	}
-
-	return nil
-}
-
 func (c *Controller) resyncVpcDNSConfig() {
 	cm, err := c.configMapsLister.ConfigMaps(c.config.PodNamespace).Get(util.VpcDNSConfig)
 	if err != nil && !k8serrors.IsNotFound(err) {

pkg/daemon/config.go

@@ -64,6 +64,7 @@ type Configuration struct {
 	TCPConnCheckPort          int
 	UDPConnCheckPort          int
 	EnableTProxy              bool
+	OVSVsctlConcurrency       int32
 }
 
 // ParseFlags will parse cmd args then init kubeClient and configuration
@@ -102,6 +103,7 @@ func ParseFlags() *Configuration {
 		argTCPConnectivityCheckPort  = pflag.Int("tcp-conn-check-port", 8100, "TCP connectivity Check Port")
 		argUDPConnectivityCheckPort  = pflag.Int("udp-conn-check-port", 8101, "UDP connectivity Check Port")
 		argEnableTProxy              = pflag.Bool("enable-tproxy", false, "enable tproxy for vpc pod liveness or readiness probe")
+		argOVSVsctlConcurrency       = pflag.Int32("ovs-vsctl-concurrency", 100, "concurrency limit of ovs-vsctl")
 	)
 
 	// mute info log for ipset lib
@@ -157,6 +159,7 @@ func ParseFlags() *Configuration {
 		TCPConnCheckPort:          *argTCPConnectivityCheckPort,
 		UDPConnCheckPort:          *argUDPConnectivityCheckPort,
 		EnableTProxy:              *argEnableTProxy,
+		OVSVsctlConcurrency:       *argOVSVsctlConcurrency,
 	}
 	return config
 }

pkg/ovs/ovs-vsctl.go

@@ -1,6 +1,7 @@
 package ovs
 
 import (
+	"context"
 	"fmt"
 	"os/exec"
 	"regexp"
@@ -13,25 +14,57 @@ import (
 	"github.com/kubeovn/kube-ovn/pkg/util"
 )
 
+var limiter *Limiter
+
+func init() {
+	limiter = new(Limiter)
+}
+
+func UpdateOVSVsctlLimiter(c int32) {
+	if c >= 0 {
+		limiter.Update(c)
+		klog.V(4).Infof("update ovs-vsctl concurrency limit to %d", limiter.Limit())
+	}
+}
+
 // Glory belongs to openvswitch/ovn-kubernetes
 // https://github.com/openvswitch/ovn-kubernetes/blob/master/go-controller/pkg/util/ovs.go
 
 var podNetNsRegexp = regexp.MustCompile(`pod_netns="([^"]+)"`)
 
 func Exec(args ...string) (string, error) {
-	start := time.Now()
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+
+	var (
+		start        time.Time
+		elapsed      float64
+		output       []byte
+		method, code string
+		err          error
+	)
+
+	if err = limiter.Wait(ctx); err != nil {
+		klog.V(4).Infof("command %s %s waiting for execution timeout by concurrency limit of %d", OvsVsCtl, strings.Join(args, " "), limiter.Limit())
+		return "", err
+	}
+	defer limiter.Done()
+	klog.V(4).Infof("command %s %s waiting for execution concurrency %d/%d", OvsVsCtl, strings.Join(args, " "), limiter.Current(), limiter.Limit())
+
+	start = time.Now()
 	args = append([]string{"--timeout=30"}, args...)
-	output, err := exec.Command(OvsVsCtl, args...).CombinedOutput()
-	elapsed := float64((time.Since(start)) / time.Millisecond)
+	output, err = exec.Command(OvsVsCtl, args...).CombinedOutput()
+	elapsed = float64((time.Since(start)) / time.Millisecond)
 	klog.V(4).Infof("command %s %s in %vms", OvsVsCtl, strings.Join(args, " "), elapsed)
-	method := ""
+
 	for _, arg := range args {
 		if !strings.HasPrefix(arg, "--") {
 			method = arg
 			break
 		}
 	}
-	code := "0"
+
+	code = "0"
 	defer func() {
 		ovsClientRequestLatency.WithLabelValues("ovsdb", method, code).Observe(elapsed)
 	}()