From 1f3ba42314e11d03a4388a7448f55e03aa08a77b Mon Sep 17 00:00:00 2001 From: ChrisLiu Date: Fri, 14 Jun 2024 18:30:38 +0800 Subject: [PATCH] Enhance: support custom health checks for AlibabaCloud-SLB Signed-off-by: ChrisLiu --- cloudprovider/alibabacloud/nlb.go | 5 +- cloudprovider/alibabacloud/slb.go | 194 ++++++++++++++++-- cloudprovider/alibabacloud/slb_test.go | 114 +++++++--- docs/en/user_manuals/network.md | 66 ++++++ ...21\347\273\234\346\250\241\345\236\213.md" | 66 ++++++ 5 files changed, 402 insertions(+), 43 deletions(-) diff --git a/cloudprovider/alibabacloud/nlb.go b/cloudprovider/alibabacloud/nlb.go index 48ff9ce1..2d8b17a3 100644 --- a/cloudprovider/alibabacloud/nlb.go +++ b/cloudprovider/alibabacloud/nlb.go @@ -235,7 +235,10 @@ func (n *NlbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.C func (n *NlbPlugin) OnPodDeleted(c client.Client, pod *corev1.Pod, ctx context.Context) cperrors.PluginError { networkManager := utils.NewNetworkManager(pod, c) networkConfig := networkManager.GetNetworkConfig() - sc := parseLbConfig(networkConfig) + sc, err := parseNlbConfig(networkConfig) + if err != nil { + return cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) + } var podKeys []string if sc.isFixed { diff --git a/cloudprovider/alibabacloud/slb.go b/cloudprovider/alibabacloud/slb.go index 0da2e42a..7e65ee48 100644 --- a/cloudprovider/alibabacloud/slb.go +++ b/cloudprovider/alibabacloud/slb.go @@ -18,6 +18,7 @@ package alibabacloud import ( "context" + "fmt" gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" "github.com/openkruise/kruise-game/cloudprovider" cperrors "github.com/openkruise/kruise-game/cloudprovider/errors" @@ -49,6 +50,16 @@ const ( SlbConfigHashKey = "game.kruise.io/network-config-hash" ) +const ( + // annotations provided by AlibabaCloud Cloud Controller Manager + LBHealthCheckSwitchAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-health-check-switch" + LBHealthCheckProtocolPortAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-protocol-port" + + // ConfigNames defined by OKG + LBHealthCheckSwitchConfigName = "LBHealthCheckSwitch" + LBHealthCheckProtocolPortConfigName = "LBHealthCheckProtocolPort" +) + type portAllocated map[int32]bool type SlbPlugin struct { @@ -64,6 +75,18 @@ type slbConfig struct { targetPorts []int protocols []corev1.Protocol isFixed bool + + lBHealthCheckSwitch string + lBHealthCheckProtocolPort string + lBHealthCheckFlag string + lBHealthCheckType string + lBHealthCheckConnectTimeout string + lBHealthCheckInterval string + lBHealthCheckUri string + lBHealthCheckDomain string + lBHealthCheckMethod string + lBHealthyThreshold string + lBUnhealthyThreshold string } func (s *SlbPlugin) Name() string { @@ -127,18 +150,21 @@ func (s *SlbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.C networkManager := utils.NewNetworkManager(pod, c) networkStatus, _ := networkManager.GetNetworkStatus() - networkConfig := networkManager.GetNetworkConfig() - sc := parseLbConfig(networkConfig) if networkStatus == nil { pod, err := networkManager.UpdateNetworkStatus(gamekruiseiov1alpha1.NetworkStatus{ CurrentNetworkState: gamekruiseiov1alpha1.NetworkNotReady, }, pod) return pod, cperrors.ToPluginError(err, cperrors.InternalError) } + networkConfig := networkManager.GetNetworkConfig() + sc, err := parseLbConfig(networkConfig) + if err != nil { + return pod, cperrors.ToPluginError(err, cperrors.ParameterError) + } // get svc svc := &corev1.Service{} - err := c.Get(ctx, types.NamespacedName{ + err = c.Get(ctx, types.NamespacedName{ Name: pod.GetName(), Namespace: pod.GetNamespace(), }, svc) @@ -232,7 +258,10 @@ func (s *SlbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.C func (s *SlbPlugin) OnPodDeleted(c client.Client, pod *corev1.Pod, ctx context.Context) cperrors.PluginError { networkManager := utils.NewNetworkManager(pod, c) networkConfig := networkManager.GetNetworkConfig() - sc := parseLbConfig(networkConfig) + sc, err := parseLbConfig(networkConfig) + if err != nil { + return cperrors.NewPluginError(cperrors.ParameterError, err.Error()) + } var podKeys []string if sc.isFixed { @@ -335,11 +364,23 @@ func init() { alibabaCloudProvider.registerPlugin(&slbPlugin) } -func parseLbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) *slbConfig { +func parseLbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) (*slbConfig, error) { var lbIds []string ports := make([]int, 0) protocols := make([]corev1.Protocol, 0) isFixed := false + + lBHealthCheckSwitch := "on" + lBHealthCheckProtocolPort := "" + lBHealthCheckFlag := "off" + lBHealthCheckType := "tcp" + lBHealthCheckConnectTimeout := "5" + lBHealthCheckInterval := "10" + lBUnhealthyThreshold := "2" + lBHealthyThreshold := "2" + lBHealthCheckUri := "" + lBHealthCheckDomain := "" + lBHealthCheckMethod := "" for _, c := range conf { switch c.Name { case SlbIdsConfigName: @@ -368,14 +409,100 @@ func parseLbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) *slbConfig { continue } isFixed = v + case LBHealthCheckSwitchConfigName: + checkSwitch := strings.ToLower(c.Value) + if checkSwitch != "on" && checkSwitch != "off" { + return nil, fmt.Errorf("invalid lb health check switch value: %s", c.Value) + } + lBHealthCheckSwitch = checkSwitch + case LBHealthCheckFlagConfigName: + flag := strings.ToLower(c.Value) + if flag != "on" && flag != "off" { + return nil, fmt.Errorf("invalid lb health check flag value: %s", c.Value) + } + lBHealthCheckFlag = flag + case LBHealthCheckTypeConfigName: + checkType := strings.ToLower(c.Value) + if checkType != "tcp" && checkType != "http" { + return nil, fmt.Errorf("invalid lb health check type: %s", c.Value) + } + lBHealthCheckType = checkType + case LBHealthCheckProtocolPortConfigName: + if validateHttpProtocolPort(c.Value) != nil { + return nil, fmt.Errorf("invalid lb health check protocol port: %s", c.Value) + } + lBHealthCheckProtocolPort = c.Value + case LBHealthCheckConnectTimeoutConfigName: + timeoutInt, err := strconv.Atoi(c.Value) + if err != nil { + return nil, fmt.Errorf("invalid lb health check connect timeout: %s", c.Value) + } + if timeoutInt < 1 || timeoutInt > 300 { + return nil, fmt.Errorf("invalid lb health check connect timeout: %d", timeoutInt) + } + lBHealthCheckConnectTimeout = c.Value + case LBHealthCheckIntervalConfigName: + intervalInt, err := strconv.Atoi(c.Value) + if err != nil { + return nil, fmt.Errorf("invalid lb health check interval: %s", c.Value) + } + if intervalInt < 1 || intervalInt > 50 { + return nil, fmt.Errorf("invalid lb health check interval: %d", intervalInt) + } + lBHealthCheckInterval = c.Value + case LBHealthyThresholdConfigName: + thresholdInt, err := strconv.Atoi(c.Value) + if err != nil { + return nil, fmt.Errorf("invalid lb healthy threshold: %s", c.Value) + } + if thresholdInt < 2 || thresholdInt > 10 { + return nil, fmt.Errorf("invalid lb healthy threshold: %d", thresholdInt) + } + lBHealthyThreshold = c.Value + case LBUnhealthyThresholdConfigName: + thresholdInt, err := strconv.Atoi(c.Value) + if err != nil { + return nil, fmt.Errorf("invalid lb unhealthy threshold: %s", c.Value) + } + if thresholdInt < 2 || thresholdInt > 10 { + return nil, fmt.Errorf("invalid lb unhealthy threshold: %d", thresholdInt) + } + lBUnhealthyThreshold = c.Value + case LBHealthCheckUriConfigName: + if validateUri(c.Value) != nil { + return nil, fmt.Errorf("invalid lb health check uri: %s", c.Value) + } + lBHealthCheckUri = c.Value + case LBHealthCheckDomainConfigName: + if validateDomain(c.Value) != nil { + return nil, fmt.Errorf("invalid lb health check domain: %s", c.Value) + } + lBHealthCheckDomain = c.Value + case LBHealthCheckMethodConfigName: + method := strings.ToLower(c.Value) + if method != "get" && method != "head" { + return nil, fmt.Errorf("invalid lb health check method: %s", c.Value) + } + lBHealthCheckMethod = method } } return &slbConfig{ - lbIds: lbIds, - protocols: protocols, - targetPorts: ports, - isFixed: isFixed, - } + lbIds: lbIds, + protocols: protocols, + targetPorts: ports, + isFixed: isFixed, + lBHealthCheckSwitch: lBHealthCheckSwitch, + lBHealthCheckFlag: lBHealthCheckFlag, + lBHealthCheckType: lBHealthCheckType, + lBHealthCheckProtocolPort: lBHealthCheckProtocolPort, + lBHealthCheckConnectTimeout: lBHealthCheckConnectTimeout, + lBHealthCheckInterval: lBHealthCheckInterval, + lBHealthCheckUri: lBHealthCheckUri, + lBHealthCheckDomain: lBHealthCheckDomain, + lBHealthCheckMethod: lBHealthCheckMethod, + lBHealthyThreshold: lBHealthyThreshold, + lBUnhealthyThreshold: lBUnhealthyThreshold, + }, nil } func getPorts(ports []corev1.ServicePort) []int32 { @@ -409,15 +536,32 @@ func (s *SlbPlugin) consSvc(sc *slbConfig, pod *corev1.Pod, c client.Client, ctx }) } + svcAnnotations := map[string]string{ + SlbListenerOverrideKey: "true", + SlbIdAnnotationKey: lbId, + SlbConfigHashKey: util.GetHash(sc), + LBHealthCheckFlagAnnotationKey: sc.lBHealthCheckFlag, + LBHealthCheckSwitchAnnotationKey: sc.lBHealthCheckSwitch, + } + if sc.lBHealthCheckSwitch == "on" { + svcAnnotations[LBHealthCheckTypeAnnotationKey] = sc.lBHealthCheckType + svcAnnotations[LBHealthCheckConnectTimeoutAnnotationKey] = sc.lBHealthCheckConnectTimeout + svcAnnotations[LBHealthCheckIntervalAnnotationKey] = sc.lBHealthCheckInterval + svcAnnotations[LBHealthyThresholdAnnotationKey] = sc.lBHealthyThreshold + svcAnnotations[LBUnhealthyThresholdAnnotationKey] = sc.lBUnhealthyThreshold + if sc.lBHealthCheckType == "http" { + svcAnnotations[LBHealthCheckProtocolPortAnnotationKey] = sc.lBHealthCheckProtocolPort + svcAnnotations[LBHealthCheckDomainAnnotationKey] = sc.lBHealthCheckDomain + svcAnnotations[LBHealthCheckUriAnnotationKey] = sc.lBHealthCheckUri + svcAnnotations[LBHealthCheckMethodAnnotationKey] = sc.lBHealthCheckMethod + } + } + svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: pod.GetName(), - Namespace: pod.GetNamespace(), - Annotations: map[string]string{ - SlbListenerOverrideKey: "true", - SlbIdAnnotationKey: lbId, - SlbConfigHashKey: util.GetHash(sc), - }, + Name: pod.GetName(), + Namespace: pod.GetNamespace(), + Annotations: svcAnnotations, OwnerReferences: getSvcOwnerReference(c, ctx, pod, sc.isFixed), }, Spec: corev1.ServiceSpec{ @@ -459,3 +603,19 @@ func getSvcOwnerReference(c client.Client, ctx context.Context, pod *corev1.Pod, } return ownerReferences } + +func validateHttpProtocolPort(protocolPort string) error { + protocolPorts := strings.Split(protocolPort, ",") + for _, pp := range protocolPorts { + protocol := strings.Split(pp, ":")[0] + if protocol != "http" && protocol != "https" { + return fmt.Errorf("invalid http protocol: %s", protocol) + } + port := strings.Split(pp, ":")[1] + _, err := strconv.Atoi(port) + if err != nil { + return fmt.Errorf("invalid http port: %s", port) + } + } + return nil +} diff --git a/cloudprovider/alibabacloud/slb_test.go b/cloudprovider/alibabacloud/slb_test.go index c25266f4..db9202b5 100644 --- a/cloudprovider/alibabacloud/slb_test.go +++ b/cloudprovider/alibabacloud/slb_test.go @@ -18,7 +18,6 @@ package alibabacloud import ( gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" - "github.com/openkruise/kruise-game/pkg/util" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -72,10 +71,7 @@ func TestAllocateDeAllocate(t *testing.T) { func TestParseLbConfig(t *testing.T) { tests := []struct { conf []gamekruiseiov1alpha1.NetworkConfParams - lbIds []string - ports []int - protocols []corev1.Protocol - isFixed bool + slbConfig *slbConfig }{ { conf: []gamekruiseiov1alpha1.NetworkConfParams{ @@ -87,11 +83,72 @@ func TestParseLbConfig(t *testing.T) { Name: PortProtocolsConfigName, Value: "80", }, + { + Name: LBHealthCheckSwitchConfigName, + Value: "off", + }, + { + Name: LBHealthCheckFlagConfigName, + Value: "off", + }, + { + Name: LBHealthCheckTypeConfigName, + Value: "HTTP", + }, + { + Name: LBHealthCheckConnectPortConfigName, + Value: "6000", + }, + { + Name: LBHealthCheckConnectTimeoutConfigName, + Value: "100", + }, + { + Name: LBHealthCheckIntervalConfigName, + Value: "30", + }, + { + Name: LBHealthCheckUriConfigName, + Value: "/another?valid", + }, + { + Name: LBHealthCheckDomainConfigName, + Value: "www.test.com", + }, + { + Name: LBHealthCheckMethodConfigName, + Value: "HEAD", + }, + { + Name: LBHealthyThresholdConfigName, + Value: "5", + }, + { + Name: LBUnhealthyThresholdConfigName, + Value: "5", + }, + { + Name: LBHealthCheckProtocolPortConfigName, + Value: "http:80", + }, + }, + slbConfig: &slbConfig{ + lbIds: []string{"xxx-A"}, + targetPorts: []int{80}, + protocols: []corev1.Protocol{corev1.ProtocolTCP}, + isFixed: false, + lBHealthCheckSwitch: "off", + lBHealthCheckFlag: "off", + lBHealthCheckType: "http", + lBHealthCheckConnectTimeout: "100", + lBHealthCheckInterval: "30", + lBHealthCheckUri: "/another?valid", + lBHealthCheckDomain: "www.test.com", + lBHealthCheckMethod: "head", + lBHealthyThreshold: "5", + lBUnhealthyThreshold: "5", + lBHealthCheckProtocolPort: "http:80", }, - lbIds: []string{"xxx-A"}, - ports: []int{80}, - protocols: []corev1.Protocol{corev1.ProtocolTCP}, - isFixed: false, }, { conf: []gamekruiseiov1alpha1.NetworkConfParams{ @@ -108,26 +165,33 @@ func TestParseLbConfig(t *testing.T) { Value: "true", }, }, - lbIds: []string{"xxx-A", "xxx-B"}, - ports: []int{81, 82, 83}, - protocols: []corev1.Protocol{corev1.ProtocolUDP, corev1.ProtocolTCP, corev1.ProtocolTCP}, - isFixed: true, + slbConfig: &slbConfig{ + lbIds: []string{"xxx-A", "xxx-B"}, + targetPorts: []int{81, 82, 83}, + protocols: []corev1.Protocol{corev1.ProtocolUDP, corev1.ProtocolTCP, corev1.ProtocolTCP}, + isFixed: true, + lBHealthCheckSwitch: "on", + lBHealthCheckFlag: "off", + lBHealthCheckType: "tcp", + lBHealthCheckConnectTimeout: "5", + lBHealthCheckInterval: "10", + lBUnhealthyThreshold: "2", + lBHealthyThreshold: "2", + lBHealthCheckUri: "", + lBHealthCheckDomain: "", + lBHealthCheckMethod: "", + lBHealthCheckProtocolPort: "", + }, }, } - for _, test := range tests { - sc := parseLbConfig(test.conf) - if !reflect.DeepEqual(test.lbIds, sc.lbIds) { - t.Errorf("lbId expect: %v, actual: %v", test.lbIds, sc.lbIds) - } - if !util.IsSliceEqual(test.ports, sc.targetPorts) { - t.Errorf("ports expect: %v, actual: %v", test.ports, sc.targetPorts) - } - if !reflect.DeepEqual(test.protocols, sc.protocols) { - t.Errorf("protocols expect: %v, actual: %v", test.protocols, sc.protocols) + for i, test := range tests { + sc, err := parseLbConfig(test.conf) + if err != nil { + t.Error(err) } - if test.isFixed != sc.isFixed { - t.Errorf("isFixed expect: %v, actual: %v", test.isFixed, sc.isFixed) + if !reflect.DeepEqual(test.slbConfig, sc) { + t.Errorf("case %d: lbId expect: %v, actual: %v", i, test.slbConfig, sc) } } } diff --git a/docs/en/user_manuals/network.md b/docs/en/user_manuals/network.md index 4cb223e8..d105e156 100644 --- a/docs/en/user_manuals/network.md +++ b/docs/en/user_manuals/network.md @@ -436,6 +436,72 @@ AllowNotReadyContainers - Value: {containerName_0},{containerName_1},... Example：sidecar - Configuration change supported or not: It cannot be changed during the in-place updating process. +LBHealthCheckSwitch + +- Meaning：Whether to enable health check +- Format："on" means on, "off" means off. Default is on +- Whether to support changes: Yes + +LBHealthCheckFlag + +- Meaning: Whether to enable http type health check +- Format: "on" means on, "off" means off. Default is on +- Whether to support changes: Yes + +LBHealthCheckType + +- Meaning: Health Check Protocol +- Format: fill in "tcp" or "http", the default is tcp +- Whether to support changes: Yes + +LBHealthCheckConnectTimeout + +- Meaning: Maximum timeout for health check response. +- Format: Unit: seconds. The value range is [1, 300]. The default value is "5" +- Whether to support changes: Yes + +LBHealthyThreshold + +- Meaning: After the number of consecutive successful health checks, the health check status of the server will be determined from failure to success. +- Format: Value range [2, 10]. Default value is "2" +- Whether to support changes: Yes + +LBUnhealthyThreshold + +- Meaning: After the number of consecutive health check failures, the health check status of the server will be determined from success to failure. +- Format: Value range [2, 10]. The default value is "2" +- Whether to support changes: Yes + +LBHealthCheckInterval + +- Meaning: health check interval. +- Format: Unit: seconds. The value range is [1, 50]. The default value is "10" +- Whether to support changes: Yes + +LBHealthCheckProtocolPort + +- Meaning：the protocols & ports of HTTP type health check. +- Format：Multiple values are separated by ','. e.g. https:443,http:80 +- Whether to support changes: Yes + +LBHealthCheckUri + +- Meaning: The corresponding uri when the health check type is HTTP. +- Format: The length is 1~80 characters, only letters, numbers, and characters can be used. Must start with a forward slash (/). Such as "/test/index.html" +- Whether to support changes: Yes + +LBHealthCheckDomain + +- Meaning: The corresponding domain name when the health check type is HTTP. +- Format: The length of a specific domain name is limited to 1~80 characters. Only lowercase letters, numbers, dashes (-), and half-width periods (.) can be used. +- Whether to support changes: Yes + +LBHealthCheckMethod + +- Meaning: The corresponding method when the health check type is HTTP. +- Format: "GET" or "HEAD" +- Whether to support changes: Yes + #### Plugin configuration ``` [alibabacloud] diff --git "a/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/\347\275\221\347\273\234\346\250\241\345\236\213.md" "b/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/\347\275\221\347\273\234\346\250\241\345\236\213.md" index 15258430..448c6e53 100644 --- "a/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/\347\275\221\347\273\234\346\250\241\345\236\213.md" +++ "b/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/\347\275\221\347\273\234\346\250\241\345\236\213.md" @@ -439,6 +439,72 @@ AllowNotReadyContainers - 格式：{containerName_0},{containerName_1},... 例如：sidecar - 是否支持变更：在原地升级过程中不可变更。 +LBHealthCheckSwitch + +- 含义：是否开启健康检查 +- 格式：“on”代表开启，“off”代表关闭。默认为on +- 是否支持变更：支持 + +LBHealthCheckFlag + +- 含义：是否开启http类型健康检查 +- 格式：“on”代表开启，“off”代表关闭。默认为off +- 是否支持变更：支持 + +LBHealthCheckType + +- 含义：健康检查协议 +- 格式：填写 “tcp” 或者 “http”，默认为tcp +- 是否支持变更：支持 + +LBHealthCheckConnectTimeout + +- 含义：健康检查响应的最大超时时间。 +- 格式：单位：秒。取值范围[1, 300]。默认值为“5” +- 是否支持变更：支持 + +LBHealthyThreshold + +- 含义：健康检查连续成功多少次后，将服务器的健康检查状态由失败判定为成功。 +- 格式：取值范围[2, 10]。默认值为“2” +- 是否支持变更：支持 + +LBUnhealthyThreshold + +- 含义：健康检查连续失败多少次后，将服务器的健康检查状态由成功判定为失败。 +- 格式：取值范围[2, 10]。默认值为“2” +- 是否支持变更：支持 + +LBHealthCheckInterval + +- 含义：健康检查的时间间隔。 +- 格式：单位：秒。取值范围[1, 50]。默认值为“10” +- 是否支持变更：支持 + +LBHealthCheckProtocolPort + +- 含义：http类型健康检查的协议及端口。 +- 格式：多个值之间用英文半角逗号（,）分隔。如https:443,http:80 +- 是否支持变更：支持 + +LBHealthCheckUri + +- 含义：健康检查类型为HTTP时对应的检查路径。 +- 格式：长度为1~80个字符，只能使用字母、数字、字符。 必须以正斜线（/）开头。 +- 是否支持变更：支持 + +LBHealthCheckDomain + +- 含义：健康检查类型为HTTP时对应的域名。 +- 格式：特定域名长度限制1~80个字符，只能使用小写字母、数字、短划线（-）、半角句号（.）。 +- 是否支持变更：支持 + +LBHealthCheckMethod + +- 含义：健康检查类型为HTTP时对应的方法。 +- 格式：“GET” 或者 “HEAD” +- 是否支持变更：支持 + #### 插件配置 ``` [alibabacloud]