From 3ae9008efacaf9ce6743cb6496e8fb79a2ab86fb Mon Sep 17 00:00:00 2001 From: hhr <691129301@qq.com> Date: Tue, 5 Nov 2024 13:11:48 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20add=20jdcloud=20provider=20and=20th?= =?UTF-8?q?e=20nlb=E3=80=81eip=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cloudprovider/config.go | 3 + cloudprovider/jdcloud/README.md | 208 +++++++ cloudprovider/jdcloud/README.zh_CN.md | 209 +++++++ cloudprovider/jdcloud/apis/v1/doc.go | 1 + cloudprovider/jdcloud/apis/v1/jdnlb_types.go | 37 ++ cloudprovider/jdcloud/eip.go | 103 ++++ cloudprovider/jdcloud/eip_test.go | 1 + cloudprovider/jdcloud/jdcloud.go | 61 +++ cloudprovider/jdcloud/nlb.go | 546 +++++++++++++++++++ cloudprovider/jdcloud/nlb_test.go | 344 ++++++++++++ cloudprovider/manager/provider_manager.go | 11 + cloudprovider/options/jdcloud_options.go | 28 + config/manager/config.toml | 7 + 13 files changed, 1559 insertions(+) create mode 100644 cloudprovider/jdcloud/README.md create mode 100644 cloudprovider/jdcloud/README.zh_CN.md create mode 100644 cloudprovider/jdcloud/apis/v1/doc.go create mode 100644 cloudprovider/jdcloud/apis/v1/jdnlb_types.go create mode 100644 cloudprovider/jdcloud/eip.go create mode 100644 cloudprovider/jdcloud/eip_test.go create mode 100644 cloudprovider/jdcloud/jdcloud.go create mode 100644 cloudprovider/jdcloud/nlb.go create mode 100644 cloudprovider/jdcloud/nlb_test.go create mode 100644 cloudprovider/options/jdcloud_options.go diff --git a/cloudprovider/config.go b/cloudprovider/config.go index abd5496..9744469 100644 --- a/cloudprovider/config.go +++ b/cloudprovider/config.go @@ -48,6 +48,7 @@ type CloudProviderConfig struct { VolcengineOptions CloudProviderOptions AmazonsWebServicesOptions CloudProviderOptions TencentCloudOptions CloudProviderOptions + JdCloudOptions CloudProviderOptions } type tomlConfigs struct { @@ -56,6 +57,7 @@ type tomlConfigs struct { Volcengine options.VolcengineOptions `toml:"volcengine"` AmazonsWebServices options.AmazonsWebServicesOptions `toml:"aws"` TencentCloud options.TencentCloudOptions `toml:"tencentcloud"` + JdCloud options.JdCloudOptions `toml:"jdcloud"` } func (cf *ConfigFile) Parse() *CloudProviderConfig { @@ -70,6 +72,7 @@ func (cf *ConfigFile) Parse() *CloudProviderConfig { VolcengineOptions: config.Volcengine, AmazonsWebServicesOptions: config.AmazonsWebServices, TencentCloudOptions: config.TencentCloud, + JdCloudOptions: config.JdCloud, } } diff --git a/cloudprovider/jdcloud/README.md b/cloudprovider/jdcloud/README.md new file mode 100644 index 0000000..386e0fb --- /dev/null +++ b/cloudprovider/jdcloud/README.md @@ -0,0 +1,208 @@ +English | [中文](./README.md) + +Based on JdCloud Container Service, for game scenarios, combine OKG to provide various network model plugins. + +## JdCloud-NLB configuration + +JdCloud Container Service supports the reuse of NLB (Network Load Balancer) in Kubernetes. Different services (svcs) can use different ports of the same NLB. As a result, the JdCloud-NLB network plugin will record the port allocation for each NLB. For services that specify the network type as JdCloud-NLB, the JdCloud-NLB network plugin will automatically allocate a port and create a service object. Once it detects that the public IP of the svc has been successfully created, the GameServer's network will transition to the Ready state, completing the process. + +### plugin configuration +```toml +[jdcloud] +enable = true +[jdcloud.nlb] +#To allocate external access ports for Pods, you need to define the idle port ranges that the NLB (Network Load Balancer) can use. The maximum range for each port segment is 200 ports. +max_port = 700 +min_port = 500 +``` + +### Parameter +#### NlbIds +- Meaning：fill in the id of the clb. You can fill in more than one. You need to create the clb in [JdCloud]. +- Value：each clbId is divided by `,` . For example：`netlb-aaa,netlb-bbb,...` +- Configurable：Y + +#### PortProtocols +- Meaning：the ports and protocols exposed by the pod, support filling in multiple ports/protocols +- Value：`port1/protocol1`,`port2/protocol2`,... The protocol names must be in uppercase letters. +- Configurable：Y + +#### Fixed +- Meaning：whether the mapping relationship is fixed. If the mapping relationship is fixed, the mapping relationship remains unchanged even if the pod is deleted and recreated. +- Value：false / true +- Configurable：Y + +#### AllocateLoadBalancerNodePorts +- Meaning：Whether the generated service is assigned nodeport, this can be set to false only in nlb passthrough mode +- Value：false / true +- Configurable：Y + +#### AllowNotReadyContainers +- Meaning：the container names that are allowed not ready when inplace updating, when traffic will not be cut. +- Value：{containerName_0},{containerName_1},... eg：sidecar +- Configurable：It cannot be changed during the in-place updating process. + +#### Annotations +- Meaning：the anno added to the service +- Value：key1:value1,key2:value2... +- Configurable：Y + + +### Example +```yaml +cat <= minPort { + newCache[lbId][port] = true + ports = append(ports, port) + } + } + if len(ports) != 0 { + newPodAllocate[svc.GetNamespace()+"/"+svc.GetName()] = lbId + ":" + util.Int32SliceToString(ports, ",") + } + } + } + log.Infof("[%s] podAllocate cache complete initialization: %v", NlbNetwork, newPodAllocate) + return newCache, newPodAllocate +} + +func (c *NlbPlugin) OnPodAdded(client client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) { + return pod, nil +} + +func (c *NlbPlugin) OnPodUpdated(client client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) { + networkManager := utils.NewNetworkManager(pod, client) + + networkStatus, err := networkManager.GetNetworkStatus() + if err != nil { + return pod, cperrors.ToPluginError(err, cperrors.InternalError) + } + networkConfig := networkManager.GetNetworkConfig() + config := parseLbConfig(networkConfig) + if networkStatus == nil { + pod, err := networkManager.UpdateNetworkStatus(gamekruiseiov1alpha1.NetworkStatus{ + CurrentNetworkState: gamekruiseiov1alpha1.NetworkNotReady, + }, pod) + return pod, cperrors.ToPluginError(err, cperrors.InternalError) + } + + // get svc + svc := &corev1.Service{} + err = client.Get(ctx, types.NamespacedName{ + Name: pod.GetName(), + Namespace: pod.GetNamespace(), + }, svc) + if err != nil { + if errors.IsNotFound(err) { + return pod, cperrors.ToPluginError(client.Create(ctx, c.consSvc(config, pod, client, ctx)), cperrors.ApiCallError) + } + return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) + } + + // update svc + if util.GetHash(config) != svc.GetAnnotations()[NlbConfigHashKey] { + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error()) + } + return pod, cperrors.ToPluginError(client.Update(ctx, c.consSvc(config, pod, client, ctx)), cperrors.ApiCallError) + } + + // disable network + if networkManager.GetNetworkDisabled() && svc.Spec.Type == corev1.ServiceTypeLoadBalancer { + svc.Spec.Type = corev1.ServiceTypeClusterIP + return pod, cperrors.ToPluginError(client.Update(ctx, svc), cperrors.ApiCallError) + } + + // enable network + if !networkManager.GetNetworkDisabled() && svc.Spec.Type == corev1.ServiceTypeClusterIP { + svc.Spec.Type = corev1.ServiceTypeLoadBalancer + return pod, cperrors.ToPluginError(client.Update(ctx, svc), cperrors.ApiCallError) + } + + // network not ready + if len(svc.Status.LoadBalancer.Ingress) == 0 { + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + return pod, cperrors.ToPluginError(err, cperrors.InternalError) + } + + // allow not ready containers + if util.IsAllowNotReadyContainers(networkManager.GetNetworkConfig()) { + toUpDateSvc, err := utils.AllowNotReadyContainers(client, ctx, pod, svc, false) + if err != nil { + return pod, err + } + + if toUpDateSvc { + err := client.Update(ctx, svc) + if err != nil { + return pod, cperrors.ToPluginError(err, cperrors.ApiCallError) + } + } + } + + // network ready + internalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0) + externalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0) + for _, port := range svc.Spec.Ports { + instrIPort := port.TargetPort + instrEPort := intstr.FromInt(int(port.Port)) + internalAddress := gamekruiseiov1alpha1.NetworkAddress{ + IP: pod.Status.PodIP, + Ports: []gamekruiseiov1alpha1.NetworkPort{ + { + Name: instrIPort.String(), + Port: &instrIPort, + Protocol: port.Protocol, + }, + }, + } + externalAddress := gamekruiseiov1alpha1.NetworkAddress{ + IP: svc.Status.LoadBalancer.Ingress[0].IP, + Ports: []gamekruiseiov1alpha1.NetworkPort{ + { + Name: instrIPort.String(), + Port: &instrEPort, + Protocol: port.Protocol, + }, + }, + } + internalAddresses = append(internalAddresses, internalAddress) + externalAddresses = append(externalAddresses, externalAddress) + } + networkStatus.InternalAddresses = internalAddresses + networkStatus.ExternalAddresses = externalAddresses + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + return pod, cperrors.ToPluginError(err, cperrors.InternalError) +} + +func (c *NlbPlugin) OnPodDeleted(client client.Client, pod *corev1.Pod, ctx context.Context) cperrors.PluginError { + networkManager := utils.NewNetworkManager(pod, client) + networkConfig := networkManager.GetNetworkConfig() + sc := parseLbConfig(networkConfig) + + var podKeys []string + if sc.isFixed { + gss, err := util.GetGameServerSetOfPod(pod, client, ctx) + if err != nil && !errors.IsNotFound(err) { + return cperrors.ToPluginError(err, cperrors.ApiCallError) + } + // gss exists in cluster, do not deAllocate. + if err == nil && gss.GetDeletionTimestamp() == nil { + return nil + } + // gss not exists in cluster, deAllocate all the ports related to it. + for key := range c.podAllocate { + gssName := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey] + if strings.Contains(key, pod.GetNamespace()+"/"+gssName) { + podKeys = append(podKeys, key) + } + } + } else { + podKeys = append(podKeys, pod.GetNamespace()+"/"+pod.GetName()) + } + + for _, podKey := range podKeys { + c.deAllocate(podKey) + } + + return nil +} + +func (c *NlbPlugin) allocate(lbIds []string, num int, nsName string) (string, []int32) { + c.mutex.Lock() + defer c.mutex.Unlock() + + var ports []int32 + var lbId string + + // find lb with adequate ports + for _, nlbId := range lbIds { + sum := 0 + for i := c.minPort; i < c.maxPort; i++ { + if !c.cache[nlbId][i] { + sum++ + } + if sum >= num { + lbId = nlbId + break + } + } + } + + // select ports + for i := 0; i < num; i++ { + var port int32 + if c.cache[lbId] == nil { + c.cache[lbId] = make(portAllocated, c.maxPort-c.minPort) + for i := c.minPort; i < c.maxPort; i++ { + c.cache[lbId][i] = false + } + } + + for p, allocated := range c.cache[lbId] { + if !allocated { + port = p + break + } + } + c.cache[lbId][port] = true + ports = append(ports, port) + } + + c.podAllocate[nsName] = lbId + ":" + util.Int32SliceToString(ports, ",") + log.Infof("pod %s allocate nlb %s ports %v", nsName, lbId, ports) + return lbId, ports +} + +func (c *NlbPlugin) deAllocate(nsName string) { + c.mutex.Lock() + defer c.mutex.Unlock() + + allocatedPorts, exist := c.podAllocate[nsName] + if !exist { + return + } + + nlbPorts := strings.Split(allocatedPorts, ":") + lbId := nlbPorts[0] + ports := util.StringToInt32Slice(nlbPorts[1], ",") + for _, port := range ports { + c.cache[lbId][port] = false + } + + delete(c.podAllocate, nsName) + log.Infof("pod %s deallocate nlb %s ports %v", nsName, lbId, ports) +} + +func init() { + JdNlbPlugin := NlbPlugin{ + mutex: sync.RWMutex{}, + } + jdcloudProvider.registerPlugin(&JdNlbPlugin) +} + +func parseLbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) *nlbConfig { + var lbIds []string + ports := make([]int, 0) + protocols := make([]corev1.Protocol, 0) + isFixed := false + allocateLoadBalancerNodePorts := true + annotations := map[string]string{} + algo := string(v1.JdNLBAlgorithmRoundRobin) + idleTime := v1.JdNLBDefaultConnIdleTime + for _, c := range conf { + switch c.Name { + case NlbIdsConfigName: + for _, clbId := range strings.Split(c.Value, ",") { + if clbId != "" { + lbIds = append(lbIds, clbId) + } + } + case PortProtocolsConfigName: + for _, pp := range strings.Split(c.Value, ",") { + ppSlice := strings.Split(pp, "/") + port, err := strconv.Atoi(ppSlice[0]) + if err != nil { + continue + } + ports = append(ports, port) + if len(ppSlice) != 2 { + protocols = append(protocols, corev1.ProtocolTCP) + } else { + protocols = append(protocols, corev1.Protocol(ppSlice[1])) + } + } + case FixedConfigName: + v, err := strconv.ParseBool(c.Value) + if err != nil { + continue + } + isFixed = v + case NlbAlgorithm: + algo = c.Value + case NlbConnectionIdleTime: + t, err := strconv.Atoi(c.Value) + if err == nil { + idleTime = t + } + case AllocateLoadBalancerNodePorts: + v, err := strconv.ParseBool(c.Value) + if err != nil { + continue + } + allocateLoadBalancerNodePorts = v + case NlbAnnotations: + for _, anno := range strings.Split(c.Value, ",") { + annoKV := strings.Split(anno, ":") + if len(annoKV) == 2 { + annotations[annoKV[0]] = annoKV[1] + } else { + log.Warningf("nlb annotation %s is invalid", annoKV[0]) + } + } + } + } + return &nlbConfig{ + lbIds: lbIds, + protocols: protocols, + targetPorts: ports, + isFixed: isFixed, + annotations: annotations, + allocateLoadBalancerNodePorts: allocateLoadBalancerNodePorts, + algorithm: algo, + connIdleTimeSeconds: idleTime, + } +} + +func getPorts(ports []corev1.ServicePort) []int32 { + var ret []int32 + for _, port := range ports { + ret = append(ret, port.Port) + } + return ret +} + +func (c *NlbPlugin) consSvc(config *nlbConfig, pod *corev1.Pod, client client.Client, ctx context.Context) *corev1.Service { + var ports []int32 + var lbId string + podKey := pod.GetNamespace() + "/" + pod.GetName() + allocatedPorts, exist := c.podAllocate[podKey] + if exist { + nlbPorts := strings.Split(allocatedPorts, ":") + lbId = nlbPorts[0] + ports = util.StringToInt32Slice(nlbPorts[1], ",") + } else { + lbId, ports = c.allocate(config.lbIds, len(config.targetPorts), podKey) + } + + svcPorts := make([]corev1.ServicePort, 0) + for i := 0; i < len(config.targetPorts); i++ { + svcPorts = append(svcPorts, corev1.ServicePort{ + Name: strconv.Itoa(config.targetPorts[i]), + Port: ports[i], + Protocol: config.protocols[i], + TargetPort: intstr.FromInt(config.targetPorts[i]), + }) + } + + annotations := map[string]string{ + NlbIdLabelKey: lbId, + NlbSpecAnnotationKey: getNLBSpec(svcPorts, lbId, config.algorithm, config.connIdleTimeSeconds), + NlbConfigHashKey: util.GetHash(config), + } + for key, value := range config.annotations { + annotations[key] = value + } + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: pod.GetName(), + Namespace: pod.GetNamespace(), + Annotations: annotations, + OwnerReferences: getSvcOwnerReference(client, ctx, pod, config.isFixed), + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Selector: map[string]string{ + SvcSelectorKey: pod.GetName(), + }, + Ports: svcPorts, + AllocateLoadBalancerNodePorts: ptr.To[bool](config.allocateLoadBalancerNodePorts), + }, + } + return svc +} + +func getNLBSpec(ports []corev1.ServicePort, lbId, algorithm string, connIdleTimeSeconds int) string { + jdNlb := _getNLBSpec(ports, lbId, algorithm, connIdleTimeSeconds) + bytes, err := json.Marshal(jdNlb) + if err != nil { + return "" + } + return string(bytes) +} + +func _getNLBSpec(ports []corev1.ServicePort, lbId, algorithm string, connIdleTimeSeconds int) *v1.JdNLB { + var listeners = make([]*v1.JdNLBListener, len(ports)) + for k, v := range ports { + listeners[k] = &v1.JdNLBListener{ + Protocol: strings.ToUpper(string(v.Protocol)), + ConnectionIdleTimeSeconds: connIdleTimeSeconds, + Backend: &v1.JdNLBListenerBackend{ + Algorithm: v1.JdNLBAlgorithm(algorithm), + }, + } + } + return &v1.JdNLB{ + Version: JdNLBVersion, + LoadBalancerId: lbId, + LoadBalancerType: v1.LbType_NLB, + Internal: false, + Listeners: listeners, + } +} + +func getSvcOwnerReference(c client.Client, ctx context.Context, pod *corev1.Pod, isFixed bool) []metav1.OwnerReference { + ownerReferences := []metav1.OwnerReference{ + { + APIVersion: pod.APIVersion, + Kind: pod.Kind, + Name: pod.GetName(), + UID: pod.GetUID(), + Controller: ptr.To[bool](true), + BlockOwnerDeletion: ptr.To[bool](true), + }, + } + if isFixed { + gss, err := util.GetGameServerSetOfPod(pod, c, ctx) + if err == nil { + ownerReferences = []metav1.OwnerReference{ + { + APIVersion: gss.APIVersion, + Kind: gss.Kind, + Name: gss.GetName(), + UID: gss.GetUID(), + Controller: ptr.To[bool](true), + BlockOwnerDeletion: ptr.To[bool](true), + }, + } + } + } + return ownerReferences +} diff --git a/cloudprovider/jdcloud/nlb_test.go b/cloudprovider/jdcloud/nlb_test.go new file mode 100644 index 0000000..8141290 --- /dev/null +++ b/cloudprovider/jdcloud/nlb_test.go @@ -0,0 +1,344 @@ +/* +Copyright 2024 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jdcloud + +import ( + "context" + "k8s.io/utils/ptr" + "reflect" + "sync" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/pkg/util" +) + +func TestAllocateDeAllocate(t *testing.T) { + test := struct { + lbIds []string + nlb *NlbPlugin + num int + podKey string + }{ + lbIds: []string{"xxx-A"}, + nlb: &NlbPlugin{ + maxPort: int32(700), + minPort: int32(500), + cache: make(map[string]portAllocated), + podAllocate: make(map[string]string), + mutex: sync.RWMutex{}, + }, + podKey: "xxx/xxx", + num: 3, + } + + lbId, ports := test.nlb.allocate(test.lbIds, test.num, test.podKey) + if _, exist := test.nlb.podAllocate[test.podKey]; !exist { + t.Errorf("podAllocate[%s] is empty after allocated", test.podKey) + } + for _, port := range ports { + if port > test.nlb.maxPort || port < test.nlb.minPort { + t.Errorf("allocate port %d, unexpected", port) + } + if test.nlb.cache[lbId][port] == false { + t.Errorf("Allocate port %d failed", port) + } + } + + test.nlb.deAllocate(test.podKey) + for _, port := range ports { + if test.nlb.cache[lbId][port] == true { + t.Errorf("deAllocate port %d failed", port) + } + } + if _, exist := test.nlb.podAllocate[test.podKey]; exist { + t.Errorf("podAllocate[%s] is not empty after deallocated", test.podKey) + } +} + +func TestParseLbConfig(t *testing.T) { + tests := []struct { + conf []gamekruiseiov1alpha1.NetworkConfParams + lbIds []string + ports []int + protocols []corev1.Protocol + isFixed bool + }{ + { + conf: []gamekruiseiov1alpha1.NetworkConfParams{ + { + Name: NlbIdsConfigName, + Value: "xxx-A", + }, + { + Name: PortProtocolsConfigName, + Value: "80", + }, + }, + lbIds: []string{"xxx-A"}, + ports: []int{80}, + protocols: []corev1.Protocol{corev1.ProtocolTCP}, + isFixed: false, + }, + { + conf: []gamekruiseiov1alpha1.NetworkConfParams{ + { + Name: NlbIdsConfigName, + Value: "xxx-A,xxx-B,", + }, + { + Name: PortProtocolsConfigName, + Value: "81/UDP,82,83/TCP", + }, + { + Name: FixedConfigName, + Value: "true", + }, + }, + lbIds: []string{"xxx-A", "xxx-B"}, + ports: []int{81, 82, 83}, + protocols: []corev1.Protocol{corev1.ProtocolUDP, corev1.ProtocolTCP, corev1.ProtocolTCP}, + isFixed: true, + }, + } + + for _, test := range tests { + sc := parseLbConfig(test.conf) + if !reflect.DeepEqual(test.lbIds, sc.lbIds) { + t.Errorf("lbId expect: %v, actual: %v", test.lbIds, sc.lbIds) + } + if !util.IsSliceEqual(test.ports, sc.targetPorts) { + t.Errorf("ports expect: %v, actual: %v", test.ports, sc.targetPorts) + } + if !reflect.DeepEqual(test.protocols, sc.protocols) { + t.Errorf("protocols expect: %v, actual: %v", test.protocols, sc.protocols) + } + if test.isFixed != sc.isFixed { + t.Errorf("isFixed expect: %v, actual: %v", test.isFixed, sc.isFixed) + } + } +} + +func TestInitLbCache(t *testing.T) { + test := struct { + svcList []corev1.Service + minPort int32 + maxPort int32 + cache map[string]portAllocated + podAllocate map[string]string + }{ + minPort: 500, + maxPort: 700, + cache: map[string]portAllocated{ + "xxx-A": map[int32]bool{ + 666: true, + }, + "xxx-B": map[int32]bool{ + 555: true, + }, + }, + podAllocate: map[string]string{ + "ns-0/name-0": "xxx-A:666", + "ns-1/name-1": "xxx-B:555", + }, + svcList: []corev1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + NlbIdLabelKey: "xxx-A", + }, + Namespace: "ns-0", + Name: "name-0", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Selector: map[string]string{ + SvcSelectorKey: "pod-A", + }, + Ports: []corev1.ServicePort{ + { + TargetPort: intstr.FromInt(80), + Port: 666, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + NlbIdLabelKey: "xxx-B", + }, + Namespace: "ns-1", + Name: "name-1", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Selector: map[string]string{ + SvcSelectorKey: "pod-B", + }, + Ports: []corev1.ServicePort{ + { + TargetPort: intstr.FromInt(8080), + Port: 555, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + }, + } + + actualCache, actualPodAllocate := initLbCache(test.svcList, test.minPort, test.maxPort) + for lb, pa := range test.cache { + for port, isAllocated := range pa { + if actualCache[lb][port] != isAllocated { + t.Errorf("lb %s port %d isAllocated, expect: %t, actual: %t", lb, port, isAllocated, actualCache[lb][port]) + } + } + } + if !reflect.DeepEqual(actualPodAllocate, test.podAllocate) { + t.Errorf("podAllocate expect %v, but actully got %v", test.podAllocate, actualPodAllocate) + } +} + +func TestNlbPlugin_consSvc(t *testing.T) { + type fields struct { + maxPort int32 + minPort int32 + cache map[string]portAllocated + podAllocate map[string]string + } + type args struct { + config *nlbConfig + pod *corev1.Pod + client client.Client + ctx context.Context + } + tests := []struct { + name string + fields fields + args args + want *corev1.Service + }{ + { + name: "convert svc cache exist", + fields: fields{ + maxPort: 3000, + minPort: 1, + cache: map[string]portAllocated{ + "default/test-pod": map[int32]bool{}, + }, + podAllocate: map[string]string{ + "default/test-pod": "nlb-xxx:80,81", + }, + }, + args: args{ + config: &nlbConfig{ + lbIds: []string{"nlb-xxx"}, + targetPorts: []int{82}, + protocols: []corev1.Protocol{ + corev1.ProtocolTCP, + }, + isFixed: false, + annotations: map[string]string{ + "service.beta.kubernetes.io/jdcloud-load-balancer-spec": "{}", + }, + allocateLoadBalancerNodePorts: true, + }, + pod: &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + UID: "32fqwfqfew", + }, + }, + client: nil, + ctx: context.Background(), + }, + want: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + NlbConfigHashKey: util.GetHash(&nlbConfig{ + lbIds: []string{"nlb-xxx"}, + targetPorts: []int{82}, + protocols: []corev1.Protocol{ + corev1.ProtocolTCP, + }, + isFixed: false, + annotations: map[string]string{ + "service.beta.kubernetes.io/jdcloud-load-balancer-spec": "{}", + }, + allocateLoadBalancerNodePorts: true, + }), + "service.beta.kubernetes.io/jdcloud-load-balancer-spec": "{}", + "service.beta.kubernetes.io/jdcloud-loadbalancer-id": "nlb-xxx", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "pod", + Name: "test-pod", + UID: "32fqwfqfew", + Controller: ptr.To[bool](true), + BlockOwnerDeletion: ptr.To[bool](true), + }, + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Selector: map[string]string{ + SvcSelectorKey: "test-pod", + }, + Ports: []corev1.ServicePort{{ + Name: "82", + Port: 80, + Protocol: "TCP", + TargetPort: intstr.IntOrString{ + Type: 0, + IntVal: 82, + }, + }, + }, + AllocateLoadBalancerNodePorts: ptr.To[bool](true), + }, + }, + }, + } + for _, tt := range tests { + c := &NlbPlugin{ + maxPort: tt.fields.maxPort, + minPort: tt.fields.minPort, + cache: tt.fields.cache, + podAllocate: tt.fields.podAllocate, + } + if got := c.consSvc(tt.args.config, tt.args.pod, tt.args.client, tt.args.ctx); !reflect.DeepEqual(got, tt.want) { + t.Errorf("consSvc() = %v, want %v", got, tt.want) + } + } +} diff --git a/cloudprovider/manager/provider_manager.go b/cloudprovider/manager/provider_manager.go index 9806d66..2c18086 100644 --- a/cloudprovider/manager/provider_manager.go +++ b/cloudprovider/manager/provider_manager.go @@ -18,6 +18,7 @@ package manager import ( "context" + "github.com/openkruise/kruise-game/cloudprovider/jdcloud" "github.com/openkruise/kruise-game/apis/v1alpha1" "github.com/openkruise/kruise-game/cloudprovider" @@ -150,5 +151,15 @@ func NewProviderManager() (*ProviderManager, error) { } } + if configs.JdCloudOptions.Valid() && configs.JdCloudOptions.Enabled() { + // build and register tencent cloud provider + tcp, err := jdcloud.NewJdcloudProvider() + if err != nil { + log.Errorf("Failed to initialize jdcloud provider.because of %s", err.Error()) + } else { + pm.RegisterCloudProvider(tcp, configs.JdCloudOptions) + } + } + return pm, nil } diff --git a/cloudprovider/options/jdcloud_options.go b/cloudprovider/options/jdcloud_options.go new file mode 100644 index 0000000..144e1fa --- /dev/null +++ b/cloudprovider/options/jdcloud_options.go @@ -0,0 +1,28 @@ +package options + +type JdCloudOptions struct { + Enable bool `toml:"enable"` + NLBOptions JdNLBOptions `toml:"nlb"` +} + +type JdNLBOptions struct { + MaxPort int32 `toml:"max_port"` + MinPort int32 `toml:"min_port"` +} + +func (v JdCloudOptions) Valid() bool { + nlbOptions := v.NLBOptions + + if nlbOptions.MaxPort > 65535 { + return false + } + + if nlbOptions.MinPort < 1 { + return false + } + return true +} + +func (v JdCloudOptions) Enabled() bool { + return v.Enable +} diff --git a/config/manager/config.toml b/config/manager/config.toml index 93b3b1e..3091f6c 100644 --- a/config/manager/config.toml +++ b/config/manager/config.toml @@ -26,3 +26,10 @@ enable = false [aws.nlb] max_port = 30050 min_port = 30001 + +[jdcloud] +enable = false +[jdcloud.nlb] +max_port = 700 +min_port = 500 + From 942002f594e658713af40a6d02eb87b4893b721d Mon Sep 17 00:00:00 2001 From: hhr <691129301@qq.com> Date: Tue, 5 Nov 2024 16:34:42 +0800 Subject: [PATCH 2/2] fix: fix eip networkConf & move jdnlb_types to nlb --- cloudprovider/jdcloud/README.md | 41 +++++---------- cloudprovider/jdcloud/README.zh_CN.md | 43 +++++---------- cloudprovider/jdcloud/apis/v1/doc.go | 1 - cloudprovider/jdcloud/apis/v1/jdnlb_types.go | 37 ------------- cloudprovider/jdcloud/eip.go | 40 ++++++++------ cloudprovider/jdcloud/nlb.go | 55 ++++++++++++++++---- 6 files changed, 95 insertions(+), 122 deletions(-) delete mode 100644 cloudprovider/jdcloud/apis/v1/doc.go delete mode 100644 cloudprovider/jdcloud/apis/v1/jdnlb_types.go diff --git a/cloudprovider/jdcloud/README.md b/cloudprovider/jdcloud/README.md index 386e0fb..d1a2a09 100644 --- a/cloudprovider/jdcloud/README.md +++ b/cloudprovider/jdcloud/README.md @@ -124,59 +124,37 @@ JdCloud Container Service supports binding an Elastic Public IP directly to a po - Install the EIP-Controller component. - The Elastic Public IP will not be deleted when the pod is destroyed. -### Annotation Parameter +### Parameter -#### EnableEIPAnnotationKey -- Meaning：Whether to enable Elastic Public IP -- Value：string，"true"/"false" - -#### BandwidthAnnotationkey +#### BandwidthConfigName - Meaning：The bandwidth of the Elastic Public IP, measured in Mbps, has a value range of [1, 1024]. - Value：Must be an integer - Configurable：Y -#### ChargeTypeConfigAnnotationkey +#### ChargeTypeConfigName - Meaning：The billing method for the Elastic Public IP - Value：string, `postpaid_by_usage`/`postpaid_by_duration` - Configurable：Y -#### FixedEIPAnnotationKey +#### FixedEIPConfigName - Meaning：Whether to fixed the Elastic Public IP,if so, the EIP will not be changed when the pod is recreated. - Value：string, "false" / "true" - Configurable：Y -#### AssignEIPAnnotationKey +#### AssignEIPConfigName - Meaning：Whether to designate a specific Elastic Public IP. If true, provide the ID of the Elastic Public IP; otherwise, an EIP will be automatically allocated. - Value：string, "false" / "true" -#### EIPIdAnnotationKey +#### EIPIdConfigName - Meaning：If a specific Elastic Public IP is designated, the ID of the Elastic Public IP must be provided, and the component will automatically perform the lookup and binding. - Value：string，for example：`fip-xxxxxxxx` -Note that when creating a Gss with the JdCloud-EIP network type, the above configuration must be added to the gss annotations, not in the NetworkConf. Otherwise, the EIP resource will not be created properly. - -```YAML -apiVersion: game.kruise.io/v1alpha1 -kind: GameServerSet -metadata: - annotations: - jdos.jd.com/eip.bandwith: "10" - jdos.jd.com/eip.chargeMode: postpaid_by_usage - jdos.jd.com/eip.enable: "true" - jdos.jd.com/eip.static: "true" -``` - ### Example ```yaml cat <