From 270f0dcb9cb507de27d21a8258387c75d07c29cf Mon Sep 17 00:00:00 2001 From: rockerchen Date: Mon, 28 Oct 2024 10:04:58 +0800 Subject: [PATCH] feat(*): add tencent cloud provider and clb plugin Signed-off-by: rockerchen --- apis/v1alpha1/gameserverset_types.go | 1 + cloudprovider/config.go | 8 +- cloudprovider/manager/provider_manager.go | 12 + cloudprovider/options/tencentcloud_options.go | 29 ++ .../v1alpha1/dedicatedclblistener_types.go | 93 ++++ .../tencentcloud/apis/v1alpha1/doc.go | 4 + .../tencentcloud/apis/v1alpha1/register.go | 38 ++ .../apis/v1alpha1/zz_generated.deepcopy.go | 135 ++++++ cloudprovider/tencentcloud/clb.go | 432 ++++++++++++++++++ cloudprovider/tencentcloud/clb_test.go | 190 ++++++++ cloudprovider/tencentcloud/tencentcloud.go | 59 +++ ...oud.tencent.com_dedicatedclblisteners.yaml | 121 +++++ docs/en/user_manuals/network.md | 105 ++++- ...21\347\273\234\346\250\241\345\236\213.md" | 105 ++++- main.go | 4 +- 15 files changed, 1330 insertions(+), 6 deletions(-) create mode 100644 cloudprovider/options/tencentcloud_options.go create mode 100644 cloudprovider/tencentcloud/apis/v1alpha1/dedicatedclblistener_types.go create mode 100644 cloudprovider/tencentcloud/apis/v1alpha1/doc.go create mode 100644 cloudprovider/tencentcloud/apis/v1alpha1/register.go create mode 100644 cloudprovider/tencentcloud/apis/v1alpha1/zz_generated.deepcopy.go create mode 100644 cloudprovider/tencentcloud/clb.go create mode 100644 cloudprovider/tencentcloud/clb_test.go create mode 100644 cloudprovider/tencentcloud/tencentcloud.go create mode 100644 config/crd/bases/networking.cloud.tencent.com_dedicatedclblisteners.yaml diff --git a/apis/v1alpha1/gameserverset_types.go b/apis/v1alpha1/gameserverset_types.go index 052d397f..014d3745 100644 --- a/apis/v1alpha1/gameserverset_types.go +++ b/apis/v1alpha1/gameserverset_types.go @@ -30,6 +30,7 @@ import ( const ( GameServerOwnerGssKey = "game.kruise.io/owner-gss" + OwnerPodKey = "game.kruise.io/owner-pod" GameServerSetReserveIdsKey = "game.kruise.io/reserve-ids" AstsHashKey = "game.kruise.io/asts-hash" PpmHashKey = "game.kruise.io/ppm-hash" diff --git a/cloudprovider/config.go b/cloudprovider/config.go index cb63b4f7..abd54969 100644 --- a/cloudprovider/config.go +++ b/cloudprovider/config.go @@ -17,13 +17,13 @@ limitations under the License. package cloudprovider import ( + "flag" + "github.com/BurntSushi/toml" "github.com/openkruise/kruise-game/cloudprovider/options" "k8s.io/klog/v2" ) -import "flag" - var Opt *Options type Options struct { @@ -47,6 +47,7 @@ type CloudProviderConfig struct { AlibabaCloudOptions CloudProviderOptions VolcengineOptions CloudProviderOptions AmazonsWebServicesOptions CloudProviderOptions + TencentCloudOptions CloudProviderOptions } type tomlConfigs struct { @@ -54,10 +55,10 @@ type tomlConfigs struct { AlibabaCloud options.AlibabaCloudOptions `toml:"alibabacloud"` Volcengine options.VolcengineOptions `toml:"volcengine"` AmazonsWebServices options.AmazonsWebServicesOptions `toml:"aws"` + TencentCloud options.TencentCloudOptions `toml:"tencentcloud"` } func (cf *ConfigFile) Parse() *CloudProviderConfig { - var config tomlConfigs if _, err := toml.DecodeFile(cf.Path, &config); err != nil { klog.Fatal(err) @@ -68,6 +69,7 @@ func (cf *ConfigFile) Parse() *CloudProviderConfig { AlibabaCloudOptions: config.AlibabaCloud, VolcengineOptions: config.Volcengine, AmazonsWebServicesOptions: config.AmazonsWebServices, + TencentCloudOptions: config.TencentCloud, } } diff --git a/cloudprovider/manager/provider_manager.go b/cloudprovider/manager/provider_manager.go index 87298d53..45182cdd 100644 --- a/cloudprovider/manager/provider_manager.go +++ b/cloudprovider/manager/provider_manager.go @@ -18,11 +18,13 @@ package manager import ( "context" + "github.com/openkruise/kruise-game/apis/v1alpha1" "github.com/openkruise/kruise-game/cloudprovider" "github.com/openkruise/kruise-game/cloudprovider/alibabacloud" aws "github.com/openkruise/kruise-game/cloudprovider/amazonswebservices" "github.com/openkruise/kruise-game/cloudprovider/kubernetes" + "github.com/openkruise/kruise-game/cloudprovider/tencentcloud" volcengine "github.com/openkruise/kruise-game/cloudprovider/volcengine" corev1 "k8s.io/api/core/v1" log "k8s.io/klog/v2" @@ -138,5 +140,15 @@ func NewProviderManager() (*ProviderManager, error) { } } + if configs.TencentCloudOptions.Valid() && configs.TencentCloudOptions.Enabled() { + // build and register amazon web services provider + tcp, err := tencentcloud.NewTencentCloudProvider() + if err != nil { + log.Errorf("Failed to initialize tencentcloud provider.because of %s", err.Error()) + } else { + pm.RegisterCloudProvider(tcp, configs.TencentCloudOptions) + } + } + return pm, nil } diff --git a/cloudprovider/options/tencentcloud_options.go b/cloudprovider/options/tencentcloud_options.go new file mode 100644 index 00000000..e0ff8cd3 --- /dev/null +++ b/cloudprovider/options/tencentcloud_options.go @@ -0,0 +1,29 @@ +package options + +type TencentCloudOptions struct { + Enable bool `toml:"enable"` + CLBOptions TencentCloudCLBOptions `toml:"clb"` +} + +type TencentCloudCLBOptions struct { + MaxPort int32 `toml:"max_port"` + MinPort int32 `toml:"min_port"` +} + +func (o TencentCloudOptions) Valid() bool { + clbOptions := o.CLBOptions + + if clbOptions.MaxPort > 65535 { + return false + } + + if clbOptions.MinPort < 1 { + return false + } + + return true +} + +func (o TencentCloudOptions) Enabled() bool { + return o.Enable +} diff --git a/cloudprovider/tencentcloud/apis/v1alpha1/dedicatedclblistener_types.go b/cloudprovider/tencentcloud/apis/v1alpha1/dedicatedclblistener_types.go new file mode 100644 index 00000000..fd390bf1 --- /dev/null +++ b/cloudprovider/tencentcloud/apis/v1alpha1/dedicatedclblistener_types.go @@ -0,0 +1,93 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// DedicatedCLBListenerSpec defines the desired state of DedicatedCLBListener +type DedicatedCLBListenerSpec struct { + // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable" + LbId string `json:"lbId"` + // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable" + // +optional + LbRegion string `json:"lbRegion,omitempty"` + // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable" + LbPort int64 `json:"lbPort"` + // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable" + // +kubebuilder:validation:Enum=TCP;UDP + Protocol string `json:"protocol"` + // +optional + ExtensiveParameters string `json:"extensiveParameters,omitempty"` + // +optional + TargetPod *TargetPod `json:"targetPod,omitempty"` +} + +type TargetPod struct { + PodName string `json:"podName"` + TargetPort int64 `json:"targetPort"` +} + +// DedicatedCLBListenerStatus defines the observed state of DedicatedCLBListener +type DedicatedCLBListenerStatus struct { + ListenerId string `json:"listenerId,omitempty"` + // +kubebuilder:validation:Enum=Bound;Available;Pending;Failed;Deleting + State string `json:"state,omitempty"` + Message string `json:"message,omitempty"` + Address string `json:"address,omitempty"` +} + +const ( + DedicatedCLBListenerStateBound = "Bound" + DedicatedCLBListenerStateAvailable = "Available" + DedicatedCLBListenerStatePending = "Pending" + DedicatedCLBListenerStateFailed = "Failed" + DedicatedCLBListenerStateDeleting = "Deleting" +) + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="LbId",type="string",JSONPath=".spec.lbId",description="CLB ID" +// +kubebuilder:printcolumn:name="LbPort",type="integer",JSONPath=".spec.lbPort",description="Port of CLB Listener" +// +kubebuilder:printcolumn:name="Pod",type="string",JSONPath=".spec.targetPod.podName",description="Pod name of target pod" +// +kubebuilder:printcolumn:name="State",type="string",JSONPath=".status.state",description="State of the dedicated clb listener" + +// DedicatedCLBListener is the Schema for the dedicatedclblisteners API +type DedicatedCLBListener struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec DedicatedCLBListenerSpec `json:"spec,omitempty"` + Status DedicatedCLBListenerStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// DedicatedCLBListenerList contains a list of DedicatedCLBListener +type DedicatedCLBListenerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []DedicatedCLBListener `json:"items"` +} + +func init() { + SchemeBuilder.Register(&DedicatedCLBListener{}, &DedicatedCLBListenerList{}) +} diff --git a/cloudprovider/tencentcloud/apis/v1alpha1/doc.go b/cloudprovider/tencentcloud/apis/v1alpha1/doc.go new file mode 100644 index 00000000..ac49e5db --- /dev/null +++ b/cloudprovider/tencentcloud/apis/v1alpha1/doc.go @@ -0,0 +1,4 @@ +// Package v1alpha1 contains API Schema definitions for the tencentcloud v1alpha1 API group +// +k8s:deepcopy-gen=package,register +// +groupName=networking.cloud.tencent.com +package v1alpha1 diff --git a/cloudprovider/tencentcloud/apis/v1alpha1/register.go b/cloudprovider/tencentcloud/apis/v1alpha1/register.go new file mode 100644 index 00000000..86e84450 --- /dev/null +++ b/cloudprovider/tencentcloud/apis/v1alpha1/register.go @@ -0,0 +1,38 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the networking v1alpha1 API group +// +kubebuilder:validation:Required +// +kubebuilder:object:generate=true +// +groupName=networking.cloud.tencent.com + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "networking.cloud.tencent.com", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/cloudprovider/tencentcloud/apis/v1alpha1/zz_generated.deepcopy.go b/cloudprovider/tencentcloud/apis/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 00000000..a8bfca79 --- /dev/null +++ b/cloudprovider/tencentcloud/apis/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,135 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DedicatedCLBListener) DeepCopyInto(out *DedicatedCLBListener) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListener. +func (in *DedicatedCLBListener) DeepCopy() *DedicatedCLBListener { + if in == nil { + return nil + } + out := new(DedicatedCLBListener) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DedicatedCLBListener) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DedicatedCLBListenerList) DeepCopyInto(out *DedicatedCLBListenerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]DedicatedCLBListener, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListenerList. +func (in *DedicatedCLBListenerList) DeepCopy() *DedicatedCLBListenerList { + if in == nil { + return nil + } + out := new(DedicatedCLBListenerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DedicatedCLBListenerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DedicatedCLBListenerSpec) DeepCopyInto(out *DedicatedCLBListenerSpec) { + *out = *in + if in.TargetPod != nil { + in, out := &in.TargetPod, &out.TargetPod + *out = new(TargetPod) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListenerSpec. +func (in *DedicatedCLBListenerSpec) DeepCopy() *DedicatedCLBListenerSpec { + if in == nil { + return nil + } + out := new(DedicatedCLBListenerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DedicatedCLBListenerStatus) DeepCopyInto(out *DedicatedCLBListenerStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListenerStatus. +func (in *DedicatedCLBListenerStatus) DeepCopy() *DedicatedCLBListenerStatus { + if in == nil { + return nil + } + out := new(DedicatedCLBListenerStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TargetPod) DeepCopyInto(out *TargetPod) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetPod. +func (in *TargetPod) DeepCopy() *TargetPod { + if in == nil { + return nil + } + out := new(TargetPod) + in.DeepCopyInto(out) + return out +} diff --git a/cloudprovider/tencentcloud/clb.go b/cloudprovider/tencentcloud/clb.go new file mode 100644 index 00000000..c5929e20 --- /dev/null +++ b/cloudprovider/tencentcloud/clb.go @@ -0,0 +1,432 @@ +package tencentcloud + +import ( + "context" + "fmt" + "reflect" + "slices" + "strconv" + "strings" + "sync" + + kruisev1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/cloudprovider" + cperrors "github.com/openkruise/kruise-game/cloudprovider/errors" + provideroptions "github.com/openkruise/kruise-game/cloudprovider/options" + "github.com/openkruise/kruise-game/cloudprovider/tencentcloud/apis/v1alpha1" + "github.com/openkruise/kruise-game/cloudprovider/utils" + "github.com/openkruise/kruise-game/pkg/util" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + log "k8s.io/klog/v2" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + ClbNetwork = "TencentCloud-CLB" + AliasCLB = "CLB-Network" + ClbIdsConfigName = "ClbIds" + PortProtocolsConfigName = "PortProtocols" + MinPortConfigName = "MinPort" + MaxPortConfigName = "MaxPort" +) + +type portAllocated map[int32]bool + +type ClbPlugin struct { + maxPort int32 + minPort int32 + cache map[string]portAllocated + podAllocate map[string][]string + mutex sync.RWMutex +} + +type portProtocol struct { + port int + protocol string +} + +type clbConfig struct { + lbIds []string + targetPorts []portProtocol +} + +func (p *ClbPlugin) Name() string { + return ClbNetwork +} + +func (p *ClbPlugin) Alias() string { + return AliasCLB +} + +func (p *ClbPlugin) Init(c client.Client, options cloudprovider.CloudProviderOptions, ctx context.Context) error { + p.mutex.Lock() + defer p.mutex.Unlock() + clbOptions := options.(provideroptions.TencentCloudOptions).CLBOptions + p.minPort = clbOptions.MinPort + p.maxPort = clbOptions.MaxPort + + listenerList := &v1alpha1.DedicatedCLBListenerList{} + err := c.List(ctx, listenerList) + if err != nil { + return err + } + p.cache, p.podAllocate = initLbCache(listenerList.Items, p.minPort, p.maxPort) + log.Infof("[%s] podAllocate cache complete initialization: %v", ClbNetwork, p.podAllocate) + return nil +} + +func initLbCache(listenerList []v1alpha1.DedicatedCLBListener, minPort, maxPort int32) (map[string]portAllocated, map[string][]string) { + newCache := make(map[string]portAllocated) + newPodAllocate := make(map[string][]string) + for _, lis := range listenerList { + podName, exist := lis.GetLabels()[kruisev1alpha1.OwnerPodKey] + if !exist || podName == "" { + continue + } + if lis.Spec.LbPort > int64(maxPort) || lis.Spec.LbPort < int64(minPort) { + continue + } + lbId := lis.Spec.LbId + if newCache[lbId] == nil { + newCache[lbId] = make(portAllocated, maxPort-minPort) + for i := minPort; i < maxPort; i++ { + newCache[lbId][i] = false + } + } + newCache[lbId][int32(lis.Spec.LbPort)] = true + podKey := lis.GetNamespace() + "/" + podName + newPodAllocate[podKey] = append(newPodAllocate[podKey], fmt.Sprintf("%s:%d", lbId, lis.Spec.LbPort)) + } + return newCache, newPodAllocate +} + +func (p *ClbPlugin) OnPodAdded(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) { + return pod, nil +} + +func (p *ClbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) { + if pod.DeletionTimestamp != nil { + return pod, nil + } + networkManager := utils.NewNetworkManager(pod, c) + networkStatus, _ := networkManager.GetNetworkStatus() + if networkStatus == nil { + pod, err := networkManager.UpdateNetworkStatus(kruisev1alpha1.NetworkStatus{ + CurrentNetworkState: kruisev1alpha1.NetworkNotReady, + }, pod) + return pod, cperrors.ToPluginError(err, cperrors.InternalError) + } + networkConfig := networkManager.GetNetworkConfig() + clbConf, err := parseLbConfig(networkConfig) + if err != nil { + return pod, cperrors.ToPluginError(err, cperrors.ParameterError) + } + + gss, err := util.GetGameServerSetOfPod(pod, c, ctx) + if err != nil && !errors.IsNotFound(err) { + return pod, cperrors.ToPluginError(err, cperrors.ApiCallError) + } + + // get related dedicated clb listeners + listeners := &v1alpha1.DedicatedCLBListenerList{} + if err := c.List( + ctx, listeners, + client.InNamespace(pod.Namespace), + client.MatchingLabels{ + kruisev1alpha1.OwnerPodKey: pod.Name, + kruisev1alpha1.GameServerOwnerGssKey: gss.Name, + }, + ); err != nil { + return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) + } + + // reconcile + lisMap := make(map[portProtocol]v1alpha1.DedicatedCLBListener) + + for _, lis := range listeners.Items { + // ignore deleting dedicated clb listener + if lis.DeletionTimestamp != nil { + continue + } + // old dedicated clb listener remain + if lis.OwnerReferences[0].Kind == "Pod" && lis.OwnerReferences[0].UID != pod.UID { + log.Infof("[%s] waitting old dedicated clb listener %s/%s deleted. old owner pod uid is %s, but now is %s", ClbNetwork, lis.Namespace, lis.Name, lis.OwnerReferences[0].UID, pod.UID) + return pod, nil + } + + // delete dedicated clb listener if necessary (target pod not match) + targetPod := lis.Spec.TargetPod + if targetPod == nil || targetPod.PodName != pod.Name { + err := c.Delete(ctx, &lis) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) + } + continue + } + port := portProtocol{ + port: int(targetPod.TargetPort), + protocol: lis.Spec.Protocol, + } + lisMap[port] = lis + } + + internalAddresses := make([]kruisev1alpha1.NetworkAddress, 0) + externalAddresses := make([]kruisev1alpha1.NetworkAddress, 0) + + for _, port := range clbConf.targetPorts { + if lis, ok := lisMap[port]; !ok { // no dedicated clb listener, try to create one + // ensure not ready while creating the listener + networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error()) + } + // allocate and create new listener bound to pod + newLis, err := p.consLis(clbConf, pod, port, gss.Name) + if err != nil { + return pod, cperrors.ToPluginError(err, cperrors.InternalError) + } + err = c.Create(ctx, newLis) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) + } + } else { // already created dedicated clb listener bound to pod + delete(lisMap, port) + // recreate dedicated clb listener if necessary (config changed) + if !slices.Contains(clbConf.lbIds, lis.Spec.LbId) || lis.Spec.LbPort > int64(p.maxPort) || lis.Spec.LbPort < int64(p.minPort) || lis.Spec.Protocol != port.protocol || lis.Spec.TargetPod.TargetPort != int64(port.port) { + // ensure not ready while recreating the listener + networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error()) + } + + // delete old listener + err := c.Delete(ctx, &lis) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) + } + + // allocate and create new listener bound to pod + newLis, err := p.consLis(clbConf, pod, port, gss.Name) + if err != nil { + return pod, cperrors.ToPluginError(err, cperrors.InternalError) + } + err = c.Create(ctx, newLis) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) + } + } else { // dedicated clb listener is desired, check status + if lis.Status.State == v1alpha1.DedicatedCLBListenerStateBound && lis.Status.Address != "" { // network ready + ss := strings.Split(lis.Status.Address, ":") + if len(ss) != 2 { + return pod, cperrors.NewPluginError(cperrors.InternalError, fmt.Sprintf("invalid dedicated clb listener address %s", lis.Status.Address)) + } + lbPort, err := strconv.Atoi(ss[1]) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.InternalError, fmt.Sprintf("invalid dedicated clb listener port %s", ss[1])) + } + instrIPort := intstr.FromInt(int(port.port)) + instrEPort := intstr.FromInt(lbPort) + internalAddresses = append(internalAddresses, kruisev1alpha1.NetworkAddress{ + IP: pod.Status.PodIP, + Ports: []kruisev1alpha1.NetworkPort{ + { + Name: instrIPort.String(), + Port: &instrIPort, + Protocol: corev1.Protocol(port.protocol), + }, + }, + }) + externalAddresses = append(externalAddresses, kruisev1alpha1.NetworkAddress{ + IP: ss[0], + Ports: []kruisev1alpha1.NetworkPort{ + { + Name: instrIPort.String(), + Port: &instrEPort, + Protocol: corev1.Protocol(port.protocol), + }, + }, + }) + } else { // network not ready + networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error()) + } + } + } + } + } + + // other dedicated clb listener is not used, delete it + for _, lis := range lisMap { + err := c.Delete(ctx, &lis) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) + } + } + + // set network status to ready when all lb port is ready + if len(externalAddresses) == len(clbConf.targetPorts) { + // change network status to ready if necessary + if !reflect.DeepEqual(externalAddresses, networkStatus.ExternalAddresses) || networkStatus.CurrentNetworkState != kruisev1alpha1.NetworkReady { + networkStatus.InternalAddresses = internalAddresses + networkStatus.ExternalAddresses = externalAddresses + networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error()) + } + } + } + + return pod, nil +} + +func (p *ClbPlugin) OnPodDeleted(c client.Client, pod *corev1.Pod, ctx context.Context) cperrors.PluginError { + p.deAllocate(pod.GetNamespace() + "/" + pod.GetName()) + return nil +} + +func (p *ClbPlugin) consLis(clbConf *clbConfig, pod *corev1.Pod, port portProtocol, gssName string) (*v1alpha1.DedicatedCLBListener, error) { + lbId, lbPort := p.allocate(clbConf.lbIds, pod.GetNamespace()+"/"+pod.GetName()) + if lbId == "" { + return nil, fmt.Errorf("there are no avaialable ports for %v", clbConf.lbIds) + } + lis := &v1alpha1.DedicatedCLBListener{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: pod.Name + "-", + Namespace: pod.Namespace, + Labels: map[string]string{ + kruisev1alpha1.OwnerPodKey: pod.Name, + kruisev1alpha1.GameServerOwnerGssKey: gssName, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: pod.APIVersion, + Kind: pod.Kind, + Name: pod.GetName(), + UID: pod.GetUID(), + Controller: ptr.To[bool](true), + BlockOwnerDeletion: ptr.To[bool](true), + }, + }, + }, + Spec: v1alpha1.DedicatedCLBListenerSpec{ + LbId: lbId, + LbPort: int64(lbPort), + Protocol: port.protocol, + TargetPod: &v1alpha1.TargetPod{ + PodName: pod.Name, + TargetPort: int64(port.port), + }, + }, + } + return lis, nil +} + +func init() { + clbPlugin := ClbPlugin{ + mutex: sync.RWMutex{}, + } + tencentCloudProvider.registerPlugin(&clbPlugin) +} + +func parseLbConfig(conf []kruisev1alpha1.NetworkConfParams) (*clbConfig, error) { + var lbIds []string + ports := []portProtocol{} + for _, c := range conf { + switch c.Name { + case ClbIdsConfigName: + for _, clbId := range strings.Split(c.Value, ",") { + if clbId != "" { + lbIds = append(lbIds, clbId) + } + } + case PortProtocolsConfigName: + for _, pp := range strings.Split(c.Value, ",") { + ppSlice := strings.Split(pp, "/") + port, err := strconv.Atoi(ppSlice[0]) + if err != nil { + continue + } + protocol := "TCP" + if len(ppSlice) == 2 { + protocol = ppSlice[1] + } + ports = append(ports, portProtocol{ + port: port, + protocol: protocol, + }) + } + } + } + return &clbConfig{ + lbIds: lbIds, + targetPorts: ports, + }, nil +} + +func (p *ClbPlugin) allocate(lbIds []string, podKey string) (string, int32) { + p.mutex.Lock() + defer p.mutex.Unlock() + + var lbId string + var port int32 + + // find avaialable port + for _, clbId := range lbIds { + for i := p.minPort; i < p.maxPort; i++ { + if !p.cache[clbId][i] { + lbId = clbId + port = i + break + } + } + } + // update cache + if lbId != "" { + if p.cache[lbId] == nil { // init lb cache if not exist + p.cache[lbId] = make(portAllocated, p.maxPort-p.minPort) + for i := p.minPort; i < p.maxPort; i++ { + p.cache[lbId][i] = false + } + } + p.cache[lbId][port] = true + p.podAllocate[podKey] = append(p.podAllocate[podKey], fmt.Sprintf("%s:%d", lbId, port)) + log.Infof("pod %s allocate clb %s port %d", podKey, lbId, port) + } + return lbId, port +} + +func (p *ClbPlugin) deAllocate(podKey string) { + p.mutex.Lock() + defer p.mutex.Unlock() + + allocatedPorts, exist := p.podAllocate[podKey] + if !exist { + return + } + + for _, port := range allocatedPorts { + ss := strings.Split(port, ":") + if len(ss) != 2 { + log.Errorf("bad allocated port cache format %s", port) + continue + } + lbId := ss[0] + lbPort, err := strconv.Atoi(ss[1]) + if err != nil { + log.Errorf("failed to parse allocated port %s: %s", port, err.Error()) + continue + } + p.cache[lbId][int32(lbPort)] = false + log.Infof("pod %s deallocate clb %s ports %d", podKey, lbId, lbPort) + } + delete(p.podAllocate, podKey) +} diff --git a/cloudprovider/tencentcloud/clb_test.go b/cloudprovider/tencentcloud/clb_test.go new file mode 100644 index 00000000..da3b756c --- /dev/null +++ b/cloudprovider/tencentcloud/clb_test.go @@ -0,0 +1,190 @@ +package tencentcloud + +import ( + "reflect" + "sync" + "testing" + + kruisev1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/cloudprovider/tencentcloud/apis/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestAllocateDeAllocate(t *testing.T) { + test := struct { + lbIds []string + clb *ClbPlugin + podKey string + }{ + lbIds: []string{"lb-xxx"}, + clb: &ClbPlugin{ + maxPort: int32(712), + minPort: int32(512), + cache: make(map[string]portAllocated), + podAllocate: make(map[string][]string), + mutex: sync.RWMutex{}, + }, + podKey: "xxx/xxx", + } + + lbId, port := test.clb.allocate(test.lbIds, test.podKey) + if _, exist := test.clb.podAllocate[test.podKey]; !exist { + t.Errorf("podAllocate[%s] is empty after allocated", test.podKey) + } + if port > test.clb.maxPort || port < test.clb.minPort { + t.Errorf("allocate port %d, unexpected", port) + } + if test.clb.cache[lbId][port] == false { + t.Errorf("Allocate port %d failed", port) + } + test.clb.deAllocate(test.podKey) + if test.clb.cache[lbId][port] == true { + t.Errorf("deAllocate port %d failed", port) + } + if _, exist := test.clb.podAllocate[test.podKey]; exist { + t.Errorf("podAllocate[%s] is not empty after deallocated", test.podKey) + } +} + +func TestParseLbConfig(t *testing.T) { + tests := []struct { + conf []kruisev1alpha1.NetworkConfParams + clbConfig *clbConfig + }{ + { + conf: []kruisev1alpha1.NetworkConfParams{ + { + Name: ClbIdsConfigName, + Value: "xxx-A", + }, + { + Name: PortProtocolsConfigName, + Value: "80", + }, + }, + clbConfig: &clbConfig{ + lbIds: []string{"xxx-A"}, + targetPorts: []portProtocol{ + { + port: 80, + protocol: "TCP", + }, + }, + }, + }, + { + conf: []kruisev1alpha1.NetworkConfParams{ + { + Name: ClbIdsConfigName, + Value: "xxx-A,xxx-B,", + }, + { + Name: PortProtocolsConfigName, + Value: "81/UDP,82,83/TCP", + }, + }, + clbConfig: &clbConfig{ + lbIds: []string{"xxx-A", "xxx-B"}, + targetPorts: []portProtocol{ + { + port: 81, + protocol: "UDP", + }, + { + port: 82, + protocol: "TCP", + }, + { + port: 83, + protocol: "TCP", + }, + }, + }, + }, + } + + for i, test := range tests { + lc, err := parseLbConfig(test.conf) + if err != nil { + t.Error(err) + } + if !reflect.DeepEqual(test.clbConfig, lc) { + t.Errorf("case %d: lbId expect: %v, actual: %v", i, test.clbConfig, lc) + } + } +} + +func TestInitLbCache(t *testing.T) { + test := struct { + listenerList []v1alpha1.DedicatedCLBListener + minPort int32 + maxPort int32 + cache map[string]portAllocated + podAllocate map[string][]string + }{ + minPort: 512, + maxPort: 712, + cache: map[string]portAllocated{ + "xxx-A": map[int32]bool{ + 666: true, + }, + "xxx-B": map[int32]bool{ + 555: true, + }, + }, + podAllocate: map[string][]string{ + "ns-0/name-0": {"xxx-A:666"}, + "ns-1/name-1": {"xxx-B:555"}, + }, + listenerList: []v1alpha1.DedicatedCLBListener{ + { + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + kruisev1alpha1.OwnerPodKey: "name-0", + }, + Namespace: "ns-0", + Name: "name-0-xxx", + }, + Spec: v1alpha1.DedicatedCLBListenerSpec{ + LbId: "xxx-A", + LbPort: 666, + Protocol: "TCP", + TargetPod: &v1alpha1.TargetPod{ + PodName: "name-0", + TargetPort: 80, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + kruisev1alpha1.OwnerPodKey: "name-1", + }, + Namespace: "ns-1", + Name: "name-1-xxx", + }, + Spec: v1alpha1.DedicatedCLBListenerSpec{ + LbId: "xxx-B", + LbPort: 555, + Protocol: "TCP", + TargetPod: &v1alpha1.TargetPod{ + PodName: "name-1", + TargetPort: 80, + }, + }, + }, + }, + } + + actualCache, actualPodAllocate := initLbCache(test.listenerList, test.minPort, test.maxPort) + for lb, pa := range test.cache { + for port, isAllocated := range pa { + if actualCache[lb][port] != isAllocated { + t.Errorf("lb %s port %d isAllocated, expect: %t, actual: %t", lb, port, isAllocated, actualCache[lb][port]) + } + } + } + if !reflect.DeepEqual(actualPodAllocate, test.podAllocate) { + t.Errorf("podAllocate expect %v, but actully got %v", test.podAllocate, actualPodAllocate) + } +} diff --git a/cloudprovider/tencentcloud/tencentcloud.go b/cloudprovider/tencentcloud/tencentcloud.go new file mode 100644 index 00000000..c1494633 --- /dev/null +++ b/cloudprovider/tencentcloud/tencentcloud.go @@ -0,0 +1,59 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tencentcloud + +import ( + "github.com/openkruise/kruise-game/cloudprovider" + "k8s.io/klog/v2" +) + +const ( + TencentCloud = "TencentCloud" +) + +var tencentCloudProvider = &Provider{ + plugins: make(map[string]cloudprovider.Plugin), +} + +type Provider struct { + plugins map[string]cloudprovider.Plugin +} + +func (ap *Provider) Name() string { + return TencentCloud +} + +func (ap *Provider) ListPlugins() (map[string]cloudprovider.Plugin, error) { + if ap.plugins == nil { + return make(map[string]cloudprovider.Plugin), nil + } + + return ap.plugins, nil +} + +// register plugin of cloud provider and different cloud providers +func (ap *Provider) registerPlugin(plugin cloudprovider.Plugin) { + name := plugin.Name() + if name == "" { + klog.Fatal("empty plugin name") + } + ap.plugins[name] = plugin +} + +func NewTencentCloudProvider() (cloudprovider.CloudProvider, error) { + return tencentCloudProvider, nil +} diff --git a/config/crd/bases/networking.cloud.tencent.com_dedicatedclblisteners.yaml b/config/crd/bases/networking.cloud.tencent.com_dedicatedclblisteners.yaml new file mode 100644 index 00000000..61c98299 --- /dev/null +++ b/config/crd/bases/networking.cloud.tencent.com_dedicatedclblisteners.yaml @@ -0,0 +1,121 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: dedicatedclblisteners.networking.cloud.tencent.com +spec: + group: networking.cloud.tencent.com + names: + kind: DedicatedCLBListener + listKind: DedicatedCLBListenerList + plural: dedicatedclblisteners + singular: dedicatedclblistener + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: CLB ID + jsonPath: .spec.lbId + name: LbId + type: string + - description: Port of CLB Listener + jsonPath: .spec.lbPort + name: LbPort + type: integer + - description: Pod name of target pod + jsonPath: .spec.targetPod.podName + name: Pod + type: string + - description: State of the dedicated clb listener + jsonPath: .status.state + name: State + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: DedicatedCLBListener is the Schema for the dedicatedclblisteners + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DedicatedCLBListenerSpec defines the desired state of DedicatedCLBListener + properties: + extensiveParameters: + type: string + lbId: + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + lbPort: + format: int64 + type: integer + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + lbRegion: + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + protocol: + enum: + - TCP + - UDP + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + targetPod: + properties: + podName: + type: string + targetPort: + format: int64 + type: integer + required: + - podName + - targetPort + type: object + required: + - lbId + - lbPort + - protocol + type: object + status: + description: DedicatedCLBListenerStatus defines the observed state of + DedicatedCLBListener + properties: + address: + type: string + listenerId: + type: string + message: + type: string + state: + enum: + - Bound + - Available + - Pending + - Failed + - Deleting + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/docs/en/user_manuals/network.md b/docs/en/user_manuals/network.md index d105e156..be3307e8 100644 --- a/docs/en/user_manuals/network.md +++ b/docs/en/user_manuals/network.md @@ -969,4 +969,107 @@ nlb-26jbknebrjlejt5abu 192.168.0.8:80,192.168.0.82:80,192.168.63.228:80 ``` -After waiting for the entire update process to end, you can find that there are no changes in the ep, indicating that no extraction has been performed. \ No newline at end of file +After waiting for the entire update process to end, you can find that there are no changes in the ep, indicating that no extraction has been performed. + +--- + +### TencentCloud-CLB + +#### Plugin name + +`TencentCloud-CLB` + +#### Cloud Provider + +TencentCloud + +#### Plugin description + + +- TencentCloud-CLB enables game servers to be accessed from the Internet by using Cloud Load Balancer (CLB) of Tencent Cloud. CLB is a type of Server Load Balancer (CLB). TencentCloud-CLB uses different ports for different game servers. The CLB instance only forwards traffic, but does not implement load balancing. +The [tke-extend-network-controller](https://github.com/tkestack/tke-extend-network-controller) network plugin needs to be installed (can be installed through the TKE application market). +- This network plugin does not support network isolation. + +#### Network parameters + +- Meaning: the CLB instance ID. You can fill in multiple ids. +- Value: in the format of slbId-0,slbId-1,... An example value can be "lb-9zeo7prq1m25ctpfrw1m7,lb-bp1qz7h50yd3w58h2f8je" +- Configuration change supported or not: yes. You can add new slbIds at the end. However, it is recommended not to change existing slbId that is in use. + +PortProtocols + +- Meaning: the ports in the pod to be exposed and the protocols. You can specify multiple ports and protocols. +- Value: in the format of port1/protocol1,port2/protocol2,... The protocol names must be in uppercase letters. +- Configuration change supported or not: yes. + +#### Plugin configuration + +``` +[tencentcloud] +enable = true +[tencentcloud.clb] +# Specify the range of available ports of the CLB instance. Ports in this range can be used to forward Internet traffic to pods. In this example, the range includes 200 ports. +min_port = 1000 +max_port = 1100 +``` + +#### Example + +```yaml +apiVersion: game.kruise.io/v1alpha1 +kind: GameServerSet +metadata: + name: clb-nginx + namespace: default +spec: + replicas: 1 + updateStrategy: + rollingUpdate: + podUpdatePolicy: InPlaceIfPossible + network: + networkType: TencentCloud-CLB + networkConf: + - name: ClbIds + value: "lb-3ip9k5kr,lb-4ia8k0yh" + - name: PortProtocols + value: "80/TCP,7777/UDP" + gameServerTemplate: + spec: + containers: + - image: nginx + name: nginx +``` + +The network status of GameServer would be as follows: + +```yaml + networkStatus: + createTime: "2024-10-28T03:16:20Z" + currentNetworkState: Ready + desiredNetworkState: Ready + externalAddresses: + - ip: 139.155.64.52 + ports: + - name: "80" + port: 1002 + protocol: TCP + - ip: 139.155.64.52 + ports: + - name: "7777" + port: 1003 + protocol: UDP + internalAddresses: + - ip: 172.16.7.106 + ports: + - name: "80" + port: 80 + protocol: TCP + - ip: 172.16.7.106 + ports: + - name: "7777" + port: 7777 + protocol: UDP + lastTransitionTime: "2024-10-28T03:16:20Z" + networkType: TencentCloud-CLB +``` + diff --git "a/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/\347\275\221\347\273\234\346\250\241\345\236\213.md" "b/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/\347\275\221\347\273\234\346\250\241\345\236\213.md" index 448c6e53..21523d11 100644 --- "a/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/\347\275\221\347\273\234\346\250\241\345\236\213.md" +++ "b/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/\347\275\221\347\273\234\346\250\241\345\236\213.md" @@ -972,6 +972,109 @@ nlb-26jbknebrjlejt5abu 192.168.0.8:80,192.168.0.82:80,192.168.63.228:80 等待整个更新过程结束，可以发现ep没有任何变化，说明并未进行摘流。 +--- + +### TencentCloud-CLB + +#### 插件名称 + +`TencentCloud-CLB` + +#### Cloud Provider + +TencentCloud + +#### 插件说明 + +- TencentCloud-CLB 使用腾讯云负载均衡器（CLB）作为对外服务的承载实体，在此模式下，不同游戏服使用 CLB 的不同端口对外暴露，此时 CLB 只做转发，并未均衡流量。 +- 需安装 [tke-extend-network-controller](https://github.com/tkestack/tke-extend-network-controller) 网络插件(可通过 TKE 应用市场安装)。 +- 是否支持网络隔离：否。 + +#### 网络参数 + +ClbIds + +- 含义：填写clb的id。可填写多个。 +- 填写格式：各个clbId用,分割。例如：lb-xxxx,lb-yyyy,... +- 是否支持变更：支持。 + +PortProtocols + +- 含义：pod暴露的端口及协议，支持填写多个端口/协议。 +- 格式：port1/protocol1,port2/protocol2,...（协议需大写） +- 是否支持变更：支持。 + +#### 插件配置 + +``` +[tencentcloud] +enable = true +[tencentcloud.clb] +# 填写clb可使用的空闲端口段，用于为pod分配外部接入端口 +min_port = 1000 +max_port = 1100 +``` + +#### 示例说明 + +```yaml +apiVersion: game.kruise.io/v1alpha1 +kind: GameServerSet +metadata: + name: clb-nginx + namespace: default +spec: + replicas: 1 + updateStrategy: + rollingUpdate: + podUpdatePolicy: InPlaceIfPossible + network: + networkType: TencentCloud-CLB + networkConf: + - name: ClbIds + value: "lb-3ip9k5kr,lb-4ia8k0yh" + - name: PortProtocols + value: "80/TCP,7777/UDP" + gameServerTemplate: + spec: + containers: + - image: nginx + name: nginx +``` + +生成的 gameserver clb-nginx-0 networkStatus 字段如下所示： + +```yaml + networkStatus: + createTime: "2024-10-28T03:16:20Z" + currentNetworkState: Ready + desiredNetworkState: Ready + externalAddresses: + - ip: 139.155.64.52 + ports: + - name: "80" + port: 1002 + protocol: TCP + - ip: 139.155.64.52 + ports: + - name: "7777" + port: 1003 + protocol: UDP + internalAddresses: + - ip: 172.16.7.106 + ports: + - name: "80" + port: 80 + protocol: TCP + - ip: 172.16.7.106 + ports: + - name: "7777" + port: 7777 + protocol: UDP + lastTransitionTime: "2024-10-28T03:16:20Z" + networkType: TencentCloud-CLB +``` + ## 获取网络信息 GameServer Network Status可以通过两种方式获取 @@ -1051,4 +1154,4 @@ func getNetwork() { // 访问networkStatus各个字段 } -``` \ No newline at end of file +``` diff --git a/main.go b/main.go index 8e02f511..592e9dc6 100644 --- a/main.go +++ b/main.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -42,6 +43,7 @@ import ( "github.com/openkruise/kruise-game/cloudprovider" aliv1beta1 "github.com/openkruise/kruise-game/cloudprovider/alibabacloud/apis/v1beta1" cpmanager "github.com/openkruise/kruise-game/cloudprovider/manager" + tencentv1alpha1 "github.com/openkruise/kruise-game/cloudprovider/tencentcloud/apis/v1alpha1" kruisegameclientset "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" kruisegamevisions "github.com/openkruise/kruise-game/pkg/client/informers/externalversions" controller "github.com/openkruise/kruise-game/pkg/controllers" @@ -67,6 +69,7 @@ func init() { utilruntime.Must(kruiseV1alpha1.AddToScheme(scheme)) utilruntime.Must(aliv1beta1.AddToScheme(scheme)) + utilruntime.Must(tencentv1alpha1.AddToScheme(scheme)) utilruntime.Must(ackv1alpha1.AddToScheme(scheme)) utilruntime.Must(elbv2api.AddToScheme(scheme)) @@ -138,7 +141,6 @@ func main() { SyncPeriod: syncPeriod, NewClient: utilclient.NewClient, }) - if err != nil { setupLog.Error(err, "unable to start kruise-game-manager") os.Exit(1)