diff --git a/apis/v1alpha1/gameserverset_types.go b/apis/v1alpha1/gameserverset_types.go index 379c2f93..ffa19d99 100644 --- a/apis/v1alpha1/gameserverset_types.go +++ b/apis/v1alpha1/gameserverset_types.go @@ -36,6 +36,10 @@ const ( GsTemplateMetadataHashKey = "game.kruise.io/gsTemplate-metadata-hash" ) +const ( + InplaceUpdateNotReadyBlocker = "game.kruise.io/inplace-update-not-ready-blocker" +) + // GameServerSetSpec defines the desired state of GameServerSet type GameServerSetSpec struct { // replicas is the desired number of replicas of the given Template. @@ -69,6 +73,10 @@ type Network struct { type NetworkConfParams KVParams +const ( + AllowNotReadyContainersNetworkConfName = "AllowNotReadyContainers" +) + type KVParams struct { Name string `json:"name,omitempty"` Value string `json:"value,omitempty"` diff --git a/cloudprovider/alibabacloud/nlb_sp.go b/cloudprovider/alibabacloud/nlb_sp.go new file mode 100644 index 00000000..d49b4ffb --- /dev/null +++ b/cloudprovider/alibabacloud/nlb_sp.go @@ -0,0 +1,239 @@ +package alibabacloud + +import ( + "context" + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/cloudprovider" + cperrors "github.com/openkruise/kruise-game/cloudprovider/errors" + "github.com/openkruise/kruise-game/cloudprovider/utils" + "github.com/openkruise/kruise-game/pkg/util" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + "strconv" +) + +const ( + NlbSPNetwork = "AlibabaCloud-NLB-SharedPort" + NlbIdsConfigName = "NlbIds" +) + +func init() { + alibabaCloudProvider.registerPlugin(&NlbSpPlugin{}) +} + +type NlbSpPlugin struct { +} + +func (N *NlbSpPlugin) Name() string { + return NlbSPNetwork +} + +func (N *NlbSpPlugin) Alias() string { + return "" +} + +func (N *NlbSpPlugin) Init(client client.Client, options cloudprovider.CloudProviderOptions, ctx context.Context) error { + return nil +} + +func (N *NlbSpPlugin) OnPodAdded(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) { + networkManager := utils.NewNetworkManager(pod, c) + podNetConfig := parseNLbSpConfig(networkManager.GetNetworkConfig()) + + pod.Labels[SlbIdLabelKey] = podNetConfig.lbId + + // Get Svc + svc := &corev1.Service{} + err := c.Get(ctx, types.NamespacedName{ + Namespace: pod.GetNamespace(), + Name: podNetConfig.lbId, + }, svc) + if err != nil { + if errors.IsNotFound(err) { + // Create Svc + return pod, cperrors.ToPluginError(c.Create(ctx, consNlbSvc(podNetConfig, pod, c, ctx)), cperrors.ApiCallError) + } + return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) + } + return pod, nil +} + +func (N *NlbSpPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) { + networkManager := utils.NewNetworkManager(pod, c) + networkStatus, _ := networkManager.GetNetworkStatus() + if networkStatus == nil { + pod, err := networkManager.UpdateNetworkStatus(gamekruiseiov1alpha1.NetworkStatus{ + CurrentNetworkState: gamekruiseiov1alpha1.NetworkNotReady, + }, pod) + return pod, cperrors.ToPluginError(err, cperrors.InternalError) + } + + networkConfig := networkManager.GetNetworkConfig() + podNetConfig := parseNLbSpConfig(networkConfig) + + // Get Svc + svc := &corev1.Service{} + err := c.Get(context.Background(), types.NamespacedName{ + Namespace: pod.GetNamespace(), + Name: podNetConfig.lbId, + }, svc) + if err != nil { + if errors.IsNotFound(err) { + // Create Svc + return pod, cperrors.ToPluginError(c.Create(ctx, consNlbSvc(podNetConfig, pod, c, ctx)), cperrors.ApiCallError) + } + return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) + } + + // update svc + if util.GetHash(podNetConfig) != svc.GetAnnotations()[SlbConfigHashKey] { + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error()) + } + return pod, cperrors.ToPluginError(c.Update(ctx, consNlbSvc(podNetConfig, pod, c, ctx)), cperrors.ApiCallError) + } + + _, hasLabel := pod.Labels[SlbIdLabelKey] + // disable network + if networkManager.GetNetworkDisabled() && hasLabel { + newLabels := pod.GetLabels() + delete(newLabels, SlbIdLabelKey) + pod.Labels = newLabels + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + return pod, cperrors.ToPluginError(err, cperrors.InternalError) + } + + // enable network + if !networkManager.GetNetworkDisabled() && !hasLabel { + pod.Labels[SlbIdLabelKey] = podNetConfig.lbId + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + return pod, cperrors.ToPluginError(err, cperrors.InternalError) + } + + // network not ready + if svc.Status.LoadBalancer.Ingress == nil { + return pod, nil + } + + // allow not ready containers + if util.IsAllowNotReadyContainers(networkConfig) { + toUpDateSvc, err := utils.AllowNotReadyContainers(c, ctx, pod, svc, true) + if err != nil { + return pod, err + } + + if toUpDateSvc { + err := c.Update(ctx, svc) + if err != nil { + return pod, cperrors.ToPluginError(err, cperrors.ApiCallError) + } + } + } + + // network ready + internalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0) + externalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0) + for _, port := range svc.Spec.Ports { + instrIPort := port.TargetPort + instrEPort := intstr.FromInt(int(port.Port)) + internalAddress := gamekruiseiov1alpha1.NetworkAddress{ + IP: pod.Status.PodIP, + Ports: []gamekruiseiov1alpha1.NetworkPort{ + { + Name: instrIPort.String(), + Port: &instrIPort, + Protocol: port.Protocol, + }, + }, + } + externalAddress := gamekruiseiov1alpha1.NetworkAddress{ + EndPoint: svc.Status.LoadBalancer.Ingress[0].Hostname, + Ports: []gamekruiseiov1alpha1.NetworkPort{ + { + Name: instrIPort.String(), + Port: &instrEPort, + Protocol: port.Protocol, + }, + }, + } + internalAddresses = append(internalAddresses, internalAddress) + externalAddresses = append(externalAddresses, externalAddress) + } + networkStatus.InternalAddresses = internalAddresses + networkStatus.ExternalAddresses = externalAddresses + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + return pod, cperrors.ToPluginError(err, cperrors.InternalError) +} + +func (N *NlbSpPlugin) OnPodDeleted(client client.Client, pod *corev1.Pod, ctx context.Context) cperrors.PluginError { + return nil +} + +type nlbConfig struct { + lbId string + ports []int + protocols []corev1.Protocol +} + +func parseNLbSpConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) *nlbConfig { + var lbIds string + var ports []int + var protocols []corev1.Protocol + for _, c := range conf { + switch c.Name { + case NlbIdsConfigName: + lbIds = c.Value + case PortProtocolsConfigName: + ports, protocols = parsePortProtocols(c.Value) + } + } + return &nlbConfig{ + lbId: lbIds, + ports: ports, + protocols: protocols, + } +} + +func consNlbSvc(nc *nlbConfig, pod *corev1.Pod, c client.Client, ctx context.Context) *corev1.Service { + svcPorts := make([]corev1.ServicePort, 0) + for i := 0; i < len(nc.ports); i++ { + svcPorts = append(svcPorts, corev1.ServicePort{ + Name: strconv.Itoa(nc.ports[i]), + Port: int32(nc.ports[i]), + Protocol: nc.protocols[i], + TargetPort: intstr.FromInt(nc.ports[i]), + }) + } + loadBalancerClass := "alibabacloud.com/nlb" + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: nc.lbId, + Namespace: pod.GetNamespace(), + Annotations: map[string]string{ + SlbListenerOverrideKey: "true", + SlbIdAnnotationKey: nc.lbId, + SlbConfigHashKey: util.GetHash(nc), + }, + OwnerReferences: getSvcOwnerReference(c, ctx, pod, true), + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Selector: map[string]string{ + SlbIdLabelKey: nc.lbId, + }, + Ports: svcPorts, + LoadBalancerClass: &loadBalancerClass, + }, + } + return svc +} diff --git a/cloudprovider/alibabacloud/nlb_sp_test.go b/cloudprovider/alibabacloud/nlb_sp_test.go new file mode 100644 index 00000000..214e1278 --- /dev/null +++ b/cloudprovider/alibabacloud/nlb_sp_test.go @@ -0,0 +1,58 @@ +package alibabacloud + +import ( + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + corev1 "k8s.io/api/core/v1" + "reflect" + "testing" +) + +func TestParseNLbSpConfig(t *testing.T) { + tests := []struct { + conf []gamekruiseiov1alpha1.NetworkConfParams + nc *nlbConfig + }{ + { + conf: []gamekruiseiov1alpha1.NetworkConfParams{ + { + Name: NlbIdsConfigName, + Value: "nlb-xxx", + }, + { + Name: PortProtocolsConfigName, + Value: "80/UDP", + }, + }, + nc: &nlbConfig{ + protocols: []corev1.Protocol{corev1.ProtocolUDP}, + ports: []int{80}, + lbId: "nlb-xxx", + }, + }, + { + conf: []gamekruiseiov1alpha1.NetworkConfParams{ + { + Name: NlbIdsConfigName, + Value: "nlb-xxx", + }, + { + Name: PortProtocolsConfigName, + Value: "80", + }, + }, + nc: &nlbConfig{ + protocols: []corev1.Protocol{corev1.ProtocolTCP}, + ports: []int{80}, + lbId: "nlb-xxx", + }, + }, + } + + for i, test := range tests { + expect := test.nc + actual := parseNLbSpConfig(test.conf) + if !reflect.DeepEqual(expect, actual) { + t.Errorf("case %d: expect nlbConfig is %v, but actually is %v", i, expect, actual) + } + } +} diff --git a/cloudprovider/alibabacloud/slb.go b/cloudprovider/alibabacloud/slb.go index b384114b..ef5787b2 100644 --- a/cloudprovider/alibabacloud/slb.go +++ b/cloudprovider/alibabacloud/slb.go @@ -182,6 +182,21 @@ func (s *SlbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.C return pod, cperrors.ToPluginError(err, cperrors.InternalError) } + // allow not ready containers + if util.IsAllowNotReadyContainers(networkManager.GetNetworkConfig()) { + toUpDateSvc, err := utils.AllowNotReadyContainers(c, ctx, pod, svc, false) + if err != nil { + return pod, err + } + + if toUpDateSvc { + err := c.Update(ctx, svc) + if err != nil { + return pod, cperrors.ToPluginError(err, cperrors.ApiCallError) + } + } + } + // network ready internalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0) externalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0) diff --git a/cloudprovider/alibabacloud/slb_sp.go b/cloudprovider/alibabacloud/slb_sp.go index 85057aca..da3d31a0 100644 --- a/cloudprovider/alibabacloud/slb_sp.go +++ b/cloudprovider/alibabacloud/slb_sp.go @@ -7,6 +7,7 @@ import ( "github.com/openkruise/kruise-game/cloudprovider" cperrors "github.com/openkruise/kruise-game/cloudprovider/errors" "github.com/openkruise/kruise-game/cloudprovider/utils" + "github.com/openkruise/kruise-game/pkg/util" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -130,6 +131,21 @@ func (s *SlbSpPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context return pod, nil } + // allow not ready containers + if util.IsAllowNotReadyContainers(networkManager.GetNetworkConfig()) { + toUpDateSvc, err := utils.AllowNotReadyContainers(c, ctx, pod, svc, true) + if err != nil { + return pod, err + } + + if toUpDateSvc { + err := c.Update(ctx, svc) + if err != nil { + return pod, cperrors.ToPluginError(err, cperrors.ApiCallError) + } + } + } + // network ready internalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0) externalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0) diff --git a/cloudprovider/alibabacloud/slb_sp_test.go b/cloudprovider/alibabacloud/slb_sp_test.go index 0f0ea9fd..8d2a81e5 100644 --- a/cloudprovider/alibabacloud/slb_sp_test.go +++ b/cloudprovider/alibabacloud/slb_sp_test.go @@ -3,6 +3,7 @@ package alibabacloud import ( "fmt" gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/pkg/util" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "reflect" @@ -121,3 +122,32 @@ func TestParseLbSpConfig(t *testing.T) { } } } + +func TestParsePortProtocols(t *testing.T) { + tests := []struct { + value string + ports []int + protocols []corev1.Protocol + }{ + { + value: "80", + ports: []int{80}, + protocols: []corev1.Protocol{corev1.ProtocolTCP}, + }, + { + value: "8080/UDP,80/TCP", + ports: []int{8080, 80}, + protocols: []corev1.Protocol{corev1.ProtocolUDP, corev1.ProtocolTCP}, + }, + } + + for i, test := range tests { + actualPorts, actualProtocols := parsePortProtocols(test.value) + if !util.IsSliceEqual(actualPorts, test.ports) { + t.Errorf("case %d: expect ports is %v, but actually is %v", i, test.ports, actualPorts) + } + if !reflect.DeepEqual(actualProtocols, test.protocols) { + t.Errorf("case %d: expect protocols is %v, but actually is %v", i, test.protocols, actualProtocols) + } + } +} diff --git a/cloudprovider/utils/service.go b/cloudprovider/utils/service.go new file mode 100644 index 00000000..36069447 --- /dev/null +++ b/cloudprovider/utils/service.go @@ -0,0 +1,85 @@ +package utils + +import ( + "context" + kruisePub "github.com/openkruise/kruise-api/apps/pub" + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + cperrors "github.com/openkruise/kruise-game/cloudprovider/errors" + "github.com/openkruise/kruise-game/pkg/util" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + "strings" +) + +func AllowNotReadyContainers(c client.Client, ctx context.Context, pod *corev1.Pod, svc *corev1.Service, isSvcShared bool) (bool, cperrors.PluginError) { + // get lifecycleState + lifecycleState, exist := pod.GetLabels()[kruisePub.LifecycleStateKey] + + // get gss + gss, err := util.GetGameServerSetOfPod(pod, c, ctx) + if err != nil { + return false, cperrors.ToPluginError(err, cperrors.ApiCallError) + } + + // get allowNotReadyContainers + var allowNotReadyContainers []string + for _, kv := range gss.Spec.Network.NetworkConf { + if kv.Name == gamekruiseiov1alpha1.AllowNotReadyContainersNetworkConfName { + for _, allowNotReadyContainer := range strings.Split(kv.Value, ",") { + if allowNotReadyContainer != "" { + allowNotReadyContainers = append(allowNotReadyContainers, allowNotReadyContainer) + } + } + } + } + + // PreInplaceUpdating + if exist && lifecycleState == string(kruisePub.LifecycleStatePreparingUpdate) { + // ensure PublishNotReadyAddresses is true when containers pre-updating + if !svc.Spec.PublishNotReadyAddresses && util.IsContainersPreInplaceUpdating(pod, gss, allowNotReadyContainers) { + svc.Spec.PublishNotReadyAddresses = true + return true, nil + } + + // ensure remove finalizer + if svc.Spec.PublishNotReadyAddresses || !util.IsContainersPreInplaceUpdating(pod, gss, allowNotReadyContainers) { + pod.GetLabels()[gamekruiseiov1alpha1.InplaceUpdateNotReadyBlocker] = "false" + } + } else { + pod.GetLabels()[gamekruiseiov1alpha1.InplaceUpdateNotReadyBlocker] = "true" + if !svc.Spec.PublishNotReadyAddresses { + return false, nil + } + if isSvcShared { + // ensure PublishNotReadyAddresses is false when all pods are updated + if gss.Status.UpdatedReplicas == gss.Status.Replicas { + podList := &corev1.PodList{} + err := c.List(ctx, podList, &client.ListOptions{ + Namespace: gss.GetNamespace(), + LabelSelector: labels.SelectorFromSet(map[string]string{ + gamekruiseiov1alpha1.GameServerOwnerGssKey: gss.GetName(), + })}) + if err != nil { + return false, cperrors.ToPluginError(err, cperrors.ApiCallError) + } + for _, p := range podList.Items { + _, condition := util.GetPodConditionFromList(p.Status.Conditions, corev1.PodReady) + if condition == nil || condition.Status != corev1.ConditionTrue { + return false, nil + } + } + svc.Spec.PublishNotReadyAddresses = false + return true, nil + } + } else { + _, condition := util.GetPodConditionFromList(pod.Status.Conditions, corev1.PodReady) + if condition == nil || condition.Status != corev1.ConditionTrue { + return false, nil + } + svc.Spec.PublishNotReadyAddresses = false + return true, nil + } + } + return false, nil +} diff --git a/cloudprovider/utils/service_test.go b/cloudprovider/utils/service_test.go new file mode 100644 index 00000000..36c51f2d --- /dev/null +++ b/cloudprovider/utils/service_test.go @@ -0,0 +1,289 @@ +package utils + +import ( + "context" + kruisePub "github.com/openkruise/kruise-api/apps/pub" + kruiseV1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1" + kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "testing" +) + +var ( + scheme = runtime.NewScheme() +) + +func init() { + utilruntime.Must(gamekruiseiov1alpha1.AddToScheme(scheme)) + utilruntime.Must(kruiseV1beta1.AddToScheme(scheme)) + utilruntime.Must(kruiseV1alpha1.AddToScheme(scheme)) + utilruntime.Must(corev1.AddToScheme(scheme)) +} + +func TestAllowNotReadyContainers(t *testing.T) { + tests := []struct { + // input + pod *corev1.Pod + svc *corev1.Service + gss *gamekruiseiov1alpha1.GameServerSet + isSvcShared bool + podElse []*corev1.Pod + // output + inplaceUpdateNotReadyBlocker string + isSvcUpdated bool + }{ + // When svc is not shared, pod updated, svc should not publish NotReadyAddresses + { + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "xxx", + Name: "case0-0", + UID: "xxx0", + Labels: map[string]string{ + kruisePub.LifecycleStateKey: string(kruisePub.LifecycleStateUpdating), + gamekruiseiov1alpha1.GameServerOwnerGssKey: "case0", + }, + }, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }, + }, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "name_A", + Image: "v1.0", + }, + { + Name: "name_B", + Image: "v1.0", + }, + }, + }, + }, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + PublishNotReadyAddresses: true, + }, + }, + gss: &gamekruiseiov1alpha1.GameServerSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "GameServerSet", + APIVersion: "game.kruise.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "xxx", + Name: "case0", + UID: "xxx0", + }, + Spec: gamekruiseiov1alpha1.GameServerSetSpec{ + Network: &gamekruiseiov1alpha1.Network{ + NetworkConf: []gamekruiseiov1alpha1.NetworkConfParams{ + { + Name: gamekruiseiov1alpha1.AllowNotReadyContainersNetworkConfName, + Value: "name_B", + }, + }, + }, + GameServerTemplate: gamekruiseiov1alpha1.GameServerTemplate{ + PodTemplateSpec: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "name_A", + Image: "v1.0", + }, + { + Name: "name_B", + Image: "v1.0", + }, + }, + }, + }, + }, + }, + }, + isSvcShared: false, + inplaceUpdateNotReadyBlocker: "true", + isSvcUpdated: true, + }, + // When svc is not shared & pod is pre-updating & svc PublishNotReadyAddresses is false, svc should publish NotReadyAddresses + { + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "xxx", + Name: "case1-0", + UID: "xxx0", + Labels: map[string]string{ + kruisePub.LifecycleStateKey: string(kruisePub.LifecycleStatePreparingUpdate), + gamekruiseiov1alpha1.InplaceUpdateNotReadyBlocker: "true", + gamekruiseiov1alpha1.GameServerOwnerGssKey: "case1", + }, + }, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }, + }, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "name_A", + Image: "v1.0", + }, + { + Name: "name_B", + Image: "v1.0", + }, + }, + }, + }, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + PublishNotReadyAddresses: false, + }, + }, + gss: &gamekruiseiov1alpha1.GameServerSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "GameServerSet", + APIVersion: "game.kruise.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "xxx", + Name: "case1", + UID: "xxx0", + }, + Spec: gamekruiseiov1alpha1.GameServerSetSpec{ + Network: &gamekruiseiov1alpha1.Network{ + NetworkConf: []gamekruiseiov1alpha1.NetworkConfParams{ + { + Name: gamekruiseiov1alpha1.AllowNotReadyContainersNetworkConfName, + Value: "name_B", + }, + }, + }, + GameServerTemplate: gamekruiseiov1alpha1.GameServerTemplate{ + PodTemplateSpec: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "name_A", + Image: "v1.0", + }, + { + Name: "name_B", + Image: "v2.0", + }, + }, + }, + }, + }, + }, + }, + isSvcShared: false, + inplaceUpdateNotReadyBlocker: "true", + isSvcUpdated: true, + }, + // When svc is not shared & pod is pre-updating & svc PublishNotReadyAddresses is true, finalizer of pod should be removed to enter next stage + { + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "xxx", + Name: "case2-0", + UID: "xxx0", + Labels: map[string]string{ + kruisePub.LifecycleStateKey: string(kruisePub.LifecycleStatePreparingUpdate), + gamekruiseiov1alpha1.GameServerOwnerGssKey: "case2", + }, + }, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }, + }, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "name_A", + Image: "v1.0", + }, + { + Name: "name_B", + Image: "v1.0", + }, + }, + }, + }, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + PublishNotReadyAddresses: true, + }, + }, + gss: &gamekruiseiov1alpha1.GameServerSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "GameServerSet", + APIVersion: "game.kruise.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "xxx", + Name: "case2", + UID: "xxx0", + }, + Spec: gamekruiseiov1alpha1.GameServerSetSpec{ + Network: &gamekruiseiov1alpha1.Network{ + NetworkConf: []gamekruiseiov1alpha1.NetworkConfParams{ + { + Name: gamekruiseiov1alpha1.AllowNotReadyContainersNetworkConfName, + Value: "name_B", + }, + }, + }, + GameServerTemplate: gamekruiseiov1alpha1.GameServerTemplate{ + PodTemplateSpec: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "name_A", + Image: "v1.0", + }, + { + Name: "name_B", + Image: "v1.0", + }, + }, + }, + }, + }, + }, + }, + isSvcShared: false, + inplaceUpdateNotReadyBlocker: "false", + isSvcUpdated: false, + }, + } + + for i, test := range tests { + objs := []client.Object{test.gss, test.pod, test.svc} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() + actual, err := AllowNotReadyContainers(c, context.TODO(), test.pod, test.svc, test.isSvcShared) + if err != nil { + t.Errorf("case %d: %s", i, err.Error()) + } + if actual != test.isSvcUpdated { + t.Errorf("case %d: expect isSvcUpdated is %v but actually got %v", i, test.isSvcUpdated, actual) + } + if test.pod.GetLabels()[gamekruiseiov1alpha1.InplaceUpdateNotReadyBlocker] != test.inplaceUpdateNotReadyBlocker { + t.Errorf("case %d: expect inplaceUpdateNotReadyBlocker is %v but actually got %v", i, test.inplaceUpdateNotReadyBlocker, test.pod.GetLabels()[gamekruiseiov1alpha1.InplaceUpdateNotReadyBlocker]) + } + } +} diff --git a/docs/en/user_manuals/network.md b/docs/en/user_manuals/network.md index 3dd0b89d..5c9d9ee3 100644 --- a/docs/en/user_manuals/network.md +++ b/docs/en/user_manuals/network.md @@ -430,6 +430,12 @@ Fixed - Value: false or true. - Configuration change supported or not: yes. +AllowNotReadyContainers + +- Meaning: the container names that are allowed not ready when inplace updating, when traffic will not be cut. +- Value: {containerName_0},{containerName_1},... Example：sidecar +- Configuration change supported or not: It cannot be changed during the in-place updating process. + #### Plugin configuration ``` [alibabacloud] @@ -473,6 +479,12 @@ PortProtocols - Value: in the format of port1/protocol1,port2/protocol2,... The protocol names must be in uppercase letters. - Configuration change supported or not: no. The configuration change can be supported in future. +AllowNotReadyContainers + +- Meaning: the container names that are allowed not ready when inplace updating, when traffic will not be cut. +- Value: {containerName_0},{containerName_1},... Example：sidecar +- Configuration change supported or not: It cannot be changed during the in-place updating process. + #### Plugin configuration None @@ -616,4 +628,107 @@ status: status: InUse ``` -In addition, the generated EIP resource will be named after {pod namespace}/{pod name} in the Alibaba Cloud console, which corresponds to each game server one by one. \ No newline at end of file +In addition, the generated EIP resource will be named after {pod namespace}/{pod name} in the Alibaba Cloud console, which corresponds to each game server one by one. + +--- + +### AlibabaCloud-NLB-SharedPort + +#### Plugin name + +`AlibabaCloud-NLB-SharedPort` + +#### Cloud Provider + +AlibabaCloud + +#### Plugin description + +- AlibabaCloud-NLB-SharedPort enables game servers to be accessed from the Internet by using Layer 4 NLB of Alibaba Cloud, which is similar to AlibabaCloud-SLB-SharedPort. + This network plugin applies to stateless network services, such as proxy or gateway, in gaming scenarios. + +- This network plugin supports network isolation. + +#### Network parameters + +SlbIds + +- Meaning: the CLB instance IDs. You can specify multiple NLB instance IDs. +- Value: an example value can be nlb-9zeo7prq1m25ctpfrw1m7 +- Configuration change supported or not: no. + +PortProtocols + +- Meaning: the ports in the pod to be exposed and the protocols. You can specify multiple ports and protocols. +- Value: in the format of port1/protocol1,port2/protocol2,... The protocol names must be in uppercase letters. +- Configuration change supported or not: no. + +AllowNotReadyContainers + +- Meaning: the container names that are allowed not ready when inplace updating, when traffic will not be cut. +- Value: {containerName_0},{containerName_1},... Example：sidecar +- Configuration change supported or not: It cannot be changed during the in-place updating process. + +#### Plugin configuration + +None + +#### Example + +Deploy a GameServerSet with two containers, one named app-2048 and the other named sidecar. + +Specify the network parameter AllowNotReadyContainers as sidecar, +then the entire pod will still provide services when the sidecar is updated in place. + +```yaml +apiVersion: game.kruise.io/v1alpha1 +kind: GameServerSet +metadata: + name: gss-2048-nlb + namespace: default +spec: + replicas: 3 + updateStrategy: + rollingUpdate: + maxUnavailable: 100% + podUpdatePolicy: InPlaceIfPossible + network: + networkType: AlibabaCloud-NLB-SharedPort + networkConf: + - name: NlbIds + value: nlb-26jbknebrjlejt5abu + - name: PortProtocols + value: 80/TCP + - name: AllowNotReadyContainers + value: sidecar + gameServerTemplate: + spec: + containers: + - image: registry.cn-beijing.aliyuncs.com/acs/2048:v1.0 + name: app-2048 + volumeMounts: + - name: shared-dir + mountPath: /var/www/html/js + - image: registry.cn-beijing.aliyuncs.com/acs/2048-sidecar:v1.0 + name: sidecar + args: + - bash + - -c + - rsync -aP /app/js/* /app/scripts/ && while true; do echo 11;sleep 2; done + volumeMounts: + - name: shared-dir + mountPath: /app/scripts + volumes: + - name: shared-dir + emptyDir: {} +``` + +After successful deployment, update the sidecar image to v2.0 and observe the corresponding endpoint: + +```bash +kubectl get ep -w | grep nlb-26jbknebrjlejt5abu +nlb-26jbknebrjlejt5abu 192.168.0.8:80,192.168.0.82:80,192.168.63.228:80 10m + +``` + +After waiting for the entire update process to end, you can find that there are no changes in the ep, indicating that no extraction has been performed. \ No newline at end of file diff --git "a/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/\347\275\221\347\273\234\346\250\241\345\236\213.md" "b/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/\347\275\221\347\273\234\346\250\241\345\236\213.md" index 0007a067..e6b7268c 100644 --- "a/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/\347\275\221\347\273\234\346\250\241\345\236\213.md" +++ "b/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/\347\275\221\347\273\234\346\250\241\345\236\213.md" @@ -427,6 +427,12 @@ Fixed - 填写格式：false / true - 是否支持变更：支持 +AllowNotReadyContainers + +- 含义：在容器原地升级时允许不断流的对应容器名称，可填写多个 +- 格式：{containerName_0},{containerName_1},... 例如：sidecar +- 是否支持变更：在原地升级过程中不可变更。 + #### 插件配置 ``` [alibabacloud] @@ -470,6 +476,12 @@ PortProtocols - 格式：port1/protocol1,port2/protocol2,...（协议需大写） - 是否支持变更：暂不支持。未来将支持 +AllowNotReadyContainers + +- 含义：在容器原地升级时允许不断流的对应容器名称，可填写多个 +- 格式：{containerName_0},{containerName_1},... 例如：sidecar +- 是否支持变更：在原地升级过程中不可变更。 + #### 插件配置 无 @@ -616,6 +628,104 @@ status: 此外，生成的EIP资源在阿里云控制台中会以{pod namespace}/{pod name}命名，与每一个游戏服一一对应。 +### AlibabaCloud-NLB-SharedPort + +#### 插件名称 + +`AlibabaCloud-NLB-SharedPort` + +#### Cloud Provider + +AlibabaCloud + +#### 插件说明 + +- AlibabaCloud-NLB-SharedPort 使用阿里云网络型负载均衡（NLB）作为对外服务的承载实体。其与AlibabaCloud-SLB-SharedPort作用类似。 + 适用于游戏场景下代理（proxy）或网关等无状态网络服务。 + +- 是否支持网络隔离：是 + +#### 网络参数 + +SlbIds + +- 含义：填写nlb的id，暂不支持填写多例 +- 填写格式：例如：nlb-9zeo7prq1m25ctpfrw1m7 +- 是否支持变更：暂不支持。 + +PortProtocols + +- 含义：pod暴露的端口及协议，支持填写多个端口/协议 +- 格式：port1/protocol1,port2/protocol2,...（协议需大写） +- 是否支持变更：暂不支持。 + +AllowNotReadyContainers + +- 含义：在容器原地升级时允许不断流的对应容器名称，可填写多个 +- 格式：{containerName_0},{containerName_1},... 例如：sidecar +- 是否支持变更：在原地升级过程中不可变更。 + +#### 插件配置 + +无 + +#### 示例说明 + +部署一个具有两个容器的GameServerSet，一个容器名为app-2048，另一个为sidecar。 +指定网络参数 AllowNotReadyContainers 为 sidecar，则在sidecar原地更新时整个pod依然会提供服务，不会断流。 + +```yaml +apiVersion: game.kruise.io/v1alpha1 +kind: GameServerSet +metadata: + name: gss-2048-nlb + namespace: default +spec: + replicas: 3 + updateStrategy: + rollingUpdate: + maxUnavailable: 100% + podUpdatePolicy: InPlaceIfPossible + network: + networkType: AlibabaCloud-NLB-SharedPort + networkConf: + - name: NlbIds + value: nlb-26jbknebrjlejt5abu + - name: PortProtocols + value: 80/TCP + - name: AllowNotReadyContainers + value: sidecar + gameServerTemplate: + spec: + containers: + - image: registry.cn-beijing.aliyuncs.com/acs/2048:v1.0 + name: app-2048 + volumeMounts: + - name: shared-dir + mountPath: /var/www/html/js + - image: registry.cn-beijing.aliyuncs.com/acs/2048-sidecar:v1.0 + name: sidecar + args: + - bash + - -c + - rsync -aP /app/js/* /app/scripts/ && while true; do echo 11;sleep 2; done + volumeMounts: + - name: shared-dir + mountPath: /app/scripts + volumes: + - name: shared-dir + emptyDir: {} +``` + +部署成功后，将sidecar镜像更新到v2.0版本，同时观察对应endpoint情况: +```bash +kubectl get ep -w | grep nlb-26jbknebrjlejt5abu +nlb-26jbknebrjlejt5abu 192.168.0.8:80,192.168.0.82:80,192.168.63.228:80 10m + +``` + +等待整个更新过程结束，可以发现ep没有任何变化，说明并未进行摘流。 + ## 获取网络信息 GameServer Network Status可以通过两种方式获取 diff --git a/pkg/util/gameserver.go b/pkg/util/gameserver.go index 6c24ecfc..c50d2d9b 100644 --- a/pkg/util/gameserver.go +++ b/pkg/util/gameserver.go @@ -132,6 +132,16 @@ func GetNewAstsFromGss(gss *gameKruiseV1alpha1.GameServerSet, asts *kruiseV1beta readinessGates = append(readinessGates, corev1.PodReadinessGate{ConditionType: appspub.InPlaceUpdateReady}) asts.Spec.Template.Spec.ReadinessGates = readinessGates + // AllowNotReadyContainers + if gss.Spec.Network != nil && IsAllowNotReadyContainers(gss.Spec.Network.NetworkConf) { + // set lifecycle + asts.Spec.Lifecycle = &appspub.Lifecycle{ + InPlaceUpdate: &appspub.LifecycleHook{ + LabelsHandler: map[string]string{gameKruiseV1alpha1.InplaceUpdateNotReadyBlocker: "true"}, + }, + } + } + // set VolumeClaimTemplates asts.Spec.VolumeClaimTemplates = gss.Spec.GameServerTemplate.VolumeClaimTemplates @@ -210,3 +220,12 @@ func GetGameServerSetOfPod(pod *corev1.Pod, c client.Client, ctx context.Context }, gss) return gss, err } + +func IsAllowNotReadyContainers(networkConfParams []gameKruiseV1alpha1.NetworkConfParams) bool { + for _, networkConfParam := range networkConfParams { + if networkConfParam.Name == gameKruiseV1alpha1.AllowNotReadyContainersNetworkConfName { + return true + } + } + return false +} diff --git a/pkg/util/gameserver_test.go b/pkg/util/gameserver_test.go index fdf6fe23..e9b9d57c 100644 --- a/pkg/util/gameserver_test.go +++ b/pkg/util/gameserver_test.go @@ -325,3 +325,37 @@ func TestGetGsTemplateMetadataHash(t *testing.T) { } } } + +func TestIsAllowNotReadyContainers(t *testing.T) { + tests := []struct { + networkConfParams []gameKruiseV1alpha1.NetworkConfParams + isAllowNotReadyContainers bool + }{ + { + networkConfParams: []gameKruiseV1alpha1.NetworkConfParams{ + { + Name: gameKruiseV1alpha1.AllowNotReadyContainersNetworkConfName, + Value: "xxx", + }, + }, + isAllowNotReadyContainers: true, + }, + { + networkConfParams: []gameKruiseV1alpha1.NetworkConfParams{ + { + Name: "xxx", + Value: "xxx", + }, + }, + isAllowNotReadyContainers: false, + }, + } + + for i, test := range tests { + actual := IsAllowNotReadyContainers(test.networkConfParams) + expect := test.isAllowNotReadyContainers + if actual != expect { + t.Errorf("case %d: expect isAllowNotReadyContainers is %v but actually got %v", i, expect, actual) + } + } +} diff --git a/pkg/util/pod.go b/pkg/util/pod.go index 38828a3b..b450f662 100644 --- a/pkg/util/pod.go +++ b/pkg/util/pod.go @@ -16,7 +16,10 @@ limitations under the License. package util -import corev1 "k8s.io/api/core/v1" +import ( + gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + corev1 "k8s.io/api/core/v1" +) // GetPodConditionFromList extracts the provided condition from the given list of condition and // returns the index of the condition and the condition. Returns -1 and nil if the condition is not present. @@ -31,3 +34,20 @@ func GetPodConditionFromList(conditions []corev1.PodCondition, conditionType cor } return -1, nil } + +func IsContainersPreInplaceUpdating(pod *corev1.Pod, gss *gameKruiseV1alpha1.GameServerSet, containerNames []string) bool { + var diffNames []string + for _, actual := range pod.Status.ContainerStatuses { + for _, expect := range gss.Spec.GameServerTemplate.Spec.Containers { + if actual.Name == expect.Name && actual.Image != expect.Image { + diffNames = append(diffNames, actual.Name) + } + } + } + for _, containerName := range containerNames { + if IsStringInList(containerName, diffNames) { + return true + } + } + return false +} diff --git a/pkg/util/pod_test.go b/pkg/util/pod_test.go index b1ee0fb8..393c1233 100644 --- a/pkg/util/pod_test.go +++ b/pkg/util/pod_test.go @@ -17,6 +17,7 @@ limitations under the License. package util import ( + gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" corev1 "k8s.io/api/core/v1" "reflect" "testing" @@ -59,3 +60,135 @@ func TestGetPodConditionFromList(t *testing.T) { } } } + +func TestIsContainersPreInplaceUpdating(t *testing.T) { + tests := []struct { + pod *corev1.Pod + gss *gameKruiseV1alpha1.GameServerSet + containerNames []string + isUpdating bool + }{ + { + pod: &corev1.Pod{ + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "name_A", + Image: "v1.0", + }, + { + Name: "name_B", + Image: "v1.0", + }, + }, + }, + }, + gss: &gameKruiseV1alpha1.GameServerSet{ + Spec: gameKruiseV1alpha1.GameServerSetSpec{ + GameServerTemplate: gameKruiseV1alpha1.GameServerTemplate{ + PodTemplateSpec: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "name_A", + Image: "v1.0", + }, + { + Name: "name_B", + Image: "v2.0", + }, + }, + }, + }, + }, + }, + }, + containerNames: []string{"name_B"}, + isUpdating: true, + }, + { + pod: &corev1.Pod{ + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "name_A", + Image: "v1.0", + }, + { + Name: "name_B", + Image: "v1.0", + }, + }, + }, + }, + gss: &gameKruiseV1alpha1.GameServerSet{ + Spec: gameKruiseV1alpha1.GameServerSetSpec{ + GameServerTemplate: gameKruiseV1alpha1.GameServerTemplate{ + PodTemplateSpec: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "name_A", + Image: "v1.0", + }, + { + Name: "name_B", + Image: "v2.0", + }, + }, + }, + }, + }, + }, + }, + containerNames: []string{"name_A"}, + isUpdating: false, + }, + { + pod: &corev1.Pod{ + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "name_A", + Image: "v1.0", + }, + { + Name: "name_B", + Image: "v1.0", + }, + }, + }, + }, + gss: &gameKruiseV1alpha1.GameServerSet{ + Spec: gameKruiseV1alpha1.GameServerSetSpec{ + GameServerTemplate: gameKruiseV1alpha1.GameServerTemplate{ + PodTemplateSpec: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "name_A", + Image: "v1.0", + }, + { + Name: "name_B", + Image: "v1.0", + }, + }, + }, + }, + }, + }, + }, + containerNames: []string{"name_B"}, + isUpdating: false, + }, + } + + for i, test := range tests { + actual := IsContainersPreInplaceUpdating(test.pod, test.gss, test.containerNames) + expect := test.isUpdating + if actual != expect { + t.Errorf("case %d: expect IsContainersPreInplaceUpdating is %v but actually got %v", i, expect, actual) + } + } +}