diff --git a/apis/v1alpha1/gameserverset_types.go b/apis/v1alpha1/gameserverset_types.go index ffa19d99..f9fbff52 100644 --- a/apis/v1alpha1/gameserverset_types.go +++ b/apis/v1alpha1/gameserverset_types.go @@ -64,8 +64,22 @@ type GameServerTemplate struct { // +kubebuilder:validation:Schemaless corev1.PodTemplateSpec `json:",inline"` VolumeClaimTemplates []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"` + // LifecyclePolicyType indicates the policy for GameServer creation and deletion. + LifecyclePolicyType LifecyclePolicyType `json:"lifecyclePolicyType,omitempty"` } +type LifecyclePolicyType string + +const ( + // FollowByPodLifecyclePolicyType indicates that GameServer is created when the pod is created + // and deleted when the pod is deleted. The age of GameServer is exactly the same as that of the pod. + FollowByPodLifecyclePolicyType LifecyclePolicyType = "FollowByPod" + // WhenScaledLifecyclePolicyType indicates that GameServers will be created when replicas of GameServerSet + // increases and will be deleted when replicas of GameServerSet decreases. The GameServer will not be deleted + // when the corresponding pod is deleted due to manual deletion, update, eviction, etc. + WhenScaledLifecyclePolicyType LifecyclePolicyType = "WhenScaled" +) + type Network struct { NetworkType string `json:"networkType,omitempty"` NetworkConf []NetworkConfParams `json:"networkConf,omitempty"` diff --git a/config/crd/bases/game.kruise.io_gameserversets.yaml b/config/crd/bases/game.kruise.io_gameserversets.yaml index 12d1cce5..940a2c4d 100644 --- a/config/crd/bases/game.kruise.io_gameserversets.yaml +++ b/config/crd/bases/game.kruise.io_gameserversets.yaml @@ -70,6 +70,10 @@ spec: description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster Important: Run "make" to regenerate code after modifying this file' properties: + lifecyclePolicyType: + description: LifecyclePolicyType indicates the policy for GameServer + creation and deletion. + type: string volumeClaimTemplates: items: description: PersistentVolumeClaim is a user's request for and diff --git a/docs/en/user_manuals/CRD_field_description.md b/docs/en/user_manuals/CRD_field_description.md index 8f05b14b..980ff853 100644 --- a/docs/en/user_manuals/CRD_field_description.md +++ b/docs/en/user_manuals/CRD_field_description.md @@ -38,8 +38,22 @@ type GameServerTemplate struct { // Requests and claims for persistent volumes. VolumeClaimTemplates []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"` + + // LifecyclePolicyType indicates the policy for GameServer creation and deletion. + LifecyclePolicyType LifecyclePolicyType `json:"lifecyclePolicyType,omitempty"` } +type LifecyclePolicyType string + +// FollowByPodLifecyclePolicyType indicates that GameServer is created when the pod is created +// and deleted when the pod is deleted. The age of GameServer is exactly the same as that of the pod. +FollowByPodLifecyclePolicyType LifecyclePolicyType = "FollowByPod" + +// WhenScaledLifecyclePolicyType indicates that GameServers will be created when replicas of GameServerSet +// increases and will be deleted when replicas of GameServerSet decreases. The GameServer will not be deleted +// when the corresponding pod is deleted due to manual deletion, update, eviction, etc. +WhenScaledLifecyclePolicyType LifecyclePolicyType = "WhenScaled" + ``` #### UpdateStrategy diff --git "a/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/CRD\345\255\227\346\256\265\350\257\264\346\230\216.md" "b/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/CRD\345\255\227\346\256\265\350\257\264\346\230\216.md" index fae99331..7afbc7ed 100644 --- "a/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/CRD\345\255\227\346\256\265\350\257\264\346\230\216.md" +++ "b/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/CRD\345\255\227\346\256\265\350\257\264\346\230\216.md" @@ -72,7 +72,21 @@ type GameServerTemplate struct { // 对持久卷的请求和声明 VolumeClaimTemplates []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"` + + // LifecyclePolicyType 表明GameServer创建与删除的策略 + LifecyclePolicyType LifecyclePolicyType `json:"lifecyclePolicyType,omitempty"` } + +type LifecyclePolicyType string + +// FollowByPodLifecyclePolicyType 表示创建 pod 时创建 GameServer, 并在删除 pod 时删除。 +// 此策略类型下，GameServer 的年龄与 pod 的年龄完全相同。 +FollowByPodLifecyclePolicyType LifecyclePolicyType = "FollowByPod" + +// WhenScaledLifecyclePolicyType 表示当 GameServerSet 的副本增加时将创建 GameServers，当 GameServerSet 的副本减少时将被删除。 +// 由于手动删除、更新、驱逐等原因删除相应的 pod 时，GameServer 不会被删除。 +WhenScaledLifecyclePolicyType LifecyclePolicyType = "WhenScaled" + ``` ### UpdateStrategy diff --git a/pkg/controllers/gameserver/gameserver_controller.go b/pkg/controllers/gameserver/gameserver_controller.go index f423fb16..af7ff77a 100644 --- a/pkg/controllers/gameserver/gameserver_controller.go +++ b/pkg/controllers/gameserver/gameserver_controller.go @@ -26,7 +26,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" @@ -173,8 +172,15 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) } if podFound && !gsFound { - err := r.initGameServer(pod) - if err != nil && !errors.IsAlreadyExists(err) && !errors.IsNotFound(err) { + gss, err := r.getGameServerSet(pod) + if err != nil { + if errors.IsNotFound(err) { + return reconcile.Result{}, nil + } + return reconcile.Result{}, err + } + err = r.initGameServerByPod(gss, pod) + if err != nil && !errors.IsAlreadyExists(err) { klog.Errorf("failed to create GameServer %s in %s, because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) return reconcile.Result{}, err } @@ -237,57 +243,24 @@ func (r *GameServerReconciler) getGameServerSet(pod *corev1.Pod) (*gamekruiseiov return gss, err } -func (r *GameServerReconciler) initGameServer(pod *corev1.Pod) error { - gs := &gamekruiseiov1alpha1.GameServer{} - gs.Name = pod.GetName() - gs.Namespace = pod.GetNamespace() - - // set owner reference - gss, err := r.getGameServerSet(pod) - if err != nil { - return err - } - ors := make([]metav1.OwnerReference, 0) - or := metav1.OwnerReference{ - APIVersion: gss.APIVersion, - Kind: gss.Kind, - Name: gss.GetName(), - UID: gss.GetUID(), - Controller: pointer.BoolPtr(true), - BlockOwnerDeletion: pointer.BoolPtr(true), - } - ors = append(ors, or) - gs.OwnerReferences = ors - - // set Labels - gsLabels := gss.Spec.GameServerTemplate.GetLabels() - if gsLabels == nil { - gsLabels = make(map[string]string) - } - gsLabels[gamekruiseiov1alpha1.GameServerOwnerGssKey] = gss.GetName() - gs.SetLabels(gsLabels) - - // set Annotations - gsAnnotations := gss.Spec.GameServerTemplate.GetAnnotations() - if gsAnnotations == nil { - gsAnnotations = make(map[string]string) +func (r *GameServerReconciler) initGameServerByPod(gss *gamekruiseiov1alpha1.GameServerSet, pod *corev1.Pod) error { + // default fields + gs := util.InitGameServer(gss, pod.Name) + + if gss.Spec.GameServerTemplate.LifecyclePolicyType == gamekruiseiov1alpha1.FollowByPodLifecyclePolicyType || gss.Spec.GameServerTemplate.LifecyclePolicyType == "" { + // rewrite ownerReferences + ors := make([]metav1.OwnerReference, 0) + or := metav1.OwnerReference{ + APIVersion: pod.APIVersion, + Kind: pod.Kind, + Name: pod.GetName(), + UID: pod.GetUID(), + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + } + ors = append(ors, or) + gs.OwnerReferences = ors } - gsAnnotations[gamekruiseiov1alpha1.GsTemplateMetadataHashKey] = util.GetGsTemplateMetadataHash(gss) - gs.SetAnnotations(gsAnnotations) - - // set NetWork - gs.Spec.NetworkDisabled = false - - // set OpsState - gs.Spec.OpsState = gamekruiseiov1alpha1.None - - // set UpdatePriority - updatePriority := intstr.FromInt(0) - gs.Spec.UpdatePriority = &updatePriority - - // set deletionPriority - deletionPriority := intstr.FromInt(0) - gs.Spec.DeletionPriority = &deletionPriority return r.Client.Create(context.Background(), gs) } diff --git a/pkg/controllers/gameserverset/gameserverset_controller.go b/pkg/controllers/gameserverset/gameserverset_controller.go index 26a12c1f..94d7bf93 100644 --- a/pkg/controllers/gameserverset/gameserverset_controller.go +++ b/pkg/controllers/gameserverset/gameserverset_controller.go @@ -180,6 +180,15 @@ func (r *GameServerSetReconciler) Reconcile(ctx context.Context, req ctrl.Reques err = r.Get(ctx, namespacedName, asts) if err != nil { if errors.IsNotFound(err) { + // init GameServers when owner is GameServerSet + if gss.Spec.GameServerTemplate.LifecyclePolicyType == gamekruiseiov1alpha1.WhenScaledLifecyclePolicyType { + newManageIds, _ := computeToScaleGs(gss.Spec.ReserveGameServerIds, []int{}, []int{}, int(*gss.Spec.Replicas), []corev1.Pod{}, gamekruiseiov1alpha1.GeneralScaleDownStrategyType) + err := SyncGameServer(gss, r.Client, newManageIds, []int{}) + if err != nil { + klog.Errorf("failed to sync GameServers %s in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + } err = r.initAsts(gss) if err != nil { klog.Errorf("failed to create advanced statefulset %s in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) @@ -252,12 +261,6 @@ func (r *GameServerSetReconciler) Reconcile(ctx context.Context, req ctrl.Reques return reconcile.Result{}, err } - err = gsm.SyncGameServerReplicas() - if err != nil { - klog.Errorf("GameServerSet %s failed to adjust the replicas of GameServers to match that of Pods in %s, because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) - return reconcile.Result{}, err - } - return ctrl.Result{}, nil } diff --git a/pkg/controllers/gameserverset/gameserverset_manager.go b/pkg/controllers/gameserverset/gameserverset_manager.go index 7a6e17a3..10458685 100644 --- a/pkg/controllers/gameserverset/gameserverset_manager.go +++ b/pkg/controllers/gameserverset/gameserverset_manager.go @@ -18,12 +18,6 @@ package gameserverset import ( "context" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/tools/record" - "sort" - "sync" - "time" - kruiseV1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1" kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" corev1 "k8s.io/api/core/v1" @@ -32,9 +26,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/json" + "k8s.io/client-go/tools/record" "k8s.io/klog/v2" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" + "sort" + "strconv" + "sync" gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" "github.com/openkruise/kruise-game/pkg/util" @@ -47,7 +45,7 @@ type Control interface { IsNeedToScale() bool IsNeedToUpdateWorkload() bool SyncPodProbeMarker() error - SyncGameServerReplicas() error + //SyncGameServerReplicas() error GetReplicasAfterKilling() *int32 } @@ -129,7 +127,15 @@ func (manager *GameServerSetManager) GameServerScale() error { klog.Infof("GameServers %s/%s already has %d replicas, expect to have %d replicas.", gss.GetNamespace(), gss.GetName(), currentReplicas, expectedReplicas) manager.eventRecorder.Eventf(gss, corev1.EventTypeNormal, ScaleReason, "scale from %d to %d", currentReplicas, expectedReplicas) - newReserveIds := computeToScaleGs(gssReserveIds, reserveIds, notExistIds, expectedReplicas, podList, gss.Spec.ScaleStrategy.ScaleDownStrategyType) + newManageIds, newReserveIds := computeToScaleGs(gssReserveIds, reserveIds, notExistIds, expectedReplicas, podList, gss.Spec.ScaleStrategy.ScaleDownStrategyType) + + if gss.Spec.GameServerTemplate.LifecyclePolicyType == gameKruiseV1alpha1.WhenScaledLifecyclePolicyType { + err := SyncGameServer(gss, c, newManageIds, util.GetIndexListFromPodList(podList)) + if err != nil { + return err + } + } + asts.Spec.ReserveOrdinals = newReserveIds asts.Spec.Replicas = gss.Spec.Replicas asts.Spec.ScaleStrategy = &kruiseV1beta1.StatefulSetScaleStrategy{ @@ -157,7 +163,7 @@ func (manager *GameServerSetManager) GameServerScale() error { return nil } -func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedReplicas int, pods []corev1.Pod, scaleDownType gameKruiseV1alpha1.ScaleDownStrategyType) []int { +func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedReplicas int, pods []corev1.Pod, scaleDownType gameKruiseV1alpha1.ScaleDownStrategyType) ([]int, []int) { workloadManageIds := util.GetIndexListFromPodList(pods) var toAdd []int @@ -215,12 +221,13 @@ func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedRepl } } + newManageIds := append(workloadManageIds, util.GetSliceInANotInB(toAdd, workloadManageIds)...) + newManageIds = util.GetSliceInANotInB(newManageIds, toDelete) + if scaleDownType == gameKruiseV1alpha1.ReserveIdsScaleDownStrategyType { - return append(gssReserveIds, util.GetSliceInANotInB(toDelete, gssReserveIds)...) + return newManageIds, append(gssReserveIds, util.GetSliceInANotInB(toDelete, gssReserveIds)...) } - newManageIds := append(workloadManageIds, util.GetSliceInANotInB(toAdd, workloadManageIds)...) - newManageIds = util.GetSliceInANotInB(newManageIds, toDelete) var newReserveIds []int if len(newManageIds) != 0 { sort.Ints(newManageIds) @@ -231,63 +238,89 @@ func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedRepl } } - return newReserveIds + return newManageIds, newReserveIds } -func (manager *GameServerSetManager) SyncGameServerReplicas() error { - gss := manager.gameServerSet - gsList := &gameKruiseV1alpha1.GameServerList{} - err := manager.client.List(context.Background(), gsList, &client.ListOptions{ - Namespace: gss.GetNamespace(), - LabelSelector: labels.SelectorFromSet(map[string]string{ - gameKruiseV1alpha1.GameServerOwnerGssKey: gss.GetName(), - })}) - if err != nil { - return err - } - podIds := util.GetIndexListFromPodList(manager.podList) - - gsMap := make(map[int]int) - deleteIds := make([]int, 0) - for id, gs := range gsList.Items { - gsId := util.GetIndexFromGsName(gs.Name) - if !util.IsNumInList(gsId, podIds) { - gsMap[gsId] = id - deleteIds = append(deleteIds, gsId) - } - } - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) +func SyncGameServer(gss *gameKruiseV1alpha1.GameServerSet, c client.Client, newManageIds, oldManageIds []int) error { + ctx, cancel := context.WithCancel(context.Background()) defer cancel() - errch := make(chan error, len(deleteIds)) + addIds := util.GetSliceInANotInB(newManageIds, oldManageIds) + klog.Infof("GameServerSet %s/%s will add GameServer Id: %v", gss.Namespace, gss.Name, addIds) + deleteIds := util.GetSliceInANotInB(oldManageIds, newManageIds) + klog.Infof("GameServerSet %s/%s will delete GameServer Id: %v", gss.Namespace, gss.Name, deleteIds) + + errch := make(chan error, len(addIds)+len(deleteIds)) var wg sync.WaitGroup - for _, gsId := range deleteIds { + for _, gsId := range append(addIds, deleteIds...) { wg.Add(1) id := gsId go func(ctx context.Context) { defer wg.Done() defer ctx.Done() - gs := gsList.Items[gsMap[id]].DeepCopy() - gsLabels := make(map[string]string) - gsLabels[gameKruiseV1alpha1.GameServerDeletingKey] = "true" - patchGs := map[string]interface{}{"metadata": map[string]map[string]string{"labels": gsLabels}} - patchBytes, err := json.Marshal(patchGs) + gs := &gameKruiseV1alpha1.GameServer{} + gsName := gss.Name + "-" + strconv.Itoa(id) + err := c.Get(ctx, types.NamespacedName{ + Name: gsName, + Namespace: gss.Namespace, + }, gs) if err != nil { + if errors.IsNotFound(err) { + if util.IsNumInList(id, addIds) { + err = c.Create(ctx, util.InitGameServer(gss, gsName)) + if err != nil { + errch <- err + } + klog.Infof("GameServer %s/%s added", gss.Namespace, gsName) + } + return + } errch <- err + return } - err = manager.client.Patch(context.TODO(), gs, client.RawPatch(types.MergePatchType, patchBytes)) - if err != nil && !errors.IsNotFound(err) { - errch <- err + + if util.IsNumInList(id, addIds) && gs.GetLabels()[gameKruiseV1alpha1.GameServerDeletingKey] == "true" { + gsLabels := make(map[string]string) + gsLabels[gameKruiseV1alpha1.GameServerDeletingKey] = "false" + patchGs := map[string]interface{}{"metadata": map[string]map[string]string{"labels": gsLabels}} + patchBytes, err := json.Marshal(patchGs) + if err != nil { + errch <- err + return + } + err = c.Patch(ctx, gs, client.RawPatch(types.MergePatchType, patchBytes)) + if err != nil && !errors.IsNotFound(err) { + errch <- err + return + } + klog.Infof("GameServer %s/%s DeletingKey turn into false", gss.Namespace, gsName) } + + if util.IsNumInList(id, deleteIds) && gs.GetLabels()[gameKruiseV1alpha1.GameServerDeletingKey] != "true" { + gsLabels := make(map[string]string) + gsLabels[gameKruiseV1alpha1.GameServerDeletingKey] = "true" + patchGs := map[string]interface{}{"metadata": map[string]map[string]string{"labels": gsLabels}} + patchBytes, err := json.Marshal(patchGs) + if err != nil { + errch <- err + return + } + err = c.Patch(ctx, gs, client.RawPatch(types.MergePatchType, patchBytes)) + if err != nil && !errors.IsNotFound(err) { + errch <- err + return + } + klog.Infof("GameServer %s/%s DeletingKey turn into true, who will be deleted", gss.Namespace, gsName) + } + }(ctx) } + wg.Wait() close(errch) - err = <-errch + err := <-errch if err != nil { - klog.Errorf("failed to delete GameServers %s in %s because of %s.", gss.GetName(), gss.GetNamespace(), err.Error()) return err } diff --git a/pkg/controllers/gameserverset/gameserverset_manager_test.go b/pkg/controllers/gameserverset/gameserverset_manager_test.go index e1f1e7fb..a68afc73 100644 --- a/pkg/controllers/gameserverset/gameserverset_manager_test.go +++ b/pkg/controllers/gameserverset/gameserverset_manager_test.go @@ -8,7 +8,6 @@ import ( "github.com/openkruise/kruise-game/pkg/util" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -16,6 +15,7 @@ import ( "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "strconv" "testing" ) @@ -39,6 +39,7 @@ func TestComputeToScaleGs(t *testing.T) { scaleDownStrategyType gameKruiseV1alpha1.ScaleDownStrategyType pods []corev1.Pod newReserveIds []int + newManageIds []int }{ { newGssReserveIds: []int{2, 3, 4}, @@ -85,6 +86,7 @@ func TestComputeToScaleGs(t *testing.T) { }, }, newReserveIds: []int{2, 3, 4, 5}, + newManageIds: []int{0, 1, 6}, }, { newGssReserveIds: []int{0, 2, 3}, @@ -140,6 +142,7 @@ func TestComputeToScaleGs(t *testing.T) { }, }, newReserveIds: []int{0, 2, 3, 4, 5}, + newManageIds: []int{1, 6, 7}, }, { newGssReserveIds: []int{0}, @@ -195,6 +198,7 @@ func TestComputeToScaleGs(t *testing.T) { }, }, newReserveIds: []int{0}, + newManageIds: []int{1}, }, { newGssReserveIds: []int{0, 2, 3}, @@ -250,6 +254,7 @@ func TestComputeToScaleGs(t *testing.T) { }, }, newReserveIds: []int{0, 2, 3, 5}, + newManageIds: []int{1, 4, 6, 7}, }, { newGssReserveIds: []int{0, 3, 5}, @@ -296,6 +301,7 @@ func TestComputeToScaleGs(t *testing.T) { }, }, newReserveIds: []int{0, 3, 5, 2, 4, 6}, + newManageIds: []int{1}, }, { newGssReserveIds: []int{1, 2}, @@ -324,14 +330,38 @@ func TestComputeToScaleGs(t *testing.T) { }, }, newReserveIds: []int{1, 2}, + newManageIds: []int{0, 3}, + }, + { + newGssReserveIds: []int{}, + oldGssreserveIds: []int{}, + notExistIds: []int{}, + expectedReplicas: 3, + scaleDownStrategyType: gameKruiseV1alpha1.GeneralScaleDownStrategyType, + pods: []corev1.Pod{}, + newReserveIds: []int{}, + newManageIds: []int{0, 1, 2}, + }, + { + newGssReserveIds: []int{1, 2}, + oldGssreserveIds: []int{}, + notExistIds: []int{}, + expectedReplicas: 3, + scaleDownStrategyType: gameKruiseV1alpha1.GeneralScaleDownStrategyType, + pods: []corev1.Pod{}, + newReserveIds: []int{1, 2}, + newManageIds: []int{0, 3, 4}, }, } for i, test := range tests { - newReserveIds := computeToScaleGs(test.newGssReserveIds, test.oldGssreserveIds, test.notExistIds, test.expectedReplicas, test.pods, test.scaleDownStrategyType) + newManageIds, newReserveIds := computeToScaleGs(test.newGssReserveIds, test.oldGssreserveIds, test.notExistIds, test.expectedReplicas, test.pods, test.scaleDownStrategyType) if !util.IsSliceEqual(newReserveIds, test.newReserveIds) { t.Errorf("case %d: expect newNotExistIds %v but got %v", i, test.newReserveIds, newReserveIds) } + if !util.IsSliceEqual(newManageIds, test.newManageIds) { + t.Errorf("case %d: expect newManageIds %v but got %v", i, test.newManageIds, newManageIds) + } } } @@ -700,13 +730,17 @@ func TestGameServerScale(t *testing.T) { } } -func TestSyncGameServerReplicas(t *testing.T) { +func TestSyncGameServer(t *testing.T) { tests := []struct { - gss *gameKruiseV1alpha1.GameServerSet - podList []corev1.Pod - gsList []*gameKruiseV1alpha1.GameServer - toDelete []int + gss *gameKruiseV1alpha1.GameServerSet + gsList []*gameKruiseV1alpha1.GameServer + newManageIds []int + oldManageIds []int + IdsToCreate []int + IdsLabelTure []int + IdsLabelFalse []int }{ + // case 0 { gss: &gameKruiseV1alpha1.GameServerSet{ ObjectMeta: metav1.ObjectMeta{ @@ -714,23 +748,6 @@ func TestSyncGameServerReplicas(t *testing.T) { Name: "xxx", }, }, - podList: []corev1.Pod{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "xxx-0", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "xxx-2", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "xxx-4", - }, - }, - }, gsList: []*gameKruiseV1alpha1.GameServer{ { ObjectMeta: metav1.ObjectMeta{ @@ -769,41 +786,101 @@ func TestSyncGameServerReplicas(t *testing.T) { }, }, }, - toDelete: []int{3}, + oldManageIds: []int{0, 2, 3, 4}, + newManageIds: []int{0, 1}, + IdsToCreate: []int{1}, + IdsLabelTure: []int{2, 3, 4}, + IdsLabelFalse: []int{}, + }, + + // case 1 + { + gss: &gameKruiseV1alpha1.GameServerSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "xxx", + Name: "xxx", + }, + }, + gsList: []*gameKruiseV1alpha1.GameServer{}, + oldManageIds: []int{}, + newManageIds: []int{0, 1, 3}, + IdsToCreate: []int{0, 1, 3}, + IdsLabelTure: []int{}, + IdsLabelFalse: []int{}, + }, + + // case 2 + { + gss: &gameKruiseV1alpha1.GameServerSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "xxx", + Name: "xxx", + }, + }, + gsList: []*gameKruiseV1alpha1.GameServer{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "xxx", + Name: "xxx-0", + Labels: map[string]string{ + gameKruiseV1alpha1.GameServerOwnerGssKey: "xxx", + gameKruiseV1alpha1.GameServerDeletingKey: "true", + }, + }, + }, + }, + oldManageIds: []int{}, + newManageIds: []int{0}, + IdsToCreate: []int{}, + IdsLabelTure: []int{}, + IdsLabelFalse: []int{0}, }, } - for _, test := range tests { + for i, test := range tests { objs := []client.Object{test.gss} for _, gs := range test.gsList { objs = append(objs, gs) } c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() - recorder := record.NewFakeRecorder(100) - manager := &GameServerSetManager{ - gameServerSet: test.gss, - podList: test.podList, - eventRecorder: recorder, - client: c, - } - if err := manager.SyncGameServerReplicas(); err != nil { + if err := SyncGameServer(test.gss, c, test.newManageIds, test.oldManageIds); err != nil { t.Error(err) } - gsList := &gameKruiseV1alpha1.GameServerList{} - if err := manager.client.List(context.Background(), gsList, &client.ListOptions{ - Namespace: test.gss.GetNamespace(), - LabelSelector: labels.SelectorFromSet(map[string]string{ - gameKruiseV1alpha1.GameServerOwnerGssKey: test.gss.GetName(), - gameKruiseV1alpha1.GameServerDeletingKey: "true", - })}); err != nil { - t.Error(err) + for _, id := range test.IdsToCreate { + gs := &gameKruiseV1alpha1.GameServer{} + if err := c.Get(context.Background(), types.NamespacedName{ + Namespace: test.gss.GetNamespace(), + Name: test.gss.GetName() + "-" + strconv.Itoa(id), + }, gs); err != nil { + t.Error(err) + } + } + + for _, id := range test.IdsLabelTure { + gs := &gameKruiseV1alpha1.GameServer{} + if err := c.Get(context.Background(), types.NamespacedName{ + Namespace: test.gss.GetNamespace(), + Name: test.gss.GetName() + "-" + strconv.Itoa(id), + }, gs); err != nil { + t.Error(err) + } + if gs.GetLabels()[gameKruiseV1alpha1.GameServerDeletingKey] != "true" { + t.Errorf("case %d: gs %d GameServerDeletingKey is not true", i, id) + } } - actual := util.GetIndexListFromGsList(gsList.Items) - expect := test.toDelete - if !util.IsSliceEqual(actual, expect) { - t.Errorf("expect to delete gameservers %v but actually %v", expect, actual) + for _, id := range test.IdsLabelFalse { + gs := &gameKruiseV1alpha1.GameServer{} + if err := c.Get(context.Background(), types.NamespacedName{ + Namespace: test.gss.GetNamespace(), + Name: test.gss.GetName() + "-" + strconv.Itoa(id), + }, gs); err != nil { + t.Error(err) + } + if gs.GetLabels()[gameKruiseV1alpha1.GameServerDeletingKey] != "false" { + t.Errorf("case %d: gs %d GameServerDeletingKey is not false", i, id) + } } } } diff --git a/pkg/util/gameserver.go b/pkg/util/gameserver.go index c50d2d9b..e20fea79 100644 --- a/pkg/util/gameserver.go +++ b/pkg/util/gameserver.go @@ -26,6 +26,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "strconv" "strings" @@ -229,3 +231,53 @@ func IsAllowNotReadyContainers(networkConfParams []gameKruiseV1alpha1.NetworkCon } return false } + +func InitGameServer(gss *gameKruiseV1alpha1.GameServerSet, name string) *gameKruiseV1alpha1.GameServer { + gs := &gameKruiseV1alpha1.GameServer{} + gs.Name = name + gs.Namespace = gss.GetNamespace() + + ors := make([]metav1.OwnerReference, 0) + or := metav1.OwnerReference{ + APIVersion: gss.APIVersion, + Kind: gss.Kind, + Name: gss.GetName(), + UID: gss.GetUID(), + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + } + ors = append(ors, or) + gs.OwnerReferences = ors + + // set Labels + gsLabels := gss.Spec.GameServerTemplate.GetLabels() + if gsLabels == nil { + gsLabels = make(map[string]string) + } + gsLabels[gameKruiseV1alpha1.GameServerOwnerGssKey] = gss.GetName() + gs.SetLabels(gsLabels) + + // set Annotations + gsAnnotations := gss.Spec.GameServerTemplate.GetAnnotations() + if gsAnnotations == nil { + gsAnnotations = make(map[string]string) + } + gsAnnotations[gameKruiseV1alpha1.GsTemplateMetadataHashKey] = GetGsTemplateMetadataHash(gss) + gs.SetAnnotations(gsAnnotations) + + // set NetWork + gs.Spec.NetworkDisabled = false + + // set OpsState + gs.Spec.OpsState = gameKruiseV1alpha1.None + + // set UpdatePriority + updatePriority := intstr.FromInt(0) + gs.Spec.UpdatePriority = &updatePriority + + // set deletionPriority + deletionPriority := intstr.FromInt(0) + gs.Spec.DeletionPriority = &deletionPriority + + return gs +} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index efd6dd7d..5a00c785 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -10,6 +10,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" @@ -51,7 +52,27 @@ func (f *Framework) AfterSuit() error { } func (f *Framework) AfterEach() error { - return f.client.DeleteGameServerSet() + return wait.PollImmediate(5*time.Second, 3*time.Minute, + func() (done bool, err error) { + err = f.client.DeleteGameServerSet() + if err != nil && !apierrors.IsNotFound(err) { + { + return false, err + } + } + + labelSelector := labels.SelectorFromSet(map[string]string{ + gamekruiseiov1alpha1.GameServerOwnerGssKey: client.GameServerSet, + }).String() + podList, err := f.client.GetPodList(labelSelector) + if err != nil { + return false, err + } + if len(podList.Items) != 0 { + return false, nil + } + return true, nil + }) } func (f *Framework) DeployGameServerSet() (*gamekruiseiov1alpha1.GameServerSet, error) { @@ -59,6 +80,12 @@ func (f *Framework) DeployGameServerSet() (*gamekruiseiov1alpha1.GameServerSet, return f.client.CreateGameServerSet(gss) } +func (f *Framework) DeployGameServerSetWithOwner(owner gamekruiseiov1alpha1.LifecyclePolicyType) (*gamekruiseiov1alpha1.GameServerSet, error) { + gss := f.client.DefaultGameServerSet() + gss.Spec.GameServerTemplate.LifecyclePolicyType = owner + return f.client.CreateGameServerSet(gss) +} + func (f *Framework) DeployGssWithServiceQualities() (*gamekruiseiov1alpha1.GameServerSet, error) { gss := f.client.DefaultGameServerSet() up := intstr.FromInt(20) @@ -218,7 +245,7 @@ func (f *Framework) ExpectGssCorrect(gss *gamekruiseiov1alpha1.GameServerSet, ex podIndexList := util.GetIndexListFromPodList(podList.Items) if !util.IsSliceEqual(expectIndex, podIndexList) { - return fmt.Errorf("current pods and expected pods do not correspond") + return fmt.Errorf("current pods and expected pods do not correspond, actual podIndexList: %v", podIndexList) } return nil @@ -255,8 +282,51 @@ func (f *Framework) WaitForGsDeletionPriorityUpdated(gsName string, deletionPrio } func (f *Framework) DeletePodDirectly(index int) error { - gsName := client.GameServerSet + "-" + strconv.Itoa(index) - return f.client.DeletePod(gsName) + var uid types.UID + podName := client.GameServerSet + "-" + strconv.Itoa(index) + + // get + if err := wait.PollImmediate(5*time.Second, 3*time.Minute, + func() (done bool, err error) { + + pod, err := f.client.GetPod(podName) + if err != nil { + return false, err + } + uid = pod.UID + return true, nil + }); err != nil { + return err + } + + // delete + if err := f.client.DeletePod(podName); err != nil { + return err + } + + // check + return wait.PollImmediate(5*time.Second, 3*time.Minute, + func() (done bool, err error) { + pod, err := f.client.GetPod(podName) + if err != nil { + return false, err + } + if pod.UID == uid { + return false, nil + } + return true, nil + }) +} + +func (f *Framework) WaitForPodDeleted(podName string) error { + return wait.PollImmediate(5*time.Second, 3*time.Minute, + func() (done bool, err error) { + _, err = f.client.GetPod(podName) + if apierrors.IsNotFound(err) { + return true, nil + } + return false, nil + }) } func (f *Framework) ExpectGsCorrect(gsName, opsState, dp, up string) error { diff --git a/test/e2e/testcase/testcase.go b/test/e2e/testcase/testcase.go index 05a301b9..3aeabf3a 100644 --- a/test/e2e/testcase/testcase.go +++ b/test/e2e/testcase/testcase.go @@ -92,10 +92,10 @@ func RunTestCases(f *framework.Framework) { gomega.Expect(err).To(gomega.BeNil()) }) - ginkgo.It("GameServer lifecycle", func() { + ginkgo.It("GameServer lifecycle(gs owner is GameServerSet)", func() { - // deploy - gss, err := f.DeployGameServerSet() + // Deploy a gss, and the owner of the GameServers it manages is GameServerSet + gss, err := f.DeployGameServerSetWithOwner("GameServerSet") gomega.Expect(err).To(gomega.BeNil()) err = f.ExpectGssCorrect(gss, []int{0, 1, 2}) @@ -113,5 +113,27 @@ func RunTestCases(f *framework.Framework) { err = f.ExpectGsCorrect(gss.GetName()+"-1", "None", "100", "0") gomega.Expect(err).To(gomega.BeNil()) }) + + ginkgo.It("GameServer lifecycle(gs owner is Pod)", func() { + + // Deploy a gss, and the owner of the GameServers it manages is Pod + gss, err := f.DeployGameServerSet() + gomega.Expect(err).To(gomega.BeNil()) + + err = f.ExpectGssCorrect(gss, []int{0, 1, 2}) + gomega.Expect(err).To(gomega.BeNil()) + + _, err = f.ChangeGameServerDeletionPriority(gss.GetName()+"-1", "100") + gomega.Expect(err).To(gomega.BeNil()) + + err = f.WaitForGsDeletionPriorityUpdated(gss.GetName()+"-1", "100") + gomega.Expect(err).To(gomega.BeNil()) + + err = f.DeletePodDirectly(1) + gomega.Expect(err).To(gomega.BeNil()) + + err = f.ExpectGsCorrect(gss.GetName()+"-1", "None", "0", "0") + gomega.Expect(err).To(gomega.BeNil()) + }) }) }