diff --git a/apis/v1alpha1/gameserverset_types.go b/apis/v1alpha1/gameserverset_types.go index ffa19d99..a863c757 100644 --- a/apis/v1alpha1/gameserverset_types.go +++ b/apis/v1alpha1/gameserverset_types.go @@ -64,8 +64,22 @@ type GameServerTemplate struct { // +kubebuilder:validation:Schemaless corev1.PodTemplateSpec `json:",inline"` VolumeClaimTemplates []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"` + // ReclaimPolicy indicates the reclaim policy for GameServer. + // Default is Cascade. + ReclaimPolicy GameServerReclaimPolicy `json:"reclaimPolicy,omitempty"` } +type GameServerReclaimPolicy string + +const ( + // CascadeGameServerReclaimPolicy indicates that GameServer is deleted when the pod is deleted. + // The age of GameServer is exactly the same as that of the pod. + CascadeGameServerReclaimPolicy GameServerReclaimPolicy = "Cascade" + // DeleteGameServerReclaimPolicy indicates that GameServers will be deleted when replicas of GameServerSet decreases. + // The GameServer will not be deleted when the corresponding pod is deleted due to manual deletion, update, eviction, etc. + DeleteGameServerReclaimPolicy GameServerReclaimPolicy = "Delete" +) + type Network struct { NetworkType string `json:"networkType,omitempty"` NetworkConf []NetworkConfParams `json:"networkConf,omitempty"` diff --git a/config/crd/bases/game.kruise.io_gameserversets.yaml b/config/crd/bases/game.kruise.io_gameserversets.yaml index 12d1cce5..8a2a4fe9 100644 --- a/config/crd/bases/game.kruise.io_gameserversets.yaml +++ b/config/crd/bases/game.kruise.io_gameserversets.yaml @@ -70,6 +70,10 @@ spec: description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster Important: Run "make" to regenerate code after modifying this file' properties: + reclaimPolicy: + description: ReclaimPolicy indicates the reclaim policy for GameServer. + Default is Cascade. + type: string volumeClaimTemplates: items: description: PersistentVolumeClaim is a user's request for and diff --git a/docs/en/user_manuals/CRD_field_description.md b/docs/en/user_manuals/CRD_field_description.md index 8f05b14b..674bc473 100644 --- a/docs/en/user_manuals/CRD_field_description.md +++ b/docs/en/user_manuals/CRD_field_description.md @@ -38,8 +38,24 @@ type GameServerTemplate struct { // Requests and claims for persistent volumes. VolumeClaimTemplates []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"` + + // ReclaimPolicy indicates the reclaim policy for GameServer. + // Default is Cascade. + ReclaimPolicy GameServerReclaimPolicy `json:"reclaimPolicy,omitempty"` } +type GameServerReclaimPolicy string + +const ( + // CascadeGameServerReclaimPolicy indicates that GameServer is deleted when the pod is deleted. + // The age of GameServer is exactly the same as that of the pod. + CascadeGameServerReclaimPolicy GameServerReclaimPolicy = "Cascade" + + // DeleteGameServerReclaimPolicy indicates that GameServers will be deleted when replicas of GameServerSet decreases. + // The GameServer will not be deleted when the corresponding pod is deleted due to manual deletion, update, eviction, etc. + DeleteGameServerReclaimPolicy GameServerReclaimPolicy = "Delete" +) + ``` #### UpdateStrategy diff --git "a/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/CRD\345\255\227\346\256\265\350\257\264\346\230\216.md" "b/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/CRD\345\255\227\346\256\265\350\257\264\346\230\216.md" index fae99331..cb424451 100644 --- "a/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/CRD\345\255\227\346\256\265\350\257\264\346\230\216.md" +++ "b/docs/\344\270\255\346\226\207/\347\224\250\346\210\267\346\211\213\345\206\214/CRD\345\255\227\346\256\265\350\257\264\346\230\216.md" @@ -72,7 +72,23 @@ type GameServerTemplate struct { // 对持久卷的请求和声明 VolumeClaimTemplates []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"` + + // ReclaimPolicy 表明GameServer的回收策略 + // 当前支持两种，Cascade与Delete。默认为Cascade + ReclaimPolicy GameServerReclaimPolicy `json:"reclaimPolicy,omitempty"` } + +type GameServerReclaimPolicy string + +const ( + // Cascade 表明pod删除时GameServer一并删除。GameServer的生命周期与pod相同 + CascadeGameServerReclaimPolicy GameServerReclaimPolicy = "Cascade" + + // Delete 表明 GameServers 只会在GameServerSet副本数目减少时被删除。 + // 当对应的pod被手动删除、更新重建、被驱逐时，GameServer都不会被删除。 + DeleteGameServerReclaimPolicy GameServerReclaimPolicy = "Delete" +) + ``` ### UpdateStrategy diff --git a/pkg/controllers/gameserver/gameserver_controller.go b/pkg/controllers/gameserver/gameserver_controller.go index f423fb16..9f00efb9 100644 --- a/pkg/controllers/gameserver/gameserver_controller.go +++ b/pkg/controllers/gameserver/gameserver_controller.go @@ -26,7 +26,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" @@ -173,8 +172,15 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) } if podFound && !gsFound { - err := r.initGameServer(pod) - if err != nil && !errors.IsAlreadyExists(err) && !errors.IsNotFound(err) { + gss, err := r.getGameServerSet(pod) + if err != nil { + if errors.IsNotFound(err) { + return reconcile.Result{}, nil + } + return reconcile.Result{}, err + } + err = r.initGameServerByPod(gss, pod) + if err != nil && !errors.IsAlreadyExists(err) { klog.Errorf("failed to create GameServer %s in %s, because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) return reconcile.Result{}, err } @@ -237,57 +243,24 @@ func (r *GameServerReconciler) getGameServerSet(pod *corev1.Pod) (*gamekruiseiov return gss, err } -func (r *GameServerReconciler) initGameServer(pod *corev1.Pod) error { - gs := &gamekruiseiov1alpha1.GameServer{} - gs.Name = pod.GetName() - gs.Namespace = pod.GetNamespace() - - // set owner reference - gss, err := r.getGameServerSet(pod) - if err != nil { - return err - } - ors := make([]metav1.OwnerReference, 0) - or := metav1.OwnerReference{ - APIVersion: gss.APIVersion, - Kind: gss.Kind, - Name: gss.GetName(), - UID: gss.GetUID(), - Controller: pointer.BoolPtr(true), - BlockOwnerDeletion: pointer.BoolPtr(true), - } - ors = append(ors, or) - gs.OwnerReferences = ors - - // set Labels - gsLabels := gss.Spec.GameServerTemplate.GetLabels() - if gsLabels == nil { - gsLabels = make(map[string]string) - } - gsLabels[gamekruiseiov1alpha1.GameServerOwnerGssKey] = gss.GetName() - gs.SetLabels(gsLabels) - - // set Annotations - gsAnnotations := gss.Spec.GameServerTemplate.GetAnnotations() - if gsAnnotations == nil { - gsAnnotations = make(map[string]string) +func (r *GameServerReconciler) initGameServerByPod(gss *gamekruiseiov1alpha1.GameServerSet, pod *corev1.Pod) error { + // default fields + gs := util.InitGameServer(gss, pod.Name) + + if gss.Spec.GameServerTemplate.ReclaimPolicy == gamekruiseiov1alpha1.CascadeGameServerReclaimPolicy || gss.Spec.GameServerTemplate.ReclaimPolicy == "" { + // rewrite ownerReferences + ors := make([]metav1.OwnerReference, 0) + or := metav1.OwnerReference{ + APIVersion: pod.APIVersion, + Kind: pod.Kind, + Name: pod.GetName(), + UID: pod.GetUID(), + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + } + ors = append(ors, or) + gs.OwnerReferences = ors } - gsAnnotations[gamekruiseiov1alpha1.GsTemplateMetadataHashKey] = util.GetGsTemplateMetadataHash(gss) - gs.SetAnnotations(gsAnnotations) - - // set NetWork - gs.Spec.NetworkDisabled = false - - // set OpsState - gs.Spec.OpsState = gamekruiseiov1alpha1.None - - // set UpdatePriority - updatePriority := intstr.FromInt(0) - gs.Spec.UpdatePriority = &updatePriority - - // set deletionPriority - deletionPriority := intstr.FromInt(0) - gs.Spec.DeletionPriority = &deletionPriority return r.Client.Create(context.Background(), gs) } diff --git a/pkg/controllers/gameserverset/gameserverset_controller.go b/pkg/controllers/gameserverset/gameserverset_controller.go index 26a12c1f..f1e53e48 100644 --- a/pkg/controllers/gameserverset/gameserverset_controller.go +++ b/pkg/controllers/gameserverset/gameserverset_controller.go @@ -252,12 +252,6 @@ func (r *GameServerSetReconciler) Reconcile(ctx context.Context, req ctrl.Reques return reconcile.Result{}, err } - err = gsm.SyncGameServerReplicas() - if err != nil { - klog.Errorf("GameServerSet %s failed to adjust the replicas of GameServers to match that of Pods in %s, because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) - return reconcile.Result{}, err - } - return ctrl.Result{}, nil } diff --git a/pkg/controllers/gameserverset/gameserverset_manager.go b/pkg/controllers/gameserverset/gameserverset_manager.go index 7a6e17a3..4703ff8d 100644 --- a/pkg/controllers/gameserverset/gameserverset_manager.go +++ b/pkg/controllers/gameserverset/gameserverset_manager.go @@ -18,12 +18,6 @@ package gameserverset import ( "context" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/tools/record" - "sort" - "sync" - "time" - kruiseV1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1" kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" corev1 "k8s.io/api/core/v1" @@ -32,9 +26,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/json" + "k8s.io/client-go/tools/record" "k8s.io/klog/v2" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" + "sort" + "strconv" + "sync" gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" "github.com/openkruise/kruise-game/pkg/util" @@ -47,7 +45,6 @@ type Control interface { IsNeedToScale() bool IsNeedToUpdateWorkload() bool SyncPodProbeMarker() error - SyncGameServerReplicas() error GetReplicasAfterKilling() *int32 } @@ -129,7 +126,15 @@ func (manager *GameServerSetManager) GameServerScale() error { klog.Infof("GameServers %s/%s already has %d replicas, expect to have %d replicas.", gss.GetNamespace(), gss.GetName(), currentReplicas, expectedReplicas) manager.eventRecorder.Eventf(gss, corev1.EventTypeNormal, ScaleReason, "scale from %d to %d", currentReplicas, expectedReplicas) - newReserveIds := computeToScaleGs(gssReserveIds, reserveIds, notExistIds, expectedReplicas, podList, gss.Spec.ScaleStrategy.ScaleDownStrategyType) + newManageIds, newReserveIds := computeToScaleGs(gssReserveIds, reserveIds, notExistIds, expectedReplicas, podList, gss.Spec.ScaleStrategy.ScaleDownStrategyType) + + if gss.Spec.GameServerTemplate.ReclaimPolicy == gameKruiseV1alpha1.DeleteGameServerReclaimPolicy { + err := SyncGameServer(gss, c, newManageIds, util.GetIndexListFromPodList(podList)) + if err != nil { + return err + } + } + asts.Spec.ReserveOrdinals = newReserveIds asts.Spec.Replicas = gss.Spec.Replicas asts.Spec.ScaleStrategy = &kruiseV1beta1.StatefulSetScaleStrategy{ @@ -157,7 +162,7 @@ func (manager *GameServerSetManager) GameServerScale() error { return nil } -func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedReplicas int, pods []corev1.Pod, scaleDownType gameKruiseV1alpha1.ScaleDownStrategyType) []int { +func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedReplicas int, pods []corev1.Pod, scaleDownType gameKruiseV1alpha1.ScaleDownStrategyType) ([]int, []int) { workloadManageIds := util.GetIndexListFromPodList(pods) var toAdd []int @@ -215,12 +220,13 @@ func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedRepl } } + newManageIds := append(workloadManageIds, util.GetSliceInANotInB(toAdd, workloadManageIds)...) + newManageIds = util.GetSliceInANotInB(newManageIds, toDelete) + if scaleDownType == gameKruiseV1alpha1.ReserveIdsScaleDownStrategyType { - return append(gssReserveIds, util.GetSliceInANotInB(toDelete, gssReserveIds)...) + return newManageIds, append(gssReserveIds, util.GetSliceInANotInB(toDelete, gssReserveIds)...) } - newManageIds := append(workloadManageIds, util.GetSliceInANotInB(toAdd, workloadManageIds)...) - newManageIds = util.GetSliceInANotInB(newManageIds, toDelete) var newReserveIds []int if len(newManageIds) != 0 { sort.Ints(newManageIds) @@ -231,63 +237,80 @@ func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedRepl } } - return newReserveIds + return newManageIds, newReserveIds } -func (manager *GameServerSetManager) SyncGameServerReplicas() error { - gss := manager.gameServerSet - gsList := &gameKruiseV1alpha1.GameServerList{} - err := manager.client.List(context.Background(), gsList, &client.ListOptions{ - Namespace: gss.GetNamespace(), - LabelSelector: labels.SelectorFromSet(map[string]string{ - gameKruiseV1alpha1.GameServerOwnerGssKey: gss.GetName(), - })}) - if err != nil { - return err - } - podIds := util.GetIndexListFromPodList(manager.podList) - - gsMap := make(map[int]int) - deleteIds := make([]int, 0) - for id, gs := range gsList.Items { - gsId := util.GetIndexFromGsName(gs.Name) - if !util.IsNumInList(gsId, podIds) { - gsMap[gsId] = id - deleteIds = append(deleteIds, gsId) - } - } - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) +func SyncGameServer(gss *gameKruiseV1alpha1.GameServerSet, c client.Client, newManageIds, oldManageIds []int) error { + ctx, cancel := context.WithCancel(context.Background()) defer cancel() - errch := make(chan error, len(deleteIds)) + addIds := util.GetSliceInANotInB(newManageIds, oldManageIds) + deleteIds := util.GetSliceInANotInB(oldManageIds, newManageIds) + + errch := make(chan error, len(addIds)+len(deleteIds)) var wg sync.WaitGroup - for _, gsId := range deleteIds { + for _, gsId := range append(addIds, deleteIds...) { wg.Add(1) id := gsId go func(ctx context.Context) { defer wg.Done() defer ctx.Done() - gs := gsList.Items[gsMap[id]].DeepCopy() - gsLabels := make(map[string]string) - gsLabels[gameKruiseV1alpha1.GameServerDeletingKey] = "true" - patchGs := map[string]interface{}{"metadata": map[string]map[string]string{"labels": gsLabels}} - patchBytes, err := json.Marshal(patchGs) + gs := &gameKruiseV1alpha1.GameServer{} + gsName := gss.Name + "-" + strconv.Itoa(id) + err := c.Get(ctx, types.NamespacedName{ + Name: gsName, + Namespace: gss.Namespace, + }, gs) if err != nil { + if errors.IsNotFound(err) { + return + } errch <- err + return } - err = manager.client.Patch(context.TODO(), gs, client.RawPatch(types.MergePatchType, patchBytes)) - if err != nil && !errors.IsNotFound(err) { - errch <- err + + if util.IsNumInList(id, addIds) && gs.GetLabels()[gameKruiseV1alpha1.GameServerDeletingKey] == "true" { + gsLabels := make(map[string]string) + gsLabels[gameKruiseV1alpha1.GameServerDeletingKey] = "false" + patchGs := map[string]interface{}{"metadata": map[string]map[string]string{"labels": gsLabels}} + patchBytes, err := json.Marshal(patchGs) + if err != nil { + errch <- err + return + } + err = c.Patch(ctx, gs, client.RawPatch(types.MergePatchType, patchBytes)) + if err != nil && !errors.IsNotFound(err) { + errch <- err + return + } + klog.Infof("GameServer %s/%s DeletingKey turn into false", gss.Namespace, gsName) } + + if util.IsNumInList(id, deleteIds) && gs.GetLabels()[gameKruiseV1alpha1.GameServerDeletingKey] != "true" { + gsLabels := make(map[string]string) + gsLabels[gameKruiseV1alpha1.GameServerDeletingKey] = "true" + patchGs := map[string]interface{}{"metadata": map[string]map[string]string{"labels": gsLabels}} + patchBytes, err := json.Marshal(patchGs) + if err != nil { + errch <- err + return + } + err = c.Patch(ctx, gs, client.RawPatch(types.MergePatchType, patchBytes)) + if err != nil && !errors.IsNotFound(err) { + errch <- err + return + } + klog.Infof("GameServer %s/%s DeletingKey turn into true, who will be deleted", gss.Namespace, gsName) + } + }(ctx) } + wg.Wait() close(errch) - err = <-errch + err := <-errch if err != nil { - klog.Errorf("failed to delete GameServers %s in %s because of %s.", gss.GetName(), gss.GetNamespace(), err.Error()) return err } diff --git a/pkg/controllers/gameserverset/gameserverset_manager_test.go b/pkg/controllers/gameserverset/gameserverset_manager_test.go index e1f1e7fb..37c70144 100644 --- a/pkg/controllers/gameserverset/gameserverset_manager_test.go +++ b/pkg/controllers/gameserverset/gameserverset_manager_test.go @@ -8,7 +8,6 @@ import ( "github.com/openkruise/kruise-game/pkg/util" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -16,6 +15,7 @@ import ( "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "strconv" "testing" ) @@ -39,6 +39,7 @@ func TestComputeToScaleGs(t *testing.T) { scaleDownStrategyType gameKruiseV1alpha1.ScaleDownStrategyType pods []corev1.Pod newReserveIds []int + newManageIds []int }{ { newGssReserveIds: []int{2, 3, 4}, @@ -85,6 +86,7 @@ func TestComputeToScaleGs(t *testing.T) { }, }, newReserveIds: []int{2, 3, 4, 5}, + newManageIds: []int{0, 1, 6}, }, { newGssReserveIds: []int{0, 2, 3}, @@ -140,6 +142,7 @@ func TestComputeToScaleGs(t *testing.T) { }, }, newReserveIds: []int{0, 2, 3, 4, 5}, + newManageIds: []int{1, 6, 7}, }, { newGssReserveIds: []int{0}, @@ -195,6 +198,7 @@ func TestComputeToScaleGs(t *testing.T) { }, }, newReserveIds: []int{0}, + newManageIds: []int{1}, }, { newGssReserveIds: []int{0, 2, 3}, @@ -250,6 +254,7 @@ func TestComputeToScaleGs(t *testing.T) { }, }, newReserveIds: []int{0, 2, 3, 5}, + newManageIds: []int{1, 4, 6, 7}, }, { newGssReserveIds: []int{0, 3, 5}, @@ -296,6 +301,7 @@ func TestComputeToScaleGs(t *testing.T) { }, }, newReserveIds: []int{0, 3, 5, 2, 4, 6}, + newManageIds: []int{1}, }, { newGssReserveIds: []int{1, 2}, @@ -324,14 +330,38 @@ func TestComputeToScaleGs(t *testing.T) { }, }, newReserveIds: []int{1, 2}, + newManageIds: []int{0, 3}, + }, + { + newGssReserveIds: []int{}, + oldGssreserveIds: []int{}, + notExistIds: []int{}, + expectedReplicas: 3, + scaleDownStrategyType: gameKruiseV1alpha1.GeneralScaleDownStrategyType, + pods: []corev1.Pod{}, + newReserveIds: []int{}, + newManageIds: []int{0, 1, 2}, + }, + { + newGssReserveIds: []int{1, 2}, + oldGssreserveIds: []int{}, + notExistIds: []int{}, + expectedReplicas: 3, + scaleDownStrategyType: gameKruiseV1alpha1.GeneralScaleDownStrategyType, + pods: []corev1.Pod{}, + newReserveIds: []int{1, 2}, + newManageIds: []int{0, 3, 4}, }, } for i, test := range tests { - newReserveIds := computeToScaleGs(test.newGssReserveIds, test.oldGssreserveIds, test.notExistIds, test.expectedReplicas, test.pods, test.scaleDownStrategyType) + newManageIds, newReserveIds := computeToScaleGs(test.newGssReserveIds, test.oldGssreserveIds, test.notExistIds, test.expectedReplicas, test.pods, test.scaleDownStrategyType) if !util.IsSliceEqual(newReserveIds, test.newReserveIds) { t.Errorf("case %d: expect newNotExistIds %v but got %v", i, test.newReserveIds, newReserveIds) } + if !util.IsSliceEqual(newManageIds, test.newManageIds) { + t.Errorf("case %d: expect newManageIds %v but got %v", i, test.newManageIds, newManageIds) + } } } @@ -700,13 +730,16 @@ func TestGameServerScale(t *testing.T) { } } -func TestSyncGameServerReplicas(t *testing.T) { +func TestSyncGameServer(t *testing.T) { tests := []struct { - gss *gameKruiseV1alpha1.GameServerSet - podList []corev1.Pod - gsList []*gameKruiseV1alpha1.GameServer - toDelete []int + gss *gameKruiseV1alpha1.GameServerSet + gsList []*gameKruiseV1alpha1.GameServer + newManageIds []int + oldManageIds []int + IdsLabelTure []int + IdsLabelFalse []int }{ + // case 0 { gss: &gameKruiseV1alpha1.GameServerSet{ ObjectMeta: metav1.ObjectMeta{ @@ -714,23 +747,6 @@ func TestSyncGameServerReplicas(t *testing.T) { Name: "xxx", }, }, - podList: []corev1.Pod{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "xxx-0", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "xxx-2", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "xxx-4", - }, - }, - }, gsList: []*gameKruiseV1alpha1.GameServer{ { ObjectMeta: metav1.ObjectMeta{ @@ -769,41 +785,88 @@ func TestSyncGameServerReplicas(t *testing.T) { }, }, }, - toDelete: []int{3}, + oldManageIds: []int{0, 2, 3, 4}, + newManageIds: []int{0, 1}, + IdsLabelTure: []int{2, 3, 4}, + IdsLabelFalse: []int{}, + }, + + // case 1 + { + gss: &gameKruiseV1alpha1.GameServerSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "xxx", + Name: "xxx", + }, + }, + gsList: []*gameKruiseV1alpha1.GameServer{}, + oldManageIds: []int{}, + newManageIds: []int{0, 1, 3}, + IdsLabelTure: []int{}, + IdsLabelFalse: []int{}, + }, + + // case 2 + { + gss: &gameKruiseV1alpha1.GameServerSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "xxx", + Name: "xxx", + }, + }, + gsList: []*gameKruiseV1alpha1.GameServer{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "xxx", + Name: "xxx-0", + Labels: map[string]string{ + gameKruiseV1alpha1.GameServerOwnerGssKey: "xxx", + gameKruiseV1alpha1.GameServerDeletingKey: "true", + }, + }, + }, + }, + oldManageIds: []int{}, + newManageIds: []int{0}, + IdsLabelTure: []int{}, + IdsLabelFalse: []int{0}, }, } - for _, test := range tests { + for i, test := range tests { objs := []client.Object{test.gss} for _, gs := range test.gsList { objs = append(objs, gs) } c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() - recorder := record.NewFakeRecorder(100) - manager := &GameServerSetManager{ - gameServerSet: test.gss, - podList: test.podList, - eventRecorder: recorder, - client: c, - } - if err := manager.SyncGameServerReplicas(); err != nil { + if err := SyncGameServer(test.gss, c, test.newManageIds, test.oldManageIds); err != nil { t.Error(err) } - gsList := &gameKruiseV1alpha1.GameServerList{} - if err := manager.client.List(context.Background(), gsList, &client.ListOptions{ - Namespace: test.gss.GetNamespace(), - LabelSelector: labels.SelectorFromSet(map[string]string{ - gameKruiseV1alpha1.GameServerOwnerGssKey: test.gss.GetName(), - gameKruiseV1alpha1.GameServerDeletingKey: "true", - })}); err != nil { - t.Error(err) + for _, id := range test.IdsLabelTure { + gs := &gameKruiseV1alpha1.GameServer{} + if err := c.Get(context.Background(), types.NamespacedName{ + Namespace: test.gss.GetNamespace(), + Name: test.gss.GetName() + "-" + strconv.Itoa(id), + }, gs); err != nil { + t.Errorf("case %d: err: %s", i, err.Error()) + } + if gs.GetLabels()[gameKruiseV1alpha1.GameServerDeletingKey] != "true" { + t.Errorf("case %d: gs %d GameServerDeletingKey is not true", i, id) + } } - actual := util.GetIndexListFromGsList(gsList.Items) - expect := test.toDelete - if !util.IsSliceEqual(actual, expect) { - t.Errorf("expect to delete gameservers %v but actually %v", expect, actual) + for _, id := range test.IdsLabelFalse { + gs := &gameKruiseV1alpha1.GameServer{} + if err := c.Get(context.Background(), types.NamespacedName{ + Namespace: test.gss.GetNamespace(), + Name: test.gss.GetName() + "-" + strconv.Itoa(id), + }, gs); err != nil { + t.Error(err) + } + if gs.GetLabels()[gameKruiseV1alpha1.GameServerDeletingKey] != "false" { + t.Errorf("case %d: gs %d GameServerDeletingKey is not false", i, id) + } } } } diff --git a/pkg/util/gameserver.go b/pkg/util/gameserver.go index c50d2d9b..6ee146bc 100644 --- a/pkg/util/gameserver.go +++ b/pkg/util/gameserver.go @@ -19,6 +19,7 @@ package util import ( "context" "encoding/json" + "fmt" appspub "github.com/openkruise/kruise-api/apps/pub" kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" @@ -26,6 +27,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "strconv" "strings" @@ -197,6 +200,9 @@ func GetAstsHash(gss *gameKruiseV1alpha1.GameServerSet) string { } func GetGsTemplateMetadataHash(gss *gameKruiseV1alpha1.GameServerSet) string { + fmt.Println(gss.Spec.GameServerTemplate.GetLabels()) + fmt.Println(gss.Spec.GameServerTemplate.GetAnnotations()) + return GetHash(metav1.ObjectMeta{ Labels: gss.Spec.GameServerTemplate.GetLabels(), Annotations: gss.Spec.GameServerTemplate.GetAnnotations(), @@ -229,3 +235,53 @@ func IsAllowNotReadyContainers(networkConfParams []gameKruiseV1alpha1.NetworkCon } return false } + +func InitGameServer(gss *gameKruiseV1alpha1.GameServerSet, name string) *gameKruiseV1alpha1.GameServer { + gs := &gameKruiseV1alpha1.GameServer{} + gs.Name = name + gs.Namespace = gss.GetNamespace() + + ors := make([]metav1.OwnerReference, 0) + or := metav1.OwnerReference{ + APIVersion: gss.APIVersion, + Kind: gss.Kind, + Name: gss.GetName(), + UID: gss.GetUID(), + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + } + ors = append(ors, or) + gs.OwnerReferences = ors + + // set Labels + gsLabels := gss.Spec.GameServerTemplate.DeepCopy().GetLabels() + if gsLabels == nil { + gsLabels = make(map[string]string) + } + gsLabels[gameKruiseV1alpha1.GameServerOwnerGssKey] = gss.GetName() + gs.SetLabels(gsLabels) + + // set Annotations + gsAnnotations := gss.Spec.GameServerTemplate.DeepCopy().GetAnnotations() + if gsAnnotations == nil { + gsAnnotations = make(map[string]string) + } + gsAnnotations[gameKruiseV1alpha1.GsTemplateMetadataHashKey] = GetGsTemplateMetadataHash(gss) + gs.SetAnnotations(gsAnnotations) + + // set NetWork + gs.Spec.NetworkDisabled = false + + // set OpsState + gs.Spec.OpsState = gameKruiseV1alpha1.None + + // set UpdatePriority + updatePriority := intstr.FromInt(0) + gs.Spec.UpdatePriority = &updatePriority + + // set deletionPriority + deletionPriority := intstr.FromInt(0) + gs.Spec.DeletionPriority = &deletionPriority + + return gs +} diff --git a/pkg/util/gameserver_test.go b/pkg/util/gameserver_test.go index e9b9d57c..84b21095 100644 --- a/pkg/util/gameserver_test.go +++ b/pkg/util/gameserver_test.go @@ -20,6 +20,9 @@ import ( gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" + "reflect" "sort" "testing" ) @@ -359,3 +362,81 @@ func TestIsAllowNotReadyContainers(t *testing.T) { } } } + +func TestInitGameServer(t *testing.T) { + updatePriority := intstr.FromInt(0) + deletionPriority := intstr.FromInt(0) + + tests := []struct { + gss *gameKruiseV1alpha1.GameServerSet + name string + gs *gameKruiseV1alpha1.GameServer + }{ + { + gss: &gameKruiseV1alpha1.GameServerSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "GameServerSet", + APIVersion: "game.kruise.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "xxx", + Name: "case0", + UID: "xxx0", + }, + Spec: gameKruiseV1alpha1.GameServerSetSpec{ + GameServerTemplate: gameKruiseV1alpha1.GameServerTemplate{ + PodTemplateSpec: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "label-key": "label-value", + }, + }, + }, + }, + }, + }, + name: "case0-1", + gs: &gameKruiseV1alpha1.GameServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: "case0-1", + Namespace: "xxx", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "GameServerSet", + APIVersion: "game.kruise.io/v1alpha1", + Name: "case0", + UID: "xxx0", + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }, + }, + Labels: map[string]string{ + "label-key": "label-value", + gameKruiseV1alpha1.GameServerOwnerGssKey: "case0", + }, + Annotations: map[string]string{ + gameKruiseV1alpha1.GsTemplateMetadataHashKey: GetHash(metav1.ObjectMeta{ + Labels: map[string]string{ + "label-key": "label-value", + }, + }), + }, + }, + Spec: gameKruiseV1alpha1.GameServerSpec{ + NetworkDisabled: false, + OpsState: gameKruiseV1alpha1.None, + UpdatePriority: &updatePriority, + DeletionPriority: &deletionPriority, + }, + }, + }, + } + + for i, test := range tests { + expect := test.gs + actual := InitGameServer(test.gss, test.name) + if !reflect.DeepEqual(expect, actual) { + t.Errorf("case %d: expect generated GameServer is %v but actually got %v", i, expect, actual) + } + } +} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index efd6dd7d..7a83d7e6 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -10,6 +10,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" @@ -51,7 +52,27 @@ func (f *Framework) AfterSuit() error { } func (f *Framework) AfterEach() error { - return f.client.DeleteGameServerSet() + return wait.PollImmediate(5*time.Second, 3*time.Minute, + func() (done bool, err error) { + err = f.client.DeleteGameServerSet() + if err != nil && !apierrors.IsNotFound(err) { + { + return false, err + } + } + + labelSelector := labels.SelectorFromSet(map[string]string{ + gamekruiseiov1alpha1.GameServerOwnerGssKey: client.GameServerSet, + }).String() + podList, err := f.client.GetPodList(labelSelector) + if err != nil { + return false, err + } + if len(podList.Items) != 0 { + return false, nil + } + return true, nil + }) } func (f *Framework) DeployGameServerSet() (*gamekruiseiov1alpha1.GameServerSet, error) { @@ -59,6 +80,12 @@ func (f *Framework) DeployGameServerSet() (*gamekruiseiov1alpha1.GameServerSet, return f.client.CreateGameServerSet(gss) } +func (f *Framework) DeployGameServerSetWithReclaimPolicy(reclaimPolicy gamekruiseiov1alpha1.GameServerReclaimPolicy) (*gamekruiseiov1alpha1.GameServerSet, error) { + gss := f.client.DefaultGameServerSet() + gss.Spec.GameServerTemplate.ReclaimPolicy = reclaimPolicy + return f.client.CreateGameServerSet(gss) +} + func (f *Framework) DeployGssWithServiceQualities() (*gamekruiseiov1alpha1.GameServerSet, error) { gss := f.client.DefaultGameServerSet() up := intstr.FromInt(20) @@ -160,6 +187,14 @@ func (f *Framework) WaitForGsCreated(gss *gamekruiseiov1alpha1.GameServerSet) er if len(podList.Items) != int(*gss.Spec.Replicas) { return false, nil } + gsList, err := f.client.GetGameServerList(labelSelector) + if err != nil { + return false, err + } + if len(gsList.Items) != int(*gss.Spec.Replicas) { + return false, nil + } + return true, nil }) } @@ -218,7 +253,7 @@ func (f *Framework) ExpectGssCorrect(gss *gamekruiseiov1alpha1.GameServerSet, ex podIndexList := util.GetIndexListFromPodList(podList.Items) if !util.IsSliceEqual(expectIndex, podIndexList) { - return fmt.Errorf("current pods and expected pods do not correspond") + return fmt.Errorf("current pods and expected pods do not correspond, actual podIndexList: %v", podIndexList) } return nil @@ -255,21 +290,66 @@ func (f *Framework) WaitForGsDeletionPriorityUpdated(gsName string, deletionPrio } func (f *Framework) DeletePodDirectly(index int) error { - gsName := client.GameServerSet + "-" + strconv.Itoa(index) - return f.client.DeletePod(gsName) -} + var uid types.UID + podName := client.GameServerSet + "-" + strconv.Itoa(index) -func (f *Framework) ExpectGsCorrect(gsName, opsState, dp, up string) error { - gs, err := f.client.GetGameServer(gsName) - if err != nil { + // get + if err := wait.PollImmediate(5*time.Second, 3*time.Minute, + func() (done bool, err error) { + + pod, err := f.client.GetPod(podName) + if err != nil { + return false, err + } + uid = pod.UID + return true, nil + }); err != nil { return err } - if gs.Status.DeletionPriority.String() != dp || gs.Status.UpdatePriority.String() != up || string(gs.Spec.OpsState) != opsState { - return fmt.Errorf("current GameServer is wrong") + // delete + if err := f.client.DeletePod(podName); err != nil { + return err } - return nil + // check + return wait.PollImmediate(5*time.Second, 3*time.Minute, + func() (done bool, err error) { + pod, err := f.client.GetPod(podName) + if err != nil { + return false, err + } + if pod.UID == uid { + return false, nil + } + return true, nil + }) +} + +func (f *Framework) WaitForPodDeleted(podName string) error { + return wait.PollImmediate(5*time.Second, 3*time.Minute, + func() (done bool, err error) { + _, err = f.client.GetPod(podName) + if apierrors.IsNotFound(err) { + return true, nil + } + return false, nil + }) +} + +func (f *Framework) ExpectGsCorrect(gsName, opsState, dp, up string) error { + return wait.PollImmediate(5*time.Second, 3*time.Minute, + func() (done bool, err error) { + gs, err := f.client.GetGameServer(gsName) + if err != nil { + return false, nil + } + + if gs.Status.DeletionPriority.String() != dp || gs.Status.UpdatePriority.String() != up || string(gs.Spec.OpsState) != opsState { + return false, nil + } + return true, nil + }) } func (f *Framework) WaitForGsUpdatePriorityUpdated(gsName string, updatePriority string) error { diff --git a/test/e2e/testcase/testcase.go b/test/e2e/testcase/testcase.go index 05a301b9..f27f3297 100644 --- a/test/e2e/testcase/testcase.go +++ b/test/e2e/testcase/testcase.go @@ -92,10 +92,10 @@ func RunTestCases(f *framework.Framework) { gomega.Expect(err).To(gomega.BeNil()) }) - ginkgo.It("GameServer lifecycle", func() { + ginkgo.It("GameServer lifecycle(DeleteGameServerReclaimPolicy)", func() { - // deploy - gss, err := f.DeployGameServerSet() + // Deploy a gss, and the ReclaimPolicy is Delete + gss, err := f.DeployGameServerSetWithReclaimPolicy(gameKruiseV1alpha1.DeleteGameServerReclaimPolicy) gomega.Expect(err).To(gomega.BeNil()) err = f.ExpectGssCorrect(gss, []int{0, 1, 2}) @@ -113,5 +113,27 @@ func RunTestCases(f *framework.Framework) { err = f.ExpectGsCorrect(gss.GetName()+"-1", "None", "100", "0") gomega.Expect(err).To(gomega.BeNil()) }) + + ginkgo.It("GameServer lifecycle(CascadeGameServerReclaimPolicy)", func() { + + // Deploy a gss, and the ReclaimPolicy is Cascade + gss, err := f.DeployGameServerSet() + gomega.Expect(err).To(gomega.BeNil()) + + err = f.ExpectGssCorrect(gss, []int{0, 1, 2}) + gomega.Expect(err).To(gomega.BeNil()) + + _, err = f.ChangeGameServerDeletionPriority(gss.GetName()+"-1", "100") + gomega.Expect(err).To(gomega.BeNil()) + + err = f.WaitForGsDeletionPriorityUpdated(gss.GetName()+"-1", "100") + gomega.Expect(err).To(gomega.BeNil()) + + err = f.DeletePodDirectly(1) + gomega.Expect(err).To(gomega.BeNil()) + + err = f.ExpectGsCorrect(gss.GetName()+"-1", "None", "0", "0") + gomega.Expect(err).To(gomega.BeNil()) + }) }) }