From 1d28a09840f6852bcc47be0260aa82ea8915a6a9 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Wed, 23 Oct 2024 04:07:30 +0800 Subject: [PATCH] interface: add APIs for eviction/disruption budget (#911) --- Makefile | 2 +- .../v1alpha1/disruptionbudget_types.go | 116 ++++++++++ apis/placement/v1alpha1/eviction_types.go | 125 ++++++++++ .../v1alpha1/zz_generated.deepcopy.go | 180 +++++++++++++++ ...terresourceplacementdisruptionbudgets.yaml | 148 ++++++++++++ ....io_clusterresourceplacementevictions.yaml | 188 +++++++++++++++ .../api_validation_integration_test.go | 218 ++++++++++++++++++ test/apis/placement/v1alpha1/suite_test.go | 88 +++++++ 8 files changed, 1064 insertions(+), 1 deletion(-) create mode 100644 apis/placement/v1alpha1/disruptionbudget_types.go create mode 100644 apis/placement/v1alpha1/eviction_types.go create mode 100644 config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementdisruptionbudgets.yaml create mode 100644 config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementevictions.yaml create mode 100644 test/apis/placement/v1alpha1/api_validation_integration_test.go create mode 100644 test/apis/placement/v1alpha1/suite_test.go diff --git a/Makefile b/Makefile index 6925a87c8..c1e4b3253 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ GOLANGCI_LINT_BIN := golangci-lint GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/$(GOLANGCI_LINT_BIN)-$(GOLANGCI_LINT_VER)) # ENVTEST_K8S_VERSION refers to the version of k8s binary assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.28.0 +ENVTEST_K8S_VERSION = 1.30.0 # ENVTEST_VER is the version of the ENVTEST binary ENVTEST_VER = v0.0.0-20240317073005-bd9ea79e8d18 ENVTEST_BIN := setup-envtest diff --git a/apis/placement/v1alpha1/disruptionbudget_types.go b/apis/placement/v1alpha1/disruptionbudget_types.go new file mode 100644 index 000000000..172078514 --- /dev/null +++ b/apis/placement/v1alpha1/disruptionbudget_types.go @@ -0,0 +1,116 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster,categories={fleet,fleet-placement},shortName=crpdb +// +kubebuilder:subresource:status +// +kubebuilder:storageversion + +// ClusterResourcePlacementDisruptionBudget is the policy applied to a ClusterResourcePlacement +// object that specifies its disruption budget, i.e., how many placements (clusters) can be +// down at the same time due to voluntary disruptions (e.g., evictions). Involuntary +// disruptions are not subject to this budget, but will still count against it. +// +// To apply a ClusterResourcePlacementDisruptionBudget to a ClusterResourcePlacement, use the +// same name for the ClusterResourcePlacementDisruptionBudget object as the ClusterResourcePlacement +// object. This guarantees a 1:1 link between the two objects. +type ClusterResourcePlacementDisruptionBudget struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Spec is the desired state of the ClusterResourcePlacementDisruptionBudget. + // +kubebuilder:validation:XValidation:rule="!(has(self.maxUnavailable) && has(self.minAvailable))",message="Both MaxUnavailable and MinAvailable cannot be specified" + // +required + Spec PlacementDisruptionBudgetSpec `json:"spec"` +} + +// PlacementDisruptionBudgetSpec is the desired state of the PlacementDisruptionBudget. +type PlacementDisruptionBudgetSpec struct { + // MaxUnavailable is the maximum number of placements (clusters) that can be down at the + // same time due to voluntary disruptions. For example, a setting of 1 would imply that + // a voluntary disruption (e.g., an eviction) can only happen if all placements (clusters) + // from the linked Placement object are applied and available. + // + // This can be either an absolute value (e.g., 1) or a percentage (e.g., 10%). + // + // If a percentage is specified, Fleet will calculate the corresponding absolute values + // as follows: + // * if the linked Placement object is of the PickFixed placement type, + // the percentage is against the number of clusters specified in the placement (i.e., the + // length of ClusterNames field in the placement policy); + // * if the linked Placement object is of the PickAll placement type, + // the percentage is against the total number of clusters being selected by the scheduler + // at the time of the evaluation of the disruption budget; + // * if the linked Placement object is of the PickN placement type, + // the percentage is against the number of clusters specified in the placement (i.e., the + // value of the NumberOfClusters fields in the placement policy). + // The end result will be rounded up to the nearest integer if applicable. + // + // One may use a value of 0 for this field; in this case, no voluntary disruption would be + // allowed. + // + // This field is mutually exclusive with the MinAvailable field in the spec; exactly one + // of them can be set at a time. + // + // +kubebuilder:validation:XIntOrString + // +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches('^(100|[0-9]{1,2}%)$') : self >= 0",message="If supplied value is String should match regex '^(100|[0-9]{1,2}%)$' or If supplied value is Integer must be greater than or equal to 0" + // +optional + MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` + + // MinAvailable is the minimum number of placements (clusters) that must be available at any + // time despite voluntary disruptions. For example, a setting of 10 would imply that + // a voluntary disruption (e.g., an eviction) can only happen if there are at least 11 + // placements (clusters) from the linked Placement object are applied and available. + // + // This can be either an absolute value (e.g., 1) or a percentage (e.g., 10%). + // + // If a percentage is specified, Fleet will calculate the corresponding absolute values + // as follows: + // * if the linked Placement object is of the PickFixed placement type, + // the percentage is against the number of clusters specified in the placement (i.e., the + // length of ClusterNames field in the placement policy); + // * if the linked Placement object is of the PickAll placement type, + // the percentage is against the total number of clusters being selected by the scheduler + // at the time of the evaluation of the disruption budget; + // * if the linked Placement object is of the PickN placement type, + // the percentage is against the number of clusters specified in the placement (i.e., the + // value of the NumberOfClusters fields in the placement policy). + // The end result will be rounded up to the nearest integer if applicable. + // + // One may use a value of 0 for this field; in this case, voluntary disruption would be + // allowed at any time. + // + // This field is mutually exclusive with the MaxUnavailable field in the spec; exactly one + // of them can be set at a time. + // + // +kubebuilder:validation:XIntOrString + // +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches('^(100|[0-9]{1,2}%)$') : self >= 0",message="If supplied value is String should match regex '^(100|[0-9]{1,2}%)$' or If supplied value is Integer must be greater than or equal to 0" + // +optional + MinAvailable *intstr.IntOrString `json:"minAvailable,omitempty"` +} + +// ClusterResourcePlacementDisruptionBudgetList contains a list of ClusterResourcePlacementDisruptionBudget objects. +// +kubebuilder:resource:scope=Cluster +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type ClusterResourcePlacementDisruptionBudgetList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + // Items is the list of PlacementDisruptionBudget objects. + Items []ClusterResourcePlacementDisruptionBudget `json:"items"` +} + +func init() { + SchemeBuilder.Register( + &ClusterResourcePlacementDisruptionBudget{}, + &ClusterResourcePlacementDisruptionBudgetList{}) +} diff --git a/apis/placement/v1alpha1/eviction_types.go b/apis/placement/v1alpha1/eviction_types.go new file mode 100644 index 000000000..d91769d55 --- /dev/null +++ b/apis/placement/v1alpha1/eviction_types.go @@ -0,0 +1,125 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster,categories={fleet,fleet-placement},shortName=crpe +// +kubebuilder:subresource:status +// +kubebuilder:storageversion + +// ClusterResourcePlacementEviction is an eviction attempt on a specific placement from +// a ClusterResourcePlacement object; one may use this API to force the removal of specific +// resources from a cluster. +// +// An eviction is a voluntary disruption; its execution is subject to the disruption budget +// linked with the target ClusterResourcePlacement object (if present). +// +// Beware that an eviction alone does not guarantee that a placement will not re-appear; i.e., +// after an eviction, the Fleet scheduler might still pick the previous target cluster for +// placement. To prevent this, considering adding proper taints to the target cluster before running +// an eviction that will exclude it from future placements; this is especially true in scenarios +// where one would like to perform a cluster replacement. +// +// For safety reasons, Fleet will only execute an eviction once; the spec in this object is immutable, +// and once executed, the object will be ignored after. To trigger another eviction attempt on the +// same placement from the same ClusterResourcePlacement object, one must re-create (delete and +// create) the same Eviction object. Note also that an Eviction object will be +// ignored once it is deemed invalid (e.g., such an object might be targeting a CRP object or +// a placement that does not exist yet), even if it does become valid later +// (e.g., the CRP object or the placement appears later). To fix the situation, re-create the +// Eviction object. +// +// Executed evictions might be kept around for a while for auditing purposes; the Fleet controllers might +// have a TTL set up for such objects and will garbage collect them automatically. For further +// information, see the Fleet documentation. +type ClusterResourcePlacementEviction struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Spec is the desired state of the ClusterResourcePlacementEviction. + // + // Note that all fields in the spec are immutable. + // +required + Spec PlacementEvictionSpec `json:"spec"` + + // Status is the observed state of the ClusterResourcePlacementEviction. + // +optional + Status PlacementEvictionStatus `json:"status,omitempty"` +} + +// PlacementEvictionSpec is the desired state of the parent PlacementEviction. +type PlacementEvictionSpec struct { + // PlacementName is the name of the Placement object which + // the Eviction object targets. + // +kubebuilder:validation:Required + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="The PlacementName field is immutable" + // +kubebuilder:validation:MaxLength=255 + PlacementName string `json:"placementName"` + + // ClusterName is the name of the cluster that the Eviction object targets. + // +kubebuilder:validation:Required + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="The ClusterName field is immutable" + // +kubebuilder:validation:MaxLength=255 + ClusterName string `json:"clusterName"` +} + +// PlacementEvictionStatus is the observed state of the parent PlacementEviction. +type PlacementEvictionStatus struct { + // Conditions is the list of currently observed conditions for the + // PlacementEviction object. + // + // Available condition types include: + // * Valid: whether the Eviction object is valid, i.e., it targets at a valid placement. + // * Executed: whether the Eviction object has been executed. + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// PlacementEvictionConditionType identifies a specific condition of the +// PlacementEviction. +type PlacementEvictionConditionType string + +const ( + // PlacementEvictionConditionTypeValid indicates whether the Eviction object is valid. + // + // The following values are possible: + // * True: the Eviction object is valid. + // * False: the Eviction object is invalid; it might be targeting a CRP object or a placement + // that does not exist yet. + // Note that this is a terminal state; once an Eviction object is deemed invalid, it will + // not be evaluated again, even if the target appears later. + PlacementEvictionConditionTypeValid PlacementEvictionConditionType = "Valid" + + // PlacementEvictionConditionTypeExecuted indicates whether the Eviction object has been executed. + // + // The following values are possible: + // * True: the Eviction object has been executed. + // Note that this is a terminal state; once an Eviction object is executed, it will not be + // executed again. + // * False: the Eviction object has not been executed yet. + PlacementEvictionConditionTypeExecuted PlacementEvictionConditionType = "Executed" +) + +// ClusterResourcePlacementEvictionList contains a list of ClusterResourcePlacementEviction objects. +// +kubebuilder:resource:scope=Cluster +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type ClusterResourcePlacementEvictionList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + // Items is the list of ClusterResourcePlacementEviction objects. + Items []ClusterResourcePlacementEviction `json:"items"` +} + +func init() { + SchemeBuilder.Register( + &ClusterResourcePlacementEviction{}, + &ClusterResourcePlacementEvictionList{}) +} diff --git a/apis/placement/v1alpha1/zz_generated.deepcopy.go b/apis/placement/v1alpha1/zz_generated.deepcopy.go index f5c9c8c2d..d9caa6b5c 100644 --- a/apis/placement/v1alpha1/zz_generated.deepcopy.go +++ b/apis/placement/v1alpha1/zz_generated.deepcopy.go @@ -13,6 +13,7 @@ import ( "go.goms.io/fleet/apis/placement/v1beta1" "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -313,6 +314,123 @@ func (in *ClusterResourceOverrideSpec) DeepCopy() *ClusterResourceOverrideSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterResourcePlacementDisruptionBudget) DeepCopyInto(out *ClusterResourcePlacementDisruptionBudget) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterResourcePlacementDisruptionBudget. +func (in *ClusterResourcePlacementDisruptionBudget) DeepCopy() *ClusterResourcePlacementDisruptionBudget { + if in == nil { + return nil + } + out := new(ClusterResourcePlacementDisruptionBudget) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterResourcePlacementDisruptionBudget) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterResourcePlacementDisruptionBudgetList) DeepCopyInto(out *ClusterResourcePlacementDisruptionBudgetList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClusterResourcePlacementDisruptionBudget, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterResourcePlacementDisruptionBudgetList. +func (in *ClusterResourcePlacementDisruptionBudgetList) DeepCopy() *ClusterResourcePlacementDisruptionBudgetList { + if in == nil { + return nil + } + out := new(ClusterResourcePlacementDisruptionBudgetList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterResourcePlacementDisruptionBudgetList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterResourcePlacementEviction) DeepCopyInto(out *ClusterResourcePlacementEviction) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterResourcePlacementEviction. +func (in *ClusterResourcePlacementEviction) DeepCopy() *ClusterResourcePlacementEviction { + if in == nil { + return nil + } + out := new(ClusterResourcePlacementEviction) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterResourcePlacementEviction) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterResourcePlacementEvictionList) DeepCopyInto(out *ClusterResourcePlacementEvictionList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClusterResourcePlacementEviction, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterResourcePlacementEvictionList. +func (in *ClusterResourcePlacementEvictionList) DeepCopy() *ClusterResourcePlacementEvictionList { + if in == nil { + return nil + } + out := new(ClusterResourcePlacementEvictionList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterResourcePlacementEvictionList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterUpdatingStatus) DeepCopyInto(out *ClusterUpdatingStatus) { *out = *in @@ -410,6 +528,68 @@ func (in *OverrideRule) DeepCopy() *OverrideRule { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlacementDisruptionBudgetSpec) DeepCopyInto(out *PlacementDisruptionBudgetSpec) { + *out = *in + if in.MaxUnavailable != nil { + in, out := &in.MaxUnavailable, &out.MaxUnavailable + *out = new(intstr.IntOrString) + **out = **in + } + if in.MinAvailable != nil { + in, out := &in.MinAvailable, &out.MinAvailable + *out = new(intstr.IntOrString) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlacementDisruptionBudgetSpec. +func (in *PlacementDisruptionBudgetSpec) DeepCopy() *PlacementDisruptionBudgetSpec { + if in == nil { + return nil + } + out := new(PlacementDisruptionBudgetSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlacementEvictionSpec) DeepCopyInto(out *PlacementEvictionSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlacementEvictionSpec. +func (in *PlacementEvictionSpec) DeepCopy() *PlacementEvictionSpec { + if in == nil { + return nil + } + out := new(PlacementEvictionSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlacementEvictionStatus) DeepCopyInto(out *PlacementEvictionStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlacementEvictionStatus. +func (in *PlacementEvictionStatus) DeepCopy() *PlacementEvictionStatus { + if in == nil { + return nil + } + out := new(PlacementEvictionStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PlacementReference) DeepCopyInto(out *PlacementReference) { *out = *in diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementdisruptionbudgets.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementdisruptionbudgets.yaml new file mode 100644 index 000000000..21f85083c --- /dev/null +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementdisruptionbudgets.yaml @@ -0,0 +1,148 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: clusterresourceplacementdisruptionbudgets.placement.kubernetes-fleet.io +spec: + group: placement.kubernetes-fleet.io + names: + categories: + - fleet + - fleet-placement + kind: ClusterResourcePlacementDisruptionBudget + listKind: ClusterResourcePlacementDisruptionBudgetList + plural: clusterresourceplacementdisruptionbudgets + shortNames: + - crpdb + singular: clusterresourceplacementdisruptionbudget + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + ClusterResourcePlacementDisruptionBudget is the policy applied to a ClusterResourcePlacement + object that specifies its disruption budget, i.e., how many placements (clusters) can be + down at the same time due to voluntary disruptions (e.g., evictions). Involuntary + disruptions are not subject to this budget, but will still count against it. + + + To apply a ClusterResourcePlacementDisruptionBudget to a ClusterResourcePlacement, use the + same name for the ClusterResourcePlacementDisruptionBudget object as the ClusterResourcePlacement + object. This guarantees a 1:1 link between the two objects. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec is the desired state of the ClusterResourcePlacementDisruptionBudget. + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + MaxUnavailable is the maximum number of placements (clusters) that can be down at the + same time due to voluntary disruptions. For example, a setting of 1 would imply that + a voluntary disruption (e.g., an eviction) can only happen if all placements (clusters) + from the linked Placement object are applied and available. + + + This can be either an absolute value (e.g., 1) or a percentage (e.g., 10%). + + + If a percentage is specified, Fleet will calculate the corresponding absolute values + as follows: + * if the linked Placement object is of the PickFixed placement type, + the percentage is against the number of clusters specified in the placement (i.e., the + length of ClusterNames field in the placement policy); + * if the linked Placement object is of the PickAll placement type, + the percentage is against the total number of clusters being selected by the scheduler + at the time of the evaluation of the disruption budget; + * if the linked Placement object is of the PickN placement type, + the percentage is against the number of clusters specified in the placement (i.e., the + value of the NumberOfClusters fields in the placement policy). + The end result will be rounded up to the nearest integer if applicable. + + + One may use a value of 0 for this field; in this case, no voluntary disruption would be + allowed. + + + This field is mutually exclusive with the MinAvailable field in the spec; exactly one + of them can be set at a time. + x-kubernetes-int-or-string: true + x-kubernetes-validations: + - message: If supplied value is String should match regex '^(100|[0-9]{1,2}%)$' + or If supplied value is Integer must be greater than or equal + to 0 + rule: 'type(self) == string ? self.matches(''^(100|[0-9]{1,2}%)$'') + : self >= 0' + minAvailable: + anyOf: + - type: integer + - type: string + description: |- + MinAvailable is the minimum number of placements (clusters) that must be available at any + time despite voluntary disruptions. For example, a setting of 10 would imply that + a voluntary disruption (e.g., an eviction) can only happen if there are at least 11 + placements (clusters) from the linked Placement object are applied and available. + + + This can be either an absolute value (e.g., 1) or a percentage (e.g., 10%). + + + If a percentage is specified, Fleet will calculate the corresponding absolute values + as follows: + * if the linked Placement object is of the PickFixed placement type, + the percentage is against the number of clusters specified in the placement (i.e., the + length of ClusterNames field in the placement policy); + * if the linked Placement object is of the PickAll placement type, + the percentage is against the total number of clusters being selected by the scheduler + at the time of the evaluation of the disruption budget; + * if the linked Placement object is of the PickN placement type, + the percentage is against the number of clusters specified in the placement (i.e., the + value of the NumberOfClusters fields in the placement policy). + The end result will be rounded up to the nearest integer if applicable. + + + One may use a value of 0 for this field; in this case, voluntary disruption would be + allowed at any time. + + + This field is mutually exclusive with the MaxUnavailable field in the spec; exactly one + of them can be set at a time. + x-kubernetes-int-or-string: true + x-kubernetes-validations: + - message: If supplied value is String should match regex '^(100|[0-9]{1,2}%)$' + or If supplied value is Integer must be greater than or equal + to 0 + rule: 'type(self) == string ? self.matches(''^(100|[0-9]{1,2}%)$'') + : self >= 0' + type: object + x-kubernetes-validations: + - message: Both MaxUnavailable and MinAvailable cannot be specified + rule: '!(has(self.maxUnavailable) && has(self.minAvailable))' + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementevictions.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementevictions.yaml new file mode 100644 index 000000000..96044b4f4 --- /dev/null +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementevictions.yaml @@ -0,0 +1,188 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: clusterresourceplacementevictions.placement.kubernetes-fleet.io +spec: + group: placement.kubernetes-fleet.io + names: + categories: + - fleet + - fleet-placement + kind: ClusterResourcePlacementEviction + listKind: ClusterResourcePlacementEvictionList + plural: clusterresourceplacementevictions + shortNames: + - crpe + singular: clusterresourceplacementeviction + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + ClusterResourcePlacementEviction is an eviction attempt on a specific placement from + a ClusterResourcePlacement object; one may use this API to force the removal of specific + resources from a cluster. + + + An eviction is a voluntary disruption; its execution is subject to the disruption budget + linked with the target ClusterResourcePlacement object (if present). + + + Beware that an eviction alone does not guarantee that a placement will not re-appear; i.e., + after an eviction, the Fleet scheduler might still pick the previous target cluster for + placement. To prevent this, considering adding proper taints to the target cluster before running + an eviction that will exclude it from future placements; this is especially true in scenarios + where one would like to perform a cluster replacement. + + + For safety reasons, Fleet will only execute an eviction once; the spec in this object is immutable, + and once executed, the object will be ignored after. To trigger another eviction attempt on the + same placement from the same ClusterResourcePlacement object, one must re-create (delete and + create) the same Eviction object. Note also that an Eviction object will be + ignored once it is deemed invalid (e.g., such an object might be targeting a CRP object or + a placement that does not exist yet), even if it does become valid later + (e.g., the CRP object or the placement appears later). To fix the situation, re-create the + Eviction object. + + + Executed evictions might be kept around for a while for auditing purposes; the Fleet controllers might + have a TTL set up for such objects and will garbage collect them automatically. For further + information, see the Fleet documentation. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + Spec is the desired state of the ClusterResourcePlacementEviction. + + + Note that all fields in the spec are immutable. + properties: + clusterName: + description: ClusterName is the name of the cluster that the Eviction + object targets. + maxLength: 255 + type: string + x-kubernetes-validations: + - message: The ClusterName field is immutable + rule: self == oldSelf + placementName: + description: |- + PlacementName is the name of the Placement object which + the Eviction object targets. + maxLength: 255 + type: string + x-kubernetes-validations: + - message: The PlacementName field is immutable + rule: self == oldSelf + required: + - clusterName + - placementName + type: object + status: + description: Status is the observed state of the ClusterResourcePlacementEviction. + properties: + conditions: + description: |- + Conditions is the list of currently observed conditions for the + PlacementEviction object. + + + Available condition types include: + * Valid: whether the Eviction object is valid, i.e., it targets at a valid placement. + * Executed: whether the Eviction object has been executed. + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/test/apis/placement/v1alpha1/api_validation_integration_test.go b/test/apis/placement/v1alpha1/api_validation_integration_test.go new file mode 100644 index 000000000..7a5f17d10 --- /dev/null +++ b/test/apis/placement/v1alpha1/api_validation_integration_test.go @@ -0,0 +1,218 @@ +package v1alpha1 + +import ( + "errors" + "fmt" + "reflect" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + placementv1alpha1 "go.goms.io/fleet/apis/placement/v1alpha1" +) + +const ( + crpdbNameTemplate = "test-crpdb-%d" +) + +var _ = Describe("Test placement v1alpha1 API validation", func() { + Context("Test ClusterPlacementDisruptionBudget API validation - valid cases", func() { + It("should allow creation of ClusterPlacementDisruptionBudget with valid maxUnavailable - int", func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(crpdbNameTemplate, GinkgoParallelProcess()), + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 2}, + }, + } + Expect(hubClient.Create(ctx, &crpdb)).Should(Succeed()) + }) + + It("should allow creation of ClusterPlacementDisruptionBudget with valid maxUnavailable - string", func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(crpdbNameTemplate, GinkgoParallelProcess()), + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{Type: intstr.String, StrVal: "10%"}, + }, + } + Expect(hubClient.Create(ctx, &crpdb)).Should(Succeed()) + }) + + It("should allow creation of ClusterPlacementDisruptionBudget with valid minAvailable - int", func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(crpdbNameTemplate, GinkgoParallelProcess()), + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 2}, + }, + } + Expect(hubClient.Create(ctx, &crpdb)).Should(Succeed()) + }) + + It("should allow creation of ClusterPlacementDisruptionBudget with valid minAvailable - string", func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(crpdbNameTemplate, GinkgoParallelProcess()), + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{Type: intstr.String, StrVal: "10%"}, + }, + } + Expect(hubClient.Create(ctx, &crpdb)).Should(Succeed()) + }) + + AfterEach(func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(crpdbNameTemplate, GinkgoParallelProcess()), + }, + } + Expect(hubClient.Delete(ctx, &crpdb)).Should(Succeed()) + }) + }) + + Context("Test ClusterPlacementDisruptionBudget API validation - invalid cases", func() { + It("should deny creation of ClusterPlacementDisruptionBudget when both maxUnavailable and minAvailable are specified", func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(crpdbNameTemplate, GinkgoParallelProcess()), + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 1}, + MinAvailable: &intstr.IntOrString{Type: intstr.String, StrVal: "10%"}, + }, + } + err := hubClient.Create(ctx, &crpdb) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create CRPDB call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("Both MaxUnavailable and MinAvailable cannot be specified")) + }) + + It("should deny creation of ClusterPlacementDisruptionBudget with invalid maxUnavailable - negative int", func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(crpdbNameTemplate, GinkgoParallelProcess()), + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: -1}, + }, + } + err := hubClient.Create(ctx, &crpdb) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create CRPDB call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid: spec.maxUnavailable")) + }) + + It("should deny creation of ClusterPlacementDisruptionBudget with invalid maxUnavailable - negative percentage", func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(crpdbNameTemplate, GinkgoParallelProcess()), + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{Type: intstr.String, StrVal: "-1%"}, + }, + } + err := hubClient.Create(ctx, &crpdb) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create CRPDB call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid: spec.maxUnavailable")) + }) + + It("should deny creation of ClusterPlacementDisruptionBudget with invalid maxUnavailable - greater than 100", func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(crpdbNameTemplate, GinkgoParallelProcess()), + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{Type: intstr.String, StrVal: "101%"}, + }, + } + err := hubClient.Create(ctx, &crpdb) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create CRPDB call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid: spec.maxUnavailable")) + }) + + It("should deny creation of ClusterPlacementDisruptionBudget with invalid maxUnavailable - no percentage specified", func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(crpdbNameTemplate, GinkgoParallelProcess()), + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{Type: intstr.String, StrVal: "-1"}, + }, + } + err := hubClient.Create(ctx, &crpdb) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create CRPDB call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid: spec.maxUnavailable")) + }) + + It("should deny creation of ClusterPlacementDisruptionBudget with invalid minAvailable - negative int", func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(crpdbNameTemplate, GinkgoParallelProcess()), + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{Type: intstr.Int, IntVal: -1}, + }, + } + err := hubClient.Create(ctx, &crpdb) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create CRPDB call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid: spec.minAvailable")) + }) + + It("should deny creation of ClusterPlacementDisruptionBudget with invalid minAvailable - negative percentage", func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(crpdbNameTemplate, GinkgoParallelProcess()), + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{Type: intstr.String, StrVal: "-1%"}, + }, + } + err := hubClient.Create(ctx, &crpdb) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create CRPDB call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid: spec.minAvailable")) + }) + + It("should deny creation of ClusterPlacementDisruptionBudget with invalid minAvailable - greater than 100", func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(crpdbNameTemplate, GinkgoParallelProcess()), + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{Type: intstr.String, StrVal: "101%"}, + }, + } + err := hubClient.Create(ctx, &crpdb) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create CRPDB call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid: spec.minAvailable")) + }) + + It("should deny creation of ClusterPlacementDisruptionBudget with invalid minAvailable - no percentage specified", func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(crpdbNameTemplate, GinkgoParallelProcess()), + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{Type: intstr.String, StrVal: "-1"}, + }, + } + err := hubClient.Create(ctx, &crpdb) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create CRPDB call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid: spec.minAvailable")) + }) + }) +}) diff --git a/test/apis/placement/v1alpha1/suite_test.go b/test/apis/placement/v1alpha1/suite_test.go new file mode 100644 index 000000000..45af00112 --- /dev/null +++ b/test/apis/placement/v1alpha1/suite_test.go @@ -0,0 +1,88 @@ +package v1alpha1 + +import ( + "context" + "flag" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + "k8s.io/klog/v2/textlogger" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + placementv1alpha1 "go.goms.io/fleet/apis/placement/v1alpha1" +) + +var ( + hubTestEnv *envtest.Environment + hubClient client.Client + ctx context.Context + cancel context.CancelFunc +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "ClusterResourcePlacement Controller Suite") +} + +var _ = BeforeSuite(func() { + By("Setup klog") + fs := flag.NewFlagSet("klog", flag.ContinueOnError) + klog.InitFlags(fs) + Expect(fs.Parse([]string{"--v", "5", "-add_dir_header", "true"})).Should(Succeed()) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrap the test environment") + // Start the cluster. + hubTestEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("..", "..", "..", "..", "config", "crd", "bases"), + }, + ErrorIfCRDPathMissing: true, + } + hubCfg, err := hubTestEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(hubCfg).NotTo(BeNil()) + + Expect(placementv1alpha1.AddToScheme(scheme.Scheme)).Should(Succeed()) + + klog.InitFlags(flag.CommandLine) + flag.Parse() + // Create the hub controller manager. + hubCtrlMgr, err := ctrl.NewManager(hubCfg, ctrl.Options{ + Scheme: scheme.Scheme, + Metrics: metricsserver.Options{ + BindAddress: "0", + }, + Logger: textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(4))), + }) + Expect(err).NotTo(HaveOccurred()) + + // Set up the client. + // The client must be one with cache (i.e. configured by the controller manager) to make + // use of the cache indexes. + hubClient = hubCtrlMgr.GetClient() + Expect(hubClient).NotTo(BeNil()) + + go func() { + defer GinkgoRecover() + err = hubCtrlMgr.Start(ctx) + Expect(err).ToNot(HaveOccurred(), "failed to start manager for hub") + }() +}) + +var _ = AfterSuite(func() { + defer klog.Flush() + cancel() + + By("tearing down the test environment") + Expect(hubTestEnv.Stop()).Should(Succeed()) +})