Add feature toggle annotations to Space & NSTemplateSet

controllers/space/space_controller.go

@@ -152,7 +152,6 @@ func (r *Reconciler) ensureNSTemplateSet(ctx context.Context, space *toolchainv1
 		// look-up and delete the NSTemplateSet on the current member cluster
 		if isBeingDeleted, err := r.deleteNSTemplateSetFromCluster(ctx, space, space.Status.TargetCluster); err != nil {
 			return norequeue, r.setStatusRetargetFailed(ctx, space, err)
-
 		} else if isBeingDeleted {
 			logger.Info("wait while NSTemplateSet is being deleted", "member_cluster", space.Status.TargetCluster)
 			return norequeue, r.setStatusRetargeting(ctx, space)
@@ -334,9 +333,11 @@ func (r *Reconciler) manageNSTemplateSet(ctx context.Context, space *toolchainv1
 
 	// update the NSTemplateSet if needed (including in case of missing space roles)
 	nsTmplSetSpec := NewNSTemplateSetSpec(space, spaceBindings, tmplTier)
-	if !reflect.DeepEqual(nsTmplSet.Spec, nsTmplSetSpec) {
+	featureToggleAnnotationUpdated := ensureFeatureToggleAnnotation(space, nsTmplSet) // also check if the feature annotation was updated
+	if !reflect.DeepEqual(nsTmplSet.Spec, nsTmplSetSpec) || featureToggleAnnotationUpdated {
 		logger.Info("NSTemplateSet is not up-to-date")
-		// postpone NSTemplateSet updates if needed (but only for NSTemplateTier updates, not tier promotions or changes in spacebindings)
+		// postpone NSTemplateSet updates if needed
+		// but only for NSTemplateTier updates, not tier promotions or changes in spacebindings or feature toggle annotation
 		if space.Labels[hash.TemplateTierHashLabelKey(space.Spec.TierName)] != "" &&
 			condition.IsTrue(space.Status.Conditions, toolchainv1alpha1.ConditionReady) {
 			// postpone if needed, so we don't overflow the cluster with too many concurrent updates
@@ -387,9 +388,34 @@ func NewNSTemplateSet(namespace string, space *toolchainv1alpha1.Space, bindings
 		},
 	}
 	nsTmplSet.Spec = NewNSTemplateSetSpec(space, bindings, tmplTier)
+
+	// propagate the feature annotation from the space
+	ensureFeatureToggleAnnotation(space, nsTmplSet)
 	return nsTmplSet
 }
 
+// ensureFeatureToggleAnnotation propagates the feature toggle annotation from the parent Space to the child NSTemplateSet
+// if it's present in the space. If there is no annotation in the Space then the annotation is deleted from the NSTemplateSet too.
+// Returns true if there is any changes in the NSTemplateSet's annotation.
+func ensureFeatureToggleAnnotation(space *toolchainv1alpha1.Space, nsTemplateSet *toolchainv1alpha1.NSTemplateSet) bool {
+	winners, found := space.Annotations[toolchainv1alpha1.FeatureToggleNameAnnotationKey]
+	oldNSTemplateAnnotation, oldNSTemplateAnnotationFound := nsTemplateSet.Annotations[toolchainv1alpha1.FeatureToggleNameAnnotationKey]
+	if found {
+		// Propagate from Space to NSTemplateSet
+		if nsTemplateSet.Annotations == nil {
+			nsTemplateSet.Annotations = make(map[string]string)
+		}
+		nsTemplateSet.Annotations[toolchainv1alpha1.FeatureToggleNameAnnotationKey] = winners
+		return oldNSTemplateAnnotation != winners
+	}
+	if oldNSTemplateAnnotationFound {
+		// Delete the annotation from NSTemplateSet because it doesn't exist in the Space anymore
+		delete(nsTemplateSet.Annotations, toolchainv1alpha1.FeatureToggleNameAnnotationKey)
+		return true
+	}
+	return false
+}
+
 func NewNSTemplateSetSpec(space *toolchainv1alpha1.Space, bindings []toolchainv1alpha1.SpaceBinding, tmplTier *toolchainv1alpha1.NSTemplateTier) toolchainv1alpha1.NSTemplateSetSpec {
 	s := toolchainv1alpha1.NSTemplateSetSpec{
 		TierName: space.Spec.TierName,

controllers/toolchainconfig/configuration.go

@@ -318,6 +318,32 @@
 	return duration
 }
 
+func (d TiersConfig) FeatureToggles() []FeatureToggle {
+	toggles := make([]FeatureToggle, 0, len(d.tiers.FeatureToggles))
+	for _, t := range d.tiers.FeatureToggles {
+		toggles = append(toggles, FeatureToggle{t})
+	}
+	return toggles
+}
+
+type FeatureToggle struct {
+	toggle toolchainv1alpha1.FeatureToggle
+}
+
+func NewFeatureToggle(t toolchainv1alpha1.FeatureToggle) FeatureToggle {
+	return FeatureToggle{
+		toggle: t,
+	}
+}
+
+func (t FeatureToggle) Name() string {
+	return t.toggle.Name
+}
+
+func (t FeatureToggle) Weight() uint {
+	return commonconfig.GetUint(t.toggle.Weight, 100)
+}
+
 type ToolchainStatusConfig struct {
 	t toolchainv1alpha1.ToolchainStatusConfig
 }

controllers/toolchainconfig/configuration_test.go

@@ -442,6 +442,7 @@ func TestTiers(t *testing.T) {
 		assert.Equal(t, "deactivate30", toolchainCfg.Tiers().DefaultUserTier())
 		assert.Equal(t, "base", toolchainCfg.Tiers().DefaultSpaceTier())
 		assert.Equal(t, 24*time.Hour, toolchainCfg.Tiers().DurationBeforeChangeTierRequestDeletion())
+		assert.Empty(t, toolchainCfg.Tiers().FeatureToggles())
 	})
 	t.Run("invalid", func(t *testing.T) {
 		cfg := commonconfig.NewToolchainConfigObjWithReset(t, testconfig.Tiers().DurationBeforeChangeTierRequestDeletion("rapid"))
@@ -450,15 +451,23 @@ func TestTiers(t *testing.T) {
 		assert.Equal(t, 24*time.Hour, toolchainCfg.Tiers().DurationBeforeChangeTierRequestDeletion())
 	})
 	t.Run("non-default", func(t *testing.T) {
+		weight10 := uint(10)
 		cfg := commonconfig.NewToolchainConfigObjWithReset(t, testconfig.Tiers().
 			DefaultUserTier("deactivate90").
 			DefaultSpaceTier("advanced").
-			DurationBeforeChangeTierRequestDeletion("48h"))
+			DurationBeforeChangeTierRequestDeletion("48h").
+			FeatureToggle("feature-1", nil). // With default weight
+			FeatureToggle("feature-2", &weight10))
 		toolchainCfg := newToolchainConfig(cfg, map[string]map[string]string{})
 
 		assert.Equal(t, "deactivate90", toolchainCfg.Tiers().DefaultUserTier())
 		assert.Equal(t, "advanced", toolchainCfg.Tiers().DefaultSpaceTier())
 		assert.Equal(t, 48*time.Hour, toolchainCfg.Tiers().DurationBeforeChangeTierRequestDeletion())
+		assert.Len(t, toolchainCfg.Tiers().FeatureToggles(), 2)
+		assert.Equal(t, "feature-1", toolchainCfg.Tiers().FeatureToggles()[0].Name())
+		assert.Equal(t, uint(100), toolchainCfg.Tiers().FeatureToggles()[0].Weight()) // default weight
+		assert.Equal(t, "feature-2", toolchainCfg.Tiers().FeatureToggles()[1].Name())
+		assert.Equal(t, weight10, toolchainCfg.Tiers().FeatureToggles()[1].Weight())
 	})
 }
 

controllers/usersignup/usersignup_controller.go

@@ -359,7 +359,7 @@ func (r *Reconciler) checkIfMurAlreadyExists(
 		logger.Info("MasterUserRecord exists", "Name", mur.Name)
 
 		if shouldManageSpace(userSignup) {
-			space, created, err := r.ensureSpace(ctx, userSignup, mur, spaceTier)
+			space, created, err := r.ensureSpace(ctx, userSignup, mur, spaceTier, config)
 			if err != nil {
 				return true, r.wrapErrorWithStatusUpdate(ctx, userSignup, r.setStatusFailedToCreateSpace, err, "error creating Space")
 			}
@@ -689,6 +689,7 @@ func (r *Reconciler) ensureSpace(
 	userSignup *toolchainv1alpha1.UserSignup,
 	mur *toolchainv1alpha1.MasterUserRecord,
 	spaceTier *toolchainv1alpha1.NSTemplateTier,
+	config toolchainconfig.ToolchainConfig,
 ) (*toolchainv1alpha1.Space, bool, error) {
 	logger := log.FromContext(ctx)
 	logger.Info("Ensuring Space", "UserSignup", userSignup.Name, "MUR", mur.Name, "NSTemplateTier", spaceTier.Name)
@@ -715,7 +716,7 @@ func (r *Reconciler) ensureSpace(
 	}
 	tCluster := targetCluster(mur.Spec.UserAccounts[0].TargetCluster)
 
-	space = spaceutil.NewSpace(userSignup, tCluster.getClusterName(), mur.Name, spaceTier.Name)
+	space = spaceutil.NewSpaceWithFeatureToggles(userSignup, tCluster.getClusterName(), mur.Name, spaceTier.Name, config.Tiers().FeatureToggles())
 
 	err = r.Client.Create(ctx, space)
 	if err != nil {

controllers/usersignup/usersignup_controller_test.go

@@ -206,14 +206,27 @@ func TestUserSignupCreateSpaceAndSpaceBindingOk(t *testing.T) {
 			commonsignup.WithTargetCluster("member1"),
 			commonsignup.WithStateLabel(toolchainv1alpha1.UserSignupStateLabelValueNotReady),
 			commonsignup.WithAnnotation(toolchainv1alpha1.SkipAutoCreateSpaceAnnotationKey, "true")),
+		"with feature toggles": commonsignup.NewUserSignup(
+			commonsignup.ApprovedManually(),
+			commonsignup.WithTargetCluster("member1"),
+			commonsignup.WithStateLabel(toolchainv1alpha1.UserSignupStateLabelValueNotReady),
+			commonsignup.WithoutAnnotation(toolchainv1alpha1.SkipAutoCreateSpaceAnnotationKey)),
 	} {
 		t.Run(testname, func(t *testing.T) {
 			// given
 			defer counter.Reset()
 
 			mur := newMasterUserRecord(userSignup, "member1", deactivate30Tier.Name, "foo")
 			mur.Labels = map[string]string{toolchainv1alpha1.MasterUserRecordOwnerLabelKey: userSignup.Name}
-			r, req, _ := prepareReconcile(t, userSignup.Name, spaceProvisionerConfig, userSignup, mur, baseNSTemplateTier, base2NSTemplateTier, deactivate30Tier, deactivate80Tier, event)
+
+			config := &toolchainv1alpha1.ToolchainConfig{}
+			switch testname {
+			case "with feature toggles":
+				weight100 := uint(100)
+				weight0 := uint(0)
+				config = commonconfig.NewToolchainConfigObjWithReset(t, testconfig.Tiers().FeatureToggle("feature-on", &weight100).FeatureToggle("feature-off", &weight0))
+			}
+			r, req, _ := prepareReconcile(t, userSignup.Name, spaceProvisionerConfig, config, userSignup, mur, baseNSTemplateTier, base2NSTemplateTier, deactivate30Tier, deactivate80Tier, event)
 
 			// when
 			res, err := r.Reconcile(context.TODO(), req)
@@ -230,15 +243,26 @@ func TestUserSignupCreateSpaceAndSpaceBindingOk(t *testing.T) {
 					HasLabelWithValue(toolchainv1alpha1.SpaceCreatorLabelKey, userSignup.Name).
 					HasSpecTargetCluster("member1").
 					HasSpecTargetClusterRoles([]string{cluster.RoleLabel(cluster.Tenant)}).
-					HasTier("base")
+					HasTier("base").
+					DoesNotHaveAnnotation(toolchainv1alpha1.FeatureToggleNameAnnotationKey)
+				AssertThatUserSignup(t, req.Namespace, userSignup.Name, r.Client).HasHomeSpace("foo")
+			case "with feature toggles":
+				spacetest.AssertThatSpace(t, test.HostOperatorNs, "foo", r.Client).
+					Exists().
+					HasLabelWithValue(toolchainv1alpha1.SpaceCreatorLabelKey, userSignup.Name).
+					HasSpecTargetCluster("member1").
+					HasSpecTargetClusterRoles([]string{cluster.RoleLabel(cluster.Tenant)}).
+					HasTier("base").
+					HasAnnotationWithValue(toolchainv1alpha1.FeatureToggleNameAnnotationKey, "feature-on")
 				AssertThatUserSignup(t, req.Namespace, userSignup.Name, r.Client).HasHomeSpace("foo")
 			case "with social event":
 				spacetest.AssertThatSpace(t, test.HostOperatorNs, "foo", r.Client).
 					Exists().
 					HasLabelWithValue(toolchainv1alpha1.SpaceCreatorLabelKey, userSignup.Name).
 					HasSpecTargetCluster("member1").
 					HasSpecTargetClusterRoles([]string{cluster.RoleLabel(cluster.Tenant)}).
-					HasTier("base2")
+					HasTier("base2").
+					DoesNotHaveAnnotation(toolchainv1alpha1.FeatureToggleNameAnnotationKey)
 				AssertThatUserSignup(t, req.Namespace, userSignup.Name, r.Client).HasHomeSpace("foo")
 			case "with skip space creation annotation set to true":
 				spacetest.AssertThatSpace(t, test.HostOperatorNs, "foo", r.Client).
@@ -263,6 +287,20 @@ func TestUserSignupCreateSpaceAndSpaceBindingOk(t *testing.T) {
 						HasLabelWithValue(toolchainv1alpha1.SpaceBindingSpaceLabelKey, "foo").
 						HasSpec("foo", "foo", "admin")
 					AssertThatUserSignup(t, req.Namespace, userSignup.Name, r.Client).HasHomeSpace("foo")
+					spacetest.AssertThatSpace(t, test.HostOperatorNs, "foo", r.Client).
+						Exists().
+						DoesNotHaveAnnotation(toolchainv1alpha1.FeatureToggleNameAnnotationKey)
+				case "with feature toggles":
+					spacebindingtest.AssertThatSpaceBinding(t, test.HostOperatorNs, "foo", "foo", r.Client).
+						Exists().
+						HasLabelWithValue(toolchainv1alpha1.SpaceCreatorLabelKey, userSignup.Name).
+						HasLabelWithValue(toolchainv1alpha1.SpaceBindingMasterUserRecordLabelKey, "foo").
+						HasLabelWithValue(toolchainv1alpha1.SpaceBindingSpaceLabelKey, "foo").
+						HasSpec("foo", "foo", "admin")
+					AssertThatUserSignup(t, req.Namespace, userSignup.Name, r.Client).HasHomeSpace("foo")
+					spacetest.AssertThatSpace(t, test.HostOperatorNs, "foo", r.Client).
+						Exists().
+						HasAnnotationWithValue(toolchainv1alpha1.FeatureToggleNameAnnotationKey, "feature-on")
 				case "with skip space creation annotation set to true":
 					spacebindingtest.AssertThatSpaceBinding(t, test.HostOperatorNs, "foo", "foo", r.Client).
 						DoesNotExist()

go.mod

@@ -2,8 +2,8 @@ module github.com/codeready-toolchain/host-operator
 
 require (
 	cloud.google.com/go/recaptchaenterprise/v2 v2.13.0
-	github.com/codeready-toolchain/api v0.0.0-20240620205422-a29b6c658d03
-	github.com/codeready-toolchain/toolchain-common v0.0.0-20240627121104-7ec286970742
+	github.com/codeready-toolchain/api v0.0.0-20240627084210-f4a765461e75
+	github.com/codeready-toolchain/toolchain-common v0.0.0-20240628050807-7f226f55a34a
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
 	github.com/go-logr/logr v1.4.1

go.sum

@@ -137,10 +137,10 @@ github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWH
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
-github.com/codeready-toolchain/api v0.0.0-20240620205422-a29b6c658d03 h1:uqApdceOkNQI1wy2gwA6pEUzhLlqqbbc+ChTvEshIW0=
-github.com/codeready-toolchain/api v0.0.0-20240620205422-a29b6c658d03/go.mod h1:ie9p4LenCCS0LsnbWp6/xwpFDdCWYE0KWzUO6Sk1g0E=
-github.com/codeready-toolchain/toolchain-common v0.0.0-20240627121104-7ec286970742 h1:+eveKbSWfqtnLrWXWiSGiBpcncfby5SVc/ut6J9VLyk=
-github.com/codeready-toolchain/toolchain-common v0.0.0-20240627121104-7ec286970742/go.mod h1:ZwfGkrTArynbvIPOJwQQCwFVW7Dj0XOgi5XzkNG1ov8=
+github.com/codeready-toolchain/api v0.0.0-20240627084210-f4a765461e75 h1:LOrOMTw3bplM+VwzSxi3j9GNseujhcebO/LGr8b95GE=
+github.com/codeready-toolchain/api v0.0.0-20240627084210-f4a765461e75/go.mod h1:ie9p4LenCCS0LsnbWp6/xwpFDdCWYE0KWzUO6Sk1g0E=
+github.com/codeready-toolchain/toolchain-common v0.0.0-20240628050807-7f226f55a34a h1:hXS9egkRRCyvUE//MJf+LDhH3mt1DDaWNFwTenex2dA=
+github.com/codeready-toolchain/toolchain-common v0.0.0-20240628050807-7f226f55a34a/go.mod h1:ZwfGkrTArynbvIPOJwQQCwFVW7Dj0XOgi5XzkNG1ov8=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=

pkg/space/space.go

@@ -2,9 +2,13 @@ package space
 
 import (
 	"fmt"
+	"math/rand"
+	"strings"
 
 	toolchainv1alpha1 "github.com/codeready-toolchain/api/api/v1alpha1"
+	"github.com/codeready-toolchain/host-operator/controllers/toolchainconfig"
 	"github.com/codeready-toolchain/toolchain-common/pkg/cluster"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -29,6 +33,37 @@ func NewSpace(userSignup *toolchainv1alpha1.UserSignup, targetClusterName string
 	return space
 }
 
+// NewSpaceWithFeatureToggles is the same as NewSpace() but also does a feature toggle lottery drawing
+// and adds the corresponding feature annotations for features which "won" and should be enabled for the space.
+func NewSpaceWithFeatureToggles(userSignup *toolchainv1alpha1.UserSignup, targetClusterName string, compliantUserName, tier string, toggles []toolchainconfig.FeatureToggle) *toolchainv1alpha1.Space {
+	s := NewSpace(userSignup, targetClusterName, compliantUserName, tier)
+	addFeatureToggles(s, toggles)
+	return s
+}
+
+// addFeatureToggles does "lottery" drawing for all given feature toggles according to their weights.
+// And it adds the corresponding feature annotation to the space for features that won and should be enabled for the space.
+func addFeatureToggles(space *toolchainv1alpha1.Space, toggles []toolchainconfig.FeatureToggle) {
+	var winners []string
+	for _, t := range toggles {
+		weight := int(t.Weight())
+		// We generate a random number between 0 and 100. If the number is equal to or lower than the weight
+		// then the feature wins.
+		// We don't use recommended crypto/rand here because we don't need crypto grade random generator
+		// and math/rand with pseudo random numbers, which is much cheaper, is sufficient, so we are disabling the linter
+		if weight == 100 || (weight > 0 && rand.Intn(100) <= weight) { // nolint:gosec
+			// Winner!
+			winners = append(winners, t.Name())
+		}
+	}
+	if len(winners) > 0 {
+		if space.Annotations == nil {
+			space.Annotations = make(map[string]string)
+		}
+		space.Annotations[toolchainv1alpha1.FeatureToggleNameAnnotationKey] = strings.Join(winners, ",")
+	}
+}
+
 // NewSubSpace creates a space CR for a SpaceRequest object.
 func NewSubSpace(spaceRequest *toolchainv1alpha1.SpaceRequest, parentSpace *toolchainv1alpha1.Space) *toolchainv1alpha1.Space {
 	labels := map[string]string{

pkg/space/space_test.go

@@ -2,6 +2,8 @@ package space
 
 import (
 	"fmt"
+	"github.com/codeready-toolchain/host-operator/controllers/toolchainconfig"
+	"strings"
 	"testing"
 
 	toolchainv1alpha1 "github.com/codeready-toolchain/api/api/v1alpha1"
@@ -30,6 +32,63 @@ func TestNewSpace(t *testing.T) {
 	assert.Equal(t, expectedSpace, space)
 }
 
+func TestNewSpaceWithFeatureToggles(t *testing.T) {
+	// given
+	userSignup := commonsignup.NewUserSignup()
+	weight100 := uint(100)
+	weight0 := uint(0)
+	weight50 := uint(50)
+	featureToggles := []toolchainconfig.FeatureToggle{
+		toolchainconfig.NewFeatureToggle(toolchainv1alpha1.FeatureToggle{
+			Name:   "my-cool-feature",
+			Weight: &weight100,
+		}),
+		toolchainconfig.NewFeatureToggle(toolchainv1alpha1.FeatureToggle{
+			Name:   "my-not-so-cool-feature",
+			Weight: &weight0,
+		}),
+		toolchainconfig.NewFeatureToggle(toolchainv1alpha1.FeatureToggle{
+			Name:   "my-so-so-feature",
+			Weight: &weight50,
+		}),
+	}
+
+	// when
+	// create space 100 times and verify that the feature with weight 100 is always added
+	// the feature with weight 0 never added
+	// and the feature with weight 50 is added at least once and not added at least once too
+	var myCoolFeatureCount, myNotSoCoolFeatureCount, mySoSoFeatureCount int
+	for i := 0; i < 100; i++ {
+		space := NewSpaceWithFeatureToggles(userSignup, test.MemberClusterName, "johny", "advanced", featureToggles)
+
+		// then
+		expectedSpace := spacetest.NewSpace(test.HostOperatorNs, "johny",
+			spacetest.WithTierName("advanced"),
+			spacetest.WithSpecTargetCluster("member-cluster"),
+			spacetest.WithSpecTargetClusterRoles([]string{commoncluster.RoleLabel(commoncluster.Tenant)}),
+			spacetest.WithCreatorLabel(userSignup.Name))
+		assert.Equal(t, expectedSpace.Spec, space.Spec)
+		features := space.GetAnnotations()[toolchainv1alpha1.FeatureToggleNameAnnotationKey]
+		for _, feature := range strings.Split(features, ",") {
+			switch feature {
+			case "my-cool-feature":
+				myCoolFeatureCount++
+			case "my-not-so-cool-feature":
+				myNotSoCoolFeatureCount++
+			case "my-so-so-feature":
+				mySoSoFeatureCount++
+			default:
+				assert.Fail(t, "unknown feature", feature)
+			}
+		}
+	}
+
+	// then
+	assert.Equal(t, 100, myCoolFeatureCount)
+	assert.Equal(t, 0, myNotSoCoolFeatureCount)
+	assert.True(t, mySoSoFeatureCount > 0 && mySoSoFeatureCount < 100, "Feature with weight 50 should be present at least once but less than 100 times", mySoSoFeatureCount)
+}
+
 func TestNewSubSpace(t *testing.T) {
 	// given
 	srClusterRoles := []string{commoncluster.RoleLabel(commoncluster.Tenant)}