From d379d579cfb500ba36148e2109e56f2479a3c4f3 Mon Sep 17 00:00:00 2001 From: Denis Arsh Date: Sun, 14 Jul 2024 18:11:06 +0300 Subject: [PATCH] roll application on feature flags change --- api/v1/apptypes.go | 9 +- .../mlops/crds/mlops.cnvrg.io_cnvrgapps.yaml | 4 + .../crd/bases/mlops.cnvrg.io_cnvrgapps.yaml | 4 + controllers/app/controller.go | 126 +++++++++++--- controllers/app/readiness.go | 160 +++++++++++++++--- controllers/app/utils.go | 23 +++ controllers/test/controller_test.go | 56 ++++++ controllers/test/suite_test.go | 4 + controllers/test/suite_utils_test.go | 24 +++ .../tmpl/crds/mlops.cnvrg.io_cnvrgapps.yaml | 4 + .../mlops.cnvrg.io_cnvrgthirdparties.yaml | 3 + pkg/app/controlplane/tmpl/scheduler/dep.tpl | 1 + .../controlplane/tmpl/sidekiqs/searchkiq.tpl | 1 + .../controlplane/tmpl/sidekiqs/sidekiq.tpl | 1 + .../controlplane/tmpl/sidekiqs/systemkiq.tpl | 1 + pkg/app/controlplane/tmpl/webapp/dep.tpl | 1 + 16 files changed, 374 insertions(+), 48 deletions(-) diff --git a/api/v1/apptypes.go b/api/v1/apptypes.go index a364f117..b9a0e15d 100644 --- a/api/v1/apptypes.go +++ b/api/v1/apptypes.go @@ -96,8 +96,9 @@ func DefaultCnvrgAppSpec() CnvrgAppSpec { } type Status struct { - Status OperatorStatus `json:"status,omitempty"` - Message string `json:"message,omitempty"` - Progress int `json:"progress,omitempty"` - StackReadiness map[string]bool `json:"stackReadiness,omitempty"` + Status OperatorStatus `json:"status,omitempty"` + Message string `json:"message,omitempty"` + LastFeatureFlagsHash string `json:"lastFeatureFlagHash"` + Progress int `json:"progress,omitempty"` + StackReadiness map[string]bool `json:"stackReadiness,omitempty"` } diff --git a/charts/mlops/crds/mlops.cnvrg.io_cnvrgapps.yaml b/charts/mlops/crds/mlops.cnvrg.io_cnvrgapps.yaml index 041079e8..45868517 100644 --- a/charts/mlops/crds/mlops.cnvrg.io_cnvrgapps.yaml +++ b/charts/mlops/crds/mlops.cnvrg.io_cnvrgapps.yaml @@ -894,6 +894,8 @@ spec: type: object status: properties: + lastFeatureFlagHash: + type: string message: type: string progress: @@ -904,6 +906,8 @@ spec: type: object status: type: string + required: + - lastFeatureFlagHash type: object type: object served: true diff --git a/config/crd/bases/mlops.cnvrg.io_cnvrgapps.yaml b/config/crd/bases/mlops.cnvrg.io_cnvrgapps.yaml index 041079e8..45868517 100644 --- a/config/crd/bases/mlops.cnvrg.io_cnvrgapps.yaml +++ b/config/crd/bases/mlops.cnvrg.io_cnvrgapps.yaml @@ -894,6 +894,8 @@ spec: type: object status: properties: + lastFeatureFlagHash: + type: string message: type: string progress: @@ -904,6 +906,8 @@ spec: type: object status: type: string + required: + - lastFeatureFlagHash type: object type: object served: true diff --git a/controllers/app/controller.go b/controllers/app/controller.go index e02e8b8e..9bcf88b3 100644 --- a/controllers/app/controller.go +++ b/controllers/app/controller.go @@ -35,6 +35,7 @@ import ( const systemStatusHealthCheckLabelName = "cnvrg-system-status-check" const CnvrgappFinalizer = "cnvrgapp.mlops.cnvrg.io/finalizer" +const RolloutAnnotation = "kubectl.kubernetes.io/restartedAt" type CnvrgAppReconciler struct { client.Client @@ -47,7 +48,6 @@ type CnvrgAppReconciler struct { // +kubebuilder:rbac:groups=mlops.cnvrg.io,namespace=cnvrg,resources=cnvrgapps/status,verbs=get;update;patch func (r *CnvrgAppReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - log = r.Log.WithValues("name", req.NamespacedName) log.Info("starting cnvrgapp reconciliation") @@ -90,20 +90,34 @@ func (r *CnvrgAppReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return ctrl.Result{}, nil } // check if enabled control plane workloads are all in ready status - ready, percentageReady, stackReadiness, err := r.getControlPlaneReadinessStatus(cnvrgApp) + stackReadiness, err := r.getControlPlaneReadinessStatus(cnvrgApp) if err != nil { return ctrl.Result{}, err } // even if all control plane workloads are ready, let operator finish the full reconcile loop - if percentageReady == 100 { - percentageReady = 99 + if stackReadiness.percentageReady == 100 { + stackReadiness.percentageReady = 99 } + + //check are feature flags were updated + var featureFlagsChanged bool + newFeatureFlagsHash := hashStringsMap(cnvrgApp.Spec.ControlPlane.BaseConfig.FeatureFlags) + + lastFeatureFlagsHash := cnvrgApp.Status.LastFeatureFlagsHash + + // feature flags are changed + if newFeatureFlagsHash != lastFeatureFlagsHash && lastFeatureFlagsHash != "" { + featureFlagsChanged = true + } + s := mlopsv1.Status{ - Status: mlopsv1.StatusReconciling, - Message: fmt.Sprintf("reconciling... (%d%%)", percentageReady), - Progress: percentageReady, - StackReadiness: stackReadiness} + Status: mlopsv1.StatusReconciling, + Message: fmt.Sprintf("reconciling... (%d%%)", stackReadiness.percentageReady), + Progress: stackReadiness.percentageReady, + StackReadiness: stackReadiness.readyState, + LastFeatureFlagsHash: newFeatureFlagsHash, + } r.updateStatusMessage(s, cnvrgApp) // apply spec manifests @@ -113,27 +127,82 @@ func (r *CnvrgAppReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c } // get control plan readiness - ready, percentageReady, stackReadiness, err = r.getControlPlaneReadinessStatus(cnvrgApp) + stackReadiness, err = r.getControlPlaneReadinessStatus(cnvrgApp) if err != nil { return ctrl.Result{}, err } - statusMsg := fmt.Sprintf("successfully reconciled, ready (%d%%)", percentageReady) + + var rollingOnFeatureFlagUpdate bool + + //feature flags were changed, therefore need rollout + if featureFlagsChanged { + rollingOnFeatureFlagUpdate = true + + if cnvrgApp.Spec.ControlPlane.WebApp.Enabled { + err := r.RollDeployment(types.NamespacedName{Name: cnvrgApp.Spec.ControlPlane.WebApp.SvcName, Namespace: cnvrgApp.Namespace}) + if err != nil { + return ctrl.Result{}, err + } + } + + if cnvrgApp.Spec.ControlPlane.Sidekiq.Enabled { + err = r.RollDeployment(types.NamespacedName{Name: sidekiq, Namespace: cnvrgApp.Namespace}) + if err != nil { + return ctrl.Result{}, err + } + } + + if cnvrgApp.Spec.ControlPlane.Searchkiq.Enabled { + err = r.RollDeployment(types.NamespacedName{Name: searchkiq, Namespace: cnvrgApp.Namespace}) + if err != nil { + return ctrl.Result{}, err + } + } + + if cnvrgApp.Spec.ControlPlane.Systemkiq.Enabled { + err = r.RollDeployment(types.NamespacedName{Name: systemkiq, Namespace: cnvrgApp.Namespace}) + if err != nil { + return ctrl.Result{}, err + } + } + + if cnvrgApp.Spec.ControlPlane.CnvrgScheduler.Enabled { + err = r.RollDeployment(types.NamespacedName{Name: scheduler, Namespace: cnvrgApp.Namespace}) + if err != nil { + return ctrl.Result{}, err + } + } + } + + statusMsg := fmt.Sprintf("successfully reconciled, ready (%d%%)", stackReadiness.percentageReady) log.Info(statusMsg) - if ready { // u r done, done + if stackReadiness.isReady && !rollingOnFeatureFlagUpdate { // u r done and no need to roll due to feature flags change, done s := mlopsv1.Status{ - Status: mlopsv1.StatusReady, - Message: statusMsg, - Progress: percentageReady, - StackReadiness: stackReadiness} + Status: mlopsv1.StatusReady, + Message: statusMsg, + Progress: stackReadiness.percentageReady, + StackReadiness: stackReadiness.readyState, + LastFeatureFlagsHash: newFeatureFlagsHash, + } r.updateStatusMessage(s, cnvrgApp) log.Info("stack is ready!") r.recorder.Event(cnvrgApp, "Normal", "Created", fmt.Sprintf("cnvrgapp %s successfully deployed", req.NamespacedName)) return ctrl.Result{}, nil } else { // reconcile again requeueAfter, _ := time.ParseDuration("30s") - log.Info("stack not ready yet, requeuing...") - r.recorder.Event(cnvrgApp, "Normal", "Creating", fmt.Sprintf("cnvrgapp %s not ready yet, done: %d%%", req.NamespacedName, percentageReady)) + var logMessage, eventMessage string + if rollingOnFeatureFlagUpdate { + logMessage = "rolling apps due to feature flags change..." + eventMessage = "rolling: feature flags changed" + + } else { + logMessage = "stack not ready yet, requeuing..." + eventMessage = fmt.Sprintf("cnvrgapp %s not ready yet, done: %d%%", req.NamespacedName, stackReadiness.percentageReady) + + } + log.Info(logMessage) + r.recorder.Event(cnvrgApp, "Normal", "Creating", eventMessage) return ctrl.Result{RequeueAfter: requeueAfter}, nil } } @@ -254,6 +323,7 @@ func (r *CnvrgAppReconciler) updateStatusMessage(status mlopsv1.Status, app *mlo if status.StackReadiness != nil { app.Status.StackReadiness = status.StackReadiness } + app.Status.LastFeatureFlagsHash = status.LastFeatureFlagsHash if err := r.Status().Update(context.Background(), app); err != nil { r.recorder.Event(app, "Warning", "StatusUpdateError", err.Error()) } @@ -401,7 +471,6 @@ func (r *CnvrgAppReconciler) SetupWithManager(mgr ctrl.Manager) error { } func (r *CnvrgAppReconciler) appOwnsPredicateFuncs() predicate.Funcs { - return predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { @@ -456,7 +525,6 @@ func (r *CnvrgAppReconciler) CheckDeploymentReadiness(name types.NamespacedName) } func (r *CnvrgAppReconciler) CheckStatefulSetReadiness(name types.NamespacedName) (bool, error) { - ctx := context.Background() sts := &v1apps.StatefulSet{} @@ -472,3 +540,23 @@ func (r *CnvrgAppReconciler) CheckStatefulSetReadiness(name types.NamespacedName return false, nil } + +func (r *CnvrgAppReconciler) RollDeployment(name types.NamespacedName) error { + ctx := context.Background() + deployment := &v1apps.Deployment{} + + if err := r.Get(ctx, name, deployment); err != nil { + return fmt.Errorf("failed to get deployment for rollout %s/%s : %v", name.Namespace, name.Name, err) + } + + if deployment.Spec.Template.ObjectMeta.Annotations == nil { + deployment.Spec.Template.ObjectMeta.Annotations = make(map[string]string) + } + deployment.Spec.Template.ObjectMeta.Annotations[RolloutAnnotation] = time.Now().Format(time.RFC3339) + + if err := r.Client.Update(ctx, deployment); err != nil { + return fmt.Errorf("failed to get update deployment for rollout %s/%s : %v", name.Namespace, name.Name, err) + } + + return nil +} diff --git a/controllers/app/readiness.go b/controllers/app/readiness.go index dc37b1d8..2cf07cd1 100644 --- a/controllers/app/readiness.go +++ b/controllers/app/readiness.go @@ -5,8 +5,27 @@ import ( "k8s.io/apimachinery/pkg/types" ) -func (r *CnvrgAppReconciler) getControlPlaneReadinessStatus(app *mlopsv1.CnvrgApp) (bool, int, map[string]bool, error) { +const ( + webApp = "webapp" + sidekiq = "sidekiq" + searchkiq = "searchkiq" + systemkiq = "systemkiq" + pg = "pg" + minio = "minio" + redis = "redis" + es = "es" + kibana = "kibana" + prometheus = "prometheus" + scheduler = "scheduler" +) + +type StackReadiness struct { + readyState map[string]bool + isReady bool + percentageReady int +} +func (r *CnvrgAppReconciler) getControlPlaneReadinessStatus(app *mlopsv1.CnvrgApp) (*StackReadiness, error) { readyState := make(map[string]bool) // check webapp status @@ -14,39 +33,39 @@ func (r *CnvrgAppReconciler) getControlPlaneReadinessStatus(app *mlopsv1.CnvrgAp name := types.NamespacedName{Name: app.Spec.ControlPlane.WebApp.SvcName, Namespace: app.Namespace} ready, err := r.CheckDeploymentReadiness(name) if err != nil { - return false, 0, nil, err + return nil, err } - readyState["webApp"] = ready + readyState[webApp] = ready } // check sidekiq status if app.Spec.ControlPlane.Sidekiq.Enabled { - name := types.NamespacedName{Name: "sidekiq", Namespace: app.Namespace} + name := types.NamespacedName{Name: sidekiq, Namespace: app.Namespace} ready, err := r.CheckDeploymentReadiness(name) if err != nil { - return false, 0, nil, err + return nil, err } - readyState["sidekiq"] = ready + readyState[sidekiq] = ready } // check searchkiq status if app.Spec.ControlPlane.Searchkiq.Enabled { - name := types.NamespacedName{Name: "searchkiq", Namespace: app.Namespace} + name := types.NamespacedName{Name: searchkiq, Namespace: app.Namespace} ready, err := r.CheckDeploymentReadiness(name) if err != nil { - return false, 0, nil, err + return nil, err } - readyState["searchkiq"] = ready + readyState[searchkiq] = ready } // check systemkiq status if app.Spec.ControlPlane.Systemkiq.Enabled { - name := types.NamespacedName{Name: "systemkiq", Namespace: app.Namespace} + name := types.NamespacedName{Name: systemkiq, Namespace: app.Namespace} ready, err := r.CheckDeploymentReadiness(name) if err != nil { - return false, 0, nil, err + return nil, err } - readyState["systemkiq"] = ready + readyState[systemkiq] = ready } // check postgres status @@ -54,9 +73,9 @@ func (r *CnvrgAppReconciler) getControlPlaneReadinessStatus(app *mlopsv1.CnvrgAp name := types.NamespacedName{Name: app.Spec.Dbs.Pg.SvcName, Namespace: app.Namespace} ready, err := r.CheckDeploymentReadiness(name) if err != nil { - return false, 0, nil, err + return nil, err } - readyState["pg"] = ready + readyState[pg] = ready } // check minio status @@ -64,9 +83,9 @@ func (r *CnvrgAppReconciler) getControlPlaneReadinessStatus(app *mlopsv1.CnvrgAp name := types.NamespacedName{Name: app.Spec.Dbs.Minio.SvcName, Namespace: app.Namespace} ready, err := r.CheckDeploymentReadiness(name) if err != nil { - return false, 0, nil, err + return nil, err } - readyState["minio"] = ready + readyState[minio] = ready } // check redis status @@ -74,9 +93,9 @@ func (r *CnvrgAppReconciler) getControlPlaneReadinessStatus(app *mlopsv1.CnvrgAp name := types.NamespacedName{Name: app.Spec.Dbs.Redis.SvcName, Namespace: app.Namespace} ready, err := r.CheckDeploymentReadiness(name) if err != nil { - return false, 0, nil, err + return nil, err } - readyState["redis"] = ready + readyState[redis] = ready } // check es status @@ -84,9 +103,9 @@ func (r *CnvrgAppReconciler) getControlPlaneReadinessStatus(app *mlopsv1.CnvrgAp name := types.NamespacedName{Name: app.Spec.Dbs.Es.SvcName, Namespace: app.Namespace} ready, err := r.CheckStatefulSetReadiness(name) if err != nil { - return false, 0, nil, err + return nil, err } - readyState["es"] = ready + readyState[es] = ready } // check kibana status @@ -94,9 +113,9 @@ func (r *CnvrgAppReconciler) getControlPlaneReadinessStatus(app *mlopsv1.CnvrgAp name := types.NamespacedName{Name: app.Spec.Dbs.Es.Kibana.SvcName, Namespace: app.Namespace} ready, err := r.CheckDeploymentReadiness(name) if err != nil { - return false, 0, nil, err + return nil, err } - readyState["kibana"] = ready + readyState[kibana] = ready } // check prometheus status @@ -104,9 +123,19 @@ func (r *CnvrgAppReconciler) getControlPlaneReadinessStatus(app *mlopsv1.CnvrgAp name := types.NamespacedName{Name: app.Spec.Dbs.Prom.SvcName, Namespace: app.Namespace} ready, err := r.CheckDeploymentReadiness(name) if err != nil { - return false, 0, nil, err + return nil, err } - readyState["prometheus"] = ready + readyState[prometheus] = ready + } + + // check scheduler status + if app.Spec.ControlPlane.CnvrgScheduler.Enabled { + name := types.NamespacedName{Name: scheduler, Namespace: app.Namespace} + ready, err := r.CheckDeploymentReadiness(name) + if err != nil { + return nil, err + } + readyState[scheduler] = ready } percentageReady := 0 @@ -126,5 +155,86 @@ func (r *CnvrgAppReconciler) getControlPlaneReadinessStatus(app *mlopsv1.CnvrgAp percentageReady = 100 } - return readyCount == len(readyState), percentageReady, readyState, nil + return &StackReadiness{ + isReady: readyCount == len(readyState), + percentageReady: percentageReady, + readyState: readyState, + }, nil +} + +func (s *StackReadiness) webAppReady() bool { + if r, ok := s.readyState[webApp]; ok { + return r + } + return false +} + +func (s *StackReadiness) sidekiqReady() bool { + if r, ok := s.readyState[sidekiq]; ok { + return r + } + return false +} + +func (s *StackReadiness) searchkiqReady() bool { + if r, ok := s.readyState[searchkiq]; ok { + return r + } + return false +} + +func (s *StackReadiness) systemkiqReady() bool { + if r, ok := s.readyState[systemkiq]; ok { + return r + } + return false +} + +func (s *StackReadiness) pgReady() bool { + if r, ok := s.readyState[pg]; ok { + return r + } + return false +} + +func (s *StackReadiness) minioReady() bool { + if r, ok := s.readyState[minio]; ok { + return r + } + return false +} + +func (s *StackReadiness) redisReady() bool { + if r, ok := s.readyState[redis]; ok { + return r + } + return false +} + +func (s *StackReadiness) esReady() bool { + if r, ok := s.readyState[es]; ok { + return r + } + return false +} + +func (s *StackReadiness) kibanaReady() bool { + if r, ok := s.readyState[kibana]; ok { + return r + } + return false +} + +func (s *StackReadiness) prometheusReady() bool { + if r, ok := s.readyState[prometheus]; ok { + return r + } + return false +} + +func (s *StackReadiness) schedulerReady() bool { + if r, ok := s.readyState[scheduler]; ok { + return r + } + return false } diff --git a/controllers/app/utils.go b/controllers/app/utils.go index f6caf24d..142bb98a 100644 --- a/controllers/app/utils.go +++ b/controllers/app/utils.go @@ -2,6 +2,7 @@ package app import ( "context" + "crypto/sha256" "fmt" mlopsv1 "github.com/AccessibleAI/cnvrg-operator/api/v1" "github.com/AccessibleAI/cnvrg-operator/controllers" @@ -197,3 +198,25 @@ func RandomString(length int) string { } return output.String() } + +func hashStringsMap(m map[string]string) string { + h := sha256.New() + + keys := make([]string, len(m)) + i := 0 + for k := range m { + keys[i] = k + i++ + } + sort.Strings(keys) + for _, k := range keys { + v := m[k] + + b := sha256.Sum256([]byte(fmt.Sprintf("%v", k))) + h.Write(b[:]) + b = sha256.Sum256([]byte(fmt.Sprintf("%v", v))) + h.Write(b[:]) + } + + return fmt.Sprintf("%x", h.Sum(nil)) +} diff --git a/controllers/test/controller_test.go b/controllers/test/controller_test.go index c34800bb..7bb5e4ff 100644 --- a/controllers/test/controller_test.go +++ b/controllers/test/controller_test.go @@ -2,9 +2,12 @@ package test import ( cnvrgv1 "github.com/AccessibleAI/cnvrg-operator/api/v1" + "github.com/AccessibleAI/cnvrg-operator/controllers/app" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" v1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) var _ = Describe("Cnvrg App controller", func() { @@ -68,5 +71,58 @@ var _ = Describe("Cnvrg App controller", func() { }) }) + + It("should roll application on feature flags change", func() { + nsName := GenName("ns") + namespace := genNS(nsName) + ExpectCreate(&namespace) + + By("validating relevant components are rolled on feature flag change", func() { + appName := GenName("cnvrg") + appKey := Key(nsName, appName) + obj := genApp(nsName, appName) + obj.Spec.ControlPlane.WebApp.Enabled = true + obj.Spec.ControlPlane.Sidekiq.Enabled = true + obj.Spec.ControlPlane.Systemkiq.Enabled = true + obj.Spec.ControlPlane.Searchkiq.Enabled = true + obj.Spec.ControlPlane.CnvrgScheduler.Enabled = true + + var depObj v1.Deployment + + appDeployKey := Key(nsName, obj.Spec.ControlPlane.WebApp.SvcName) + sidekiqDeployKey := Key(nsName, "sidekiq") + searchkiqDeployKey := Key(nsName, "searchkiq") + systemkiqDeployKey := Key(nsName, "systemkiq") + cnvrgSchedulerDeployKey := Key(nsName, "scheduler") + + By("creating cnvrg app", func() { + ExpectCreate(&obj) + EventuallyGet(appKey, &obj) + }) + + By("changing feature flags", func() { + EventuallyUpdate(types.NamespacedName{Name: appName, Namespace: nsName}, &obj, func(o client.Object) { + obj.Spec.ControlPlane.BaseConfig.FeatureFlags = map[string]string{ + "new-feature": "true", + } + }) + + }) + + By("verify rollout annotations are present", func() { + EventuallyTemplateAnnotationIsPresent(appDeployKey, depObj, app.RolloutAnnotation) // web app is rolled on feature flag change + EventuallyTemplateAnnotationIsPresent(sidekiqDeployKey, depObj, app.RolloutAnnotation) // sidekiq app is rolled on feature flag change + EventuallyTemplateAnnotationIsPresent(systemkiqDeployKey, depObj, app.RolloutAnnotation) // systemkiq app is rolled on feature flag change + EventuallyTemplateAnnotationIsPresent(searchkiqDeployKey, depObj, app.RolloutAnnotation) // searchkiq app is rolled on feature flag change + EventuallyTemplateAnnotationIsPresent(cnvrgSchedulerDeployKey, depObj, app.RolloutAnnotation) // scheduler app is rolled on feature flag change + }) + + By("deleting app", func() { + ExpectDelete(appKey, &obj) + EventuallyGone(appKey, &obj) + }) + }) + + }) }, ) diff --git a/controllers/test/suite_test.go b/controllers/test/suite_test.go index 6648e951..9107b719 100644 --- a/controllers/test/suite_test.go +++ b/controllers/test/suite_test.go @@ -15,6 +15,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager/signals" "testing" "time" @@ -50,6 +52,8 @@ var _ = BeforeSuite(func() { done := make(chan struct{}) go func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + projectBaseDir := "../../" crdPaths := []string{ filepath.Join(projectBaseDir, "charts/mlops/crds/"), diff --git a/controllers/test/suite_utils_test.go b/controllers/test/suite_utils_test.go index 8b97da6b..efb683ed 100644 --- a/controllers/test/suite_utils_test.go +++ b/controllers/test/suite_utils_test.go @@ -7,6 +7,7 @@ import ( . "github.com/onsi/gomega" "github.com/onsi/gomega/gstruct" "github.com/onsi/gomega/types" + v1 "k8s.io/api/apps/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" ktypes "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -57,6 +58,17 @@ func EventuallyGet(key ktypes.NamespacedName, obj client.Object, intervals ...in EventuallyWithOffset(1, GetF(key, obj), intervals...).ShouldNot(BeNil()) } +// EventuallyUpdate helps with updates that tend to race with controller updates. +func EventuallyUpdate(key ktypes.NamespacedName, obj client.Object, modify func(o client.Object), opts ...client.UpdateOption) { + EventuallyWithOffset(1, func() error { + if err := Get(key, obj); err != nil { + return err + } + modify(obj) + return Update(obj, opts...) + }).Should(Succeed()) +} + // GetF returns a function that wraps the k8s client get. // It helps with the use of async matchers. func GetF(key ktypes.NamespacedName, obj client.Object) func() (interface{}, error) { @@ -66,6 +78,13 @@ func GetF(key ktypes.NamespacedName, obj client.Object) func() (interface{}, err } } +func GetTemplateAnnotationsF(key ktypes.NamespacedName, obj v1.Deployment) func() (interface{}, error) { + return func() (interface{}, error) { + err := Get(key, &obj) + return obj.Spec.Template.Annotations, err + } +} + // Get wraps the k8s client get. func Get(key ktypes.NamespacedName, obj client.Object) error { return tc.client.Get(tc.ctx, key, obj) @@ -148,3 +167,8 @@ func EventuallyFinalize(key ktypes.NamespacedName, obj client.Object, intervals func Update(obj client.Object, opts ...client.UpdateOption) error { return tc.client.Update(tc.ctx, obj, opts...) } + +// EventuallyTemplateAnnotationIsPresent helps with the asserting on the annotation filed in deployment template, assert if annotation key is present +func EventuallyTemplateAnnotationIsPresent(key ktypes.NamespacedName, obj v1.Deployment, annotation string, intervals ...interface{}) { + EventuallyWithOffset(1, GetTemplateAnnotationsF(key, obj), intervals...).Should(gomega.HaveKey(annotation)) +} diff --git a/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgapps.yaml b/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgapps.yaml index 1ebc35ff..73c2ec20 100644 --- a/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgapps.yaml +++ b/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgapps.yaml @@ -897,6 +897,8 @@ spec: type: object status: properties: + lastFeatureFlagHash: + type: string message: type: string progress: @@ -907,6 +909,8 @@ spec: type: object status: type: string + required: + - lastFeatureFlagHash type: object type: object served: true diff --git a/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgthirdparties.yaml b/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgthirdparties.yaml index 5b21b657..f153bdaa 100644 --- a/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgthirdparties.yaml +++ b/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgthirdparties.yaml @@ -42,6 +42,9 @@ metadata: mlops.cnvrg.io/default-loader: "true" mlops.cnvrg.io/own: "false" mlops.cnvrg.io/updatable: "true" + mlops.cnvrg.io/default-loader: "true" + mlops.cnvrg.io/own: "false" + mlops.cnvrg.io/updatable: "true" controller-gen.kubebuilder.io/version: v0.9.2 creationTimestamp: null name: cnvrgthirdparties.mlops.cnvrg.io diff --git a/pkg/app/controlplane/tmpl/scheduler/dep.tpl b/pkg/app/controlplane/tmpl/scheduler/dep.tpl index 80b7f0f9..d3f0a02a 100644 --- a/pkg/app/controlplane/tmpl/scheduler/dep.tpl +++ b/pkg/app/controlplane/tmpl/scheduler/dep.tpl @@ -25,6 +25,7 @@ spec: template: metadata: annotations: + cnvrg-component: scheduler {{- range $k, $v := .Spec.Annotations }} {{$k}}: "{{$v}}" {{- end }} diff --git a/pkg/app/controlplane/tmpl/sidekiqs/searchkiq.tpl b/pkg/app/controlplane/tmpl/sidekiqs/searchkiq.tpl index a1af02f4..4a38f827 100644 --- a/pkg/app/controlplane/tmpl/sidekiqs/searchkiq.tpl +++ b/pkg/app/controlplane/tmpl/sidekiqs/searchkiq.tpl @@ -28,6 +28,7 @@ spec: template: metadata: annotations: + cnvrg-component: searchkiq {{- range $k, $v := .Spec.Annotations }} {{$k}}: "{{$v}}" {{- end }} diff --git a/pkg/app/controlplane/tmpl/sidekiqs/sidekiq.tpl b/pkg/app/controlplane/tmpl/sidekiqs/sidekiq.tpl index 9626c5a9..1953e1d2 100644 --- a/pkg/app/controlplane/tmpl/sidekiqs/sidekiq.tpl +++ b/pkg/app/controlplane/tmpl/sidekiqs/sidekiq.tpl @@ -28,6 +28,7 @@ spec: template: metadata: annotations: + cnvrg-component: sidekiq {{- range $k, $v := .Spec.Annotations }} {{$k}}: "{{$v}}" {{- end }} diff --git a/pkg/app/controlplane/tmpl/sidekiqs/systemkiq.tpl b/pkg/app/controlplane/tmpl/sidekiqs/systemkiq.tpl index f60956eb..32454b85 100644 --- a/pkg/app/controlplane/tmpl/sidekiqs/systemkiq.tpl +++ b/pkg/app/controlplane/tmpl/sidekiqs/systemkiq.tpl @@ -28,6 +28,7 @@ spec: template: metadata: annotations: + cnvrg-component: systemkiq {{- range $k, $v := .Spec.Annotations }} {{$k}}: "{{$v}}" {{- end }} diff --git a/pkg/app/controlplane/tmpl/webapp/dep.tpl b/pkg/app/controlplane/tmpl/webapp/dep.tpl index 180652fe..caa8d401 100644 --- a/pkg/app/controlplane/tmpl/webapp/dep.tpl +++ b/pkg/app/controlplane/tmpl/webapp/dep.tpl @@ -33,6 +33,7 @@ spec: template: metadata: annotations: + cnvrg-component: webapp {{- range $k, $v := .Spec.Annotations }} {{$k}}: "{{$v}}" {{- end }}