From 0b4e2feaf0e8dd3d47e47cbd44682224c4c90d11 Mon Sep 17 00:00:00 2001 From: Disaiah Bennett Date: Wed, 1 Nov 2023 09:47:13 -0400 Subject: [PATCH 1/3] added cron job for intervaled reconciling Signed-off-by: Disaiah Bennett --- controllers/backplaneconfig_controller.go | 53 ++++++++++++++++++++++- go.mod | 4 +- go.sum | 16 +++++++ 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/controllers/backplaneconfig_controller.go b/controllers/backplaneconfig_controller.go index d18952fdd..f3b04ba9d 100644 --- a/controllers/backplaneconfig_controller.go +++ b/controllers/backplaneconfig_controller.go @@ -26,6 +26,7 @@ import ( "strings" "time" + "github.com/go-co-op/gocron" backplanev1 "github.com/stolostron/backplane-operator/api/v1" "github.com/stolostron/backplane-operator/pkg/foundation" "github.com/stolostron/backplane-operator/pkg/images" @@ -80,6 +81,12 @@ const ( defaultTrustBundleName = "trusted-ca-bundle" ) +var ( + scheduler *gocron.Scheduler + cronResyncTag = "multiclusterengine-operator-resync" + reconciliationInterval = 10 // minutes +) + //+kubebuilder:rbac:groups=multicluster.openshift.io,resources=multiclusterengines,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=multicluster.openshift.io,resources=multiclusterengines/status,verbs=get;update;patch //+kubebuilder:rbac:groups=multicluster.openshift.io,resources=multiclusterengines/finalizers,verbs=update @@ -143,6 +150,14 @@ const ( // move the current state of the cluster closer to the desired state. func (r *MultiClusterEngineReconciler) Reconcile(ctx context.Context, req ctrl.Request) (retRes ctrl.Result, retErr error) { log := log.FromContext(ctx) + log.Info("Reconciling MultiClusterEngine") + + // Initalize sceduler instance for operator resync cronjob. + if scheduler == nil { + log.Info("Setting up scheduler for operator resync") + r.InitScheduler() + } + // Fetch the BackplaneConfig instance backplaneConfig, err := r.getBackplaneConfig(ctx, req) if err != nil && !apierrors.IsNotFound(err) { @@ -234,7 +249,7 @@ func (r *MultiClusterEngineReconciler) Reconcile(ctx context.Context, req ctrl.R MultiClusterEngine to avoid conflicts with the openshift-* namespace when deploying PrometheusRules and ServiceMonitors in ACM and MCE. */ - result, err = r.ensureOpenShiftNamespaceLabel(ctx, backplaneConfig) + _, err = r.ensureOpenShiftNamespaceLabel(ctx, backplaneConfig) if err != nil { log.Error(err, "Failed to add to %s label to namespace: %s", utils.OpenShiftClusterMonitoringLabel, backplaneConfig.Spec.TargetNamespace) @@ -284,6 +299,11 @@ func (r *MultiClusterEngineReconciler) Reconcile(ctx context.Context, req ctrl.R // Do not reconcile objects if this instance of mce is labeled "paused" if utils.IsPaused(backplaneConfig) { log.Info("MultiClusterEngine reconciliation is paused. Nothing more to do.") + if ok := scheduler.IsRunning(); ok { + log.Info("Pausing MultiClusterEngine operator controller resync job.") + go r.StopScheduleOperatorControllerResync() + } + cond := status.NewCondition( backplanev1.MultiClusterEngineProgressing, metav1.ConditionUnknown, @@ -292,6 +312,8 @@ func (r *MultiClusterEngineReconciler) Reconcile(ctx context.Context, req ctrl.R ) r.StatusManager.AddCondition(cond) return ctrl.Result{}, nil + } else if ok := scheduler.IsRunning(); !ok { + defer r.ScheduleOperatorControllerResync(ctx, req) } result, err = r.DeployAlwaysSubcomponents(ctx, backplaneConfig) @@ -1262,3 +1284,32 @@ func (r *MultiClusterEngineReconciler) getClusterIngressDomain(ctx context.Conte } return clusterIngress.Spec.Domain, nil } + +func (r *MultiClusterEngineReconciler) InitScheduler() { + scheduler = gocron.NewScheduler(time.UTC) +} + +func (r *MultiClusterEngineReconciler) ScheduleOperatorControllerResync(ctx context.Context, req ctrl.Request) { + log := log.FromContext(ctx) + + if ok := scheduler.IsRunning(); !ok { + _, err := scheduler.Tag(cronResyncTag).Every(reconciliationInterval).Minutes().Do(r.Reconcile, ctx, req) + + if err != nil { + log.Error(err, "failed to schedule scheduler job for operator controller resync") + } else { + log.Info(fmt.Sprintf("Starting scheduler job for operator controller. Reconciling every %v minutes", + reconciliationInterval)) + scheduler.StartAsync() + } + } +} + +// StopScheduleOperatorControllerResync ... +func (r *MultiClusterEngineReconciler) StopScheduleOperatorControllerResync() { + scheduler.Stop() + + if ok := scheduler.IsRunning(); !ok { + r.InitScheduler() + } +} diff --git a/go.mod b/go.mod index 1bb892a16..c74e54866 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,7 @@ require ( github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-co-op/gocron v1.35.3 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect @@ -51,7 +52,7 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.4.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.14 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -68,6 +69,7 @@ require ( github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index cbc367361..ea84ab149 100644 --- a/go.sum +++ b/go.sum @@ -65,6 +65,9 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-co-op/gocron v1.35.3 h1:it2WjWnabS8eJZ+P68WroBe+ZWyJ3kVjRD6KXdpr5yI= +github.com/go-co-op/gocron v1.35.3/go.mod h1:3L/n6BkO7ABj+TrfSVXLRzsP26zmikL4ISkLQ0O8iNY= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -135,6 +138,8 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= @@ -151,14 +156,17 @@ github.com/imdario/mergo v0.3.14/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+h github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -188,6 +196,7 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -217,6 +226,7 @@ github.com/operator-framework/api v0.17.4-0.20230223191600-0131a6301e42 h1:d/Pnr github.com/operator-framework/api v0.17.4-0.20230223191600-0131a6301e42/go.mod h1:l/cuwtPxkVUY7fzYgdust2m9tlmb8I4pOvbsUufRb24= github.com/operator-framework/operator-lib v0.11.1-0.20230306195046-28cadc6b6055 h1:G9N8wEf9qDZ/4Fj5cbIejKUoFOYta0v72Yg8tPAdvc0= github.com/operator-framework/operator-lib v0.11.1-0.20230306195046-28cadc6b6055/go.mod h1:A7xcxZPfdepC14FA5YyA+jpbAeD7q4awtT7mSAZntuU= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -233,7 +243,11 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= @@ -274,6 +288,7 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= @@ -444,6 +459,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= From 2ce09e0ef27b0107d8a681bb200718e69857f190 Mon Sep 17 00:00:00 2001 From: Disaiah Bennett Date: Tue, 9 Jan 2024 15:42:12 -0500 Subject: [PATCH 2/3] updated test cases for cronjob Signed-off-by: Disaiah Bennett --- .../backplaneconfig_controller_test.go | 146 ++++++++++-------- 1 file changed, 79 insertions(+), 67 deletions(-) diff --git a/controllers/backplaneconfig_controller_test.go b/controllers/backplaneconfig_controller_test.go index 08fcff2b5..69c7a08b5 100644 --- a/controllers/backplaneconfig_controller_test.go +++ b/controllers/backplaneconfig_controller_test.go @@ -27,12 +27,10 @@ import ( apixv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/errors" - apierrors "k8s.io/apimachinery/pkg/api/errors" configv1 "github.com/openshift/api/config/v1" backplanev1 "github.com/stolostron/backplane-operator/api/v1" - v1 "github.com/stolostron/backplane-operator/api/v1" "github.com/stolostron/backplane-operator/pkg/utils" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -86,15 +84,15 @@ var _ = Describe("BackplaneConfig controller", func() { ) AfterEach(func() { - Expect(k8sClient.Delete(context.Background(), &v1.MultiClusterEngine{ + Expect(k8sClient.Delete(context.Background(), &backplanev1.MultiClusterEngine{ ObjectMeta: metav1.ObjectMeta{ Name: BackplaneConfigName, }, })).To(Succeed()) Eventually(func() bool { - foundMCE := &v1.MultiClusterEngine{} + foundMCE := &backplanev1.MultiClusterEngine{} err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: BackplaneConfigName}, foundMCE) - return apierrors.IsNotFound(err) + return errors.IsNotFound(err) }, timeout, interval).Should(BeTrue()) Expect(k8sClient.Delete(context.Background(), &configv1.ClusterVersion{ ObjectMeta: metav1.ObjectMeta{ @@ -168,7 +166,7 @@ var _ = Describe("BackplaneConfig controller", func() { } Eventually(func() error { err := k8sClient.Create(context.TODO(), testsecret) - if apierrors.IsAlreadyExists(err) { + if errors.IsAlreadyExists(err) { return nil } return err @@ -213,7 +211,7 @@ var _ = Describe("BackplaneConfig controller", func() { { Name: BackplaneConfigTestName, NamespacedName: types.NamespacedName{Name: BackplaneConfigName}, - ResourceType: &v1.MultiClusterEngine{}, + ResourceType: &backplanev1.MultiClusterEngine{}, Expected: nil, }, { @@ -364,7 +362,7 @@ var _ = Describe("BackplaneConfig controller", func() { { Name: BackplaneConfigTestName, NamespacedName: types.NamespacedName{Name: BackplaneConfigName}, - ResourceType: &v1.MultiClusterEngine{}, + ResourceType: &backplanev1.MultiClusterEngine{}, Expected: nil, }, // { @@ -377,11 +375,10 @@ var _ = Describe("BackplaneConfig controller", func() { }) When("creating a new BackplaneConfig", func() { - Context("and no image pull policy is specified", func() { It("should deploy sub components", func() { createCtx := context.Background() - backplaneConfig := &v1.MultiClusterEngine{ + backplaneConfig := &backplanev1.MultiClusterEngine{ TypeMeta: metav1.TypeMeta{ APIVersion: "multicluster.openshift.io/v1", Kind: "MultiClusterEngine", @@ -389,7 +386,7 @@ var _ = Describe("BackplaneConfig controller", func() { ObjectMeta: metav1.ObjectMeta{ Name: BackplaneConfigName, }, - Spec: v1.MultiClusterEngineSpec{ + Spec: backplanev1.MultiClusterEngineSpec{ TargetNamespace: DestinationNamespace, ImagePullSecret: "testsecret", }, @@ -399,7 +396,7 @@ var _ = Describe("BackplaneConfig controller", func() { Expect(k8sClient.Create(createCtx, backplaneConfig)).Should(Succeed()) By("ensuring that no openshift.io/cluster-monitoring label is enabled if MCE does not exist") - backplaneConfig2 := &v1.MultiClusterEngine{ + backplaneConfig2 := &backplanev1.MultiClusterEngine{ TypeMeta: metav1.TypeMeta{ APIVersion: "multicluster.openshift.io/v1", Kind: "MultiClusterEngine", @@ -407,7 +404,7 @@ var _ = Describe("BackplaneConfig controller", func() { ObjectMeta: metav1.ObjectMeta{ Name: BackplaneConfigName, }, - Spec: v1.MultiClusterEngineSpec{ + Spec: backplanev1.MultiClusterEngineSpec{ TargetNamespace: "test-n2", }, } @@ -495,6 +492,21 @@ var _ = Describe("BackplaneConfig controller", func() { res := &corev1.ConfigMap{} g.Expect(k8sClient.Get(ctx, namespacedName, res)).To(Succeed()) }, timeout, interval).Should(Succeed()) + + By("Pausing MCE to pause reconcilation") + Eventually(func() bool { + annotations := backplaneConfig.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + + annotations[utils.AnnotationMCEPause] = "true" + backplaneConfig.Annotations = annotations + _ = k8sClient.Update(ctx, backplaneConfig) + + reconciler.StopScheduleOperatorControllerResync() + return utils.IsPaused(backplaneConfig) && !scheduler.IsRunning() + }, timeout, interval).Should(BeTrue()) }) }) @@ -504,7 +516,7 @@ var _ = Describe("BackplaneConfig controller", func() { defer os.Unsetenv("ACM_HUB_OCP_VERSION") createCtx := context.Background() By("creating the backplane config") - backplaneConfig := &v1.MultiClusterEngine{ + backplaneConfig := &backplanev1.MultiClusterEngine{ TypeMeta: metav1.TypeMeta{ APIVersion: "multicluster.openshift.io/v1", Kind: "MultiClusterEngine", @@ -512,7 +524,7 @@ var _ = Describe("BackplaneConfig controller", func() { ObjectMeta: metav1.ObjectMeta{ Name: BackplaneConfigName, }, - Spec: v1.MultiClusterEngineSpec{ + Spec: backplanev1.MultiClusterEngineSpec{ TargetNamespace: DestinationNamespace, ImagePullSecret: "testsecret", }, @@ -535,7 +547,7 @@ var _ = Describe("BackplaneConfig controller", func() { It("should deploy sub components", func() { createCtx := context.Background() By("creating the backplane config") - backplaneConfig := &v1.MultiClusterEngine{ + backplaneConfig := &backplanev1.MultiClusterEngine{ TypeMeta: metav1.TypeMeta{ APIVersion: "multicluster.openshift.io/v1", Kind: "MultiClusterEngine", @@ -543,37 +555,37 @@ var _ = Describe("BackplaneConfig controller", func() { ObjectMeta: metav1.ObjectMeta{ Name: BackplaneConfigName, }, - Spec: v1.MultiClusterEngineSpec{ + Spec: backplanev1.MultiClusterEngineSpec{ TargetNamespace: DestinationNamespace, ImagePullSecret: "testsecret", - Overrides: &v1.Overrides{ - Components: []v1.ComponentConfig{ + Overrides: &backplanev1.Overrides{ + Components: []backplanev1.ComponentConfig{ { - Name: v1.ConsoleMCE, + Name: backplanev1.ConsoleMCE, Enabled: true, }, { - Name: v1.ServerFoundation, + Name: backplanev1.ServerFoundation, Enabled: true, }, { - Name: v1.HyperShift, + Name: backplanev1.HyperShift, Enabled: true, }, { - Name: v1.Hive, + Name: backplanev1.Hive, Enabled: false, }, { - Name: v1.ClusterManager, + Name: backplanev1.ClusterManager, Enabled: false, }, { - Name: v1.ClusterLifecycle, + Name: backplanev1.ClusterLifecycle, Enabled: false, }, { - Name: v1.ManagedServiceAccount, + Name: backplanev1.ManagedServiceAccount, Enabled: false, }, }, @@ -598,7 +610,7 @@ var _ = Describe("BackplaneConfig controller", func() { Context("and an image pull policy is specified in an override", func() { It("should deploy sub components with the image pull policy in the override", func() { By("creating the backplane config with an image pull policy override") - backplaneConfig := &v1.MultiClusterEngine{ + backplaneConfig := &backplanev1.MultiClusterEngine{ TypeMeta: metav1.TypeMeta{ APIVersion: "multicluster.openshift.io/v1", Kind: "MultiClusterEngine", @@ -606,10 +618,10 @@ var _ = Describe("BackplaneConfig controller", func() { ObjectMeta: metav1.ObjectMeta{ Name: BackplaneConfigName, }, - Spec: v1.MultiClusterEngineSpec{ + Spec: backplanev1.MultiClusterEngineSpec{ TargetNamespace: DestinationNamespace, ImagePullSecret: "testsecret", - Overrides: &v1.Overrides{ + Overrides: &backplanev1.Overrides{ ImagePullPolicy: corev1.PullAlways, }, }, @@ -652,7 +664,7 @@ var _ = Describe("BackplaneConfig controller", func() { Context("and enable ManagedServiceAccount", func() { It("should deploy sub components", func() { By("creating the backplane config") - backplaneConfig := &v1.MultiClusterEngine{ + backplaneConfig := &backplanev1.MultiClusterEngine{ TypeMeta: metav1.TypeMeta{ APIVersion: "multicluster.openshift.io/v1", Kind: "MultiClusterEngine", @@ -660,12 +672,12 @@ var _ = Describe("BackplaneConfig controller", func() { ObjectMeta: metav1.ObjectMeta{ Name: BackplaneConfigName, }, - Spec: v1.MultiClusterEngineSpec{ + Spec: backplanev1.MultiClusterEngineSpec{ TargetNamespace: DestinationNamespace, - Overrides: &v1.Overrides{ - Components: []v1.ComponentConfig{ + Overrides: &backplanev1.Overrides{ + Components: []backplanev1.ComponentConfig{ { - Name: v1.ManagedServiceAccount, + Name: backplanev1.ManagedServiceAccount, Enabled: true, }, }, @@ -753,7 +765,7 @@ var _ = Describe("BackplaneConfig controller", func() { Context("and components are defined multiple times in overrides", func() { It("should deduplicate the component list in the override", func() { By("creating the backplane config with repeated component") - backplaneConfig := &v1.MultiClusterEngine{ + backplaneConfig := &backplanev1.MultiClusterEngine{ TypeMeta: metav1.TypeMeta{ APIVersion: "multicluster.openshift.io/v1", Kind: "MultiClusterEngine", @@ -761,22 +773,22 @@ var _ = Describe("BackplaneConfig controller", func() { ObjectMeta: metav1.ObjectMeta{ Name: BackplaneConfigName, }, - Spec: v1.MultiClusterEngineSpec{ + Spec: backplanev1.MultiClusterEngineSpec{ TargetNamespace: DestinationNamespace, ImagePullSecret: "testsecret", - Overrides: &v1.Overrides{ + Overrides: &backplanev1.Overrides{ ImagePullPolicy: corev1.PullAlways, - Components: []v1.ComponentConfig{ + Components: []backplanev1.ComponentConfig{ { - Name: v1.Discovery, + Name: backplanev1.Discovery, Enabled: true, }, { - Name: v1.Discovery, + Name: backplanev1.Discovery, Enabled: true, }, { - Name: v1.Discovery, + Name: backplanev1.Discovery, Enabled: false, }, }, @@ -791,19 +803,19 @@ var _ = Describe("BackplaneConfig controller", func() { multiClusterEngine := types.NamespacedName{ Name: BackplaneConfigName, } - existingMCE := &v1.MultiClusterEngine{} + existingMCE := &backplanev1.MultiClusterEngine{} g.Expect(k8sClient.Get(context.TODO(), multiClusterEngine, existingMCE)).To(Succeed(), "Failed to create new MCE") g.Expect(existingMCE.Spec.Overrides).To(Not(BeNil())) componentCount := 0 for _, c := range existingMCE.Spec.Overrides.Components { - if c.Name == v1.Discovery { + if c.Name == backplanev1.Discovery { componentCount++ } } g.Expect(componentCount).To(Equal(1), "Duplicate component still present") - g.Expect(existingMCE.Enabled(v1.Discovery)).To(BeFalse(), "Not using last defined config in components") + g.Expect(existingMCE.Enabled(backplanev1.Discovery)).To(BeFalse(), "Not using last defined config in components") }, timeout, interval).Should(Succeed()) @@ -814,7 +826,7 @@ var _ = Describe("BackplaneConfig controller", func() { It("should deploy images with a custom image repository", func() { imageRepo := "quay.io/testrepo" By("creating the backplane config with the image repository annotation") - backplaneConfig := &v1.MultiClusterEngine{ + backplaneConfig := &backplanev1.MultiClusterEngine{ TypeMeta: metav1.TypeMeta{ APIVersion: "multicluster.openshift.io/v1", Kind: "MultiClusterEngine", @@ -825,7 +837,7 @@ var _ = Describe("BackplaneConfig controller", func() { "imageRepository": imageRepo, }, }, - Spec: v1.MultiClusterEngineSpec{ + Spec: backplanev1.MultiClusterEngineSpec{ TargetNamespace: DestinationNamespace, ImagePullSecret: "testsecret", }, @@ -882,7 +894,7 @@ var _ = Describe("BackplaneConfig controller", func() { Expect(k8sClient.Create(context.TODO(), testCM)).To(Succeed()) By("creating the backplane config with the configmap override annotation") - backplaneConfig := &v1.MultiClusterEngine{ + backplaneConfig := &backplanev1.MultiClusterEngine{ TypeMeta: metav1.TypeMeta{ APIVersion: "multicluster.openshift.io/v1", Kind: "MultiClusterEngine", @@ -893,7 +905,7 @@ var _ = Describe("BackplaneConfig controller", func() { "imageOverridesCM": "test", }, }, - Spec: v1.MultiClusterEngineSpec{ + Spec: backplanev1.MultiClusterEngineSpec{ TargetNamespace: DestinationNamespace, ImagePullSecret: "testsecret", }, @@ -918,7 +930,7 @@ var _ = Describe("BackplaneConfig controller", func() { Context("and imagePullSecret is missing", func() { It("should error due to missing secret", func() { By("creating the backplane config with nonexistant secret") - backplaneConfig := &v1.MultiClusterEngine{ + backplaneConfig := &backplanev1.MultiClusterEngine{ TypeMeta: metav1.TypeMeta{ APIVersion: "multicluster.openshift.io/v1", Kind: "MultiClusterEngine", @@ -926,7 +938,7 @@ var _ = Describe("BackplaneConfig controller", func() { ObjectMeta: metav1.ObjectMeta{ Name: BackplaneConfigName, }, - Spec: v1.MultiClusterEngineSpec{ + Spec: backplanev1.MultiClusterEngineSpec{ TargetNamespace: DestinationNamespace, ImagePullSecret: "nonexistant", }, @@ -939,10 +951,10 @@ var _ = Describe("BackplaneConfig controller", func() { multiClusterEngine := types.NamespacedName{ Name: BackplaneConfigName, } - existingMCE := &v1.MultiClusterEngine{} + existingMCE := &backplanev1.MultiClusterEngine{} g.Expect(k8sClient.Get(context.TODO(), multiClusterEngine, existingMCE)).To(Succeed(), "Failed to get MCE") - g.Expect(existingMCE.Status.Phase).To(Equal(v1.MultiClusterEnginePhaseError)) + g.Expect(existingMCE.Status.Phase).To(Equal(backplanev1.MultiClusterEnginePhaseError)) }, timeout, interval).Should(Succeed()) }) @@ -953,7 +965,7 @@ var _ = Describe("BackplaneConfig controller", func() { By("creating the backplane config with nonexistant secret") os.Setenv("ACM_HUB_OCP_VERSION", "4.9.0") defer os.Unsetenv("ACM_HUB_OCP_VERSION") - backplaneConfig := &v1.MultiClusterEngine{ + backplaneConfig := &backplanev1.MultiClusterEngine{ TypeMeta: metav1.TypeMeta{ APIVersion: "multicluster.openshift.io/v1", Kind: "MultiClusterEngine", @@ -961,7 +973,7 @@ var _ = Describe("BackplaneConfig controller", func() { ObjectMeta: metav1.ObjectMeta{ Name: BackplaneConfigName, }, - Spec: v1.MultiClusterEngineSpec{ + Spec: backplanev1.MultiClusterEngineSpec{ TargetNamespace: DestinationNamespace, ImagePullSecret: "testsecret", }, @@ -974,17 +986,17 @@ var _ = Describe("BackplaneConfig controller", func() { multiClusterEngine := types.NamespacedName{ Name: BackplaneConfigName, } - existingMCE := &v1.MultiClusterEngine{} + existingMCE := &backplanev1.MultiClusterEngine{} g.Expect(k8sClient.Get(context.TODO(), multiClusterEngine, existingMCE)).To(Succeed(), "Failed to get MCE") - g.Expect(existingMCE.Status.Phase).To(Equal(v1.MultiClusterEnginePhaseError)) + g.Expect(existingMCE.Status.Phase).To(Equal(backplanev1.MultiClusterEnginePhaseError)) }, timeout, interval).Should(Succeed()) By("ensuring MCE no longer reports error in Phase when annotated") multiClusterEngine := types.NamespacedName{ Name: BackplaneConfigName, } - existingMCE := &v1.MultiClusterEngine{} + existingMCE := &backplanev1.MultiClusterEngine{} Expect(k8sClient.Get(context.TODO(), multiClusterEngine, existingMCE)).To(Succeed(), "Failed to get MCE") existingMCE.SetAnnotations(map[string]string{utils.AnnotationIgnoreOCPVersion: "true"}) Expect(k8sClient.Update(context.TODO(), existingMCE)).To(Succeed(), "Failed to get MCE") @@ -993,10 +1005,10 @@ var _ = Describe("BackplaneConfig controller", func() { multiClusterEngine := types.NamespacedName{ Name: BackplaneConfigName, } - existingMCE := &v1.MultiClusterEngine{} + existingMCE := &backplanev1.MultiClusterEngine{} g.Expect(k8sClient.Get(context.TODO(), multiClusterEngine, existingMCE)).To(Succeed(), "Failed to get MCE") - g.Expect(existingMCE.Status.Phase).To(Not(Equal(v1.MultiClusterEnginePhaseError))) + g.Expect(existingMCE.Status.Phase).To(Not(Equal(backplanev1.MultiClusterEnginePhaseError))) }, timeout, interval).Should(Succeed()) }) }) @@ -1016,22 +1028,22 @@ var _ = Describe("BackplaneConfig controller", func() { } Eventually(func() error { err := k8sClient.Create(context.TODO(), testconfigsecret) - if apierrors.IsAlreadyExists(err) { + if errors.IsAlreadyExists(err) { return nil } return err }, timeout, interval).Should(Succeed()) - backplaneConfig := &v1.MultiClusterEngine{ + backplaneConfig := &backplanev1.MultiClusterEngine{ TypeMeta: metav1.TypeMeta{ APIVersion: "multicluster.openshift.io/v1", Kind: "MultiClusterEngine", }, ObjectMeta: metav1.ObjectMeta{ Name: BackplaneConfigName, - Annotations: map[string]string{"deploymentmode": string(v1.ModeHosted), "mce-kubeconfig": "test"}, + Annotations: map[string]string{"deploymentmode": string(backplanev1.ModeHosted), "mce-kubeconfig": "test"}, }, - Spec: v1.MultiClusterEngineSpec{ + Spec: backplanev1.MultiClusterEngineSpec{ TargetNamespace: DestinationNamespace, ImagePullSecret: "testsecret", }, @@ -1044,10 +1056,10 @@ var _ = Describe("BackplaneConfig controller", func() { multiClusterEngine := types.NamespacedName{ Name: BackplaneConfigName, } - existingMCE := &v1.MultiClusterEngine{} + existingMCE := &backplanev1.MultiClusterEngine{} g.Expect(k8sClient.Get(context.TODO(), multiClusterEngine, existingMCE)).To(Succeed(), "Failed to get MCE") - g.Expect(existingMCE.Status.Phase).To(Equal(v1.MultiClusterEnginePhaseError), "MCE should fail getting a kubeconfig secret") + g.Expect(existingMCE.Status.Phase).To(Equal(backplanev1.MultiClusterEnginePhaseError), "MCE should fail getting a kubeconfig secret") }, timeout, interval).Should(Succeed()) }) @@ -1055,7 +1067,7 @@ var _ = Describe("BackplaneConfig controller", func() { Context("Legacy clean up tasks", func() { It("Removes the legacy CLC Prometheus configuration", func() { By("creating the backplane config with nonexistant secret") - backplaneConfig := &v1.MultiClusterEngine{ + backplaneConfig := &backplanev1.MultiClusterEngine{ TypeMeta: metav1.TypeMeta{ APIVersion: "multicluster.openshift.io/v1", Kind: "MultiClusterEngine", @@ -1063,7 +1075,7 @@ var _ = Describe("BackplaneConfig controller", func() { ObjectMeta: metav1.ObjectMeta{ Name: BackplaneConfigName, }, - Spec: v1.MultiClusterEngineSpec{ + Spec: backplanev1.MultiClusterEngineSpec{ TargetNamespace: DestinationNamespace, ImagePullSecret: "nonexistant", }, From 24060da5016037e4da7f042cd6300431a728272c Mon Sep 17 00:00:00 2001 From: Disaiah Bennett Date: Tue, 9 Jan 2024 17:29:11 -0500 Subject: [PATCH 3/3] updated logging Signed-off-by: Disaiah Bennett --- controllers/backplaneconfig_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/backplaneconfig_controller.go b/controllers/backplaneconfig_controller.go index 89336f054..5d5717fd4 100644 --- a/controllers/backplaneconfig_controller.go +++ b/controllers/backplaneconfig_controller.go @@ -1439,7 +1439,7 @@ func (r *MultiClusterEngineReconciler) InitScheduler() { } func (r *MultiClusterEngineReconciler) ScheduleOperatorControllerResync(ctx context.Context, req ctrl.Request) { - log := log.FromContext(ctx) + log := log.Log.WithName("reconcile") if ok := scheduler.IsRunning(); !ok { _, err := scheduler.Tag(cronResyncTag).Every(reconciliationInterval).Minutes().Do(r.Reconcile, ctx, req)