wip

cmd/operator/main.go

@@ -8,6 +8,7 @@ import (
 	"github.com/spf13/viper"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"os"
@@ -110,26 +111,20 @@ func setParams(params []param, command *cobra.Command) {
 
 func runOperator() {
 	ctrl.SetLogger(zapr.NewLogger(initZapLog()))
-
-	namespaces := []string{"cnvrg"} // List of Namespaces
-	defaultNamespaces := make(map[string]cache.Config)
-
-	for _, ns := range namespaces {
-		defaultNamespaces[ns] = cache.Config{}
+	selector, err := labels.Parse("name=cnvrg")
+	if err != nil {
+		zap.S().Error(err)
+		return
 	}
+	cacheCfg := cache.Config{LabelSelector: selector}
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme: scheme,
 		Cache: cache.Options{
-			DefaultNamespaces: defaultNamespaces,
-			//DefaultNamespaces: map[string]cache.Config{
-			//	"namespace": {},
-			//},
+			DefaultNamespaces: map[string]cache.Config{"cnvrg": cacheCfg},
 		},
 		Metrics:                metricsserver.Options{BindAddress: viper.GetString("metrics-addr")},
 		HealthProbeBindAddress: viper.GetString("health-probe-addr"),
-		LeaderElection:         viper.GetBool("enable-leader-election"),
-		LeaderElectionID:       "99748453.cnvrg.io",
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")

controllers/app/controller.go

@@ -11,7 +11,6 @@ import (
 	"github.com/AccessibleAI/cnvrg-operator/pkg/app/networking"
 	"github.com/AccessibleAI/cnvrg-operator/pkg/app/registry"
 	sso2 "github.com/AccessibleAI/cnvrg-operator/pkg/app/sso"
-	"github.com/AccessibleAI/cnvrg-operator/pkg/desired"
 	"github.com/go-logr/logr"
 	"github.com/spf13/viper"
 	"gopkg.in/d4l3k/messagediff.v1"
@@ -20,7 +19,6 @@ import (
 	v1core "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
@@ -32,10 +30,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
-	"strings"
 	"time"
 )
 
+const systemStatusHealthCheckLabelName = "cnvrg-system-status-check"
 const CnvrgappFinalizer = "cnvrgapp.mlops.cnvrg.io/finalizer"
 
 type CnvrgAppReconciler struct {
@@ -389,11 +387,6 @@ func (r *CnvrgAppReconciler) cleanupDbInitCm(desiredSpec *mlopsv1.CnvrgApp) erro
 }
 
 func (r *CnvrgAppReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	if viper.GetBool("create-crds") {
-		if err := controlplane.NewControlPlaneCrdsStateManager(r.Client, r.Scheme, r.Log).Apply(); err != nil {
-			return err
-		}
-	}
 
 	appPredicate := predicate.Funcs{
 
@@ -421,28 +414,34 @@ func (r *CnvrgAppReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		},
 	}
 
-	appOwnsPredicate := predicate.Funcs{
+	appOwnsPredicate := r.appOwnsPredicateFuncs()
+
+	r.recorder = mgr.GetEventRecorderFor("cnvrgapp")
+	a := &v1apps.Deployment{}
+	a.GroupVersionKind()
+	cnvrgAppController := ctrl.
+		NewControllerManagedBy(mgr).
+		Owns(&v1apps.Deployment{}, builder.WithPredicates(appOwnsPredicate)).
+		Owns(&v1apps.StatefulSet{}, builder.WithPredicates(appOwnsPredicate)).
+		For(&mlopsv1.CnvrgApp{}, builder.WithPredicates(appPredicate))
+
+	r.Log.Info(fmt.Sprintf("max concurrent reconciles: %d", viper.GetInt("max-concurrent-reconciles")))
+
+	return cnvrgAppController.
+		WithOptions(controller.Options{MaxConcurrentReconciles: viper.GetInt("max-concurrent-reconciles")}).
+		Complete(r)
+}
+
+func (r *CnvrgAppReconciler) appOwnsPredicateFuncs() predicate.Funcs {
+
+	return predicate.Funcs{
 
 		UpdateFunc: func(e event.UpdateEvent) bool {
-			gvk := e.ObjectNew.GetObjectKind().GroupVersionKind()
-			app := mlopsv1.DefaultCnvrgAppSpec()
-			healthCheckWorkloads := []string{
-				"ingresscheck",
-				"sidekiq",
-				"searchkiq",
-				"systemkiq",
-				app.ControlPlane.WebApp.SvcName,
-				app.Dbs.Pg.SvcName,
-				app.Dbs.Minio.SvcName,
-				app.Dbs.Redis.SvcName,
-				app.Dbs.Es.SvcName,
-			}
 
-			if gvk == desired.Kinds[desired.DeploymentGVK] || gvk == desired.Kinds[desired.StatefulSetGVK] || gvk == desired.Kinds[desired.JobGVK] {
-				if controllers.ContainsString(healthCheckWorkloads, e.ObjectNew.GetName()) {
-					return true
-				}
+			if controllers.ContainsString(labelsMapToList(e.ObjectNew.GetLabels()), systemStatusHealthCheckLabelName) {
+				return true
 			}
+
 			r.Log.V(1).Info("received update event", "objectName", e.ObjectNew.GetName())
 			return false
 		},
@@ -452,31 +451,6 @@ func (r *CnvrgAppReconciler) SetupWithManager(mgr ctrl.Manager) error {
 			return true
 		},
 	}
-	r.recorder = mgr.GetEventRecorderFor("cnvrgapp")
-	cnvrgAppController := ctrl.
-		NewControllerManagedBy(mgr).
-		For(&mlopsv1.CnvrgApp{}, builder.WithPredicates(appPredicate))
-
-	for _, v := range desired.Kinds {
-
-		if strings.Contains(v.Group, "istio.io") {
-			continue
-		}
-		if strings.Contains(v.Group, "openshift.io") {
-			continue
-		}
-		if strings.Contains(v.Group, "coreos.com") {
-			continue
-		}
-		u := &unstructured.Unstructured{}
-		u.SetGroupVersionKind(v)
-		cnvrgAppController.Owns(u, builder.WithPredicates(appOwnsPredicate))
-	}
-
-	r.Log.Info(fmt.Sprintf("max concurrent reconciles: %d", viper.GetInt("max-concurrent-reconciles")))
-	return cnvrgAppController.
-		WithOptions(controller.Options{MaxConcurrentReconciles: viper.GetInt("max-concurrent-reconciles")}).
-		Complete(r)
 }
 
 func (r *CnvrgAppReconciler) CheckJobReadiness(name types.NamespacedName) (bool, error) {

controllers/app/utils.go

@@ -21,10 +21,6 @@ import (
 
 var log logr.Logger
 
-func discoverAICloudHost(clientset client.Client) {
-
-}
-
 func discoverOcpDefaultRouteHost(clientset client.Client) (ocpDefaultRouteHost string, err error) {
 	routeCfg := &unstructured.Unstructured{}
 	routeCfg.SetGroupVersionKind(desired.Kinds["OcpIngressCfgGVK"])
@@ -168,3 +164,10 @@ func CalculateAndApplyAppDefaults(app *mlopsv1.CnvrgApp, defaultSpec *mlopsv1.Cn
 
 	return nil
 }
+
+func labelsMapToList(labels map[string]string) (labelList []string) {
+	for labelName, _ := range labels {
+		labelList = append(labelList, labelName)
+	}
+	return labelList
+}

go.mod

@@ -20,7 +20,7 @@ require (
 	k8s.io/api v0.29.0
 	k8s.io/apimachinery v0.29.0
 	k8s.io/client-go v0.29.0
-	sigs.k8s.io/controller-runtime v0.16.3
+	sigs.k8s.io/controller-runtime v0.17.0
 )
 
 require (
@@ -77,7 +77,7 @@ require (
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.1 // indirect
 	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
-	github.com/evanphx/json-patch/v5 v5.7.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.8.0 // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect

go.sum

@@ -165,6 +165,8 @@ github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ
 github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc=
 github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
+github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro=
+github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
 github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4=
 github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
@@ -403,8 +405,10 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
 github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
+github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY=
 github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
 github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
+github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI=
@@ -735,6 +739,8 @@ oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY=
 oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324=
 sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4=
 sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0=
+sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s=
+sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s=
 sigs.k8s.io/gateway-api v0.6.0 h1:v2FqrN2ROWZLrSnI2o91taHR8Sj3s+Eh3QU7gLNWIqA=
 sigs.k8s.io/gateway-api v0.6.0/go.mod h1:EYJT+jlPWTeNskjV0JTki/03WX1cyAnBhwBJfYHpV/0=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=

pkg/app/controlplane/tmpl/sidekiqs/searchkiq.tpl

@@ -14,6 +14,7 @@ metadata:
     app: searchkiq
     owner: cnvrg-control-plane
     cnvrg-component: searchkiq
+    cnvrg-system-status-check: "true"
     {{- range $k, $v := .Spec.Labels }}
     {{$k}}: "{{$v}}"
     {{- end }}

pkg/app/controlplane/tmpl/sidekiqs/sidekiq.tpl

@@ -14,6 +14,7 @@ metadata:
     app: sidekiq
     owner: cnvrg-control-plane
     cnvrg-component: sidekiq
+    cnvrg-system-status-check: "true"
     {{- range $k, $v := .Spec.Labels }}
     {{$k}}: "{{$v}}"
     {{- end }}

pkg/app/controlplane/tmpl/sidekiqs/systemkiq.tpl

@@ -14,6 +14,7 @@ metadata:
     app: systemkiq
     owner: cnvrg-control-plane
     cnvrg-component: systemkiq
+    cnvrg-system-status-check: "true"
     {{- range $k, $v := .Spec.Labels }}
     {{$k}}: "{{$v}}"
     {{- end }}

pkg/app/controlplane/tmpl/webapp/dep.tpl

@@ -14,6 +14,7 @@ metadata:
     app: {{ .Spec.ControlPlane.WebApp.SvcName }}
     owner: cnvrg-control-plane
     cnvrg-component: webapp
+    cnvrg-system-status-check: "true"
     {{- range $k, $v := .Spec.Labels }}
     {{$k}}: "{{$v}}"
     {{- end }}

pkg/app/dbs/tmpl/es/elastic/sts.tpl

@@ -18,6 +18,7 @@ metadata:
     {{- end }}
   labels:
     app: {{ .Spec.Dbs.Es.SvcName }}
+    cnvrg-system-status-check: "true"
     {{- range $k, $v := .Spec.Labels }}
     {{$k}}: "{{$v}}"
     {{- end }}

pkg/app/dbs/tmpl/minio/dep.tpl

@@ -13,6 +13,7 @@ metadata:
   labels:
     app: {{ .Spec.Dbs.Minio.SvcName }}
     cnvrg-component: minio
+    cnvrg-system-status-check: "true"
     {{- range $k, $v := .Spec.Labels }}
     {{$k}}: "{{$v}}"
     {{- end }}

pkg/app/dbs/tmpl/pg/dep.tpl

@@ -13,6 +13,7 @@ metadata:
   labels:
     app: {{.Spec.Dbs.Pg.SvcName }}
     cnvrg-component: pg
+    cnvrg-system-status-check: "true"
     {{- range $k, $v := .Spec.Labels }}
     {{$k}}: "{{$v}}"
     {{- end }}

pkg/app/dbs/tmpl/redis/dep.tpl

@@ -13,6 +13,7 @@ metadata:
   labels:
     app: {{.Spec.Dbs.Redis.SvcName }}
     cnvrg-component: redis
+    cnvrg-system-status-check: "true"
     {{- range $k, $v := .Spec.Labels }}
     {{$k}}: "{{$v}}"
     {{- end }}