Devops 1705 mergemaster

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.22.6 as builder
+FROM golang:1.23.0 as builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests

README.md

@@ -21,4 +21,49 @@ helm install cnvrg cnvrg/mlops \
  --set registry.user="<CNVRG-USERNAME>" \
  --set registry.password="<CNVRG-PASSWORD>" \
  --set controlPlane.baseConfig.agentCustomTag="<AGENT-CUSTOM-TAG>"
-```
+```
+
+## Using external secret for SMTP server
+It's an option to specify external secret for SMTP server credintials instead setting it in helm chart values or cnvrgapp CRD . 
+The parameter to reference the secret is `controlPlane.smtp.CredentialsSecretRef` and the keys in the secret should be `username` and `password`.
+
+```bash
+helm install cnvrg cnvrg/mlops \
+ --create-namespace -n cnvrg \
+ --set controlPlane.smtp.credentialsSecretRef="SECRET-NAME"
+```
+secret example
+```bash
+apiVersion: v1
+kind: Secret
+metadata:
+  name: SECRET-NAME
+  namespace: cnvrg
+type: Opaque
+data:
+  username: YWRtaW4=
+  password: c2VjcmV0
+```
+
+## Using external secret for OAuth2 client configuration
+
+It's an option to specify external secret for OAuth2 client configuration instead setting it in helm chart values or cnvrgapp CRD. The parameter to reference the secret is `sso.central.credentialsSecretRef` and the keys in the secret should be `clientId`, `clientSecret`
+
+```bash
+helm install cnvrg cnvrg/mlops \
+ --create-namespace -n cnvrg \
+ --set sso.central.credentialsSecretRef="SECRET-NAME"
+```
+
+secret example
+```bash
+apiVersion: v1
+kind: Secret
+metadata:
+  name: SECRET-NAME
+  namespace: cnvrg
+type: Opaque
+data:
+  clientId: YWRtaW4=
+  clientSecret: c2VjcmV0
+```

api/v1/app.go

@@ -122,13 +122,14 @@ type Ldap struct {
 }
 
 type SMTP struct {
-	Server            string `json:"server,omitempty"`
-	Port              int    `json:"port,omitempty"`
-	Username          string `json:"username,omitempty"`
-	Password          string `json:"password,omitempty"`
-	Domain            string `json:"domain,omitempty"`
-	OpensslVerifyMode string `json:"opensslVerifyMode,omitempty"`
-	Sender            string `json:"sender,omitempty"`
+	Server               string `json:"server,omitempty"`
+	Port                 int    `json:"port,omitempty"`
+	CredentialsSecretRef string `json:"credentialsSecretRef,omitempty"`
+	Username             string `json:"username,omitempty"`
+	Password             string `json:"password,omitempty"`
+	Domain               string `json:"domain,omitempty"`
+	OpensslVerifyMode    string `json:"opensslVerifyMode,omitempty"`
+	Sender               string `json:"sender,omitempty"`
 }
 
 type ObjectStorage struct {
@@ -407,6 +408,7 @@ type CentralSSO struct {
 	EmailDomain                      []string          `json:"emailDomain,omitempty"`
 	ClientID                         string            `json:"clientId,omitempty"`
 	ClientSecret                     string            `json:"clientSecret,omitempty"`
+	CredentialsSecretRef             string            `json:"credentialsSecretRef,omitempty"`
 	OidcIssuerURL                    string            `json:"oidcIssuerUrl,omitempty"`
 	ServiceUrl                       string            `json:"serviceUrl,omitempty"`
 	Scope                            string            `json:"scope,omitempty"`

charts/cvatdbs/values.yaml

@@ -20,12 +20,12 @@ pg:
       cpu: 200m
       memory: 1Gi
 redis:
-  image: redis:7.0.5
+  image: redis:7.4.0
   confRef: cvat-redis
   resources:
     limits:
       cpu: 1000m
       memory: 2Gi
     requests:
       cpu: 100m
-      memory: 200Mi
+      memory: 200Mi

charts/mlops/crds/mlops.cnvrg.io_cnvrgapps.yaml

@@ -267,6 +267,8 @@ spec:
                     type: object
                   smtp:
                     properties:
+                      credentialsSecretRef:
+                        type: string
                       domain:
                         type: string
                       opensslVerifyMode:
@@ -768,6 +770,8 @@ spec:
                         type: string
                       cookieDomain:
                         type: string
+                      credentialsSecretRef:
+                        type: string
                       emailDomain:
                         items:
                           type: string

charts/mlops/templates/cap.yml

@@ -138,6 +138,7 @@ spec:
       port: {{.Values.controlPlane.smtp.port}}
       username: {{.Values.controlPlane.smtp.username}}
       password: {{.Values.controlPlane.smtp.password}}
+      credentialsSecretRef: {{.Values.controlPlane.smtp.credentialsSecretRef}}
       domain: {{.Values.controlPlane.smtp.domain}}
       opensslVerifyMode: {{.Values.controlPlane.smtp.opensslVerifyMode}}
       sender: {{.Values.controlPlane.smtp.sender}}
@@ -338,6 +339,7 @@ spec:
       emailDomain: {{ toJson .Values.sso.central.emailDomain }}
       clientId: {{.Values.sso.central.clientId}}
       clientSecret: {{.Values.sso.central.clientSecret}}
+      credentialsSecretRef: {{.Values.sso.central.credentialsSecretRef}}
       oidcIssuerUrl: {{.Values.sso.central.oidcIssuerUrl}}
       serviceUrl: {{.Values.sso.central.serviceUrl}}
       scope: {{.Values.sso.central.scope}}

charts/mlops/values.yaml

@@ -72,7 +72,7 @@ controlPlane:
       maxReplicas: 5
   hyper:
     enabled: true
-    image: hyper-server:latest
+    image: hyper-server:v2.0
     port: 5050
     replicas: 1
     nodePort: 30050
@@ -128,6 +128,7 @@ controlPlane:
     domain: ''
     opensslVerifyMode: ''
     sender: [email protected]
+    credentialsSecretRef: ''
   objectStorage:
     type: minio
     bucket: cnvrg-storage
@@ -148,7 +149,7 @@ dbs:
   pg:
     enabled: true
     serviceAccount: pg
-    image: postgresql-12-centos7:latest
+    image: postgresql-12-centos7:pg13
     port: 5432
     storageSize: 80Gi
     svcName: postgres
@@ -172,7 +173,7 @@ dbs:
   redis:
     enabled: true
     serviceAccount: redis
-    image: cnvrg-redis:v7.0.0
+    image: redis:7.4.0
     svcName: redis
     port: 6379
     storageSize: 10Gi
@@ -190,7 +191,7 @@ dbs:
     enabled: true
     serviceAccount: minio
     replicas: 1
-    image: minio:RELEASE.2021-05-22T02-34-39Z
+    image: minio:RELEASE.2024-09-13T20-26-02Z
     port: 9000
     storageSize: 100Gi
     svcName: minio
@@ -244,8 +245,8 @@ dbs:
       credsRef: kibana-creds
     elastalert:
       enabled: true
-      image: elastalert:3.0.0-beta.1
-      authProxyImage: nginx:1.20
+      image: elastalert:v5.1
+      authProxyImage: nginx:1.27.1
       credsRef: elastalert-creds
       port: 8080
       nodePort: 32030
@@ -264,12 +265,12 @@ dbs:
     enabled: true
     credsRef: prom-creds
     extraScrapeConfigs:
-    image: prometheus:v2.37.1
+    image: prometheus:v2.45.0-stable
     storageClass: ""
     storageSize: 50Gi
     grafana:
       enabled: true
-      image: grafana/grafana-oss:9.5.20
+      image: grafana/grafana-oss:11.2.0
       svcName: grafana
       port: 8080
       nodePort: 30012
@@ -311,7 +312,7 @@ sso:
     enabled: false
     name: cnvrg-jwks
     image: jwks:ns-watch-scope
-    cacheImage: redis:7.0.5
+    cacheImage: redis:7.4.0
   central:
     enabled: false
     publicUrl: ''
@@ -323,6 +324,7 @@ sso:
       - "*"
     clientId: ''
     clientSecret: ''
+    credentialsSecretRef: ''
     oidcIssuerUrl: ''
     serviceUrl: ''
     scope: openid email profile
@@ -363,4 +365,3 @@ mpi:
 metastorageprovisioner:
   enabled: true
   version: ''
-

charts/mpi/values.yaml

@@ -1,6 +1,6 @@
 imageHub: docker.io
-image: mpioperator/mpi-operator:v0.2.3
-kubectlDeliveryImage: mpioperator/kubectl-delivery:v0.2.3
+image: mpioperator/mpi-operator:v0.5
+kubectlDeliveryImage: mpioperator/kubectl-delivery:v0.5
 imagePullSecretRef: cnvrg-app-registry
 scc: false
 registry:

charts/nomex/values.yaml

@@ -1,3 +1,3 @@
 imageHub: docker.io/cnvrg
-image: nomex:v1.0.0
-imagePullSecretRef: cnvrg-app-registry
+image: nomex:v2.0.0
+imagePullSecretRef: cnvrg-app-registry

charts/optionals/helmfile.yaml

@@ -1,21 +1,21 @@
 releases:
   - name: istio-base
     chart: istio/base
-    version: 1.21.2
+    version: 1.23.1
     namespace: istio-system
     labels:
       stage: istio-infra
 
   - name: istiod
     chart: istio/istiod
-    version: 1.21.2
+    version: 1.23.1
     namespace: istio-system
     labels:
       stage: istio-infra
 
   - name: istio-ingress
     chart: istio/gateway
-    version: 1.21.2
+    version: 1.23.1
     namespace: istio-system
     labels:
       stage: istio-gateway

config/crd/bases/mlops.cnvrg.io_cnvrgapps.yaml

@@ -267,6 +267,8 @@ spec:
                     type: object
                   smtp:
                     properties:
+                      credentialsSecretRef:
+                        type: string
                       domain:
                         type: string
                       opensslVerifyMode:
@@ -768,6 +770,8 @@ spec:
                         type: string
                       cookieDomain:
                         type: string
+                      credentialsSecretRef:
+                        type: string
                       emailDomain:
                         items:
                           type: string

go.mod

@@ -1,8 +1,6 @@
 module github.com/AccessibleAI/cnvrg-operator
 
-go 1.22.0
-
-toolchain go1.22.2
+go 1.23.0
 
 require (
 	dario.cat/mergo v1.0.0

go.sum

@@ -107,8 +107,6 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI=
 github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro=
-github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
 github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
@@ -329,12 +327,11 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY=
-github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM=
+github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8=
+github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE=
-github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk=
+github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk=
 github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
@@ -615,30 +612,18 @@ helm.sh/helm/v3 v3.15.3 h1:HcZDaVFe9uHa6hpsR54mJjYyRy4uz/pc6csg27nxFOc=
 helm.sh/helm/v3 v3.15.3/go.mod h1:FzSIP8jDQaa6WAVg9F+OkKz7J0ZmAga4MABtTbsb9WQ=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/api v0.30.0 h1:siWhRq7cNjy2iHssOB9SCGNCl2spiF1dO3dABqZ8niA=
-k8s.io/api v0.30.0/go.mod h1:OPlaYhoHs8EQ1ql0R/TsUgaRPhpKNxIMrKQfWUp8QSE=
 k8s.io/api v0.30.1 h1:kCm/6mADMdbAxmIh0LBjS54nQBE+U4KmbCfIkF5CpJY=
 k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM=
-k8s.io/apiextensions-apiserver v0.30.0 h1:jcZFKMqnICJfRxTgnC4E+Hpcq8UEhT8B2lhBcQ+6uAs=
-k8s.io/apiextensions-apiserver v0.30.0/go.mod h1:N9ogQFGcrbWqAY9p2mUAL5mGxsLqwgtUce127VtRX5Y=
 k8s.io/apiextensions-apiserver v0.30.1 h1:4fAJZ9985BmpJG6PkoxVRpXv9vmPUOVzl614xarePws=
 k8s.io/apiextensions-apiserver v0.30.1/go.mod h1:R4GuSrlhgq43oRY9sF2IToFh7PVlF1JjfWdoG3pixk4=
-k8s.io/apimachinery v0.30.0 h1:qxVPsyDM5XS96NIh9Oj6LavoVFYff/Pon9cZeDIkHHA=
-k8s.io/apimachinery v0.30.0/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
 k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U=
 k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
-k8s.io/apiserver v0.30.0 h1:QCec+U72tMQ+9tR6A0sMBB5Vh6ImCEkoKkTDRABWq6M=
-k8s.io/apiserver v0.30.0/go.mod h1:smOIBq8t0MbKZi7O7SyIpjPsiKJ8qa+llcFCluKyqiY=
 k8s.io/apiserver v0.30.1 h1:BEWEe8bzS12nMtDKXzCF5Q5ovp6LjjYkSp8qOPk8LZ8=
 k8s.io/apiserver v0.30.1/go.mod h1:i87ZnQ+/PGAmSbD/iEKM68bm1D5reX8fO4Ito4B01mo=
 k8s.io/cli-runtime v0.30.0 h1:0vn6/XhOvn1RJ2KJOC6IRR2CGqrpT6QQF4+8pYpWQ48=
 k8s.io/cli-runtime v0.30.0/go.mod h1:vATpDMATVTMA79sZ0YUCzlMelf6rUjoBzlp+RnoM+cg=
-k8s.io/client-go v0.30.0 h1:sB1AGGlhY/o7KCyCEQ0bPWzYDL0pwOZO4vAtTSh/gJQ=
-k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY=
 k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q=
 k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc=
-k8s.io/component-base v0.30.0 h1:cj6bp38g0ainlfYtaOQuRELh5KSYjhKxM+io7AUIk4o=
-k8s.io/component-base v0.30.0/go.mod h1:V9x/0ePFNaKeKYA3bOvIbrNoluTSG+fSJKjLdjOoeXQ=
 k8s.io/component-base v0.30.1 h1:bvAtlPh1UrdaZL20D9+sWxsJljMi0QZ3Lmw+kmZAaxQ=
 k8s.io/component-base v0.30.1/go.mod h1:e/X9kDiOebwlI41AvBHuWdqFriSRrX50CdwA9TFaHLI=
 k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw=
@@ -651,8 +636,6 @@ k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCf
 k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo=
 oras.land/oras-go v1.2.5/go.mod h1:PuAwRShRZCsZb7g8Ar3jKKQR/2A/qN+pkYxIOd/FAoo=
-sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s=
-sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s=
 sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw=
 sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=