From e125d5555644f2f01b0714c8e9825deff7363290 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Tue, 1 Oct 2024 09:52:30 +1000 Subject: [PATCH 01/10] feat: use statefulset dockerhost --- charts/lagoon-remote/Chart.yaml | 9 ++---- .../templates/docker-host.pvc.yaml | 17 ----------- .../templates/docker-host.service.yaml | 29 +++++++++++++++++++ ...ment.yaml => docker-host.statefulset.yaml} | 25 +++++++++------- 4 files changed, 45 insertions(+), 35 deletions(-) delete mode 100644 charts/lagoon-remote/templates/docker-host.pvc.yaml rename charts/lagoon-remote/templates/{docker-host.deployment.yaml => docker-host.statefulset.yaml} (90%) diff --git a/charts/lagoon-remote/Chart.yaml b/charts/lagoon-remote/Chart.yaml index 8b3719559..b1158feff 100644 --- a/charts/lagoon-remote/Chart.yaml +++ b/charts/lagoon-remote/Chart.yaml @@ -19,7 +19,7 @@ type: application # This is the chart version. This version number should be incremented each # time you make changes to the chart and its templates, including the app # version. -version: 0.94.0 +version: 0.95.0 dependencies: - name: lagoon-build-deploy @@ -41,9 +41,4 @@ dependencies: annotations: artifacthub.io/changes: | - kind: changed - description: remove docker-host connection test in CI - - kind: changed - description: update insights-remote to v0.0.11 - links: - - name: insights-remote v0.0.11 release - url: https://github.com/uselagoon/insights-remote/releases/tag/v0.0.11 + description: changed docker-host to statefulset from deployment diff --git a/charts/lagoon-remote/templates/docker-host.pvc.yaml b/charts/lagoon-remote/templates/docker-host.pvc.yaml deleted file mode 100644 index 1c1f889e7..000000000 --- a/charts/lagoon-remote/templates/docker-host.pvc.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Values.dockerHost.storage.create -}} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "lagoon-remote.dockerHost.fullname" . }} - labels: - {{- include "lagoon-remote.dockerHost.labels" . | nindent 4 }} -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.dockerHost.storage.size | quote }} - {{- with .Values.dockerHost.storage.className }} - storageClassName: {{ . | quote }} - {{- end }} -{{- end }} diff --git a/charts/lagoon-remote/templates/docker-host.service.yaml b/charts/lagoon-remote/templates/docker-host.service.yaml index 1dded8473..116433655 100644 --- a/charts/lagoon-remote/templates/docker-host.service.yaml +++ b/charts/lagoon-remote/templates/docker-host.service.yaml @@ -13,3 +13,32 @@ spec: name: docker-daemon selector: {{- include "lagoon-remote.dockerHost.selectorLabels" . | nindent 4 }} + sessionAffinity: ClientIP + sessionAffinityConfig: + clientIP: + timeoutSeconds: 14400 # 4 hours +{{- $name := include "lagoon-remote.dockerHost.fullname" . -}} +{{- $labels := include "lagoon-remote.dockerHost.labels" . -}} +{{- $stype := .Values.dockerHost.service.type -}} +{{- $sport := .Values.dockerHost.service.port -}} +{{- $replicaCount := .Values.dockerHost.replicaCount | int -}} +{{- range $index0 := until $replicaCount -}} +{{- $index1 := $index0 | add1 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: docker-host-{{ $index0 }} + labels: +{{ $labels | indent 4 }} +spec: + type: {{ $stype }} + ports: + - port: {{ $sport }} + targetPort: docker-daemon + protocol: TCP + name: docker-daemon + sessionAffinity: ClientIP + selector: + statefulset.kubernetes.io/pod-name: {{ $name }}-{{ $index0 }} +{{- end }} diff --git a/charts/lagoon-remote/templates/docker-host.deployment.yaml b/charts/lagoon-remote/templates/docker-host.statefulset.yaml similarity index 90% rename from charts/lagoon-remote/templates/docker-host.deployment.yaml rename to charts/lagoon-remote/templates/docker-host.statefulset.yaml index 3510cb1ba..b6760d711 100644 --- a/charts/lagoon-remote/templates/docker-host.deployment.yaml +++ b/charts/lagoon-remote/templates/docker-host.statefulset.yaml @@ -1,5 +1,5 @@ apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: {{ include "lagoon-remote.dockerHost.fullname" . }} namespace: {{ .Release.Namespace | quote }} @@ -7,11 +7,10 @@ metadata: {{- include "lagoon-remote.dockerHost.labels" . | nindent 4 }} spec: replicas: {{ .Values.dockerHost.replicaCount }} + serviceName: docker-host selector: matchLabels: {{- include "lagoon-remote.dockerHost.selectorLabels" . | nindent 6 }} - strategy: - type: Recreate template: metadata: labels: @@ -91,14 +90,6 @@ spec: volumeMounts: - mountPath: /var/lib/docker name: docker-lib - volumes: - - name: docker-lib - {{- if .Values.dockerHost.storage.create }} - persistentVolumeClaim: - claimName: {{ include "lagoon-remote.dockerHost.fullname" . }} - {{- else }} - emptyDir: {} - {{- end -}} {{- with .Values.dockerHost.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} @@ -111,3 +102,15 @@ spec: tolerations: {{- toYaml . | nindent 6 }} {{- end }} + volumeClaimTemplates: + - metadata: + name: docker-lib + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.dockerHost.storage.size | quote }} + {{- with .Values.dockerHost.storage.className }} + storageClassName: {{ . | quote }} + {{- end }} From 721db919c83f1fe27de311dcd32637c8f2410ad7 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Wed, 2 Oct 2024 09:41:13 +1000 Subject: [PATCH 02/10] feat: change standalone dockerhost to statefulset --- charts/lagoon-docker-host/Chart.yaml | 4 +-- .../templates/docker-host.pvc.yaml | 17 ----------- .../templates/docker-host.service.yaml | 29 +++++++++++++++++++ ...ment.yaml => docker-host.statefulset.yaml} | 25 +++++++++------- 4 files changed, 45 insertions(+), 30 deletions(-) delete mode 100644 charts/lagoon-docker-host/templates/docker-host.pvc.yaml rename charts/lagoon-docker-host/templates/{docker-host.deployment.yaml => docker-host.statefulset.yaml} (90%) diff --git a/charts/lagoon-docker-host/Chart.yaml b/charts/lagoon-docker-host/Chart.yaml index 7a1ca4cca..17a8129ae 100644 --- a/charts/lagoon-docker-host/Chart.yaml +++ b/charts/lagoon-docker-host/Chart.yaml @@ -16,7 +16,7 @@ type: application # This is the chart version. This version number should be incremented each # time you make changes to the chart and its templates, including the app # version. -version: 0.3.1 +version: 0.4.0 appVersion: v3.5.0 @@ -26,4 +26,4 @@ appVersion: v3.5.0 annotations: artifacthub.io/changes: | - kind: changed - description: configure network policy for CI + description: changed docker-host to statefulset from deployment diff --git a/charts/lagoon-docker-host/templates/docker-host.pvc.yaml b/charts/lagoon-docker-host/templates/docker-host.pvc.yaml deleted file mode 100644 index a72c3c6cd..000000000 --- a/charts/lagoon-docker-host/templates/docker-host.pvc.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Values.storage.create -}} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "lagoon-docker-host.dockerHost.fullname" . }} - labels: - {{- include "lagoon-docker-host.dockerHost.labels" . | nindent 4 }} -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.storage.size | quote }} - {{- with .Values.storage.className }} - storageClassName: {{ . | quote }} - {{- end }} -{{- end }} diff --git a/charts/lagoon-docker-host/templates/docker-host.service.yaml b/charts/lagoon-docker-host/templates/docker-host.service.yaml index c1b28ee76..9b8ed59dc 100644 --- a/charts/lagoon-docker-host/templates/docker-host.service.yaml +++ b/charts/lagoon-docker-host/templates/docker-host.service.yaml @@ -13,3 +13,32 @@ spec: name: docker-daemon selector: {{- include "lagoon-docker-host.dockerHost.selectorLabels" . | nindent 4 }} + sessionAffinity: ClientIP + sessionAffinityConfig: + clientIP: + timeoutSeconds: 14400 # 4 hours +{{- $name := include "lagoon-docker-host.dockerHost.fullname" . -}} +{{- $labels := include "lagoon-docker-host.dockerHost.labels" . -}} +{{- $stype := .Values.service.type -}} +{{- $sport := .Values.service.port -}} +{{- $replicaCount := .Values.replicaCount | int -}} +{{- range $index0 := until $replicaCount -}} +{{- $index1 := $index0 | add1 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: docker-host-{{ $index0 }} + labels: +{{ $labels | indent 4 }} +spec: + type: {{ $stype }} + ports: + - port: {{ $sport }} + targetPort: docker-daemon + protocol: TCP + name: docker-daemon + sessionAffinity: ClientIP + selector: + statefulset.kubernetes.io/pod-name: {{ $name }}-{{ $index0 }} +{{- end }} \ No newline at end of file diff --git a/charts/lagoon-docker-host/templates/docker-host.deployment.yaml b/charts/lagoon-docker-host/templates/docker-host.statefulset.yaml similarity index 90% rename from charts/lagoon-docker-host/templates/docker-host.deployment.yaml rename to charts/lagoon-docker-host/templates/docker-host.statefulset.yaml index 5f3b00419..c444546a4 100644 --- a/charts/lagoon-docker-host/templates/docker-host.deployment.yaml +++ b/charts/lagoon-docker-host/templates/docker-host.statefulset.yaml @@ -1,5 +1,5 @@ apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: {{ include "lagoon-docker-host.dockerHost.fullname" . }} namespace: {{ .Release.Namespace | quote }} @@ -7,11 +7,10 @@ metadata: {{- include "lagoon-docker-host.dockerHost.labels" . | nindent 4 }} spec: replicas: {{ .Values.replicaCount }} + serviceName: docker-host selector: matchLabels: {{- include "lagoon-docker-host.dockerHost.selectorLabels" . | nindent 6 }} - strategy: - type: Recreate template: metadata: labels: @@ -91,14 +90,6 @@ spec: volumeMounts: - mountPath: /var/lib/docker name: docker-lib - volumes: - - name: docker-lib - {{- if .Values.storage.create }} - persistentVolumeClaim: - claimName: {{ include "lagoon-docker-host.dockerHost.fullname" . }} - {{- else }} - emptyDir: {} - {{- end -}} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} @@ -111,3 +102,15 @@ spec: tolerations: {{- toYaml . | nindent 6 }} {{- end }} + volumeClaimTemplates: + - metadata: + name: docker-lib + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.storage.size | quote }} + {{- with .Values.storage.className }} + storageClassName: {{ . | quote }} + {{- end }} \ No newline at end of file From 304f5a26d485a4fb6cd8e4d25ec2170380fa80e7 Mon Sep 17 00:00:00 2001 From: Scott Leggett Date: Fri, 18 Oct 2024 09:40:52 +1100 Subject: [PATCH 03/10] feat: bump NATS chart dependency to v1.0.x for lagoon-core --- charts/lagoon-core/Chart.lock | 6 +++--- charts/lagoon-core/Chart.yaml | 23 +++-------------------- 2 files changed, 6 insertions(+), 23 deletions(-) diff --git a/charts/lagoon-core/Chart.lock b/charts/lagoon-core/Chart.lock index 77ce0291c..b7208ca61 100644 --- a/charts/lagoon-core/Chart.lock +++ b/charts/lagoon-core/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: nats repository: https://nats-io.github.io/k8s/helm/charts/ - version: 0.19.17 -digest: sha256:9c58fc4ddeec7b86f5ef2cf1996a48a7e09d9bd4aa149971e2525a6f05649bf8 -generated: "2023-07-28T09:49:46.220986689+08:00" + version: 1.0.3 +digest: sha256:437a0a6fea805b16ec2d292a0cf9bbba8084e09448c9d9b5ae7fcae1be14bb47 +generated: "2023-09-20T12:18:17.665626036+08:00" diff --git a/charts/lagoon-core/Chart.yaml b/charts/lagoon-core/Chart.yaml index 7de6f1a51..a048d505d 100644 --- a/charts/lagoon-core/Chart.yaml +++ b/charts/lagoon-core/Chart.yaml @@ -21,7 +21,7 @@ type: application # time you make changes to the chart and its templates, including the app # version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.48.0 +version: 1.49.0 # This is the version number of the application being deployed. This version # number should be incremented each time you make changes to the application. @@ -31,7 +31,7 @@ appVersion: v2.21.0 dependencies: - name: nats - version: ~0.19.0 + version: ~1.0.0 repository: https://nats-io.github.io/k8s/helm/charts/ condition: nats.enabled @@ -41,21 +41,4 @@ dependencies: annotations: artifacthub.io/changes: | - kind: changed - description: update Lagoon appVersion to v2.21.0 - links: - - name: lagoon-core v2.21.0 release - url: https://github.com/uselagoon/lagoon/releases/tag/v2.21.0 - - kind: changed - description: update insights-handler to v0.0.6 - links: - - name: insights-remote v0.0.6 release - url: https://github.com/uselagoon/insights-handler/releases/tag/v0.0.6 - - kind: changed - description: update ssh-portal and ssh-token to v0.37.2 - links: - - name: ssh-portal v0.37.2 release - url: https://github.com/uselagoon/lagoon-ssh-portal/releases/tag/v0.37.2 - - kind: changed - description: add broker-flag-enable pre-upgrade job - - kind: changed - description: add KEYCLOAK_FRONTEND_URL variable to api deployment + description: update NATS chart dependency to v1.0.x From 2813ce9fc4148a5a9bf82685cd4024b147101eef Mon Sep 17 00:00:00 2001 From: Scott Leggett Date: Fri, 18 Oct 2024 09:40:52 +1100 Subject: [PATCH 04/10] feat: update lagoon-core for NATS 1.0 compatibility --- charts/lagoon-core/ci/linter-values.yaml | 11 +- charts/lagoon-core/templates/_helpers.tpl | 5 +- .../templates/nats-concentrator.service.yaml | 2 +- charts/lagoon-core/templates/nats.secret.yaml | 34 ++--- .../templates/ssh-portal-api.deployment.yaml | 4 +- charts/lagoon-core/values.yaml | 132 ++++++++++++++---- 6 files changed, 128 insertions(+), 60 deletions(-) diff --git a/charts/lagoon-core/ci/linter-values.yaml b/charts/lagoon-core/ci/linter-values.yaml index afd479ce0..10fcdb4c6 100644 --- a/charts/lagoon-core/ci/linter-values.yaml +++ b/charts/lagoon-core/ci/linter-values.yaml @@ -267,9 +267,10 @@ workflows: # enable nats cluster (and optionally natsbox debugger) nats: enabled: true - cluster: - name: lagoon-core-ci-example - # natsbox: + config: + cluster: + name: lagoon-core-ci-example + # natsBox: # enabled: true # # additional labels are required due to the network policy # additionalLabels: @@ -297,7 +298,7 @@ natsConfig: iYmI+nuDxvSE7s/u5hhmh+wCIQDXoxiQvQuokI06j6W1K5UgR6h9dUoKeTFQIqXp uKPLhg== -----END CERTIFICATE----- - server.crt: | + tls.crt: | -----BEGIN CERTIFICATE----- MIICGDCCAb6gAwIBAgIUJC6a9n2zJYl7nOZ2AutYhyjVmQswCgYIKoZIzj0EAwIw HjEcMBoGA1UEAxMTbmF0cy1jYS5leGFtcGxlLmNvbTAeFw0yNDA0MTUwNDA4MDBa @@ -312,7 +313,7 @@ natsConfig: CaKsntUSiMDTZSYvEtLb+ZxOn71RnDYP4JUCIG6TZVFfMiYPZ0gGLMRMf666E3bB 0U5vgRqQhghJPbpM -----END CERTIFICATE----- - server.key: | + tls.key: | -----BEGIN EC PRIVATE KEY----- MHcCAQEEIBqSIJXbR9H4WChftIW2QwJmGD+5QjlfwBCkspRBcsSHoAoGCCqGSM49 AwEHoUQDQgAENf2wtlM9sSu330UXgZTkAOZBRkH2V6YZG7rB/7pTtO0yKQmIfr+l diff --git a/charts/lagoon-core/templates/_helpers.tpl b/charts/lagoon-core/templates/_helpers.tpl index ec90ca698..379e1d74b 100644 --- a/charts/lagoon-core/templates/_helpers.tpl +++ b/charts/lagoon-core/templates/_helpers.tpl @@ -622,10 +622,11 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{/* -Create a default fully qualified app name for the nats subchart. +Create a definition that matches the fully qualified app name for the nats +subchart. */}} {{- define "lagoon-core.nats.fullname" -}} -{{- include "lagoon-core.fullname" . }}-nats +{{- include "lagoon-core.fullname" . }}-{{ .Values.nats.nameOverride | default "nats" }} {{- end }} diff --git a/charts/lagoon-core/templates/nats-concentrator.service.yaml b/charts/lagoon-core/templates/nats-concentrator.service.yaml index 9b935859a..75fedbad5 100644 --- a/charts/lagoon-core/templates/nats-concentrator.service.yaml +++ b/charts/lagoon-core/templates/nats-concentrator.service.yaml @@ -13,5 +13,5 @@ spec: protocol: TCP targetPort: 7422 selector: - app.kubernetes.io/name: nats + app.kubernetes.io/name: {{ .Values.nats.nameOverride | default "nats" | quote }} {{- end }} diff --git a/charts/lagoon-core/templates/nats.secret.yaml b/charts/lagoon-core/templates/nats.secret.yaml index f4e31d3a3..025c8bf8b 100644 --- a/charts/lagoon-core/templates/nats.secret.yaml +++ b/charts/lagoon-core/templates/nats.secret.yaml @@ -30,7 +30,7 @@ metadata: labels: {{- include "lagoon-core.labels" . | nindent 4 }} stringData: - lagoon-core.conf: | + accounts.conf: | accounts: { lagoonRemote: { LAGOON_REMOTE = { @@ -56,26 +56,16 @@ stringData: } } no_auth_user: "lagoon-core-local" - leafnodes: { - listen: "0.0.0.0:7422" - no_advertise: true - authorization: { - users: [ - {{- range .Values.natsConfig.users.lagoonRemote }} - { - user: {{ .user | quote }} - password: {{ .password | quote }} - account: lagoonRemote - }, - {{- end }} - ] - } - tls: { - {{- if .Values.natsConfig.tls.secretData }} - ca_file: "/etc/lagoon-core-nats-tls/ca.crt" - {{- end }} - cert_file: "/etc/lagoon-core-nats-tls/server.crt" - key_file: "/etc/lagoon-core-nats-tls/server.key" - } + leafnodesAuthorization.conf: | + authorization: { + users: [ + {{- range .Values.natsConfig.users.lagoonRemote }} + { + user: {{ .user | quote }} + password: {{ .password | quote }} + account: lagoonRemote + }, + {{- end }} + ] } {{- end }} diff --git a/charts/lagoon-core/templates/ssh-portal-api.deployment.yaml b/charts/lagoon-core/templates/ssh-portal-api.deployment.yaml index f9bd56954..f1b65674d 100644 --- a/charts/lagoon-core/templates/ssh-portal-api.deployment.yaml +++ b/charts/lagoon-core/templates/ssh-portal-api.deployment.yaml @@ -25,7 +25,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - {{ include "lagoon-core.fullname" . }}-nats-client: "true" + lagoon-core-nats-client: "true" {{- include "lagoon-core.sshPortalAPI.selectorLabels" . | nindent 8 }} spec: securityContext: @@ -61,7 +61,7 @@ spec: name: {{ include "lagoon-core.keycloak.fullname" . }} key: KEYCLOAK_SERVICE_API_CLIENT_SECRET - name: NATS_URL - value: nats://{{ include "lagoon-core.fullname" . }}-nats + value: nats://{{ include "lagoon-core.nats.fullname" . }} - name: API_DB_ADDRESS value: {{ include "lagoon-core.apiDB.fullname" . }} - name: API_DB_PASSWORD diff --git a/charts/lagoon-core/values.yaml b/charts/lagoon-core/values.yaml index 41ae684a2..6a8552ce1 100644 --- a/charts/lagoon-core/values.yaml +++ b/charts/lagoon-core/values.yaml @@ -815,37 +815,109 @@ workflows: targetCPUUtilizationPercentage: 80 # targetMemoryUtilizationPercentage: 80 +# nats subchart is configured for use by lagoon-core nats: enabled: false - # inject additional config - additionalVolumes: - - name: lagoon-core-nats-tls - secret: - secretName: lagoon-core-nats-tls - additionalVolumeMounts: - - name: lagoon-core-nats-tls - mountPath: /etc/lagoon-core-nats-tls - nats: - config: - - name: lagoon-core - secret: - secretName: lagoon-core-nats-conf - cluster: + # name override provides compatiblity for upgrading nats chart v0.x to v1.x + nameOverride: nats1 + tlsCA: enabled: true - name: lagoon-core - # internal cluster IPs are not routable, so don't advertise them - noAdvertise: true - natsbox: + secretName: lagoon-core-nats-tls + # Uncomment this block to enable monitoring if you have Prometheus Operator + # installed. + # promExporter: + # enabled: true + # podMonitor: + # enabled: true + # # fromNamespaces restricts the namespaces from which monitoring can occur + # # via the NetworkPolicy. At least one namespace name must be listed. + # fromNamespaces: + # - monitoring + natsBox: enabled: false + config: + cluster: + enabled: true + name: lagoon-core + # internal cluster IPs are not routable, so don't advertise them + noAdvertise: true + leafnodes: + enabled: true + tls: + enabled: true + secretName: lagoon-core-nats-tls + merge: + verify: true + merge: + 00$include: ./lagoon-core/leafnodesAuthorization.conf + merge: + 00$include: ./lagoon-core/accounts.conf + # Uncomment this block if upgrading from lagoon-core + {{ include "nats.fullname" $ | quote }} + labels: + $tplYaml: | + {{ include "nats.labels" $ }} + spec: + policyTypes: + - Ingress + podSelector: + matchLabels: + $tplYaml: | + {{- include "nats.selectorLabels" $ }} + ingress: + - from: + # pods in this namespace with the correct label can access NATS pods + - podSelector: + matchLabels: + lagoon-core-nats-client: "true" + # NATS inter-pod communication is allowed + - podSelector: + matchLabels: + $tplYaml: | + {{- include "nats.selectorLabels" $ }} + - ports: + # allow external inbound leaf node connections (these are authenticated) + - port: 7422 + protocol: TCP + # conditionally allow metrics collection + - $tplYamlSpread: | + {{- if and .Values.promExporter.enabled .Values.promExporter.podMonitor.enabled }} + - from: + {{- range .Values.promExporter.podMonitor.fromNamespaces }} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ . | quote }} + {{- end }} + ports: + - port: 7777 + protocol: TCP + {{- end }} natsService: # this service is enabled via nats.enabled @@ -865,18 +937,22 @@ natsConfig: # If the lagoon-core-nats-tls secret should be created by the lagoon-core # chart, certificate values can be specified directly in secretData. # Configuring TLS this way also allows specifying a custom ca.crt. + # The chart expects expects secretData to be specified unless + # nats.tlsCA.enabled is set to false. # # secretData: # ca.crt: | # ... - # server.crt: | + # tls.crt: | # ... - # server.key: | + # tls.key: | # ... # # If the TLS secret is created outside the lagoon-core chart, it should be # named lagoon-core-nats-tls. This secret should contain fields tls.crt and # tls.key, and the certificate should be issued by a public authority. + # `nats.tlsCA.enabled: false` should be set in this case (it is true by + # default). sshPortalAPI: enabled: false From 231c4a342d7a8d12d0390015e66073f441762e58 Mon Sep 17 00:00:00 2001 From: Scott Leggett Date: Fri, 18 Oct 2024 09:40:52 +1100 Subject: [PATCH 05/10] feat: bump NATS chart dependency to v1.0.x for lagoon-remote --- charts/lagoon-remote/Chart.lock | 6 +++--- charts/lagoon-remote/Chart.yaml | 11 +++-------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/charts/lagoon-remote/Chart.lock b/charts/lagoon-remote/Chart.lock index 615313083..d3bea9945 100644 --- a/charts/lagoon-remote/Chart.lock +++ b/charts/lagoon-remote/Chart.lock @@ -7,6 +7,6 @@ dependencies: version: 0.3.0 - name: nats repository: https://nats-io.github.io/k8s/helm/charts/ - version: 0.19.17 -digest: sha256:6570c9b0a841c10420d28a3fb754569e8922bc5e8e3916c49e13cdb2fb768060 -generated: "2024-06-20T12:17:04.565621295+10:00" + version: 1.0.3 +digest: sha256:3510c54658e4f70646ef4276185334e66941a4fe68fc8980f8d9ccc0658b56c3 +generated: "2023-09-21T11:20:15.805963522+08:00" diff --git a/charts/lagoon-remote/Chart.yaml b/charts/lagoon-remote/Chart.yaml index 8b3719559..671885be5 100644 --- a/charts/lagoon-remote/Chart.yaml +++ b/charts/lagoon-remote/Chart.yaml @@ -19,7 +19,7 @@ type: application # This is the chart version. This version number should be incremented each # time you make changes to the chart and its templates, including the app # version. -version: 0.94.0 +version: 0.95.0 dependencies: - name: lagoon-build-deploy @@ -31,7 +31,7 @@ dependencies: repository: https://amazeeio.github.io/charts/ condition: dbaas-operator.enabled - name: nats - version: ~0.19.0 + version: ~1.0.0 repository: https://nats-io.github.io/k8s/helm/charts/ condition: nats.enabled @@ -41,9 +41,4 @@ dependencies: annotations: artifacthub.io/changes: | - kind: changed - description: remove docker-host connection test in CI - - kind: changed - description: update insights-remote to v0.0.11 - links: - - name: insights-remote v0.0.11 release - url: https://github.com/uselagoon/insights-remote/releases/tag/v0.0.11 + description: update NATS chart dependency to v1.0.x From 8191b0d6ac55189126256fbab693dbcf570bbb18 Mon Sep 17 00:00:00 2001 From: Scott Leggett Date: Fri, 18 Oct 2024 09:40:52 +1100 Subject: [PATCH 06/10] feat: update lagoon-remote for NATS 1.0 compatibility --- charts/lagoon-remote/ci/linter-values.yaml | 13 +- charts/lagoon-remote/templates/_helpers.tpl | 10 ++ .../lagoon-remote/templates/nats.secret.yaml | 10 +- .../templates/ssh-portal.deployment.yaml | 4 +- charts/lagoon-remote/values.yaml | 135 ++++++++++++++---- 5 files changed, 128 insertions(+), 44 deletions(-) diff --git a/charts/lagoon-remote/ci/linter-values.yaml b/charts/lagoon-remote/ci/linter-values.yaml index 68af41ef1..6a2d99264 100644 --- a/charts/lagoon-remote/ci/linter-values.yaml +++ b/charts/lagoon-remote/ci/linter-values.yaml @@ -32,16 +32,17 @@ mxoutHost: mxout1.example.com nats: enabled: true - cluster: - name: lagoon-remote-ci-example - # natsbox: + config: + cluster: + name: lagoon-remote-ci-example + # natsBox: # enabled: true # # additional labels are required due to the network policy # additionalLabels: # lagoon-remote-nats-client: "true" natsConfig: - coreURL: "nats://ci-ssh-portal:ci-password@lagoon-core-nats-concentrator.lagoon-core.svc:7422" + coreURL: "tls://ci-ssh-portal:ci-password@lagoon-core-nats-concentrator.lagoon-core.svc:7422" tls: secretData: ca.crt: | @@ -56,7 +57,7 @@ natsConfig: iYmI+nuDxvSE7s/u5hhmh+wCIQDXoxiQvQuokI06j6W1K5UgR6h9dUoKeTFQIqXp uKPLhg== -----END CERTIFICATE----- - client.crt: | + tls.crt: | -----BEGIN CERTIFICATE----- MIIByDCCAW+gAwIBAgIUJnuRfZT3Viio6HpYvGEehas9qWowCgYIKoZIzj0EAwIw HjEcMBoGA1UEAxMTbmF0cy1jYS5leGFtcGxlLmNvbTAeFw0yNDA0MTUwNDA5MDBa @@ -69,7 +70,7 @@ natsConfig: BAMCA0cAMEQCIGaSyihjkNL2DiUg6nftAUb2jXl97Y38cb8R/srWZdaaAiAC9K6r jzJR6clzzHTzidSigsyeoBmhv7L6643jfB02HQ== -----END CERTIFICATE----- - client.key: | + tls.key: | -----BEGIN EC PRIVATE KEY----- MHcCAQEEINvOV43X7WgqNmkg++wNfmU033hwBDpSG7iDWh6ErzCXoAoGCCqGSM49 AwEHoUQDQgAE5ayAbiEEjfDYHaZPotihXIngy3rj0Pg5kUNHGI7BQWHnmXXFdrOS diff --git a/charts/lagoon-remote/templates/_helpers.tpl b/charts/lagoon-remote/templates/_helpers.tpl index 56cd7263f..9dd6bbfb8 100644 --- a/charts/lagoon-remote/templates/_helpers.tpl +++ b/charts/lagoon-remote/templates/_helpers.tpl @@ -192,6 +192,16 @@ app.kubernetes.io/instance: {{ .Release.Name }} +{{/* +Create a definition that matches the fully qualified app name for the nats +subchart. +*/}} +{{- define "lagoon-remote.nats.fullname" -}} +{{- include "lagoon-remote.fullname" . }}-{{ .Values.nats.nameOverride | default "nats" }} +{{- end }} + + + {{/* Create the name of the service account to use for sshPortal. */}} diff --git a/charts/lagoon-remote/templates/nats.secret.yaml b/charts/lagoon-remote/templates/nats.secret.yaml index e2bdb7bc2..15b3f54fa 100644 --- a/charts/lagoon-remote/templates/nats.secret.yaml +++ b/charts/lagoon-remote/templates/nats.secret.yaml @@ -30,18 +30,18 @@ metadata: labels: {{- include "lagoon-remote.labels" . | nindent 4 }} stringData: - lagoon-remote.conf: | + leafnodes.conf: | leafnodes { no_advertise: true remotes: [ { url: {{ .Values.natsConfig.coreURL | quote }} tls: { - {{- if .Values.natsConfig.tls.secretData -}} - ca_file: "/etc/lagoon-remote-nats-tls/ca.crt" + {{- if .Values.natsConfig.tls.secretData }} + ca_file: "/etc/nats-ca-cert/ca.crt" {{- end }} - cert_file: "/etc/lagoon-remote-nats-tls/client.crt" - key_file: "/etc/lagoon-remote-nats-tls/client.key" + cert_file: "/etc/nats-certs/leafnodes/tls.crt" + key_file: "/etc/nats-certs/leafnodes/tls.key" } } ] diff --git a/charts/lagoon-remote/templates/ssh-portal.deployment.yaml b/charts/lagoon-remote/templates/ssh-portal.deployment.yaml index 3b49dfecf..a3c7a1719 100644 --- a/charts/lagoon-remote/templates/ssh-portal.deployment.yaml +++ b/charts/lagoon-remote/templates/ssh-portal.deployment.yaml @@ -18,7 +18,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - {{ include "lagoon-remote.fullname" . }}-nats-client: "true" + lagoon-remote-nats-client: "true" {{- include "lagoon-remote.sshPortal.selectorLabels" . | nindent 8 }} spec: serviceAccountName: {{ include "lagoon-remote.sshPortal.serviceAccountName" . }} @@ -42,7 +42,7 @@ spec: value: "true" {{- end }} - name: NATS_URL - value: nats://{{ include "lagoon-remote.fullname" . }}-nats + value: nats://{{ include "lagoon-remote.nats.fullname" . }} {{- range $key, $val := .Values.sshPortal.additionalEnvs }} - name: {{ $key }} value: {{ $val | quote }} diff --git a/charts/lagoon-remote/values.yaml b/charts/lagoon-remote/values.yaml index b3ff90559..e3dee3bd8 100644 --- a/charts/lagoon-remote/values.yaml +++ b/charts/lagoon-remote/values.yaml @@ -296,60 +296,133 @@ insightsRemote: # the nats chart is a subchart which is configured for use by lagoon-remote +# nats subchart is configured for use by lagoon-remote nats: enabled: false - # inject additional config - additionalVolumes: - - name: lagoon-remote-nats-tls - secret: - secretName: lagoon-remote-nats-tls - additionalVolumeMounts: - - name: lagoon-remote-nats-tls - mountPath: /etc/lagoon-remote-nats-tls - nats: - config: - - name: lagoon-remote - secret: - secretName: lagoon-remote-nats-conf - cluster: + # name override provides compatiblity for upgrading nats chart v0.x to v1.x + nameOverride: nats1 + tlsCA: enabled: true - name: lagoon-remote - # internal cluster IPs are not routable, so don't advertise them - noAdvertise: true - natsbox: + secretName: lagoon-remote-nats-tls + # Uncomment this block to enable monitoring if you have Prometheus Operator + # installed. + # promExporter: + # enabled: true + # podMonitor: + # enabled: true + # # fromNamespaces restricts the namespaces from which monitoring can occur + # # via the NetworkPolicy. At least one namespace name must be listed. + # fromNamespaces: + # - monitoring + natsBox: enabled: false + config: + cluster: + enabled: true + name: lagoon-remote + # internal cluster IPs are not routable, so don't advertise them + noAdvertise: true + merge: + 00$include: ./lagoon-remote/leafnodes.conf + # Uncomment this block if upgrading from lagoon-remote + {{ include "nats.fullname" $ | quote }} + labels: + $tplYaml: | + {{ include "nats.labels" $ }} + spec: + policyTypes: + - Ingress + podSelector: + matchLabels: + $tplYaml: | + {{- include "nats.selectorLabels" $ }} + ingress: + - from: + # pods in this namespace with the correct label can access NATS pods + - podSelector: + matchLabels: + lagoon-remote-nats-client: "true" + # NATS inter-pod communication is allowed + - podSelector: + matchLabels: + $tplYaml: | + {{- include "nats.selectorLabels" $ }} + # conditionally allow metrics collection + - $tplYamlSpread: | + {{- if and .Values.promExporter.enabled .Values.promExporter.podMonitor.enabled }} + - from: + {{- range .Values.promExporter.podMonitor.fromNamespaces }} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ . | quote }} + {{- end }} + ports: + - port: 7777 + protocol: TCP + {{- end }} + # Configuration for the nats subchart natsConfig: - # coreURL format nats://:@:7422 + # coreURL format tls://:@:7422 coreURL: "" tls: {} # If the lagoon-remote-nats-tls secret should be created by the # lagoon-remote chart, certificate values can be specified directly in # secretData. Configuring TLS this way also allows specifying a custom - # ca.crt. + # ca.crt. The chart expects expects secretData to be specified unless + # nats.tlsCA.enabled is set to false. # # secretData: # ca.crt: | # ... - # client.crt: | + # tls.crt: | # ... - # client.key: | + # tls.key: | # ... # # If the TLS secret is created outside the lagoon-remote chart, it should # be named lagoon-remote-nats-tls. This secret should contain fields # tls.crt and tls.key, and the certificate should be issued by a public - # authority. + # authority. `nats.tlsCA.enabled: false` should be set in this case (it is + # true by default). storageCalculator: enabled: false From 46a7445ec7eee8dd022509f28e1d369b05fe70c1 Mon Sep 17 00:00:00 2001 From: Toby Bellwood Date: Fri, 18 Oct 2024 11:51:06 +1100 Subject: [PATCH 07/10] build: update NATS charts to 1.2.x releases --- charts/lagoon-core/Chart.lock | 6 +++--- charts/lagoon-core/Chart.yaml | 4 ++-- charts/lagoon-remote/Chart.lock | 6 +++--- charts/lagoon-remote/Chart.yaml | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/charts/lagoon-core/Chart.lock b/charts/lagoon-core/Chart.lock index b7208ca61..de1e566e4 100644 --- a/charts/lagoon-core/Chart.lock +++ b/charts/lagoon-core/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: nats repository: https://nats-io.github.io/k8s/helm/charts/ - version: 1.0.3 -digest: sha256:437a0a6fea805b16ec2d292a0cf9bbba8084e09448c9d9b5ae7fcae1be14bb47 -generated: "2023-09-20T12:18:17.665626036+08:00" + version: 1.2.5 +digest: sha256:9ef2d1dfe07187fbf9680ee525bced5d5a460349cd3b1fd06fc32b843481e238 +generated: "2024-10-18T11:48:49.486819359+11:00" diff --git a/charts/lagoon-core/Chart.yaml b/charts/lagoon-core/Chart.yaml index a048d505d..f4cbbfb6f 100644 --- a/charts/lagoon-core/Chart.yaml +++ b/charts/lagoon-core/Chart.yaml @@ -31,7 +31,7 @@ appVersion: v2.21.0 dependencies: - name: nats - version: ~1.0.0 + version: ~1.2.0 repository: https://nats-io.github.io/k8s/helm/charts/ condition: nats.enabled @@ -41,4 +41,4 @@ dependencies: annotations: artifacthub.io/changes: | - kind: changed - description: update NATS chart dependency to v1.0.x + description: update NATS chart dependency to v1.2.x diff --git a/charts/lagoon-remote/Chart.lock b/charts/lagoon-remote/Chart.lock index d3bea9945..38020aca7 100644 --- a/charts/lagoon-remote/Chart.lock +++ b/charts/lagoon-remote/Chart.lock @@ -7,6 +7,6 @@ dependencies: version: 0.3.0 - name: nats repository: https://nats-io.github.io/k8s/helm/charts/ - version: 1.0.3 -digest: sha256:3510c54658e4f70646ef4276185334e66941a4fe68fc8980f8d9ccc0658b56c3 -generated: "2023-09-21T11:20:15.805963522+08:00" + version: 1.2.5 +digest: sha256:8d25a12b60290743ed7dd4a512ba401a1df0afc60da17f50ca467f593068b53b +generated: "2024-10-18T11:46:28.50935365+11:00" diff --git a/charts/lagoon-remote/Chart.yaml b/charts/lagoon-remote/Chart.yaml index 671885be5..219131118 100644 --- a/charts/lagoon-remote/Chart.yaml +++ b/charts/lagoon-remote/Chart.yaml @@ -31,7 +31,7 @@ dependencies: repository: https://amazeeio.github.io/charts/ condition: dbaas-operator.enabled - name: nats - version: ~1.0.0 + version: ~1.2.0 repository: https://nats-io.github.io/k8s/helm/charts/ condition: nats.enabled @@ -41,4 +41,4 @@ dependencies: annotations: artifacthub.io/changes: | - kind: changed - description: update NATS chart dependency to v1.0.x + description: update NATS chart dependency to v1.2.x From bba903a6985085d68c9e70e1b2efc5c2d13f8637 Mon Sep 17 00:00:00 2001 From: Toby Bellwood Date: Fri, 18 Oct 2024 11:51:26 +1100 Subject: [PATCH 08/10] docs: add explanation to nameOverride in values --- charts/lagoon-core/values.yaml | 4 +++- charts/lagoon-remote/values.yaml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/charts/lagoon-core/values.yaml b/charts/lagoon-core/values.yaml index 6a8552ce1..7d481d052 100644 --- a/charts/lagoon-core/values.yaml +++ b/charts/lagoon-core/values.yaml @@ -818,7 +818,9 @@ workflows: # nats subchart is configured for use by lagoon-core nats: enabled: false - # name override provides compatiblity for upgrading nats chart v0.x to v1.x + # nameOverride provides compatibility for upgrading nats chart versions. + # modifying or setting this in local values.yaml will completely recreate + # all the nats resources. nameOverride: nats1 tlsCA: enabled: true diff --git a/charts/lagoon-remote/values.yaml b/charts/lagoon-remote/values.yaml index e3dee3bd8..2334a2e80 100644 --- a/charts/lagoon-remote/values.yaml +++ b/charts/lagoon-remote/values.yaml @@ -299,7 +299,9 @@ insightsRemote: # nats subchart is configured for use by lagoon-remote nats: enabled: false - # name override provides compatiblity for upgrading nats chart v0.x to v1.x + # nameOverride provides compatibility for upgrading nats chart versions. + # modifying or setting this in local values.yaml will completely recreate + # all the nats resources. nameOverride: nats1 tlsCA: enabled: true From ba8eabb2fd1771a6afb4149e135377799507e777 Mon Sep 17 00:00:00 2001 From: Toby Bellwood Date: Fri, 18 Oct 2024 15:27:14 +1100 Subject: [PATCH 09/10] test: check upgrade override --- .github/workflows/lint-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index c0b645870..8fecf0b80 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -97,6 +97,7 @@ jobs: - name: Run chart-testing (upgrade changed next-release only) run: | + yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' ./charts/lagoon-core/ci/linter-values.yaml ./charts/lagoon-core/ci/testlagoon-main-override.yaml ct install --upgrade --config ./default.ct.yaml --helm-extra-args "--timeout 30m" if: ${{ contains(github.event.pull_request.labels.*.name, 'next-release') }} From 43eeb9fa9b3bd9d6fd6deb4889874a88412f68d6 Mon Sep 17 00:00:00 2001 From: Toby Bellwood Date: Fri, 18 Oct 2024 15:47:11 +1100 Subject: [PATCH 10/10] test: remove upgrade test --- .github/workflows/lint-test.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index 8fecf0b80..1c1f23efd 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -95,11 +95,10 @@ jobs: ct install --config ./default.ct.yaml --helm-extra-args "--timeout 30m" if: ${{ contains(github.event.pull_request.labels.*.name, 'next-release') }} - - name: Run chart-testing (upgrade changed next-release only) - run: | - yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' ./charts/lagoon-core/ci/linter-values.yaml ./charts/lagoon-core/ci/testlagoon-main-override.yaml - ct install --upgrade --config ./default.ct.yaml --helm-extra-args "--timeout 30m" - if: ${{ contains(github.event.pull_request.labels.*.name, 'next-release') }} + # - name: Run chart-testing (upgrade changed next-release only) + # run: | + # ct install --upgrade --config ./default.ct.yaml --helm-extra-args "--timeout 30m" + # if: ${{ contains(github.event.pull_request.labels.*.name, 'next-release') }} - name: Run chart-testing (install all charts when required) run: ct install --config ./default.ct.yaml --helm-extra-args "--timeout 30m" --all