Merge pull request #922 from Azure/cluster-service-helm-chart

convert cs deployment to helm chart

.yamllint.yml

@@ -3,6 +3,8 @@ yaml-files:
   - '*.yaml'
   - '*.yml'
   - '.yamllint'
+ignore:
+  - 'cluster-service/deploy/helm/templates/azure-operators-managed-identities-config.configmap.yaml'
 
 rules:
   brackets: enable

cluster-service/Makefile

@@ -3,59 +3,48 @@ DEPLOY_ENV ?= personal-dev
 $(shell ../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk)
 include config.mk
 
-deploy: deploy-namespace-template deploy-istio-configurations-template ${DB_SECRET_TARGET}
+deploy: provision-shard
+	ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${RESOURCEGROUP} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \
+	kubectl create namespace cluster-service --dry-run=client -o json | kubectl apply -f - && \
+	kubectl label namespace cluster-service "istio.io/rev=$${ISTO_VERSION}" --overwrite=true && \
 	AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n clusters-service --query clientId -o tsv) && \
 	CS_SERVICE_PRINCIPAL_CREDS_BASE64='$(shell az keyvault secret show --vault-name "${SERVICE_KV}" --name "aro-hcp-dev-sp-cs" | jq .value -r | base64 | tr -d '\n')' && \
 	TENANT_ID=$(shell az account show --query tenantId --output tsv) && \
 	OIDC_BLOB_SERVICE_ENDPOINT=$(shell az storage account show -n ${OIDC_STORAGE_ACCOUNT} -g ${RESOURCEGROUP} --query primaryEndpoints.blob -o tsv) && \
 	OIDC_ISSUER_BASE_ENDPOINT=$(shell az storage account show -n ${OIDC_STORAGE_ACCOUNT} -g ${RESOURCEGROUP} --query primaryEndpoints.web -o tsv) && \
 	OCP_ACR_URL=$(shell az acr show -n ${OCP_ACR_NAME} --query loginServer -o tsv) && \
 	OCP_ACR_RESOURCE_ID=$(shell az acr show -n ${OCP_ACR_NAME} --query id -o tsv) && \
-	oc process --local -f deploy/openshift-templates/arohcp-service-template.yml \
-	  -p AZURE_CS_MI_CLIENT_ID=$${AZURE_CS_MI_CLIENT_ID} \
-	  -p TENANT_ID=$${TENANT_ID} \
-	  -p REGION=${REGION} \
-	  -p SERVICE_KEYVAULT_NAME=${SERVICE_KV} \
-	  -p CS_SERVICE_PRINCIPAL_CREDS_BASE64=$${CS_SERVICE_PRINCIPAL_CREDS_BASE64} \
-	  -p IMAGE_REGISTRY=${ACR_NAME}.azurecr.io \
-	  -p IMAGE_REPOSITORY=${IMAGE_REPO} \
-	  -p AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID=${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} \
-	  -p FPA_CERT_NAME=${FPA_CERT_NAME} \
-	  -p IMAGE_TAG=${IMAGE_TAG} \
-	  -p OCP_ACR_RESOURCE_ID=$${OCP_ACR_RESOURCE_ID} \
-	  -p OCP_ACR_URL=$${OCP_ACR_URL} \
-	  -p DATABASE_DISABLE_TLS=${DATABASE_DISABLE_TLS} \
-	  -p OIDC_ISSUER_BASE_URL=$${OIDC_ISSUER_BASE_ENDPOINT} \
-	  -p OIDC_ISSUER_BLOB_SERVICE_URL=$${OIDC_BLOB_SERVICE_ENDPOINT} \
-	  -p AZURE_MI_MOCK_SERVICE_PRINCIPAL_PRINCIPAL_ID=${AZURE_MI_MOCK_SERVICE_PRINCIPAL_PRINCIPAL_ID} \
-	  -p AZURE_MI_MOCK_SERVICE_PRINCIPAL_CLIENT_ID=${AZURE_MI_MOCK_SERVICE_PRINCIPAL_CLIENT_ID} \
-	  -p AZURE_MI_MOCK_SERVICE_PRINCIPAL_CERT_NAME=${MI_MOCK_SERVICE_PRINCIPAL_CERT_NAME} \
-	  -p DATABASE_AUTH_METHOD=${DATABASE_AUTH_METHOD} \
-	  -p AZURE_ARM_HELPER_IDENTITY_CLIENT_ID=${AZURE_ARM_HELPER_IDENTITY_CLIENT_ID} \
-	  -p AZURE_ARM_HELPER_IDENTITY_CERT_NAME=${ARM_HELPER_CERT_NAME} \
-	  -p AZURE_ARM_HELPER_MOCK_FPA_PRINCIPAL_ID=${AZURE_ARM_HELPER_MOCK_FPA_PRINCIPAL_ID} \
-	  | oc apply -f -
-
-deploy-namespace-template:
-	ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${RESOURCEGROUP} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \
-	oc process --local -f deploy/openshift-templates/arohcp-namespace-template.yml \
-		-p ISTIO_VERSION=$${ISTO_VERSION} | oc apply -f -
-
-deploy-istio-configurations-template:
-	kubectl apply -f deploy/istio.yml
-
-deploy-local-db-secret: provision-shard
-	oc process --local -f deploy/openshift-templates/arohcp-secrets-template.yml \
-		-p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/provisioning-shards.yml)" | oc apply -f -
-	oc process --local -f deploy/openshift-templates/arohcp-db-template.yml | oc apply -f -
-
-deploy-azure-db-secret: provision-shard
-	oc process --local -f deploy/openshift-templates/arohcp-secrets-template.yml \
-		-p DATABASE_USER=clusters-service \
-		-p DATABASE_NAME=clusters-service \
-		-p DATABASE_PASSWORD="" \
-		-p DATABASE_HOST=$(shell az postgres flexible-server show --resource-group ${RESOURCEGROUP} -n ${DATABASE_SERVER_NAME} --query fullyQualifiedDomainName -o tsv) \
-		-p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/provisioning-shards.yml)" | oc apply -f -
+	helm upgrade --install cluster-service --namespace cluster-service \
+	  deploy/helm/ \
+	  --set azureCsMiClientId=$${AZURE_CS_MI_CLIENT_ID} \
+	  --set oidcIssuerBlobServiceUrl=$${OIDC_BLOB_SERVICE_ENDPOINT} \
+	  --set oidcIssuerBaseUrl=$${OIDC_ISSUER_BASE_ENDPOINT} \
+	  --set tenantId=$${TENANT_ID} \
+	  --set region=${REGION} \
+	  --set serviceKeyvaultName=${SERVICE_KV} \
+	  --set csServicePrincipalCredsBase64=$${CS_SERVICE_PRINCIPAL_CREDS_BASE64} \
+	  --set imageRegistry=${ACR_NAME}.azurecr.io \
+	  --set imageRepository=${IMAGE_REPO} \
+	  --set imageTag=${IMAGE_TAG} \
+	  --set azureFirstPartyApplicationClientId=${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} \
+	  --set fpaCertName=${FPA_CERT_NAME} \
+	  --set ocpAcrResourceId=$${OCP_ACR_RESOURCE_ID} \
+	  --set ocpAcrUrl=$${OCP_ACR_URL} \
+	  --set databaseDisableTls=${DATABASE_DISABLE_TLS} \
+	  --set databaseAuthMethod=${DATABASE_AUTH_METHOD} \
+	  --set provisionShardsConfig="$(shell base64 -i -w 0 deploy/provisioning-shards.yml)" \
+	  --set deployLocalDatabase=${DEPLOY_LOCAL_DB} \
+	  --set databaseHost=${DB_HOST} \
+	  --set databaseName=${DB_NAME} \
+	  --set databaseUser=${DB_USERNAME} \
+	  --set databasePassword=${DB_PASSWORD} \
+	  --set azureMiMockServicePrincipalPrincipalId=${AZURE_MI_MOCK_SERVICE_PRINCIPAL_PRINCIPAL_ID} \
+	  --set azureMiMockServicePrincipalClientId=${AZURE_MI_MOCK_SERVICE_PRINCIPAL_CLIENT_ID} \
+	  --set azureMiMockServicePrincipalCertName=${MI_MOCK_SERVICE_PRINCIPAL_CERT_NAME} \
+	  --set azureArmHelperIdentityCertName=${ARM_HELPER_CERT_NAME} \
+	  --set azureArmHelperIdentityClientId=${AZURE_ARM_HELPER_IDENTITY_CLIENT_ID} \
+	  --set azureArmHelperMockFpaPrincipalId=${AZURE_ARM_HELPER_MOCK_FPA_PRINCIPAL_ID} \
+	  --set azureOperatorsManagedIdentitiesConfig=${AZURE_OPERATORS_MANAGED_IDENTITIES_CONFIG}
 
 deploy-pr-env-deps:
 	AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n clusters-service --query clientId -o tsv) && \

cluster-service/config.tmpl.mk

@@ -23,10 +23,17 @@ DATABASE_DISABLE_TLS ?= {{ not .clusterService.postgres.deploy }}
 DATABASE_AUTH_METHOD ?= {{ ternary "az-entra" "postgres" .clusterService.postgres.deploy }}
 DATABASE_SERVER_NAME ?= {{ .clusterService.postgres.name }}
 DB_SECRET_TARGET = {{ ternary "deploy-azure-db-secret" "deploy-local-db-secret" .clusterService.postgres.deploy }}
+DEPLOY_LOCAL_DB ?= {{ ternary "false" "true" .clusterService.postgres.deploy }}
+DB_HOST ?= {{ ternary "$(shell az postgres flexible-server show --resource-group ${RESOURCEGROUP} -n ${DATABASE_SERVER_NAME} --query fullyQualifiedDomainName -o tsv)" "ocm-cs-db" .clusterService.postgres.deploy }}
+DB_NAME ?= {{ ternary "clusters-service" "ocm-cs-db" .clusterService.postgres.deploy }}
+DB_USERNAME ?= {{ ternary "clusters-service" "ocm" .clusterService.postgres.deploy }}
+DB_PASSWORD ?= {{ ternary "" "TheBlurstOfTimes" .clusterService.postgres.deploy }}
 
 DEVOPS_MSI_ID ?= {{ .aroDevopsMsiId }}
 
 # MGMT CLUSTER KVs
 MGMT_RESOURCEGROUP ?= {{ .mgmt.rg }}
 CX_SECRETS_KV_NAME ?= {{ .cxKeyVault.name }}
 CX_MI_KV_NAME ?= {{ .msiKeyVault.name }}
+
+AZURE_OPERATORS_MANAGED_IDENTITIES_CONFIG ?= {{ .clusterService.azureOperatorsManagedIdentitiesConfig | b64enc }}

cluster-service/deploy/helm/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

cluster-service/deploy/helm/Chart.yaml

@@ -0,0 +1,13 @@
+apiVersion: v2
+name: cluster-service
+description: A Helm chart for Cluster Service
+type: application
+version: 0.1.0
+
+appVersion: "1.16.0"
+
+dependencies:
+  - name: database
+    version: 0.1.0
+    repository: "file://charts/database"
+    condition: deployLocalDatabase

cluster-service/deploy/helm/charts/database/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: database
+description: A Helm chart for a Local Cluster Service Database
+type: application
+version: 0.1.0

cluster-service/deploy/helm/charts/database/templates/database.deployment.yaml

@@ -0,0 +1,69 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.databaseServiceName }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: {{ .Values.databaseServiceName }}
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        name: {{ .Values.databaseServiceName }}
+    spec:
+      containers:
+        - env:
+            - name: POSTGRES_USER
+              valueFrom:
+                secretKeyRef:
+                  key: db.user
+                  name: {{ .Values.databaseK8sSecretName }}
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  key: db.password
+                  name: {{ .Values.databaseK8sSecretName }}
+            - name: POSTGRES_DB
+              valueFrom:
+                secretKeyRef:
+                  key: db.name
+                  name: {{ .Values.databaseK8sSecretName }}
+            - name: PGDATA
+              value: /var/lib/pgsql/data/pgdata
+          image: docker.io/library/postgres:16.2
+          imagePullPolicy: IfNotPresent
+          livenessProbe:
+            tcpSocket:
+              port: 5432
+            initialDelaySeconds: 120
+            timeoutSeconds: 10
+          name: postgresql
+          ports:
+            - containerPort: 5432
+              protocol: TCP
+          readinessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - exec /usr/bin/pg_isready -U $POSTGRES_USER -d $POSTGRES_DB -h localhost -p 5432
+            initialDelaySeconds: 5
+            timeoutSeconds: 1
+          securityContext:
+            capabilities: {}
+            privileged: false
+          terminationMessagePath: /dev/termination-log
+          volumeMounts:
+            - mountPath: /var/lib/pgsql/data
+              name: '{{ .Values.databaseServiceName }}-data'
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      volumes:
+        - name: '{{ .Values.databaseServiceName }}-data'
+          persistentVolumeClaim:
+            claimName: {{ .Values.databaseServiceName }}

cluster-service/deploy/helm/charts/database/templates/database.pvc.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ .Values.databaseServiceName }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ .Values.databaseVolumeCapacity }}

cluster-service/deploy/helm/charts/database/templates/database.service.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.databaseServiceName }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  ports:
+    - name: postgresql
+      nodePort: 0
+      port: 5432
+      protocol: TCP
+      targetPort: 5432
+  selector:
+    name: {{ .Values.databaseServiceName }}
+  sessionAffinity: None
+  type: ClusterIP

cluster-service/deploy/helm/charts/database/values.yaml

@@ -0,0 +1,26 @@
+# The name of the OpenShift Service exposed for the database.
+databaseServiceName: ocm-cs-db
+
+# The name of the K8s secret where CS DB connection information is placed
+databaseK8sSecretName: ocm-cs-db
+
+# Volume space available for data, e.g. 512Mi, 2Gi.
+databaseVolumeCapacity: 512Mi
+
+# Version of PostgreSQL image to be used (10 or latest).
+postgresqlVersion: "12"
+
+# The hostname of the postgres server/service. It can be a K8s service name
+databaseHost: "ocm-cs-db"
+
+# Username for PostgreSQL user that will be used for accessing the database.
+databaseUser: "ocm"
+
+# Password for the PostgreSQL connection user.
+databasePassword: "TheBlurstOfTimes"
+
+# Name of the PostgreSQL database accessed.
+databaseName: "ocm-cs-db"
+
+# Host port
+databasePort: "5432"

cluster-service/deploy/helm/templates/authentication.configmap.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: authentication
+  namespace: {{ .Release.Namespace }}
+data:
+  jwks.json: ""
+  acl.yml: ""

cluster-service/deploy/helm/templates/azure-credentials.secret.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: azure-credentials
+  namespace: {{ .Release.Namespace }}
+type: Opaque
+data:
+  azure-auth-config: {{ .Values.csServicePrincipalCredsBase64 }}

cluster-service/deploy/helm/templates/azure-operators-managed-identities-config.configmap.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: azure-operators-managed-identities-config
+  namespace: {{ .Release.Namespace }}
+data:
+  azure-operators-managed-identities-config.yaml: |
+{{ .Values.azureOperatorsManagedIdentitiesConfig | b64dec | indent 4 }}

cluster-service/deploy/helm/templates/azure-runtime-config.configmap.yaml

@@ -0,0 +1,23 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: azure-runtime-config
+  namespace: {{ .Release.Namespace }}
+data:
+  config.json: |
+    {
+      "cloudEnvironment": "AzurePublicCloud",
+      "managedIdentitiesDataPlaneAudienceResource": "{{ .Values.managedIdentitiesDataPlaneAudienceResource }}",
+      "tenantId": "{{ .Values.tenantId }}",
+      "ocpImagesAcr": {
+        "resourceId": "{{ .Values.ocpAcrResourceId }}",
+        "url": "{{ .Values.ocpAcrUrl }}",
+        "scopeMapName": "_repositories_pull"
+      },
+      "dataplane_identities_oidc_configuration": {
+        "storage_account_blob_container_name": "$web",
+        "storage_account_blob_service_url": "{{ .Values.oidcIssuerBlobServiceUrl }}",
+        "oidc_issuer_base_url": "{{ .Values.oidcIssuerBaseUrl }}"
+      }
+    }

cluster-service/deploy/helm/templates/cloud-resource-constraints-config.configmap.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cloud-resource-constraints-config
+  namespace: {{ .Release.Namespace }}
+data:
+  instance-type-constraints.yaml: |
+    instance_types:
+      - id: Standard_D8s_v3
+        ccs_only: true
+        enabled: true
+  cloud-region-constraints.yaml: |
+    cloud_regions:
+      - id: {{ .Values.region }}
+        enabled: true
+        govcloud: false
+        ccs_only: false

cluster-service/deploy/helm/templates/cloud-resources-config.configmap.yaml

@@ -0,0 +1,23 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cloud-resources-config
+  namespace: {{ .Release.Namespace }}
+data:
+  instance-types.yaml: |
+    instance_types:
+      - id: Standard_D8s_v3
+        name: Standard_D8s_v3 - General purpose
+        cloud_provider_id: azure
+        cpu_cores: 8
+        memory: 34359738368
+        category: general_purpose
+        size: d8s_v3
+        generic_name: standard-d8s_v3
+  cloud-regions.yaml: |
+    cloud_regions:
+      - id: {{ .Values.region }}
+        cloud_provider_id: azure
+        display_name: Azure East US
+        supports_multi_az: true