From 1af07b1ce7305e35a6c1300b23a0076fbc2a1dd9 Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Thu, 16 May 2024 16:49:53 +0200 Subject: [PATCH] AKS template for maestro the new template `service-template-aro-hcp.yml` is a first draft for deploying the maestro server on ARO-HCP. it leverages the pre-provisoned secrets that can be found on such a cluster, like the MQTT secret and CSI secret store configuration to transparently load client certificates from key vault. part of https://issues.redhat.com/browse/ARO-7234 --- templates/db-template.yml | 23 +- templates/service-template-aro-hcp.yml | 351 +++++++++++++++++++++++++ 2 files changed, 370 insertions(+), 4 deletions(-) create mode 100755 templates/service-template-aro-hcp.yml diff --git a/templates/db-template.yml b/templates/db-template.yml index c5f223b5..23c2efd7 100755 --- a/templates/db-template.yml +++ b/templates/db-template.yml @@ -46,6 +46,11 @@ parameters: required: true value: "postgres:14.2" + - name: DB_SECRET_NAME + displayName: Database Secret Name + description: Name of the secret containing database connection information + value: "maestro-db" + objects: - apiVersion: v1 @@ -102,17 +107,17 @@ objects: valueFrom: secretKeyRef: key: database-user - name: ${DATABASE_SERVICE_NAME} + name: ${DATABASE_SERVICE_NAME}-creds - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: key: database-password - name: ${DATABASE_SERVICE_NAME} + name: ${DATABASE_SERVICE_NAME}-creds - name: POSTGRES_DB valueFrom: secretKeyRef: key: database-name - name: ${DATABASE_SERVICE_NAME} + name: ${DATABASE_SERVICE_NAME}-creds - name: PGDATA value: /var/lib/pgsql/data/pgdata image: ${POSTGRES_IMAGE} @@ -155,9 +160,19 @@ objects: template.openshift.io/expose-database_name: "{.data['database-name']}" template.openshift.io/expose-password: "{.data['database-password']}" template.openshift.io/expose-username: "{.data['database-user']}" - name: ${DATABASE_SERVICE_NAME} + name: ${DATABASE_SERVICE_NAME}-creds stringData: database-name: ${DATABASE_NAME} database-password: ${DATABASE_PASSWORD} database-user: ${DATABASE_USER} + - apiVersion: v1 + kind: Secret + metadata: + name: ${DB_SECRET_NAME} + stringData: + db.host: ${DATABASE_SERVICE_NAME} + db.port: "5432" + db.name: ${DATABASE_NAME} + db.user: ${DATABASE_USER} + db.password: ${DATABASE_PASSWORD} diff --git a/templates/service-template-aro-hcp.yml b/templates/service-template-aro-hcp.yml new file mode 100755 index 00000000..a1e01cb1 --- /dev/null +++ b/templates/service-template-aro-hcp.yml @@ -0,0 +1,351 @@ +--- +kind: Template +apiVersion: v1 +metadata: + name: maestro-service + annotations: + openshift.io/display-name: maestro + description: Example Service API for the Unified Hybrid Cloud deployment + tags: golang,uhc,service-delivery + iconClass: icon-shadowman + template.openshift.io/provider-display-name: Red Hat, Inc. + template.openshift.io/documentation-url: https://gitlab.cee.redhat.com/service/ +labels: + template: maestro +parameters: + +- name: ENVIRONMENT + displayName: Environment + description: Which Account Manager environment to use for this deployment + value: production + +- name: IMAGE_REGISTRY + displayName: Image Registry + required: true + +- name: IMAGE_REPOSITORY + displayName: Image Repository + required: true + +- name: VERSION + displayName: Image version + value: latest + +- name: IMAGE_TAG + displayName: Image tag + value: latest + +- name: GLOG_V + displayName: GLOG V Level + description: Log verbosity level + value: "10" + +- name: MEMORY_REQUEST + description: Memory request for the API pods. + value: "512Mi" + +- name: MEMORY_LIMIT + description: Memory limit for the API pods. + value: "1Gi" + +- name: CPU_REQUEST + description: CPU request for the API pods. + value: "200m" + +- name: CPU_LIMIT + description: CPU limit for the API pods. + value: "1" + +- name: OCM_BASE_URL + displayName: OCM API Base URL + description: Base path for all OCM APIs + +- name: OCM_DEBUG + displayName: OCM API Debug mode + description: Debug mode for OCM API client + value: "false" + +- name: REPLICAS + description: Number of replicas of the service to run. + value: "1" + +- name: ENABLE_JWT + displayName: Enable JWT + description: Enable JWT authentication validation + value: "true" + +- name: ENABLE_HTTPS + displayName: Enable HTTPS + description: Enable HTTPS rather than HTTP + value: "true" + +- name: HTTP_SERVER_BINDPORT + displayName: HTTP Server Bindport + description: HTTP server bind port + value: "8000" + +- name: GRPC_SERVER_BINDPORT + displayName: gRPC Server Bindport + description: gRPC server bind port + value: "8090" + +- name: METRICS_SERVER_BINDPORT + displayName: Metrics Server Bindport + description: Metrics server bind port + value: "8080" + +- name: HEALTH_CHECK_SERVER_BINDPORT + displayName: Health check Server Bindport + description: Health check server bind port + value: "8083" + +- name: HTTP_SERVER_HOSTNAME + displayName: HTTP Server Hostname + description: Server's public hostname + value: "" + +- name: ENABLE_AUTHZ + displayName: Enable Authz + description: Enable Authorization on endpoints, should only be disabled for debug + value: "true" + +- name: DB_MAX_OPEN_CONNS + displayName: Maximum Open Database Connections + description: Maximum number of open database connections per pod + value: "50" + +- name: DB_SSLMODE + displayName: DB SSLmode + description: Database ssl mode (disable | require | verify-ca | verify-full) + value: "verify-full" + +- name: ENABLE_DB_DEBUG + displayName: Enable DB Debug + description: framework's debug mode + value: "false" + +- name: ENABLE_METRICS_HTTPS + displayName: Enable Metrics HTTPS + description: Enable HTTPS for metrics server + value: "false" + +- name: ENABLE_OCM_MOCK + displayName: Enable OCM Mock + description: Enable mock uhc client + value: "false" + +- name: HTTP_READ_TIMEOUT + displayName: HTTP Read Timeout + description: HTTP server read timeout + value: 5s + +- name: HTTP_WRITE_TIMEOUT + displayName: HTTP Write Timeout + description: HTTP server write timeout + value: 30s + +- name: LABEL_METRICS_INCLUSION_DURATION + displayName: Label metrics inclusion duration + description: A cluster's last telemetry date needs be within in this duration in order to have labels collected + value: "168h" + +- name: DB_SECRET_NAME + displayName: Database Secret Name + description: Name of the secret containing database connection information + value: "maestro-db" + +- name: MAESTRO_SERVER_EXPOSURE + displayName: The type of service to be used to expose the Maestro Server + value: "ClusterIP" + +objects: + - kind: Deployment + apiVersion: apps/v1 + metadata: + name: maestro + labels: + app: maestro + spec: + selector: + matchLabels: + app: maestro + replicas: ${{REPLICAS}} + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + app: maestro + spec: + serviceAccountName: maestro + volumes: + - name: db + secret: + secretName: ${DB_SECRET_NAME} + - name: mqtt + secret: + secretName: maestro-mqtt + - name: mqtt-creds + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "maestro" + initContainers: + - name: migration + image: ${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}:${IMAGE_TAG} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: db + mountPath: /secrets/db + command: + - /usr/local/bin/maestro + - migration + - --db-host-file=/secrets/db/db.host + - --db-port-file=/secrets/db/db.port + - --db-user-file=/secrets/db/db.user + - --db-password-file=/secrets/db/db.password + - --db-name-file=/secrets/db/db.name + - --db-sslmode=${DB_SSLMODE} + - --alsologtostderr + - -v=${GLOG_V} + containers: + - name: service + image: ${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}:${IMAGE_TAG} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: db + mountPath: /secrets/db + - name: mqtt + mountPath: /secrets/mqtt + - name: mqtt-creds + mountPath: /secrets/mqtt-creds + readOnly: true + env: + - name: "AMS_ENV" + value: "${ENVIRONMENT}" + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + command: + - /usr/local/bin/maestro + - server + - --client-id=maestro-$(POD_NAME) + - --db-host-file=/secrets/db/db.host + - --db-port-file=/secrets/db/db.port + - --db-user-file=/secrets/db/db.user + - --db-password-file=/secrets/db/db.password + - --db-name-file=/secrets/db/db.name + - --db-rootcert=/secrets/db/db.ca_cert + - --db-sslmode=${DB_SSLMODE} + - --mqtt-config-file=/secrets/mqtt/config.yaml + - --enable-ocm-mock=${ENABLE_OCM_MOCK} + - --enable-jwt=${ENABLE_JWT} + - --enable-https=${ENABLE_HTTPS} + - --server-hostname=${HTTP_SERVER_HOSTNAME} + - --http-server-bindport=${HTTP_SERVER_BINDPORT} + - --grpc-server-bindport=${GRPC_SERVER_BINDPORT} + - --health-check-server-bindport=${HEALTH_CHECK_SERVER_BINDPORT} + - --enable-health-check-https=${ENABLE_HTTPS} + - --db-sslmode=${DB_SSLMODE} + - --db-max-open-connections=${DB_MAX_OPEN_CONNS} + - --enable-authz=${ENABLE_AUTHZ} + - --enable-db-debug=${ENABLE_DB_DEBUG} + - --enable-metrics-https=${ENABLE_METRICS_HTTPS} + - --enable-sentry=false + - --http-read-timeout=${HTTP_READ_TIMEOUT} + - --http-write-timeout=${HTTP_WRITE_TIMEOUT} + - --label-metrics-inclusion-duration=${LABEL_METRICS_INCLUSION_DURATION} + - --alsologtostderr + - -v=${GLOG_V} + resources: + requests: + cpu: ${CPU_REQUEST} + memory: ${MEMORY_REQUEST} + limits: + cpu: ${CPU_LIMIT} + memory: ${MEMORY_LIMIT} + livenessProbe: + httpGet: + path: /api/maestro + port: 8000 + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /healthcheck + port: 8083 + scheme: HTTP + httpHeaders: + - name: User-Agent + value: Probe + initialDelaySeconds: 20 + periodSeconds: 10 + + - kind: Service + apiVersion: v1 + metadata: + name: maestro + labels: + app: maestro + port: api + spec: + type: ${MAESTRO_SERVER_EXPOSURE} + selector: + app: maestro + ports: + - port: 8000 + targetPort: 8000 + protocol: TCP + + # Services for diagnostic ports (not part of main service because we + # don't want exposing them externally through same route). + - kind: Service + apiVersion: v1 + metadata: + name: maestro-metrics + labels: + app: maestro + port: metrics + spec: + selector: + app: maestro + ports: + - port: 8080 + targetPort: 8080 + name: metrics + + - kind: Service + apiVersion: v1 + metadata: + name: maestro-grpc + labels: + app: maestro-grpc + port: grpc + spec: + selector: + app: maestro + ports: + - port: 8090 + targetPort: 8090 + protocol: TCP + + - apiVersion: v1 + kind: Service + metadata: + name: maestro-healthcheck + labels: + app: maestro + port: healthcheck + spec: + selector: + app: maestro + ports: + - port: 8083 + targetPort: 8083