From 877034ee21ebc346ce0815128c144690dd7db0a7 Mon Sep 17 00:00:00 2001 From: akutz Date: Fri, 18 Oct 2024 11:54:54 -0500 Subject: [PATCH] Distinct webhook deployments This path introduces distinct webhook deployments for the admission and conversion webhooks. This is to provide a more stable and performant experience. The webhook pods will not require nearly as much memory as the controller-manager pods. Additionally, if the webhooks ever go down, it is a bad time. Admission webhooks being unavailable means no one can apply any changes to the VM Op APIs. The conversion webhooks being offline means no one can even *read* the VM Op APIs. --- Makefile | 13 +- config/certmanager/certificate.yaml | 12 +- config/certmanager/kustomization.yaml | 6 +- .../admission_webhook_health_and_metrics.yaml | 23 +++ ...conversion_webhook_health_and_metrics.yaml | 23 +++ config/default/kustomization.yaml | 39 +--- ...h.yaml => manager_health_and_metrics.yaml} | 9 + config/local/kustomization.yaml | 1 + config/manager/kustomization.yaml | 3 + config/manager/manager.yaml | 5 +- config/replacements/kustomization.yaml | 183 +++++++++++++++++- .../web-console-validator/kustomization.yaml | 3 + config/webhook/kustomization.yaml | 14 -- .../admission/deployment.yaml} | 43 ++-- config/webhooks/admission/kustomization.yaml | 79 ++++++++ .../admission}/kustomizeconfig.yaml | 0 .../admission}/manifests.yaml | 0 .../admission}/manifests_label_patch.yaml | 2 +- .../admission}/service.yaml | 6 +- .../storage_quota_webhook_configuration.yaml | 1 + .../admission}/webhookcainjection_patch.yaml | 4 +- config/webhooks/conversion/deployment.yaml | 65 +++++++ config/webhooks/conversion/kustomization.yaml | 40 ++++ config/webhooks/conversion/service.yaml | 13 ++ controllers/infra/controllers.go | 4 - main.go | 26 ++- test/builder/test_suite.go | 7 +- webhooks/webhooks.go | 70 ++++--- 28 files changed, 564 insertions(+), 130 deletions(-) create mode 100644 config/default/admission_webhook_health_and_metrics.yaml create mode 100644 config/default/conversion_webhook_health_and_metrics.yaml rename config/default/{manager_auth_proxy_patch.yaml => manager_health_and_metrics.yaml} (79%) delete mode 100644 config/webhook/kustomization.yaml rename config/{default/manager_webhook_patch.yaml => webhooks/admission/deployment.yaml} (55%) create mode 100644 config/webhooks/admission/kustomization.yaml rename config/{webhook => webhooks/admission}/kustomizeconfig.yaml (100%) rename config/{webhook => webhooks/admission}/manifests.yaml (100%) rename config/{webhook => webhooks/admission}/manifests_label_patch.yaml (88%) rename config/{webhook => webhooks/admission}/service.yaml (56%) rename config/{webhook => webhooks/admission}/storage_quota_webhook_configuration.yaml (99%) rename config/{webhook => webhooks/admission}/webhookcainjection_patch.yaml (70%) create mode 100644 config/webhooks/conversion/deployment.yaml create mode 100644 config/webhooks/conversion/kustomization.yaml create mode 100644 config/webhooks/conversion/service.yaml diff --git a/Makefile b/Makefile index 922667e1d..ff7d4b2c5 100644 --- a/Makefile +++ b/Makefile @@ -85,11 +85,12 @@ GOVULNCHECK := $(TOOLS_BIN_DIR)/govulncheck KIND := $(TOOLS_BIN_DIR)/kind # Allow overriding manifest generation destination directory -MANIFEST_ROOT ?= config -CRD_ROOT ?= $(MANIFEST_ROOT)/crd/bases -EXTERNAL_CRD_ROOT ?= $(MANIFEST_ROOT)/crd/external-crds -WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook -RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac +MANIFEST_ROOT ?= config +CRD_ROOT ?= $(MANIFEST_ROOT)/crd/bases +EXTERNAL_CRD_ROOT ?= $(MANIFEST_ROOT)/crd/external-crds +WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhooks +ADMISSION_WEBHOOK_ROOT ?= $(WEBHOOK_ROOT)/admission +RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac # Image URL to use all building/pushing image targets IMAGE ?= vmoperator-controller @@ -332,7 +333,7 @@ generate-manifests: ## Generate manifests e.g. CRD, RBAC etc. output:none $(CONTROLLER_GEN) \ paths=./webhooks/... \ - output:webhook:dir=$(WEBHOOK_ROOT) \ + output:webhook:dir=$(ADMISSION_WEBHOOK_ROOT) \ webhook $(CONTROLLER_GEN) \ paths=./controllers/... \ diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml index 2c93395ae..b9750faa7 100644 --- a/config/certmanager/certificate.yaml +++ b/config/certmanager/certificate.yaml @@ -1,6 +1,6 @@ +--- # The following manifests contain a self-signed issuer CR and a certificate CR. # More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for breaking changes apiVersion: cert-manager.io/v1 kind: Issuer metadata: @@ -15,11 +15,13 @@ metadata: name: serving-cert namespace: system spec: - # WEBHOOK_SERVICE_NAME_PLACEHOLDER and WEBHOOK_SERVICE_NAMESPACE_PLACEHOLDER will be substituted by kustomize dnsNames: - - WEBHOOK_SERVICE_NAME_PLACEHOLDER.WEBHOOK_SERVICE_NAMESPACE_PLACEHOLDER.svc - - WEBHOOK_SERVICE_NAME_PLACEHOLDER.WEBHOOK_SERVICE_NAMESPACE_PLACEHOLDER.svc.cluster.local + - ADMISSION_WEBHOOK_SERVICE_NAME_PLACEHOLDER.ADMISSION_WEBHOOK_SERVICE_NAMESPACE_PLACEHOLDER.svc + - ADMISSION_WEBHOOK_SERVICE_NAME_PLACEHOLDER.ADMISSION_WEBHOOK_SERVICE_NAMESPACE_PLACEHOLDER.svc.cluster.local + - CONVERSION_WEBHOOK_SERVICE_NAME_PLACEHOLDER.CONVERSION_WEBHOOK_SERVICE_NAMESPACE_PLACEHOLDER.svc + - CONVERSION_WEBHOOK_SERVICE_NAME_PLACEHOLDER.CONVERSION_WEBHOOK_SERVICE_NAMESPACE_PLACEHOLDER.svc.cluster.local issuerRef: kind: Issuer name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize + # This secret will not be prefixed, since it not managed by kustomize. + secretName: webhook-server-cert diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml index 438e93c2c..c3fb2f189 100644 --- a/config/certmanager/kustomization.yaml +++ b/config/certmanager/kustomization.yaml @@ -1,8 +1,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -resources: -- certificate.yaml - configurations: - kustomizeconfig.yaml + +resources: +- certificate.yaml diff --git a/config/default/admission_webhook_health_and_metrics.yaml b/config/default/admission_webhook_health_and_metrics.yaml new file mode 100644 index 000000000..053f679f9 --- /dev/null +++ b/config/default/admission_webhook_health_and_metrics.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admission-webhook + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--enable-leader-election=false" + - "--health-addr=127.0.0.1:9446" + - "--metrics-addr=0" + - "--profiler-address=0" + ports: + - containerPort: 9446 + name: health-probe + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: health-probe diff --git a/config/default/conversion_webhook_health_and_metrics.yaml b/config/default/conversion_webhook_health_and_metrics.yaml new file mode 100644 index 000000000..e18745de6 --- /dev/null +++ b/config/default/conversion_webhook_health_and_metrics.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: conversion-webhook + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--enable-leader-election=false" + - "--health-addr=127.0.0.1:9447" + - "--metrics-addr=0" + - "--profiler-address=0" + ports: + - containerPort: 9447 + name: health-probe + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: health-probe diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 7f6cd3a09..39e6227a7 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -6,48 +6,19 @@ resources: - ../manager - ../web-console-validator - ../rbac -- ../webhook +- ../webhooks/admission +- ../webhooks/conversion - ../certmanager - ../crd/external-crds/encryption.vmware.com_encryptionclasses.yaml patches: - path: manager_default_container_patch.yaml -- path: manager_auth_proxy_patch.yaml -- path: manager_webhook_patch.yaml - path: manager_replicas_patch.yaml - path: manager_pod_info_patch.yaml - path: manager_tolerations_patch.yaml - path: manager_update_strategy_patch.yaml - path: manager_leader_election_id_patch.yaml - path: manager_max_concurrent_reconciles_patch.yaml - -replacements: -- source: - fieldPath: spec.template.spec.containers.[name=manager].ports.[name=webhook-server].containerPort - group: apps - version: v1 - kind: Deployment - namespace: system - name: controller-manager - targets: - - select: - group: apps - version: v1 - kind: Deployment - namespace: system - name: controller-manager - fieldPaths: - - spec.template.spec.containers.[name=manager].env.[name=WEBHOOK_SERVICE_CONTAINER_PORT].value -- source: - fieldPath: spec.template.spec.containers.[name=manager].volumeMounts.[name=cert].mountPath - group: apps - version: v1 - kind: Deployment - namespace: system - name: controller-manager - targets: - - select: - kind: Deployment - name: controller-manager - fieldPaths: - - spec.template.spec.containers.[name=manager].env.[name=WEBHOOK_SECRET_VOLUME_MOUNT_PATH].value +- path: manager_health_and_metrics.yaml +- path: admission_webhook_health_and_metrics.yaml +- path: conversion_webhook_health_and_metrics.yaml diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_health_and_metrics.yaml similarity index 79% rename from config/default/manager_auth_proxy_patch.yaml rename to config/default/manager_health_and_metrics.yaml index c45166a11..d7aa3cc6e 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_health_and_metrics.yaml @@ -12,6 +12,15 @@ spec: - name: manager args: - "--metrics-addr=127.0.0.1:8080" + - "--health-addr=127.0.0.1:9445" + ports: + - containerPort: 9445 + name: health-probe + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: health-probe - name: kube-rbac-proxy image: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.0 args: diff --git a/config/local/kustomization.yaml b/config/local/kustomization.yaml index f1c80ef36..12e4997e5 100644 --- a/config/local/kustomization.yaml +++ b/config/local/kustomization.yaml @@ -1,5 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization + resources: - ./vmoperator/ diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 5c5f0b84c..dfce22e6c 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,2 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + resources: - manager.yaml diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index ed56fb8e5..23ad8d02a 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -41,6 +41,9 @@ spec: requests: cpu: 100m memory: 75Mi + env: + - name: VMOP_CONTAINER_TYPE + value: controller-manager nodeSelector: node-role.kubernetes.io/control-plane: "" - terminationGracePeriodSeconds: 10 + terminationGracePeriodSeconds: 10 \ No newline at end of file diff --git a/config/replacements/kustomization.yaml b/config/replacements/kustomization.yaml index 8ba5a5812..7b2441b97 100644 --- a/config/replacements/kustomization.yaml +++ b/config/replacements/kustomization.yaml @@ -23,13 +23,13 @@ replacements: namespace: system name: controller-manager -# WEBHOOK_SERVICE_NAME +# WEBHOOK_SERVICE_NAME (Admission) - source: fieldPath: metadata.name version: v1 kind: Service namespace: system - name: webhook-service + name: admission-webhook-service targets: - fieldPaths: - spec.template.spec.containers.[name=manager].env.[name=WEBHOOK_SERVICE_NAME].value @@ -38,7 +38,7 @@ replacements: version: v1 kind: Deployment namespace: system - name: controller-manager + name: admission-webhook - fieldPaths: - spec.dnsNames.0 options: @@ -62,13 +62,13 @@ replacements: namespace: system name: serving-cert -# WEBHOOK_SERVICE_NAMESPACE +# WEBHOOK_SERVICE_NAMESPACE (Admission) - source: fieldPath: metadata.namespace version: v1 kind: Service namespace: system - name: webhook-service + name: admission-webhook-service targets: - fieldPaths: - spec.template.spec.containers.[name=manager].env.[name=WEBHOOK_SERVICE_NAMESPACE].value @@ -77,7 +77,7 @@ replacements: version: v1 kind: Deployment namespace: system - name: controller-manager + name: admission-webhook - fieldPaths: - spec.dnsNames.0 options: @@ -101,6 +101,84 @@ replacements: namespace: system name: serving-cert +# WEBHOOK_SERVICE_NAME (Conversion) +- source: + fieldPath: metadata.name + version: v1 + kind: Service + namespace: system + name: conversion-webhook-service + targets: + - fieldPaths: + - spec.template.spec.containers.[name=manager].env.[name=WEBHOOK_SERVICE_NAME].value + select: + group: apps + version: v1 + kind: Deployment + namespace: system + name: conversion-webhook + - fieldPaths: + - spec.dnsNames.2 + options: + delimiter: . + index: 0 + select: + version: v1 + group: cert-manager.io + kind: Certificate + namespace: system + name: serving-cert + - fieldPaths: + - spec.dnsNames.3 + options: + delimiter: . + index: 0 + select: + version: v1 + group: cert-manager.io + kind: Certificate + namespace: system + name: serving-cert + +# WEBHOOK_SERVICE_NAMESPACE (Conversion) +- source: + fieldPath: metadata.namespace + version: v1 + kind: Service + namespace: system + name: conversion-webhook-service + targets: + - fieldPaths: + - spec.template.spec.containers.[name=manager].env.[name=WEBHOOK_SERVICE_NAMESPACE].value + select: + group: apps + version: v1 + kind: Deployment + namespace: system + name: conversion-webhook + - fieldPaths: + - spec.dnsNames.2 + options: + delimiter: . + index: 1 + select: + version: v1 + group: cert-manager.io + kind: Certificate + namespace: system + name: serving-cert + - fieldPaths: + - spec.dnsNames.3 + options: + delimiter: . + index: 1 + select: + version: v1 + group: cert-manager.io + kind: Certificate + namespace: system + name: serving-cert + # WEBHOOK_CERTIFICATE_NAME - source: fieldPath: metadata.name @@ -219,7 +297,16 @@ replacements: version: v1 kind: Deployment namespace: system - name: controller-manager + name: admission-webhook + - fieldPaths: + - spec.template.spec.containers.[name=manager].env.[name=WEBHOOK_SECRET_NAME].value + - spec.template.spec.volumes.[name=cert].secret.secretName + select: + group: apps + version: v1 + kind: Deployment + namespace: system + name: conversion-webhook # WEBHOOK_CERTIFICATE_NAMESPACE @@ -239,7 +326,15 @@ replacements: version: v1 kind: Deployment namespace: system - name: controller-manager + name: admission-webhook + - fieldPaths: + - spec.template.spec.containers.[name=manager].env.[name=WEBHOOK_SECRET_NAMESPACE].value + select: + group: apps + version: v1 + kind: Deployment + namespace: system + name: conversion-webhook - fieldPaths: - metadata.annotations.[cert-manager.io/inject-ca-from] options: @@ -330,3 +425,75 @@ replacements: version: v1 kind: CustomResourceDefinition name: virtualmachinesetresourcepolicies.vmoperator.vmware.com + +# Update the CRDs with the name of the conversion webhook service. +- source: + fieldPath: metadata.name + version: v1 + kind: Service + namespace: system + name: conversion-webhook-service + targets: + - select: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: clustervirtualmachineimages.vmoperator.vmware.com + fieldPaths: + - spec.conversion.webhook.clientConfig.service.name + options: + create: true + - select: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: virtualmachineclasses.vmoperator.vmware.com + fieldPaths: + - spec.conversion.webhook.clientConfig.service.name + options: + create: true + - select: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: virtualmachineimages.vmoperator.vmware.com + fieldPaths: + - spec.conversion.webhook.clientConfig.service.name + options: + create: true + - select: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: virtualmachinepublishrequests.vmoperator.vmware.com + fieldPaths: + - spec.conversion.webhook.clientConfig.service.name + options: + create: true + - select: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: virtualmachines.vmoperator.vmware.com + fieldPaths: + - spec.conversion.webhook.clientConfig.service.name + options: + create: true + - select: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: virtualmachineservices.vmoperator.vmware.com + fieldPaths: + - spec.conversion.webhook.clientConfig.service.name + options: + create: true + - select: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: virtualmachinesetresourcepolicies.vmoperator.vmware.com + fieldPaths: + - spec.conversion.webhook.clientConfig.service.name + options: + create: true diff --git a/config/web-console-validator/kustomization.yaml b/config/web-console-validator/kustomization.yaml index 9b2933ba6..4a57bf3e7 100644 --- a/config/web-console-validator/kustomization.yaml +++ b/config/web-console-validator/kustomization.yaml @@ -1,2 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + resources: - web_console_validator.yaml diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml deleted file mode 100644 index 412001848..000000000 --- a/config/webhook/kustomization.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -configurations: -- kustomizeconfig.yaml - -resources: -- service.yaml -- manifests.yaml -- storage_quota_webhook_configuration.yaml - -patches: -- path: manifests_label_patch.yaml -- path: webhookcainjection_patch.yaml diff --git a/config/default/manager_webhook_patch.yaml b/config/webhooks/admission/deployment.yaml similarity index 55% rename from config/default/manager_webhook_patch.yaml rename to config/webhooks/admission/deployment.yaml index eceb3a307..1c07f9849 100644 --- a/config/default/manager_webhook_patch.yaml +++ b/config/webhooks/admission/deployment.yaml @@ -1,15 +1,40 @@ +--- apiVersion: apps/v1 kind: Deployment metadata: - name: controller-manager + name: admission-webhook namespace: system + labels: + webhook: admission + controller-tools.k8s.io: "1.0" spec: + selector: + matchLabels: + webhook: admission + controller-tools.k8s.io: "1.0" template: + metadata: + labels: + webhook: admission + controller-tools.k8s.io: "1.0" + name: vmoperator-admission-webhook spec: - # There's an assumption in kustomization.yaml that manager is container[0] containers: - name: manager + command: + - /manager + image: controller:latest + imagePullPolicy: Never + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 100m + memory: 75Mi env: + - name: VMOP_CONTAINER_TYPE + value: admission-webhook - name: WEBHOOK_SERVICE_NAMESPACE value: WEBHOOK_SERVICE_NAMESPACE_PLACEHOLDER - name: WEBHOOK_SERVICE_NAME @@ -23,20 +48,16 @@ spec: - name: WEBHOOK_SERVICE_CONTAINER_PORT value: WEBHOOK_SERVICE_CONTAINER_PORT_STRING ports: - - containerPort: 9878 - name: webhook-server + - name: https + containerPort: 9878 protocol: TCP - - containerPort: 9445 - name: health-probe - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: health-probe volumeMounts: - name: cert mountPath: /tmp/k8s-webhook-server/serving-certs readOnly: true + nodeSelector: + node-role.kubernetes.io/control-plane: "" + terminationGracePeriodSeconds: 10 volumes: - name: cert secret: diff --git a/config/webhooks/admission/kustomization.yaml b/config/webhooks/admission/kustomization.yaml new file mode 100644 index 000000000..143a4ef64 --- /dev/null +++ b/config/webhooks/admission/kustomization.yaml @@ -0,0 +1,79 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +configurations: +- kustomizeconfig.yaml + +resources: +- deployment.yaml +- service.yaml +- manifests.yaml +- storage_quota_webhook_configuration.yaml + +patches: +- path: manifests_label_patch.yaml +- path: webhookcainjection_patch.yaml + +replacements: +- source: + fieldPath: spec.template.spec.containers.[name=manager].ports.[name=https].containerPort + group: apps + version: v1 + kind: Deployment + namespace: system + name: admission-webhook + targets: + - select: + group: apps + version: v1 + kind: Deployment + namespace: system + name: admission-webhook + fieldPaths: + - spec.template.spec.containers.[name=manager].env.[name=WEBHOOK_SERVICE_CONTAINER_PORT].value +- source: + fieldPath: spec.template.spec.containers.[name=manager].volumeMounts.[name=cert].mountPath + group: apps + version: v1 + kind: Deployment + namespace: system + name: admission-webhook + targets: + - select: + group: apps + version: v1 + kind: Deployment + namespace: system + name: admission-webhook + fieldPaths: + - spec.template.spec.containers.[name=manager].env.[name=WEBHOOK_SECRET_VOLUME_MOUNT_PATH].value + +# Update the name of the service in the Mutating and Validating webhook configs. +- source: + fieldPath: metadata.name + version: v1 + kind: Service + namespace: system + name: admission-webhook-service + targets: + - select: + group: admissionregistration.k8s.io + version: v1 + kind: MutatingWebhookConfiguration + name: mutating-webhook-configuration + fieldPaths: + - webhooks.*.clientConfig.service.name +- source: + fieldPath: metadata.name + version: v1 + kind: Service + namespace: system + name: admission-webhook-service + targets: + - select: + group: admissionregistration.k8s.io + version: v1 + kind: ValidatingWebhookConfiguration + name: validating-webhook-configuration + fieldPaths: + - webhooks.*.clientConfig.service.name \ No newline at end of file diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhooks/admission/kustomizeconfig.yaml similarity index 100% rename from config/webhook/kustomizeconfig.yaml rename to config/webhooks/admission/kustomizeconfig.yaml diff --git a/config/webhook/manifests.yaml b/config/webhooks/admission/manifests.yaml similarity index 100% rename from config/webhook/manifests.yaml rename to config/webhooks/admission/manifests.yaml diff --git a/config/webhook/manifests_label_patch.yaml b/config/webhooks/admission/manifests_label_patch.yaml similarity index 88% rename from config/webhook/manifests_label_patch.yaml rename to config/webhooks/admission/manifests_label_patch.yaml index 5dfbad64d..b5f8ef2d8 100644 --- a/config/webhook/manifests_label_patch.yaml +++ b/config/webhooks/admission/manifests_label_patch.yaml @@ -12,4 +12,4 @@ kind: MutatingWebhookConfiguration metadata: name: mutating-webhook-configuration labels: - "webhooks.vmoperator.vmware.com": "true" + webhooks.vmoperator.vmware.com: "true" diff --git a/config/webhook/service.yaml b/config/webhooks/admission/service.yaml similarity index 56% rename from config/webhook/service.yaml rename to config/webhooks/admission/service.yaml index a1a58e048..b964f11b9 100644 --- a/config/webhook/service.yaml +++ b/config/webhooks/admission/service.yaml @@ -2,12 +2,12 @@ apiVersion: v1 kind: Service metadata: - name: webhook-service + name: admission-webhook-service namespace: system spec: ports: - name: https port: 443 - targetPort: webhook-server + targetPort: https selector: - control-plane: controller-manager + webhook: admission diff --git a/config/webhook/storage_quota_webhook_configuration.yaml b/config/webhooks/admission/storage_quota_webhook_configuration.yaml similarity index 99% rename from config/webhook/storage_quota_webhook_configuration.yaml rename to config/webhooks/admission/storage_quota_webhook_configuration.yaml index 1f2175ac1..8eedd6217 100644 --- a/config/webhook/storage_quota_webhook_configuration.yaml +++ b/config/webhooks/admission/storage_quota_webhook_configuration.yaml @@ -1,3 +1,4 @@ +--- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: diff --git a/config/webhook/webhookcainjection_patch.yaml b/config/webhooks/admission/webhookcainjection_patch.yaml similarity index 70% rename from config/webhook/webhookcainjection_patch.yaml rename to config/webhooks/admission/webhookcainjection_patch.yaml index 85a58d967..3c16653e0 100644 --- a/config/webhook/webhookcainjection_patch.yaml +++ b/config/webhooks/admission/webhookcainjection_patch.yaml @@ -1,6 +1,4 @@ -# This patch add annotation to admission webhook config and -# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. -# uncomment the following lines to enable mutating webhook +--- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: diff --git a/config/webhooks/conversion/deployment.yaml b/config/webhooks/conversion/deployment.yaml new file mode 100644 index 000000000..ceefb29e1 --- /dev/null +++ b/config/webhooks/conversion/deployment.yaml @@ -0,0 +1,65 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: conversion-webhook + namespace: system + labels: + webhook: conversion + controller-tools.k8s.io: "1.0" +spec: + selector: + matchLabels: + webhook: conversion + controller-tools.k8s.io: "1.0" + template: + metadata: + labels: + webhook: conversion + controller-tools.k8s.io: "1.0" + name: vmoperator-conversion-webhook + spec: + containers: + - name: manager + command: + - /manager + image: controller:latest + imagePullPolicy: Never + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 100m + memory: 75Mi + env: + - name: VMOP_CONTAINER_TYPE + value: conversion-webhook + - name: WEBHOOK_SERVICE_NAMESPACE + value: WEBHOOK_SERVICE_NAMESPACE_PLACEHOLDER + - name: WEBHOOK_SERVICE_NAME + value: WEBHOOK_SERVICE_NAME_PLACEHOLDER + - name: WEBHOOK_SECRET_NAMESPACE + value: WEBHOOK_SECRET_NAMESPACE_PLACEHOLDER + - name: WEBHOOK_SECRET_NAME + value: WEBHOOK_SECRET_NAME_PLACEHOLDER + - name: WEBHOOK_SECRET_VOLUME_MOUNT_PATH + value: WEBHOOK_SECRET_VOLUME_MOUNT_PATH_VALUE + - name: WEBHOOK_SERVICE_CONTAINER_PORT + value: WEBHOOK_SERVICE_CONTAINER_PORT_STRING + ports: + - name: https + containerPort: 9879 + protocol: TCP + volumeMounts: + - name: cert + mountPath: /tmp/k8s-webhook-server/serving-certs + readOnly: true + nodeSelector: + node-role.kubernetes.io/control-plane: "" + terminationGracePeriodSeconds: 10 + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: WEBHOOK_SECRET_NAME_PLACEHOLDER diff --git a/config/webhooks/conversion/kustomization.yaml b/config/webhooks/conversion/kustomization.yaml new file mode 100644 index 000000000..76a5097ea --- /dev/null +++ b/config/webhooks/conversion/kustomization.yaml @@ -0,0 +1,40 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- deployment.yaml +- service.yaml + +replacements: +- source: + fieldPath: spec.template.spec.containers.[name=manager].ports.[name=https].containerPort + group: apps + version: v1 + kind: Deployment + namespace: system + name: conversion-webhook + targets: + - select: + group: apps + version: v1 + kind: Deployment + namespace: system + name: conversion-webhook + fieldPaths: + - spec.template.spec.containers.[name=manager].env.[name=WEBHOOK_SERVICE_CONTAINER_PORT].value +- source: + fieldPath: spec.template.spec.containers.[name=manager].volumeMounts.[name=cert].mountPath + group: apps + version: v1 + kind: Deployment + namespace: system + name: conversion-webhook + targets: + - select: + group: apps + version: v1 + kind: Deployment + namespace: system + name: conversion-webhook + fieldPaths: + - spec.template.spec.containers.[name=manager].env.[name=WEBHOOK_SECRET_VOLUME_MOUNT_PATH].value diff --git a/config/webhooks/conversion/service.yaml b/config/webhooks/conversion/service.yaml new file mode 100644 index 000000000..010ab720b --- /dev/null +++ b/config/webhooks/conversion/service.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: conversion-webhook-service + namespace: system +spec: + ports: + - name: https + port: 443 + targetPort: https + selector: + webhook: conversion diff --git a/controllers/infra/controllers.go b/controllers/infra/controllers.go index b2d471e97..b36dd44f8 100644 --- a/controllers/infra/controllers.go +++ b/controllers/infra/controllers.go @@ -8,7 +8,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" - "github.com/vmware-tanzu/vm-operator/controllers/infra/capability" "github.com/vmware-tanzu/vm-operator/controllers/infra/configmap" "github.com/vmware-tanzu/vm-operator/controllers/infra/node" "github.com/vmware-tanzu/vm-operator/controllers/infra/secret" @@ -20,9 +19,6 @@ import ( // AddToManager adds the controllers to the provided manager. func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) error { - if err := capability.AddToManager(ctx, mgr); err != nil { - return fmt.Errorf("failed to initialize infra capability controller: %w", err) - } if err := configmap.AddToManager(ctx, mgr); err != nil { return fmt.Errorf("failed to initialize infra configmap controller: %w", err) } diff --git a/main.go b/main.go index 66d728593..335f1919a 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "context" "crypto/tls" "flag" + "fmt" "os" "path" "time" @@ -25,6 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/vmware-tanzu/vm-operator/controllers" + "github.com/vmware-tanzu/vm-operator/controllers/infra/capability" "github.com/vmware-tanzu/vm-operator/pkg" pkgcfg "github.com/vmware-tanzu/vm-operator/pkg/config" "github.com/vmware-tanzu/vm-operator/pkg/config/capabilities" @@ -195,15 +197,29 @@ func main() { ctx *pkgctx.ControllerManagerContext, mgr ctrlmgr.Manager) error { - if err := controllers.AddToManager(ctx, mgr); err != nil { - return err + // Always load the capability controller. + if err := capability.AddToManager(ctx, mgr); err != nil { + return fmt.Errorf("failed to initialize infra capability controller: %w", err) } - if pkgcfg.FromContext(ctx).Features.WorkloadDomainIsolation { - if err := services.AddToManager(ctx, mgr); err != nil { + + containerType := os.Getenv("VMOP_CONTAINER_TYPE") + switch containerType { + case "controller-manager": + if err := controllers.AddToManager(ctx, mgr); err != nil { return err } + if pkgcfg.FromContext(ctx).Features.WorkloadDomainIsolation { + if err := services.AddToManager(ctx, mgr); err != nil { + return err + } + } + case "admission-webhook", "conversion-webhook": + return webhooks.AddToManager(ctx, mgr) + default: + return fmt.Errorf("invalid container type: %s", containerType) } - return webhooks.AddToManager(ctx, mgr) + + return nil } setupLog.Info("creating controller manager") diff --git a/test/builder/test_suite.go b/test/builder/test_suite.go index 63561c3a8..af78fdf12 100644 --- a/test/builder/test_suite.go +++ b/test/builder/test_suite.go @@ -466,7 +466,12 @@ func (s *TestSuite) postConfigureManager() { if s.isWebhookTest() { By("installing the webhook(s)", func() { // ASSERT that the file for validating webhook file exists. - validatingWebhookFile := path.Join(testutil.GetRootDirOrDie(), "config", "webhook", "manifests.yaml") + validatingWebhookFile := path.Join( + testutil.GetRootDirOrDie(), + "config", + "webhooks", + "admission", + "manifests.yaml") Expect(validatingWebhookFile).Should(BeAnExistingFile()) // UNMARSHAL the contents of the validating webhook file into MutatingWebhookConfiguration and diff --git a/webhooks/webhooks.go b/webhooks/webhooks.go index 5720ed265..3c9e756ba 100644 --- a/webhooks/webhooks.go +++ b/webhooks/webhooks.go @@ -5,6 +5,7 @@ package webhooks import ( "fmt" + "os" ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" @@ -24,41 +25,48 @@ import ( // AddToManager adds all webhooks and a certificate manager to the provided controller manager. func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr ctrlmgr.Manager) error { - if err := conversion.AddToManager(ctx, mgr); err != nil { - return fmt.Errorf("failed to initialize conversion webhooks: %w", err) - } - if err := persistentvolumeclaim.AddToManager(ctx, mgr); err != nil { - return fmt.Errorf("failed to initialize PersistentVolumeClaim webhook: %w", err) - } - if err := virtualmachine.AddToManager(ctx, mgr); err != nil { - return fmt.Errorf("failed to initialize VirtualMachine webhooks: %w", err) - } - if err := virtualmachineclass.AddToManager(ctx, mgr); err != nil { - return fmt.Errorf("failed to initialize VirtualMachineClass webhooks: %w", err) - } - if err := virtualmachinepublishrequest.AddToManager(ctx, mgr); err != nil { - return fmt.Errorf("failed to initialize VirtualMachinePublishRequest webhooks: %w", err) - } - if err := virtualmachineservice.AddToManager(ctx, mgr); err != nil { - return fmt.Errorf("failed to initialize VirtualMachineService webhooks: %w", err) - } - if err := virtualmachinesetresourcepolicy.AddToManager(ctx, mgr); err != nil { - return fmt.Errorf("failed to initialize VirtualMachineSetResourcePolicy webhooks: %w", err) - } - if err := virtualmachinewebconsolerequest.AddToManager(ctx, mgr); err != nil { - return fmt.Errorf("failed to initialize VirtualMachineWebConsoleRequest webhooks: %w", err) - } + containerType := os.Getenv("VMOP_CONTAINER_TYPE") + switch containerType { + case "admission-webhook": + if err := persistentvolumeclaim.AddToManager(ctx, mgr); err != nil { + return fmt.Errorf("failed to initialize PersistentVolumeClaim webhook: %w", err) + } + if err := virtualmachine.AddToManager(ctx, mgr); err != nil { + return fmt.Errorf("failed to initialize VirtualMachine webhooks: %w", err) + } + if err := virtualmachineclass.AddToManager(ctx, mgr); err != nil { + return fmt.Errorf("failed to initialize VirtualMachineClass webhooks: %w", err) + } + if err := virtualmachinepublishrequest.AddToManager(ctx, mgr); err != nil { + return fmt.Errorf("failed to initialize VirtualMachinePublishRequest webhooks: %w", err) + } + if err := virtualmachineservice.AddToManager(ctx, mgr); err != nil { + return fmt.Errorf("failed to initialize VirtualMachineService webhooks: %w", err) + } + if err := virtualmachinesetresourcepolicy.AddToManager(ctx, mgr); err != nil { + return fmt.Errorf("failed to initialize VirtualMachineSetResourcePolicy webhooks: %w", err) + } + if err := virtualmachinewebconsolerequest.AddToManager(ctx, mgr); err != nil { + return fmt.Errorf("failed to initialize VirtualMachineWebConsoleRequest webhooks: %w", err) + } - if pkgcfg.FromContext(ctx).Features.K8sWorkloadMgmtAPI { - if err := virtualmachinereplicaset.AddToManager(ctx, mgr); err != nil { - return fmt.Errorf("failed to initialize VirtualMachineReplicaSet webhooks: %w", err) + if pkgcfg.FromContext(ctx).Features.K8sWorkloadMgmtAPI { + if err := virtualmachinereplicaset.AddToManager(ctx, mgr); err != nil { + return fmt.Errorf("failed to initialize VirtualMachineReplicaSet webhooks: %w", err) + } } - } - if pkgcfg.FromContext(ctx).Features.UnifiedStorageQuota { - if err := unifiedstoragequota.AddToManager(ctx, mgr); err != nil { - return fmt.Errorf("failed to initialize UnifiedStorageQuota webhooks: %w", err) + if pkgcfg.FromContext(ctx).Features.UnifiedStorageQuota { + if err := unifiedstoragequota.AddToManager(ctx, mgr); err != nil { + return fmt.Errorf("failed to initialize UnifiedStorageQuota webhooks: %w", err) + } + } + case "conversion-webhook": + if err := conversion.AddToManager(ctx, mgr); err != nil { + return fmt.Errorf("failed to initialize conversion webhooks: %w", err) } + default: + return fmt.Errorf("invalid container type: %s", containerType) } return nil