Distinct webhook deployments

This path introduces distinct webhook deployments for the
admission and conversion webhooks. This is to provide a more
stable and performant experience. The webhook pods will not require
nearly as much memory as the controller-manager pods. Additionally,
if the webhooks ever go down, it is a bad time. Admission webhooks
being unavailable means no one can apply any changes to the VM Op
APIs. The conversion webhooks being offline means no one can even
*read* the VM Op APIs.

This patch also removes the vestigial use of kustomizeconfig
files since they are no longer needed, having been replaced by, well,
the replacements feature.

Makefile

@@ -85,11 +85,12 @@ GOVULNCHECK        := $(TOOLS_BIN_DIR)/govulncheck
 KIND               := $(TOOLS_BIN_DIR)/kind
 
 # Allow overriding manifest generation destination directory
-MANIFEST_ROOT     ?= config
-CRD_ROOT          ?= $(MANIFEST_ROOT)/crd/bases
-EXTERNAL_CRD_ROOT ?= $(MANIFEST_ROOT)/crd/external-crds
-WEBHOOK_ROOT      ?= $(MANIFEST_ROOT)/webhook
-RBAC_ROOT         ?= $(MANIFEST_ROOT)/rbac
+MANIFEST_ROOT          ?= config
+CRD_ROOT               ?= $(MANIFEST_ROOT)/crd/bases
+EXTERNAL_CRD_ROOT      ?= $(MANIFEST_ROOT)/crd/external-crds
+WEBHOOK_ROOT           ?= $(MANIFEST_ROOT)/webhooks
+ADMISSION_WEBHOOK_ROOT ?= $(WEBHOOK_ROOT)/admission
+RBAC_ROOT              ?= $(MANIFEST_ROOT)/rbac
 
 # Image URL to use all building/pushing image targets
 IMAGE ?= vmoperator-controller
@@ -332,7 +333,7 @@ generate-manifests: ## Generate manifests e.g. CRD, RBAC etc.
 		output:none
 	$(CONTROLLER_GEN) \
 		paths=./webhooks/... \
-		output:webhook:dir=$(WEBHOOK_ROOT) \
+		output:webhook:dir=$(ADMISSION_WEBHOOK_ROOT) \
 		webhook
 	$(CONTROLLER_GEN) \
 		paths=./controllers/... \

config/certmanager/certificate.yaml

@@ -1,6 +1,6 @@
+---
 # The following manifests contain a self-signed issuer CR and a certificate CR.
 # More document can be found at https://docs.cert-manager.io
-# WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for breaking changes
 apiVersion: cert-manager.io/v1
 kind: Issuer
 metadata:
@@ -12,14 +12,16 @@ spec:
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
-  name: serving-cert  # this name should match the one appeared in kustomizeconfig.yaml
+  name: serving-cert
   namespace: system
 spec:
-  # WEBHOOK_SERVICE_NAME_PLACEHOLDER and WEBHOOK_SERVICE_NAMESPACE_PLACEHOLDER will be substituted by kustomize
   dnsNames:
-  - WEBHOOK_SERVICE_NAME_PLACEHOLDER.WEBHOOK_SERVICE_NAMESPACE_PLACEHOLDER.svc
-  - WEBHOOK_SERVICE_NAME_PLACEHOLDER.WEBHOOK_SERVICE_NAMESPACE_PLACEHOLDER.svc.cluster.local
+  - ADMISSION_WEBHOOK_SERVICE_NAME_PLACEHOLDER.ADMISSION_WEBHOOK_SERVICE_NAMESPACE_PLACEHOLDER.svc
+  - ADMISSION_WEBHOOK_SERVICE_NAME_PLACEHOLDER.ADMISSION_WEBHOOK_SERVICE_NAMESPACE_PLACEHOLDER.svc.cluster.local
+  - CONVERSION_WEBHOOK_SERVICE_NAME_PLACEHOLDER.CONVERSION_WEBHOOK_SERVICE_NAMESPACE_PLACEHOLDER.svc
+  - CONVERSION_WEBHOOK_SERVICE_NAME_PLACEHOLDER.CONVERSION_WEBHOOK_SERVICE_NAMESPACE_PLACEHOLDER.svc.cluster.local
   issuerRef:
     kind: Issuer
     name: selfsigned-issuer
-  secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
+  # This secret will not be prefixed, since it not managed by kustomize.
+  secretName: webhook-server-cert

config/certmanager/kustomization.yaml

@@ -2,6 +2,3 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
 - certificate.yaml
-
-configurations:
-- kustomizeconfig.yaml

config/crd/kustomization.yaml

@@ -1,9 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-configurations:
-- kustomizeconfig.yaml
-
 resources:
 - bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml
 - bases/vmoperator.vmware.com_contentsources.yaml

config/default/admission_webhook_health_and_metrics.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: admission-webhook
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        args:
+        - "--enable-leader-election=false"
+        - "--health-addr=127.0.0.1:9446"
+        - "--metrics-addr=0"
+        - "--profiler-address=0"
+        ports:
+        - containerPort: 9446
+          name: health-probe
+          protocol: TCP
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: health-probe

config/default/conversion_webhook_health_and_metrics.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: conversion-webhook
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        args:
+        - "--enable-leader-election=false"
+        - "--health-addr=127.0.0.1:9447"
+        - "--metrics-addr=0"
+        - "--profiler-address=0"
+        ports:
+        - containerPort: 9447
+          name: health-probe
+          protocol: TCP
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: health-probe

config/default/kustomization.yaml

@@ -1,56 +1,24 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-configurations:
-- kustomizeconfig.yaml
-
 resources:
 - ../crd
 - ../manager
 - ../web-console-validator
 - ../rbac
-- ../webhook
+- ../webhooks/admission
+- ../webhooks/conversion
 - ../certmanager
 - ../crd/external-crds/encryption.vmware.com_encryptionclasses.yaml
 
 patches:
 - path: manager_default_container_patch.yaml
-- path: manager_auth_proxy_patch.yaml
-- path: manager_webhook_patch.yaml
 - path: manager_replicas_patch.yaml
 - path: manager_pod_info_patch.yaml
 - path: manager_tolerations_patch.yaml
 - path: manager_update_strategy_patch.yaml
 - path: manager_leader_election_id_patch.yaml
 - path: manager_max_concurrent_reconciles_patch.yaml
-
-replacements:
-- source:
-    fieldPath: spec.template.spec.containers.[name=manager].ports.[name=webhook-server].containerPort
-    group: apps
-    version: v1
-    kind: Deployment
-    namespace: system
-    name: controller-manager
-  targets:
-  - select:
-      group: apps
-      version: v1
-      kind: Deployment
-      namespace: system
-      name: controller-manager
-    fieldPaths:
-    - spec.template.spec.containers.[name=manager].env.[name=WEBHOOK_SERVICE_CONTAINER_PORT].value
-- source:
-    fieldPath: spec.template.spec.containers.[name=manager].volumeMounts.[name=cert].mountPath
-    group: apps
-    version: v1
-    kind: Deployment
-    namespace: system
-    name: controller-manager
-  targets:
-  - select:
-      kind: Deployment
-      name: controller-manager
-    fieldPaths:
-    - spec.template.spec.containers.[name=manager].env.[name=WEBHOOK_SECRET_VOLUME_MOUNT_PATH].value
+- path: manager_health_and_metrics.yaml
+- path: admission_webhook_health_and_metrics.yaml
+- path: conversion_webhook_health_and_metrics.yaml

config/default/manager_auth_proxy_patch.yaml → config/default/manager_health_and_metrics.yaml

@@ -12,6 +12,15 @@ spec:
       - name: manager
         args:
         - "--metrics-addr=127.0.0.1:8080"
+        - "--health-addr=127.0.0.1:9445"
+        ports:
+        - containerPort: 9445
+          name: health-probe
+          protocol: TCP
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: health-probe
       - name: kube-rbac-proxy
         image: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.0
         args:

config/local/kustomization.yaml

@@ -1,5 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
+
 resources:
 - ./vmoperator/
 

config/manager/kustomization.yaml

@@ -1,2 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
 resources:
 - manager.yaml

config/manager/manager.yaml

@@ -41,6 +41,9 @@ spec:
           requests:
             cpu: 100m
             memory: 75Mi
+        env:
+        - name: VMOP_CONTAINER_TYPE
+          value: controller-manager
       nodeSelector:
         node-role.kubernetes.io/control-plane: ""
-      terminationGracePeriodSeconds: 10
+      terminationGracePeriodSeconds: 10