refactor: sidecar injector (#141)

Signed-off-by: Charles-Edouard Brétéché <[email protected]>
kyverno · Oct 25, 2024 · 6096cc5 · 6096cc5
1 parent 5a72331
commit 6096cc5
Show file tree

Hide file tree

Showing 21 changed files with 318 additions and 359 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -86,16 +86,16 @@ jobs:
         with:
           go-version-file: go.mod
           cache-dependency-path: go.sum
-      - name: Run tests
-        run: |
-          set -e
-          make kind-create-cluster
-          make kind-load-taged-image
       - name: Install Cosign
         uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da # v3.7.0
       - name: Install chainsaw
         uses: kyverno/action-install-chainsaw@d311eacde764f806c9658574ff64c9c3b21f8397 # v0.2.11
         with:
           verify: true
+      - name: Setup test environment
+        run: |
+          set -e
+          make kind-create-cluster
+          make chart-install
       - name: Run Chainsaw Tests
         run: chainsaw test tests/e2e-test
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -19,7 +19,6 @@
             "program": "${workspaceFolder}",
             "args": [
                 "sidecar-injector",
-                "--local"
             ],
         }
     ]

diff --git a/charts/kyverno-envoy-plugin/templates/_helpers/_deployment.tpl b/charts/kyverno-envoy-plugin/templates/_helpers/_deployment.tpl
@@ -0,0 +1,10 @@
+{{/* vim: set filetype=mustache: */}}
+
+{{- define "kyverno.deployment.replicas" -}}
+  {{- if and (not (kindIs "invalid" .)) (not (kindIs "string" .)) -}}
+  {{- if eq (int .) 0 -}}
+    {{- fail "Kyverno does not support running with 0 replicas. Please provide a non-zero integer value." -}}
+  {{- end -}}
+  {{- end -}}
+  {{- . -}}
+{{- end -}}
diff --git a/charts/kyverno-envoy-plugin/templates/sidecar-injector/certificates.yaml b/charts/kyverno-envoy-plugin/templates/sidecar-injector/certificates.yaml
@@ -1,32 +1,56 @@
 {{- if .Values.sidecarInjector.enabled -}}
-{{- if .Values.sidecarInjector.certificates.selfSigned -}}
 {{- $ca := genCA (printf "*.%s.svc" (include "kyverno.namespace" .)) 1024 -}}
 {{- $svcName := (printf "%s.%s.svc" (include "kyverno.sidecar-injector.name" .) (include "kyverno.namespace" .)) -}}
-{{- $cert := genSignedCert $svcName nil (list $svcName) 1024 $ca -}}
+{{- $tls := genSignedCert $svcName nil (list $svcName) 1024 $ca -}}
+{{- if .Values.sidecarInjector.certificates.selfSigned -}}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ template "kyverno.sidecar-injector.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-ca
+  name: {{ template "kyverno.sidecar-injector.name" . }}
   namespace: {{ template "kyverno.namespace" . }}
   labels:
     {{- include "kyverno.sidecar-injector.labels" . | nindent 4 }}
+  annotations:
+    self-signed-cert: "true"
 type: kubernetes.io/tls
 data:
-  tls.key: {{ $ca.Key | b64enc }}
-  tls.crt: {{ $ca.Cert | b64enc }}
+  tls.key: {{ $tls.Key | b64enc }}
+  tls.crt: {{ $tls.Cert | b64enc }}
+  ca.crt: {{ $ca.Cert | b64enc }}
+{{- end }}
 ---
-apiVersion: v1
-kind: Secret
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
 metadata:
-  name: {{ template "kyverno.sidecar-injector.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-pair
-  namespace: {{ template "kyverno.namespace" . }}
+  name: {{ template "kyverno.sidecar-injector.name" . }}
   labels:
     {{- include "kyverno.sidecar-injector.labels" . | nindent 4 }}
-  annotations:
-    self-signed-cert: "true"
-type: kubernetes.io/tls
-data:
-  tls.key: {{ $cert.Key | b64enc }}
-  tls.crt: {{ $cert.Cert | b64enc }}
-{{- end -}}
+webhooks:
+  - name: kyverno-envoy-sidecar.kyverno-envoy-sidecar-injector.svc
+    clientConfig:
+      service:
+        name: {{ template "kyverno.sidecar-injector.name" . }}
+        namespace: {{ template "kyverno.namespace" . }}
+        path: "/mutate"
+      caBundle: {{ $ca.Cert | b64enc }}
+    failurePolicy: Fail
+    sideEffects: None
+    admissionReviewVersions:
+    - v1
+    rules:
+      - apiGroups:
+          - ''
+        apiVersions:
+          - v1
+        resources:
+          - pods
+        operations:
+          - CREATE
+        scope: '*'  
+    objectSelector:
+      matchExpressions:
+        - key: kyverno-envoy-sidecar/injection
+          operator: In
+          values:
+          - enabled
 {{- end -}}
diff --git a/charts/kyverno-envoy-plugin/templates/sidecar-injector/deployment.yaml b/charts/kyverno-envoy-plugin/templates/sidecar-injector/deployment.yaml
@@ -74,6 +74,10 @@ spec:
           {{- tpl (toYaml .) $ | nindent 10 }}
         {{- end }}
       serviceAccountName: {{ template "kyverno.sidecar-injector.service-account.name" . }}
+      volumes:
+        - name: certs
+          secret:
+            secretName: {{ template "kyverno.sidecar-injector.name" . }}
       containers:
         {{- with .Values.sidecarInjector.containers.injector }}
         - name: injector
@@ -107,5 +111,9 @@ spec:
           args:
             {{- tpl (toYaml .) $ | nindent 12 }}
           {{- end }}
+          volumeMounts:
+            - name: certs
+              mountPath: /opt/kubernetes-sidecar-injector/certs
+              readOnly: true
         {{- end }}
 {{- end -}}
diff --git a/charts/kyverno-envoy-plugin/templates/sidecar-injector/mutating-webhook-configuration.yaml b/charts/kyverno-envoy-plugin/templates/sidecar-injector/mutating-webhook-configuration.yaml
diff --git a/charts/kyverno-envoy-plugin/templates/sidecar-injector/service.yaml b/charts/kyverno-envoy-plugin/templates/sidecar-injector/service.yaml
@@ -6,13 +6,21 @@ metadata:
   namespace: {{ template "kyverno.namespace" . }}
   labels:
     {{- include "kyverno.sidecar-injector.labels" . | nindent 4 }}
+  {{- with .Values.sidecarInjector.service.annotations }}
+  annotations:
+    {{- tpl (toYaml .) $ | nindent 4 }}
+  {{- end }}
 spec:
-  type: ClusterIP
+  type: {{ .Values.sidecarInjector.service.type }}
   ports:
   - name: https
+    port: {{ .Values.sidecarInjector.service.port }}
     protocol: TCP
-    port: 443
-    targetPort: 8443
+    appProtocol: https
+    targetPort: https
+    {{- if and (eq .Values.sidecarInjector.service.type "NodePort") (not (empty .Values.sidecarInjector.service.nodePort)) }}
+    nodePort: {{ .Values.sidecarInjector.service.nodePort }}
+    {{- end }}
   selector:
-    {{- include "kyverno.sidecar-injector.labels" . | nindent 4 }}
+    {{- include "kyverno.sidecar-injector.labels.match" . | nindent 4 }}
 {{- end -}}
diff --git a/charts/kyverno-envoy-plugin/values.yaml b/charts/kyverno-envoy-plugin/values.yaml
@@ -165,7 +165,7 @@ sidecarInjector:
       # @default -- See [values.yaml](values.yaml)
       startupProbe:
         httpGet:
-          path: /health/liveness
+          path: /livez
           port: 9443
           scheme: HTTPS
         failureThreshold: 20
@@ -178,7 +178,7 @@ sidecarInjector:
       # @default -- See [values.yaml](values.yaml)
       livenessProbe:
         httpGet:
-          path: /health/liveness
+          path: /livez
           port: 9443
           scheme: HTTPS
         initialDelaySeconds: 15
@@ -193,7 +193,7 @@ sidecarInjector:
       # @default -- See [values.yaml](values.yaml)
       readinessProbe:
         httpGet:
-          path: /health/readiness
+          path: /readyz
           port: 9443
           scheme: HTTPS
         initialDelaySeconds: 5
@@ -211,7 +211,9 @@ sidecarInjector:
       # -- Container args.
       args:
         - sidecar-injector
-        - --port=9443
+        - --address=:9443
+        - --cert-file=/opt/kubernetes-sidecar-injector/certs/tls.crt
+        - --key-file=/opt/kubernetes-sidecar-injector/certs/tls.key
 
   service:
 

diff --git a/go.mod b/go.mod
@@ -11,6 +11,7 @@ require (
 	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.9.0
 	go.uber.org/multierr v1.11.0
+	gomodules.xyz/jsonpatch/v2 v2.4.0
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142
 	google.golang.org/grpc v1.67.1
 	k8s.io/apimachinery v0.29.3
@@ -108,7 +109,7 @@ require (
 	k8s.io/component-base v0.29.2 // indirect
 	k8s.io/klog/v2 v2.120.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20240221221325-2ac9dc51f3f1 // indirect
-	k8s.io/utils v0.0.0-20240102154912-e7106e64919e // indirect
+	k8s.io/utils v0.0.0-20240102154912-e7106e64919e
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kubectl-validate v0.0.2-0.20240102223437-fe143bcde89f // indirect

diff --git a/go.sum b/go.sum
@@ -300,6 +300,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
+gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY=