refactor: sidecar injector

Signed-off-by: Charles-Edouard Brétéché <[email protected]>

.github/workflows/tests.yaml

@@ -86,16 +86,16 @@ jobs:
         with:
           go-version-file: go.mod
           cache-dependency-path: go.sum
-      - name: Run tests
-        run: |
-          set -e
-          make kind-create-cluster
-          make kind-load-taged-image
       - name: Install Cosign
         uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da # v3.7.0
       - name: Install chainsaw
         uses: kyverno/action-install-chainsaw@d311eacde764f806c9658574ff64c9c3b21f8397 # v0.2.11
         with:
           verify: true
+      - name: Run tests
+        run: |
+          set -e
+          make kind-create-cluster
+          make chart-install
       - name: Run Chainsaw Tests
         run: chainsaw test tests/e2e-test

.vscode/launch.json

@@ -19,7 +19,6 @@
             "program": "${workspaceFolder}",
             "args": [
                 "sidecar-injector",
-                "--local"
             ],
         }
     ]

charts/kyverno-envoy-plugin/templates/_helpers/_deployment.tpl

@@ -0,0 +1,10 @@
+{{/* vim: set filetype=mustache: */}}
+
+{{- define "kyverno.deployment.replicas" -}}
+  {{- if and (not (kindIs "invalid" .)) (not (kindIs "string" .)) -}}
+  {{- if eq (int .) 0 -}}
+    {{- fail "Kyverno does not support running with 0 replicas. Please provide a non-zero integer value." -}}
+  {{- end -}}
+  {{- end -}}
+  {{- . -}}
+{{- end -}}

charts/kyverno-envoy-plugin/templates/sidecar-injector/deployment.yaml

@@ -74,6 +74,10 @@ spec:
           {{- tpl (toYaml .) $ | nindent 10 }}
         {{- end }}
       serviceAccountName: {{ template "kyverno.sidecar-injector.service-account.name" . }}
+      volumes:
+        - name: certs
+          secret:
+            secretName: {{ template "kyverno.sidecar-injector.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-pair
       containers:
         {{- with .Values.sidecarInjector.containers.injector }}
         - name: injector
@@ -107,5 +111,9 @@ spec:
           args:
             {{- tpl (toYaml .) $ | nindent 12 }}
           {{- end }}
+          volumeMounts:
+            - name: certs
+              mountPath: /opt/kubernetes-sidecar-injector/certs
+              readOnly: true
         {{- end }}
 {{- end -}}

charts/kyverno-envoy-plugin/values.yaml

@@ -165,7 +165,7 @@ sidecarInjector:
       # @default -- See [values.yaml](values.yaml)
       startupProbe:
         httpGet:
-          path: /health/liveness
+          path: /livez
           port: 9443
           scheme: HTTPS
         failureThreshold: 20
@@ -178,7 +178,7 @@ sidecarInjector:
       # @default -- See [values.yaml](values.yaml)
       livenessProbe:
         httpGet:
-          path: /health/liveness
+          path: /livez
           port: 9443
           scheme: HTTPS
         initialDelaySeconds: 15
@@ -193,7 +193,7 @@ sidecarInjector:
       # @default -- See [values.yaml](values.yaml)
       readinessProbe:
         httpGet:
-          path: /health/readiness
+          path: /readyz
           port: 9443
           scheme: HTTPS
         initialDelaySeconds: 5
@@ -212,6 +212,8 @@ sidecarInjector:
       args:
         - sidecar-injector
         - --port=9443
+        - --certFile=/opt/kubernetes-sidecar-injector/certs/tls.crt
+        - --keyFile=/opt/kubernetes-sidecar-injector/certs/tls.key
 
   service:
 

pkg/commands/inject/command.go

@@ -1,10 +1,18 @@
 package inject
 
 import (
+	"context"
+	"errors"
 	"fmt"
+	"net/http"
+	"time"
 
 	"github.com/kyverno/kyverno-envoy-plugin/pkg/httpd"
+	"github.com/kyverno/kyverno-envoy-plugin/pkg/server/handlers"
+	"github.com/kyverno/kyverno-envoy-plugin/pkg/signals"
 	"github.com/spf13/cobra"
+	"go.uber.org/multierr"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 func Command() *cobra.Command {
@@ -13,14 +21,51 @@ func Command() *cobra.Command {
 		Use:   "sidecar-injector",
 		Short: "Responsible for injecting sidecars into pod containers",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			fmt.Printf("SimpleServer starting to listen in port %v", httpdConf.Port)
-			return httpdConf.Start()
+			return runServer(context.Background(), fmt.Sprintf(":%d", httpdConf.Port))
 		},
 	}
 	command.Flags().IntVar(&httpdConf.Port, "port", 443, "server port.")
-	command.Flags().StringVar(&httpdConf.CertFile, "certFile", "/etc/mutator/certs/tls.crt", "File containing tls certificate")
-	command.Flags().StringVar(&httpdConf.KeyFile, "keyFile", "/etc/mutator/certs/tls.key", "File containing tls private key")
-	command.Flags().BoolVar(&httpdConf.Local, "local", false, "Local run mode")
-	command.Flags().StringVar(&(&httpdConf.Patcher).SidecarDataKey, "sidecarDataKey", "sidecars.yaml", "ConfigMap Sidecar Data Key")
+	command.Flags().StringVar(&httpdConf.CertFile, "certFile", "", "File containing tls certificate")
+	command.Flags().StringVar(&httpdConf.KeyFile, "keyFile", "", "File containing tls private key")
+	// command.Flags().BoolVar(&httpdConf.Local, "local", false, "Local run mode")
+	// command.Flags().StringVar(&(&httpdConf.Patcher).SidecarDataKey, "sidecarDataKey", "sidecars.yaml", "ConfigMap Sidecar Data Key")
 	return command
 }
+
+func setupMux() http.Handler {
+	mux := http.NewServeMux()
+	mux.Handle("/livez", handlers.Health())
+	mux.Handle("/readyz", handlers.Health())
+	mux.Handle("/mutate", handlers.AdmissionReview(nil))
+	return mux
+}
+
+func setupServer(addr string) *http.Server {
+	return &http.Server{
+		Addr:    addr,
+		Handler: setupMux(),
+	}
+}
+
+func runServer(ctx context.Context, addr string) error {
+	var group wait.Group
+	server := setupServer(addr)
+	err := func() error {
+		signalsCtx, signalsCancel := signals.Context(ctx)
+		defer signalsCancel()
+		var shutdownErr error
+		group.StartWithContext(signalsCtx, func(ctx context.Context) {
+			<-ctx.Done()
+			shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer shutdownCancel()
+			shutdownErr = server.Shutdown(shutdownCtx)
+		})
+		serveErr := server.ListenAndServe()
+		if errors.Is(serveErr, http.ErrServerClosed) {
+			serveErr = nil
+		}
+		return multierr.Combine(serveErr, shutdownErr)
+	}()
+	group.Wait()
+	return err
+}

pkg/httpd/simpleserver.go

@@ -3,68 +3,91 @@ package httpd
 import (
 	"fmt"
 	"net/http"
-	"os"
-	"path/filepath"
 
-	"github.com/kyverno/kyverno-envoy-plugin/pkg/admission"
-	"github.com/kyverno/kyverno-envoy-plugin/pkg/webhook"
-	"github.com/pkg/errors"
-	log "github.com/sirupsen/logrus"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
+	"github.com/kyverno/kyverno-envoy-plugin/pkg/server/handlers"
 )
 
-/*SimpleServer is the required config to create httpd server*/
 type SimpleServer struct {
-	Local    bool
+	// Local    bool
 	Port     int
 	CertFile string
 	KeyFile  string
-	Patcher  webhook.SidecarInjectorPatcher
-	Debug    bool
+	// Patcher  webhook.SidecarInjectorPatcher
 }
 
+//	&http.Server{
+//		Addr: fmt.Sprintf(":%d", webhookServerPort),
+//		TLSConfig: &tls.Config{
+//			GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+//				certPem, keyPem, err := tlsProvider()
+//				if err != nil {
+//					return nil, err
+//				}
+//				pair, err := tls.X509KeyPair(certPem, keyPem)
+//				if err != nil {
+//					return nil, err
+//				}
+//				return &pair, nil
+//			},
+//			MinVersion: tls.VersionTLS12,
+//			CipherSuites: []uint16{
+//				// AEADs w/ ECDHE
+//				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+//				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+//				tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+//				tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+//				tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+//				tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+//			},
+//		},
+//		Handler:           mux,
+//		ReadTimeout:       30 * time.Second,
+//		WriteTimeout:      30 * time.Second,
+//		ReadHeaderTimeout: 30 * time.Second,
+//		IdleTimeout:       5 * time.Minute,
+//
 // Start the simple http server supporting TLS
 func (simpleServer *SimpleServer) Start() error {
-	k8sClient, err := simpleServer.CreateClient()
-	if err != nil {
-		return err
-	}
-	simpleServer.Patcher.K8sClient = k8sClient
+	// k8sClient, err := simpleServer.CreateClient()
+	// if err != nil {
+	// 	return err
+	// }
+	// simpleServer.Patcher.K8sClient = k8sClient
 	server := &http.Server{
 		Addr: fmt.Sprintf(":%d", simpleServer.Port),
 	}
 	mux := http.NewServeMux()
 	server.Handler = mux
-	admissionHandler := &admission.Handler{
-		Handler: &admission.PodAdmissionRequestHandler{
-			PodHandler: &simpleServer.Patcher,
-		},
-	}
-	mux.HandleFunc("/healthz", webhook.HealthCheckHandler)
-	mux.HandleFunc("/mutate", admissionHandler.HandleAdmission)
-	if simpleServer.Local {
+	// admissionHandler := &admission.Handler{
+	// 	Handler: &admission.PodAdmissionRequestHandler{
+	// 		PodHandler: &simpleServer.Patcher,
+	// 	},
+	// }
+	mux.Handle("/livez", handlers.Health())
+	mux.Handle("/readyz", handlers.Health())
+	mux.Handle("/mutate", handlers.AdmissionReview(nil))
+	if simpleServer.CertFile != "" && simpleServer.KeyFile != "" {
+		return server.ListenAndServeTLS(simpleServer.CertFile, simpleServer.KeyFile)
+	} else {
 		return server.ListenAndServe()
 	}
-	return server.ListenAndServeTLS(simpleServer.CertFile, simpleServer.KeyFile)
 }
 
-// CreateClient Create the server
-func (simpleServer *SimpleServer) CreateClient() (*kubernetes.Clientset, error) {
-	config, err := simpleServer.buildConfig()
-	if err != nil {
-		return nil, errors.Wrapf(err, "error setting up cluster config")
-	}
-	return kubernetes.NewForConfig(config)
-}
+// // CreateClient Create the server
+// func (simpleServer *SimpleServer) CreateClient() (*kubernetes.Clientset, error) {
+// 	config, err := simpleServer.buildConfig()
+// 	if err != nil {
+// 		return nil, errors.Wrapf(err, "error setting up cluster config")
+// 	}
+// 	return kubernetes.NewForConfig(config)
+// }
 
-func (simpleServer *SimpleServer) buildConfig() (*rest.Config, error) {
-	if simpleServer.Local {
-		log.Debug("Using local kubeconfig.")
-		kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config")
-		return clientcmd.BuildConfigFromFlags("", kubeconfig)
-	}
-	log.Debug("Using in cluster kubeconfig.")
-	return rest.InClusterConfig()
-}
+// func (simpleServer *SimpleServer) buildConfig() (*rest.Config, error) {
+// 	if simpleServer.Local {
+// 		log.Debug("Using local kubeconfig.")
+// 		kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config")
+// 		return clientcmd.BuildConfigFromFlags("", kubeconfig)
+// 	}
+// 	log.Debug("Using in cluster kubeconfig.")
+// 	return rest.InClusterConfig()
+// }

pkg/server/handlers/admission.go

@@ -0,0 +1,36 @@
+package handlers
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	admissionv1 "k8s.io/api/admission/v1"
+)
+
+func AdmissionReview(inner func(*admissionv1.AdmissionRequest) *admissionv1.AdmissionResponse) http.HandlerFunc {
+	return func(writer http.ResponseWriter, request *http.Request) {
+		if request.Body == nil {
+			// HttpError(request.Context(), writer, request, logger, errors.New("empty body"), http.StatusBadRequest)
+			return
+		}
+		defer request.Body.Close()
+		body, err := io.ReadAll(request.Body)
+		if err != nil {
+			// HttpError(request.Context(), writer, request, logger, err, http.StatusBadRequest)
+			return
+		}
+		contentType := request.Header.Get("Content-Type")
+		if contentType != "application/json" {
+			// HttpError(request.Context(), writer, request, logger, errors.New("invalid Content-Type"), http.StatusUnsupportedMediaType)
+			return
+		}
+		fmt.Println(string(body))
+		var admissionReview admissionv1.AdmissionReview
+		if err := json.Unmarshal(body, &admissionReview); err != nil {
+			// HttpError(request.Context(), writer, request, logger, err, http.StatusExpectationFailed)
+			return
+		}
+	}
+}

pkg/signals/context.go

@@ -2,11 +2,29 @@ package signals
 
 import (
 	"context"
-	"os"
+	"fmt"
 	"os/signal"
 	"syscall"
+
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 func Context(ctx context.Context) (context.Context, context.CancelFunc) {
-	return signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
+	return signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM)
+}
+
+func WithContext(ctx context.Context, funcs ...func(context.Context)) {
+	var group wait.Group
+	func() {
+		fmt.Println("Setting up signal aware context")
+		ctx, cancel := Context(ctx)
+		defer cancel()
+		fmt.Println("Starting group routines")
+		for _, f := range funcs {
+			group.StartWithContext(ctx, f)
+		}
+		<-ctx.Done()
+	}()
+	fmt.Println("Waiting for group routines to terminate")
+	group.Wait()
 }