From 8a9ee47699303c35f45f1bdfb0a0dd1959e34ecc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?= Date: Fri, 25 Oct 2024 11:26:34 +0200 Subject: [PATCH] refactor: sidecar injector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Charles-Edouard Brétéché --- .github/workflows/tests.yaml | 10 +- .vscode/launch.json | 1 - .../sidecar-injector/deployment.yaml | 8 ++ charts/kyverno-envoy-plugin/values.yaml | 8 +- pkg/commands/inject/command.go | 19 ++- pkg/httpd/simpleserver.go | 111 +++++++++++------- pkg/server/handlers/admission.go | 36 ++++++ pkg/signals/context.go | 22 +++- pkg/webhook/health.go | 10 -- .../{ => sidecar-injector}/chainsaw-test.yaml | 0 .../{ => sidecar-injector}/deployment.yaml | 0 11 files changed, 154 insertions(+), 71 deletions(-) create mode 100644 pkg/server/handlers/admission.go delete mode 100644 pkg/webhook/health.go rename tests/e2e-test/{ => sidecar-injector}/chainsaw-test.yaml (100%) rename tests/e2e-test/{ => sidecar-injector}/deployment.yaml (100%) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ea6dde69..8c31f4d0 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -86,16 +86,16 @@ jobs: with: go-version-file: go.mod cache-dependency-path: go.sum - - name: Run tests - run: | - set -e - make kind-create-cluster - make kind-load-taged-image - name: Install Cosign uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da # v3.7.0 - name: Install chainsaw uses: kyverno/action-install-chainsaw@d311eacde764f806c9658574ff64c9c3b21f8397 # v0.2.11 with: verify: true + - name: Run tests + run: | + set -e + make kind-create-cluster + make chart-install - name: Run Chainsaw Tests run: chainsaw test tests/e2e-test diff --git a/.vscode/launch.json b/.vscode/launch.json index 404706c0..5506068c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -19,7 +19,6 @@ "program": "${workspaceFolder}", "args": [ "sidecar-injector", - "--local" ], } ] diff --git a/charts/kyverno-envoy-plugin/templates/sidecar-injector/deployment.yaml b/charts/kyverno-envoy-plugin/templates/sidecar-injector/deployment.yaml index 30f92053..1ddb5ba2 100644 --- a/charts/kyverno-envoy-plugin/templates/sidecar-injector/deployment.yaml +++ b/charts/kyverno-envoy-plugin/templates/sidecar-injector/deployment.yaml @@ -74,6 +74,10 @@ spec: {{- tpl (toYaml .) $ | nindent 10 }} {{- end }} serviceAccountName: {{ template "kyverno.sidecar-injector.service-account.name" . }} + volumes: + - name: certs + secret: + secretName: {{ template "kyverno.sidecar-injector.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-pair containers: {{- with .Values.sidecarInjector.containers.injector }} - name: injector @@ -107,5 +111,9 @@ spec: args: {{- tpl (toYaml .) $ | nindent 12 }} {{- end }} + volumeMounts: + - name: certs + mountPath: /opt/kubernetes-sidecar-injector/certs + readOnly: true {{- end }} {{- end -}} diff --git a/charts/kyverno-envoy-plugin/values.yaml b/charts/kyverno-envoy-plugin/values.yaml index 192be520..acb4faaf 100644 --- a/charts/kyverno-envoy-plugin/values.yaml +++ b/charts/kyverno-envoy-plugin/values.yaml @@ -165,7 +165,7 @@ sidecarInjector: # @default -- See [values.yaml](values.yaml) startupProbe: httpGet: - path: /health/liveness + path: /livez port: 9443 scheme: HTTPS failureThreshold: 20 @@ -178,7 +178,7 @@ sidecarInjector: # @default -- See [values.yaml](values.yaml) livenessProbe: httpGet: - path: /health/liveness + path: /livez port: 9443 scheme: HTTPS initialDelaySeconds: 15 @@ -193,7 +193,7 @@ sidecarInjector: # @default -- See [values.yaml](values.yaml) readinessProbe: httpGet: - path: /health/readiness + path: /readyz port: 9443 scheme: HTTPS initialDelaySeconds: 5 @@ -212,6 +212,8 @@ sidecarInjector: args: - sidecar-injector - --port=9443 + - --certFile=/opt/kubernetes-sidecar-injector/certs/tls.crt + - --keyFile=/opt/kubernetes-sidecar-injector/certs/tls.key service: diff --git a/pkg/commands/inject/command.go b/pkg/commands/inject/command.go index c727d9d8..2d0cea31 100644 --- a/pkg/commands/inject/command.go +++ b/pkg/commands/inject/command.go @@ -1,9 +1,11 @@ package inject import ( + "context" "fmt" "github.com/kyverno/kyverno-envoy-plugin/pkg/httpd" + "github.com/kyverno/kyverno-envoy-plugin/pkg/signals" "github.com/spf13/cobra" ) @@ -13,14 +15,19 @@ func Command() *cobra.Command { Use: "sidecar-injector", Short: "Responsible for injecting sidecars into pod containers", RunE: func(cmd *cobra.Command, args []string) error { - fmt.Printf("SimpleServer starting to listen in port %v", httpdConf.Port) - return httpdConf.Start() + signals.WithContext(context.Background(), func(ctx context.Context) { + fmt.Println("SimpleServer starting to listen in port", httpdConf.Port) + if err := httpdConf.Start(); err != nil { + panic(err) + } + }) + return nil }, } command.Flags().IntVar(&httpdConf.Port, "port", 443, "server port.") - command.Flags().StringVar(&httpdConf.CertFile, "certFile", "/etc/mutator/certs/tls.crt", "File containing tls certificate") - command.Flags().StringVar(&httpdConf.KeyFile, "keyFile", "/etc/mutator/certs/tls.key", "File containing tls private key") - command.Flags().BoolVar(&httpdConf.Local, "local", false, "Local run mode") - command.Flags().StringVar(&(&httpdConf.Patcher).SidecarDataKey, "sidecarDataKey", "sidecars.yaml", "ConfigMap Sidecar Data Key") + command.Flags().StringVar(&httpdConf.CertFile, "certFile", "", "File containing tls certificate") + command.Flags().StringVar(&httpdConf.KeyFile, "keyFile", "", "File containing tls private key") + // command.Flags().BoolVar(&httpdConf.Local, "local", false, "Local run mode") + // command.Flags().StringVar(&(&httpdConf.Patcher).SidecarDataKey, "sidecarDataKey", "sidecars.yaml", "ConfigMap Sidecar Data Key") return command } diff --git a/pkg/httpd/simpleserver.go b/pkg/httpd/simpleserver.go index 2c7f7613..0095ba93 100644 --- a/pkg/httpd/simpleserver.go +++ b/pkg/httpd/simpleserver.go @@ -3,68 +3,91 @@ package httpd import ( "fmt" "net/http" - "os" - "path/filepath" - "github.com/kyverno/kyverno-envoy-plugin/pkg/admission" - "github.com/kyverno/kyverno-envoy-plugin/pkg/webhook" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" + "github.com/kyverno/kyverno-envoy-plugin/pkg/server/handlers" ) -/*SimpleServer is the required config to create httpd server*/ type SimpleServer struct { - Local bool + // Local bool Port int CertFile string KeyFile string - Patcher webhook.SidecarInjectorPatcher - Debug bool + // Patcher webhook.SidecarInjectorPatcher } +// &http.Server{ +// Addr: fmt.Sprintf(":%d", webhookServerPort), +// TLSConfig: &tls.Config{ +// GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { +// certPem, keyPem, err := tlsProvider() +// if err != nil { +// return nil, err +// } +// pair, err := tls.X509KeyPair(certPem, keyPem) +// if err != nil { +// return nil, err +// } +// return &pair, nil +// }, +// MinVersion: tls.VersionTLS12, +// CipherSuites: []uint16{ +// // AEADs w/ ECDHE +// tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +// tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +// tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +// tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, +// tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, +// tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +// }, +// }, +// Handler: mux, +// ReadTimeout: 30 * time.Second, +// WriteTimeout: 30 * time.Second, +// ReadHeaderTimeout: 30 * time.Second, +// IdleTimeout: 5 * time.Minute, +// // Start the simple http server supporting TLS func (simpleServer *SimpleServer) Start() error { - k8sClient, err := simpleServer.CreateClient() - if err != nil { - return err - } - simpleServer.Patcher.K8sClient = k8sClient + // k8sClient, err := simpleServer.CreateClient() + // if err != nil { + // return err + // } + // simpleServer.Patcher.K8sClient = k8sClient server := &http.Server{ Addr: fmt.Sprintf(":%d", simpleServer.Port), } mux := http.NewServeMux() server.Handler = mux - admissionHandler := &admission.Handler{ - Handler: &admission.PodAdmissionRequestHandler{ - PodHandler: &simpleServer.Patcher, - }, - } - mux.HandleFunc("/healthz", webhook.HealthCheckHandler) - mux.HandleFunc("/mutate", admissionHandler.HandleAdmission) - if simpleServer.Local { + // admissionHandler := &admission.Handler{ + // Handler: &admission.PodAdmissionRequestHandler{ + // PodHandler: &simpleServer.Patcher, + // }, + // } + mux.Handle("/livez", handlers.Health()) + mux.Handle("/readyz", handlers.Health()) + mux.Handle("/mutate", handlers.AdmissionReview(nil)) + if simpleServer.CertFile != "" && simpleServer.KeyFile != "" { + return server.ListenAndServeTLS(simpleServer.CertFile, simpleServer.KeyFile) + } else { return server.ListenAndServe() } - return server.ListenAndServeTLS(simpleServer.CertFile, simpleServer.KeyFile) } -// CreateClient Create the server -func (simpleServer *SimpleServer) CreateClient() (*kubernetes.Clientset, error) { - config, err := simpleServer.buildConfig() - if err != nil { - return nil, errors.Wrapf(err, "error setting up cluster config") - } - return kubernetes.NewForConfig(config) -} +// // CreateClient Create the server +// func (simpleServer *SimpleServer) CreateClient() (*kubernetes.Clientset, error) { +// config, err := simpleServer.buildConfig() +// if err != nil { +// return nil, errors.Wrapf(err, "error setting up cluster config") +// } +// return kubernetes.NewForConfig(config) +// } -func (simpleServer *SimpleServer) buildConfig() (*rest.Config, error) { - if simpleServer.Local { - log.Debug("Using local kubeconfig.") - kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config") - return clientcmd.BuildConfigFromFlags("", kubeconfig) - } - log.Debug("Using in cluster kubeconfig.") - return rest.InClusterConfig() -} +// func (simpleServer *SimpleServer) buildConfig() (*rest.Config, error) { +// if simpleServer.Local { +// log.Debug("Using local kubeconfig.") +// kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config") +// return clientcmd.BuildConfigFromFlags("", kubeconfig) +// } +// log.Debug("Using in cluster kubeconfig.") +// return rest.InClusterConfig() +// } diff --git a/pkg/server/handlers/admission.go b/pkg/server/handlers/admission.go new file mode 100644 index 00000000..b8b25da8 --- /dev/null +++ b/pkg/server/handlers/admission.go @@ -0,0 +1,36 @@ +package handlers + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + + admissionv1 "k8s.io/api/admission/v1" +) + +func AdmissionReview(inner func(*admissionv1.AdmissionRequest) *admissionv1.AdmissionResponse) http.HandlerFunc { + return func(writer http.ResponseWriter, request *http.Request) { + if request.Body == nil { + // HttpError(request.Context(), writer, request, logger, errors.New("empty body"), http.StatusBadRequest) + return + } + defer request.Body.Close() + body, err := io.ReadAll(request.Body) + if err != nil { + // HttpError(request.Context(), writer, request, logger, err, http.StatusBadRequest) + return + } + contentType := request.Header.Get("Content-Type") + if contentType != "application/json" { + // HttpError(request.Context(), writer, request, logger, errors.New("invalid Content-Type"), http.StatusUnsupportedMediaType) + return + } + fmt.Println(string(body)) + var admissionReview admissionv1.AdmissionReview + if err := json.Unmarshal(body, &admissionReview); err != nil { + // HttpError(request.Context(), writer, request, logger, err, http.StatusExpectationFailed) + return + } + } +} diff --git a/pkg/signals/context.go b/pkg/signals/context.go index f3fbccba..8f9be68e 100644 --- a/pkg/signals/context.go +++ b/pkg/signals/context.go @@ -2,11 +2,29 @@ package signals import ( "context" - "os" + "fmt" "os/signal" "syscall" + + "k8s.io/apimachinery/pkg/util/wait" ) func Context(ctx context.Context) (context.Context, context.CancelFunc) { - return signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) + return signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) +} + +func WithContext(ctx context.Context, funcs ...func(context.Context)) { + var group wait.Group + func() { + fmt.Println("Setting up signal aware context") + ctx, cancel := Context(ctx) + defer cancel() + fmt.Println("Starting group routines") + for _, f := range funcs { + group.StartWithContext(ctx, f) + } + <-ctx.Done() + }() + fmt.Println("Waiting for group routines to terminate") + group.Wait() } diff --git a/pkg/webhook/health.go b/pkg/webhook/health.go deleted file mode 100644 index f015807b..00000000 --- a/pkg/webhook/health.go +++ /dev/null @@ -1,10 +0,0 @@ -package webhook - -import ( - "net/http" -) - -// HealthCheckHandler HttpServer function to handle Health check -func HealthCheckHandler(writer http.ResponseWriter, _ *http.Request) { - writer.WriteHeader(http.StatusOK) -} diff --git a/tests/e2e-test/chainsaw-test.yaml b/tests/e2e-test/sidecar-injector/chainsaw-test.yaml similarity index 100% rename from tests/e2e-test/chainsaw-test.yaml rename to tests/e2e-test/sidecar-injector/chainsaw-test.yaml diff --git a/tests/e2e-test/deployment.yaml b/tests/e2e-test/sidecar-injector/deployment.yaml similarity index 100% rename from tests/e2e-test/deployment.yaml rename to tests/e2e-test/sidecar-injector/deployment.yaml