feat: Add webhook support for hub-agent (#224)

* feat: Add webhook support for hub-agent

* fix review comments

* Add a missing err handling

* fix lint error

Co-authored-by: guofei <[email protected]>

charts/hub-agent/templates/deployment.yaml

@@ -20,11 +20,19 @@ spec:
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           args:
+            - --leader-elect=true
+            - --enable-webhook={{ .Values.enableWebhook }}
             - --v={{ .Values.logVerbosity }}
           ports:
             - name: http
               containerPort: 80
               protocol: TCP
+          env:
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: metadata.namespace
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       {{- with .Values.affinity }}

charts/hub-agent/templates/webhookservice.yaml

@@ -0,0 +1,24 @@
+# We use a headless service for webhook assuming the apiserver's dns can resolve it.
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    {{- include "hub-agent.labels" . | nindent 4 }}      
+  name: fleetwebhook
+  namespace: {{ .Values.namespace }} 
+spec:
+  clusterIP: None
+  clusterIPs:
+  - None
+  ipFamilies:
+  - IPv4
+  ipFamilyPolicy: SingleStack
+  ports:
+  - name: client
+    port: 9443
+    protocol: TCP
+    targetPort: 9443
+  selector:
+    {{- include "hub-agent.selectorLabels" . | nindent 4 }}
+  sessionAffinity: None
+  type: ClusterIP

charts/hub-agent/values.yaml

@@ -12,6 +12,8 @@ image:
 
 logVerbosity: 2
 
+enableWebhook: false
+
 namespace:
   fleet-system
 

cmd/hubagent/main.go

@@ -5,6 +5,7 @@ Licensed under the MIT license.
 package main
 
 import (
+	"context"
 	"flag"
 	"os"
 
@@ -14,23 +15,30 @@ import (
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	fleetmetrics "go.goms.io/fleet/pkg/metrics"
+	"go.goms.io/fleet/pkg/webhook"
 
 	fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1"
 	"go.goms.io/fleet/pkg/controllers/membercluster"
 	//+kubebuilder:scaffold:imports
 )
 
 var (
-	scheme               = runtime.NewScheme()
-	probeAddr            = flag.String("health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
-	metricsAddr          = flag.String("metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
-	enableLeaderElection = flag.Bool("leader-elect", false,
-		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
-	networkingAgentsEnabled = flag.Bool("networking-agents-enabled", false,
-		"Whether the networking agents are enabled or not.")
+	scheme                  = runtime.NewScheme()
+	probeAddr               = flag.String("health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
+	metricsAddr             = flag.String("metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
+	enableLeaderElection    = flag.Bool("leader-elect", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
+	enableWebhook           = flag.Bool("enable-webhook", false, "If set, the fleet webhook is enabled.")
+	networkingAgentsEnabled = flag.Bool("networking-agents-enabled", false, "Whether the networking agents are enabled or not.")
+)
+
+const (
+	FleetWebhookCertDir     = "/tmp/k8s-webhook-server/serving-certs"
+	FleetWebhookPort        = 9443
+	LeaderElectionNamespace = "kube-system"
 )
 
 func init() {
@@ -49,12 +57,14 @@ func main() {
 	defer klog.Flush()
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:                 scheme,
-		MetricsBindAddress:     *metricsAddr,
-		Port:                   9446,
-		HealthProbeBindAddress: *probeAddr,
-		LeaderElection:         *enableLeaderElection,
-		LeaderElectionID:       "984738fa.hub.fleet.azure.com",
+		Scheme:                  scheme,
+		MetricsBindAddress:      *metricsAddr,
+		Port:                    FleetWebhookPort,
+		CertDir:                 FleetWebhookCertDir,
+		HealthProbeBindAddress:  *probeAddr,
+		LeaderElection:          *enableLeaderElection,
+		LeaderElectionNamespace: LeaderElectionNamespace,
+		LeaderElectionID:        "984738fa.hub.fleet.azure.com",
 	})
 	if err != nil {
 		klog.ErrorS(err, "unable to start controller manager.")
@@ -79,10 +89,50 @@ func main() {
 		klog.ErrorS(err, "unable to set up ready check")
 		os.Exit(1)
 	}
+
+	if *enableWebhook {
+		// Generate self-signed key and crt files in FleetWebhookCertDir for the webhook server to start
+		caPEM, err := webhook.GenCertificate(FleetWebhookCertDir)
+		if err != nil {
+			klog.ErrorS(err, "fail to generate certificates for webhook server")
+			os.Exit(1)
+		}
+
+		if err := mgr.Add(&webhookApiserverConfigurator{
+			mgr:   mgr,
+			caPEM: caPEM,
+			port:  FleetWebhookPort,
+		}); err != nil {
+			klog.ErrorS(err, "unable to add webhookApiserverConfigurator")
+			os.Exit(1)
+		}
+		if err := webhook.AddToManager(mgr); err != nil {
+			klog.ErrorS(err, "unable to register webhooks to the manager")
+			os.Exit(1)
+		}
+	}
+
 	//+kubebuilder:scaffold:builder
 
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		klog.ErrorS(err, "problem starting manager")
 		os.Exit(1)
 	}
 }
+
+type webhookApiserverConfigurator struct {
+	mgr   manager.Manager
+	caPEM []byte
+	port  int
+}
+
+var _ manager.Runnable = &webhookApiserverConfigurator{}
+
+func (c *webhookApiserverConfigurator) Start(ctx context.Context) error {
+	klog.V(2).InfoS("setting up webhooks in apiserver from the leader")
+	if err := webhook.CreateFleetWebhookConfiguration(ctx, c.mgr.GetClient(), c.caPEM, c.port); err != nil {
+		klog.ErrorS(err, "unable to setup webhook configurations in apiserver")
+		os.Exit(1)
+	}
+	return nil
+}

pkg/webhook/add_pod.go

@@ -0,0 +1,13 @@
+/*
+Copyright (c) Microsoft Corporation.
+Licensed under the MIT license.
+*/
+package webhook
+
+import (
+	"go.goms.io/fleet/pkg/webhook/pod"
+)
+
+func init() {
+	AddToManagerFuncs = append(AddToManagerFuncs, pod.Add)
+}

pkg/webhook/pod/pod_validating_webhook.go

@@ -0,0 +1,52 @@
+/*
+Copyright (c) Microsoft Corporation.
+Licensed under the MIT license.
+*/
+
+package pod
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	admissionv1 "k8s.io/api/admission/v1"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+)
+
+// Add registers the webhook for K8s bulit-in object types.
+func Add(mgr manager.Manager) error {
+	hookServer := mgr.GetWebhookServer()
+	hookServer.Register("/validate-v1-pod", &webhook.Admission{Handler: &podValidator{Client: mgr.GetClient()}})
+	return nil
+}
+
+type podValidator struct {
+	Client  client.Client
+	decoder *admission.Decoder
+}
+
+// podValidator denies a pod if it is not created in the system namespaces.
+func (v *podValidator) Handle(ctx context.Context, req admission.Request) admission.Response {
+	if req.Operation == admissionv1.Create {
+		pod := &corev1.Pod{}
+		err := v.decoder.Decode(req, pod)
+		if err != nil {
+			return admission.Errored(http.StatusBadRequest, err)
+		}
+		if pod.Namespace != "kube-system" && pod.Namespace != "fleet-system" {
+			return admission.Denied(fmt.Sprintf("Pod %s/%s creation is disallowed in the fleet hub cluster", pod.Namespace, pod.Name))
+		}
+	}
+	return admission.Allowed("")
+}
+
+// InjectDecoder injects the decoder.
+func (v *podValidator) InjectDecoder(d *admission.Decoder) error {
+	v.decoder = d
+	return nil
+}