feat: Enabled webhook in upstream & prod via config. (#330)

* Added configuration for Pod & CRP webhooks to either be service ref or URL (DNS).
* Added E2E tests.

Co-authored-by: Sean Hobbs <[email protected]>

Makefile

@@ -140,7 +140,9 @@ install-hub-agent-helm:
     --set image.repository=$(REGISTRY)/$(HUB_AGENT_IMAGE_NAME) \
     --set image.tag=$(HUB_AGENT_IMAGE_VERSION) \
     --set logVerbosity=5 \
-    --set namespace=fleet-system
+    --set namespace=fleet-system \
+    --set enableWebhook=true \
+    --set webhookClientConnectionType=service
 
 .PHONY: e2e-hub-kubeconfig-secret
 e2e-hub-kubeconfig-secret:

charts/hub-agent/templates/deployment.yaml

@@ -22,6 +22,7 @@ spec:
           args:
             - --leader-elect=true
             - --enable-webhook={{ .Values.enableWebhook }}
+            - --webhook-client-connection-type={{.Values.webhookClientConnectionType}}
             - --v={{ .Values.logVerbosity }}
             - -add_dir_header
           ports:

charts/hub-agent/templates/webhookservice.yaml

@@ -1,4 +1,4 @@
-# We use a headless service for webhook assuming the apiserver's dns can resolve it.
+# The webhook will normally use a service reference with a cluster assigned IP.
 apiVersion: v1
 kind: Service
 metadata:
@@ -7,9 +7,6 @@ metadata:
   name: fleetwebhook
   namespace: {{ .Values.namespace }} 
 spec:
-  clusterIP: None
-  clusterIPs:
-  - None
   ipFamilies:
   - IPv4
   ipFamilyPolicy: SingleStack

charts/hub-agent/values.yaml

@@ -12,7 +12,8 @@ image:
 
 logVerbosity: 5
 
-enableWebhook: false
+enableWebhook: true
+webhookClientConnectionType: service
 
 namespace:
   fleet-system

cmd/hubagent/main.go

@@ -2,10 +2,10 @@
 Copyright (c) Microsoft Corporation.
 Licensed under the MIT license.
 */
+
 package main
 
 import (
-	"context"
 	"flag"
 	"os"
 
@@ -25,7 +25,7 @@ import (
 	"go.goms.io/fleet/pkg/controllers/membercluster"
 	fleetmetrics "go.goms.io/fleet/pkg/metrics"
 	"go.goms.io/fleet/pkg/webhook"
-	//+kubebuilder:scaffold:imports
+	// +kubebuilder:scaffold:imports
 )
 
 var (
@@ -49,7 +49,7 @@ func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 	utilruntime.Must(fleetv1alpha1.AddToScheme(scheme))
 	utilruntime.Must(workv1alpha1.AddToScheme(scheme))
-	//+kubebuilder:scaffold:scheme
+	// +kubebuilder:scaffold:scheme
 	klog.InitFlags(nil)
 
 	metrics.Registry.MustRegister(fleetmetrics.JoinResultMetrics, fleetmetrics.LeaveResultMetrics, fleetmetrics.PlacementApplyFailedCount, fleetmetrics.PlacementApplySucceedCount)
@@ -109,7 +109,7 @@ func main() {
 	}
 
 	if opts.EnableWebhook {
-		if err := SetupWebhook(mgr); err != nil {
+		if err := SetupWebhook(mgr, options.WebhookClientConnectionType(opts.WebhookClientConnectionType)); err != nil {
 			klog.ErrorS(err, "unable to set up webhook")
 			exitWithErrorFunc()
 		}
@@ -121,51 +121,29 @@ func main() {
 		exitWithErrorFunc()
 	}
 
-	//+kubebuilder:scaffold:builder
+	// +kubebuilder:scaffold:builder
 
 	if err := mgr.Start(ctx); err != nil {
 		klog.ErrorS(err, "problem starting manager")
 		exitWithErrorFunc()
 	}
 }
 
-// SetupWebhook generate the webhook cert and then setup the webhook configurator
-func SetupWebhook(mgr manager.Manager) error {
-	// Generate self-signed key and crt files in FleetWebhookCertDir for the webhook server to start
-	caPEM, err := webhook.GenCertificate(FleetWebhookCertDir)
+// SetupWebhook generates the webhook cert and then set up the webhook configurator.
+func SetupWebhook(mgr manager.Manager, webhookClientConnectionType options.WebhookClientConnectionType) error {
+	// Generate self-signed key and crt files in FleetWebhookCertDir for the webhook server to start.
+	w, err := webhook.NewWebhookConfig(mgr, FleetWebhookPort, &webhookClientConnectionType, FleetWebhookCertDir)
 	if err != nil {
-		klog.ErrorS(err, "fail to generate certificates for webhook server")
+		klog.ErrorS(err, "fail to generate WebhookConfig")
 		return err
 	}
-
-	if err := mgr.Add(&webhookApiserverConfigurator{
-		mgr:   mgr,
-		caPEM: caPEM,
-		port:  FleetWebhookPort,
-	}); err != nil {
-		klog.ErrorS(err, "unable to add webhookApiserverConfigurator")
+	if err = mgr.Add(w); err != nil {
+		klog.ErrorS(err, "unable to add WebhookConfig")
 		return err
 	}
-	if err := webhook.AddToManager(mgr); err != nil {
+	if err = webhook.AddToManager(mgr); err != nil {
 		klog.ErrorS(err, "unable to register webhooks to the manager")
 		return err
 	}
 	return nil
 }
-
-type webhookApiserverConfigurator struct {
-	mgr   manager.Manager
-	caPEM []byte
-	port  int
-}
-
-var _ manager.Runnable = &webhookApiserverConfigurator{}
-
-func (c *webhookApiserverConfigurator) Start(ctx context.Context) error {
-	klog.V(2).InfoS("setting up webhooks in apiserver from the leader")
-	if err := webhook.CreateFleetWebhookConfiguration(ctx, c.mgr.GetClient(), c.caPEM, c.port); err != nil {
-		klog.ErrorS(err, "unable to setup webhook configurations in apiserver")
-		return err
-	}
-	return nil
-}

cmd/hubagent/options/options.go

@@ -2,6 +2,7 @@
 Copyright (c) Microsoft Corporation.
 Licensed under the MIT license.
 */
+
 package options
 
 import (
@@ -34,6 +35,8 @@ type Options struct {
 	MetricsBindAddress string
 	// EnableWebhook indicates if we will run a webhook
 	EnableWebhook bool
+	// Sets the connection type for the webhook.
+	WebhookClientConnectionType string
 	// NetworkingAgentsEnabled indicates if we enable network agents
 	NetworkingAgentsEnabled bool
 	// ClusterUnhealthyThreshold is the duration of failure for the cluster to be considered unhealthy.
@@ -87,7 +90,8 @@ func (o *Options) AddFlags(flags *flag.FlagSet) {
 	flags.BoolVar(&o.LeaderElection.LeaderElect, "leader-elect", false, "Start a leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.")
 	flags.DurationVar(&o.LeaderElection.LeaseDuration.Duration, "leader-lease-duration", 15*time.Second, "This is effectively the maximum duration that a leader can be stopped before someone else will replace it.")
 	flag.StringVar(&o.LeaderElection.ResourceNamespace, "leader-election-namespace", utils.FleetSystemNamespace, "The namespace in which the leader election resource will be created.")
-	flag.BoolVar(&o.EnableWebhook, "enable-webhook", false, "If set, the fleet webhook is enabled.")
+	flag.BoolVar(&o.EnableWebhook, "enable-webhook", true, "If set, the fleet webhook is enabled.")
+	flag.StringVar(&o.WebhookClientConnectionType, "webhook-client-connection-type", "url", "Sets the connection type used by the webhook client. Only URL or Service is valid.")
 	flag.BoolVar(&o.NetworkingAgentsEnabled, "networking-agents-enabled", false, "Whether the networking agents are enabled or not.")
 	flags.DurationVar(&o.ClusterUnhealthyThreshold.Duration, "cluster-unhealthy-threshold", 60*time.Second, "The duration for a member cluster to be in a degraded state before considered unhealthy.")
 	flags.DurationVar(&o.WorkPendingGracePeriod.Duration, "work-pending-grace-period", 15*time.Second,

cmd/hubagent/options/validation.go

@@ -29,5 +29,10 @@ func (o *Options) Validate() field.ErrorList {
 		errs = append(errs, field.Invalid(newPath.Child("WorkPendingGracePeriod"), o.WorkPendingGracePeriod, "must be greater than 0"))
 	}
 
+	connectionType := o.WebhookClientConnectionType
+	if _, err := parseWebhookClientConnectionString(connectionType); err != nil {
+		errs = append(errs, field.Invalid(newPath.Child("WebhookClientConnectionType"), o.EnableWebhook, err.Error()))
+	}
+
 	return errs
 }

cmd/hubagent/options/webhookconnectiontype.go

@@ -0,0 +1,33 @@
+/*
+Copyright (c) Microsoft Corporation.
+Licensed under the MIT license.
+*/
+
+package options
+
+import (
+	"errors"
+	"strings"
+)
+
+type WebhookClientConnectionType string
+
+const (
+	URL     WebhookClientConnectionType = "url"
+	Service WebhookClientConnectionType = "service"
+)
+
+var (
+	capabilitiesMap = map[string]WebhookClientConnectionType{
+		"service": Service,
+		"url":     URL,
+	}
+)
+
+func parseWebhookClientConnectionString(str string) (WebhookClientConnectionType, error) {
+	t, ok := capabilitiesMap[strings.ToLower(str)]
+	if !ok {
+		return "", errors.New("must be \"service\" or \"url\"")
+	}
+	return t, nil
+}

pkg/webhook/clusterresourceplacement/clusterresourceplacement_webhook.go

@@ -11,6 +11,11 @@ import (
 	fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1"
 )
 
+const (
+	// ValidationPath is the webhook service path which admission requests are routed to for validating ClusterResourcePlacement resources.
+	ValidationPath = "/validate-fleet-azure-com-v1alpha1-clusterresourceplacement"
+)
+
 func Add(mgr manager.Manager) error {
 	return (&fleetv1alpha1.ClusterResourcePlacement{}).SetupWebhookWithManager(mgr)
 }

pkg/webhook/pod/pod_validating_webhook.go

@@ -18,10 +18,15 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 )
 
+const (
+	// ValidationPath is the webhook service path which admission requests are routed to for validating Pod resources.
+	ValidationPath = "/validate-v1-pod"
+)
+
 // Add registers the webhook for K8s bulit-in object types.
 func Add(mgr manager.Manager) error {
 	hookServer := mgr.GetWebhookServer()
-	hookServer.Register("/validate-v1-pod", &webhook.Admission{Handler: &podValidator{Client: mgr.GetClient()}})
+	hookServer.Register(ValidationPath, &webhook.Admission{Handler: &podValidator{Client: mgr.GetClient()}})
 	return nil
 }