From ce5f2015bf4d333a91181b48972c0a4ec4bc20a8 Mon Sep 17 00:00:00 2001 From: Sean Hobbs Date: Mon, 31 Oct 2022 09:17:27 -0700 Subject: [PATCH] feat: Enabled webhook in upstream & prod via config. (#330) * Added configuration for Pod & CRP webhooks to either be service ref or URL (DNS). * Added E2E tests. Co-authored-by: Sean Hobbs --- Makefile | 4 +- charts/hub-agent/templates/deployment.yaml | 1 + .../hub-agent/templates/webhookservice.yaml | 5 +- charts/hub-agent/values.yaml | 3 +- cmd/hubagent/main.go | 48 +- cmd/hubagent/options/options.go | 6 +- cmd/hubagent/options/validation.go | 5 + cmd/hubagent/options/webhookconnectiontype.go | 33 ++ .../clusterresourceplacement_webhook.go | 5 + pkg/webhook/pod/pod_validating_webhook.go | 7 +- pkg/webhook/webhook.go | 150 ++++-- test/e2e/e2e_test.go | 2 +- test/e2e/webhook_test.go | 499 ++++++++++++++++++ 13 files changed, 684 insertions(+), 84 deletions(-) create mode 100644 cmd/hubagent/options/webhookconnectiontype.go create mode 100644 test/e2e/webhook_test.go diff --git a/Makefile b/Makefile index 848cb333b..9d9ccad61 100644 --- a/Makefile +++ b/Makefile @@ -140,7 +140,9 @@ install-hub-agent-helm: --set image.repository=$(REGISTRY)/$(HUB_AGENT_IMAGE_NAME) \ --set image.tag=$(HUB_AGENT_IMAGE_VERSION) \ --set logVerbosity=5 \ - --set namespace=fleet-system + --set namespace=fleet-system \ + --set enableWebhook=true \ + --set webhookClientConnectionType=service .PHONY: e2e-hub-kubeconfig-secret e2e-hub-kubeconfig-secret: diff --git a/charts/hub-agent/templates/deployment.yaml b/charts/hub-agent/templates/deployment.yaml index b7be0c89b..e7b238ba1 100644 --- a/charts/hub-agent/templates/deployment.yaml +++ b/charts/hub-agent/templates/deployment.yaml @@ -22,6 +22,7 @@ spec: args: - --leader-elect=true - --enable-webhook={{ .Values.enableWebhook }} + - --webhook-client-connection-type={{.Values.webhookClientConnectionType}} - --v={{ .Values.logVerbosity }} - -add_dir_header ports: diff --git a/charts/hub-agent/templates/webhookservice.yaml b/charts/hub-agent/templates/webhookservice.yaml index 16a641b0f..70cb92e09 100644 --- a/charts/hub-agent/templates/webhookservice.yaml +++ b/charts/hub-agent/templates/webhookservice.yaml @@ -1,4 +1,4 @@ -# We use a headless service for webhook assuming the apiserver's dns can resolve it. +# The webhook will normally use a service reference with a cluster assigned IP. apiVersion: v1 kind: Service metadata: @@ -7,9 +7,6 @@ metadata: name: fleetwebhook namespace: {{ .Values.namespace }} spec: - clusterIP: None - clusterIPs: - - None ipFamilies: - IPv4 ipFamilyPolicy: SingleStack diff --git a/charts/hub-agent/values.yaml b/charts/hub-agent/values.yaml index 0f3884609..121e38f9f 100644 --- a/charts/hub-agent/values.yaml +++ b/charts/hub-agent/values.yaml @@ -12,7 +12,8 @@ image: logVerbosity: 5 -enableWebhook: false +enableWebhook: true +webhookClientConnectionType: service namespace: fleet-system diff --git a/cmd/hubagent/main.go b/cmd/hubagent/main.go index 45945b26a..6df887790 100644 --- a/cmd/hubagent/main.go +++ b/cmd/hubagent/main.go @@ -2,10 +2,10 @@ Copyright (c) Microsoft Corporation. Licensed under the MIT license. */ + package main import ( - "context" "flag" "os" @@ -25,7 +25,7 @@ import ( "go.goms.io/fleet/pkg/controllers/membercluster" fleetmetrics "go.goms.io/fleet/pkg/metrics" "go.goms.io/fleet/pkg/webhook" - //+kubebuilder:scaffold:imports + // +kubebuilder:scaffold:imports ) var ( @@ -49,7 +49,7 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(fleetv1alpha1.AddToScheme(scheme)) utilruntime.Must(workv1alpha1.AddToScheme(scheme)) - //+kubebuilder:scaffold:scheme + // +kubebuilder:scaffold:scheme klog.InitFlags(nil) metrics.Registry.MustRegister(fleetmetrics.JoinResultMetrics, fleetmetrics.LeaveResultMetrics, fleetmetrics.PlacementApplyFailedCount, fleetmetrics.PlacementApplySucceedCount) @@ -109,7 +109,7 @@ func main() { } if opts.EnableWebhook { - if err := SetupWebhook(mgr); err != nil { + if err := SetupWebhook(mgr, options.WebhookClientConnectionType(opts.WebhookClientConnectionType)); err != nil { klog.ErrorS(err, "unable to set up webhook") exitWithErrorFunc() } @@ -121,7 +121,7 @@ func main() { exitWithErrorFunc() } - //+kubebuilder:scaffold:builder + // +kubebuilder:scaffold:builder if err := mgr.Start(ctx); err != nil { klog.ErrorS(err, "problem starting manager") @@ -129,43 +129,21 @@ func main() { } } -// SetupWebhook generate the webhook cert and then setup the webhook configurator -func SetupWebhook(mgr manager.Manager) error { - // Generate self-signed key and crt files in FleetWebhookCertDir for the webhook server to start - caPEM, err := webhook.GenCertificate(FleetWebhookCertDir) +// SetupWebhook generates the webhook cert and then set up the webhook configurator. +func SetupWebhook(mgr manager.Manager, webhookClientConnectionType options.WebhookClientConnectionType) error { + // Generate self-signed key and crt files in FleetWebhookCertDir for the webhook server to start. + w, err := webhook.NewWebhookConfig(mgr, FleetWebhookPort, &webhookClientConnectionType, FleetWebhookCertDir) if err != nil { - klog.ErrorS(err, "fail to generate certificates for webhook server") + klog.ErrorS(err, "fail to generate WebhookConfig") return err } - - if err := mgr.Add(&webhookApiserverConfigurator{ - mgr: mgr, - caPEM: caPEM, - port: FleetWebhookPort, - }); err != nil { - klog.ErrorS(err, "unable to add webhookApiserverConfigurator") + if err = mgr.Add(w); err != nil { + klog.ErrorS(err, "unable to add WebhookConfig") return err } - if err := webhook.AddToManager(mgr); err != nil { + if err = webhook.AddToManager(mgr); err != nil { klog.ErrorS(err, "unable to register webhooks to the manager") return err } return nil } - -type webhookApiserverConfigurator struct { - mgr manager.Manager - caPEM []byte - port int -} - -var _ manager.Runnable = &webhookApiserverConfigurator{} - -func (c *webhookApiserverConfigurator) Start(ctx context.Context) error { - klog.V(2).InfoS("setting up webhooks in apiserver from the leader") - if err := webhook.CreateFleetWebhookConfiguration(ctx, c.mgr.GetClient(), c.caPEM, c.port); err != nil { - klog.ErrorS(err, "unable to setup webhook configurations in apiserver") - return err - } - return nil -} diff --git a/cmd/hubagent/options/options.go b/cmd/hubagent/options/options.go index b1c3f51d7..7a5039577 100644 --- a/cmd/hubagent/options/options.go +++ b/cmd/hubagent/options/options.go @@ -2,6 +2,7 @@ Copyright (c) Microsoft Corporation. Licensed under the MIT license. */ + package options import ( @@ -34,6 +35,8 @@ type Options struct { MetricsBindAddress string // EnableWebhook indicates if we will run a webhook EnableWebhook bool + // Sets the connection type for the webhook. + WebhookClientConnectionType string // NetworkingAgentsEnabled indicates if we enable network agents NetworkingAgentsEnabled bool // ClusterUnhealthyThreshold is the duration of failure for the cluster to be considered unhealthy. @@ -87,7 +90,8 @@ func (o *Options) AddFlags(flags *flag.FlagSet) { flags.BoolVar(&o.LeaderElection.LeaderElect, "leader-elect", false, "Start a leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.") flags.DurationVar(&o.LeaderElection.LeaseDuration.Duration, "leader-lease-duration", 15*time.Second, "This is effectively the maximum duration that a leader can be stopped before someone else will replace it.") flag.StringVar(&o.LeaderElection.ResourceNamespace, "leader-election-namespace", utils.FleetSystemNamespace, "The namespace in which the leader election resource will be created.") - flag.BoolVar(&o.EnableWebhook, "enable-webhook", false, "If set, the fleet webhook is enabled.") + flag.BoolVar(&o.EnableWebhook, "enable-webhook", true, "If set, the fleet webhook is enabled.") + flag.StringVar(&o.WebhookClientConnectionType, "webhook-client-connection-type", "url", "Sets the connection type used by the webhook client. Only URL or Service is valid.") flag.BoolVar(&o.NetworkingAgentsEnabled, "networking-agents-enabled", false, "Whether the networking agents are enabled or not.") flags.DurationVar(&o.ClusterUnhealthyThreshold.Duration, "cluster-unhealthy-threshold", 60*time.Second, "The duration for a member cluster to be in a degraded state before considered unhealthy.") flags.DurationVar(&o.WorkPendingGracePeriod.Duration, "work-pending-grace-period", 15*time.Second, diff --git a/cmd/hubagent/options/validation.go b/cmd/hubagent/options/validation.go index ad44aee3d..28f6f19d2 100644 --- a/cmd/hubagent/options/validation.go +++ b/cmd/hubagent/options/validation.go @@ -29,5 +29,10 @@ func (o *Options) Validate() field.ErrorList { errs = append(errs, field.Invalid(newPath.Child("WorkPendingGracePeriod"), o.WorkPendingGracePeriod, "must be greater than 0")) } + connectionType := o.WebhookClientConnectionType + if _, err := parseWebhookClientConnectionString(connectionType); err != nil { + errs = append(errs, field.Invalid(newPath.Child("WebhookClientConnectionType"), o.EnableWebhook, err.Error())) + } + return errs } diff --git a/cmd/hubagent/options/webhookconnectiontype.go b/cmd/hubagent/options/webhookconnectiontype.go new file mode 100644 index 000000000..6c1bd8562 --- /dev/null +++ b/cmd/hubagent/options/webhookconnectiontype.go @@ -0,0 +1,33 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package options + +import ( + "errors" + "strings" +) + +type WebhookClientConnectionType string + +const ( + URL WebhookClientConnectionType = "url" + Service WebhookClientConnectionType = "service" +) + +var ( + capabilitiesMap = map[string]WebhookClientConnectionType{ + "service": Service, + "url": URL, + } +) + +func parseWebhookClientConnectionString(str string) (WebhookClientConnectionType, error) { + t, ok := capabilitiesMap[strings.ToLower(str)] + if !ok { + return "", errors.New("must be \"service\" or \"url\"") + } + return t, nil +} diff --git a/pkg/webhook/clusterresourceplacement/clusterresourceplacement_webhook.go b/pkg/webhook/clusterresourceplacement/clusterresourceplacement_webhook.go index 1b83ce3d3..5e1a2a4b6 100644 --- a/pkg/webhook/clusterresourceplacement/clusterresourceplacement_webhook.go +++ b/pkg/webhook/clusterresourceplacement/clusterresourceplacement_webhook.go @@ -11,6 +11,11 @@ import ( fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1" ) +const ( + // ValidationPath is the webhook service path which admission requests are routed to for validating ClusterResourcePlacement resources. + ValidationPath = "/validate-fleet-azure-com-v1alpha1-clusterresourceplacement" +) + func Add(mgr manager.Manager) error { return (&fleetv1alpha1.ClusterResourcePlacement{}).SetupWebhookWithManager(mgr) } diff --git a/pkg/webhook/pod/pod_validating_webhook.go b/pkg/webhook/pod/pod_validating_webhook.go index c2a25bf6f..a16d491f1 100644 --- a/pkg/webhook/pod/pod_validating_webhook.go +++ b/pkg/webhook/pod/pod_validating_webhook.go @@ -18,10 +18,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) +const ( + // ValidationPath is the webhook service path which admission requests are routed to for validating Pod resources. + ValidationPath = "/validate-v1-pod" +) + // Add registers the webhook for K8s bulit-in object types. func Add(mgr manager.Manager) error { hookServer := mgr.GetWebhookServer() - hookServer.Register("/validate-v1-pod", &webhook.Admission{Handler: &podValidator{Client: mgr.GetClient()}}) + hookServer.Register(ValidationPath, &webhook.Admission{Handler: &podValidator{Client: mgr.GetClient()}}) return nil } diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index d46b26cd2..9868f86f1 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -13,6 +13,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "errors" "fmt" "math/big" "os" @@ -20,31 +21,26 @@ import ( "time" admv1 "k8s.io/api/admissionregistration/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/manager" fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1" + "go.goms.io/fleet/cmd/hubagent/options" + "go.goms.io/fleet/pkg/webhook/clusterresourceplacement" + "go.goms.io/fleet/pkg/webhook/pod" ) const ( FleetWebhookCertFileName = "tls.crt" FleetWebhookKeyFileName = "tls.key" FleetWebhookCfgName = "fleet-validating-webhook-configuration" + FleetWebhookSvcName = "fleetwebhook" ) -var WebhookServiceNs string - -func init() { - // We assume the Pod namespace should be passed to env through downward API in the Pod spec - WebhookServiceNs = os.Getenv("POD_NAMESPACE") - if WebhookServiceNs == "" { - panic("Fail to obtain Pod namespace from env") - } -} - var AddToManagerFuncs []func(manager.Manager) error // AddToManager adds all Controllers to the Manager @@ -57,16 +53,57 @@ func AddToManager(m manager.Manager) error { return nil } -// CreateFleetWebhookConfiguration creates the ValidatingWebhookConfiguration object for the webhook -func CreateFleetWebhookConfiguration(ctx context.Context, client client.Client, caPEM []byte, port int) error { +type Config struct { + mgr manager.Manager + + // webhook server info + serviceNamespace string + servicePort int32 + serviceURL string + + // caPEM is a PEM encoded CA bundle which will be used to validate the webhook's server certificate. + caPEM []byte + + clientConnectionType *options.WebhookClientConnectionType +} + +func NewWebhookConfig(mgr manager.Manager, port int, clientConnectionType *options.WebhookClientConnectionType, certDir string) (*Config, error) { + // We assume the Pod namespace should be passed to env through downward API in the Pod spec. + namespace := os.Getenv("POD_NAMESPACE") + if namespace == "" { + return nil, errors.New("fail to obtain Pod namespace from POD_NAMESPACE") + } + w := Config{ + mgr: mgr, + servicePort: int32(port), + serviceNamespace: namespace, + serviceURL: fmt.Sprintf("https://%s.%s.svc.cluster.local:%d", FleetWebhookSvcName, namespace, port), + clientConnectionType: clientConnectionType, + } + caPEM, err := w.genCertificate(certDir) + if err != nil { + return nil, err // TODO + } + w.caPEM = caPEM + return &w, err +} + +func (w *Config) Start(ctx context.Context) error { + klog.V(2).InfoS("setting up webhooks in apiserver from the leader") + if err := w.createFleetWebhookConfiguration(ctx); err != nil { + klog.ErrorS(err, "unable to setup webhook configurations in apiserver") + return err + } + return nil +} + +// createFleetWebhookConfiguration creates the ValidatingWebhookConfiguration object for the webhook +func (w *Config) createFleetWebhookConfiguration(ctx context.Context) error { failPolicy := admv1.Fail // reject request if the webhook doesn't work sideEffortsNone := admv1.SideEffectClassNone - - // We assume a headless service named fleetwebhook has been created in the pod namespace (e.g., via helm chart) - podWebhookURL := fmt.Sprintf("https://fleetwebhook.%s.svc.cluster.local:%d/validate-v1-pod", WebhookServiceNs, port) - crpWebhookURL := fmt.Sprintf("https://fleetwebhook.%s.svc.cluster.local:%d/validate-fleet-azure-com-v1alpha1-clusterresourceplacement", WebhookServiceNs, port) namespacedScope := admv1.NamespacedScope clusterScope := admv1.ClusterScope + whCfg := admv1.ValidatingWebhookConfiguration{ ObjectMeta: metav1.ObjectMeta{ Name: FleetWebhookCfgName, @@ -76,11 +113,8 @@ func CreateFleetWebhookConfiguration(ctx context.Context, client client.Client, }, Webhooks: []admv1.ValidatingWebhook{ { - Name: "fleet.pod.validating", - ClientConfig: admv1.WebhookClientConfig{ - URL: &podWebhookURL, - CABundle: caPEM, - }, + Name: "fleet.pod.validating", + ClientConfig: w.createClientConfig(corev1.Pod{}), FailurePolicy: &failPolicy, SideEffects: &sideEffortsNone, AdmissionReviewVersions: []string{"v1", "v1beta1"}, @@ -88,7 +122,7 @@ func CreateFleetWebhookConfiguration(ctx context.Context, client client.Client, Rules: []admv1.RuleWithOperations{ { Operations: []admv1.OperationType{ - admv1.OperationAll, + admv1.Create, }, Rule: admv1.Rule{ APIGroups: []string{""}, @@ -100,11 +134,8 @@ func CreateFleetWebhookConfiguration(ctx context.Context, client client.Client, }, }, { - Name: "fleet.clusterresourceplacement.validating", - ClientConfig: admv1.WebhookClientConfig{ - URL: &crpWebhookURL, - CABundle: caPEM, - }, + Name: "fleet.clusterresourceplacement.validating", + ClientConfig: w.createClientConfig(fleetv1alpha1.ClusterResourcePlacement{}), FailurePolicy: &failPolicy, SideEffects: &sideEffortsNone, AdmissionReviewVersions: []string{"v1", "v1beta1"}, @@ -112,7 +143,8 @@ func CreateFleetWebhookConfiguration(ctx context.Context, client client.Client, Rules: []admv1.RuleWithOperations{ { Operations: []admv1.OperationType{ - admv1.OperationAll, + admv1.Create, + admv1.Update, }, Rule: admv1.Rule{ APIGroups: []string{"fleet.azure.com"}, @@ -126,17 +158,17 @@ func CreateFleetWebhookConfiguration(ctx context.Context, client client.Client, }, } - if err := client.Create(ctx, &whCfg); err != nil { + if err := w.mgr.GetClient().Create(ctx, &whCfg); err != nil { if !apierrors.IsAlreadyExists(err) { return err } klog.V(2).InfoS("validatingwebhookconfiguration exists, need to update", "name", FleetWebhookCfgName) // Here we simply use delete/create pattern to implement full overwrite - err := client.Delete(ctx, &whCfg) + err := w.mgr.GetClient().Delete(ctx, &whCfg) if err != nil { return err } - err = client.Create(ctx, &whCfg) + err = w.mgr.GetClient().Create(ctx, &whCfg) if err != nil { return err } @@ -146,9 +178,37 @@ func CreateFleetWebhookConfiguration(ctx context.Context, client client.Client, return nil } -// GenCertificate generates the serving cerficiate for the webhook server -func GenCertificate(certDir string) ([]byte, error) { - caPEM, certPEM, keyPEM, err := genSelfSignedCert() +func (w *Config) createClientConfig(webhookInterface interface{}) admv1.WebhookClientConfig { + serviceRef := admv1.ServiceReference{ + Namespace: w.serviceNamespace, + Name: FleetWebhookSvcName, + Port: pointer.Int32(w.servicePort), + } + var serviceEndpoint string + switch webhookInterface.(type) { + case corev1.Pod: + serviceEndpoint = w.serviceURL + pod.ValidationPath + serviceRef.Path = pointer.String(pod.ValidationPath) + case fleetv1alpha1.ClusterResourcePlacement: + serviceEndpoint = w.serviceURL + clusterresourceplacement.ValidationPath + serviceRef.Path = pointer.String(clusterresourceplacement.ValidationPath) + } + + config := admv1.WebhookClientConfig{ + CABundle: w.caPEM, + } + switch *w.clientConnectionType { + case options.Service: + config.Service = &serviceRef + case options.URL: + config.URL = pointer.String(serviceEndpoint) + } + return config +} + +// genCertificate generates the serving cerficiate for the webhook server +func (w *Config) genCertificate(certDir string) ([]byte, error) { + caPEM, certPEM, keyPEM, err := w.genSelfSignedCert() if err != nil { klog.ErrorS(err, "fail to generate self-signed cert") return nil, err @@ -163,12 +223,17 @@ func GenCertificate(certDir string) ([]byte, error) { } // genSelfSignedCert generates the self signed Certificate/Key pair -func genSelfSignedCert() (caPEMByte, certPEMByte, keyPEMByte []byte, err error) { +func (w *Config) genSelfSignedCert() (caPEMByte, certPEMByte, keyPEMByte []byte, err error) { // CA config ca := &x509.Certificate{ SerialNumber: big.NewInt(2022), Subject: pkix.Name{ - Organization: []string{"fleet.azure.com"}, + CommonName: "fleet.azure.com", + OrganizationalUnit: []string{"Azure Kubernetes Service"}, + Organization: []string{"Microsoft"}, + Locality: []string{"Redmond"}, + Province: []string{"Washington"}, + Country: []string{"United States of America"}, }, NotBefore: time.Now(), NotAfter: time.Now().AddDate(10, 0, 0), // Set expiration time to be 10 years for now @@ -201,15 +266,20 @@ func genSelfSignedCert() (caPEMByte, certPEMByte, keyPEMByte []byte, err error) caPEMByte = caPEM.Bytes() dnsNames := []string{ - fmt.Sprintf("fleetwebhook.%s.svc.cluster.local", WebhookServiceNs), + fmt.Sprintf("%s.%s.svc", FleetWebhookSvcName, w.serviceNamespace), + fmt.Sprintf("%s.%s.svc.cluster.local", FleetWebhookSvcName, w.serviceNamespace), } // server cert config cert := &x509.Certificate{ DNSNames: dnsNames, SerialNumber: big.NewInt(2022), Subject: pkix.Name{ - CommonName: "fleetWebhookServer", - Organization: []string{"fleet.azure.com"}, + CommonName: fmt.Sprintf("%s.cert.server", FleetWebhookSvcName), + OrganizationalUnit: []string{"Azure Kubernetes Service"}, + Organization: []string{"Microsoft"}, + Locality: []string{"Redmond"}, + Province: []string{"Washington"}, + Country: []string{"United States of America"}, }, NotBefore: time.Now(), NotAfter: time.Now().AddDate(10, 0, 0), diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 3ecaf95ea..a723f01a1 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -164,7 +164,7 @@ var _ = BeforeSuite(func() { // hub setup HubCluster.HubURL = hubURL framework.GetClusterClient(HubCluster) - //member setup + // member setup MemberCluster.HubURL = hubURL framework.GetClusterClient(MemberCluster) diff --git a/test/e2e/webhook_test.go b/test/e2e/webhook_test.go new file mode 100644 index 000000000..191b4696f --- /dev/null +++ b/test/e2e/webhook_test.go @@ -0,0 +1,499 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package e2e + +import ( + errors "errors" + "fmt" + "reflect" + "regexp" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + admv1 "k8s.io/api/admissionregistration/v1" + corev1 "k8s.io/api/core/v1" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1" + "go.goms.io/fleet/pkg/utils" + testUtils "go.goms.io/fleet/test/e2e/utils" +) + +const ( + kubeSystemNs = "kube-system" + fleetSystemNs = "fleet-system" +) + +var ( + whitelistedNamespaces = []corev1.Namespace{ + {ObjectMeta: metav1.ObjectMeta{Name: kubeSystemNs}}, + {ObjectMeta: metav1.ObjectMeta{Name: fleetSystemNs}}, + } +) + +var _ = Describe("Fleet's Hub cluster webhook tests", func() { + Context("Pod validation webhook", func() { + It("should admit operations on Pods within whitelisted namespaces", func() { + for _, ns := range whitelistedNamespaces { + objKey := client.ObjectKey{Name: utils.RandStr(), Namespace: ns.ObjectMeta.Name} + nginxPod := &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: objKey.Name, + Namespace: objKey.Namespace, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx:1.14.2", + Ports: []corev1.ContainerPort{ + { + Name: "http", + Protocol: corev1.ProtocolTCP, + ContainerPort: 80, + }, + }, + }, + }, + }, + } + + By(fmt.Sprintf("expecting admission of operation CREATE of Pod in whitelisted namespace %s", ns.ObjectMeta.Name)) + Expect(HubCluster.KubeClient.Create(ctx, nginxPod)).Should(Succeed()) + + By(fmt.Sprintf("expecting admission of operation UPDATE of Pod in whitelisted namespace %s", ns.ObjectMeta.Name)) + var podV2 *corev1.Pod + Eventually(func() error { + var currentPod corev1.Pod + Expect(HubCluster.KubeClient.Get(ctx, objKey, ¤tPod)).Should(Succeed()) + podV2 = currentPod.DeepCopy() + podV2.Labels = map[string]string{utils.RandStr(): utils.RandStr()} + return HubCluster.KubeClient.Update(ctx, podV2) + }, testUtils.PollTimeout, testUtils.PollInterval).Should(Succeed()) + + By(fmt.Sprintf("expecting admission of operation DELETE of Pod in whitelisted namespace %s", ns.ObjectMeta.Name)) + Expect(HubCluster.KubeClient.Delete(ctx, nginxPod)).Should(Succeed()) + } + }) + It("should deny create operation on Pods within any non-whitelisted namespaces", func() { + rndNs := corev1.Namespace{ + TypeMeta: metav1.TypeMeta{ + Kind: "namespace", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: utils.RandStr(), + }, + } + Expect(HubCluster.KubeClient.Create(ctx, &rndNs)).Should(Succeed()) + + nginxPod := &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: utils.RandStr(), + Namespace: rndNs.Name, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx:1.14.2", + Ports: []corev1.ContainerPort{ + { + Name: "http", + Protocol: corev1.ProtocolTCP, + ContainerPort: 80, + }, + }, + }, + }, + }, + } + + By(fmt.Sprintf("expecting denial of operation %s of Pod in non-whitelisted namespace %s", admv1.Create, rndNs.Name)) + err := HubCluster.KubeClient.Create(ctx, nginxPod) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create Pod call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(`admission webhook.*denied the request.*`)) + }) + }) + Context("ClusterResourcePlacement validation webhook", func() { + It("should admit write operations for valid ClusterResourcePlacement resources", func() { + By("attempting to create a CRP") + validCRP := fleetv1alpha1.ClusterResourcePlacement{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterResourcePlacement", + APIVersion: "v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: utils.RandStr(), + }, + Spec: fleetv1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []fleetv1alpha1.ClusterResourceSelector{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: utils.RandStr(), + }, + }, + }, + } + Expect(HubCluster.KubeClient.Create(ctx, &validCRP)).Should(Succeed()) + + By("attempting to update a CRP") + var createdCRP fleetv1alpha1.ClusterResourcePlacement + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, client.ObjectKey{Name: validCRP.Name}, &createdCRP); err != nil { + return err + } + // check conditions to infer we have latest + if len(createdCRP.Status.Conditions) == 0 { + return fmt.Errorf("failed to get crp condition, want not empty") + } + return nil + }, testUtils.PollTimeout, testUtils.PollInterval).Should(Succeed()) + + createdCRP.Spec.ResourceSelectors[0].Name = utils.RandStr() + Expect(HubCluster.KubeClient.Update(ctx, &createdCRP)).Should(Succeed()) + + By("attempting to delete a CRP") + Expect(HubCluster.KubeClient.Delete(ctx, &createdCRP)).Should(Succeed()) + }) + It("should deny write operations for ClusterResourcePlacements which specify both a label & name within a resource selector", func() { + invalidCRP := fleetv1alpha1.ClusterResourcePlacement{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterResourcePlacement", + APIVersion: "v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: utils.RandStr(), + }, + Spec: fleetv1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []fleetv1alpha1.ClusterResourceSelector{ + { + Group: "core", + Version: "v1", + Kind: "Pod", + Name: utils.RandStr(), + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"SomeKey": "SomeValue"}, + }, + }, + }, + }, + } + + By("attempting to create the invalid resource") + err := HubCluster.KubeClient.Create(ctx, &invalidCRP) + Expect(err).Should(HaveOccurred()) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create ClusterResourcePlacement call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(`admission webhook "fleet.clusterresourceplacement.validating" denied the request`)) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("the labelSelector and name fields are mutually exclusive")) + + By("attempting to update an existing resource with the invalid spec") + // Create a valid CRP. + validCRP := fleetv1alpha1.ClusterResourcePlacement{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterResourcePlacement", + APIVersion: "v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: utils.RandStr(), + }, + Spec: fleetv1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []fleetv1alpha1.ClusterResourceSelector{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: utils.RandStr(), + }, + }, + }, + } + Expect(HubCluster.KubeClient.Create(ctx, &validCRP)).Should(Succeed()) + + // Get the created CRP + var createdCRP fleetv1alpha1.ClusterResourcePlacement + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, client.ObjectKey{Name: validCRP.Name}, &createdCRP); err != nil { + return err + } + // check conditions to infer we have latest + if len(createdCRP.Status.Conditions) == 0 { + return fmt.Errorf("failed to get crp condition, want not empty") + } + return nil + }, testUtils.PollTimeout, testUtils.PollInterval).Should(Succeed()) + + // Set the spec to be invalid & attempt update + createdCRP.Spec = invalidCRP.Spec + err = HubCluster.KubeClient.Update(ctx, &createdCRP) + Expect(err).Should(HaveOccurred()) + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create ClusterResourcePlacement call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(`admission webhook "fleet.clusterresourceplacement.validating" denied the request`)) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("the labelSelector and name fields are mutually exclusive")) + }) + It("should deny write operations for ClusterResourcePlacements which specify an invalid cluster label selector", func() { + invalidCRP := fleetv1alpha1.ClusterResourcePlacement{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterResourcePlacement", + APIVersion: "v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: utils.RandStr(), + }, + Spec: fleetv1alpha1.ClusterResourcePlacementSpec{ + Policy: &fleetv1alpha1.PlacementPolicy{ + ClusterNames: nil, + Affinity: &fleetv1alpha1.Affinity{ + ClusterAffinity: &fleetv1alpha1.ClusterAffinity{ + ClusterSelectorTerms: []fleetv1alpha1.ClusterSelectorTerm{ + { + LabelSelector: metav1.LabelSelector{ + MatchLabels: nil, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "SomeKey", + Operator: "invalid-operator", + Values: []string{"SomeValue"}, + }, + }, + }, + }, + }, + }, + }, + }, + ResourceSelectors: []fleetv1alpha1.ClusterResourceSelector{}, + }, + } + + By("attempting to create the invalid resource") + err := HubCluster.KubeClient.Create(ctx, &invalidCRP) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create ClusterResourcePlacement call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(`admission webhook "fleet.clusterresourceplacement.validating" denied the request`)) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(regexp.QuoteMeta(fmt.Sprintf("the labelSelector in cluster selector %+v is invalid:", invalidCRP.Spec.Policy.Affinity.ClusterAffinity.ClusterSelectorTerms[0])))) + + By("attempting to update an existing resource with the invalid spec") + // Create a valid CRP. + validCRP := fleetv1alpha1.ClusterResourcePlacement{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterResourcePlacement", + APIVersion: "v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: utils.RandStr(), + }, + Spec: fleetv1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []fleetv1alpha1.ClusterResourceSelector{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: utils.RandStr(), + }, + }, + }, + } + Expect(HubCluster.KubeClient.Create(ctx, &validCRP)).Should(Succeed()) + + // Get the created CRP + var createdCRP fleetv1alpha1.ClusterResourcePlacement + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, client.ObjectKey{Name: validCRP.Name}, &createdCRP); err != nil { + return err + } + // check conditions to infer we have latest + if len(createdCRP.Status.Conditions) == 0 { + return fmt.Errorf("failed to get crp condition, want not empty") + } + return nil + }, testUtils.PollTimeout, testUtils.PollInterval).Should(Succeed()) + + // Set the spec to be invalid & attempt update + createdCRP.Spec = invalidCRP.Spec + err = HubCluster.KubeClient.Update(ctx, &createdCRP) + Expect(err).Should(HaveOccurred()) + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create ClusterResourcePlacement call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(`admission webhook "fleet.clusterresourceplacement.validating" denied the request`)) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(regexp.QuoteMeta(fmt.Sprintf("the labelSelector in cluster selector %+v is invalid:", invalidCRP.Spec.Policy.Affinity.ClusterAffinity.ClusterSelectorTerms[0])))) + }) + It("should deny write operations for ClusterResourcePlacements which specify an invalid resource label selector", func() { + invalidCRP := fleetv1alpha1.ClusterResourcePlacement{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterResourcePlacement", + APIVersion: "v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: utils.RandStr(), + }, + Spec: fleetv1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []fleetv1alpha1.ClusterResourceSelector{ + { + Group: "", + Version: "v1", + Kind: "namespace", + Name: "", + LabelSelector: &metav1.LabelSelector{ + MatchLabels: nil, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "SomeKey", + Operator: "invalid-operator", + Values: []string{"SomeValue"}, + }, + }, + }, + }, + }, + }, + } + + By("attempting to create the invalid resource") + err := HubCluster.KubeClient.Create(ctx, &invalidCRP) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create ClusterResourcePlacement call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(`admission webhook "fleet.clusterresourceplacement.validating" denied the request`)) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(regexp.QuoteMeta(fmt.Sprintf("the labelSelector in resource selector %+v is invalid:", invalidCRP.Spec.ResourceSelectors[0])))) + + By("attempting to update an existing resource with the invalid spec") + // Create a valid CRP. + validCRP := fleetv1alpha1.ClusterResourcePlacement{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterResourcePlacement", + APIVersion: "v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: utils.RandStr(), + }, + Spec: fleetv1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []fleetv1alpha1.ClusterResourceSelector{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: utils.RandStr(), + }, + }, + }, + } + Expect(HubCluster.KubeClient.Create(ctx, &validCRP)).Should(Succeed()) + + // Get the created CRP + var createdCRP fleetv1alpha1.ClusterResourcePlacement + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, client.ObjectKey{Name: validCRP.Name}, &createdCRP); err != nil { + return err + } + // check conditions to infer we have latest + if len(createdCRP.Status.Conditions) == 0 { + return fmt.Errorf("failed to get crp condition, want not empty") + } + return nil + }, testUtils.PollTimeout, testUtils.PollInterval).Should(Succeed()) + + // Set the spec to be invalid & attempt update + createdCRP.Spec = invalidCRP.Spec + err = HubCluster.KubeClient.Update(ctx, &createdCRP) + Expect(err).Should(HaveOccurred()) + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create ClusterResourcePlacement call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(`admission webhook "fleet.clusterresourceplacement.validating" denied the request`)) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(regexp.QuoteMeta(fmt.Sprintf("the labelSelector in resource selector %+v is invalid:", invalidCRP.Spec.ResourceSelectors[0])))) + }) + It("should deny write operations for ClusterResourcePlacements which specify an invalid GVK within the resource selector", func() { + invalidCRP := fleetv1alpha1.ClusterResourcePlacement{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterResourcePlacement", + APIVersion: "v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: utils.RandStr(), + }, + Spec: fleetv1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []fleetv1alpha1.ClusterResourceSelector{ + { + Group: utils.RandStr(), + Version: utils.RandStr(), + Kind: utils.RandStr(), + Name: utils.RandStr(), + LabelSelector: &metav1.LabelSelector{}, + }, + }, + }, + } + + invalidGVK := metav1.GroupVersionKind{ + Group: invalidCRP.Spec.ResourceSelectors[0].Group, + Version: invalidCRP.Spec.ResourceSelectors[0].Version, + Kind: invalidCRP.Spec.ResourceSelectors[0].Kind, + } + + By("attempting to create the invalid resource") + err := HubCluster.KubeClient.Create(ctx, &invalidCRP) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create ClusterResourcePlacement call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(`admission webhook "fleet.clusterresourceplacement.validating" denied the request`)) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(regexp.QuoteMeta(fmt.Sprintf("the resource is not found in schema (please retry) or it is not a cluster scoped resource: %s", invalidGVK)))) + + By("attempting to update an existing resource with the invalid spec") + // Create a valid CRP. + validCRP := fleetv1alpha1.ClusterResourcePlacement{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterResourcePlacement", + APIVersion: "v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: utils.RandStr(), + }, + Spec: fleetv1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []fleetv1alpha1.ClusterResourceSelector{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: utils.RandStr(), + }, + }, + }, + } + Expect(HubCluster.KubeClient.Create(ctx, &validCRP)).Should(Succeed()) + + // Get the created CRP + var createdCRP fleetv1alpha1.ClusterResourcePlacement + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, client.ObjectKey{Name: validCRP.Name}, &createdCRP); err != nil { + return err + } + // check conditions to infer we have latest + if len(createdCRP.Status.Conditions) == 0 { + return fmt.Errorf("failed to get crp condition, want not empty") + } + return nil + }, testUtils.PollTimeout, testUtils.PollInterval).Should(Succeed()) + + // Set the spec to be invalid & attempt update + createdCRP.Spec = invalidCRP.Spec + err = HubCluster.KubeClient.Update(ctx, &createdCRP) + Expect(err).Should(HaveOccurred()) + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create ClusterResourcePlacement call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(`admission webhook "fleet.clusterresourceplacement.validating" denied the request`)) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(regexp.QuoteMeta(fmt.Sprintf("the resource is not found in schema (please retry) or it is not a cluster scoped resource: %s", invalidGVK)))) + }) + }) +})