From 9dc0f6347a38afb205ce38e18656e279457ce218 Mon Sep 17 00:00:00 2001 From: akutz Date: Wed, 27 Dec 2023 12:00:56 -0600 Subject: [PATCH] Support e2e integration tests w all webhooks & controller This patch introduces the new function "NewTestSuiteWithOptions" for creating a test suite that supports all webhooks, both conversion and admission, as well as controllers. This enables end-to-end testing of a controller via integration tests where the production webhooks are installed and part of the reconciliation. --- api/v1alpha1/virtualmachine_conversion.go | 4 + api/v1alpha1/virtualmachine_webhook.go | 4 +- api/v1alpha2/virtualmachine_webhook.go | 7 +- .../virtualmachine_controller_intg_test.go | 2 +- .../virtualmachine_controller_suite_test.go | 52 +- pkg/manager/manager.go | 2 + pkg/manager/options.go | 6 +- test/builder/test_suite.go | 470 ++++++++++++++---- test/builder/util.go | 4 + .../mutation/virtualmachine_mutator.go | 3 + .../validation/virtualmachine_validator.go | 2 + .../mutation/virtualmachine_mutator.go | 3 + .../validation/virtualmachine_validator.go | 2 + 13 files changed, 459 insertions(+), 102 deletions(-) diff --git a/api/v1alpha1/virtualmachine_conversion.go b/api/v1alpha1/virtualmachine_conversion.go index 77bdbcb51..5be5ddf8c 100644 --- a/api/v1alpha1/virtualmachine_conversion.go +++ b/api/v1alpha1/virtualmachine_conversion.go @@ -882,6 +882,8 @@ func restore_v1alpha2_VirtualMachineReadinessProbeSpec( // ConvertTo converts this VirtualMachine to the Hub version. func (src *VirtualMachine) ConvertTo(dstRaw conversion.Hub) error { + fmt.Println("v1a1/vm convert to") + dst := dstRaw.(*v1alpha2.VirtualMachine) if err := Convert_v1alpha1_VirtualMachine_To_v1alpha2_VirtualMachine(src, dst, nil); err != nil { return err @@ -904,6 +906,8 @@ func (src *VirtualMachine) ConvertTo(dstRaw conversion.Hub) error { // ConvertFrom converts the hub version to this VirtualMachine. func (dst *VirtualMachine) ConvertFrom(srcRaw conversion.Hub) error { + fmt.Println("v1a1/vm convert from") + src := srcRaw.(*v1alpha2.VirtualMachine) if err := Convert_v1alpha2_VirtualMachine_To_v1alpha1_VirtualMachine(src, dst, nil); err != nil { return err diff --git a/api/v1alpha1/virtualmachine_webhook.go b/api/v1alpha1/virtualmachine_webhook.go index de33a451f..dcd0c34e2 100644 --- a/api/v1alpha1/virtualmachine_webhook.go +++ b/api/v1alpha1/virtualmachine_webhook.go @@ -3,7 +3,9 @@ package v1alpha1 -import ctrl "sigs.k8s.io/controller-runtime" +import ( + ctrl "sigs.k8s.io/controller-runtime" +) func (r *VirtualMachine) SetupWebhookWithManager(mgr ctrl.Manager) error { return ctrl.NewWebhookManagedBy(mgr). diff --git a/api/v1alpha2/virtualmachine_webhook.go b/api/v1alpha2/virtualmachine_webhook.go index f4f73e0a7..7a03a67b1 100644 --- a/api/v1alpha2/virtualmachine_webhook.go +++ b/api/v1alpha2/virtualmachine_webhook.go @@ -3,9 +3,14 @@ package v1alpha2 -import ctrl "sigs.k8s.io/controller-runtime" +import ( + "fmt" + + ctrl "sigs.k8s.io/controller-runtime" +) func (r *VirtualMachine) SetupWebhookWithManager(mgr ctrl.Manager) error { + fmt.Println("v1a2/vm convert") return ctrl.NewWebhookManagedBy(mgr). For(r). Complete() diff --git a/controllers/virtualmachine/v1alpha2/virtualmachine_controller_intg_test.go b/controllers/virtualmachine/v1alpha2/virtualmachine_controller_intg_test.go index 46852c1d1..3ab432fe0 100644 --- a/controllers/virtualmachine/v1alpha2/virtualmachine_controller_intg_test.go +++ b/controllers/virtualmachine/v1alpha2/virtualmachine_controller_intg_test.go @@ -116,7 +116,7 @@ func intgTests() { }) }) - It("Reconciles after VirtualMachine creation", func() { + FIt("Reconciles after VirtualMachine creation", func() { Expect(ctx.Client.Create(ctx, vm)).To(Succeed()) By("VirtualMachine should have finalizer added", func() { diff --git a/controllers/virtualmachine/v1alpha2/virtualmachine_controller_suite_test.go b/controllers/virtualmachine/v1alpha2/virtualmachine_controller_suite_test.go index 3c9c44dbf..7de5c41d8 100644 --- a/controllers/virtualmachine/v1alpha2/virtualmachine_controller_suite_test.go +++ b/controllers/virtualmachine/v1alpha2/virtualmachine_controller_suite_test.go @@ -10,22 +10,60 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" + vmopv1a1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + vmopv1a2 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" virtualmachine "github.com/vmware-tanzu/vm-operator/controllers/virtualmachine/v1alpha2" ctrlContext "github.com/vmware-tanzu/vm-operator/pkg/context" "github.com/vmware-tanzu/vm-operator/pkg/lib" + pkgmgr "github.com/vmware-tanzu/vm-operator/pkg/manager" providerfake "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/fake" "github.com/vmware-tanzu/vm-operator/test/builder" + mutationv1a1 "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachine/v1alpha1/mutation" + validationv1a1 "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachine/v1alpha1/validation" + mutationv1a2 "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachine/v1alpha2/mutation" + validationv1a2 "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachine/v1alpha2/validation" ) var intgFakeVMProvider = providerfake.NewVMProviderA2() -var suite = builder.NewTestSuiteForControllerWithFSS( - virtualmachine.AddToManager, - func(ctx *ctrlContext.ControllerManagerContext, _ ctrlmgr.Manager) error { - ctx.VMProviderA2 = intgFakeVMProvider - return nil - }, - map[string]bool{lib.VMServiceV1Alpha2FSS: true}) +var suite = builder.NewTestSuiteWithOptions( + builder.TestSuiteOptions{ + InitProviderFn: func(ctx *ctrlContext.ControllerManagerContext, _ ctrlmgr.Manager) error { + ctx.VMProviderA2 = intgFakeVMProvider + return nil + }, + FeatureStates: map[string]bool{lib.VMServiceV1Alpha2FSS: true}, + Controllers: []pkgmgr.AddToManagerFunc{virtualmachine.AddToManager}, + ConversionWebhooks: []builder.TestSuiteConversionWebhookOptions{ + { + Name: "virtualmachines.vmoperator.vmware.com", + AddToManagerFn: []func(ctrlmgr.Manager) error{ + (&vmopv1a1.VirtualMachine{}).SetupWebhookWithManager, + (&vmopv1a2.VirtualMachine{}).SetupWebhookWithManager, + }, + }, + }, + MutationWebhooks: []builder.TestSuiteMutationWebhookOptions{ + { + Name: "default.mutating.virtualmachine.v1alpha1.vmoperator.vmware.com", + AddToManagerFn: mutationv1a1.AddToManager, + }, + { + Name: "default.mutating.virtualmachine.v1alpha2.vmoperator.vmware.com", + AddToManagerFn: mutationv1a2.AddToManager, + }, + }, + ValidationWebhooks: []builder.TestSuiteValidationWebhookOptions{ + { + Name: "default.validating.virtualmachine.v1alpha1.vmoperator.vmware.com", + AddToManagerFn: validationv1a1.AddToManager, + }, + { + Name: "default.validating.virtualmachine.v1alpha2.vmoperator.vmware.com", + AddToManagerFn: validationv1a2.AddToManager, + }, + }, + }) func TestVirtualMachine(t *testing.T) { suite.Register(t, "VirtualMachine controller suite", intgTests, unitTests) diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 374763b4a..f7135a6cd 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" @@ -46,6 +47,7 @@ func New(opts Options) (Manager, error) { opts.defaults() _ = clientgoscheme.AddToScheme(opts.Scheme) + _ = apiextensionsv1.AddToScheme(opts.Scheme) _ = vmopv1.AddToScheme(opts.Scheme) _ = ncpv1alpha1.AddToScheme(opts.Scheme) _ = cnsv1alpha1.AddToScheme(opts.Scheme) diff --git a/pkg/manager/options.go b/pkg/manager/options.go index 7f02fca82..e9496e605 100644 --- a/pkg/manager/options.go +++ b/pkg/manager/options.go @@ -37,7 +37,7 @@ var InitializeProvidersNoopFn InitializeProvidersFunc = func(_ *context.Controll return nil } -// Options describes the options used to create a new GCM manager. +// Options describes the options used to create a new manager. type Options struct { // LeaderElectionEnabled is a flag that enables leader election. LeaderElectionEnabled bool @@ -46,8 +46,8 @@ type Options struct { // locking resource when configuring leader election. LeaderElectionID string - // HealthProbeBindAddress is the TCP address that the controller should bind to - // for serving health probes + // HealthProbeBindAddress is the TCP address to which the controller should + // bind for serving health probes. HealthProbeBindAddress string // SyncPeriod is the amount of time to wait between syncing the local diff --git a/test/builder/test_suite.go b/test/builder/test_suite.go index 9ab9e8200..796b17a40 100644 --- a/test/builder/test_suite.go +++ b/test/builder/test_suite.go @@ -33,7 +33,9 @@ import ( "k8s.io/client-go/rest" "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -74,33 +76,58 @@ type TestSuite struct { envTest envtest.Environment config *rest.Config integrationTestClient client.Client + integrationTest bool + initProvidersFn pkgmgr.InitializeProvidersFunc + + // Controller manager specific fields. + manager pkgmgr.Manager + managerRunning bool + managerRunningMutex sync.Mutex // Cancel function that will be called to close the Done channel of the // Context, which will then stop the manager. cancelFuncMutex sync.Mutex cancelFunc context.CancelFunc - // Controller specific fields - addToManagerFn pkgmgr.AddToManagerFunc - initProvidersFn pkgmgr.InitializeProvidersFunc - integrationTest bool - manager pkgmgr.Manager - managerRunning bool - managerRunningMutex sync.Mutex + // Controller specific fields. + controllers []pkgmgr.AddToManagerFunc - // Webhook specific fields - webhookName string - certDir string - validatorFn builder.ValidatorFunc - mutatorFn builder.MutatorFunc - pki pkiToolchain - webhookYaml []byte + // Webhook specific fields. + webhook testSuiteWebhookConfig + // Feature state switches. fssMap map[string]bool } +type testSuiteWebhookConfig struct { + certDir string + pki pkiToolchain + conversionOpts []TestSuiteConversionWebhookOptions + conversionName []string + mutationOpts []TestSuiteMutationWebhookOptions + mutationYAML []byte + validationOpts []TestSuiteValidationWebhookOptions + validationYAML []byte +} + func (s *TestSuite) isWebhookTest() bool { - return s.webhookName != "" + return s.isAdmissionWebhookTest() || s.isConversionWebhookTest() +} + +func (s *TestSuite) isAdmissionWebhookTest() bool { + return s.isMutationWebhookTest() || s.isValidationWebhookTest() +} + +func (s *TestSuite) isConversionWebhookTest() bool { + return len(s.webhook.conversionOpts) > 0 +} + +func (s *TestSuite) isMutationWebhookTest() bool { + return len(s.webhook.mutationOpts) > 0 +} + +func (s *TestSuite) isValidationWebhookTest() bool { + return len(s.webhook.validationOpts) > 0 } func (s *TestSuite) GetEnvTestConfig() *rest.Config { @@ -149,7 +176,7 @@ func NewTestSuiteForControllerWithFSS(addToManagerFn pkgmgr.AddToManagerFunc, testSuite := &TestSuite{ Context: context.Background(), integrationTest: true, - addToManagerFn: addToManagerFn, + controllers: []pkgmgr.AddToManagerFunc{addToManagerFn}, initProvidersFn: initProvidersFn, fssMap: fssMap, } @@ -158,6 +185,73 @@ func NewTestSuiteForControllerWithFSS(addToManagerFn pkgmgr.AddToManagerFunc, return testSuite } +type TestSuiteOptions struct { + InitProviderFn pkgmgr.InitializeProvidersFunc + FeatureStates map[string]bool + Controllers []pkgmgr.AddToManagerFunc + ValidationWebhooks []TestSuiteValidationWebhookOptions + MutationWebhooks []TestSuiteMutationWebhookOptions + ConversionWebhooks []TestSuiteConversionWebhookOptions +} + +type TestSuiteValidationWebhookOptions struct { + Name string + AddToManagerFn pkgmgr.AddToManagerFunc + ValidatorFn builder.ValidatorFunc +} + +type TestSuiteMutationWebhookOptions struct { + Name string + AddToManagerFn pkgmgr.AddToManagerFunc + MutatorFn builder.MutatorFunc +} + +type TestSuiteConversionWebhookOptions struct { + Name string + AddToManagerFn []func(ctrl.Manager) error +} + +// NewTestSuiteWithOptions returns a new test suite used for controller integration test with FSS set. +func NewTestSuiteWithOptions(opts TestSuiteOptions) *TestSuite { + + if len(opts.Controllers) == 0 && + len(opts.ValidationWebhooks) == 0 && + len(opts.MutationWebhooks) == 0 && + len(opts.ConversionWebhooks) == 0 { + + panic("there are no addToManager functions") + } + if opts.InitProviderFn == nil { + panic("initProvidersFn is nil") + } + + testSuite := &TestSuite{ + Context: context.Background(), + integrationTest: true, + controllers: opts.Controllers, + initProvidersFn: opts.InitProviderFn, + fssMap: opts.FeatureStates, + webhook: testSuiteWebhookConfig{ + conversionOpts: opts.ConversionWebhooks, + mutationOpts: opts.MutationWebhooks, + validationOpts: opts.ValidationWebhooks, + }, + } + + if testSuite.isWebhookTest() { + // Create a temp directory for the certs needed for testing webhooks. + certDir, err := os.MkdirTemp(os.TempDir(), "") + if err != nil { + panic(errors.Wrap(err, "failed to create temp dir for certs")) + } + testSuite.webhook.certDir = certDir + } + + testSuite.init() + + return testSuite +} + // NewTestSuiteForValidatingWebhook returns a new test suite used for unit and // integration testing validating webhooks created using the "pkg/builder" // package. @@ -214,17 +308,27 @@ func newTestSuiteForWebhook( testSuite := &TestSuite{ Context: context.Background(), integrationTest: true, - addToManagerFn: addToManagerFn, initProvidersFn: pkgmgr.InitializeProvidersNoopFn, - webhookName: webhookName, fssMap: fssMap, } - if newValidatorFn != nil { - testSuite.validatorFn = newValidatorFn - } if newMutatorFn != nil { - testSuite.mutatorFn = newMutatorFn + testSuite.webhook.mutationOpts = []TestSuiteMutationWebhookOptions{ + { + Name: webhookName, + MutatorFn: newMutatorFn, + AddToManagerFn: addToManagerFn, + }, + } + } + if newValidatorFn != nil { + testSuite.webhook.validationOpts = []TestSuiteValidationWebhookOptions{ + { + Name: webhookName, + ValidatorFn: newValidatorFn, + AddToManagerFn: addToManagerFn, + }, + } } // Create a temp directory for the certs needed for testing webhooks. @@ -232,7 +336,7 @@ func newTestSuiteForWebhook( if err != nil { panic(errors.Wrap(err, "failed to create temp dir for certs")) } - testSuite.certDir = certDir + testSuite.webhook.certDir = certDir testSuite.init() @@ -298,7 +402,7 @@ func (s *TestSuite) Register(t *testing.T, name string, runIntegrationTestsFn, r // suite's reconciler. // // Returns nil if unit testing is disabled. -func (s *TestSuite) NewUnitTestContextForController(initObjects ...client.Object) *UnitTestContextForController { +func (s *TestSuite) NewUnitTestContextForController(initObjects ...ctrlclient.Object) *UnitTestContextForController { if s.flags.UnitTestsEnabled { ctx := NewUnitTestContextForController(initObjects) return ctx @@ -312,10 +416,10 @@ func (s *TestSuite) NewUnitTestContextForController(initObjects ...client.Object // Returns nil if unit testing is disabled. func (s *TestSuite) NewUnitTestContextForValidatingWebhook( obj, oldObj *unstructured.Unstructured, - initObjects ...client.Object) *UnitTestContextForValidatingWebhook { + initObjects ...ctrlclient.Object) *UnitTestContextForValidatingWebhook { if s.flags.UnitTestsEnabled { - ctx := NewUnitTestContextForValidatingWebhook(s.validatorFn, obj, oldObj, initObjects...) + ctx := NewUnitTestContextForValidatingWebhook(s.webhook.validationOpts[0].ValidatorFn, obj, oldObj, initObjects...) return ctx } return nil @@ -327,7 +431,7 @@ func (s *TestSuite) NewUnitTestContextForValidatingWebhook( // Returns nil if unit testing is disabled. func (s *TestSuite) NewUnitTestContextForMutatingWebhook(obj *unstructured.Unstructured) *UnitTestContextForMutatingWebhook { if s.flags.UnitTestsEnabled { - ctx := NewUnitTestContextForMutatingWebhook(s.mutatorFn, obj) + ctx := NewUnitTestContextForMutatingWebhook(s.webhook.mutationOpts[0].MutatorFn, obj) return ctx } return nil @@ -354,8 +458,35 @@ func (s *TestSuite) createManager() { opts := pkgmgr.Options{ KubeConfig: s.config, MetricsAddr: "0", - AddToManager: s.addToManagerFn, InitializeProviders: s.initProvidersFn, + AddToManager: func( + ctx *ctrlCtx.ControllerManagerContext, + m manager.Manager) error { + + for i := range s.controllers { + if err := s.controllers[i](ctx, m); err != nil { + return err + } + } + for i := range s.webhook.conversionOpts { + for j := range s.webhook.conversionOpts[i].AddToManagerFn { + if err := s.webhook.conversionOpts[i].AddToManagerFn[j](m); err != nil { + return err + } + } + } + for i := range s.webhook.mutationOpts { + if err := s.webhook.mutationOpts[i].AddToManagerFn(ctx, m); err != nil { + return err + } + } + for i := range s.webhook.validationOpts { + if err := s.webhook.validationOpts[i].AddToManagerFn(ctx, m); err != nil { + return err + } + } + return nil + }, } if enabled, ok := s.fssMap[lib.VMServiceV1Alpha2FSS]; ok && enabled { @@ -376,7 +507,7 @@ func (s *TestSuite) initializeManager() { svr := s.manager.GetWebhookServer().(*webhook.DefaultServer) svr.Options.Host = "127.0.0.1" svr.Options.Port = randomTCPPort() - svr.Options.CertDir = s.certDir + svr.Options.CertDir = s.webhook.certDir }) } } @@ -419,30 +550,46 @@ func (s *TestSuite) postConfigureManager() { // for the webhook server to come online. if s.isWebhookTest() { By("installing the webhook(s)", func() { - // ASSERT that the file for validating webhook file exists. - validatingWebhookFile := path.Join(testutil.GetRootDirOrDie(), "config", "webhook", "manifests.yaml") - Expect(validatingWebhookFile).Should(BeAnExistingFile()) - - // UNMARSHAL the contents of the validating webhook file into MutatingWebhookConfiguration and - // ValidatingWebhookConfiguration. - mutatingWebhookConfig, validatingWebhookConfig := parseWebhookConfig(validatingWebhookFile) - - // MARSHAL the webhook config back to YAML. - if s.mutatorFn != nil { - By("installing the mutating webhook(s)") - svr := s.manager.GetWebhookServer().(*webhook.DefaultServer) - s.webhookYaml = updateMutatingWebhookConfig(mutatingWebhookConfig, s.webhookName, svr.Options.Host, svr.Options.Port, s.pki.publicKeyPEM) - } else { - By("installing the validating webhook(s)") - svr := s.manager.GetWebhookServer().(*webhook.DefaultServer) - s.webhookYaml = updateValidatingWebhookConfig(validatingWebhookConfig, s.webhookName, svr.Options.Host, svr.Options.Port, s.pki.publicKeyPEM) + svr := s.manager.GetWebhookServer().(*webhook.DefaultServer) + + if s.isConversionWebhookTest() { + By("installing the conversion webhook(s)", func() { + updateConversionWebhookConfig( + s, s.integrationTestClient, + s.webhook.conversionOpts, + svr.Options.Host, svr.Options.Port, + s.webhook.pki.publicKeyPEM, + &s.webhook.conversionName) + }) } - // ASSERT that eventually the webhook config gets successfully - // applied to the API server. - Eventually(func() error { - return remote.ApplyYAML(s, s.integrationTestClient, s.webhookYaml) - }).Should(Succeed()) + if s.isAdmissionWebhookTest() { + By("installing the admission webhook(s)", func() { + // ASSERT the admission webhook manifest file exists. + admissionWebhookManifestFilePath := path.Join( + testutil.GetRootDirOrDie(), + "config", "webhook", "manifests.yaml") + Expect(admissionWebhookManifestFilePath).Should(BeAnExistingFile()) + + // UNMARSHAL the contents of the admission webhook manifest file. + mutationWebhookConfig, validationWebhookConfig := parseAdmissionWebhookManifestFile( + admissionWebhookManifestFilePath) + + updateMutationWebhookConfig( + s, s.integrationTestClient, + s.webhook.mutationOpts, mutationWebhookConfig, + svr.Options.Host, svr.Options.Port, + s.webhook.pki.publicKeyPEM, + &s.webhook.mutationYAML) + + updateValidationWebhookConfig( + s, s.integrationTestClient, + s.webhook.validationOpts, validationWebhookConfig, + svr.Options.Host, svr.Options.Port, + s.webhook.pki.publicKeyPEM, + &s.webhook.validationYAML) + }) + } }) // It can take a few seconds for the webhook server to come online. @@ -492,11 +639,13 @@ func (s *TestSuite) beforeSuiteForIntegrationTesting() { // PKI toolchain to use with the webhook server. if s.isWebhookTest() { By("generating the pki toolchain", func() { - s.pki, err = generatePKIToolchain() + s.webhook.pki, err = generatePKIToolchain() Expect(err).ToNot(HaveOccurred()) // Write the CA pub key and cert pub and private keys to the cert dir. - Expect(os.WriteFile(path.Join(s.certDir, "tls.crt"), s.pki.publicKeyPEM, 0400)).To(Succeed()) - Expect(os.WriteFile(path.Join(s.certDir, "tls.key"), s.pki.privateKeyPEM, 0400)).To(Succeed()) + tlsCrtPath := path.Join(s.webhook.certDir, "tls.crt") + tlsKeyPath := path.Join(s.webhook.certDir, "tls.key") + Expect(os.WriteFile(tlsCrtPath, s.webhook.pki.publicKeyPEM, 0400)).To(Succeed()) + Expect(os.WriteFile(tlsKeyPath, s.webhook.pki.privateKeyPEM, 0400)).To(Succeed()) }) } @@ -506,7 +655,7 @@ func (s *TestSuite) beforeSuiteForIntegrationTesting() { s.initializeManager() }) - s.integrationTestClient, err = client.New(s.manager.GetConfig(), client.Options{Scheme: s.manager.GetScheme()}) + s.integrationTestClient, err = ctrlclient.New(s.manager.GetConfig(), ctrlclient.Options{Scheme: s.manager.GetScheme()}) Expect(err).NotTo(HaveOccurred()) By("create pod namespace", func() { @@ -543,9 +692,38 @@ func (s *TestSuite) afterSuiteForIntegrationTesting() { Eventually(s.getManagerRunning).Should(BeFalse()) - if s.webhookYaml != nil { + if data := s.webhook.conversionName; len(data) > 0 { + for i := range data { + name := data[i] + crd := apiextensionsv1.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apiextensions.k8s.io/v1", + Kind: "CustomResourceDefinition", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + Expect(s.integrationTestClient.Get(s, ctrlclient.ObjectKey{Name: name}, &crd)).To(Succeed()) + Expect(crd.Spec.Conversion).ToNot(BeNil()) + crd.Spec.Conversion = nil + Expect(s.integrationTestClient.Update(s, &crd)).To(Succeed()) + Expect(s.integrationTestClient.Get(s, ctrlclient.ObjectKey{Name: name}, &crd)).To(Succeed()) + Expect(crd.Spec.Conversion).ToNot(BeNil()) + Expect(crd.Spec.Conversion.Strategy).To(Equal(apiextensionsv1.NoneConverter)) + Expect(crd.Spec.Conversion.Webhook).To(BeNil()) + } + } + + if data := s.webhook.mutationYAML; len(data) > 0 { + Eventually(func() error { + return remote.DeleteYAML(s, s.integrationTestClient, data) + }).Should(Succeed()) + } + + if data := s.webhook.validationYAML; len(data) > 0 { Eventually(func() error { - return remote.DeleteYAML(s, s.integrationTestClient, s.webhookYaml) + return remote.DeleteYAML(s, s.integrationTestClient, data) }).Should(Succeed()) } }) @@ -604,7 +782,7 @@ func (s *TestSuite) UpdateCRDScope(oldCrd *apiextensionsv1.CustomResourceDefinit s.envTest.CRDs = append(s.envTest.CRDs, crds...) } -func parseWebhookConfig(path string) ( +func parseAdmissionWebhookManifestFile(path string) ( mutatingWebhookConfig admissionregv1.MutatingWebhookConfiguration, validatingWebhookConfig admissionregv1.ValidatingWebhookConfiguration) { @@ -629,44 +807,158 @@ func parseWebhookConfig(path string) ( return mutatingWebhookConfig, validatingWebhookConfig } -func updateValidatingWebhookConfig(webhookConfig admissionregv1.ValidatingWebhookConfiguration, webhookName, host string, port int, key []byte) []byte { +func updateConversionWebhookConfig( + ctx context.Context, + client ctrlclient.Client, + webhookOption []TestSuiteConversionWebhookOptions, + host string, + port int, + key []byte, + addrOfName *[]string) { + + if len(webhookOption) == 0 { + return + } + + for _, in := range webhookOption { + crd := apiextensionsv1.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apiextensions.k8s.io/v1", + Kind: "CustomResourceDefinition", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: in.Name, + }, + } + Expect(client.Get(ctx, ctrlclient.ObjectKey{Name: in.Name}, &crd)).To(Succeed()) + + crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{ + Strategy: apiextensionsv1.WebhookConverter, + Webhook: &apiextensionsv1.WebhookConversion{ + ConversionReviewVersions: []string{"v1", "v1beta1"}, + ClientConfig: &apiextensionsv1.WebhookClientConfig{}, + }, + } + updateApiExtensionWebhookConfig( + host, port, "/convert", + key, crd.Spec.Conversion.Webhook.ClientConfig) + + Expect(client.Update(ctx, &crd)).To(Succeed()) + + *addrOfName = append(*addrOfName, in.Name) + } +} + +func updateMutationWebhookConfig( + ctx context.Context, + client ctrlclient.Client, + webhookOption []TestSuiteMutationWebhookOptions, + webhookConfig admissionregv1.MutatingWebhookConfiguration, + host string, + port int, + key []byte, + addrOfYAML *[]byte) { + + if len(webhookOption) == 0 { + return + } + webhookConfigToInstall := webhookConfig.DeepCopy() - // ITERATE over all of the defined webhooks and find the webhook - // the test suite is testing and update its client config to point - // to the test webhook server. - // 1. Use the test CA - // 2. Use the test webhook endpoint - for _, webhook := range webhookConfig.Webhooks { - if webhook.Name == webhookName { - url := fmt.Sprintf("https://%s:%d%s", host, port, *webhook.ClientConfig.Service.Path) - webhook.ClientConfig.CABundle = key - webhook.ClientConfig.Service = nil - webhook.ClientConfig.URL = &url - webhookConfigToInstall.Webhooks = []admissionregv1.ValidatingWebhook{webhook} + webhookConfigToInstall.Webhooks = nil + + for _, in := range webhookOption { + for _, out := range webhookConfig.Webhooks { + if in.Name == out.Name { + updateAdmissionWebhookConfig( + host, port, *out.ClientConfig.Service.Path, key, + &out.ClientConfig) + webhookConfigToInstall.Webhooks = append( + webhookConfigToInstall.Webhooks, + out) + } } } - result, err := yaml.Marshal(webhookConfigToInstall) + + if len(webhookConfigToInstall.Webhooks) == 0 { + return + } + + By(fmt.Sprintf("Installing %d mutation webhooks", + len(webhookConfigToInstall.Webhooks))) + + data, err := yaml.Marshal(webhookConfigToInstall) Expect(err).ShouldNot(HaveOccurred()) - return result -} + *addrOfYAML = data + + // ASSERT that eventually the webhook config gets successfully + // applied to the API server. + Eventually(func() error { + return remote.ApplyYAML(ctx, client, data) + }).Should(Succeed()) +} + +func updateValidationWebhookConfig( + ctx context.Context, + client ctrlclient.Client, + webhookOption []TestSuiteValidationWebhookOptions, + webhookConfig admissionregv1.ValidatingWebhookConfiguration, + host string, + port int, + key []byte, + addrOfYAML *[]byte) { + + if len(webhookOption) == 0 { + return + } -func updateMutatingWebhookConfig(webhookConfig admissionregv1.MutatingWebhookConfiguration, webhookName, host string, port int, key []byte) []byte { webhookConfigToInstall := webhookConfig.DeepCopy() - // ITERATE over all of the defined webhooks and find the webhook - // the test suite is testing and update its client config to point - // to the test webhook server. - // 1. Use the test CA - // 2. Use the test webhook endpoint - for _, webhook := range webhookConfig.Webhooks { - if webhook.Name == webhookName { - url := fmt.Sprintf("https://%s:%d%s", host, port, *webhook.ClientConfig.Service.Path) - webhook.ClientConfig.CABundle = key - webhook.ClientConfig.Service = nil - webhook.ClientConfig.URL = &url - webhookConfigToInstall.Webhooks = []admissionregv1.MutatingWebhook{webhook} + webhookConfigToInstall.Webhooks = nil + + for _, in := range webhookOption { + for _, out := range webhookConfig.Webhooks { + if in.Name == out.Name { + updateAdmissionWebhookConfig( + host, port, *out.ClientConfig.Service.Path, key, + &out.ClientConfig) + webhookConfigToInstall.Webhooks = append( + webhookConfigToInstall.Webhooks, + out) + } } } - result, err := yaml.Marshal(webhookConfigToInstall) + + if len(webhookConfigToInstall.Webhooks) == 0 { + return + } + + By(fmt.Sprintf("Installing %d validation webhooks", + len(webhookConfigToInstall.Webhooks))) + + data, err := yaml.Marshal(webhookConfigToInstall) Expect(err).ShouldNot(HaveOccurred()) - return result + *addrOfYAML = data + + // ASSERT that eventually the webhook config gets successfully + // applied to the API server. + Eventually(func() error { + return remote.ApplyYAML(ctx, client, data) + }).Should(Succeed()) +} + +func updateAdmissionWebhookConfig( + host string, port int, path string, key []byte, + webhookConfig *admissionregv1.WebhookClientConfig) { + + webhookConfig.CABundle = key + webhookConfig.Service = nil + webhookConfig.URL = addrOf(fmt.Sprintf("https://%s:%d%s", host, port, path)) +} + +func updateApiExtensionWebhookConfig( + host string, port int, path string, key []byte, + webhookConfig *apiextensionsv1.WebhookClientConfig) { + + webhookConfig.CABundle = key + webhookConfig.Service = nil + webhookConfig.URL = addrOf(fmt.Sprintf("https://%s:%d%s", host, port, path)) } diff --git a/test/builder/util.go b/test/builder/util.go index 60cdcf9c3..480665585 100644 --- a/test/builder/util.go +++ b/test/builder/util.go @@ -662,3 +662,7 @@ func readDocuments(fp string) ([][]byte, error) { } return docs, nil } + +func addrOf[T any](t T) *T { + return &t +} diff --git a/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator.go b/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator.go index c66babe65..d6a57b894 100644 --- a/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator.go +++ b/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator.go @@ -5,6 +5,7 @@ package mutation import ( "encoding/json" + "fmt" "net/http" "os" "reflect" @@ -91,6 +92,8 @@ type mutator struct { } func (m mutator) Mutate(ctx *context.WebhookRequestContext) admission.Response { + fmt.Println("v1a1/vm mutate") + if ctx.Op == admissionv1.Delete { return admission.Allowed("") } diff --git a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go index b179281bc..cbca8038e 100644 --- a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go +++ b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go @@ -103,6 +103,8 @@ func (v validator) For() schema.GroupVersionKind { } func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission.Response { + fmt.Println("v1a1/vm validate create") + vm, err := v.vmFromUnstructured(ctx.Obj) if err != nil { return webhook.Errored(http.StatusBadRequest, err) diff --git a/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator.go b/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator.go index b655523e6..62f93563d 100644 --- a/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator.go +++ b/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator.go @@ -5,6 +5,7 @@ package mutation import ( "encoding/json" + "fmt" "net/http" "reflect" "strings" @@ -95,6 +96,8 @@ type mutator struct { } func (m mutator) Mutate(ctx *context.WebhookRequestContext) admission.Response { + fmt.Println("v1a2/vm mutate") + if ctx.Op == admissionv1.Delete { return admission.Allowed("") } diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go index 36b8f288b..98030be34 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go @@ -97,6 +97,8 @@ func (v validator) For() schema.GroupVersionKind { } func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission.Response { + fmt.Println("v1a2/vm validate create") + vm, err := v.vmFromUnstructured(ctx.Obj) if err != nil { return webhook.Errored(http.StatusBadRequest, err)