diff --git a/Makefile b/Makefile index e9cb87f..65963fe 100644 --- a/Makefile +++ b/Makefile @@ -160,10 +160,20 @@ test-e2e-teardown: $(KIND) $(e2e_targets):: test-e2e-setup test-e2e:: $(e2e_tests) ## Run e2e tests -test-e2e-ansible:: image/ansible-operator ## Run Ansible e2e tests +test-e2e-ansible:: image/ansible-operator test-e2e-ansible-run ## Run Ansible e2e tests + +.PHONY: test-e2e-ansible-run +test-e2e-ansible-run: go test ./test/e2e/ansible -v -ginkgo.v -test-e2e-ansible-molecule:: install dev-install image/ansible-operator ## Run molecule-based Ansible e2e tests + +test-e2e-ansible-molecule:: install dev-install image/ansible-operator test-e2e-ansible-molecule-generate test-e2e-ansible-molecule-run ## Run molecule-based Ansible e2e tests + +.PHONY: test-e2e-ansible-molecule-generate +test-e2e-ansible-molecule-generate: go run ./hack/generate/samples/molecule/generate.go + +.PHONY: test-e2e-ansible-molecule-run +test-e2e-ansible-molecule-run: ./hack/tests/e2e-ansible-molecule.sh ## Location to install dependencies to diff --git a/hack/generate/samples/ansible/constants.go b/hack/generate/samples/ansible/constants.go index 2932922..8e85d0b 100644 --- a/hack/generate/samples/ansible/constants.go +++ b/hack/generate/samples/ansible/constants.go @@ -518,6 +518,25 @@ const rolesForBaseOperator = ` - watch #+kubebuilder:scaffold:rules ` +const rolesForProject = ` + ## + ## Apply customize roles related to project.openshift.io + ## + - apiGroups: + - project.openshift.io + resources: + - projectrequests + - projects + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +# +kubebuilder:scaffold:rules +` const customMetricsTest = ` - name: Search for all running pods diff --git a/hack/generate/samples/ansible/generate.go b/hack/generate/samples/ansible/generate.go index c797f4d..cb3d1cc 100644 --- a/hack/generate/samples/ansible/generate.go +++ b/hack/generate/samples/ansible/generate.go @@ -42,6 +42,12 @@ var memcachedGVK = schema.GroupVersionKind{ Kind: "Memcached", } +var skipSecretGeneration = "" + +func init() { + skipSecretGeneration = os.Getenv("SKIP_SECRET_GENERATION") +} + func getCli() *cli.CLI { ansibleBundle, _ := plugin.NewBundleWithOptions( plugin.WithName(golang.DefaultNameQualifier), @@ -128,21 +134,25 @@ func GenerateMoleculeSample(samplesPath string) []sample.Sample { ) pkg.CheckError("attempting to create sample cli", err) - addIgnore, err := samplecli.NewCliSample( - samplecli.WithCLI(getCli()), - samplecli.WithCommandContext(ansibleMoleculeMemcached.CommandContext()), - samplecli.WithGvk( - schema.GroupVersionKind{ - Group: "ignore", - Version: "v1", - Kind: "Secret", - }, - ), - samplecli.WithPlugins("ansible"), - samplecli.WithExtraApiOptions("--generate-role"), - samplecli.WithName(ansibleMoleculeMemcached.Name()), - ) - pkg.CheckError("creating ignore samples", err) + var addIgnore sample.Sample + + if skipSecretGeneration == "" { + addIgnore, err = samplecli.NewCliSample( + samplecli.WithCLI(getCli()), + samplecli.WithCommandContext(ansibleMoleculeMemcached.CommandContext()), + samplecli.WithGvk( + schema.GroupVersionKind{ + Group: "ignore", + Version: "v1", + Kind: "Secret", + }, + ), + samplecli.WithPlugins("ansible"), + samplecli.WithExtraApiOptions("--generate-role"), + samplecli.WithName(ansibleMoleculeMemcached.Name()), + ) + pkg.CheckError("creating ignore samples", err) + } // remove sample directory if it already exists err = os.RemoveAll(ansibleMoleculeMemcached.Dir()) @@ -159,9 +169,11 @@ func GenerateMoleculeSample(samplesPath string) []sample.Sample { err = e2e.AllowProjectBeMultiGroup(ansibleMoleculeMemcached) pkg.CheckError("updating PROJECT file", err) - ignoreGen := sample.NewGenerator(sample.WithNoInit(), sample.WithNoWebhook()) - err = ignoreGen.GenerateSamples(addIgnore) - pkg.CheckError("generating ansible molecule sample - ignore", err) + if skipSecretGeneration == "" { + ignoreGen := sample.NewGenerator(sample.WithNoInit(), sample.WithNoWebhook()) + err = ignoreGen.GenerateSamples(addIgnore) + pkg.CheckError("generating ansible molecule sample - ignore", err) + } ImplementMemcached(ansibleMoleculeMemcached, bundleImage) diff --git a/hack/generate/samples/ansible/memcached_molecule.go b/hack/generate/samples/ansible/memcached_molecule.go index 6a32f16..9282af4 100644 --- a/hack/generate/samples/ansible/memcached_molecule.go +++ b/hack/generate/samples/ansible/memcached_molecule.go @@ -40,12 +40,14 @@ func ImplementMemcachedMolecule(sample sample.Sample, image string) { err := kbutil.InsertCode(moleculeTaskPath, targetMoleculeCheckDeployment, molecuTaskToCheckConfigMap) pkg.CheckError("replacing memcached task to add config map check", err) - log.Info("insert molecule task to ensure to check secret") - err = kbutil.InsertCode(moleculeTaskPath, memcachedCustomStatusMoleculeTarget, testSecretMoleculeCheck) - pkg.CheckError("replacing memcached task to add secret check", err) + if skipSecretGeneration == "" { + log.Info("insert molecule task to ensure to check secret") + err = kbutil.InsertCode(moleculeTaskPath, memcachedCustomStatusMoleculeTarget, testSecretMoleculeCheck) + pkg.CheckError("replacing memcached task to add secret check", err) + } log.Info("insert molecule task to ensure to foo ") - err = kbutil.InsertCode(moleculeTaskPath, testSecretMoleculeCheck, testFooMoleculeCheck) + err = kbutil.InsertCode(moleculeTaskPath, memcachedCustomStatusMoleculeTarget, testFooMoleculeCheck) pkg.CheckError("replacing memcached task to add foo check", err) log.Info("insert molecule task to check custom metrics") @@ -57,6 +59,11 @@ func ImplementMemcachedMolecule(sample sample.Sample, image string) { err = kbutil.InsertCode(filepath.Join(sample.Dir(), "roles", strings.ToLower(gvk.Kind), "tasks", "main.yml"), roleFragment, memcachedWithBlackListTask) pkg.CheckError("replacing in tasks/main.yml", err) + + log.Info("adding RBAC permissions for project.openshift.io") + err = kbutil.ReplaceInFile(filepath.Join(sample.Dir(), "config", "rbac", "role.yaml"), + "# +kubebuilder:scaffold:rules", rolesForProject) + pkg.CheckError("replacing in role.yml", err) } if gvk.Kind != "Memcached" { @@ -93,35 +100,37 @@ func ImplementMemcachedMolecule(sample sample.Sample, image string) { "playbook: playbooks/memcached.yml", memcachedWatchCustomizations) pkg.CheckError("replacing in watches", err) - log.Info("removing ignore group for the secret from watches as an workaround to work with core types") - err = kbutil.ReplaceInFile(filepath.Join(sample.Dir(), "watches.yaml"), - "ignore.example.com", "\"\"") - pkg.CheckError("replacing the watches file", err) - - log.Info("removing molecule test for the Secret since it is a core type") - cmd := exec.Command("rm", "-rf", filepath.Join(sample.Dir(), "molecule", "default", "tasks", "secret_test.yml")) - _, err = sample.CommandContext().Run(cmd) - pkg.CheckError("removing secret test file", err) - - log.Info("adding Secret task to the role") - err = kbutil.ReplaceInFile(filepath.Join(sample.Dir(), "roles", "secret", "tasks", "main.yml"), - originalTaskSecret, taskForSecret) - pkg.CheckError("replacing in secret/tasks/main.yml file", err) - - log.Info("adding ManageStatus == false for role secret") - err = kbutil.ReplaceInFile(filepath.Join(sample.Dir(), "watches.yaml"), - "role: secret", manageStatusFalseForRoleSecret) - pkg.CheckError("replacing in watches.yaml", err) - - // prevent high load of controller caused by watching all the secrets in the cluster - watchNamespacePatchFileName := "watch_namespace_patch.yaml" - log.Info("adding WATCH_NAMESPACE env patch to watch own namespace") - err = os.WriteFile(filepath.Join(sample.Dir(), "config", "testing", watchNamespacePatchFileName), []byte(watchNamespacePatch), 0644) - pkg.CheckError("adding watch_namespace_patch.yaml", err) - - log.Info("adding WATCH_NAMESPACE env patch to patch list to be applied") - err = kbutil.InsertCode(filepath.Join(sample.Dir(), "config", "testing", "kustomization.yaml"), "patches:", - fmt.Sprintf("\n- path: %s", watchNamespacePatchFileName)) - pkg.CheckError("inserting in kustomization.yaml", err) + if skipSecretGeneration == "" { + log.Info("removing ignore group for the secret from watches as an workaround to work with core types") + err = kbutil.ReplaceInFile(filepath.Join(sample.Dir(), "watches.yaml"), + "ignore.example.com", "\"\"") + pkg.CheckError("replacing the watches file", err) + + log.Info("removing molecule test for the Secret since it is a core type") + cmd := exec.Command("rm", "-rf", filepath.Join(sample.Dir(), "molecule", "default", "tasks", "secret_test.yml")) + _, err = sample.CommandContext().Run(cmd) + pkg.CheckError("removing secret test file", err) + + log.Info("adding Secret task to the role") + err = kbutil.ReplaceInFile(filepath.Join(sample.Dir(), "roles", "secret", "tasks", "main.yml"), + originalTaskSecret, taskForSecret) + pkg.CheckError("replacing in secret/tasks/main.yml file", err) + + log.Info("adding ManageStatus == false for role secret") + err = kbutil.ReplaceInFile(filepath.Join(sample.Dir(), "watches.yaml"), + "role: secret", manageStatusFalseForRoleSecret) + pkg.CheckError("replacing in watches.yaml", err) + + // prevent high load of controller caused by watching all the secrets in the cluster + watchNamespacePatchFileName := "watch_namespace_patch.yaml" + log.Info("adding WATCH_NAMESPACE env patch to watch own namespace") + err = os.WriteFile(filepath.Join(sample.Dir(), "config", "testing", watchNamespacePatchFileName), []byte(watchNamespacePatch), 0644) + pkg.CheckError("adding watch_namespace_patch.yaml", err) + + log.Info("adding WATCH_NAMESPACE env patch to patch list to be applied") + err = kbutil.InsertCode(filepath.Join(sample.Dir(), "config", "testing", "kustomization.yaml"), "patches:", + fmt.Sprintf("\n- path: %s", watchNamespacePatchFileName)) + pkg.CheckError("inserting in kustomization.yaml", err) + } } diff --git a/hack/tests/e2e-ansible-molecule.sh b/hack/tests/e2e-ansible-molecule.sh index 263b858..eb9f582 100755 --- a/hack/tests/e2e-ansible-molecule.sh +++ b/hack/tests/e2e-ansible-molecule.sh @@ -40,21 +40,27 @@ ROOTDIR="$(pwd)" cp -r $ROOTDIR/testdata/ansible/memcached-molecule-operator/ $TMPDIR/memcached-molecule-operator cp -r $ROOTDIR/testdata/ansible/advanced-molecule-operator/ $TMPDIR/advanced-molecule-operator -pushd $TMPDIR/memcached-molecule-operator +# Skip Kind test with memcached-molecule-operator if ADVANCED_MOLECULE_OPERATOR_IMAGE has a value. +if [ -z "${ADVANCED_MOLECULE_OPERATOR_IMAGE-}" ] ; then + pushd $TMPDIR/memcached-molecule-operator -header_text "Running Kind test with memcached-molecule-operator" -make kustomize -if [ -f ./bin/kustomize ] ; then - KUSTOMIZE="$(realpath ./bin/kustomize)" -else - KUSTOMIZE="$(which kustomize)" + header_text "Running Kind test with memcached-molecule-operator" + make kustomize + if [ -f ./bin/kustomize ] ; then + KUSTOMIZE="$(realpath ./bin/kustomize)" + else + KUSTOMIZE="$(which kustomize)" + fi + KUSTOMIZE_PATH=${KUSTOMIZE} TEST_OPERATOR_NAMESPACE=default molecule test -s kind + popd fi -KUSTOMIZE_PATH=${KUSTOMIZE} TEST_OPERATOR_NAMESPACE=default molecule test -s kind -popd header_text "Running Default test with advanced-molecule-operator" -make test-e2e-setup +# Skip creation of Kind cluster if ADVANCED_MOLECULE_OPERATOR_IMAGE has a value. +if [ -z "${ADVANCED_MOLECULE_OPERATOR_IMAGE-}" ] ; then + make test-e2e-setup +fi pushd $TMPDIR/advanced-molecule-operator make kustomize @@ -64,8 +70,16 @@ else KUSTOMIZE="$(which kustomize)" fi -DEST_IMAGE="quay.io/example/advanced-molecule-operator:v0.0.1" -docker build -t "$DEST_IMAGE" --no-cache . -load_image_if_kind "$DEST_IMAGE" -KUSTOMIZE_PATH=$KUSTOMIZE OPERATOR_PULL_POLICY=Never OPERATOR_IMAGE=${DEST_IMAGE} TEST_OPERATOR_NAMESPACE=osdk-test molecule test +# Check if ADVANCED_MOLECULE_OPERATOR_IMAGE has value or not. If it doesn't have a value then proceed with the test +# using a Kind cluster, otherwise proceed with the test without the Kind cluster. +if [ -z "${ADVANCED_MOLECULE_OPERATOR_IMAGE-}" ] ; then + DEST_IMAGE="quay.io/example/advanced-molecule-operator:v0.0.1" + docker build -t "$DEST_IMAGE" --no-cache . + load_image_if_kind "$DEST_IMAGE" + IMAGE_PULL_POLICY="Never" +else + DEST_IMAGE=${ADVANCED_MOLECULE_OPERATOR_IMAGE} + IMAGE_PULL_POLICY="IfNotPresent" +fi +KUSTOMIZE_PATH=$KUSTOMIZE OPERATOR_PULL_POLICY=${IMAGE_PULL_POLICY} OPERATOR_IMAGE=${DEST_IMAGE} TEST_OPERATOR_NAMESPACE=osdk-test molecule test popd diff --git a/pkg/testutils/e2e/metrics/helpers.go b/pkg/testutils/e2e/metrics/helpers.go index bbf915e..009ae4e 100644 --- a/pkg/testutils/e2e/metrics/helpers.go +++ b/pkg/testutils/e2e/metrics/helpers.go @@ -14,8 +14,25 @@ import ( // GetMetrics creates a pod with the permissions to `curl` metrics. It will then return the output of the `curl` pod func GetMetrics(sample sample.Sample, kubectl kubernetes.Kubectl, metricsClusterRoleBindingName string) string { ginkgo.By("creating a curl pod") + securityContext := ` +{ + "apiVersion": "v1", + "spec": { + "securityContext": { + "runAsNonRoot": true, + "runAsUser": 1000, + "runAsGroup": 1000, + "fsGroup": 1000, + "seccompProfile": { + "type": "RuntimeDefault" + } + } + } +} +` + overrides := fmt.Sprintf("--overrides=%s", securityContext) cmdOpts := []string{ - "run", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure", "--", + "run", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure", overrides, "--", "curl", "-v", fmt.Sprintf("http://%s-controller-manager-metrics-service.%s.svc:8443/metrics", sample.Name(), kubectl.Namespace()), } diff --git a/test/e2e/ansible/local_test.go b/test/e2e/ansible/local_test.go index 0731280..6558924 100644 --- a/test/e2e/ansible/local_test.go +++ b/test/e2e/ansible/local_test.go @@ -15,6 +15,7 @@ package e2e_ansible_test import ( + "os" "os/exec" . "github.com/onsi/ginkgo/v2" @@ -24,19 +25,30 @@ import ( var _ = Describe("Running Ansible projects", func() { Context("built with operator-sdk", func() { + var ( + skip_local_test string + ) BeforeEach(func() { - By("Installing CRD's") - err := operator.InstallCRDs(ansibleSample) - Expect(err).NotTo(HaveOccurred()) + skip_local_test = os.Getenv(SKIP_LOCAL_TEST) + if skip_local_test == "" { + By("Installing CRD's") + err := operator.InstallCRDs(ansibleSample) + Expect(err).NotTo(HaveOccurred()) + } }) AfterEach(func() { - By("Uninstalling CRD's") - err := operator.UninstallCRDs(ansibleSample) - Expect(err).NotTo(HaveOccurred()) + if skip_local_test == "" { + By("Uninstalling CRD's") + err := operator.UninstallCRDs(ansibleSample) + Expect(err).NotTo(HaveOccurred()) + } }) It("Should run correctly when run locally", func() { + if skip_local_test != "" { + Skip("Skipping local test") + } By("Running the project") cmd := exec.Command("make", "run") cmd.Dir = ansibleSample.Dir() diff --git a/test/e2e/ansible/suite_test.go b/test/e2e/ansible/suite_test.go index 3b043a4..a09b4da 100644 --- a/test/e2e/ansible/suite_test.go +++ b/test/e2e/ansible/suite_test.go @@ -56,6 +56,11 @@ var ( image = "e2e-test-ansible:temp" ) +const ( + MEMCACHED_MOLECULE_OPERATOR_IMAGE = "MEMCACHED_MOLECULE_OPERATOR_IMAGE" + SKIP_LOCAL_TEST = "SKIP_LOCAL_TEST" +) + // BeforeSuite run before any specs are run to perform the required actions for all e2e ansible tests. var _ = BeforeSuite(func() { wd, err := os.Getwd() @@ -105,9 +110,13 @@ var _ = BeforeSuite(func() { }, 3*time.Minute, time.Second).Should(Succeed()) } - By("building the project image") - err = operator.BuildOperatorImage(ansibleSample, image) - Expect(err).NotTo(HaveOccurred()) + if image_env_var, exists := os.LookupEnv(MEMCACHED_MOLECULE_OPERATOR_IMAGE); exists { + image = image_env_var + } else { + By("building the project image") + err = operator.BuildOperatorImage(ansibleSample, image) + Expect(err).NotTo(HaveOccurred()) + } onKind, err := kind.IsRunningOnKind(kctl) Expect(err).NotTo(HaveOccurred()) @@ -128,9 +137,11 @@ var _ = AfterSuite(func() { } By("destroying container image and work dir") - cmd := exec.Command("docker", "rmi", "-f", image) - if _, err := ansibleSample.CommandContext().Run(cmd); err != nil { - Expect(err).To(BeNil()) + if _, exists := os.LookupEnv(MEMCACHED_MOLECULE_OPERATOR_IMAGE); !exists { + cmd := exec.Command("docker", "rmi", "-f", image) + if _, err := ansibleSample.CommandContext().Run(cmd); err != nil { + Expect(err).To(BeNil()) + } } if err := os.RemoveAll(testdir); err != nil { Expect(err).To(BeNil()) diff --git a/testdata/memcached-molecule-operator/config/rbac/role.yaml b/testdata/memcached-molecule-operator/config/rbac/role.yaml index 3289d44..481e26e 100644 --- a/testdata/memcached-molecule-operator/config/rbac/role.yaml +++ b/testdata/memcached-molecule-operator/config/rbac/role.yaml @@ -106,6 +106,23 @@ rules: - update - watch + ## + ## Apply customize roles related to project.openshift.io + ## + - apiGroups: + - project.openshift.io + resources: + - projectrequests + - projects + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + ## ## Apply customize roles for base operator ## @@ -124,3 +141,4 @@ rules: - watch #+kubebuilder:scaffold:rules + diff --git a/testdata/memcached-molecule-operator/molecule/default/tasks/memcached_test.yml b/testdata/memcached-molecule-operator/molecule/default/tasks/memcached_test.yml index 399bd4e..2b01fc5 100644 --- a/testdata/memcached-molecule-operator/molecule/default/tasks/memcached_test.yml +++ b/testdata/memcached-molecule-operator/molecule/default/tasks/memcached_test.yml @@ -56,12 +56,6 @@ resource_name=custom_resource.metadata.name )}}' -# This will verify that the secret role was executed -- name: Verify that test-service was created - assert: - that: lookup('k8s', kind='Service', api_version='v1', namespace=namespace, resource_name='test-service') - - - name: Verify that project testing-foo was created assert: that: lookup('k8s', kind='Namespace', api_version='v1', resource_name='testing-foo') @@ -116,6 +110,12 @@ +# This will verify that the secret role was executed +- name: Verify that test-service was created + assert: + that: lookup('k8s', kind='Service', api_version='v1', namespace=namespace, resource_name='test-service') + + - when: molecule_yml.scenario.name == "test-local" block: - name: Restart the operator by killing the pod