Setup Basic Tests for Ceph-CSI-Operator

Makefile

@@ -206,3 +206,7 @@ GOBIN=$(LOCALBIN) go install $${package} ;\
 mv "$$(echo "$(1)" | sed "s/-$(3)$$//")" $(1) ;\
 }
 endef
+
+.PHONY: test-e2e
+test-e2e:
+	go test ./test/e2e -v -ginkgo.v

config/manager/kustomization.yaml

@@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: quay.io/cephcsi/ceph-csi-operator
-  newTag: latest
+  newName: example.com/ceph-csi-operator
+  newTag: v0.0.1

test/e2e/e2e_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package e2e
 
 import (
+	"context"
 	"fmt"
 	"os/exec"
 	"time"
@@ -25,101 +26,128 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/ceph/ceph-csi-operator/test/utils"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/utils/ptr"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+
+	csiv1alpha1 "github.com/ceph/ceph-csi-operator/api/v1alpha1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const namespace = "ceph-csi-operator-system"
 
-var _ = Describe("controller", Ordered, func() {
+var k8sClient client.Client
+var clientset *kubernetes.Clientset
+
+var _ = Describe("Ceph CSI Operator", Ordered, func() {
 	BeforeAll(func() {
-		By("installing prometheus operator")
-		Expect(utils.InstallPrometheusOperator()).To(Succeed())
+		var err error
+		cfg, err := config.GetConfig()
+		Expect(err).NotTo(HaveOccurred())
+
+		err = csiv1alpha1.AddToScheme(scheme.Scheme)
+		Expect(err).NotTo(HaveOccurred())
 
-		By("installing the cert-manager")
-		Expect(utils.InstallCertManager()).To(Succeed())
+		k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
+		Expect(err).NotTo(HaveOccurred())
+
+		clientset, err = kubernetes.NewForConfig(cfg)
+		Expect(err).NotTo(HaveOccurred())
 
 		By("creating manager namespace")
-		//nolint:gosec
-		cmd := exec.Command("kubectl", "create", "ns", namespace)
-		_, _ = utils.Run(cmd)
+		ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}
+		err = k8sClient.Create(context.TODO(), ns)
+		Expect(err).NotTo(HaveOccurred())
 	})
 
 	AfterAll(func() {
-		By("uninstalling the Prometheus manager bundle")
-		utils.UninstallPrometheusOperator()
-
-		By("uninstalling the cert-manager bundle")
-		utils.UninstallCertManager()
-
 		By("removing manager namespace")
-		//nolint:gosec
-		cmd := exec.Command("kubectl", "delete", "ns", namespace)
-		_, _ = utils.Run(cmd)
+		ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}
+		err := k8sClient.Delete(context.TODO(), ns)
+		Expect(err).NotTo(HaveOccurred())
 	})
 
-	Context("Operator", func() {
-		It("should run successfully", func() {
-			var controllerPodName string
+	Context("Operator Deployment", func() {
+		It("should deploy and run successfully", func() {
 			var err error
 
-			// projectimage stores the name of the image used in the example
-			var projectimage = "example.com/ceph-csi-operator:v0.0.1"
+			projectimage := "example.com/ceph-csi-operator:v0.0.1"
 
-			By("building the manager(Operator) image")
-			//nolint:gosec
+			By("building the operator image")
 			cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage))
 			_, err = utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+			Expect(err).NotTo(HaveOccurred())
 
-			By("loading the the manager(Operator) image on Kind")
-			err = utils.LoadImageToKindClusterWithName(projectimage)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+			By("loading the operator image to the cluster")
+			err = utils.LoadImageToCluster(projectimage)
+			Expect(err).NotTo(HaveOccurred())
 
 			By("installing CRDs")
-			//nolint:gosec
 			cmd = exec.Command("make", "install")
 			_, err = utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+			Expect(err).NotTo(HaveOccurred())
 
-			By("deploying the controller-manager")
-			//nolint:gosec
+			By("deploying the operator")
 			cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage))
 			_, err = utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
-
-			By("validating that the controller-manager pod is running as expected")
-			verifyControllerUp := func() error {
-				// Get pod name
-				//nolint:gosec
-				cmd = exec.Command("kubectl", "get",
-					"pods", "-l", "control-plane=controller-manager",
-					"-o", "go-template={{ range .items }}"+
-						"{{ if not .metadata.deletionTimestamp }}"+
-						"{{ .metadata.name }}"+
-						"{{ \"\\n\" }}{{ end }}{{ end }}",
-					"-n", namespace,
-				)
-
-				podOutput, err := utils.Run(cmd)
-				ExpectWithOffset(2, err).NotTo(HaveOccurred())
-				podNames := utils.GetNonEmptyLines(string(podOutput))
-				if len(podNames) != 1 {
-					return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames))
+			Expect(err).NotTo(HaveOccurred())
+
+			By("validating that the operator pod is running")
+			Eventually(func() error {
+				pods, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{
+					LabelSelector: "control-plane=ceph-csi-op-controller-manager",
+				})
+				if err != nil {
+					return err
+				}
+				if len(pods.Items) != 1 {
+					return fmt.Errorf("expected 1 operator pod, but got %d", len(pods.Items))
 				}
-				controllerPodName = podNames[0]
-				ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager"))
-
-				// Validate pod status
-				//nolint:gosec
-				cmd = exec.Command("kubectl", "get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", namespace)
-				status, err := utils.Run(cmd)
-				ExpectWithOffset(2, err).NotTo(HaveOccurred())
-				if string(status) != "Running" {
-					return fmt.Errorf("controller pod in %s status", status)
+				if pods.Items[0].Status.Phase != corev1.PodRunning {
+					return fmt.Errorf("operator pod is in %s status", pods.Items[0].Status.Phase)
 				}
 				return nil
+			}, time.Minute, time.Second).Should(Succeed())
+		})
+	})
+
+	Context("Driver CR", func() {
+		It("should create a RBD Driver CR and verify resources", func() {
+			driver := &csiv1alpha1.Driver{
+				TypeMeta: metav1.TypeMeta{
+					APIVersion: "csi.ceph.io/v1alpha1",
+					Kind:       "Driver",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test.rbd.csi.ceph.com",
+					Namespace: namespace,
+				},
+				Spec: csiv1alpha1.DriverSpec{
+					AttachRequired:   ptr.To(true),
+					NodePlugin:       &csiv1alpha1.NodePluginSpec{},
+					ControllerPlugin: &csiv1alpha1.ControllerPluginSpec{},
+				},
 			}
-			EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed())
 
+			By("creating a Driver CR")
+			Expect(k8sClient.Create(context.TODO(), driver)).To(Succeed())
+
+			By("verifying the Driver daemonset")
+			Eventually(func() error {
+				daemonset := &appsv1.DaemonSet{}
+				err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: "test.rbd.csi.ceph.com-nodeplugin", Namespace: namespace}, daemonset)
+				if err != nil {
+					return err
+				}
+				if daemonset.Status.NumberReady != daemonset.Status.DesiredNumberScheduled {
+					return fmt.Errorf("daemonset not ready, expected %d nodes, got %d", daemonset.Status.DesiredNumberScheduled, daemonset.Status.NumberReady)
+				}
+				return nil
+			}, time.Minute*5, time.Second*10).Should(Succeed())
 		})
 	})
 })

test/utils/utils.go

@@ -108,15 +108,9 @@ func InstallCertManager() error {
 	return err
 }
 
-// LoadImageToKindCluster loads a local docker image to the kind cluster
-func LoadImageToKindClusterWithName(name string) error {
-	cluster := "kind"
-	if v, ok := os.LookupEnv("KIND_CLUSTER"); ok {
-		cluster = v
-	}
-	kindOptions := []string{"load", "docker-image", name, "--name", cluster}
-	//nolint:gosec
-	cmd := exec.Command("kind", kindOptions...)
+// LoadImageToKindCluster loads a local docker image to the minikube cluster
+func LoadImageToCluster(name string) error {
+	cmd := exec.Command("minikube", "image", "load", name)
 	_, err := Run(cmd)
 	return err
 }