From cab53c493e1d2341f5e692f9f784a18d0b95d3cc Mon Sep 17 00:00:00 2001 From: Niladri Halder Date: Mon, 15 Jul 2024 15:08:05 +0530 Subject: [PATCH] feat(provisioner-localpv): Merge CAS config from PersistentVolumeClaim (#190) * Merge CAS config from PersistentVolumeClaim Signed-off-by: Nobi Signed-off-by: Niladri Halder * feat(provisioner): prefer StorageClass config over PVC config In the typical use-case for the provisioner, an admin sets the config on the StorageClass, and a user sets the config on the PVC. The admin's config may set boundaries on to the volumes which the PVC config map try to override. Preferring StorageClass config when there is a conflict may be the right way to go. Signed-off-by: Niladri Halder * tests: add integration test for pvc config Signed-off-by: Niladri Halder * chore(changelog): add changelog entry Signed-off-by: Niladri Halder --------- Signed-off-by: Nobi Signed-off-by: Niladri Halder Co-authored-by: Nobi --- Makefile | 2 +- changelogs/unreleased/190-nobiit | 1 + cmd/provisioner-localpv/app/config.go | 48 +-- go.mod | 2 +- .../api/apps/v1/deployment/deployment.go | 31 +- .../core/v1/persistentvolumeclaim/build.go | 20 +- .../api/storage/v1/storageclass/build.go | 34 +- tests/operations.go | 43 +- tests/pvc_cas_config_test.go | 402 ++++++++++++++++++ 9 files changed, 482 insertions(+), 101 deletions(-) create mode 100644 changelogs/unreleased/190-nobiit create mode 100644 tests/pvc_cas_config_test.go diff --git a/Makefile b/Makefile index c87287a7..075a22de 100644 --- a/Makefile +++ b/Makefile @@ -124,7 +124,7 @@ testv: format # Requires KUBECONFIG env and Ginkgo binary .PHONY: integration-test integration-test: - @cd tests && sudo -E env "PATH=${PATH}" ginkgo -v -failFast + @cd tests && sudo -E env "PATH=${PATH}" ginkgo -v --fail-fast .PHONY: format format: diff --git a/changelogs/unreleased/190-nobiit b/changelogs/unreleased/190-nobiit new file mode 100644 index 00000000..31f4a119 --- /dev/null +++ b/changelogs/unreleased/190-nobiit @@ -0,0 +1 @@ +Add feature to merge cas-config from PVC diff --git a/cmd/provisioner-localpv/app/config.go b/cmd/provisioner-localpv/app/config.go index 40012452..bc1e6354 100644 --- a/cmd/provisioner-localpv/app/config.go +++ b/cmd/provisioner-localpv/app/config.go @@ -1,20 +1,3 @@ -/* -Copyright 2019 The OpenEBS Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -*/ - package app import ( @@ -26,7 +9,7 @@ import ( cast "github.com/openebs/maya/pkg/castemplate/v1alpha1" hostpath "github.com/openebs/maya/pkg/hostpath/v1alpha1" "github.com/openebs/maya/pkg/util" - errors "github.com/pkg/errors" + "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" @@ -143,7 +126,7 @@ const ( ) // GetVolumeConfig creates a new VolumeConfig struct by -// parsing and merging the configuration provided in the PVC +// parsing and merging the configuration provided in the PVC/SC // annotation - cas.openebs.io/config with the // default configuration of the provisioner. func (p *Provisioner) GetVolumeConfig(ctx context.Context, pvName string, pvc *corev1.PersistentVolumeClaim) (*VolumeConfig, error) { @@ -169,16 +152,23 @@ func (p *Provisioner) GetVolumeConfig(ctx context.Context, pvName string, pvc *c } } - //TODO : extract and merge the cas volume config from pvc - //This block can be added once validation checks are added - // as to the type of config that can be passed via PVC - //pvcCASConfigStr := pvc.ObjectMeta.Annotations[string(mconfig.CASConfigKey)] - //if len(strings.TrimSpace(pvcCASConfigStr)) != 0 { - // pvcCASConfig, err := cast.UnMarshallToConfig(pvcCASConfigStr) - // if err == nil { - // pvConfig = cast.MergeConfig(pvcCASConfig, pvConfig) - // } - //} + // Extract and merge the cas config from persistentvolumeclaim. + // TODO: Validation checks for what all cas-config options can be + // set on the PVC. + pvcCASConfigStr := pvc.Annotations[string(mconfig.CASConfigKey)] + klog.V(4).Infof("PVC %v has config:%v", pvc.Name, pvcCASConfigStr) + if len(strings.TrimSpace(pvcCASConfigStr)) != 0 { + pvcCASConfig, err := cast.UnMarshallToConfig(pvcCASConfigStr) + if err == nil { + // Config keys which already exist (SC config), + // will be skipped + // i.e. SC config will have precedence over PVC config, + // if both have the same keys + pvConfig = cast.MergeConfig(pvConfig, pvcCASConfig) + } else { + return nil, errors.Wrapf(err, "failed to get config: invalid pvc config {%v}", pvcCASConfigStr) + } + } pvConfigMap, err := cast.ConfigToMap(pvConfig) if err != nil { diff --git a/go.mod b/go.mod index 428fa0e7..877eecc8 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( k8s.io/client-go v0.27.2 k8s.io/klog/v2 v2.100.1 sigs.k8s.io/sig-storage-lib-external-provisioner/v9 v9.0.3 + sigs.k8s.io/yaml v1.3.0 ) require ( @@ -84,7 +85,6 @@ require ( k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect ) replace ( diff --git a/pkg/kubernetes/api/apps/v1/deployment/deployment.go b/pkg/kubernetes/api/apps/v1/deployment/deployment.go index 840aa74f..fe54ae6a 100644 --- a/pkg/kubernetes/api/apps/v1/deployment/deployment.go +++ b/pkg/kubernetes/api/apps/v1/deployment/deployment.go @@ -1,24 +1,12 @@ -/* -Copyright 2019 The OpenEBS Authors -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package v1alpha1 import ( - templatespec "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/podtemplatespec" stringer "github.com/openebs/maya/pkg/apis/stringer/v1alpha1" - errors "github.com/pkg/errors" + "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + templatespec "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/podtemplatespec" ) // Predicate abstracts conditional logic w.r.t the deployment instance @@ -96,6 +84,19 @@ func (b *Builder) WithName(name string) *Builder { return b } +// WithGenerateName sets the Name field of deployment with a random value with the provided value as a prefix. +func (b *Builder) WithGenerateName(prefix string) *Builder { + if len(prefix) == 0 { + b.errors = append( + b.errors, + errors.New("failed to build deployment: missing prefix for generateName"), + ) + return b + } + b.deployment.object.GenerateName = prefix + "-" + return b +} + // WithNamespace sets the Namespace field of deployment with provided value. func (b *Builder) WithNamespace(namespace string) *Builder { if len(namespace) == 0 { diff --git a/pkg/kubernetes/api/core/v1/persistentvolumeclaim/build.go b/pkg/kubernetes/api/core/v1/persistentvolumeclaim/build.go index 59ce73b8..0431e509 100644 --- a/pkg/kubernetes/api/core/v1/persistentvolumeclaim/build.go +++ b/pkg/kubernetes/api/core/v1/persistentvolumeclaim/build.go @@ -1,23 +1,7 @@ -/* -Copyright 2020 The OpenEBS Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package persistentvolumeclaim import ( - errors "github.com/pkg/errors" + "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) @@ -72,7 +56,7 @@ func (b *Builder) WithGenerateName(name string) *Builder { return b } - b.pvc.object.GenerateName = name + b.pvc.object.GenerateName = name + "-" return b } diff --git a/pkg/kubernetes/api/storage/v1/storageclass/build.go b/pkg/kubernetes/api/storage/v1/storageclass/build.go index 4a0f2533..b79cb97a 100644 --- a/pkg/kubernetes/api/storage/v1/storageclass/build.go +++ b/pkg/kubernetes/api/storage/v1/storageclass/build.go @@ -1,19 +1,3 @@ -/* -Copyright 2019 The OpenEBS Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package storageclass import ( @@ -96,6 +80,24 @@ func WithLabels(labels map[string]string) StorageClassOption { } } +func WithAnnotations(annotations map[string]string) StorageClassOption { + return func(s *storagev1.StorageClass) error { + if len(annotations) == 0 { + return errors.New("Failed to set Annotations. " + + "Input is invalid.") + } + + if s.ObjectMeta.Annotations == nil { + s.ObjectMeta.Annotations = map[string]string{} + } + for key, value := range annotations { + s.ObjectMeta.Annotations[key] = value + } + + return nil + } +} + func WithParameters(parameters map[string]string) StorageClassOption { return func(s *storagev1.StorageClass) error { if len(parameters) == 0 { diff --git a/tests/operations.go b/tests/operations.go index 46160e68..5cf16530 100644 --- a/tests/operations.go +++ b/tests/operations.go @@ -1,26 +1,9 @@ -/* -Copyright 2019 The OpenEBS Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package tests import ( "bytes" "context" "fmt" - //"sort" "strconv" "strings" @@ -50,15 +33,15 @@ import ( "k8s.io/client-go/tools/remotecommand" deploy "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/apps/v1/deployment" - container "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/container" - event "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/event" + "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/container" + "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/event" pv "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/persistentvolume" pvc "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/persistentvolumeclaim" - pod "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/pod" + "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/pod" pts "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/podtemplatespec" k8svolume "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/volume" sc "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/storage/v1/storageclass" - ndmconfig "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/ndmconfig" + "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/ndmconfig" ) const ( @@ -476,6 +459,24 @@ func (ops *Operations) GetPVNameFromPVCName(namespace, pvcName string) string { return p.Spec.VolumeName } +// GetNodeAffinityLabelKeysFromPv returns the label keys for NodeSelector MatchExpressions in a PV. +func (ops *Operations) GetNodeAffinityLabelKeysFromPv(pvName string) ([]string, error) { + pv, err := ops.PVClient.Get(context.TODO(), pvName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + var nodeAffinityLabelKeys []string + + for _, selectorTerm := range pv.Spec.NodeAffinity.Required.NodeSelectorTerms { + for _, matchExpression := range selectorTerm.MatchExpressions { + nodeAffinityLabelKeys = append(nodeAffinityLabelKeys, matchExpression.Key) + } + } + + return nodeAffinityLabelKeys, nil +} + // isNotFound returns true if the original // cause of error was due to castemplate's // not found error or kubernetes not found diff --git a/tests/pvc_cas_config_test.go b/tests/pvc_cas_config_test.go new file mode 100644 index 00000000..a9507a0b --- /dev/null +++ b/tests/pvc_cas_config_test.go @@ -0,0 +1,402 @@ +package tests + +import ( + ctx "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + mayav1alpha1 "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" + "golang.org/x/net/context" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" + + deploy "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/apps/v1/deployment" + "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/container" + pvc "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/persistentvolumeclaim" + pts "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/podtemplatespec" + k8svolume "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/volume" + sc "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/storage/v1/storageclass" +) + +// createDeploymentWhichConsumesHostpath creates a single-replica Deployment whose Pod consumes hostpath PVC. +func createDeploymentWhichConsumesHostpath(namePrefix, namespace, pvcName string) (*appsv1.Deployment, error) { + labelSelector := map[string]string{ + "app": namePrefix, + } + deployment, err := deploy.NewBuilder(). + WithGenerateName(namePrefix). + WithNamespace(namespace). + WithLabelsNew(labelSelector). + WithSelectorMatchLabelsNew(labelSelector). + WithPodTemplateSpecBuilder( + pts.NewBuilder(). + WithLabelsNew(labelSelector). + WithContainerBuildersNew( + container.NewBuilder(). + WithName("busybox"). + WithImage("busybox"). + WithCommandNew( + []string{ + "sleep", + "3600", + }, + ). + WithVolumeMountsNew( + []corev1.VolumeMount{ + { + Name: "demo-vol1", + MountPath: "/mnt/store1", + }, + }, + ), + ). + WithVolumeBuilders( + k8svolume.NewBuilder(). + WithName("demo-vol1"). + WithPVCSource(pvcName), + ), + ). + Build() + if err != nil { + return nil, err + } + + return ops.DeployClient.WithNamespace(namespaceObj.Name).Create(context.TODO(), deployment) +} + +// isLabelSelectorsEqual compares two arrays of label selector keys. +func isLabelSelectorsEqual(request, result []string) bool { + if len(request) != len(result) { + return false + } + + ch := make(chan struct{}, 2) + collectFrequency := func(labelKeys []string, freq *map[string]int) { + for _, elem := range labelKeys { + (*freq)[elem]++ + } + + ch <- struct{}{} + } + + // Maps to hold the frequency of strings in the string slices. + freqRequest := make(map[string]int) + freqResult := make(map[string]int) + + go collectFrequency(request, &freqRequest) + go collectFrequency(result, &freqResult) + + for i := 0; i < 2; i++ { + select { + case <-ch: + continue + } + } + + // Compare frequencies + for key, countRequest := range freqRequest { + countResult, ok := freqResult[key] + if !ok || countRequest != countResult { + return false + } + } + + return true +} + +var _ = Describe("VOLUME PROVISIONING/DE-PROVISIONING WITH ADDITIVE CAS-CONFIGS ON PVC AND SC", func() { + var ( + pvcNamePrefix = "pvc-additive-cas-config" + scNamePrefix = "sc-additive-cas-config" + deployNamePrefix = "busybox-additive-cas-config" + deployment *appsv1.Deployment + scName string + pvcName string + pvcCapacity = "2Gi" + pvName string + pvcNodeAffinityLabelKeys = []string{"kubernetes.io/os", "kubernetes.io/hostname"} + ) + + When("an application with a PVC which has cas-config which does not have conflicts with the cas-config on the"+ + " StorageClass, is created", func() { + It("should provision the volume", func() { + By("creating the StorageClass with cas-config", func() { + storageClass, err := sc.NewStorageClass( + sc.WithGenerateName("sc-additive-cas-config"), + sc.WithLabels(map[string]string{ + "openebs.io/test-sc": "true", + }), + sc.WithLocalPV(), + + // This is the StorageClass config in question here. + sc.WithHostpath(hostpathDir), + + sc.WithVolumeBindingMode("WaitForFirstConsumer"), + sc.WithReclaimPolicy("Delete"), + ) + Expect(err).To( + BeNil(), + "while building StorageClass with name prefix {%s}", + scNamePrefix, + ) + storageClass, err = ops.SCClient.Create(ctx.TODO(), storageClass) + Expect(err).To( + BeNil(), + "while creating StorageClass with name prefix %s", + scNamePrefix, + ) + scName = storageClass.Name + }) + By("creating the PVC with additive cas-config", func() { + pvcCasConfig := []mayav1alpha1.Config{ + { + Name: "NodeAffinityLabels", // This is the config that needs to not be for the same config key name. + List: pvcNodeAffinityLabelKeys, + }, + } + pvcCasConfigStr, err := yaml.Marshal(pvcCasConfig) + Expect(err).To(BeNil(), "while marshaling cas-config") + pvc, err := pvc.NewBuilder(). + WithGenerateName(pvcNamePrefix). + WithNamespace(namespaceObj.Name). + WithAnnotations(map[string]string{ + string(mayav1alpha1.CASConfigKey): string(pvcCasConfigStr), + }). + WithStorageClass(scName). + WithAccessModes([]corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}). + WithCapacity(pvcCapacity). + Build() + Expect(err).To( + BeNil(), + "while building PVC with name prefix %s in namespace %s", + pvcNamePrefix, + namespaceObj.Name, + ) + pvc, err = ops.PVCClient.WithNamespace(namespaceObj.Name).Create(ctx.TODO(), pvc) + Expect(err).To( + BeNil(), + "while creating PVC with name prefix %s in namespace %s", + pvcNamePrefix, + namespaceObj.Name, + ) + pvcName = pvc.Name + }) + By("creating a bound PV", func() { + deployment, err = createDeploymentWhichConsumesHostpath(deployNamePrefix, namespaceObj.Name, + pvcName) + Expect(err).To( + BeNil(), + "while creating Deployment with name %s in namespace %s", + deployment.Name, + namespaceObj.Name, + ) + Expect(ops.IsPVCBoundEventually(namespaceObj.Name, pvcName)).To( + BeTrue(), + "while checking if the PVC %s in namespace %s was bound to a PV", + pvcName, namespaceObj.Name, + ) + Expect(ops.GetPodRunningCountEventually(namespaceObj.Name, "app="+deployNamePrefix, 1)).To(BeNumerically("==", + 1)) + Expect(Eventually(func() string { + pvName = ops.GetPVNameFromPVCName(namespaceObj.Name, pvcName) + return pvName + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + WithContext(ctx.TODO()). + Should(Not(BeEmpty()))).To(BeTrue()) + }) + + By("having the PVC non-conflicting cas-config set correctly", func() { + nodeAffinityLabelKeys, err := ops.GetNodeAffinityLabelKeysFromPv(pvName) + Expect(err).To(BeNil(), "while getting NodeAffinityLabels from PV '%s'", pvName) + Expect(isLabelSelectorsEqual(pvcNodeAffinityLabelKeys, nodeAffinityLabelKeys)).To( + BeTrue(), + "while checking if PV %s had the NodeAffinityLabels requested on the PVC %s on namespace %s", + pvName, pvcName, namespaceObj.Name, + ) + }) + }) + }) + + When("an application with a PVC which has cas-config which does not have conflicts with the cas-config on the"+ + " StorageClass, is deleted", func() { + It("should de-provision the volume", func() { + By("deleting the PV", func() { + podList, err := ops.PodClient.List(ctx.TODO(), metav1.ListOptions{LabelSelector: "app=" + deployNamePrefix}) + Expect(err).To(BeNil(), "while listing Pods for busybox application deployment") + Expect(len(podList.Items)).To(BeNumerically("==", 1)) + pod := &podList.Items[0] + err = ops.DeployClient.WithNamespace(namespaceObj.Name).Delete(ctx.TODO(), deployment.Name, &metav1.DeleteOptions{}) + Expect(err).To(BeNil(), "while deleting busybox application deployment") + Expect(ops.IsPodDeletedEventually(pod.Namespace, pod.Name)).To( + BeTrue(), + "while checking to see if the Pod %s in namespace %s for the busybox deployment is deleted", + pod.Name, pod.Namespace, + ) + + ops.DeletePersistentVolumeClaim(pvcName, namespaceObj.Name) + Expect(ops.IsPVCDeletedEventually(pvcName, namespaceObj.Name)).To( + BeTrue(), + "while checking if PVC %s in namespace %s is deleted", + pvcName, namespaceObj.Namespace, + ) + Expect(ops.IsPVDeletedEventually(pvName)).To( + BeTrue(), + "when checking to see if the underlying PV %s is deleted", + pvName, + ) + }) + }) + }) +}) + +var _ = Describe("VOLUME PROVISIONING/DE-PROVISIONING WITH CONFLICTING CAS-CONFIGS ON PVC AND SC", func() { + var ( + pvcNamePrefix = "pvc-conflicting-cas-config" + scNamePrefix = "sc-conflicting-cas-config" + deployNamePrefix = "busybox-conflicting-cas-config" + deployment *appsv1.Deployment + scName string + pvcName string + pvcCapacity = "2Gi" + pvName string + scNodeAffinityLabelKeys = []string{"kubernetes.io/hostname"} + ) + + When("an application with a PVC which has cas-config which has conflicts with the cas-config on the"+ + " StorageClass, is created", func() { + It("should provision the volume", func() { + By("creating the StorageClass with cas-config", func() { + storageClass, err := sc.NewStorageClass( + sc.WithGenerateName("sc-conflicting-cas-config"), + sc.WithLabels(map[string]string{ + "openebs.io/test-sc": "true", + }), + sc.WithLocalPV(), + sc.WithHostpath(hostpathDir), + // This is the config in question. + sc.WithNodeAffinityLabels(scNodeAffinityLabelKeys), + sc.WithVolumeBindingMode("WaitForFirstConsumer"), + sc.WithReclaimPolicy("Delete"), + ) + Expect(err).To( + BeNil(), + "while building StorageClass with name prefix {%s}", + scNamePrefix, + ) + storageClass, err = ops.SCClient.Create(ctx.TODO(), storageClass) + Expect(err).To( + BeNil(), + "while creating StorageClass with name prefix %s", + scNamePrefix, + ) + scName = storageClass.Name + }) + By("creating the PVC with the same cas-config key, but a different value", func() { + pvcNodeAffinityLabelKeys := []string{"kubernetes.io/os", "kubernetes.io/arch"} + pvcCasConfig := []mayav1alpha1.Config{ + { + Name: "NodeAffinityLabels", // This is the config that needs to not be for the same config key name. + List: pvcNodeAffinityLabelKeys, + }, + } + pvcCasConfigStr, err := yaml.Marshal(pvcCasConfig) + Expect(err).To(BeNil(), "while marshaling cas-config") + pvc, err := pvc.NewBuilder(). + WithGenerateName(pvcNamePrefix). + WithNamespace(namespaceObj.Name). + WithAnnotations(map[string]string{ + string(mayav1alpha1.CASConfigKey): string(pvcCasConfigStr), + }). + WithStorageClass(scName). + WithAccessModes([]corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}). + WithCapacity(pvcCapacity). + Build() + Expect(err).To( + BeNil(), + "while building PVC with name prefix %s in namespace %s", + pvcNamePrefix, + namespaceObj.Name, + ) + pvc, err = ops.PVCClient.WithNamespace(namespaceObj.Name).Create(ctx.TODO(), pvc) + Expect(err).To( + BeNil(), + "while creating PVC with name prefix %s in namespace %s", + pvcNamePrefix, + namespaceObj.Name, + ) + pvcName = pvc.Name + }) + By("creating a bound PV", func() { + deployment, err = createDeploymentWhichConsumesHostpath(deployNamePrefix, namespaceObj.Name, + pvcName) + Expect(err).To( + BeNil(), + "while creating Deployment with name %s in namespace %s", + deployment.Name, + namespaceObj.Name, + ) + Expect(ops.IsPVCBoundEventually(namespaceObj.Name, pvcName)).To( + BeTrue(), + "while checking if the PVC %s in namespace %s was bound to a PV", + pvcName, namespaceObj.Name, + ) + Expect(ops.GetPodRunningCountEventually(namespaceObj.Name, "app="+deployNamePrefix, 1)).To(BeNumerically("==", + 1)) + Expect(Eventually(func() string { + pvName = ops.GetPVNameFromPVCName(namespaceObj.Name, pvcName) + return pvName + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + WithContext(ctx.TODO()). + Should(Not(BeEmpty()))).To(BeTrue()) + }) + + By("having the SC cas-config set correctly", func() { + nodeAffinityLabelKeys, err := ops.GetNodeAffinityLabelKeysFromPv(pvName) + Expect(err).To(BeNil(), "while getting NodeAffinityLabels from PV '%s'", pvName) + Expect(isLabelSelectorsEqual(scNodeAffinityLabelKeys, nodeAffinityLabelKeys)).To( + BeTrue(), + "while checking if PV %s had the NodeAffinityLabels requested on the SC %s", + scName, + ) + }) + }) + }) + + When("an application with a PVC which has cas-config which has conflicts with the cas-config on the"+ + " StorageClass, is deleted", func() { + It("should de-provision the volume", func() { + By("deleting the PV", func() { + podList, err := ops.PodClient.List(ctx.TODO(), metav1.ListOptions{LabelSelector: "app=" + deployNamePrefix}) + Expect(err).To(BeNil(), "while listing Pods for busybox application deployment") + Expect(len(podList.Items)).To(BeNumerically("==", 1)) + pod := &podList.Items[0] + err = ops.DeployClient.WithNamespace(namespaceObj.Name).Delete(ctx.TODO(), deployment.Name, &metav1.DeleteOptions{}) + Expect(err).To(BeNil(), "while deleting busybox application deployment") + Expect(ops.IsPodDeletedEventually(pod.Namespace, pod.Name)).To( + BeTrue(), + "while checking to see if the Pod %s in namespace %s for the busybox deployment is deleted", + pod.Name, pod.Namespace, + ) + + ops.DeletePersistentVolumeClaim(pvcName, namespaceObj.Name) + Expect(ops.IsPVCDeletedEventually(pvcName, namespaceObj.Name)).To( + BeTrue(), + "while checking if PVC %s in namespace %s is deleted", + pvcName, namespaceObj.Namespace, + ) + Expect(ops.IsPVDeletedEventually(pvName)).To( + BeTrue(), + "when checking to see if the underlying PV %s is deleted", + pvName, + ) + }) + }) + }) +})