From d3e71380358431b74114741d704f14c603c91a89 Mon Sep 17 00:00:00 2001 From: rohan2794 Date: Mon, 14 Oct 2024 17:30:59 +0530 Subject: [PATCH] add interface to get device uri and setup rdma device on nodes Signed-off-by: rohan2794 --- apps/helm_utils.go | 2 +- common/controlplane/controlplane.go | 5 ++ common/controlplane/v1-rest-api/msv.go | 11 ++++ common/controlplane/v1/msv_cp.go | 9 +++ common/k8sinstall/util.go | 13 ++-- common/k8stest/msv.go | 4 ++ common/k8stest/util_helm.go | 26 ++++++++ common/k8stest/util_node.go | 12 +++- common/k8stest/util_pod.go | 3 +- common/k8stest/util_rdma.go | 91 ++++++++++++++++++++++++++ configurations/maasci_config.yaml | 2 +- 11 files changed, 169 insertions(+), 9 deletions(-) create mode 100644 common/k8stest/util_helm.go diff --git a/apps/helm_utils.go b/apps/helm_utils.go index 6f06fe0..7f01202 100644 --- a/apps/helm_utils.go +++ b/apps/helm_utils.go @@ -125,7 +125,7 @@ func UpgradeHelmChart(helmChart, namespace, releaseName string, values map[strin setVals := strings.Join(vals, ",") logf.Log.Info("executing helm upgrade ", "releaseName: ", releaseName, ", chart: ", helmChart, ", namespace: ", namespace, ", values: ", setVals) // Define the Helm installation command. - cmd := exec.Command("helm", "upgrade", releaseName, helmChart, "-n", namespace, "--set", setVals) + cmd := exec.Command("helm", "upgrade", releaseName, helmChart, "-n", namespace, "--reuse-values", "--set", setVals) // Execute the command. output, err := cmd.CombinedOutput() if err != nil { diff --git a/common/controlplane/controlplane.go b/common/controlplane/controlplane.go index c68e1c0..75159d0 100644 --- a/common/controlplane/controlplane.go +++ b/common/controlplane/controlplane.go @@ -66,6 +66,7 @@ type ControlPlaneInterface interface { GetMsvSize(uuid string) (int64, error) SetVolumeMaxSnapshotCount(uuid string, maxSnapshotCount int32) error GetMsvMaxSnapshotCount(uuid string) (int32, error) + GetMsvDeviceUri(uuid string) (string, error) // Mayastor Node abstraction @@ -301,6 +302,10 @@ func GetMsvSize(uuid string) (int64, error) { return getControlPlane().GetMsvSize(uuid) } +func GetMsvDeviceUri(uuid string) (string, error) { + return getControlPlane().GetMsvDeviceUri(uuid) +} + func SetVolumeMaxSnapshotCount(uuid string, maxSnapshotCount int32) error { return getControlPlane().SetVolumeMaxSnapshotCount(uuid, maxSnapshotCount) } diff --git a/common/controlplane/v1-rest-api/msv.go b/common/controlplane/v1-rest-api/msv.go index 64cb118..41583a4 100644 --- a/common/controlplane/v1-rest-api/msv.go +++ b/common/controlplane/v1-rest-api/msv.go @@ -240,6 +240,17 @@ func (cp CPv1RestApi) GetMsvSize(uuid string) (int64, error) { return msvSize, err } +func (cp CPv1RestApi) GetMsvDeviceUri(uuid string) (string, error) { + vol, err, _ := cp.oa.getVolume(uuid) + var deviceUri string + + if err == nil { + deviceUri = string(vol.State.Target.DeviceUri) + } + + return deviceUri, err +} + func (cp CPv1RestApi) SetVolumeMaxSnapshotCount(uuid string, maxSnapshotCount int32) error { err, _ := cp.oa.setVolumeMaxSnapshotCount(uuid, maxSnapshotCount) return err diff --git a/common/controlplane/v1/msv_cp.go b/common/controlplane/v1/msv_cp.go index 1c4832e..99fe265 100644 --- a/common/controlplane/v1/msv_cp.go +++ b/common/controlplane/v1/msv_cp.go @@ -351,6 +351,15 @@ func getMsvSize(volName string) (int64, error) { return size, err } +func (cp CPv1) GetMsvDeviceUri(volName string) (string, error) { + var deviceUri string + msv, err := getMayastorCpVolume(volName) + if err == nil { + deviceUri = msv.State.Target.DeviceUri + } + return deviceUri, err +} + func (cp CPv1) SetVolumeMaxSnapshotCount(uuid string, maxSnapshotCount int32) error { pluginpath := GetPluginPath() diff --git a/common/k8sinstall/util.go b/common/k8sinstall/util.go index 571e84e..c4ec90e 100644 --- a/common/k8sinstall/util.go +++ b/common/k8sinstall/util.go @@ -222,22 +222,27 @@ func ScaleZfsControllerViaHelm(expected_replica int32) (int32, error) { } // SetRdmaViaHelm enable and disable RDMA -func SetRdmaViaHelm(enableRdma bool, iface string) error { - e2eCfg := e2e_config.GetConfig() +func SetRdmaViaHelm(enableRdma bool, iface string, helmChart, helmRelease, helmVersion string) error { values := map[string]interface{}{ "io_engine.target.nvmf.rdma.enabled": enableRdma, "io_engine.target.nvmf.iface": iface, } - err := apps.UpgradeHelmChart(e2eCfg.Product.ChartName, + err := k8stest.UpgradeHelmChart(helmChart, common.NSMayastor(), - e2eCfg.Product.HelmReleaseName, + helmRelease, + helmVersion, values, ) if err != nil { return fmt.Errorf("failed to set RDMA via helm, error: %v", err) } + // delete io-engine deamonset pods + err = k8stest.RestartPodByPrefix(e2e_config.GetConfig().Product.IOEnginePodName) + if err != nil { + return fmt.Errorf("failed to restart %s pods, error: %v", e2e_config.GetConfig().Product.IOEnginePodName, err) + } ready, err := k8stest.OpenEBSReady(10, 340) if err != nil { return err diff --git a/common/k8stest/msv.go b/common/k8stest/msv.go index 3017484..0b39730 100644 --- a/common/k8stest/msv.go +++ b/common/k8stest/msv.go @@ -74,6 +74,10 @@ func GetMsvSize(uuid string) (int64, error) { return controlplane.GetMsvSize(uuid) } +func GetMsvDeviceUri(uuid string) (string, error) { + return controlplane.GetMsvDeviceUri(uuid) +} + func GetMsvMaxSnapshotCount(uuid string) (int32, error) { return controlplane.GetMsvMaxSnapshotCount(uuid) } diff --git a/common/k8stest/util_helm.go b/common/k8stest/util_helm.go new file mode 100644 index 0000000..7639bab --- /dev/null +++ b/common/k8stest/util_helm.go @@ -0,0 +1,26 @@ +package k8stest + +import ( + "fmt" + "os/exec" + "strings" + + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +func UpgradeHelmChart(helmChart, namespace, releaseName, version string, values map[string]interface{}) error { + var vals []string + for k, v := range values { + vals = append(vals, fmt.Sprintf("%s=%v", k, v)) + } + setVals := strings.Join(vals, ",") + logf.Log.Info("executing helm upgrade ", "releaseName: ", releaseName, ", chart: ", helmChart, "version", version, "namespace: ", namespace, ", values: ", setVals) + // Define the Helm installation command. + cmd := exec.Command("helm", "upgrade", releaseName, helmChart, "-n", namespace, "--reuse-values", "--set", setVals, "--version", version) + // Execute the command. + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to upgrade with Helm: %v\n%s", err, output) + } + return nil +} diff --git a/common/k8stest/util_node.go b/common/k8stest/util_node.go index a2f854a..46c4de9 100644 --- a/common/k8stest/util_node.go +++ b/common/k8stest/util_node.go @@ -415,7 +415,17 @@ func UpdateNodeTaints(nodeName string, taintKey string) error { } func ListWorkerNode() ([]NodeLocation, error) { - return getNodeLocs() + workerNode := make([]NodeLocation, 0) + nodes, err := getNodeLocs() + if err != nil { + return workerNode, err + } + for _, node := range nodes { + if !node.K8sControlPlane { + workerNode = append(workerNode, node) + } + } + return workerNode, nil } // ListNodesWithoutNoScheduleTaint returns list of nodes which does not have NoSchedule taint diff --git a/common/k8stest/util_pod.go b/common/k8stest/util_pod.go index 8d46171..0191d71 100644 --- a/common/k8stest/util_pod.go +++ b/common/k8stest/util_pod.go @@ -386,10 +386,9 @@ func RestartPodByPrefix(prefix string) error { return delErr } logf.Log.Info("Restarted the ", "pod ", pod.Name) - return nil } } - return fmt.Errorf("failed to restart the pod") + return nil } // CheckPodIsRunningByPrefix check pod is running by prefix name diff --git a/common/k8stest/util_rdma.go b/common/k8stest/util_rdma.go index bc47435..b7a18ec 100644 --- a/common/k8stest/util_rdma.go +++ b/common/k8stest/util_rdma.go @@ -3,11 +3,17 @@ package k8stest import ( "encoding/json" "fmt" + "net/url" + "strings" + "github.com/openebs/openebs-e2e/common/e2e_agent" agent "github.com/openebs/openebs-e2e/common/e2e_agent" + "github.com/openebs/openebs-e2e/common/e2e_config" logf "sigs.k8s.io/controller-runtime/pkg/log" ) +const RdmaDeviceName = "rxe0" + type RdmaDeviceNetworkInterface struct { IfIndex int `json:"ifindex"` IfName string `json:"ifname"` @@ -41,3 +47,88 @@ func ListRdmaDevice(node string) ([]RdmaDeviceNetworkInterface, error) { logf.Log.Info("RDMA device", "node", node, "list", rdmaDeiceList) return rdmaDeiceList, nil } + +func CreateRdmaDeviceOnNode(node string) error { + rdmaDeviceList, err := ListRdmaDevice(node) + if err != nil { + return err + } + if len(rdmaDeviceList) == 0 { + logf.Log.Info("RDMA device not found", "node", node, "list", rdmaDeviceList) + //create rdma device + nodeIp, err := GetNodeIPAddress(node) + if err != nil { + return fmt.Errorf("failed to get node %s ip, error: %v", node, err) + } + + // get interface name + iface := e2e_config.GetConfig().NetworkInterface + out, err := e2e_agent.CreateRdmaDevice(*nodeIp, RdmaDeviceName, iface) + if err != nil { + return err + } + logf.Log.Info("Device created", "node", node, "output", out, "interface", iface) + + } + rdmaDeviceList, err = ListRdmaDevice(node) + if err != nil { + return err + } + logf.Log.Info("RDMA device", "node", node, "list", rdmaDeviceList) + return nil +} + +func CreateRdmaDeviceOnAllWorkerNodes() error { + workerNodes, err := ListWorkerNode() + if err != nil { + return err + } + logf.Log.Info("Worker", "Nodes", workerNodes) + for _, node := range workerNodes { + err := CreateRdmaDeviceOnNode(node.NodeName) + if err != nil { + return err + } + } + return nil +} + +func GetVolumeProtocol(volUuid string) (string, error) { + deviceUri, err := GetMsvDeviceUri(volUuid) + if err != nil { + return "", err + } + // deviceUri: nvmf+tcp:// + // Parse the device URI + u, err := url.Parse(deviceUri) + if err != nil { + return "", fmt.Errorf("error parsing URI: %s, error: %v", deviceUri, err) + } + return u.Scheme, nil +} + +// IsVolumeAccessibleOverRdma return true if volume device uri scheme contains rdma +// if volume is accessible over rdma then device uri will be like nvmf+tcp+rdma:// +func IsVolumeAccessibleOverRdma(volUuid string) (bool, error) { + protocol, err := GetVolumeProtocol(volUuid) + if err != nil { + return false, err + } + if strings.Contains(protocol, "rdma") { + return true, nil + } + return false, nil +} + +// IsVolumeAccessibleOverTcp return true if volume device uri scheme contains tcp and not rdma +// if volume is accessible over rdma then device uri will be like nvmf+tcp:// +func IsVolumeAccessibleOverTcp(volUuid string) (bool, error) { + protocol, err := GetVolumeProtocol(volUuid) + if err != nil { + return false, err + } + if !strings.Contains(protocol, "rdma") && strings.Contains(protocol, "tcp") { + return true, nil + } + return false, nil +} diff --git a/configurations/maasci_config.yaml b/configurations/maasci_config.yaml index c384da4..bf46ae8 100644 --- a/configurations/maasci_config.yaml +++ b/configurations/maasci_config.yaml @@ -6,4 +6,4 @@ configName: maasci grpcMandated: true deferredAssert: true beforeEachCheckAndRestart: true -networkInterface: eth0 \ No newline at end of file +networkInterface: ens1f0 \ No newline at end of file