chore: add interface to get device uri and setup rdma device on nodes

apps/helm_utils.go

@@ -125,7 +125,7 @@ func UpgradeHelmChart(helmChart, namespace, releaseName string, values map[strin
 	setVals := strings.Join(vals, ",")
 	logf.Log.Info("executing helm upgrade ", "releaseName: ", releaseName, ", chart: ", helmChart, ", namespace: ", namespace, ", values: ", setVals)
 	// Define the Helm installation command.
-	cmd := exec.Command("helm", "upgrade", releaseName, helmChart, "-n", namespace, "--set", setVals)
+	cmd := exec.Command("helm", "upgrade", releaseName, helmChart, "-n", namespace, "--reuse-values", "--set", setVals)
 	// Execute the command.
 	output, err := cmd.CombinedOutput()
 	if err != nil {

common/controlplane/controlplane.go

@@ -66,6 +66,7 @@ type ControlPlaneInterface interface {
 	GetMsvSize(uuid string) (int64, error)
 	SetVolumeMaxSnapshotCount(uuid string, maxSnapshotCount int32) error
 	GetMsvMaxSnapshotCount(uuid string) (int32, error)
+	GetMsvDeviceUri(uuid string) (string, error)
 
 	// Mayastor Node abstraction
 
@@ -301,6 +302,10 @@ func GetMsvSize(uuid string) (int64, error) {
 	return getControlPlane().GetMsvSize(uuid)
 }
 
+func GetMsvDeviceUri(uuid string) (string, error) {
+	return getControlPlane().GetMsvDeviceUri(uuid)
+}
+
 func SetVolumeMaxSnapshotCount(uuid string, maxSnapshotCount int32) error {
 	return getControlPlane().SetVolumeMaxSnapshotCount(uuid, maxSnapshotCount)
 }

common/controlplane/v1-rest-api/msv.go

@@ -240,6 +240,17 @@ func (cp CPv1RestApi) GetMsvSize(uuid string) (int64, error) {
 	return msvSize, err
 }
 
+func (cp CPv1RestApi) GetMsvDeviceUri(uuid string) (string, error) {
+	vol, err, _ := cp.oa.getVolume(uuid)
+	var deviceUri string
+
+	if err == nil {
+		deviceUri = string(vol.State.Target.DeviceUri)
+	}
+
+	return deviceUri, err
+}
+
 func (cp CPv1RestApi) SetVolumeMaxSnapshotCount(uuid string, maxSnapshotCount int32) error {
 	err, _ := cp.oa.setVolumeMaxSnapshotCount(uuid, maxSnapshotCount)
 	return err

common/controlplane/v1/msv_cp.go

@@ -351,6 +351,15 @@ func getMsvSize(volName string) (int64, error) {
 	return size, err
 }
 
+func (cp CPv1) GetMsvDeviceUri(volName string) (string, error) {
+	var deviceUri string
+	msv, err := getMayastorCpVolume(volName)
+	if err == nil {
+		deviceUri = msv.State.Target.DeviceUri
+	}
+	return deviceUri, err
+}
+
 func (cp CPv1) SetVolumeMaxSnapshotCount(uuid string, maxSnapshotCount int32) error {
 	pluginpath := GetPluginPath()
 

common/k8sinstall/util.go

@@ -222,22 +222,27 @@ func ScaleZfsControllerViaHelm(expected_replica int32) (int32, error) {
 }
 
 // SetRdmaViaHelm enable and disable RDMA
-func SetRdmaViaHelm(enableRdma bool, iface string) error {
-	e2eCfg := e2e_config.GetConfig()
+func SetRdmaViaHelm(enableRdma bool, iface string, helmChart, helmRelease, helmVersion string) error {
 	values := map[string]interface{}{
 		"io_engine.target.nvmf.rdma.enabled": enableRdma,
 		"io_engine.target.nvmf.iface":        iface,
 	}
 
-	err := apps.UpgradeHelmChart(e2eCfg.Product.ChartName,
+	err := k8stest.UpgradeHelmChart(helmChart,
 		common.NSMayastor(),
-		e2eCfg.Product.HelmReleaseName,
+		helmRelease,
+		helmVersion,
 		values,
 	)
 	if err != nil {
 		return fmt.Errorf("failed to set RDMA via helm, error: %v", err)
 	}
 
+	// delete io-engine deamonset pods
+	err = k8stest.RestartPodByPrefix(e2e_config.GetConfig().Product.IOEnginePodName)
+	if err != nil {
+		return fmt.Errorf("failed to restart %s pods, error: %v", e2e_config.GetConfig().Product.IOEnginePodName, err)
+	}
 	ready, err := k8stest.OpenEBSReady(10, 340)
 	if err != nil {
 		return err

common/k8stest/msv.go

@@ -74,6 +74,10 @@ func GetMsvSize(uuid string) (int64, error) {
 	return controlplane.GetMsvSize(uuid)
 }
 
+func GetMsvDeviceUri(uuid string) (string, error) {
+	return controlplane.GetMsvDeviceUri(uuid)
+}
+
 func GetMsvMaxSnapshotCount(uuid string) (int32, error) {
 	return controlplane.GetMsvMaxSnapshotCount(uuid)
 }

common/k8stest/util_helm.go

@@ -0,0 +1,26 @@
+package k8stest
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+func UpgradeHelmChart(helmChart, namespace, releaseName, version string, values map[string]interface{}) error {
+	var vals []string
+	for k, v := range values {
+		vals = append(vals, fmt.Sprintf("%s=%v", k, v))
+	}
+	setVals := strings.Join(vals, ",")
+	logf.Log.Info("executing helm upgrade ", "releaseName: ", releaseName, ", chart: ", helmChart, "version", version, "namespace: ", namespace, ", values: ", setVals)
+	// Define the Helm installation command.
+	cmd := exec.Command("helm", "upgrade", releaseName, helmChart, "-n", namespace, "--reuse-values", "--set", setVals, "--version", version)
+	// Execute the command.
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to upgrade with Helm: %v\n%s", err, output)
+	}
+	return nil
+}

common/k8stest/util_node.go

@@ -415,7 +415,17 @@ func UpdateNodeTaints(nodeName string, taintKey string) error {
 }
 
 func ListWorkerNode() ([]NodeLocation, error) {
-	return getNodeLocs()
+	workerNode := make([]NodeLocation, 0)
+	nodes, err := getNodeLocs()
+	if err != nil {
+		return workerNode, err
+	}
+	for _, node := range nodes {
+		if !node.K8sControlPlane {
+			workerNode = append(workerNode, node)
+		}
+	}
+	return workerNode, nil
 }
 
 // ListNodesWithoutNoScheduleTaint returns list of nodes which does not have NoSchedule taint

common/k8stest/util_pod.go

@@ -386,10 +386,9 @@ func RestartPodByPrefix(prefix string) error {
 				return delErr
 			}
 			logf.Log.Info("Restarted the ", "pod ", pod.Name)
-			return nil
 		}
 	}
-	return fmt.Errorf("failed to restart the pod")
+	return nil
 }
 
 // CheckPodIsRunningByPrefix check pod is running by prefix name

common/k8stest/util_rdma.go

@@ -3,11 +3,17 @@ package k8stest
 import (
 	"encoding/json"
 	"fmt"
+	"net/url"
+	"strings"
 
+	"github.com/openebs/openebs-e2e/common/e2e_agent"
 	agent "github.com/openebs/openebs-e2e/common/e2e_agent"
+	"github.com/openebs/openebs-e2e/common/e2e_config"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 )
 
+const RdmaDeviceName = "rxe0"
+
 type RdmaDeviceNetworkInterface struct {
 	IfIndex       int    `json:"ifindex"`
 	IfName        string `json:"ifname"`
@@ -41,3 +47,88 @@ func ListRdmaDevice(node string) ([]RdmaDeviceNetworkInterface, error) {
 	logf.Log.Info("RDMA device", "node", node, "list", rdmaDeiceList)
 	return rdmaDeiceList, nil
 }
+
+func CreateRdmaDeviceOnNode(node string) error {
+	rdmaDeviceList, err := ListRdmaDevice(node)
+	if err != nil {
+		return err
+	}
+	if len(rdmaDeviceList) == 0 {
+		logf.Log.Info("RDMA device not found", "node", node, "list", rdmaDeviceList)
+		//create rdma device
+		nodeIp, err := GetNodeIPAddress(node)
+		if err != nil {
+			return fmt.Errorf("failed to get node %s ip, error: %v", node, err)
+		}
+
+		// get interface name
+		iface := e2e_config.GetConfig().NetworkInterface
+		out, err := e2e_agent.CreateRdmaDevice(*nodeIp, RdmaDeviceName, iface)
+		if err != nil {
+			return err
+		}
+		logf.Log.Info("Device created", "node", node, "output", out, "interface", iface)
+
+	}
+	rdmaDeviceList, err = ListRdmaDevice(node)
+	if err != nil {
+		return err
+	}
+	logf.Log.Info("RDMA device", "node", node, "list", rdmaDeviceList)
+	return nil
+}
+
+func CreateRdmaDeviceOnAllWorkerNodes() error {
+	workerNodes, err := ListWorkerNode()
+	if err != nil {
+		return err
+	}
+	logf.Log.Info("Worker", "Nodes", workerNodes)
+	for _, node := range workerNodes {
+		err := CreateRdmaDeviceOnNode(node.NodeName)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func GetVolumeProtocol(volUuid string) (string, error) {
+	deviceUri, err := GetMsvDeviceUri(volUuid)
+	if err != nil {
+		return "", err
+	}
+	// deviceUri: nvmf+tcp://<some-random-string>
+	// Parse the device URI
+	u, err := url.Parse(deviceUri)
+	if err != nil {
+		return "", fmt.Errorf("error parsing URI: %s, error: %v", deviceUri, err)
+	}
+	return u.Scheme, nil
+}
+
+// IsVolumeAccessibleOverRdma return true if volume device uri scheme contains rdma
+// if volume is accessible over rdma then device uri will be like nvmf+tcp+rdma://<some-random-string>
+func IsVolumeAccessibleOverRdma(volUuid string) (bool, error) {
+	protocol, err := GetVolumeProtocol(volUuid)
+	if err != nil {
+		return false, err
+	}
+	if strings.Contains(protocol, "rdma") {
+		return true, nil
+	}
+	return false, nil
+}
+
+// IsVolumeAccessibleOverTcp return true if volume device uri scheme contains tcp and not rdma
+// if volume is accessible over rdma then device uri will be like nvmf+tcp://<some-random-string>
+func IsVolumeAccessibleOverTcp(volUuid string) (bool, error) {
+	protocol, err := GetVolumeProtocol(volUuid)
+	if err != nil {
+		return false, err
+	}
+	if !strings.Contains(protocol, "rdma") && strings.Contains(protocol, "tcp") {
+		return true, nil
+	}
+	return false, nil
+}

configurations/maasci_config.yaml

@@ -6,4 +6,4 @@ configName: maasci
 grpcMandated: true
 deferredAssert: true
 beforeEachCheckAndRestart: true
-networkInterface: eth0
+networkInterface: ens1f0