Merge pull request #76 from openebs/rdma-device-create-delete

feat(RDMA): add support to create and delete rdma device on node

common/e2e_agent/client.go

@@ -100,6 +100,11 @@ type Zpool struct {
 	PoolName     string `json:"poolName"`
 }
 
+type Rdma struct {
+	DeviceName    string `json:"deviceName"`
+	InterfaceName string `json:"interfaceName"`
+}
+
 func sendRequest(reqType, url string, data interface{}) error {
 	_, err := sendRequestGetResponse(reqType, url, data, true)
 	return err
@@ -1143,3 +1148,74 @@ func LvmLvRemoveThinPool(serverAddr string, vgName string) (string, error) {
 	logf.Log.Info("LvmLvRemoveThinPool succeeded", "output", out)
 	return out, err
 }
+
+// ListRdmaDevice get pre-created and available RDMA device
+func ListRdmaDevice(serverAddr string) (string, error) {
+	logf.Log.Info("Executing GetRdmaDevice", "addr", serverAddr)
+	url := "http://" + getAgentAddress(serverAddr) + "/listrdmadevice"
+	encodedresult, err := sendRequestGetResponse("POST", url, nil, false)
+	if err != nil {
+		logf.Log.Info("sendRequestGetResponse", "encodedresult", encodedresult, "error", err.Error())
+		return encodedresult, err
+	}
+
+	out, e2eagenterrcode, err := UnwrapResult(encodedresult)
+	if err != nil {
+		logf.Log.Info("unwrap failed", "encodedresult", encodedresult, "error", err.Error())
+		return encodedresult, err
+	}
+	if e2eagenterrcode != ErrNone {
+		return out, fmt.Errorf("failed to list available RDMA device, errcode %d", e2eagenterrcode)
+	}
+	logf.Log.Info("ListRdmaDevice succeeded", "output", out)
+	return out, err
+}
+
+// CreateRdmaDevice create rdma device
+func CreateRdmaDevice(serverAddr string, deviceName string, interfaceName string) (string, error) {
+	data := Rdma{
+		DeviceName:    deviceName,
+		InterfaceName: interfaceName,
+	}
+	logf.Log.Info("Executing CreateRdmaDevice", "addr", serverAddr, "data", data)
+	url := "http://" + getAgentAddress(serverAddr) + "/createrdmadevice"
+	encodedresult, err := sendRequestGetResponse("POST", url, data, false)
+	if err != nil {
+		logf.Log.Info("sendRequestGetResponse", "encodedresult", encodedresult, "error", err.Error())
+		return encodedresult, err
+	}
+	out, e2eagenterrcode, err := UnwrapResult(encodedresult)
+	if err != nil {
+		logf.Log.Info("unwrap failed", "encodedresult", encodedresult, "error", err.Error())
+		return encodedresult, err
+	}
+	if e2eagenterrcode != ErrNone {
+		return out, fmt.Errorf("failed to create rdma device, errcode %d", e2eagenterrcode)
+	}
+	logf.Log.Info("CreateRdmaDevice succeeded", "output", out)
+	return out, err
+}
+
+// DeleteRdmaDevice destroy rdma device
+func DeleteRdmaDevice(serverAddr string, deeviceName string) (string, error) {
+	data := Rdma{
+		DeviceName: deeviceName,
+	}
+	logf.Log.Info("Executing DeleteRdmaDevice", "addr", serverAddr, "data", data)
+	url := "http://" + getAgentAddress(serverAddr) + "/deleterdmadevice"
+	encodedresult, err := sendRequestGetResponse("POST", url, data, false)
+	if err != nil {
+		logf.Log.Info("sendRequestGetResponse", "encodedresult", encodedresult, "error", err.Error())
+		return encodedresult, err
+	}
+	out, e2eagenterrcode, err := UnwrapResult(encodedresult)
+	if err != nil {
+		logf.Log.Info("unwrap failed", "encodedresult", encodedresult, "error", err.Error())
+		return encodedresult, err
+	}
+	if e2eagenterrcode != ErrNone {
+		return out, fmt.Errorf("failed to delete rdma device, errcode %d", e2eagenterrcode)
+	}
+	logf.Log.Info("DeleteRdmaDevice succeeded", "output", out)
+	return out, err
+}

common/k8stest/util_rdma.go

@@ -0,0 +1,43 @@
+package k8stest
+
+import (
+	"encoding/json"
+	"fmt"
+
+	agent "github.com/openebs/openebs-e2e/common/e2e_agent"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+type RdmaDeviceNetworkInterface struct {
+	IfIndex       int    `json:"ifindex"`
+	IfName        string `json:"ifname"`
+	Port          int    `json:"port"`
+	State         string `json:"state"`
+	PhysicalState string `json:"physical_state"`
+	NetDev        string `json:"netdev"`
+	NetDevIndex   int    `json:"netdev_index"`
+}
+
+func ListRdmaDevice(node string) ([]RdmaDeviceNetworkInterface, error) {
+	var rdmaDeiceList []RdmaDeviceNetworkInterface
+	nodeIp, err := GetNodeIPAddress(node)
+	if err != nil {
+		return rdmaDeiceList, fmt.Errorf("failed to get node %s ip, error: %v", node, err)
+	}
+
+	rdmaDevice, err := agent.ListRdmaDevice(*nodeIp)
+	if err != nil {
+		return rdmaDeiceList, fmt.Errorf("failed to list RDMA device on node %s , error: %v", node, err)
+	}
+	if rdmaDevice == "" {
+		logf.Log.Info("RDMA device list failed with empty string", "output", rdmaDevice)
+		return rdmaDeiceList, fmt.Errorf("failed to list RDMA device on node %s", node)
+	}
+	output := trimForJson(rdmaDevice)
+	if err = json.Unmarshal([]byte(output), &rdmaDeiceList); err != nil {
+		logf.Log.Info("Failed to unmarshal rdma list", "output", output)
+		return rdmaDeiceList, fmt.Errorf("failed to unmarshal rdma list on node %s , output: %s,error: %v", node, output, err)
+	}
+	logf.Log.Info("RDMA device", "node", node, "list", rdmaDeiceList)
+	return rdmaDeiceList, nil
+}

tools/e2e-agent/Dockerfile

@@ -11,7 +11,7 @@ COPY  ./ /
 # It ensures that devices are configured as soon as they are plugged in and discovered.
 # It propagates information about a processed device.
 RUN apt-get update; apt-get install net-tools iptables wget parted udev nvme-cli build-essential gettext gettext-base \
-        libinih-dev uuid-dev liburcu-dev  libblkid-dev btrfs-progs -y;
+        libinih-dev uuid-dev liburcu-dev  libblkid-dev btrfs-progs ibverbs-utils rdma-core -y;
 RUN wget https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz; \
         tar -C /usr/local/ -xzf go${GO_VERSION}.linux-amd64.tar.gz; \
         rm -rf go${GO_VERSION}.linux-amd64.tar.gz; \

tools/e2e-agent/build.sh

@@ -9,7 +9,7 @@
 # as long as we do not make breaking changes.
 set -e
 IMAGE="openebs/e2e-agent"
-TAG="v3.0.4"
+TAG="v3.0.5"
 registry=""
 tag_as_latest=""
 

tools/e2e-agent/e2e-agent.yaml

@@ -68,7 +68,7 @@ spec:
           securityContext:
             privileged: true
             allowPrivilegeEscalation: true
-          image: openebs/e2e-agent:v3.0.4
+          image: openebs/e2e-agent:v3.0.5
           imagePullPolicy: Always
           volumeMounts:
             - name: host-root

tools/e2e-agent/rdma.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"k8s.io/klog/v2"
+)
+
+type Rdma struct {
+	DeviceName    string `json:"deviceName"`
+	InterfaceName string `json:"interfaceName"`
+}
+
+// ListRdmaDevice list RDMA device
+func ListRdmaDevice(w http.ResponseWriter, r *http.Request) {
+	var msg string
+	klog.Info("List available RDMA device")
+
+	rdmaDeviceListCommand := "rdma link -j"
+	output, err := bashLocal(rdmaDeviceListCommand)
+	if err != nil {
+		msg = fmt.Sprintf("cannot list RDMA device. Error %s", err.Error())
+		klog.Error(msg)
+		WrapResult(msg, ErrExecFailed, w)
+		return
+	}
+	WrapResult(string(output), ErrNone, w)
+}
+
+// CreateRdmaDevice create rdma device
+func CreateRdmaDevice(w http.ResponseWriter, r *http.Request) {
+	var msg string
+	var rdma Rdma
+	d := json.NewDecoder(r.Body)
+	if err := d.Decode(&rdma); err != nil {
+		msg = fmt.Sprintf("failed to read JSON encoded data, Error: %s", err.Error())
+		klog.Error(msg)
+		WrapResult(msg, ErrJsonDecode, w)
+		return
+	}
+	if rdma.DeviceName == "" {
+		msg = "no device name passed"
+		klog.Error(msg)
+		WrapResult(msg, UnprocessableEntityErrorCode, w)
+		return
+	}
+	if rdma.InterfaceName == "" {
+		msg = "no interface name passed"
+		klog.Error(msg)
+		WrapResult(msg, UnprocessableEntityErrorCode, w)
+		return
+	}
+	klog.Info("creates rdma device, data: %v", rdma)
+
+	rdmaDeviceCreateCommand := fmt.Sprintf("rdma link add %s type rxe netdev %s", rdma.DeviceName, rdma.InterfaceName)
+	output, err := bashLocal(rdmaDeviceCreateCommand)
+	if err != nil {
+		msg = fmt.Sprintf("cannot create rdma device, Error %s", err.Error())
+		klog.Error(msg)
+		WrapResult(msg, ErrExecFailed, w)
+		return
+	}
+	WrapResult(string(output), ErrNone, w)
+}
+
+// DeleteRdmaDevice destroy rdma device
+func DeleteRdmaDevice(w http.ResponseWriter, r *http.Request) {
+	var msg string
+	var rdma Rdma
+	d := json.NewDecoder(r.Body)
+	if err := d.Decode(&rdma); err != nil {
+		msg = fmt.Sprintf("failed to read JSON encoded data, Error: %s", err.Error())
+		klog.Error(msg)
+		WrapResult(msg, ErrJsonDecode, w)
+		return
+	}
+	if rdma.DeviceName == "" {
+		msg = "no rdma device name passed"
+		klog.Error(msg)
+		WrapResult(msg, UnprocessableEntityErrorCode, w)
+		return
+	}
+
+	klog.Info("delete rdma device, data: %v", rdma)
+	rdmaDeviceDeleteCommand := fmt.Sprintf("rdma link delete %s", rdma.DeviceName)
+	output, err := bashLocal(rdmaDeviceDeleteCommand)
+	if err != nil {
+		msg = fmt.Sprintf("cannot delete rdma device, Error %s", err.Error())
+		klog.Error(msg)
+		WrapResult(msg, ErrExecFailed, w)
+		return
+	}
+	WrapResult(string(output), ErrNone, w)
+}

tools/e2e-agent/server.go

@@ -176,6 +176,10 @@ func handleRequests() {
 	//localPV
 	router.HandleFunc("/createhostpathdisk", CreateHostPathDisk).Methods("POST")
 	router.HandleFunc("/removehostpathdisk", RemoveHostPathDisk).Methods("POST")
+	//RDMA
+	router.HandleFunc("/listrdmadevice", ListRdmaDevice).Methods("POST")
+	router.HandleFunc("/createrdmadevice", CreateRdmaDevice).Methods("POST")
+	router.HandleFunc("/deleterdmadevice", DeleteRdmaDevice).Methods("POST")
 	log.Fatal(http.ListenAndServe(podIP+":"+restPort, router))
 }