From 8a2521015c372e5d3ca8e3f64f190b70c1880932 Mon Sep 17 00:00:00 2001 From: rohan2794 Date: Mon, 7 Oct 2024 14:20:42 +0530 Subject: [PATCH] feat(rdma): add support to create and delete rdma device on node Signed-off-by: rohan2794 --- common/e2e_agent/client.go | 76 +++++++++++++++++++++++++++ common/k8stest/util_rdma.go | 43 +++++++++++++++ tools/e2e-agent/Dockerfile | 2 +- tools/e2e-agent/build.sh | 2 +- tools/e2e-agent/e2e-agent.yaml | 2 +- tools/e2e-agent/rdma.go | 96 ++++++++++++++++++++++++++++++++++ tools/e2e-agent/server.go | 4 ++ 7 files changed, 222 insertions(+), 3 deletions(-) create mode 100644 common/k8stest/util_rdma.go create mode 100644 tools/e2e-agent/rdma.go diff --git a/common/e2e_agent/client.go b/common/e2e_agent/client.go index 10f737f..eec3045 100644 --- a/common/e2e_agent/client.go +++ b/common/e2e_agent/client.go @@ -100,6 +100,11 @@ type Zpool struct { PoolName string `json:"poolName"` } +type Rdma struct { + DeviceName string `json:"deviceName"` + InterfaceName string `json:"interfaceName"` +} + func sendRequest(reqType, url string, data interface{}) error { _, err := sendRequestGetResponse(reqType, url, data, true) return err @@ -1143,3 +1148,74 @@ func LvmLvRemoveThinPool(serverAddr string, vgName string) (string, error) { logf.Log.Info("LvmLvRemoveThinPool succeeded", "output", out) return out, err } + +// ListRdmaDevice get pre-created and available RDMA device +func ListRdmaDevice(serverAddr string) (string, error) { + logf.Log.Info("Executing GetRdmaDevice", "addr", serverAddr) + url := "http://" + getAgentAddress(serverAddr) + "/listrdmadevice" + encodedresult, err := sendRequestGetResponse("POST", url, nil, false) + if err != nil { + logf.Log.Info("sendRequestGetResponse", "encodedresult", encodedresult, "error", err.Error()) + return encodedresult, err + } + + out, e2eagenterrcode, err := UnwrapResult(encodedresult) + if err != nil { + logf.Log.Info("unwrap failed", "encodedresult", encodedresult, "error", err.Error()) + return encodedresult, err + } + if e2eagenterrcode != ErrNone { + return out, fmt.Errorf("failed to list available RDMA device, errcode %d", e2eagenterrcode) + } + logf.Log.Info("ListRdmaDevice succeeded", "output", out) + return out, err +} + +// CreateRdmaDevice create rdma device +func CreateRdmaDevice(serverAddr string, deviceName string, interfaceName string) (string, error) { + data := Rdma{ + DeviceName: deviceName, + InterfaceName: interfaceName, + } + logf.Log.Info("Executing CreateRdmaDevice", "addr", serverAddr, "data", data) + url := "http://" + getAgentAddress(serverAddr) + "/createrdmadevice" + encodedresult, err := sendRequestGetResponse("POST", url, data, false) + if err != nil { + logf.Log.Info("sendRequestGetResponse", "encodedresult", encodedresult, "error", err.Error()) + return encodedresult, err + } + out, e2eagenterrcode, err := UnwrapResult(encodedresult) + if err != nil { + logf.Log.Info("unwrap failed", "encodedresult", encodedresult, "error", err.Error()) + return encodedresult, err + } + if e2eagenterrcode != ErrNone { + return out, fmt.Errorf("failed to create rdma device, errcode %d", e2eagenterrcode) + } + logf.Log.Info("CreateRdmaDevice succeeded", "output", out) + return out, err +} + +// DeleteRdmaDevice destroy rdma device +func DeleteRdmaDevice(serverAddr string, deeviceName string) (string, error) { + data := Rdma{ + DeviceName: deeviceName, + } + logf.Log.Info("Executing DeleteRdmaDevice", "addr", serverAddr, "data", data) + url := "http://" + getAgentAddress(serverAddr) + "/deleterdmadevice" + encodedresult, err := sendRequestGetResponse("POST", url, data, false) + if err != nil { + logf.Log.Info("sendRequestGetResponse", "encodedresult", encodedresult, "error", err.Error()) + return encodedresult, err + } + out, e2eagenterrcode, err := UnwrapResult(encodedresult) + if err != nil { + logf.Log.Info("unwrap failed", "encodedresult", encodedresult, "error", err.Error()) + return encodedresult, err + } + if e2eagenterrcode != ErrNone { + return out, fmt.Errorf("failed to delete rdma device, errcode %d", e2eagenterrcode) + } + logf.Log.Info("DeleteRdmaDevice succeeded", "output", out) + return out, err +} diff --git a/common/k8stest/util_rdma.go b/common/k8stest/util_rdma.go new file mode 100644 index 0000000..bc47435 --- /dev/null +++ b/common/k8stest/util_rdma.go @@ -0,0 +1,43 @@ +package k8stest + +import ( + "encoding/json" + "fmt" + + agent "github.com/openebs/openebs-e2e/common/e2e_agent" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +type RdmaDeviceNetworkInterface struct { + IfIndex int `json:"ifindex"` + IfName string `json:"ifname"` + Port int `json:"port"` + State string `json:"state"` + PhysicalState string `json:"physical_state"` + NetDev string `json:"netdev"` + NetDevIndex int `json:"netdev_index"` +} + +func ListRdmaDevice(node string) ([]RdmaDeviceNetworkInterface, error) { + var rdmaDeiceList []RdmaDeviceNetworkInterface + nodeIp, err := GetNodeIPAddress(node) + if err != nil { + return rdmaDeiceList, fmt.Errorf("failed to get node %s ip, error: %v", node, err) + } + + rdmaDevice, err := agent.ListRdmaDevice(*nodeIp) + if err != nil { + return rdmaDeiceList, fmt.Errorf("failed to list RDMA device on node %s , error: %v", node, err) + } + if rdmaDevice == "" { + logf.Log.Info("RDMA device list failed with empty string", "output", rdmaDevice) + return rdmaDeiceList, fmt.Errorf("failed to list RDMA device on node %s", node) + } + output := trimForJson(rdmaDevice) + if err = json.Unmarshal([]byte(output), &rdmaDeiceList); err != nil { + logf.Log.Info("Failed to unmarshal rdma list", "output", output) + return rdmaDeiceList, fmt.Errorf("failed to unmarshal rdma list on node %s , output: %s,error: %v", node, output, err) + } + logf.Log.Info("RDMA device", "node", node, "list", rdmaDeiceList) + return rdmaDeiceList, nil +} diff --git a/tools/e2e-agent/Dockerfile b/tools/e2e-agent/Dockerfile index 2048c4b..04c2844 100644 --- a/tools/e2e-agent/Dockerfile +++ b/tools/e2e-agent/Dockerfile @@ -11,7 +11,7 @@ COPY ./ / # It ensures that devices are configured as soon as they are plugged in and discovered. # It propagates information about a processed device. RUN apt-get update; apt-get install net-tools iptables wget parted udev nvme-cli build-essential gettext gettext-base \ - libinih-dev uuid-dev liburcu-dev libblkid-dev btrfs-progs -y; + libinih-dev uuid-dev liburcu-dev libblkid-dev btrfs-progs ibverbs-utils rdma-core -y; RUN wget https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz; \ tar -C /usr/local/ -xzf go${GO_VERSION}.linux-amd64.tar.gz; \ rm -rf go${GO_VERSION}.linux-amd64.tar.gz; \ diff --git a/tools/e2e-agent/build.sh b/tools/e2e-agent/build.sh index e3edacf..56b4659 100755 --- a/tools/e2e-agent/build.sh +++ b/tools/e2e-agent/build.sh @@ -9,7 +9,7 @@ # as long as we do not make breaking changes. set -e IMAGE="openebs/e2e-agent" -TAG="v3.0.4" +TAG="v3.0.5" registry="" tag_as_latest="" diff --git a/tools/e2e-agent/e2e-agent.yaml b/tools/e2e-agent/e2e-agent.yaml index f1e4250..59c2568 100644 --- a/tools/e2e-agent/e2e-agent.yaml +++ b/tools/e2e-agent/e2e-agent.yaml @@ -68,7 +68,7 @@ spec: securityContext: privileged: true allowPrivilegeEscalation: true - image: openebs/e2e-agent:v3.0.4 + image: openebs/e2e-agent:v3.0.5 imagePullPolicy: Always volumeMounts: - name: host-root diff --git a/tools/e2e-agent/rdma.go b/tools/e2e-agent/rdma.go new file mode 100644 index 0000000..7b5ab5f --- /dev/null +++ b/tools/e2e-agent/rdma.go @@ -0,0 +1,96 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + + "k8s.io/klog/v2" +) + +type Rdma struct { + DeviceName string `json:"deviceName"` + InterfaceName string `json:"interfaceName"` +} + +// ListRdmaDevice list RDMA device +func ListRdmaDevice(w http.ResponseWriter, r *http.Request) { + var msg string + klog.Info("List available RDMA device") + + rdmaDeviceListCommand := "rdma link -j" + output, err := bashLocal(rdmaDeviceListCommand) + if err != nil { + msg = fmt.Sprintf("cannot list RDMA device. Error %s", err.Error()) + klog.Error(msg) + WrapResult(msg, ErrExecFailed, w) + return + } + WrapResult(string(output), ErrNone, w) +} + +// CreateRdmaDevice create rdma device +func CreateRdmaDevice(w http.ResponseWriter, r *http.Request) { + var msg string + var rdma Rdma + d := json.NewDecoder(r.Body) + if err := d.Decode(&rdma); err != nil { + msg = fmt.Sprintf("failed to read JSON encoded data, Error: %s", err.Error()) + klog.Error(msg) + WrapResult(msg, ErrJsonDecode, w) + return + } + if rdma.DeviceName == "" { + msg = "no device name passed" + klog.Error(msg) + WrapResult(msg, UnprocessableEntityErrorCode, w) + return + } + if rdma.InterfaceName == "" { + msg = "no interface name passed" + klog.Error(msg) + WrapResult(msg, UnprocessableEntityErrorCode, w) + return + } + klog.Info("creates rdma device, data: %v", rdma) + + rdmaDeviceCreateCommand := fmt.Sprintf("rdma link add %s type rxe netdev %s", rdma.DeviceName, rdma.InterfaceName) + output, err := bashLocal(rdmaDeviceCreateCommand) + if err != nil { + msg = fmt.Sprintf("cannot create rdma device, Error %s", err.Error()) + klog.Error(msg) + WrapResult(msg, ErrExecFailed, w) + return + } + WrapResult(string(output), ErrNone, w) +} + +// DeleteRdmaDevice destroy rdma device +func DeleteRdmaDevice(w http.ResponseWriter, r *http.Request) { + var msg string + var rdma Rdma + d := json.NewDecoder(r.Body) + if err := d.Decode(&rdma); err != nil { + msg = fmt.Sprintf("failed to read JSON encoded data, Error: %s", err.Error()) + klog.Error(msg) + WrapResult(msg, ErrJsonDecode, w) + return + } + if rdma.DeviceName == "" { + msg = "no rdma device name passed" + klog.Error(msg) + WrapResult(msg, UnprocessableEntityErrorCode, w) + return + } + + klog.Info("delete rdma device, data: %v", rdma) + rdmaDeviceDeleteCommand := fmt.Sprintf("rdma link delete %s", rdma.DeviceName) + output, err := bashLocal(rdmaDeviceDeleteCommand) + if err != nil { + msg = fmt.Sprintf("cannot delete rdma device, Error %s", err.Error()) + klog.Error(msg) + WrapResult(msg, ErrExecFailed, w) + return + } + WrapResult(string(output), ErrNone, w) +} diff --git a/tools/e2e-agent/server.go b/tools/e2e-agent/server.go index a9adeba..2942b33 100644 --- a/tools/e2e-agent/server.go +++ b/tools/e2e-agent/server.go @@ -176,6 +176,10 @@ func handleRequests() { //localPV router.HandleFunc("/createhostpathdisk", CreateHostPathDisk).Methods("POST") router.HandleFunc("/removehostpathdisk", RemoveHostPathDisk).Methods("POST") + //RDMA + router.HandleFunc("/listrdmadevice", ListRdmaDevice).Methods("POST") + router.HandleFunc("/createrdmadevice", CreateRdmaDevice).Methods("POST") + router.HandleFunc("/deleterdmadevice", DeleteRdmaDevice).Methods("POST") log.Fatal(http.ListenAndServe(podIP+":"+restPort, router)) }