From aa92d562028dbe9303201c453936ea721b97a60d Mon Sep 17 00:00:00 2001 From: rohan2794 Date: Mon, 23 Sep 2024 18:56:16 +0530 Subject: [PATCH] chore(lvm): add support to configure auto lvm thin pool expansion capability to e2e agent Signed-off-by: rohan2794 --- common/e2e_agent/client.go | 50 +++++++++- common/lvm/util.go | 44 +++++++++ scripts/k8s/deployer.sh | 38 ++++++++ tools/e2e-agent/Dockerfile | 2 +- tools/e2e-agent/build.sh | 2 +- tools/e2e-agent/e2e-agent.yaml | 35 ++++++- tools/e2e-agent/error.go | 13 +-- tools/e2e-agent/lvm.go | 163 +++++++++++++++++++++++++++++---- tools/e2e-agent/server.go | 2 + 9 files changed, 320 insertions(+), 29 deletions(-) diff --git a/common/e2e_agent/client.go b/common/e2e_agent/client.go index b6e159b..10f737f 100644 --- a/common/e2e_agent/client.go +++ b/common/e2e_agent/client.go @@ -77,7 +77,7 @@ type Lvm struct { Pv string `json:"pv"` // Physical volume Vg string `json:"vg"` // Volume group ThinPoolAutoExtendThreshold int `json:"thinPoolAutoExtendThreshold"` // thin pool auto extend threshold - ThinPoolAutoExtendPercent int `json:"ThinPoolAutoExtendPercent"` // thin pool auto extend percent + ThinPoolAutoExtendPercent int `json:"thinPoolAutoExtendPercent"` // thin pool auto extend percent } type LoopDevice struct { @@ -1095,3 +1095,51 @@ func RemoveHostPathDisk(serverAddr string, diskPath string, mountPoint string) e logf.Log.Info("RemoveHostPathDisk succeeded", "output", out) return err } + +// LvmLvChangeMonitor monitor lvm lv +func LvmLvChangeMonitor(serverAddr string, vgName string) (string, error) { + data := Lvm{ + Vg: vgName, + } + logf.Log.Info("Executing lvchange", "addr", serverAddr, "data", data) + url := "http://" + getAgentAddress(serverAddr) + "/lvmlvchangemonitor" + encodedresult, err := sendRequestGetResponse("POST", url, data, false) + if err != nil { + logf.Log.Info("sendRequestGetResponse", "encodedresult", encodedresult, "error", err.Error()) + return encodedresult, err + } + out, e2eagenterrcode, err := UnwrapResult(encodedresult) + if err != nil { + logf.Log.Info("unwrap failed", "encodedresult", encodedresult, "error", err.Error()) + return encodedresult, err + } + if e2eagenterrcode != ErrNone { + return out, fmt.Errorf("failed to monitor lvm lv, errcode %d", e2eagenterrcode) + } + logf.Log.Info("LvmLvChangeMonitor succeeded", "output", out) + return out, err +} + +// LvmLvRemoveThinPool delete lvm thin pool lv +func LvmLvRemoveThinPool(serverAddr string, vgName string) (string, error) { + data := Lvm{ + Vg: vgName, + } + logf.Log.Info("Executing lvchange", "addr", serverAddr, "data", data) + url := "http://" + getAgentAddress(serverAddr) + "/lvmlvremovethinpool" + encodedresult, err := sendRequestGetResponse("POST", url, data, false) + if err != nil { + logf.Log.Info("sendRequestGetResponse", "encodedresult", encodedresult, "error", err.Error()) + return encodedresult, err + } + out, e2eagenterrcode, err := UnwrapResult(encodedresult) + if err != nil { + logf.Log.Info("unwrap failed", "encodedresult", encodedresult, "error", err.Error()) + return encodedresult, err + } + if e2eagenterrcode != ErrNone { + return out, fmt.Errorf("failed to remove lvm thin pool lv, errcode %d", e2eagenterrcode) + } + logf.Log.Info("LvmLvRemoveThinPool succeeded", "output", out) + return out, err +} diff --git a/common/lvm/util.go b/common/lvm/util.go index 374b9d3..e11903c 100644 --- a/common/lvm/util.go +++ b/common/lvm/util.go @@ -168,3 +168,47 @@ func SetupLvmNodes(vgName string, size int64) (LvmNodesDevicePvVgConfig, error) err = lvmNodeConfig.ConfigureLvmNodesWithDeviceAndVg() return lvmNodeConfig, err } + +// EnableLvmThinPoolAutoExpansion enable auto extending of the Thin Pool (Configure Over-Provisioning protection) +func EnableLvmThinPoolAutoExpansion(thinPoolAutoExtendThreshold, thinPoolAutoExtendPercent int) error { + + workerNodes, err := ListLvmNode(common.NSOpenEBS()) + if err != nil { + return fmt.Errorf("failed to list lvm worker nodes, error: %v", err) + } + if len(workerNodes) == 0 { + return fmt.Errorf("lvm worker nodes not found") + } + /* + Editing the settings in the /etc/lvm/lvm.conf can allow auto growth of the thin pool when required. + By default, the threshold is 100% which means that the pool will not grow. + If we set this to, 75%, the Thin Pool will autoextend when the pool is 75% full. + It will increase by the default percentage of 20% if the value is not changed. + We can see these settings using the command grep against the file. + $ grep -E ‘^\s*thin_pool_auto’ /etc/lvm/lvm.conf + thin_pool_autoextend_threshold = 100 + thin_pool_autoextend_percent = 20 + */ + + for _, node := range workerNodes { + nodeIp, err := k8stest.GetNodeIPAddress(node) + if err != nil { + return fmt.Errorf("failed to get node %s IP, error: %v", node, err) + } + + out, err := e2e_agent.LvmThinPoolAutoExtendThreshold(*nodeIp, thinPoolAutoExtendThreshold) + if err != nil { + return fmt.Errorf("failed to set up thin_pool_autoextend_threshold value %d on node %s,output: %s error: %v", + thinPoolAutoExtendThreshold, node, out, err) + } + + out, err = e2e_agent.LvmThinPoolAutoExtendPercent(*nodeIp, thinPoolAutoExtendPercent) + if err != nil { + return fmt.Errorf("failed to set up thin_pool_autoextend_percent value %d on node %s,output: %s error: %v", + thinPoolAutoExtendPercent, node, out, err) + } + + } + + return nil +} diff --git a/scripts/k8s/deployer.sh b/scripts/k8s/deployer.sh index cd24784..806608c 100755 --- a/scripts/k8s/deployer.sh +++ b/scripts/k8s/deployer.sh @@ -6,6 +6,8 @@ SCRIPT_DIR="$(dirname "$0")" TMP_KIND="/tmp/kind/openebs-e2e" TMP_KIND_CONFIG="$TMP_KIND/config.yaml" TMP_KIND_ZFS="$TMP_KIND/zfs" +TMP_KIND_LVM="$TMP_KIND/lvm" +TMP_KIND_ZFS="$TMP_KIND/zpool" WORKERS=2 DELAY="false" CORES=1 @@ -205,6 +207,42 @@ EOF EOF fi + if [ "$SETUP_ZFS" = "true" ]; then + # Should already be installed by prereq script + ZPOOL=$(realpath $(which zpool)) + cat <>$TMP_KIND_ZPOOL + #/bin/sh + chroot /host $ZPOOL "\$@" +EOF + chmod +x $TMP_KIND_ZPOOL + cat <> "$TMP_KIND_CONFIG" + - hostPath: / + containerPath: /host + propagation: HostToContainer + - hostPath: $TMP_KIND_ZPOOL + containerPath: /sbin/zpool + propagation: HostToContainer +EOF + fi + + if [ "$SETUP_LVM" = "true" ]; then + # Should already be installed by prereq script + LVM=$(realpath $(which lvm)) + cat <>$TMP_KIND_LVM + #/bin/sh + chroot /host $LVM "\$@" +EOF + chmod +x $TMP_KIND_LVM + cat <> "$TMP_KIND_CONFIG" + - hostPath: / + containerPath: /host + propagation: HostToContainer + - hostPath: $TMP_KIND_LVM + containerPath: /sbin/lvm + propagation: HostToContainer +EOF + fi + if [ "$SETUP_MAYASTOR" = "true" ]; then if [ "$LABEL" = "true" ]; then cat <> "$TMP_KIND_CONFIG" diff --git a/tools/e2e-agent/Dockerfile b/tools/e2e-agent/Dockerfile index 513690d..2048c4b 100644 --- a/tools/e2e-agent/Dockerfile +++ b/tools/e2e-agent/Dockerfile @@ -11,7 +11,7 @@ COPY ./ / # It ensures that devices are configured as soon as they are plugged in and discovered. # It propagates information about a processed device. RUN apt-get update; apt-get install net-tools iptables wget parted udev nvme-cli build-essential gettext gettext-base \ - libinih-dev uuid-dev liburcu-dev libblkid-dev btrfs-progs lvm2 zfsutils-linux -y; + libinih-dev uuid-dev liburcu-dev libblkid-dev btrfs-progs -y; RUN wget https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz; \ tar -C /usr/local/ -xzf go${GO_VERSION}.linux-amd64.tar.gz; \ rm -rf go${GO_VERSION}.linux-amd64.tar.gz; \ diff --git a/tools/e2e-agent/build.sh b/tools/e2e-agent/build.sh index c39f82e..e3edacf 100755 --- a/tools/e2e-agent/build.sh +++ b/tools/e2e-agent/build.sh @@ -9,7 +9,7 @@ # as long as we do not make breaking changes. set -e IMAGE="openebs/e2e-agent" -TAG="v3.0.3" +TAG="v3.0.4" registry="" tag_as_latest="" diff --git a/tools/e2e-agent/e2e-agent.yaml b/tools/e2e-agent/e2e-agent.yaml index 8e715a0..f1e4250 100644 --- a/tools/e2e-agent/e2e-agent.yaml +++ b/tools/e2e-agent/e2e-agent.yaml @@ -17,6 +17,24 @@ data: else chroot /host zfs "$@" fi + ZPOOL: | + #!/bin/sh + if [ -x /host/sbin/zpool ]; then + chroot /host /sbin/zpool "$@" + elif [ -x /host/usr/sbin/zpool ]; then + chroot /host /usr/sbin/zpool "$@" + else + chroot /host zpool "$@" + fi + LVM: | + #!/bin/sh + if [ -x /host/sbin/lvm ]; then + chroot /host /sbin/lvm "$@" + elif [ -x /host/usr/sbin/lvm ]; then + chroot /host /usr/sbin/lvm "$@" + else + chroot /host lvm "$@" + fi --- kind: DaemonSet @@ -50,7 +68,7 @@ spec: securityContext: privileged: true allowPrivilegeEscalation: true - image: openebs/e2e-agent:v3.0.3 + image: openebs/e2e-agent:v3.0.4 imagePullPolicy: Always volumeMounts: - name: host-root @@ -63,6 +81,12 @@ spec: - mountPath: /sbin/zfs name: chroot-zfs subPath: ZFS + - mountPath: /sbin/lvm + name: chroot-lvm + subPath: LVM + - mountPath: /sbin/zpool + name: chroot-zpool + subPath: ZPOOL volumes: - name: host-root hostPath: @@ -80,6 +104,13 @@ spec: configMap: defaultMode: 0555 name: test-vars + - name: chroot-lvm + configMap: + defaultMode: 0555 + name: test-vars + - name: chroot-zpool + configMap: + defaultMode: 0555 + name: test-vars - --- diff --git a/tools/e2e-agent/error.go b/tools/e2e-agent/error.go index 9a4b85c..ec12ce4 100644 --- a/tools/e2e-agent/error.go +++ b/tools/e2e-agent/error.go @@ -12,12 +12,13 @@ type E2eAgentErrcode int const ( // general errors - ErrNone E2eAgentErrcode = 0 - ErrGeneral E2eAgentErrcode = 1 - ErrJsonDecode E2eAgentErrcode = 2 - ErrJsonEncode E2eAgentErrcode = 3 - ErrReadFail E2eAgentErrcode = 4 - ErrExecFailed E2eAgentErrcode = 5 + ErrNone E2eAgentErrcode = 0 + ErrGeneral E2eAgentErrcode = 1 + ErrJsonDecode E2eAgentErrcode = 2 + ErrJsonEncode E2eAgentErrcode = 3 + ErrReadFail E2eAgentErrcode = 4 + ErrExecFailed E2eAgentErrcode = 5 + ErrFileNotExist E2eAgentErrcode = 6 // event errors ErrConnectFail E2eAgentErrcode = 101 diff --git a/tools/e2e-agent/lvm.go b/tools/e2e-agent/lvm.go index 7480bee..41b936d 100644 --- a/tools/e2e-agent/lvm.go +++ b/tools/e2e-agent/lvm.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "net/http" + "os" "k8s.io/klog/v2" ) @@ -12,7 +13,7 @@ type Lvm struct { Pv string `json:"pv"` // Physical volume Vg string `json:"vg"` // Volume group ThinPoolAutoExtendThreshold int `json:"thinPoolAutoExtendThreshold"` // thin pool auto extend threshold - ThinPoolAutoExtendPercent int `json:"ThinPoolAutoExtendPercent"` // thin pool auto extend percent + ThinPoolAutoExtendPercent int `json:"thinPoolAutoExtendPercent"` // thin pool auto extend percent } // LvmVersion check lvm version installed on node @@ -36,7 +37,7 @@ func LvmListVg(w http.ResponseWriter, r *http.Request) { var msg string klog.Info("List lvm vgs") - lvmListCommand := "vgs --reportformat json" + lvmListCommand := "lvm vgs --reportformat json" output, err := bashLocal(lvmListCommand) if err != nil { msg = fmt.Sprintf("cannot list lvm vgs Error %s", err.Error()) @@ -52,7 +53,7 @@ func LvmListPv(w http.ResponseWriter, r *http.Request) { var msg string klog.Info("List lvm pvs") - lvmListCommand := "pvs --reportformat json" + lvmListCommand := "lvm pvs --reportformat json" output, err := bashLocal(lvmListCommand) if err != nil { msg = fmt.Sprintf("cannot list lvm pvs Error %s", err.Error()) @@ -82,7 +83,7 @@ func LvmCreatePv(w http.ResponseWriter, r *http.Request) { } klog.Info("creates lvm pv, data: %v", lvm) - lvmPvCreateCommand := fmt.Sprintf("pvcreate %s", lvm.Pv) + lvmPvCreateCommand := fmt.Sprintf("lvm pvcreate %s", lvm.Pv) output, err := bashLocal(lvmPvCreateCommand) if err != nil { msg = fmt.Sprintf("cannot create lvm pv Error %s", err.Error()) @@ -118,7 +119,7 @@ func LvmCreateVg(w http.ResponseWriter, r *http.Request) { } klog.Info("creates lvm vg, data: %v", lvm) - lvmVgCreateCommand := fmt.Sprintf("vgcreate %s %s", lvm.Vg, lvm.Pv) + lvmVgCreateCommand := fmt.Sprintf("lvm vgcreate %s %s", lvm.Vg, lvm.Pv) output, err := bashLocal(lvmVgCreateCommand) if err != nil { msg = fmt.Sprintf("cannot create lvm vg Error %s", err.Error()) @@ -148,7 +149,7 @@ func LvmRemovePv(w http.ResponseWriter, r *http.Request) { } klog.Info("remove lvm pv, data: %v", lvm) - lvmPvRemoveCommand := fmt.Sprintf("pvremove %s", lvm.Pv) + lvmPvRemoveCommand := fmt.Sprintf("lvm pvremove %s", lvm.Pv) output, err := bashLocal(lvmPvRemoveCommand) if err != nil { msg = fmt.Sprintf("cannot remove lvm pv Error %s", err.Error()) @@ -178,7 +179,7 @@ func LvmRemoveVg(w http.ResponseWriter, r *http.Request) { } klog.Info("remove lvm vg, data: %v", lvm) - lvmVgRemoveCommand := fmt.Sprintf("vgremove %s", lvm.Vg) + lvmVgRemoveCommand := fmt.Sprintf("lvm vgremove %s", lvm.Vg) output, err := bashLocal(lvmVgRemoveCommand) if err != nil { msg = fmt.Sprintf("cannot create lvm vg Error %s", err.Error()) @@ -206,13 +207,27 @@ func LvmThinPoolAutoExtendThreshold(w http.ResponseWriter, r *http.Request) { WrapResult(msg, UnprocessableEntityErrorCode, w) return } - klog.Info("update lvm.conf thin pool auto extend threshold value, data: %v", lvm) + lvmConfFile, err := getLvmConfFile() + if err != nil { + msg = fmt.Sprintf("lvm conf file verification error, error: %s", err.Error()) + klog.Error(msg) + WrapResult(msg, ErrFileNotExist, w) + return + } + if lvmConfFile == "" { + msg := "lvm conf file not found" + klog.Error(msg) + WrapResult(msg, ErrFileNotExist, w) + return + } + klog.Info("update %s thin pool auto extend threshold value, data: %v", lvmConfFile, lvm) - lvmThinPoolAutoExtentThresholdCommand := fmt.Sprintf("sed -i '/^[^#]*thin_pool_autoextend_threshold/ s/= .*/= %d/' /etc/lvm/lvm.conf", - lvm.ThinPoolAutoExtendThreshold) + lvmThinPoolAutoExtentThresholdCommand := fmt.Sprintf("sed -i '/^[^#]*thin_pool_autoextend_threshold/ s/= .*/= %d/' %s", + lvm.ThinPoolAutoExtendThreshold, + lvmConfFile) output, err := bashLocal(lvmThinPoolAutoExtentThresholdCommand) if err != nil { - msg = fmt.Sprintf("update lvm.conf thin pool auto extend threshold value, Error %s", err.Error()) + msg = fmt.Sprintf("update %s thin pool auto extend threshold value, Error %s", lvmConfFile, err.Error()) klog.Error(msg) WrapResult(msg, ErrExecFailed, w) return @@ -231,19 +246,131 @@ func LvmThinPoolAutoExtendPercent(w http.ResponseWriter, r *http.Request) { WrapResult(msg, ErrJsonDecode, w) return } - if lvm.ThinPoolAutoExtendThreshold <= 0 { - msg = "no thin pool auto extent percent passed" + if lvm.ThinPoolAutoExtendPercent <= 0 { + msg = "no thin pool auto extend percent passed" + klog.Error(msg) + WrapResult(msg, UnprocessableEntityErrorCode, w) + return + } + lvmConfFile, err := getLvmConfFile() + if err != nil { + msg = fmt.Sprintf("lvm conf file verification error, error: %v", err.Error()) + klog.Error(msg) + WrapResult(msg, ErrFileNotExist, w) + return + } + if lvmConfFile == "" { + msg := "lvm conf file not found" + klog.Error(msg) + WrapResult(msg, ErrFileNotExist, w) + return + } + klog.Info("update %s thin pool auto extend threshold value, data: %v", lvmConfFile, lvm) + + lvmThinPoolAutoExtendPercentCommand := fmt.Sprintf("sed -i '/^[^#]*thin_pool_autoextend_percent/ s/= .*/= %d/' %s", + lvm.ThinPoolAutoExtendPercent, + lvmConfFile) + output, err := bashLocal(lvmThinPoolAutoExtendPercentCommand) + if err != nil { + msg = fmt.Sprintf("update %s thin pool auto extend percent value, Error %s", lvmConfFile, err.Error()) + klog.Error(msg) + WrapResult(msg, ErrExecFailed, w) + return + } + WrapResult(output, ErrNone, w) +} + +// If it's Github action based kind cluster, lvm conf file will be +// expected at /host/host/etc/lvm/lvm.conf otherwise it will be at /host/etc/lvm/lvm.conf +func getLvmConfFile() (string, error) { + ghLvmConf := "/host/host/etc/lvm/lvm.conf" + isPresent, err := isFilePresent(ghLvmConf) + if err != nil { + return "", err + } + if isPresent { + return ghLvmConf, nil + } else { + lvmConf := "/host/etc/lvm/lvm.conf" + isPresent, err := isFilePresent(lvmConf) + if err != nil { + return "", err + } + if isPresent { + return lvmConf, nil + } + } + return "", nil +} + +func isFilePresent(file string) (bool, error) { + _, err := os.Stat(file) + if err != nil { + if os.IsNotExist(err) { + klog.Info("file not present", "file", file) + return false, nil + } + return false, fmt.Errorf("error while checking %s file existence, error: %v", file, err) + } + klog.Info("file present,", "file:", file) + return true, nil +} + +// LvmLvChangeMonitor monitor lvm lv +func LvmLvChangeMonitor(w http.ResponseWriter, r *http.Request) { + var msg string + var lvm Lvm + d := json.NewDecoder(r.Body) + if err := d.Decode(&lvm); err != nil { + msg = fmt.Sprintf("failed to read JSON encoded data, Error: %s", err.Error()) + klog.Error(msg) + WrapResult(msg, ErrJsonDecode, w) + return + } + + if lvm.Vg == "" { + msg = "no vg name passed" + klog.Error(msg) + WrapResult(msg, UnprocessableEntityErrorCode, w) + return + } + klog.Info("monitor lvm lv, data: %v", lvm) + + lvmLvMonitorCommand := fmt.Sprintf("lvm lvchange --monitor y %s/%s_thinpool", lvm.Vg, lvm.Vg) + output, err := bashLocal(lvmLvMonitorCommand) + if err != nil { + msg = fmt.Sprintf("cannot monitor lvm lv, Error %s", err.Error()) + klog.Error(msg) + WrapResult(msg, ErrExecFailed, w) + return + } + WrapResult(output, ErrNone, w) +} + +// LvmLvRemoveThinPool lv thin pool +func LvmLvRemoveThinPool(w http.ResponseWriter, r *http.Request) { + var msg string + var lvm Lvm + d := json.NewDecoder(r.Body) + if err := d.Decode(&lvm); err != nil { + msg = fmt.Sprintf("failed to read JSON encoded data, Error: %s", err.Error()) + klog.Error(msg) + WrapResult(msg, ErrJsonDecode, w) + return + } + + if lvm.Vg == "" { + msg = "no vg name passed" klog.Error(msg) WrapResult(msg, UnprocessableEntityErrorCode, w) return } - klog.Info("update lvm.conf thin pool auto extend threshold value, data: %v", lvm) + klog.Info("remove lvm thin pool lv, data: %v", lvm) - lvmThinPoolAutoExtentPercentCommand := fmt.Sprintf("sed -i '/^[^#]*thin_pool_autoextend_percent/ s/= .*/= %d/' /etc/lvm/lvm.conf", - lvm.ThinPoolAutoExtendPercent) - output, err := bashLocal(lvmThinPoolAutoExtentPercentCommand) + lvmLvMonitorCommand := fmt.Sprintf("lvm lvremove -f --noudevsync %s/%s_thinpool", lvm.Vg, lvm.Vg) + output, err := bashLocal(lvmLvMonitorCommand) if err != nil { - msg = fmt.Sprintf("update lvm.conf thin pool auto extend percent value, Error %s", err.Error()) + msg = fmt.Sprintf("cannot remove lvm thin pool lv, Error %s", err.Error()) klog.Error(msg) WrapResult(msg, ErrExecFailed, w) return diff --git a/tools/e2e-agent/server.go b/tools/e2e-agent/server.go index d9200f8..a9adeba 100644 --- a/tools/e2e-agent/server.go +++ b/tools/e2e-agent/server.go @@ -163,6 +163,8 @@ func handleRequests() { router.HandleFunc("/lvmremovevg", LvmRemoveVg).Methods("POST") router.HandleFunc("/lvmthinpoolautoextendthreshold", LvmThinPoolAutoExtendThreshold).Methods("POST") router.HandleFunc("/lvmthinpoolautoextendpercent", LvmThinPoolAutoExtendPercent).Methods("POST") + router.HandleFunc("/lvmlvchangemonitor", LvmLvChangeMonitor).Methods("POST") + router.HandleFunc("/lvmlvremovethinpool", LvmLvRemoveThinPool).Methods("POST") //loop device router.HandleFunc("/createloopdevice", CreateLoopDevice).Methods("POST") router.HandleFunc("/deleteloopdevice", DeleteLoopDevice).Methods("POST")