provisioner: add lvm provisioner

Signed-off-by: Vicente Cheng <[email protected]>
harvester · Jun 20, 2024 · b2080f3 · b2080f3
1 parent baf624f
commit b2080f3
Show file tree

Hide file tree

Showing 2 changed files with 254 additions and 1 deletion.
diff --git a/pkg/controller/blockdevice/controller.go b/pkg/controller/blockdevice/controller.go
@@ -133,6 +133,7 @@ func (c *Controller) OnBlockDeviceChange(_ string, device *diskv1.BlockDevice) (
 		return nil, fmt.Errorf("failed to resolve persistent dev path for block device %s", device.Name)
 	}
 
+	logrus.Debugf("Prepare to format device %s", device.Name)
 	if formatted, requeue, err := provisionerInst.Format(devPath); !formatted {
 		if requeue {
 			c.Blockdevices.EnqueueAfter(c.Namespace, device.Name, jitterEnqueueDelay())
@@ -153,6 +154,7 @@ func (c *Controller) OnBlockDeviceChange(_ string, device *diskv1.BlockDevice) (
 	 * 2. Spec.Filesystem.Provisioned = true, Status.ProvisionPhase = ProvisionPhaseUnprovisioned
 	 *   -> Provision the device
 	 */
+	logrus.Debugf("Prepare to provision/update device %s", device.Name)
 	if needProvisionerUpdate(device, deviceCpy) {
 		logrus.Infof("Prepare to check the new device tags %v with device: %s", deviceCpy.Spec.Tags, device.Name)
 		requeue, err := provisionerInst.Update()
@@ -210,7 +212,10 @@ func (c *Controller) generateProvisioner(device *diskv1.BlockDevice) (provisione
 	case provisioner.TypeLonghornV2:
 		return nil, fmt.Errorf("TBD type %s", provisionerType)
 	case provisioner.TypeLVM:
-		return nil, fmt.Errorf("TBD type %s", provisionerType)
+		if device.Spec.Provisioner.VgName == "" {
+			return nil, fmt.Errorf("LVM VG name cannot be empty")
+		}
+		return c.generateLVMProvisioner(device), nil
 	}
 	return nil, fmt.Errorf("unsupported provisioner type %s", provisionerType)
 }
@@ -226,6 +231,11 @@ func (c *Controller) generateLHv1Provisioner(device *diskv1.BlockDevice) (provis
 	return provisioner.NewLHV1Provisioner(device, c.BlockInfo, node, c.Nodes, c.NodeCache, CacheDiskTags, c.semaphore)
 }
 
+func (c *Controller) generateLVMProvisioner(device *diskv1.BlockDevice) provisioner.Provisioner {
+	vgName := device.Spec.Provisioner.VgName
+	return provisioner.NewLVMProvisioner(vgName, device, c.BlockInfo)
+}
+
 func (c *Controller) updateDeviceStatus(device *diskv1.BlockDevice, devPath string) error {
 	var newStatus diskv1.DeviceStatus
 	var needAutoProvision bool

diff --git a/pkg/provisioner/lvm.go b/pkg/provisioner/lvm.go
@@ -0,0 +1,243 @@
+package provisioner
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+
+	diskv1 "github.com/harvester/node-disk-manager/pkg/apis/harvesterhci.io/v1beta1"
+	"github.com/harvester/node-disk-manager/pkg/block"
+	"github.com/harvester/node-disk-manager/pkg/utils"
+)
+
+type LVMProvisioner struct {
+	*provisioner
+	vgName string
+}
+
+func NewLVMProvisioner(vgName string, device *diskv1.BlockDevice, blockInfo block.Info) Provisioner {
+	baseProvisioner := &provisioner{
+		name:      TypeLVM,
+		blockInfo: blockInfo,
+		device:    device,
+	}
+	return &LVMProvisioner{
+		provisioner: baseProvisioner,
+		vgName:      vgName,
+	}
+}
+
+func (l *LVMProvisioner) GetProvisionerName() string {
+	return l.name
+}
+
+func (l *LVMProvisioner) Format(_ string) (bool, bool, error) {
+	// LVM provisioner does not need format
+	return true, false, nil
+}
+
+func (l *LVMProvisioner) UnFormat() (bool, error) {
+	// LVM provisioner does not need unformat
+	return false, nil
+}
+
+func (l *LVMProvisioner) Provision() (bool, error) {
+
+	setProvisioned := func() {
+		provisionerStatus := &diskv1.ProvisionerStatus{
+			Type:   l.name,
+			VgName: l.vgName,
+		}
+		l.device.Status.Provisioner = provisionerStatus
+		logrus.Debugf("Set blockdevice CRD (%v) to provisioned", l.device)
+		msg := fmt.Sprintf("Added disk %s to volume group %s ", l.device.Name, l.vgName)
+		setCondDiskAddedToNodeTrue(l.device, msg, diskv1.ProvisionPhaseProvisioned)
+	}
+
+	if l.vgName == "" {
+		return false, fmt.Errorf("LVM VG name cannot be empty")
+	}
+	logrus.Infof("%s provisioning block device %s to vg: %s", l.name, l.device.Name, l.vgName)
+
+	pvsResult, err := getPVScanResult()
+	if err != nil {
+		return true, fmt.Errorf("failed to get pvscan result. %v", err)
+	}
+	logrus.Debugf("pvscan result: %v", pvsResult)
+	pvFound := false
+	vgFound := false
+	devPath := l.device.Status.DeviceStatus.DevPath
+	for pv, vg := range pvsResult {
+		if pv == devPath {
+			pvFound = true
+			if vg == l.vgName {
+				logrus.Debugf("Block device %s is already in VG %s", l.device.Name, l.vgName)
+				setProvisioned()
+				return false, nil
+			}
+		}
+		if vg == l.vgName {
+			vgFound = true
+		}
+	}
+
+	if !pvFound {
+		if err := doPVCreate(devPath); err != nil {
+			return true, err
+		}
+	}
+	if !vgFound {
+		if err := doVGCreate(devPath, l.vgName); err != nil {
+			return true, err
+		}
+	} else {
+		if err := doVGExtend(devPath, l.vgName); err != nil {
+			return true, err
+		}
+
+	}
+
+	setProvisioned()
+	return false, nil
+}
+
+func (l *LVMProvisioner) UnProvision() (bool, error) {
+	logrus.Infof("%s unprovisioning block device %s from vg: %s", l.name, l.device.Name, l.vgName)
+
+	setUnprovisioned := func() {
+		l.device.Status.Provisioner = nil
+		l.device.Spec.Provisioner = nil
+		logrus.Debugf("Set blockdevice CRD (%v) to unprovisioned", l.device)
+		msg := fmt.Sprintf("Removed disk %s from volume group %s ", l.device.Name, l.vgName)
+		setCondDiskAddedToNodeFalse(l.device, msg, diskv1.ProvisionPhaseUnprovisioned)
+	}
+
+	pvsResult, err := getPVScanResult()
+	if err != nil {
+		return true, fmt.Errorf("failed to get pvscan result. %v", err)
+	}
+	logrus.Debugf("pvscan result: %v", pvsResult)
+	devPath := l.device.Status.DeviceStatus.DevPath
+	pvFound := false
+	isInVG := false
+	pvCountInVG := 0
+	for pv, vg := range pvsResult {
+		if pv == devPath {
+			pvFound = true
+			if vg == l.vgName {
+				isInVG = true
+				pvCountInVG++
+			}
+		} else {
+			if vg == l.vgName {
+				pvCountInVG++
+			}
+		}
+	}
+
+	if !pvFound {
+		logrus.Debugf("Block device %s is not in pvs.", l.device.Name)
+		setUnprovisioned()
+		return false, nil
+	}
+	if isInVG {
+		if pvCountInVG > 1 {
+			if err := doVGReduce(devPath, l.vgName); err != nil {
+				return true, err
+			}
+		} else {
+			if err := doVGRemove(l.vgName); err != nil {
+				return true, err
+			}
+		}
+	}
+	if err := doPVRemove(devPath); err != nil {
+		return true, err
+	}
+
+	setUnprovisioned()
+	return false, nil
+}
+
+func (l *LVMProvisioner) Update() (bool, error) {
+	// Make sure the volume group are all active
+	logrus.Infof("%s update block device %s from vg: %s", l.name, l.device.Name, l.vgName)
+	if err := doVGActivate(); err != nil {
+		return true, err
+	}
+
+	return false, nil
+}
+
+func getPVScanResult() (map[string]string, error) {
+	ns := utils.GetHostNamespacePath(utils.HostProcPath)
+	executor, err := utils.NewExecutorWithNS(ns)
+	if err != nil {
+		return nil, fmt.Errorf("generate executor failed. %v", err)
+	}
+
+	args := []string{"--noheadings", "-o", "pv_name,vg_name"}
+	output, err := executor.Execute("pvs", args)
+	if err != nil {
+		return nil, fmt.Errorf("pvs failed. %v", err)
+	}
+	lines := strings.Split(output, "\n")
+	pvScanResult := make(map[string]string)
+	for _, line := range lines {
+		if line == "" {
+			continue
+		}
+		fields := strings.Fields(line)
+		// Format should be like: /dev/sda vg01
+		pv := fields[0]
+		vg := ""
+		if len(fields) >= 2 {
+			vg = fields[1]
+		}
+		pvScanResult[pv] = vg
+	}
+	return pvScanResult, nil
+}
+
+func executeCommandWithNS(cmd string, args []string) error {
+	ns := utils.GetHostNamespacePath(utils.HostProcPath)
+	executor, err := utils.NewExecutorWithNS(ns)
+	if err != nil {
+		return fmt.Errorf("generate executor failed. %v", err)
+	}
+
+	_, err = executor.Execute(cmd, args)
+	if err != nil {
+		return fmt.Errorf("execute command %s, args: %v failed. %v", cmd, args, err)
+	}
+	return nil
+}
+
+func doPVCreate(devPath string) error {
+	return executeCommandWithNS("pvcreate", []string{devPath})
+}
+
+func doVGCreate(devPath, vgName string) error {
+	return executeCommandWithNS("vgcreate", []string{vgName, devPath})
+}
+
+func doVGExtend(devPath, vgName string) error {
+	return executeCommandWithNS("vgextend", []string{vgName, devPath})
+}
+
+func doVGReduce(devPath, vgName string) error {
+	return executeCommandWithNS("vgreduce", []string{vgName, devPath})
+}
+
+func doVGRemove(vgName string) error {
+	return executeCommandWithNS("vgremove", []string{vgName})
+}
+
+func doPVRemove(devPath string) error {
+	return executeCommandWithNS("pvremove", []string{devPath})
+}
+
+func doVGActivate() error {
+	return executeCommandWithNS("vgchange", []string{"--activate", "y"})
+}