From df2eb1ef499481ab331ba066fe120dffa1a12e00 Mon Sep 17 00:00:00 2001 From: Vicente Cheng Date: Tue, 11 Jun 2024 23:57:45 +0800 Subject: [PATCH] provisioner: add lvm provisioner Signed-off-by: Vicente Cheng --- pkg/controller/blockdevice/controller.go | 12 +- pkg/provisioner/lvm.go | 309 +++++++++++++++++++++++ 2 files changed, 320 insertions(+), 1 deletion(-) create mode 100644 pkg/provisioner/lvm.go diff --git a/pkg/controller/blockdevice/controller.go b/pkg/controller/blockdevice/controller.go index fb5dac2e..a3dd399d 100644 --- a/pkg/controller/blockdevice/controller.go +++ b/pkg/controller/blockdevice/controller.go @@ -133,6 +133,7 @@ func (c *Controller) OnBlockDeviceChange(_ string, device *diskv1.BlockDevice) ( return nil, fmt.Errorf("failed to resolve persistent dev path for block device %s", device.Name) } + logrus.Debugf("Prepare to format device %s", device.Name) if formatted, requeue, err := provisionerInst.Format(devPath); !formatted { if requeue { c.Blockdevices.EnqueueAfter(c.Namespace, device.Name, jitterEnqueueDelay()) @@ -153,6 +154,7 @@ func (c *Controller) OnBlockDeviceChange(_ string, device *diskv1.BlockDevice) ( * 2. Spec.Filesystem.Provisioned = true, Status.ProvisionPhase = ProvisionPhaseUnprovisioned * -> Provision the device */ + logrus.Debugf("Prepare to provision/update device %s", device.Name) if needProvisionerUpdate(device, deviceCpy) { logrus.Infof("Prepare to check the new device tags %v with device: %s", deviceCpy.Spec.Tags, device.Name) requeue, err := provisionerInst.Update() @@ -210,7 +212,10 @@ func (c *Controller) generateProvisioner(device *diskv1.BlockDevice) (provisione case provisioner.TypeLonghornV2: return nil, fmt.Errorf("TBD type %s", provisionerType) case provisioner.TypeLVM: - return nil, fmt.Errorf("TBD type %s", provisionerType) + if device.Spec.Provisioner.VgName == "" { + return nil, fmt.Errorf("LVM VG name cannot be empty") + } + return c.generateLVMProvisioner(device), nil } return nil, fmt.Errorf("unsupported provisioner type %s", provisionerType) } @@ -226,6 +231,11 @@ func (c *Controller) generateLHv1Provisioner(device *diskv1.BlockDevice) (provis return provisioner.NewLHV1Provisioner(device, c.BlockInfo, node, c.Nodes, c.NodeCache, CacheDiskTags, c.semaphore) } +func (c *Controller) generateLVMProvisioner(device *diskv1.BlockDevice) provisioner.Provisioner { + vgName := device.Spec.Provisioner.VgName + return provisioner.NewLVMProvisioner(vgName, device, c.BlockInfo, c.semaphore) +} + func (c *Controller) updateDeviceStatus(device *diskv1.BlockDevice, devPath string) error { var newStatus diskv1.DeviceStatus var needAutoProvision bool diff --git a/pkg/provisioner/lvm.go b/pkg/provisioner/lvm.go new file mode 100644 index 00000000..77c02b14 --- /dev/null +++ b/pkg/provisioner/lvm.go @@ -0,0 +1,309 @@ +package provisioner + +import ( + "fmt" + "strings" + + "github.com/sirupsen/logrus" + + diskv1 "github.com/harvester/node-disk-manager/pkg/apis/harvesterhci.io/v1beta1" + "github.com/harvester/node-disk-manager/pkg/block" + "github.com/harvester/node-disk-manager/pkg/utils" +) + +type LVMProvisioner struct { + name string + vgName string + blockInfo block.Info + device *diskv1.BlockDevice + + semaphoreObj *Semaphore +} + +func NewLVMProvisioner(vgName string, device *diskv1.BlockDevice, blockInfo block.Info, semaphoreObj *Semaphore) *LVMProvisioner { + return &LVMProvisioner{ + name: TypeLVM, + vgName: vgName, + blockInfo: blockInfo, + device: device, + semaphoreObj: semaphoreObj, + } +} + +func (l *LVMProvisioner) GetProvisionerName() string { + return l.name +} + +func (l *LVMProvisioner) Format(_ string) (bool, bool, error) { + // LVM provisioner does not need format + return true, false, nil +} + +func (l *LVMProvisioner) UnFormat() (bool, error) { + // LVM provisioner does not need unformat + return false, nil +} + +func (l *LVMProvisioner) Provision() (bool, error) { + + setProvisioned := func() { + provisionerStatus := &diskv1.ProvisionerStatus{ + Type: l.name, + VgName: l.vgName, + } + l.device.Status.Provisioner = provisionerStatus + logrus.Debugf("Set blockdevice CRD (%v) to provisioned", l.device) + msg := fmt.Sprintf("Added disk %s to volume group %s ", l.device.Name, l.vgName) + setCondDiskAddedToNodeTrue(l.device, msg, diskv1.ProvisionPhaseProvisioned) + } + + if l.vgName == "" { + return false, fmt.Errorf("LVM VG name cannot be empty") + } + logrus.Infof("%s provisioning block device %s to vg: %s", l.name, l.device.Name, l.vgName) + + pvsResult, err := getPVScanResult() + if err != nil { + return true, fmt.Errorf("failed to get pvscan result. %v", err) + } + logrus.Debugf("pvscan result: %v", pvsResult) + pvFound := false + vgFound := false + devPath := l.device.Status.DeviceStatus.DevPath + for pv, vg := range pvsResult { + if pv == devPath { + pvFound = true + if vg == l.vgName { + logrus.Debugf("Block device %s is already in VG %s", l.device.Name, l.vgName) + setProvisioned() + return false, nil + } + } + if vg == l.vgName { + vgFound = true + } + } + + if !pvFound { + if err := doPVCreate(devPath); err != nil { + return true, err + } + } + if !vgFound { + if err := doVGCreate(devPath, l.vgName); err != nil { + return true, err + } + } else { + if err := doVGExtend(devPath, l.vgName); err != nil { + return true, err + } + + } + + setProvisioned() + return false, nil +} + +func (l *LVMProvisioner) UnProvision() (bool, error) { + logrus.Infof("%s unprovisioning block device %s from vg: %s", l.name, l.device.Name, l.vgName) + + setUnprovisioned := func() { + l.device.Status.Provisioner = nil + l.device.Spec.Provisioner = nil + logrus.Debugf("Set blockdevice CRD (%v) to unprovisioned", l.device) + msg := fmt.Sprintf("Removed disk %s from volume group %s ", l.device.Name, l.vgName) + setCondDiskAddedToNodeFalse(l.device, msg, diskv1.ProvisionPhaseUnprovisioned) + } + + pvsResult, err := getPVScanResult() + if err != nil { + return true, fmt.Errorf("failed to get pvscan result. %v", err) + } + logrus.Debugf("pvscan result: %v", pvsResult) + devPath := l.device.Status.DeviceStatus.DevPath + pvFound := false + isInVG := false + pvCountInVG := 0 + for pv, vg := range pvsResult { + if pv == devPath { + pvFound = true + if vg == l.vgName { + isInVG = true + pvCountInVG++ + } + } else { + if vg == l.vgName { + pvCountInVG++ + } + } + } + + if !pvFound { + logrus.Debugf("Block device %s is not in pvs.", l.device.Name) + setUnprovisioned() + return false, nil + } + if isInVG { + if pvCountInVG > 1 { + if err := doVGReduce(devPath, l.vgName); err != nil { + return true, err + } + } else { + if err := doVGRemove(l.vgName); err != nil { + return true, err + } + } + } + if err := doPVRemove(devPath); err != nil { + return true, err + } + + setUnprovisioned() + return false, nil +} + +func (l *LVMProvisioner) Update() (bool, error) { + // Make sure the volume group are all active + logrus.Infof("%s update block device %s from vg: %s", l.name, l.device.Name, l.vgName) + if err := doVGActivate(); err != nil { + return true, err + } + + return false, nil +} + +func getPVScanResult() (map[string]string, error) { + ns := utils.GetHostNamespacePath(utils.HostProcPath) + executor, err := utils.NewExecutorWithNS(ns) + if err != nil { + return nil, fmt.Errorf("generate executor failed. %v", err) + } + + args := []string{"--noheadings", "-o", "pv_name,vg_name"} + output, err := executor.Execute("pvs", args) + if err != nil { + return nil, fmt.Errorf("pvs failed. %v", err) + } + lines := strings.Split(output, "\n") + pvScanResult := make(map[string]string) + for _, line := range lines { + if line == "" { + continue + } + fields := strings.Fields(line) + // Format should be like: /dev/sda vg01 + pv := fields[0] + vg := "" + if len(fields) >= 2 { + vg = fields[1] + } + pvScanResult[pv] = vg + } + return pvScanResult, nil +} + +func doPVCreate(devPath string) error { + ns := utils.GetHostNamespacePath(utils.HostProcPath) + executor, err := utils.NewExecutorWithNS(ns) + if err != nil { + return fmt.Errorf("generate executor failed. %v", err) + } + + args := []string{devPath} + _, err = executor.Execute("pvcreate", args) + if err != nil { + return fmt.Errorf("pvcreate failed. %v", err) + } + return nil +} + +func doVGCreate(devPath, vgName string) error { + ns := utils.GetHostNamespacePath(utils.HostProcPath) + executor, err := utils.NewExecutorWithNS(ns) + if err != nil { + return fmt.Errorf("generate executor failed. %v", err) + } + + args := []string{vgName, devPath} + // 4M should be enough PE size for most cases, so we do not config PE size here + _, err = executor.Execute("vgcreate", args) + if err != nil { + return fmt.Errorf("vgcreate failed. %v", err) + } + return nil +} + +func doVGExtend(devPath, vgName string) error { + ns := utils.GetHostNamespacePath(utils.HostProcPath) + executor, err := utils.NewExecutorWithNS(ns) + if err != nil { + return fmt.Errorf("generate executor failed. %v", err) + } + + args := []string{vgName, devPath} + _, err = executor.Execute("vgextend", args) + if err != nil { + return fmt.Errorf("vgextend failed. %v", err) + } + return nil +} + +func doVGReduce(devPath, vgName string) error { + ns := utils.GetHostNamespacePath(utils.HostProcPath) + executor, err := utils.NewExecutorWithNS(ns) + if err != nil { + return fmt.Errorf("generate executor failed. %v", err) + } + + args := []string{vgName, devPath} + _, err = executor.Execute("vgreduce", args) + if err != nil { + return fmt.Errorf("vgreduce failed. %v", err) + } + return nil +} + +func doVGRemove(vgName string) error { + ns := utils.GetHostNamespacePath(utils.HostProcPath) + executor, err := utils.NewExecutorWithNS(ns) + if err != nil { + return fmt.Errorf("generate executor failed. %v", err) + } + + args := []string{vgName} + _, err = executor.Execute("vgremove", args) + if err != nil { + return fmt.Errorf("vgremove failed. %v", err) + } + return nil +} + +func doPVRemove(devPath string) error { + ns := utils.GetHostNamespacePath(utils.HostProcPath) + executor, err := utils.NewExecutorWithNS(ns) + if err != nil { + return fmt.Errorf("generate executor failed. %v", err) + } + + args := []string{devPath} + _, err = executor.Execute("pvremove", args) + if err != nil { + return fmt.Errorf("pvremove failed. %v", err) + } + return nil +} + +func doVGActivate() error { + ns := utils.GetHostNamespacePath(utils.HostProcPath) + executor, err := utils.NewExecutorWithNS(ns) + if err != nil { + return fmt.Errorf("generate executor failed. %v", err) + } + + args := []string{"--activate", "y"} + _, err = executor.Execute("vgchange", args) + if err != nil { + return fmt.Errorf("vgchange failed. %v", err) + } + return nil +}