Online and offline resize for PVCs.

Add a field in volumeStatus tracking the size with file system overhead taken into account. This field is only populated if the feature gate is enabled. If this differs from the guest capacity reported by libvirt (when the VM is running) or qemu-img (when the VM hasn't started yet), expand the disk to this size. Requeue virt-controller for matching VMIs if a PVC is updated, so we can detect resizes. Signed-off-by: Maya Rashish <[email protected]>
rayees0012 · Oct 29, 2021 · 5445e9d · 5445e9d
1 parent 692e4a9
commit 5445e9d
Show file tree

Hide file tree

Showing 31 changed files with 537 additions and 120 deletions.
diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
@@ -12923,6 +12923,10 @@
        "$ref": "#/definitions/k8s.io.apimachinery.pkg.api.resource.Quantity"
       }
      },
+     "filesystemOverhead": {
+      "description": "Percentage of filesystem's size to be reserved when resizing the PVC",
+      "type": "string"
+     },
      "preallocated": {
       "description": "Preallocated indicates if the PVC's storage is preallocated or not",
       "type": "boolean"

diff --git a/docs/disk-expansion.md b/docs/disk-expansion.md
@@ -0,0 +1,19 @@
+# Disk expansion
+
+For some storage methods, Kubernetes may support expanding storage in-use (allowVolumeExpansion feature).
+KubeVirt can respond to it by making the additional storage available for the virtual machines.
+This feature is currently off by default, and requires enabling a feature gate.
+To enable it, add the ExpandDisks feature gate in the kubevirt object:
+
+kubectl edit kubevirt -n kubevirt kubevirt
+```yaml
+spec:
+  configuration:
+    developerConfiguration:
+      featureGates:
+      - ExpandDisks
+```
+
+Enabling this feature does two things:
+- Notify the virtual machine about size changes
+- If the disk is a Filesystem PVC, the matching file is expanded to the remaining size (while reserving some space for file system overhead).
diff --git a/pkg/container-disk/validation.go b/pkg/container-disk/validation.go
@@ -11,8 +11,8 @@ const (
 type DiskInfo struct {
 	Format      string `json:"format"`
 	BackingFile string `json:"backing-filename"`
-	ActualSize  int    `json:"actual-size"`
-	VirtualSize int    `json:"virtual-size"`
+	ActualSize  int64  `json:"actual-size"`
+	VirtualSize int64  `json:"virtual-size"`
 }
 
 func VerifyQCOW2(diskInfo *DiskInfo) error {

diff --git a/pkg/controller/virtinformers.go b/pkg/controller/virtinformers.go
@@ -152,6 +152,12 @@ type KubeInformerFactory interface {
 	// Fake CDI DataSource informer used when feature gate is disabled
 	DummyDataSource() cache.SharedIndexInformer
 
+	// Watches for CDIConfig objects
+	CDIConfig() cache.SharedIndexInformer
+
+	// Fake CDIConfig informer used when feature gate is disabled
+	DummyCDIConfig() cache.SharedIndexInformer
+
 	// CRD
 	CRD() cache.SharedIndexInformer
 
@@ -547,6 +553,22 @@ func (f *kubeInformerFactory) DummyDataSource() cache.SharedIndexInformer {
 	})
 }
 
+func (f *kubeInformerFactory) CDIConfig() cache.SharedIndexInformer {
+	return f.getInformer("cdiConfigInformer", func() cache.SharedIndexInformer {
+		restClient := f.clientSet.CdiClient().CdiV1beta1().RESTClient()
+		lw := cache.NewListWatchFromClient(restClient, "cdiconfigs", k8sv1.NamespaceAll, fields.Everything())
+
+		return cache.NewSharedIndexInformer(lw, &cdiv1.CDIConfig{}, f.defaultResync, cache.Indexers{})
+	})
+}
+
+func (f *kubeInformerFactory) DummyCDIConfig() cache.SharedIndexInformer {
+	return f.getInformer("fakeCdiConfigInformer", func() cache.SharedIndexInformer {
+		informer, _ := testutils.NewFakeInformerFor(&cdiv1.CDIConfig{})
+		return informer
+	})
+}
+
 func (f *kubeInformerFactory) ApiAuthConfigMap() cache.SharedIndexInformer {
 	return f.getInformer("extensionsConfigMapInformer", func() cache.SharedIndexInformer {
 		restClient := f.clientSet.CoreV1().RESTClient()

diff --git a/pkg/handler-launcher-com/cmd/v1/cmd.pb.go b/pkg/handler-launcher-com/cmd/v1/cmd.pb.go
diff --git a/pkg/handler-launcher-com/cmd/v1/cmd.proto b/pkg/handler-launcher-com/cmd/v1/cmd.proto
@@ -85,6 +85,7 @@ message VirtualMachineOptions {
   repeated string PreallocatedVolumes = 3;
   Topology topology = 4;
   map<string, DiskInfo> DisksInfo = 5;
+  bool ExpandDisksEnabled = 6;
 }
 
 message VMIRequest {

diff --git a/pkg/virt-config/feature-gates.go b/pkg/virt-config/feature-gates.go
@@ -24,6 +24,7 @@ package virtconfig
 */
 
 const (
+	ExpandDisksGate   = "ExpandDisks"
 	CPUManager        = "CPUManager"
 	NUMAFeatureGate   = "NUMA"
 	IgnitionGate      = "ExperimentalIgnitionSupport"
@@ -54,6 +55,10 @@ func (c *ClusterConfig) isFeatureGateEnabled(featureGate string) bool {
 	return false
 }
 
+func (config *ClusterConfig) ExpandDisksEnabled() bool {
+	return config.isFeatureGateEnabled(ExpandDisksGate)
+}
+
 func (config *ClusterConfig) CPUManagerEnabled() bool {
 	return config.isFeatureGateEnabled(CPUManager)
 }

diff --git a/pkg/virt-controller/watch/application.go b/pkg/virt-controller/watch/application.go
@@ -156,6 +156,7 @@ type VirtControllerApp struct {
 	controllerRevisionInformer cache.SharedIndexInformer
 
 	dataVolumeInformer cache.SharedIndexInformer
+	cdiConfigInformer  cache.SharedIndexInformer
 
 	migrationController *MigrationController
 	migrationInformer   cache.SharedIndexInformer
@@ -332,12 +333,14 @@ func Execute() {
 
 	if app.hasCDI {
 		app.dataVolumeInformer = app.informerFactory.DataVolume()
+		app.cdiConfigInformer = app.informerFactory.CDIConfig()
 		log.Log.Infof("CDI detected, DataVolume integration enabled")
 	} else {
 		// Add a dummy DataVolume informer in the event datavolume support
 		// is disabled. This lets the controller continue to work without
 		// requiring a separate branching code path.
 		app.dataVolumeInformer = app.informerFactory.DummyDataVolume()
+		app.cdiConfigInformer = app.informerFactory.DummyCDIConfig()
 		log.Log.Infof("CDI not detected, DataVolume integration disabled")
 	}
 
@@ -518,6 +521,8 @@ func (vca *VirtControllerApp) initCommon() {
 		vca.vmiRecorder,
 		vca.clientSet,
 		vca.dataVolumeInformer,
+		vca.cdiConfigInformer,
+		vca.clusterConfig,
 		topologyHinter,
 	)
 

diff --git a/pkg/virt-controller/watch/application_test.go b/pkg/virt-controller/watch/application_test.go
@@ -114,6 +114,8 @@ var _ = Describe("Application", func() {
 			recorder,
 			virtClient,
 			dataVolumeInformer,
+			pvcInformer,
+			config,
 			topology.NewTopologyHinter(&cache.FakeCustomStore{}, &cache.FakeCustomStore{}, "amd64", nil),
 		)
 		app.rsController = NewVMIReplicaSet(vmiInformer, rsInformer, recorder, virtClient, uint(10))

diff --git a/pkg/virt-controller/watch/vmi.go b/pkg/virt-controller/watch/vmi.go
@@ -45,6 +45,7 @@ import (
 	cdiv1 "kubevirt.io/containerized-data-importer/pkg/apis/core/v1beta1"
 	"kubevirt.io/kubevirt/pkg/controller"
 	kubevirttypes "kubevirt.io/kubevirt/pkg/util/types"
+	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
 	"kubevirt.io/kubevirt/pkg/virt-controller/services"
 )
 
@@ -132,6 +133,8 @@ func NewVMIController(templateService services.TemplateService,
 	recorder record.EventRecorder,
 	clientset kubecli.KubevirtClient,
 	dataVolumeInformer cache.SharedIndexInformer,
+	cdiConfigInformer cache.SharedIndexInformer,
+	clusterConfig *virtconfig.ClusterConfig,
 	topologyHinter topology.Hinter,
 ) *VMIController {
 
@@ -147,6 +150,8 @@ func NewVMIController(templateService services.TemplateService,
 		podExpectations:    controller.NewUIDTrackingControllerExpectations(controller.NewControllerExpectations()),
 		vmiExpectations:    controller.NewUIDTrackingControllerExpectations(controller.NewControllerExpectations()),
 		dataVolumeInformer: dataVolumeInformer,
+		cdiConfigInformer:  cdiConfigInformer,
+		clusterConfig:      clusterConfig,
 		topologyHinter:     topologyHinter,
 	}
 
@@ -168,6 +173,10 @@ func NewVMIController(templateService services.TemplateService,
 		UpdateFunc: c.updateDataVolume,
 	})
 
+	c.pvcInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		UpdateFunc: c.updatePVC,
+	})
+
 	return c
 }
 
@@ -202,6 +211,8 @@ type VMIController struct {
 	podExpectations    *controller.UIDTrackingControllerExpectations
 	vmiExpectations    *controller.UIDTrackingControllerExpectations
 	dataVolumeInformer cache.SharedIndexInformer
+	cdiConfigInformer  cache.SharedIndexInformer
+	clusterConfig      *virtconfig.ClusterConfig
 }
 
 func (c *VMIController) Run(threadiness int, stopCh <-chan struct{}) {
@@ -1018,6 +1029,34 @@ func dataVolumeByNameFunc(dataVolumeInformer cache.SharedIndexInformer, dataVolu
 	}
 }
 
+func (c *VMIController) updatePVC(old, cur interface{}) {
+	curPVC := cur.(*k8sv1.PersistentVolumeClaim)
+	oldPVC := old.(*k8sv1.PersistentVolumeClaim)
+	if curPVC.ResourceVersion == oldPVC.ResourceVersion {
+		// Periodic resync will send update events for all known PVCs.
+		// Two different versions of the same PVC will always
+		// have different RVs.
+		return
+	}
+	if curPVC.DeletionTimestamp != nil {
+		return
+	}
+	if reflect.DeepEqual(curPVC.Status.Capacity, oldPVC.Status.Capacity) {
+		// We only do something when the capacity changes
+		return
+	}
+
+	vmis, err := c.listVMIsMatchingPVC(curPVC.Namespace, curPVC.Name)
+	if err != nil {
+		log.Log.V(4).Object(curPVC).Errorf("Error encountered during pvc update: %v", err)
+		return
+	}
+	for _, vmi := range vmis {
+		log.Log.V(4).Object(curPVC).Infof("PVC updated for vmi %s", vmi.Name)
+		c.enqueueVirtualMachine(vmi)
+	}
+}
+
 func (c *VMIController) addDataVolume(obj interface{}) {
 	dataVolume := obj.(*cdiv1.DataVolume)
 	if dataVolume.DeletionTimestamp != nil {
@@ -1283,6 +1322,24 @@ func (c *VMIController) resolveControllerRef(namespace string, controllerRef *v1
 	return vmi.(*virtv1.VirtualMachineInstance)
 }
 
+func (c *VMIController) listVMIsMatchingPVC(namespace string, pvcName string) ([]*virtv1.VirtualMachineInstance, error) {
+	objs, err := c.vmiInformer.GetIndexer().ByIndex(cache.NamespaceIndex, namespace)
+	if err != nil {
+		return nil, err
+	}
+	vmis := []*virtv1.VirtualMachineInstance{}
+	for _, obj := range objs {
+		vmi := obj.(*virtv1.VirtualMachineInstance)
+		for _, volume := range vmi.Spec.Volumes {
+			if volume.VolumeSource.DataVolume != nil && volume.VolumeSource.DataVolume.Name == pvcName ||
+				volume.VolumeSource.PersistentVolumeClaim != nil && volume.VolumeSource.PersistentVolumeClaim.ClaimName == pvcName {
+				vmis = append(vmis, vmi)
+			}
+		}
+	}
+	return vmis, nil
+}
+
 // takes a namespace and returns all Pods from the pod cache which run in this namespace
 func (c *VMIController) listVMIsMatchingDataVolume(namespace string, dataVolumeName string) ([]*virtv1.VirtualMachineInstance, error) {
 	objs, err := c.vmiInformer.GetIndexer().ByIndex(cache.NamespaceIndex, namespace)
@@ -1929,6 +1986,10 @@ func (c *VMIController) updateVolumeStatus(vmi *virtv1.VirtualMachineInstance, v
 					Capacity:     pvc.Status.Capacity,
 					Preallocated: kubevirttypes.IsPreallocated(pvc.ObjectMeta.Annotations),
 				}
+				filesystemOverhead, err := c.getFilesystemOverhead(pvc)
+				if err != nil {
+					status.PersistentVolumeClaimInfo.FilesystemOverhead = &filesystemOverhead
+				}
 			}
 		}
 
@@ -1959,6 +2020,26 @@ func (c *VMIController) updateVolumeStatus(vmi *virtv1.VirtualMachineInstance, v
 	return nil
 }
 
+func (c *VMIController) getFilesystemOverhead(pvc *k8sv1.PersistentVolumeClaim) (cdiv1.Percent, error) {
+	cdiConfigInterface, cdiConfigExists, _ := c.cdiConfigInformer.GetStore().GetByKey("config")
+	if !cdiConfigExists {
+		return "0", fmt.Errorf("No CDIConfig named config")
+	}
+	cdiConfig, ok := cdiConfigInterface.(*cdiv1.CDIConfig)
+	if !ok {
+		return "0", fmt.Errorf("Failed to convert CDIConfig object %v to type CDIConfig", cdiConfigInterface)
+	}
+	scName := pvc.Spec.StorageClassName
+	if scName == nil {
+		return cdiConfig.Status.FilesystemOverhead.Global, nil
+	}
+	fsOverhead, ok := cdiConfig.Status.FilesystemOverhead.StorageClass[*scName]
+	if !ok {
+		return cdiConfig.Status.FilesystemOverhead.Global, nil
+	}
+	return fsOverhead, nil
+}
+
 func (c *VMIController) canMoveToAttachedPhase(currentPhase virtv1.VolumePhase) bool {
 	return currentPhase == "" || currentPhase == virtv1.VolumeBound || currentPhase == virtv1.VolumePending ||
 		currentPhase == virtv1.HotplugVolumeAttachedToNode

diff --git a/pkg/virt-controller/watch/vmi_test.go b/pkg/virt-controller/watch/vmi_test.go
@@ -79,6 +79,7 @@ var _ = Describe("VirtualMachineInstance watcher", func() {
 
 	var dataVolumeSource *framework.FakeControllerSource
 	var dataVolumeInformer cache.SharedIndexInformer
+	var cdiConfigInformer cache.SharedIndexInformer
 	var dataVolumeFeeder *testutils.DataVolumeFeeder
 	var qemuGid int64 = 107
 	controllerOf := true
@@ -210,6 +211,7 @@ var _ = Describe("VirtualMachineInstance watcher", func() {
 
 		config, _, _, _ := testutils.NewFakeClusterConfig(&k8sv1.ConfigMap{})
 		pvcInformer, _ = testutils.NewFakeInformerFor(&k8sv1.PersistentVolumeClaim{})
+		cdiConfigInformer, _ = testutils.NewFakeInformerFor(&cdiv1.CDIConfig{})
 		controller = NewVMIController(
 			services.NewTemplateService("a", 240, "b", "c", "d", "e", "f", "g", pvcInformer.GetStore(), virtClient, config, qemuGid),
 			vmiInformer,
@@ -219,6 +221,8 @@ var _ = Describe("VirtualMachineInstance watcher", func() {
 			recorder,
 			virtClient,
 			dataVolumeInformer,
+			cdiConfigInformer,
+			config,
 			topology.NewTopologyHinter(&cache.FakeCustomStore{}, &cache.FakeCustomStore{}, "amd64", nil),
 		)
 		// Wrap our workqueue to have a way to detect when we are done processing updates
@@ -2260,6 +2264,8 @@ var _ = Describe("VirtualMachineInstance watcher", func() {
 		makeVolumeStatusesForUpdateWithMessage := func(podName, podUID string, phase virtv1.VolumePhase, message, reason string, indexes ...int) []virtv1.VolumeStatus {
 			res := make([]virtv1.VolumeStatus, 0)
 			for _, index := range indexes {
+				var fsOverhead cdiv1.Percent
+				fsOverhead = "0"
 				res = append(res, virtv1.VolumeStatus{
 					Name: fmt.Sprintf("volume%d", index),
 					HotplugVolume: &virtv1.HotplugVolumeStatus{
@@ -2273,6 +2279,7 @@ var _ = Describe("VirtualMachineInstance watcher", func() {
 						AccessModes: []k8sv1.PersistentVolumeAccessMode{
 							k8sv1.ReadOnlyMany,
 						},
+						FilesystemOverhead: &fsOverhead,
 					},
 				})
 			}
@@ -2450,7 +2457,7 @@ var _ = Describe("VirtualMachineInstance watcher", func() {
 			addVirtualMachine(vmi)
 			podInformer.GetIndexer().Add(virtlauncherPod)
 			//Modify by adding a new hotplugged disk
-			patch := `[{ "op": "test", "path": "/status/volumeStatus", "value": [{"name":"existing","target":""}] }, { "op": "replace", "path": "/status/volumeStatus", "value": [{"name":"existing","target":"","persistentVolumeClaimInfo":{}},{"name":"hotplug","target":"","phase":"Bound","reason":"PVCNotReady","message":"PVC is in phase Bound","persistentVolumeClaimInfo":{},"hotplugVolume":{}}] }]`
+			patch := `[{ "op": "test", "path": "/status/volumeStatus", "value": [{"name":"existing","target":""}] }, { "op": "replace", "path": "/status/volumeStatus", "value": [{"name":"existing","target":"","persistentVolumeClaimInfo":{"filesystemOverhead":"0"}},{"name":"hotplug","target":"","phase":"Bound","reason":"PVCNotReady","message":"PVC is in phase Bound","persistentVolumeClaimInfo":{"filesystemOverhead":"0"},"hotplugVolume":{}}] }]`
 			vmiInterface.EXPECT().Patch(vmi.Name, types.JSONPatchType, []byte(patch)).Return(vmi, nil)
 			controller.Execute()
 			testutils.ExpectEvent(recorder, SuccessfulCreatePodReason)
@@ -2540,7 +2547,7 @@ var _ = Describe("VirtualMachineInstance watcher", func() {
 			addVirtualMachine(vmi)
 			podInformer.GetIndexer().Add(virtlauncherPod)
 			//Modify by adding a new hotplugged disk
-			patch := `[{ "op": "test", "path": "/status/volumeStatus", "value": [{"name":"existing","target":""},{"name":"hotplug","target":"","hotplugVolume":{"attachPodName":"hp-volume-hotplug","attachPodUID":"abcd"}}] }, { "op": "replace", "path": "/status/volumeStatus", "value": [{"name":"existing","target":"","persistentVolumeClaimInfo":{}},{"name":"hotplug","target":"","phase":"Detaching","hotplugVolume":{"attachPodName":"hp-volume-hotplug","attachPodUID":"abcd"}}] }]`
+			patch := `[{ "op": "test", "path": "/status/volumeStatus", "value": [{"name":"existing","target":""},{"name":"hotplug","target":"","hotplugVolume":{"attachPodName":"hp-volume-hotplug","attachPodUID":"abcd"}}] }, { "op": "replace", "path": "/status/volumeStatus", "value": [{"name":"existing","target":"","persistentVolumeClaimInfo":{"filesystemOverhead":"0"}},{"name":"hotplug","target":"","phase":"Detaching","hotplugVolume":{"attachPodName":"hp-volume-hotplug","attachPodUID":"abcd"}}] }]`
 			vmiInterface.EXPECT().Patch(vmi.Name, types.JSONPatchType, []byte(patch)).Return(vmi, nil)
 			controller.Execute()
 			testutils.ExpectEvent(recorder, SuccessfulDeletePodReason)

diff --git a/pkg/virt-handler/options.go b/pkg/virt-handler/options.go
@@ -13,12 +13,14 @@ func virtualMachineOptions(
 	preallocatedVolumes []string,
 	capabilities *api.Capabilities,
 	disksInfo map[string]*containerdisk.DiskInfo,
+	expandDisksEnabled bool,
 ) *cmdv1.VirtualMachineOptions {
 	options := &cmdv1.VirtualMachineOptions{
 		MemBalloonStatsPeriod: period,
 		PreallocatedVolumes:   preallocatedVolumes,
 		Topology:              topologyToTopology(capabilities),
 		DisksInfo:             disksInfoToDisksInfo(disksInfo),
+		ExpandDisksEnabled:    expandDisksEnabled,
 	}
 	if smbios != nil {
 		options.VirtualMachineSMBios = &cmdv1.SMBios{

diff --git a/pkg/virt-handler/vm.go b/pkg/virt-handler/vm.go
@@ -2586,7 +2586,7 @@ func (d *VirtualMachineController) vmUpdateHelperMigrationTarget(origVMI *v1.Vir
 		}
 	}
 
-	options := virtualMachineOptions(nil, 0, nil, d.capabilities, disksInfo)
+	options := virtualMachineOptions(nil, 0, nil, d.capabilities, disksInfo, d.clusterConfig.ExpandDisksEnabled())
 	if err := client.SyncMigrationTarget(vmi, options); err != nil {
 		return fmt.Errorf("syncing migration target failed: %v", err)
 	}
@@ -2694,7 +2694,7 @@ func (d *VirtualMachineController) vmUpdateHelperDefault(origVMI *v1.VirtualMach
 	smbios := d.clusterConfig.GetSMBIOS()
 	period := d.clusterConfig.GetMemBalloonStatsPeriod()
 
-	options := virtualMachineOptions(smbios, period, preallocatedVolumes, d.capabilities, disksInfo)
+	options := virtualMachineOptions(smbios, period, preallocatedVolumes, d.capabilities, disksInfo, d.clusterConfig.ExpandDisksEnabled())
 
 	err = client.SyncVirtualMachine(vmi, options)
 	if err != nil {

diff --git a/pkg/virt-launcher/virtwrap/api/BUILD.bazel b/pkg/virt-launcher/virtwrap/api/BUILD.bazel
@@ -15,10 +15,12 @@ go_library(
         "//staging/src/kubevirt.io/client-go/precond:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/kubevirt.io/containerized-data-importer/pkg/apis/core/v1beta1:go_default_library",
     ],
 )