Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(vmbda): add hotplug virtual image #536

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions api/Taskfile.dist.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ tasks:
- |
cd ../ && docker run --rm \
-v ./:/tmp/virt ghcr.io/deckhouse/virtualization/prettier:3.2.5 \
sh -c "cd /tmp/virt ; prettier -w \"**/*.yaml\" \"**/*.yml\""
sh -c "cd /tmp/virt ; prettier -w \"crds/*.yaml\""

_ci:verify-gen:
desc: "Check generated files are up-to-date."
Expand Down Expand Up @@ -87,4 +87,4 @@ tasks:
- go install -mod=readonly sigs.k8s.io/controller-tools/cmd/controller-gen@v{{ .CONTROLLER_GEN_VERSION }}
status:
- |
ls $GOPATH/bin/controller-gen
$GOPATH/bin/controller-gen --version | grep -q "v{{ .CONTROLLER_GEN_VERSION }}"
Original file line number Diff line number Diff line change
Expand Up @@ -72,18 +72,20 @@ type VirtualMachineBlockDeviceAttachmentStatus struct {
type VMBDAObjectRef struct {
// The type of the block device. Options are:
// * `VirtualDisk` — use `VirtualDisk` as the disk. This type is always mounted in RW mode.
// * `VirtualImage` — use `VirtualImage` as the disk. Only for type PersistentVolumeClaim.
Kind VMBDAObjectRefKind `json:"kind,omitempty"`
// The name of block device to attach.
Name string `json:"name,omitempty"`
}

// VMBDAObjectRefKind defines the type of the block device.
//
// +kubebuilder:validation:Enum={VirtualDisk}
// +kubebuilder:validation:Enum={VirtualDisk,VirtualImage}
type VMBDAObjectRefKind string

const (
VMBDAObjectRefKindVirtualDisk VMBDAObjectRefKind = "VirtualDisk"
VMBDAObjectRefKindVirtualDisk VMBDAObjectRefKind = "VirtualDisk"
VMBDAObjectRefKindVirtualImage VMBDAObjectRefKind = "VirtualImage"
)

// BlockDeviceAttachmentPhase defines current status of resource:
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions crds/virtualmachineblockdeviceattachments.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,10 @@ spec:
description: |-
The type of the block device. Options are:
* `VirtualDisk` — use `VirtualDisk` as the disk. This type is always mounted in RW mode.
* `VirtualImage` — use `VirtualImage` as the disk. Only for type PersistentVolumeClaim.
enum:
- VirtualDisk
- VirtualImage
type: string
name:
description: The name of block device to attach.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,14 @@ func NewAttachmentService(client Client, controllerNamespace string) *Attachment

var (
ErrVolumeStatusNotReady = errors.New("hotplug is not ready")
ErrDiskIsSpecAttached = errors.New("virtual disk is already attached to the virtual machine spec")
ErrBlockDeviceIsSpecAttached = errors.New("block device is already attached to the virtual machine spec")
ErrHotPlugRequestAlreadySent = errors.New("attachment request is already sent")
ErrVirtualMachineWaitsForRestartApproval = errors.New("virtual machine waits for restart approval")
)

func (s AttachmentService) IsHotPlugged(vd *virtv2.VirtualDisk, vm *virtv2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) (bool, error) {
if vd == nil {
return false, errors.New("cannot check if a nil VirtualDisk is hot plugged")
func (s AttachmentService) IsHotPlugged(ad *AttachmentDisk, vm *virtv2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) (bool, error) {
if ad == nil {
return false, errors.New("cannot check if a empty AttachmentDisk is hot plugged")
}

if vm == nil {
Expand All @@ -67,7 +67,7 @@ func (s AttachmentService) IsHotPlugged(vd *virtv2.VirtualDisk, vm *virtv2.Virtu
}

for _, vs := range kvvmi.Status.VolumeStatus {
if vs.HotplugVolume != nil && vs.Name == kvbuilder.GenerateVMDDiskName(vd.Name) {
if vs.HotplugVolume != nil && vs.Name == ad.GenerateName {
if vs.Phase == virtv1.VolumeReady {
return true, nil
}
Expand All @@ -79,9 +79,9 @@ func (s AttachmentService) IsHotPlugged(vd *virtv2.VirtualDisk, vm *virtv2.Virtu
return false, nil
}

func (s AttachmentService) CanHotPlug(vd *virtv2.VirtualDisk, vm *virtv2.VirtualMachine, kvvm *virtv1.VirtualMachine) (bool, error) {
if vd == nil {
return false, errors.New("cannot hot plug a nil VirtualDisk")
func (s AttachmentService) CanHotPlug(ad *AttachmentDisk, vm *virtv2.VirtualMachine, kvvm *virtv1.VirtualMachine) (bool, error) {
if ad == nil {
return false, errors.New("cannot hot plug a nil AttachmentDisk")
}

if vm == nil {
Expand All @@ -93,12 +93,12 @@ func (s AttachmentService) CanHotPlug(vd *virtv2.VirtualDisk, vm *virtv2.Virtual
}

for _, bdr := range vm.Spec.BlockDeviceRefs {
if bdr.Kind == virtv2.DiskDevice && bdr.Name == vd.Name {
return false, fmt.Errorf("%w: virtual machine has a virtual disk reference, but it is not a hot-plugged volume", ErrDiskIsSpecAttached)
if bdr.Kind == ad.Kind && bdr.Name == ad.Name {
return false, fmt.Errorf("%w: virtual machine has a block device reference, but it is not a hot-plugged volume", ErrBlockDeviceIsSpecAttached)
}
}

name := kvbuilder.GenerateVMDDiskName(vd.Name)
name := ad.GenerateName

if kvvm.Spec.Template != nil {
for _, vs := range kvvm.Spec.Template.Spec.Volumes {
Expand All @@ -108,7 +108,7 @@ func (s AttachmentService) CanHotPlug(vd *virtv2.VirtualDisk, vm *virtv2.Virtual
}

if !vs.PersistentVolumeClaim.Hotpluggable {
return false, fmt.Errorf("%w: virtual machine has a virtual disk reference, but it is not a hot-plugged volume", ErrDiskIsSpecAttached)
return false, fmt.Errorf("%w: virtual machine has a block device reference, but it is not a hot-plugged volume", ErrBlockDeviceIsSpecAttached)
}

return false, ErrHotPlugRequestAlreadySent
Expand All @@ -129,9 +129,9 @@ func (s AttachmentService) CanHotPlug(vd *virtv2.VirtualDisk, vm *virtv2.Virtual
return true, nil
}

func (s AttachmentService) HotPlugDisk(ctx context.Context, vd *virtv2.VirtualDisk, vm *virtv2.VirtualMachine, kvvm *virtv1.VirtualMachine) error {
if vd == nil {
return errors.New("cannot hot plug a nil VirtualDisk")
func (s AttachmentService) HotPlugDisk(ctx context.Context, ad *AttachmentDisk, vm *virtv2.VirtualMachine, kvvm *virtv1.VirtualMachine) error {
if ad == nil {
return errors.New("cannot hot plug a nil AttachmentDisk")
}

if vm == nil {
Expand All @@ -142,23 +142,21 @@ func (s AttachmentService) HotPlugDisk(ctx context.Context, vd *virtv2.VirtualDi
return errors.New("cannot hot plug a disk into a nil KVVM")
}

name := kvbuilder.GenerateVMDDiskName(vd.Name)

hotplugRequest := virtv1.AddVolumeOptions{
Name: name,
Name: ad.GenerateName,
Disk: &virtv1.Disk{
Name: name,
Name: ad.GenerateName,
DiskDevice: virtv1.DiskDevice{
Disk: &virtv1.DiskTarget{
Bus: "scsi",
},
},
Serial: vd.Name,
Serial: ad.Name,
},
VolumeSource: &virtv1.HotplugVolumeSource{
PersistentVolumeClaim: &virtv1.PersistentVolumeClaimVolumeSource{
PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{
ClaimName: vd.Status.Target.PersistentVolumeClaim,
ClaimName: ad.PVCName,
},
Hotpluggable: true,
},
Expand All @@ -178,27 +176,27 @@ func (s AttachmentService) HotPlugDisk(ctx context.Context, vd *virtv2.VirtualDi
return nil
}

func (s AttachmentService) CanUnplug(vd *virtv2.VirtualDisk, kvvm *virtv1.VirtualMachine) bool {
if vd == nil || kvvm == nil || kvvm.Spec.Template == nil {
func (s AttachmentService) CanUnplug(ad *AttachmentDisk, kvvm *virtv1.VirtualMachine) bool {
if ad == nil || kvvm == nil || kvvm.Spec.Template == nil {
return false
}

for _, volume := range kvvm.Spec.Template.Spec.Volumes {
if kvapi.VolumeExists(volume, kvbuilder.GenerateVMDDiskName(vd.Name)) {
if kvapi.VolumeExists(volume, ad.GenerateName) {
return true
}
}

return false
}

func (s AttachmentService) UnplugDisk(ctx context.Context, vd *virtv2.VirtualDisk, kvvm *virtv1.VirtualMachine) error {
if vd == nil || kvvm == nil {
func (s AttachmentService) UnplugDisk(ctx context.Context, ad *AttachmentDisk, kvvm *virtv1.VirtualMachine) error {
if ad == nil || kvvm == nil {
return nil
}

unplugRequest := virtv1.RemoveVolumeOptions{
Name: kvbuilder.GenerateVMDDiskName(vd.Name),
Name: ad.GenerateName,
}

kv, err := kubevirt.New(ctx, s.client, s.controllerNamespace)
Expand Down Expand Up @@ -272,8 +270,12 @@ func (s AttachmentService) GetVirtualDisk(ctx context.Context, name, namespace s
return helper.FetchObject(ctx, types.NamespacedName{Namespace: namespace, Name: name}, s.client, &virtv2.VirtualDisk{})
}

func (s AttachmentService) GetPersistentVolumeClaim(ctx context.Context, vd *virtv2.VirtualDisk) (*corev1.PersistentVolumeClaim, error) {
return helper.FetchObject(ctx, types.NamespacedName{Namespace: vd.Namespace, Name: vd.Status.Target.PersistentVolumeClaim}, s.client, &corev1.PersistentVolumeClaim{})
func (s AttachmentService) GetVirtualImage(ctx context.Context, name, namespace string) (*virtv2.VirtualImage, error) {
return helper.FetchObject(ctx, types.NamespacedName{Namespace: namespace, Name: name}, s.client, &virtv2.VirtualImage{})
}

func (s AttachmentService) GetPersistentVolumeClaim(ctx context.Context, ad *AttachmentDisk) (*corev1.PersistentVolumeClaim, error) {
return helper.FetchObject(ctx, types.NamespacedName{Namespace: ad.Namespace, Name: ad.PVCName}, s.client, &corev1.PersistentVolumeClaim{})
}

func (s AttachmentService) GetVirtualMachine(ctx context.Context, name, namespace string) (*virtv2.VirtualMachine, error) {
Expand All @@ -291,3 +293,31 @@ func (s AttachmentService) GetKVVMI(ctx context.Context, vm *virtv2.VirtualMachi
func isSameBlockDeviceRefs(a, b virtv2.VMBDAObjectRef) bool {
return a.Kind == b.Kind && a.Name == b.Name
}

type AttachmentDisk struct {
Kind virtv2.BlockDeviceKind
Name string
Namespace string
GenerateName string
PVCName string
}

func NewAttachmentDiskFromVirtualDisk(vd *virtv2.VirtualDisk) *AttachmentDisk {
return &AttachmentDisk{
Kind: virtv2.DiskDevice,
Name: vd.GetName(),
Namespace: vd.GetNamespace(),
GenerateName: kvbuilder.GenerateVMDDiskName(vd.GetName()),
PVCName: vd.Status.Target.PersistentVolumeClaim,
}
}

func NewAttachmentDiskFromVirtualImage(vi *virtv2.VirtualImage) *AttachmentDisk {
return &AttachmentDisk{
Kind: virtv2.ImageDevice,
Name: vi.GetName(),
Namespace: vi.GetNamespace(),
GenerateName: kvbuilder.GenerateVMIDiskName(vi.GetName()),
PVCName: vi.Status.Target.PersistentVolumeClaim,
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,8 @@ func (h BlockDeviceReadyHandler) Handle(ctx context.Context, vmbda *virtv2.Virtu
return reconcile.Result{}, nil
}

pvc, err := h.attachment.GetPersistentVolumeClaim(ctx, vd)
ad := service.NewAttachmentDiskFromVirtualDisk(vd)
pvc, err := h.attachment.GetPersistentVolumeClaim(ctx, ad)
if err != nil {
return reconcile.Result{}, err
}
Expand All @@ -131,6 +132,71 @@ func (h BlockDeviceReadyHandler) Handle(ctx context.Context, vmbda *virtv2.Virtu
return reconcile.Result{}, nil
}

cb.Status(metav1.ConditionTrue).Reason(vmbdacondition.BlockDeviceReady)
return reconcile.Result{}, nil
case virtv2.VMBDAObjectRefKindVirtualImage:
viKey := types.NamespacedName{
Name: vmbda.Spec.BlockDeviceRef.Name,
Namespace: vmbda.Namespace,
}

vi, err := h.attachment.GetVirtualImage(ctx, viKey.Name, viKey.Namespace)
if err != nil {
return reconcile.Result{}, err
}

if vi == nil {
cb.
Status(metav1.ConditionFalse).
Reason(vmbdacondition.BlockDeviceNotReady).
Message(fmt.Sprintf("VirtualImage %q not found.", viKey.String()))
return reconcile.Result{}, nil
}
if vi.Generation != vi.Status.ObservedGeneration {
cb.
Status(metav1.ConditionFalse).
Reason(vmbdacondition.BlockDeviceNotReady).
Message(fmt.Sprintf("Waiting for the VirtualImage %q to be observed in its latest state generation.", viKey.String()))
return reconcile.Result{}, nil
}

if vi.Status.Phase != virtv2.ImageReady {
cb.
Status(metav1.ConditionFalse).
Reason(vmbdacondition.BlockDeviceNotReady).
Message(fmt.Sprintf("VirtualImage %q is not ready to be attached to the virtual machine: waiting for the VirtualImage to be ready for attachment.", viKey.String()))
return reconcile.Result{}, nil
}
if vi.Status.Target.PersistentVolumeClaim == "" {
cb.
Status(metav1.ConditionFalse).
Reason(vmbdacondition.BlockDeviceNotReady).
Message("Waiting until VirtualImage has associated PersistentVolumeClaim name.")
return reconcile.Result{}, nil
}

ad := service.NewAttachmentDiskFromVirtualImage(vi)
pvc, err := h.attachment.GetPersistentVolumeClaim(ctx, ad)
if err != nil {
return reconcile.Result{}, err
}

if pvc == nil {
cb.
Status(metav1.ConditionFalse).
Reason(vmbdacondition.BlockDeviceNotReady).
Message(fmt.Sprintf("Underlying PersistentVolumeClaim %q not found.", vi.Status.Target.PersistentVolumeClaim))
return reconcile.Result{}, nil
}

if vi.Status.Phase == virtv2.ImageReady && pvc.Status.Phase != corev1.ClaimBound {
cb.
Status(metav1.ConditionFalse).
Reason(vmbdacondition.BlockDeviceNotReady).
Message(fmt.Sprintf("Underlying PersistentVolumeClaim %q not bound.", vi.Status.Target.PersistentVolumeClaim))
return reconcile.Result{}, nil
}

cb.Status(metav1.ConditionTrue).Reason(vmbdacondition.BlockDeviceReady)
return reconcile.Result{}, nil
default:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,14 @@ func NewDeletionHandler() *DeletionHandler {
return &DeletionHandler{}
}

func (h DeletionHandler) Handle(ctx context.Context, vd *virtv2.VirtualMachineBlockDeviceAttachment) (reconcile.Result, error) {
func (h DeletionHandler) Handle(ctx context.Context, vmbda *virtv2.VirtualMachineBlockDeviceAttachment) (reconcile.Result, error) {
log := logger.FromContext(ctx).With(logger.SlogHandler(deletionHandlerName))
if vd.DeletionTimestamp != nil {
if vmbda.DeletionTimestamp != nil {
log.Info("Deletion observed: remove cleanup finalizer from VirtualMachineBlockDeviceAttachment")
controllerutil.RemoveFinalizer(vd, virtv2.FinalizerVMBDACleanup)
controllerutil.RemoveFinalizer(vmbda, virtv2.FinalizerVMBDACleanup)
return reconcile.Result{}, nil
}

controllerutil.AddFinalizer(vd, virtv2.FinalizerVMBDACleanup)
controllerutil.AddFinalizer(vmbda, virtv2.FinalizerVMBDACleanup)
return reconcile.Result{}, nil
}
Loading
Loading