From b6072595634b57e8205276aa7ccd8fe94af66c47 Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Wed, 4 Dec 2024 14:30:54 +0800 Subject: [PATCH] add diagnostic for data mover exposer Signed-off-by: Lyndon-Li --- changelogs/unreleased/8482-Lyndon-Li | 1 + pkg/exposer/csi_snapshot.go | 2 +- pkg/exposer/csi_snapshot_test.go | 400 +++++++++++++++++++++++++++ pkg/exposer/generic_restore.go | 6 +- pkg/exposer/generic_restore_test.go | 262 ++++++++++++++++++ pkg/util/csi/volume_snapshot.go | 50 ++-- pkg/util/csi/volume_snapshot_test.go | 194 +++++++++++++ pkg/util/kube/pod.go | 2 +- pkg/util/kube/pod_test.go | 46 +++ pkg/util/kube/pvc_pv.go | 10 +- pkg/util/kube/pvc_pv_test.go | 59 ++++ 11 files changed, 996 insertions(+), 36 deletions(-) create mode 100644 changelogs/unreleased/8482-Lyndon-Li diff --git a/changelogs/unreleased/8482-Lyndon-Li b/changelogs/unreleased/8482-Lyndon-Li new file mode 100644 index 0000000000..c1cec7e241 --- /dev/null +++ b/changelogs/unreleased/8482-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #8125, add diagnostic info for data mover exposers when expose timeout \ No newline at end of file diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index 85924f92f0..71c9a2a62d 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -355,7 +355,7 @@ func (e *csiSnapshotExposer) DiagnoseExpose(ctx context.Context, ownerObject cor if vs != nil { diag += csi.DiagnoseVS(vs) - if vs.Status.BoundVolumeSnapshotContentName != nil && *vs.Status.BoundVolumeSnapshotContentName != "" { + if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil && *vs.Status.BoundVolumeSnapshotContentName != "" { if vsc, err := e.csiSnapshotClient.VolumeSnapshotContents().Get(ctx, *vs.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}); err != nil { diag += fmt.Sprintf("error getting backup vsc %s, err: %v\n", *vs.Status.BoundVolumeSnapshotContentName, err) } else { diff --git a/pkg/exposer/csi_snapshot_test.go b/pkg/exposer/csi_snapshot_test.go index 0cbe65ffb4..d7e4a768ed 100644 --- a/pkg/exposer/csi_snapshot_test.go +++ b/pkg/exposer/csi_snapshot_test.go @@ -959,3 +959,403 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { }) } } + +func Test_csiSnapshotExposer_DiagnoseExpose(t *testing.T) { + backup := &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: velerov1.SchemeGroupVersion.String(), + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-backup", + UID: "fake-uid", + }, + } + + backupPodWithoutNodeName := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-backup", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: backup.APIVersion, + Kind: backup.Kind, + Name: backup.Name, + UID: backup.UID, + }, + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodInitialized, + Status: corev1.ConditionTrue, + Message: "fake-pod-message", + }, + }, + }, + } + + backupPodWithNodeName := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-backup", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: backup.APIVersion, + Kind: backup.Kind, + Name: backup.Name, + UID: backup.UID, + }, + }, + }, + Spec: corev1.PodSpec{ + NodeName: "fake-node", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodInitialized, + Status: corev1.ConditionTrue, + Message: "fake-pod-message", + }, + }, + }, + } + + backupPVCWithoutVolumeName := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-backup", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: backup.APIVersion, + Kind: backup.Kind, + Name: backup.Name, + UID: backup.UID, + }, + }, + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimPending, + }, + } + + backupPVCWithVolumeName := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-backup", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: backup.APIVersion, + Kind: backup.Kind, + Name: backup.Name, + UID: backup.UID, + }, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "fake-pv", + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimPending, + }, + } + + backupPV := corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-pv", + }, + Status: corev1.PersistentVolumeStatus{ + Phase: corev1.VolumePending, + Message: "fake-pv-message", + }, + } + + readyToUse := false + vscMessage := "fake-vsc-message" + backupVSC := snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-vsc", + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + ReadyToUse: &readyToUse, + Error: &snapshotv1api.VolumeSnapshotError{ + Message: &vscMessage, + }, + }, + } + + backupVSWithoutStatus := snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-backup", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: backup.APIVersion, + Kind: backup.Kind, + Name: backup.Name, + UID: backup.UID, + }, + }, + }, + } + + backupVSWithoutVSC := snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-backup", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: backup.APIVersion, + Kind: backup.Kind, + Name: backup.Name, + UID: backup.UID, + }, + }, + }, + Status: &snapshotv1api.VolumeSnapshotStatus{}, + } + + vsMessage := "fake-vs-message" + backupVSWithVSC := snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-backup", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: backup.APIVersion, + Kind: backup.Kind, + Name: backup.Name, + UID: backup.UID, + }, + }, + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &backupVSC.Name, + Error: &snapshotv1api.VolumeSnapshotError{ + Message: &vsMessage, + }, + }, + } + + nodeAgentPod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "node-agent-pod-1", + Labels: map[string]string{"name": "node-agent"}, + }, + Spec: corev1.PodSpec{ + NodeName: "fake-node", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + } + + tests := []struct { + name string + ownerBackup *velerov1.Backup + kubeClientObj []runtime.Object + snapshotClientObj []runtime.Object + expected string + }{ + { + name: "no pod, pvc, vs", + ownerBackup: backup, + expected: `***************************begin diagnose CSI exposer[velero/fake-backup]*************************** +error getting backup pod fake-backup, err: pods "fake-backup" not found +error getting backup pvc fake-backup, err: persistentvolumeclaims "fake-backup" not found +error getting backup vs fake-backup, err: volumesnapshots.snapshot.storage.k8s.io "fake-backup" not found +***************************end diagnose CSI exposer[velero/fake-backup]*************************** +`, + }, + { + name: "pod without node name, pvc without volume name, vs without status", + ownerBackup: backup, + kubeClientObj: []runtime.Object{ + &backupPodWithoutNodeName, + &backupPVCWithoutVolumeName, + }, + snapshotClientObj: []runtime.Object{ + &backupVSWithoutStatus, + }, + expected: `***************************begin diagnose CSI exposer[velero/fake-backup]*************************** +Pod velero/fake-backup, phase Pending, node name +Pod condition Initialized, status True, reason , message fake-pod-message +PVC velero/fake-backup, phase Pending, binding to +VS velero/fake-backup, bind to , readyToUse false, errMessage +***************************end diagnose CSI exposer[velero/fake-backup]*************************** +`, + }, + { + name: "pod without node name, pvc without volume name, vs without VSC", + ownerBackup: backup, + kubeClientObj: []runtime.Object{ + &backupPodWithoutNodeName, + &backupPVCWithoutVolumeName, + }, + snapshotClientObj: []runtime.Object{ + &backupVSWithoutVSC, + }, + expected: `***************************begin diagnose CSI exposer[velero/fake-backup]*************************** +Pod velero/fake-backup, phase Pending, node name +Pod condition Initialized, status True, reason , message fake-pod-message +PVC velero/fake-backup, phase Pending, binding to +VS velero/fake-backup, bind to , readyToUse false, errMessage +***************************end diagnose CSI exposer[velero/fake-backup]*************************** +`, + }, + { + name: "pod with node name, no node agent", + ownerBackup: backup, + kubeClientObj: []runtime.Object{ + &backupPodWithNodeName, + &backupPVCWithoutVolumeName, + }, + snapshotClientObj: []runtime.Object{ + &backupVSWithoutVSC, + }, + expected: `***************************begin diagnose CSI exposer[velero/fake-backup]*************************** +Pod velero/fake-backup, phase Pending, node name fake-node +Pod condition Initialized, status True, reason , message fake-pod-message +node-agent is not running in node fake-node +PVC velero/fake-backup, phase Pending, binding to +VS velero/fake-backup, bind to , readyToUse false, errMessage +***************************end diagnose CSI exposer[velero/fake-backup]*************************** +`, + }, + { + name: "pod with node name, node agent is running", + ownerBackup: backup, + kubeClientObj: []runtime.Object{ + &backupPodWithNodeName, + &backupPVCWithoutVolumeName, + &nodeAgentPod, + }, + snapshotClientObj: []runtime.Object{ + &backupVSWithoutVSC, + }, + expected: `***************************begin diagnose CSI exposer[velero/fake-backup]*************************** +Pod velero/fake-backup, phase Pending, node name fake-node +Pod condition Initialized, status True, reason , message fake-pod-message +PVC velero/fake-backup, phase Pending, binding to +VS velero/fake-backup, bind to , readyToUse false, errMessage +***************************end diagnose CSI exposer[velero/fake-backup]*************************** +`, + }, + { + name: "pvc with volume name, no pv", + ownerBackup: backup, + kubeClientObj: []runtime.Object{ + &backupPodWithNodeName, + &backupPVCWithVolumeName, + &nodeAgentPod, + }, + snapshotClientObj: []runtime.Object{ + &backupVSWithoutVSC, + }, + expected: `***************************begin diagnose CSI exposer[velero/fake-backup]*************************** +Pod velero/fake-backup, phase Pending, node name fake-node +Pod condition Initialized, status True, reason , message fake-pod-message +PVC velero/fake-backup, phase Pending, binding to fake-pv +error getting backup pv fake-pv, err: persistentvolumes "fake-pv" not found +VS velero/fake-backup, bind to , readyToUse false, errMessage +***************************end diagnose CSI exposer[velero/fake-backup]*************************** +`, + }, + { + name: "pvc with volume name, pv exists", + ownerBackup: backup, + kubeClientObj: []runtime.Object{ + &backupPodWithNodeName, + &backupPVCWithVolumeName, + &backupPV, + &nodeAgentPod, + }, + snapshotClientObj: []runtime.Object{ + &backupVSWithoutVSC, + }, + expected: `***************************begin diagnose CSI exposer[velero/fake-backup]*************************** +Pod velero/fake-backup, phase Pending, node name fake-node +Pod condition Initialized, status True, reason , message fake-pod-message +PVC velero/fake-backup, phase Pending, binding to fake-pv +PV fake-pv, phase Pending, reason , message fake-pv-message +VS velero/fake-backup, bind to , readyToUse false, errMessage +***************************end diagnose CSI exposer[velero/fake-backup]*************************** +`, + }, + { + name: "vs with vsc, vsc doesn't exist", + ownerBackup: backup, + kubeClientObj: []runtime.Object{ + &backupPodWithNodeName, + &backupPVCWithVolumeName, + &backupPV, + &nodeAgentPod, + }, + snapshotClientObj: []runtime.Object{ + &backupVSWithVSC, + }, + expected: `***************************begin diagnose CSI exposer[velero/fake-backup]*************************** +Pod velero/fake-backup, phase Pending, node name fake-node +Pod condition Initialized, status True, reason , message fake-pod-message +PVC velero/fake-backup, phase Pending, binding to fake-pv +PV fake-pv, phase Pending, reason , message fake-pv-message +VS velero/fake-backup, bind to fake-vsc, readyToUse false, errMessage fake-vs-message +error getting backup vsc fake-vsc, err: volumesnapshotcontents.snapshot.storage.k8s.io "fake-vsc" not found +***************************end diagnose CSI exposer[velero/fake-backup]*************************** +`, + }, + { + name: "vs with vsc, vsc exists", + ownerBackup: backup, + kubeClientObj: []runtime.Object{ + &backupPodWithNodeName, + &backupPVCWithVolumeName, + &backupPV, + &nodeAgentPod, + }, + snapshotClientObj: []runtime.Object{ + &backupVSWithVSC, + &backupVSC, + }, + expected: `***************************begin diagnose CSI exposer[velero/fake-backup]*************************** +Pod velero/fake-backup, phase Pending, node name fake-node +Pod condition Initialized, status True, reason , message fake-pod-message +PVC velero/fake-backup, phase Pending, binding to fake-pv +PV fake-pv, phase Pending, reason , message fake-pv-message +VS velero/fake-backup, bind to fake-vsc, readyToUse false, errMessage fake-vs-message +VSC fake-vsc, readyToUse false, errMessage fake-vsc-message, handle +***************************end diagnose CSI exposer[velero/fake-backup]*************************** +`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeKubeClient := fake.NewSimpleClientset(tt.kubeClientObj...) + fakeSnapshotClient := snapshotFake.NewSimpleClientset(tt.snapshotClientObj...) + e := &csiSnapshotExposer{ + kubeClient: fakeKubeClient, + csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), + log: velerotest.NewLogger(), + } + var ownerObject corev1.ObjectReference + if tt.ownerBackup != nil { + ownerObject = corev1.ObjectReference{ + Kind: tt.ownerBackup.Kind, + Namespace: tt.ownerBackup.Namespace, + Name: tt.ownerBackup.Name, + UID: tt.ownerBackup.UID, + APIVersion: tt.ownerBackup.APIVersion, + } + } + + diag := e.DiagnoseExpose(context.Background(), ownerObject) + assert.Equal(t, tt.expected, diag) + }) + } +} diff --git a/pkg/exposer/generic_restore.go b/pkg/exposer/generic_restore.go index d7d4ac235d..f523bce0a2 100644 --- a/pkg/exposer/generic_restore.go +++ b/pkg/exposer/generic_restore.go @@ -208,12 +208,12 @@ func (e *genericRestoreExposer) DiagnoseExpose(ctx context.Context, ownerObject pod, err := e.kubeClient.CoreV1().Pods(ownerObject.Namespace).Get(ctx, restorePodName, metav1.GetOptions{}) if err != nil { - diag += fmt.Sprintf("error to get restore pod %s, err: %v\n", restorePodName, err) + diag += fmt.Sprintf("error getting restore pod %s, err: %v\n", restorePodName, err) } pvc, err := e.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(ctx, restorePVCName, metav1.GetOptions{}) if err != nil { - diag += fmt.Sprintf("error to get restore pvc %s, err: %v\n", restorePVCName, err) + diag += fmt.Sprintf("error getting restore pvc %s, err: %v\n", restorePVCName, err) } if pod != nil { @@ -231,7 +231,7 @@ func (e *genericRestoreExposer) DiagnoseExpose(ctx context.Context, ownerObject if pvc.Spec.VolumeName != "" { if pv, err := e.kubeClient.CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{}); err != nil { - diag += fmt.Sprintf("error getting backup pv %s, err: %v\n", pvc.Spec.VolumeName, err) + diag += fmt.Sprintf("error getting restore pv %s, err: %v\n", pvc.Spec.VolumeName, err) } else { diag += kube.DiagnosePV(pv) } diff --git a/pkg/exposer/generic_restore_test.go b/pkg/exposer/generic_restore_test.go index 4c3221b5ca..bcc78b9810 100644 --- a/pkg/exposer/generic_restore_test.go +++ b/pkg/exposer/generic_restore_test.go @@ -507,3 +507,265 @@ func TestRestorePeekExpose(t *testing.T) { }) } } + +func Test_ReastoreDiagnoseExpose(t *testing.T) { + restore := &velerov1.Restore{ + TypeMeta: metav1.TypeMeta{ + APIVersion: velerov1.SchemeGroupVersion.String(), + Kind: "Restore", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-restore", + UID: "fake-uid", + }, + } + + restorePodWithoutNodeName := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-restore", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: restore.APIVersion, + Kind: restore.Kind, + Name: restore.Name, + UID: restore.UID, + }, + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodInitialized, + Status: corev1.ConditionTrue, + Message: "fake-pod-message", + }, + }, + }, + } + + restorePodWithNodeName := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-restore", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: restore.APIVersion, + Kind: restore.Kind, + Name: restore.Name, + UID: restore.UID, + }, + }, + }, + Spec: corev1.PodSpec{ + NodeName: "fake-node", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodInitialized, + Status: corev1.ConditionTrue, + Message: "fake-pod-message", + }, + }, + }, + } + + restorePVCWithoutVolumeName := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-restore", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: restore.APIVersion, + Kind: restore.Kind, + Name: restore.Name, + UID: restore.UID, + }, + }, + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimPending, + }, + } + + restorePVCWithVolumeName := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-restore", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: restore.APIVersion, + Kind: restore.Kind, + Name: restore.Name, + UID: restore.UID, + }, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "fake-pv", + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimPending, + }, + } + + restorePV := corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-pv", + }, + Status: corev1.PersistentVolumeStatus{ + Phase: corev1.VolumePending, + Message: "fake-pv-message", + }, + } + + nodeAgentPod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "node-agent-pod-1", + Labels: map[string]string{"name": "node-agent"}, + }, + Spec: corev1.PodSpec{ + NodeName: "fake-node", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + } + + tests := []struct { + name string + ownerRestore *velerov1.Restore + kubeClientObj []runtime.Object + expected string + }{ + { + name: "no pod, pvc", + ownerRestore: restore, + expected: `***************************begin diagnose restore exposer[velero/fake-restore]*************************** +error getting restore pod fake-restore, err: pods "fake-restore" not found +error getting restore pvc fake-restore, err: persistentvolumeclaims "fake-restore" not found +***************************end diagnose restore exposer[velero/fake-restore]*************************** +`, + }, + { + name: "pod without node name, pvc without volume name, vs without status", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + &restorePodWithoutNodeName, + &restorePVCWithoutVolumeName, + }, + expected: `***************************begin diagnose restore exposer[velero/fake-restore]*************************** +Pod velero/fake-restore, phase Pending, node name +Pod condition Initialized, status True, reason , message fake-pod-message +PVC velero/fake-restore, phase Pending, binding to +***************************end diagnose restore exposer[velero/fake-restore]*************************** +`, + }, + { + name: "pod without node name, pvc without volume name", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + &restorePodWithoutNodeName, + &restorePVCWithoutVolumeName, + }, + expected: `***************************begin diagnose restore exposer[velero/fake-restore]*************************** +Pod velero/fake-restore, phase Pending, node name +Pod condition Initialized, status True, reason , message fake-pod-message +PVC velero/fake-restore, phase Pending, binding to +***************************end diagnose restore exposer[velero/fake-restore]*************************** +`, + }, + { + name: "pod with node name, no node agent", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + &restorePodWithNodeName, + &restorePVCWithoutVolumeName, + }, + expected: `***************************begin diagnose restore exposer[velero/fake-restore]*************************** +Pod velero/fake-restore, phase Pending, node name fake-node +Pod condition Initialized, status True, reason , message fake-pod-message +node-agent is not running in node fake-node +PVC velero/fake-restore, phase Pending, binding to +***************************end diagnose restore exposer[velero/fake-restore]*************************** +`, + }, + { + name: "pod with node name, node agent is running", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + &restorePodWithNodeName, + &restorePVCWithoutVolumeName, + &nodeAgentPod, + }, + expected: `***************************begin diagnose restore exposer[velero/fake-restore]*************************** +Pod velero/fake-restore, phase Pending, node name fake-node +Pod condition Initialized, status True, reason , message fake-pod-message +PVC velero/fake-restore, phase Pending, binding to +***************************end diagnose restore exposer[velero/fake-restore]*************************** +`, + }, + { + name: "pvc with volume name, no pv", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + &restorePodWithNodeName, + &restorePVCWithVolumeName, + &nodeAgentPod, + }, + expected: `***************************begin diagnose restore exposer[velero/fake-restore]*************************** +Pod velero/fake-restore, phase Pending, node name fake-node +Pod condition Initialized, status True, reason , message fake-pod-message +PVC velero/fake-restore, phase Pending, binding to fake-pv +error getting restore pv fake-pv, err: persistentvolumes "fake-pv" not found +***************************end diagnose restore exposer[velero/fake-restore]*************************** +`, + }, + { + name: "pvc with volume name, pv exists", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + &restorePodWithNodeName, + &restorePVCWithVolumeName, + &restorePV, + &nodeAgentPod, + }, + expected: `***************************begin diagnose restore exposer[velero/fake-restore]*************************** +Pod velero/fake-restore, phase Pending, node name fake-node +Pod condition Initialized, status True, reason , message fake-pod-message +PVC velero/fake-restore, phase Pending, binding to fake-pv +PV fake-pv, phase Pending, reason , message fake-pv-message +***************************end diagnose restore exposer[velero/fake-restore]*************************** +`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) + + e := genericRestoreExposer{ + kubeClient: fakeKubeClient, + log: velerotest.NewLogger(), + } + + var ownerObject corev1api.ObjectReference + if test.ownerRestore != nil { + ownerObject = corev1api.ObjectReference{ + Kind: test.ownerRestore.Kind, + Namespace: test.ownerRestore.Namespace, + Name: test.ownerRestore.Name, + UID: test.ownerRestore.UID, + APIVersion: test.ownerRestore.APIVersion, + } + } + + diag := e.DiagnoseExpose(context.Background(), ownerObject) + assert.Equal(t, test.expected, diag) + }) + } +} diff --git a/pkg/util/csi/volume_snapshot.go b/pkg/util/csi/volume_snapshot.go index fcd6836388..738bd91481 100644 --- a/pkg/util/csi/volume_snapshot.go +++ b/pkg/util/csi/volume_snapshot.go @@ -776,42 +776,48 @@ func WaitUntilVSCHandleIsReady( func DiagnoseVS(vs *snapshotv1api.VolumeSnapshot) string { vscName := "" - if vs.Status.BoundVolumeSnapshotContentName != nil { - vscName = *vs.Status.BoundVolumeSnapshotContentName - } - readyToUse := false - if vs.Status.ReadyToUse != nil { - readyToUse = *vs.Status.ReadyToUse - } - errMessage := "" - if vs.Status.Error != nil && vs.Status.Error.Message != nil { - errMessage = *vs.Status.Error.Message + + if vs.Status != nil { + if vs.Status.BoundVolumeSnapshotContentName != nil { + vscName = *vs.Status.BoundVolumeSnapshotContentName + } + + if vs.Status.ReadyToUse != nil { + readyToUse = *vs.Status.ReadyToUse + } + + if vs.Status.Error != nil && vs.Status.Error.Message != nil { + errMessage = *vs.Status.Error.Message + } } - diag := fmt.Sprintf("VS %s/%s, bind to %s, readToUse %v, errMessage %s\n", vs.Namespace, vs.Name, vscName, readyToUse, errMessage) + diag := fmt.Sprintf("VS %s/%s, bind to %s, readyToUse %v, errMessage %s\n", vs.Namespace, vs.Name, vscName, readyToUse, errMessage) return diag } func DiagnoseVSC(vsc *snapshotv1api.VolumeSnapshotContent) string { handle := "" - if vsc.Status.SnapshotHandle != nil { - handle = *vsc.Status.SnapshotHandle - } - readyToUse := false - if vsc.Status.ReadyToUse != nil { - readyToUse = *vsc.Status.ReadyToUse - } - errMessage := "" - if vsc.Status.Error != nil && vsc.Status.Error.Message != nil { - errMessage = *vsc.Status.Error.Message + + if vsc.Status != nil { + if vsc.Status.SnapshotHandle != nil { + handle = *vsc.Status.SnapshotHandle + } + + if vsc.Status.ReadyToUse != nil { + readyToUse = *vsc.Status.ReadyToUse + } + + if vsc.Status.Error != nil && vsc.Status.Error.Message != nil { + errMessage = *vsc.Status.Error.Message + } } - diag := fmt.Sprintf("VSC %s, readToUse %v, errMessage %s, handle %s\n", vsc.Name, readyToUse, errMessage, handle) + diag := fmt.Sprintf("VSC %s, readyToUse %v, errMessage %s, handle %s\n", vsc.Name, readyToUse, errMessage, handle) return diag } diff --git a/pkg/util/csi/volume_snapshot_test.go b/pkg/util/csi/volume_snapshot_test.go index 3876d96edb..79adaf0025 100644 --- a/pkg/util/csi/volume_snapshot_test.go +++ b/pkg/util/csi/volume_snapshot_test.go @@ -1655,3 +1655,197 @@ func TestWaitUntilVSCHandleIsReady(t *testing.T) { }) } } + +func TestDiagnoseVS(t *testing.T) { + vscName := "fake-vsc" + readyToUse := true + message := "fake-message" + + testCases := []struct { + name string + vs *snapshotv1api.VolumeSnapshot + expected string + }{ + { + name: "VS with no status", + vs: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-vs", + Namespace: "fake-ns", + }, + }, + expected: "VS fake-ns/fake-vs, bind to , readyToUse false, errMessage \n", + }, + { + name: "VS with empty status", + vs: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-vs", + Namespace: "fake-ns", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{}, + }, + expected: "VS fake-ns/fake-vs, bind to , readyToUse false, errMessage \n", + }, + { + name: "VS with VSC name", + vs: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-vs", + Namespace: "fake-ns", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &vscName, + }, + }, + expected: "VS fake-ns/fake-vs, bind to fake-vsc, readyToUse false, errMessage \n", + }, + { + name: "VS with VSC name+ready", + vs: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-vs", + Namespace: "fake-ns", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &vscName, + ReadyToUse: &readyToUse, + }, + }, + expected: "VS fake-ns/fake-vs, bind to fake-vsc, readyToUse true, errMessage \n", + }, + { + name: "VS with VSC name+ready+empty error", + vs: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-vs", + Namespace: "fake-ns", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &vscName, + ReadyToUse: &readyToUse, + Error: &snapshotv1api.VolumeSnapshotError{}, + }, + }, + expected: "VS fake-ns/fake-vs, bind to fake-vsc, readyToUse true, errMessage \n", + }, + { + name: "VS with VSC name+ready+error", + vs: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-vs", + Namespace: "fake-ns", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &vscName, + ReadyToUse: &readyToUse, + Error: &snapshotv1api.VolumeSnapshotError{ + Message: &message, + }, + }, + }, + expected: "VS fake-ns/fake-vs, bind to fake-vsc, readyToUse true, errMessage fake-message\n", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + diag := DiagnoseVS(tc.vs) + assert.Equal(t, tc.expected, diag) + }) + } +} + +func TestDiagnoseVSC(t *testing.T) { + readyToUse := true + message := "fake-message" + handle := "fake-handle" + + testCases := []struct { + name string + vsc *snapshotv1api.VolumeSnapshotContent + expected string + }{ + { + name: "VS with no status", + vsc: &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-vsc", + }, + }, + expected: "VSC fake-vsc, readyToUse false, errMessage , handle \n", + }, + { + name: "VSC with empty status", + vsc: &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-vsc", + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{}, + }, + expected: "VSC fake-vsc, readyToUse false, errMessage , handle \n", + }, + { + name: "VSC with ready", + vsc: &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-vsc", + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + ReadyToUse: &readyToUse, + }, + }, + expected: "VSC fake-vsc, readyToUse true, errMessage , handle \n", + }, + { + name: "VSC with ready+handle", + vsc: &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-vsc", + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + ReadyToUse: &readyToUse, + SnapshotHandle: &handle, + }, + }, + expected: "VSC fake-vsc, readyToUse true, errMessage , handle fake-handle\n", + }, + { + name: "VSC with ready+handle+empty error", + vsc: &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-vsc", + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + ReadyToUse: &readyToUse, + SnapshotHandle: &handle, + Error: &snapshotv1api.VolumeSnapshotError{}, + }, + }, + expected: "VSC fake-vsc, readyToUse true, errMessage , handle fake-handle\n", + }, + { + name: "VSC with ready+handle+error", + vsc: &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-vsc", + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + ReadyToUse: &readyToUse, + SnapshotHandle: &handle, + Error: &snapshotv1api.VolumeSnapshotError{ + Message: &message, + }, + }, + }, + expected: "VSC fake-vsc, readyToUse true, errMessage fake-message, handle fake-handle\n", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + diag := DiagnoseVSC(tc.vsc) + assert.Equal(t, tc.expected, diag) + }) + } +} diff --git a/pkg/util/kube/pod.go b/pkg/util/kube/pod.go index 9f126a71a1..bd8bda01d5 100644 --- a/pkg/util/kube/pod.go +++ b/pkg/util/kube/pod.go @@ -262,7 +262,7 @@ func DiagnosePod(pod *corev1api.Pod) string { diag := fmt.Sprintf("Pod %s/%s, phase %s, node name %s\n", pod.Namespace, pod.Name, pod.Status.Phase, pod.Spec.NodeName) for _, condition := range pod.Status.Conditions { - diag += fmt.Sprintf("Pod condition %s, reason %s, message %s\n", condition.Type, condition.Reason, condition.Message) + diag += fmt.Sprintf("Pod condition %s, status %s, reason %s, message %s\n", condition.Type, condition.Status, condition.Reason, condition.Message) } return diag diff --git a/pkg/util/kube/pod_test.go b/pkg/util/kube/pod_test.go index 0e76899a5e..387c29d140 100644 --- a/pkg/util/kube/pod_test.go +++ b/pkg/util/kube/pod_test.go @@ -846,3 +846,49 @@ func TestToSystemAffinity(t *testing.T) { }) } } + +func TestDiagnosePod(t *testing.T) { + testCases := []struct { + name string + pod *corev1api.Pod + expected string + }{ + { + name: "pod with all info", + pod: &corev1api.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-pod", + Namespace: "fake-ns", + }, + Spec: corev1api.PodSpec{ + NodeName: "fake-node", + }, + Status: corev1api.PodStatus{ + Phase: corev1api.PodPending, + Conditions: []corev1api.PodCondition{ + { + Type: corev1api.PodInitialized, + Status: corev1api.ConditionTrue, + Reason: "fake-reason-1", + Message: "fake-message-1", + }, + { + Type: corev1api.PodScheduled, + Status: corev1api.ConditionFalse, + Reason: "fake-reason-2", + Message: "fake-message-2", + }, + }, + }, + }, + expected: "Pod fake-ns/fake-pod, phase Pending, node name fake-node\nPod condition Initialized, status True, reason fake-reason-1, message fake-message-1\nPod condition PodScheduled, status False, reason fake-reason-2, message fake-message-2\n", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + diag := DiagnosePod(tc.pod) + assert.Equal(t, tc.expected, diag) + }) + } +} diff --git a/pkg/util/kube/pvc_pv.go b/pkg/util/kube/pvc_pv.go index ac7d15fbfd..da9dfb2c74 100644 --- a/pkg/util/kube/pvc_pv.go +++ b/pkg/util/kube/pvc_pv.go @@ -414,15 +414,7 @@ func GetPVCForPodVolume(vol *corev1api.Volume, pod *corev1api.Pod, crClient crcl } func DiagnosePVC(pvc *corev1api.PersistentVolumeClaim) string { - diag := fmt.Sprintf("PVC %s/%s, phase %s\n", pvc.Namespace, pvc.Name, pvc.Status.Phase) - - for _, condition := range pvc.Status.Conditions { - diag += fmt.Sprintf("PVC condition %s, reason %s, message %s\n", condition.Type, condition.Reason, condition.Message) - } - - diag += fmt.Sprintf("PVC is binding to %s\n", pvc.Spec.VolumeName) - - return diag + return fmt.Sprintf("PVC %s/%s, phase %s, binding to %s\n", pvc.Namespace, pvc.Name, pvc.Status.Phase, pvc.Spec.VolumeName) } func DiagnosePV(pv *corev1api.PersistentVolume) string { diff --git a/pkg/util/kube/pvc_pv_test.go b/pkg/util/kube/pvc_pv_test.go index 00c3962c5c..8304ebf3be 100644 --- a/pkg/util/kube/pvc_pv_test.go +++ b/pkg/util/kube/pvc_pv_test.go @@ -1465,4 +1465,63 @@ func TestMakePodPVCAttachment(t *testing.T) { } func TestDiagnosePVC(t *testing.T) { + testCases := []struct { + name string + pvc *corev1api.PersistentVolumeClaim + expected string + }{ + { + name: "pvc with all info", + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-pvc", + Namespace: "fake-ns", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeName: "fake-pv", + }, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimPending, + }, + }, + expected: "PVC fake-ns/fake-pvc, phase Pending, binding to fake-pv\n", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + diag := DiagnosePVC(tc.pvc) + assert.Equal(t, tc.expected, diag) + }) + } +} + +func TestDiagnosePV(t *testing.T) { + testCases := []struct { + name string + pv *corev1api.PersistentVolume + expected string + }{ + { + name: "pv with all info", + pv: &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-pv", + }, + Status: corev1api.PersistentVolumeStatus{ + Phase: corev1api.VolumePending, + Message: "fake-message", + Reason: "fake-reason", + }, + }, + expected: "PV fake-pv, phase Pending, reason fake-reason, message fake-message\n", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + diag := DiagnosePV(tc.pv) + assert.Equal(t, tc.expected, diag) + }) + } }