From 26580e66388a3a7ec5b5abfe32ca2e0d9e3d1a9c Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Thu, 21 Nov 2024 22:33:53 +0800 Subject: [PATCH] Refactor the migration cases. Signed-off-by: Xun Jiang --- pkg/cmd/cli/install/install.go | 52 ++- test/e2e/e2e_suite_test.go | 6 +- test/e2e/migration/migration.go | 731 +++++++++++++++++-------------- test/e2e/test/test.go | 2 +- test/e2e/upgrade/upgrade.go | 9 +- test/util/csi/common.go | 12 +- test/util/velero/install.go | 3 - test/util/velero/velero_utils.go | 5 +- 8 files changed, 434 insertions(+), 386 deletions(-) diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 45be13d9fc3..ac28819bfcc 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -42,33 +42,31 @@ import ( // Options collects all the options for installing Velero into a Kubernetes cluster. type Options struct { - Namespace string - Image string - BucketName string - Prefix string - ProviderName string - PodAnnotations flag.Map - PodLabels flag.Map - ServiceAccountAnnotations flag.Map - ServiceAccountName string - VeleroPodCPURequest string - VeleroPodMemRequest string - VeleroPodCPULimit string - VeleroPodMemLimit string - NodeAgentPodCPURequest string - NodeAgentPodMemRequest string - NodeAgentPodCPULimit string - NodeAgentPodMemLimit string - RestoreOnly bool - SecretFile string - NoSecret bool - DryRun bool - BackupStorageConfig flag.Map - VolumeSnapshotConfig flag.Map - UseNodeAgent bool - PrivilegedNodeAgent bool - //TODO remove UseRestic when migration test out of using it - UseRestic bool + Namespace string + Image string + BucketName string + Prefix string + ProviderName string + PodAnnotations flag.Map + PodLabels flag.Map + ServiceAccountAnnotations flag.Map + ServiceAccountName string + VeleroPodCPURequest string + VeleroPodMemRequest string + VeleroPodCPULimit string + VeleroPodMemLimit string + NodeAgentPodCPURequest string + NodeAgentPodMemRequest string + NodeAgentPodCPULimit string + NodeAgentPodMemLimit string + RestoreOnly bool + SecretFile string + NoSecret bool + DryRun bool + BackupStorageConfig flag.Map + VolumeSnapshotConfig flag.Map + UseNodeAgent bool + PrivilegedNodeAgent bool Wait bool UseVolumeSnapshots bool DefaultRepoMaintenanceFrequency time.Duration diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 8e9e9bc0b45..68de2ecc46a 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -575,9 +575,9 @@ var _ = Describe( ) var _ = Describe( - "Migrate resources between clusters by Restic", - Label("Migration", "Restic"), - MigrationWithRestic, + "Migrate resources between clusters by FileSystem backup", + Label("Migration", "FSB"), + MigrationWithFS, ) var _ = Describe( "Migrate resources between clusters by snapshot", diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 16048062081..8999a805ff3 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -17,388 +17,455 @@ package migration import ( "context" - "flag" "fmt" "strings" "time" - "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "github.com/vmware-tanzu/velero/test" + "github.com/vmware-tanzu/velero/test" + framework "github.com/vmware-tanzu/velero/test/e2e/test" util "github.com/vmware-tanzu/velero/test/util/csi" - . "github.com/vmware-tanzu/velero/test/util/k8s" - . "github.com/vmware-tanzu/velero/test/util/kibishii" - . "github.com/vmware-tanzu/velero/test/util/providers" - . "github.com/vmware-tanzu/velero/test/util/velero" + k8sutil "github.com/vmware-tanzu/velero/test/util/k8s" + "github.com/vmware-tanzu/velero/test/util/kibishii" + "github.com/vmware-tanzu/velero/test/util/providers" + veleroutil "github.com/vmware-tanzu/velero/test/util/velero" ) -var migrationNamespace string -var veleroCfg VeleroConfig +type migrationE2E struct { + framework.TestCase + useVolumeSnapshots bool + veleroCLI2Version test.VeleroCLI2Version + kibishiiData kibishii.KibishiiData +} func MigrationWithSnapshots() { - veleroCfg = VeleroCfg - for _, veleroCLI2Version := range GetVersionList(veleroCfg.MigrateFromVeleroCLI, veleroCfg.MigrateFromVeleroVersion) { - MigrationTest(true, veleroCLI2Version) + for _, veleroCLI2Version := range veleroutil.GetVersionList( + test.VeleroCfg.MigrateFromVeleroCLI, + test.VeleroCfg.MigrateFromVeleroVersion) { + framework.TestFunc( + &migrationE2E{ + useVolumeSnapshots: true, + veleroCLI2Version: veleroCLI2Version, + }, + )() } } -func MigrationWithRestic() { - veleroCfg = VeleroCfg - for _, veleroCLI2Version := range GetVersionList(veleroCfg.MigrateFromVeleroCLI, veleroCfg.MigrateFromVeleroVersion) { - MigrationTest(false, veleroCLI2Version) +func MigrationWithFS() { + for _, veleroCLI2Version := range veleroutil.GetVersionList( + test.VeleroCfg.MigrateFromVeleroCLI, + test.VeleroCfg.MigrateFromVeleroVersion) { + framework.TestFunc( + &migrationE2E{ + useVolumeSnapshots: false, + veleroCLI2Version: veleroCLI2Version, + }, + )() } } -func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) { - var ( - backupName, restoreName string - backupScName, restoreScName string - kibishiiWorkerCount int - err error - ) - BeforeEach(func() { - kibishiiWorkerCount = 3 - veleroCfg = VeleroCfg - UUIDgen, err = uuid.NewRandom() - migrationNamespace = "migration-" + UUIDgen.String() - if useVolumeSnapshots && veleroCfg.CloudProvider == Kind { - Skip(fmt.Sprintf("Volume snapshots not supported on %s", Kind)) +func (m *migrationE2E) Init() error { + By("Skip check", func() { + if m.VeleroCfg.DefaultClusterContext == "" && m.VeleroCfg.StandbyClusterContext == "" { + Skip("Migration test needs 2 clusters") } - if veleroCfg.DefaultClusterContext == "" && veleroCfg.StandbyClusterContext == "" { - Skip("Migration test needs 2 clusters") + if m.useVolumeSnapshots && m.VeleroCfg.CloudProvider == test.Kind { + Skip(fmt.Sprintf("Volume snapshots not supported on %s", test.Kind)) } - // need to uninstall Velero first in case of the affection of the existing global velero installation - if InstallVelero { - By("Uninstall Velero", func() { - ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) - defer ctxCancel() - Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed()) - }) + + if m.VeleroCfg.SnapshotMoveData && !m.useVolumeSnapshots { + Skip("FSB migration test is not needed in data mover scenario") } }) - AfterEach(func() { - if CurrentSpecReport().Failed() && veleroCfg.FailFast { - fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") - } else { - By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.DefaultClusterContext), func() { - ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) - defer ctxCancel() - - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) - veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient - veleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName - Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed()) - - By(fmt.Sprintf("Delete sample workload namespace %s", migrationNamespace), func() { - Expect( - DeleteNamespace( - context.Background(), - *veleroCfg.DefaultClient, - migrationNamespace, - true), - ).To(Succeed()) - }) - }) - By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.StandbyClusterContext), func() { - ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) - defer ctxCancel() - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed()) - veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient - veleroCfg.ClusterToInstallVelero = veleroCfg.StandbyClusterName + By("Call the base E2E init", func() { + Expect(m.TestCase.Init()).To(Succeed()) + }) - By("Delete StorageClasses created by E2E") - Expect( - DeleteStorageClass( - ctx, - *veleroCfg.ClientToInstallVelero, - StorageClassName, - ), - ).To(Succeed()) - Expect( - DeleteStorageClass( - ctx, - *veleroCfg.ClientToInstallVelero, - StorageClassName2, - ), - ).To(Succeed()) + m.kibishiiData = *kibishii.DefaultKibishiiData + m.kibishiiData.ExpectedNodes = 3 + m.CaseBaseName = "migration-" + m.UUIDgen + m.BackupName = m.CaseBaseName + "-backup" + m.RestoreName = m.CaseBaseName + "-restore" + m.NSIncluded = &[]string{m.CaseBaseName} + + m.RestoreArgs = []string{ + "create", "--namespace", m.VeleroCfg.VeleroNamespace, + "restore", m.RestoreName, + "--from-backup", m.BackupName, "--wait", + } - if strings.EqualFold(veleroCfg.Features, FeatureCSI) && - veleroCfg.UseVolumeSnapshots { - By("Delete VolumeSnapshotClass created by E2E") - Expect( - KubectlDeleteByFile( - ctx, - fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", veleroCfg.CloudProvider), - ), - ).To(Succeed()) - } + // Message output by ginkgo + m.TestMsg = &framework.TestMSG{ + Desc: "Test migration workload on two clusters", + FailedMSG: "Fail to test migrate between two clusters", + Text: "Test back up on default cluster, restore on standby cluster", + } - Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed()) - - By(fmt.Sprintf("Delete sample workload namespace %s", migrationNamespace), func() { - Expect( - DeleteNamespace( - context.Background(), - *veleroCfg.StandbyClient, - migrationNamespace, - true, - ), - ).To(Succeed()) - }) - }) + // Need to uninstall Velero on the default cluster. + if test.InstallVelero { + ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) + defer ctxCancel() + Expect(veleroutil.VeleroUninstall(ctx, m.VeleroCfg)).To(Succeed()) + } - By(fmt.Sprintf("Switch to default KubeConfig context %s", veleroCfg.DefaultClusterContext), func() { - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) - veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient - veleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName - }) - } - }) - When("kibishii is the sample workload", func() { - It("should be successfully backed up and restored to the default BackupStorageLocation", func() { - var backupNames []string - if veleroCfg.SnapshotMoveData { - if !useVolumeSnapshots { - Skip("FSB migration test is not needed in data mover scenario") + return nil +} + +func (m *migrationE2E) Backup() error { + OriginVeleroCfg := m.VeleroCfg + var err error + + if m.veleroCLI2Version.VeleroCLI == "" { + //Assume tag of velero server image is identical to velero CLI version + //Download velero CLI if it's empty according to velero CLI version + By( + fmt.Sprintf("Install the expected version Velero CLI %s", + m.veleroCLI2Version.VeleroVersion), + func() { + // "self" represents 1.14.x and future versions + if m.veleroCLI2Version.VeleroVersion == "self" { + m.veleroCLI2Version.VeleroCLI = m.VeleroCfg.VeleroCLI + } else { + OriginVeleroCfg, err = veleroutil.SetImagesToDefaultValues( + OriginVeleroCfg, + m.veleroCLI2Version.VeleroVersion, + ) + Expect(err).To(Succeed(), + "Fail to set images for the migrate-from Velero installation.") + + m.veleroCLI2Version.VeleroCLI, err = veleroutil.InstallVeleroCLI( + m.veleroCLI2Version.VeleroVersion) + Expect(err).To(Succeed()) } + }, + ) + } + + By(fmt.Sprintf("Install Velero on default cluster (%s)", m.VeleroCfg.DefaultClusterContext), + func() { + Expect(k8sutil.KubectlConfigUseContext( + m.Ctx, m.VeleroCfg.DefaultClusterContext)).To(Succeed()) + OriginVeleroCfg.MigrateFromVeleroVersion = m.veleroCLI2Version.VeleroVersion + OriginVeleroCfg.VeleroCLI = m.veleroCLI2Version.VeleroCLI + OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient + OriginVeleroCfg.ClusterToInstallVelero = m.VeleroCfg.DefaultClusterName + OriginVeleroCfg.ServiceAccountNameToInstall = m.VeleroCfg.DefaultCLSServiceAccountName + OriginVeleroCfg.UseVolumeSnapshots = m.useVolumeSnapshots + OriginVeleroCfg.UseNodeAgent = !m.useVolumeSnapshots + + version, err := veleroutil.GetVeleroVersion(m.Ctx, OriginVeleroCfg.VeleroCLI, true) + Expect(err).To(Succeed(), "Fail to get Velero version") + OriginVeleroCfg.VeleroVersion = version + + if OriginVeleroCfg.SnapshotMoveData { + OriginVeleroCfg.UseNodeAgent = true } - oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60) - defer ctxCancel() - flag.Parse() - UUIDgen, err = uuid.NewRandom() - Expect(err).To(Succeed()) - supportUploaderType, err := IsSupportUploaderType(veleroCLI2Version.VeleroVersion) - Expect(err).To(Succeed()) - - OriginVeleroCfg := veleroCfg - if veleroCLI2Version.VeleroCLI == "" { - //Assume tag of velero server image is identical to velero CLI version - //Download velero CLI if it's empty according to velero CLI version - By(fmt.Sprintf("Install the expected version Velero CLI (%s) for installing Velero", - veleroCLI2Version.VeleroVersion), func() { - //"self" represents 1.14.x and future versions - if veleroCLI2Version.VeleroVersion == "self" { - veleroCLI2Version.VeleroCLI = veleroCfg.VeleroCLI - } else { - OriginVeleroCfg, err = SetImagesToDefaultValues( - OriginVeleroCfg, - veleroCLI2Version.VeleroVersion, - ) - Expect(err).To(Succeed(), "Fail to set images for the migrate-from Velero installation.") - - veleroCLI2Version.VeleroCLI, err = InstallVeleroCLI(veleroCLI2Version.VeleroVersion) - Expect(err).To(Succeed()) - } - }) + + Expect(veleroutil.VeleroInstall(m.Ctx, &OriginVeleroCfg, false)).To(Succeed()) + if m.veleroCLI2Version.VeleroVersion != "self" { + Expect(veleroutil.CheckVeleroVersion( + m.Ctx, + OriginVeleroCfg.VeleroCLI, + OriginVeleroCfg.MigrateFromVeleroVersion, + )).To(Succeed()) } + }, + ) - By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultClusterContext), func() { - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) - OriginVeleroCfg.MigrateFromVeleroVersion = veleroCLI2Version.VeleroVersion - OriginVeleroCfg.VeleroCLI = veleroCLI2Version.VeleroCLI - OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient - OriginVeleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName - OriginVeleroCfg.ServiceAccountNameToInstall = veleroCfg.DefaultCLSServiceAccountName - OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots - OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots - - version, err := GetVeleroVersion(oneHourTimeout, OriginVeleroCfg.VeleroCLI, true) - Expect(err).To(Succeed(), "Fail to get Velero version") - OriginVeleroCfg.VeleroVersion = version - - if OriginVeleroCfg.SnapshotMoveData { - OriginVeleroCfg.UseNodeAgent = true - } + By("Create namespace for sample workload", func() { + Expect(k8sutil.CreateNamespace( + m.Ctx, + *m.VeleroCfg.DefaultClient, + m.CaseBaseName, + )).To(Succeed(), + fmt.Sprintf("Failed to create namespace %s to install Kibishii workload", + m.CaseBaseName)) + }) - Expect(VeleroInstall(context.Background(), &OriginVeleroCfg, false)).To(Succeed()) - if veleroCLI2Version.VeleroVersion != "self" { - Expect(CheckVeleroVersion(context.Background(), OriginVeleroCfg.VeleroCLI, - OriginVeleroCfg.MigrateFromVeleroVersion)).To(Succeed()) - } - }) + By("Deploy sample workload of Kibishii", func() { + Expect(kibishii.KibishiiPrepareBeforeBackup( + m.Ctx, + *OriginVeleroCfg.DefaultClient, + OriginVeleroCfg.CloudProvider, + m.CaseBaseName, + OriginVeleroCfg.RegistryCredentialFile, + OriginVeleroCfg.Features, + OriginVeleroCfg.KibishiiDirectory, + OriginVeleroCfg.UseVolumeSnapshots, + &m.kibishiiData, + )).To(Succeed()) + }) - backupName = "backup-" + UUIDgen.String() - backupScName = backupName + "-sc" - restoreName = "restore-" + UUIDgen.String() - restoreScName = restoreName + "-sc" + By(fmt.Sprintf("Backup namespace %s", m.CaseBaseName), func() { + m.BackupArgs = []string{ + "create", "--namespace", m.VeleroCfg.VeleroNamespace, + "backup", m.BackupName, + "--include-namespaces", strings.Join(*m.NSIncluded, ","), + "--wait", + } - By("Create namespace for sample workload", func() { - Expect(CreateNamespace(oneHourTimeout, *veleroCfg.DefaultClient, migrationNamespace)).To(Succeed(), - fmt.Sprintf("Failed to create namespace %s to install Kibishii workload", migrationNamespace)) - }) + if m.useVolumeSnapshots { + m.BackupArgs = append(m.BackupArgs, "--snapshot-volumes=true") + } else { + m.BackupArgs = append(m.BackupArgs, "--default-volumes-to-fs-backup") + } + + if OriginVeleroCfg.SnapshotMoveData { + m.BackupArgs = append(m.BackupArgs, "--snapshot-move-data") + } + + Expect(veleroutil.VeleroBackupExec( + m.Ctx, + OriginVeleroCfg.VeleroCLI, + OriginVeleroCfg.VeleroNamespace, + m.BackupName, + m.BackupArgs, + )).To(Succeed(), func() string { + veleroutil.RunDebug( + context.Background(), + OriginVeleroCfg.VeleroCLI, + OriginVeleroCfg.VeleroNamespace, + m.BackupName, + "", + ) + return "Failed to backup resources" + }) + }) - KibishiiData := *DefaultKibishiiData - By("Deploy sample workload of Kibishii", func() { - KibishiiData.ExpectedNodes = kibishiiWorkerCount - Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.DefaultClient, veleroCfg.CloudProvider, - migrationNamespace, veleroCfg.RegistryCredentialFile, veleroCfg.Features, - veleroCfg.KibishiiDirectory, useVolumeSnapshots, &KibishiiData)).To(Succeed()) + if m.useVolumeSnapshots { + // Only wait for the snapshots.backupdriver.cnsdp.vmware.com + // when the vSphere plugin is used. + if OriginVeleroCfg.HasVspherePlugin { + By("Waiting for vSphere uploads to complete", func() { + Expect( + veleroutil.WaitForVSphereUploadCompletion( + context.Background(), + time.Hour, + m.CaseBaseName, + m.kibishiiData.ExpectedNodes, + ), + ).To(Succeed()) }) + } - By(fmt.Sprintf("Backup namespace %s", migrationNamespace), func() { - var BackupStorageClassCfg BackupConfig - BackupStorageClassCfg.BackupName = backupScName - BackupStorageClassCfg.IncludeResources = "StorageClass" - BackupStorageClassCfg.IncludeClusterResources = true - - //TODO Remove UseRestic parameter once minor version is 1.10 or upper - BackupStorageClassCfg.UseResticIfFSBackup = !supportUploaderType - Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI, - OriginVeleroCfg.VeleroNamespace, BackupStorageClassCfg)).To(Succeed(), func() string { - RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupStorageClassCfg.BackupName, "") - return "Fail to backup workload" - }) - backupNames = append(backupNames, BackupStorageClassCfg.BackupName) - - var BackupCfg BackupConfig - BackupCfg.BackupName = backupName - BackupCfg.Namespace = migrationNamespace - BackupCfg.UseVolumeSnapshots = useVolumeSnapshots - BackupCfg.BackupLocation = "" - BackupCfg.Selector = "" - BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots - //TODO Remove UseRestic parameter once minor version is 1.10 or upper - BackupCfg.UseResticIfFSBackup = !supportUploaderType - BackupCfg.SnapshotMoveData = OriginVeleroCfg.SnapshotMoveData - - Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI, - OriginVeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { - RunDebug(context.Background(), OriginVeleroCfg.VeleroCLI, OriginVeleroCfg.VeleroNamespace, BackupCfg.BackupName, "") - return "Fail to backup workload" + var snapshotCheckPoint test.SnapshotCheckPoint + snapshotCheckPoint.NamespaceBackedUp = m.CaseBaseName + + if OriginVeleroCfg.SnapshotMoveData { + //VolumeSnapshotContent should be deleted after data movement + _, err := util.CheckVolumeSnapshotCR( + *m.VeleroCfg.DefaultClient, + map[string]string{"namespace": m.CaseBaseName}, + 0, + ) + By("Check the VSC account", func() { + Expect(err).NotTo(HaveOccurred(), "VSC count is not as expected 0") + }) + } else { + // the snapshots of AWS may be still in pending status when do the restore. + // wait for a while to avoid this https://github.com/vmware-tanzu/velero/issues/1799 + if OriginVeleroCfg.CloudProvider == test.Azure && + strings.EqualFold(OriginVeleroCfg.Features, test.FeatureCSI) || + OriginVeleroCfg.CloudProvider == test.AWS { + By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { + time.Sleep(5 * time.Minute) }) - backupNames = append(backupNames, BackupCfg.BackupName) + } + + By("Snapshot should be created in cloud object store with retain policy", func() { + snapshotCheckPoint, err = veleroutil.GetSnapshotCheckPoint( + *OriginVeleroCfg.DefaultClient, + OriginVeleroCfg, + m.kibishiiData.ExpectedNodes, + m.CaseBaseName, + m.BackupName, + kibishii.GetKibishiiPVCNameList(m.kibishiiData.ExpectedNodes), + ) + + Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") + Expect(providers.CheckSnapshotsInProvider( + OriginVeleroCfg, + m.BackupName, + snapshotCheckPoint, + false, + )).To(Succeed()) }) + } + } - if useVolumeSnapshots { - // Only wait for the snapshots.backupdriver.cnsdp.vmware.com - // when the vSphere plugin is used. - if veleroCfg.HasVspherePlugin { - By("Waiting for vSphere uploads to complete", func() { - Expect(WaitForVSphereUploadCompletion(context.Background(), time.Hour, - migrationNamespace, kibishiiWorkerCount)).To(Succeed()) - }) - } + return nil +} - var snapshotCheckPoint SnapshotCheckPoint - snapshotCheckPoint.NamespaceBackedUp = migrationNamespace +func (m *migrationE2E) Restore() error { + StandbyVeleroCfg := m.VeleroCfg + + By("Install Velero in standby cluster.", func() { + // Ensure cluster-B is the target cluster + Expect(k8sutil.KubectlConfigUseContext( + m.Ctx, m.VeleroCfg.StandbyClusterContext)).To(Succeed()) + + // Check the workload namespace not exist in standby cluster. + _, err := k8sutil.GetNamespace( + m.Ctx, *m.VeleroCfg.StandbyClient, m.CaseBaseName) + Expect(err).To(HaveOccurred(), fmt.Sprintf( + "get namespace in dst cluster successfully, it's not as expected: %s", m.CaseBaseName)) + Expect(strings.Contains(fmt.Sprint(err), "namespaces \""+m.CaseBaseName+"\" not found")). + Should(BeTrue()) + + By("Install StorageClass for E2E.") + Expect(veleroutil.InstallStorageClasses( + m.VeleroCfg.StandbyClusterCloudProvider)).To(Succeed()) + + if strings.EqualFold(m.VeleroCfg.Features, test.FeatureCSI) && + m.VeleroCfg.UseVolumeSnapshots { + By("Install VolumeSnapshotClass for E2E.") + Expect( + k8sutil.KubectlApplyByFile( + m.Ctx, + fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", + m.VeleroCfg.StandbyClusterCloudProvider), + ), + ).To(Succeed()) + } - if OriginVeleroCfg.SnapshotMoveData { - //VolumeSnapshotContent should be deleted after data movement - _, err := util.CheckVolumeSnapshotCR(*veleroCfg.DefaultClient, map[string]string{"namespace": migrationNamespace}, 0) - Expect(err).NotTo(HaveOccurred(), "VSC count is not as expected 0") - } else { - // the snapshots of AWS may be still in pending status when do the restore, wait for a while - // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 - // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if veleroCfg.CloudProvider == Azure && strings.EqualFold(veleroCfg.Features, FeatureCSI) || veleroCfg.CloudProvider == AWS { - By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { - time.Sleep(5 * time.Minute) - }) - } - - By("Snapshot should be created in cloud object store with retain policy", func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, kibishiiWorkerCount, - migrationNamespace, backupName, GetKibishiiPVCNameList(kibishiiWorkerCount)) - Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - Expect(CheckSnapshotsInProvider( - veleroCfg, - backupName, - snapshotCheckPoint, - false, - )).To(Succeed()) - }) - } - } + StandbyVeleroCfg.ClientToInstallVelero = m.VeleroCfg.StandbyClient + StandbyVeleroCfg.ClusterToInstallVelero = m.VeleroCfg.StandbyClusterName + StandbyVeleroCfg.ServiceAccountNameToInstall = m.VeleroCfg.StandbyCLSServiceAccountName + StandbyVeleroCfg.UseNodeAgent = !m.useVolumeSnapshots + if StandbyVeleroCfg.SnapshotMoveData { + StandbyVeleroCfg.UseNodeAgent = true + // For SnapshotMoveData pipelines, we should use standby cluster setting + // for Velero installation. + // In nightly CI, StandbyClusterPlugins is set properly + // if pipeline is for SnapshotMoveData. + StandbyVeleroCfg.Plugins = m.VeleroCfg.StandbyClusterPlugins + StandbyVeleroCfg.ObjectStoreProvider = m.VeleroCfg.StandbyClusterObjectStoreProvider + } - By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyClusterContext), func() { - //Ensure workload of "migrationNamespace" existed in cluster-A - ns, err := GetNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace) - Expect(ns.Name).To(Equal(migrationNamespace)) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("get namespace in source cluster err: %v", err)) - - //Ensure cluster-B is the target cluster - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed()) - _, err = GetNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace) - Expect(err).To(HaveOccurred(), fmt.Sprintf("get namespace in dst cluster successfully, it's not as expected: %s", migrationNamespace)) - fmt.Println(err) - Expect(strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found")).Should(BeTrue()) - - veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient - veleroCfg.ClusterToInstallVelero = veleroCfg.StandbyClusterName - veleroCfg.ServiceAccountNameToInstall = veleroCfg.StandbyCLSServiceAccountName - veleroCfg.UseNodeAgent = !useVolumeSnapshots - veleroCfg.UseRestic = false - if veleroCfg.SnapshotMoveData { - veleroCfg.UseNodeAgent = true - // For SnapshotMoveData pipelines, we should use standby cluster setting for Velero installation - // In nightly CI, StandbyClusterPlugins is set properly if pipeline is for SnapshotMoveData. - veleroCfg.Plugins = veleroCfg.StandbyClusterPlugins - veleroCfg.ObjectStoreProvider = veleroCfg.StandbyClusterObjectStoreProvider - } + Expect(veleroutil.VeleroInstall( + context.Background(), &StandbyVeleroCfg, true)).To(Succeed()) + }) - By("Install StorageClass for E2E.") - Expect(InstallStorageClasses(veleroCfg.StandbyClusterCloudProvider)).To(Succeed()) - - if strings.EqualFold(veleroCfg.Features, FeatureCSI) && - veleroCfg.UseVolumeSnapshots { - By("Install VolumeSnapshotClass for E2E.") - Expect( - KubectlApplyByFile( - context.Background(), - fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", veleroCfg.StandbyClusterCloudProvider), - ), - ).To(Succeed()) - } + By("Waiting for backups sync to Velero in standby cluster", func() { + Expect(veleroutil.WaitForBackupToBeCreated( + m.Ctx, m.BackupName, 5*time.Minute, &StandbyVeleroCfg)).To(Succeed()) + }) - Expect(VeleroInstall(context.Background(), &veleroCfg, true)).To(Succeed()) + By(fmt.Sprintf("Restore %s", m.CaseBaseName), func() { + if m.VeleroCfg.SnapshotMoveData { + cmName := "datamover-storage-class-config" + labels := map[string]string{"velero.io/change-storage-class": "RestoreItemAction", + "velero.io/plugin-config": ""} + data := map[string]string{kibishii.KibishiiStorageClassName: test.StorageClassName} + + By(fmt.Sprintf("Create ConfigMap %s in namespace %s", + cmName, StandbyVeleroCfg.VeleroNamespace), func() { + _, err := k8sutil.CreateConfigMap( + StandbyVeleroCfg.StandbyClient.ClientGo, + StandbyVeleroCfg.VeleroNamespace, + cmName, + labels, + data, + ) + Expect(err).To(Succeed(), fmt.Sprintf( + "failed to create ConfigMap in the namespace %q", + StandbyVeleroCfg.VeleroNamespace)) }) + } - By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyClusterContext), func() { - Expect(WaitForBackupToBeCreated(context.Background(), backupName, 5*time.Minute, &veleroCfg)).To(Succeed()) - Expect(WaitForBackupToBeCreated(context.Background(), backupScName, 5*time.Minute, &veleroCfg)).To(Succeed()) - }) + Expect(veleroutil.VeleroRestore( + m.Ctx, + StandbyVeleroCfg.VeleroCLI, + StandbyVeleroCfg.VeleroNamespace, + m.RestoreName, + m.BackupName, + "", + )).To(Succeed(), func() string { + veleroutil.RunDebug( + m.Ctx, StandbyVeleroCfg.VeleroCLI, + StandbyVeleroCfg.VeleroNamespace, "", m.RestoreName) + return "Fail to restore workload" + }) + }) - By(fmt.Sprintf("Restore %s", migrationNamespace), func() { - if OriginVeleroCfg.SnapshotMoveData { - cmName := "datamover-storage-class-config" - labels := map[string]string{"velero.io/change-storage-class": "RestoreItemAction", - "velero.io/plugin-config": ""} - data := map[string]string{KibishiiStorageClassName: StorageClassName} - - By(fmt.Sprintf("Create ConfigMap %s in namespace %s", cmName, veleroCfg.VeleroNamespace), func() { - _, err := CreateConfigMap(veleroCfg.StandbyClient.ClientGo, veleroCfg.VeleroNamespace, cmName, labels, data) - Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", veleroCfg.VeleroNamespace)) - }) - } else { - Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace, restoreScName, backupScName, "StorageClass")).To(Succeed(), func() string { - RunDebug(context.Background(), veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace, "", restoreName) - return "Fail to restore workload" - }) - } - Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string { - RunDebug(context.Background(), veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace, "", restoreName) - return "Fail to restore workload" - }) - }) + return nil +} - By(fmt.Sprintf("Verify workload %s after restore ", migrationNamespace), func() { - Expect(KibishiiVerifyAfterRestore(*veleroCfg.StandbyClient, migrationNamespace, - oneHourTimeout, &KibishiiData, "")).To(Succeed(), "Fail to verify workload after restore") - }) +func (m *migrationE2E) Verify() error { + By(fmt.Sprintf("Verify workload %s after restore on standby cluster", m.CaseBaseName), func() { + Expect(kibishii.KibishiiVerifyAfterRestore( + *m.VeleroCfg.StandbyClient, + m.CaseBaseName, + m.Ctx, + &m.kibishiiData, + "", + )).To(Succeed(), "Fail to verify workload after restore") + }) - // TODO: delete backup created by case self, not all - By("Clean backups after test", func() { - veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient - Expect(DeleteBackups(context.Background(), backupNames, &veleroCfg)).To(Succeed()) - }) - }) + return nil +} + +func (m *migrationE2E) Clean() error { + By("Clean resource on default cluster.", func() { + Expect(m.TestCase.Clean()).To(Succeed()) }) + + By("Clean resource on standby cluster.", func() { + Expect(k8sutil.KubectlConfigUseContext( + m.Ctx, m.VeleroCfg.StandbyClusterContext)).To(Succeed()) + m.VeleroCfg.ClientToInstallVelero = m.VeleroCfg.StandbyClient + m.VeleroCfg.ClusterToInstallVelero = m.VeleroCfg.StandbyClusterName + + By("Delete StorageClasses created by E2E") + Expect( + k8sutil.DeleteStorageClass( + m.Ctx, + *m.VeleroCfg.ClientToInstallVelero, + test.StorageClassName, + ), + ).To(Succeed()) + Expect( + k8sutil.DeleteStorageClass( + m.Ctx, + *m.VeleroCfg.ClientToInstallVelero, + test.StorageClassName2, + ), + ).To(Succeed()) + + if strings.EqualFold(m.VeleroCfg.Features, test.FeatureCSI) && + m.VeleroCfg.UseVolumeSnapshots { + By("Delete VolumeSnapshotClass created by E2E") + Expect( + k8sutil.KubectlDeleteByFile( + m.Ctx, + fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", + m.VeleroCfg.StandbyClusterCloudProvider), + ), + ).To(Succeed()) + } + + Expect(veleroutil.VeleroUninstall(m.Ctx, m.VeleroCfg)).To(Succeed()) + + Expect( + k8sutil.DeleteNamespace( + m.Ctx, + *m.VeleroCfg.StandbyClient, + m.CaseBaseName, + true, + ), + ).To(Succeed()) + }) + + By("Switch to default KubeConfig context", func() { + Expect(k8sutil.KubectlConfigUseContext( + m.Ctx, + m.VeleroCfg.DefaultClusterContext, + )).To(Succeed()) + }) + + return nil } diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index ae849c611bf..bdad947246a 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -91,8 +91,8 @@ func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() { } func TestIt(test VeleroBackupRestoreTest) error { - test.Init() It(test.GetTestMsg().Text, func() { + test.Init() Expect(RunTestCase(test)).To(Succeed(), test.GetTestMsg().FailedMSG) }) return nil diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 38a8c8a0caf..4757fbbe907 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -138,13 +138,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC tmpCfgForOldVeleroInstall.VeleroVersion = version tmpCfgForOldVeleroInstall.UseVolumeSnapshots = useVolumeSnapshots - if supportUploaderType { - tmpCfgForOldVeleroInstall.UseRestic = false - tmpCfgForOldVeleroInstall.UseNodeAgent = !useVolumeSnapshots - } else { - tmpCfgForOldVeleroInstall.UseRestic = !useVolumeSnapshots - tmpCfgForOldVeleroInstall.UseNodeAgent = false - } + tmpCfgForOldVeleroInstall.UseNodeAgent = !useVolumeSnapshots Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall, false)).To(Succeed()) Expect(CheckVeleroVersion(context.Background(), tmpCfgForOldVeleroInstall.VeleroCLI, @@ -230,7 +224,6 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC By(fmt.Sprintf("Upgrade Velero by CLI %s", tmpCfg.VeleroCLI), func() { tmpCfg.GCFrequency = "" - tmpCfg.UseRestic = false tmpCfg.UseNodeAgent = !useVolumeSnapshots Expect(err).To(Succeed()) if supportUploaderType { diff --git a/test/util/csi/common.go b/test/util/csi/common.go index 9c10e962d9b..5e58ec37df0 100644 --- a/test/util/csi/common.go +++ b/test/util/csi/common.go @@ -113,8 +113,8 @@ func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, ba if len(pvList) != 1 { return "", errors.New(fmt.Sprintf("Only 1 PV of PVC %s pod %s should be found under namespace %s", pvcList[0], podName, namespace)) } - pv_value, err := GetPersistentVolume(context.Background(), client, "", pvList[0]) - fmt.Println(pv_value.Annotations["pv.kubernetes.io/provisioned-by"]) + pvValue, err := GetPersistentVolume(context.Background(), client, "", pvList[0]) + fmt.Println(pvValue.Annotations["pv.kubernetes.io/provisioned-by"]) if err != nil { return "", err } @@ -148,14 +148,10 @@ func CheckVolumeSnapshotCR(client TestClient, index map[string]string, expectedC if len(apiVersion) == 0 { return nil, errors.New("Fail to get APIVersion") } - // if apiVersion[0] == "v1beta1" { - // if snapshotContentNameList, err = GetCsiSnapshotHandle(client, apiVersion[0], index); err != nil { - // return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") - // } - // } else + if apiVersion[0] == "v1" { if snapshotContentNameList, err = GetCsiSnapshotHandle(client, apiVersion[0], index); err != nil { - return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") + return nil, errors.Wrap(err, "Fail to get CSI snapshot content") } } else { return nil, errors.New("API version is invalid") diff --git a/test/util/velero/install.go b/test/util/velero/install.go index cc81813bc96..25e38fbaf38 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -271,9 +271,6 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options if len(options.Image) > 0 { args = append(args, "--image", options.Image) } - if options.UseRestic { - args = append(args, "--use-restic") - } if options.UseNodeAgent { args = append(args, "--use-node-agent") } diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index 16053985290..c57ceae4c44 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -255,10 +255,7 @@ func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig, io.DefaultVolumesToFsBackup = veleroCfg.DefaultVolumesToFsBackup io.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots - if !veleroCfg.UseRestic { - io.UseNodeAgent = veleroCfg.UseNodeAgent - } - io.UseRestic = veleroCfg.UseRestic + io.UseNodeAgent = veleroCfg.UseNodeAgent io.Image = veleroCfg.VeleroImage io.Namespace = veleroCfg.VeleroNamespace io.UploaderType = veleroCfg.UploaderType