diff --git a/test/Makefile b/test/Makefile index 1210df871a..276cc76afd 100644 --- a/test/Makefile +++ b/test/Makefile @@ -197,7 +197,7 @@ run-e2e: ginkgo --standby-cluster-name=$(STANDBY_CLUSTER_NAME) \ --eks-policy-arn=$(EKS_POLICY_ARN) \ --default-cls-service-account-name=$(DEFAULT_CLS_SERVICE_ACCOUNT_NAME) \ - --standby-cls-service-account-name=$(STANDBY_CLS_SERVICE_ACCOUNT_NAME) + --standby-cls-service-account-name=$(STANDBY_CLS_SERVICE_ACCOUNT_NAME) \ --kibishii-directory=$(KIBISHII_DIRECTORY) \ --disable-informer-cache=$(DISABLE_INFORMER_CACHE) diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 1aa3524da9..8316126e6e 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -25,6 +25,7 @@ import ( "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "golang.org/x/mod/semver" . "github.com/vmware-tanzu/velero/test" util "github.com/vmware-tanzu/velero/test/util/csi" @@ -146,22 +147,26 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) OriginVeleroCfg.RestoreHelperImage = "" OriginVeleroCfg.Plugins = "" - // It is for v1.13.x migration scenario only, because since v1.14, nightly CI won't - // pass velero-plugin-for-csi to E2E test anymore, and velero installation will not - // fetch velero-plugin-for-csi automatically, so add it as hardcode below. - // TODO: remove this section from future version like v1.15, e.g. + versionWithoutPatch := semver.MajorMinor(veleroCLI2Version.VeleroVersion) + // Read migration case needs plugins from the PluginsMatrix map. + migrationNeedPlugins, ok := PluginsMatrix[versionWithoutPatch] + Expect(ok).To(BeTrue()) + if OriginVeleroCfg.CloudProvider == Azure { - OriginVeleroCfg.Plugins = "velero/velero-plugin-for-microsoft-azure:v1.9.0" + OriginVeleroCfg.Plugins = migrationNeedPlugins[Azure][0] } if OriginVeleroCfg.CloudProvider == AWS { - OriginVeleroCfg.Plugins = "velero/velero-plugin-for-aws:v1.9.0" + OriginVeleroCfg.Plugins = migrationNeedPlugins[AWS][0] } - if strings.Contains(OriginVeleroCfg.Features, FeatureCSI) { - OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + ",velero/velero-plugin-for-csi:v0.7.0" + // Because Velero CSI plugin is deprecated in v1.14, + // only need to install it for version lower than v1.14. + if strings.Contains(OriginVeleroCfg.Features, FeatureCSI) && + semver.Compare(versionWithoutPatch, "v1.14") < 0 { + OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[CSI][0] } if OriginVeleroCfg.SnapshotMoveData { if OriginVeleroCfg.CloudProvider == Azure { - OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + ",velero/velero-plugin-for-aws:v1.9.0" + OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[AWS][0] } } veleroCLI2Version.VeleroCLI, err = InstallVeleroCLI(veleroCLI2Version.VeleroVersion) diff --git a/test/testdata/storage-class/vsphere-csi.yaml b/test/testdata/storage-class/vsphere-csi.yaml index c6d11bc7b0..911751cfba 100644 --- a/test/testdata/storage-class/vsphere-csi.yaml +++ b/test/testdata/storage-class/vsphere-csi.yaml @@ -5,7 +5,8 @@ metadata: annotations: storageclass.kubernetes.io/is-default-class: "false" parameters: - StoragePolicyName: "vSAN Default Storage Policy" + # StoragePolicyName: "vSAN Default Storage Policy" # This is used for the TKGm environment. + svStorageClass: worker-storagepolicy # This is used for TKGs/uTKG environment. provisioner: csi.vsphere.vmware.com reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer \ No newline at end of file diff --git a/test/testdata/storage-class/vsphere.yaml b/test/testdata/storage-class/vsphere.yaml index 3d06ffdf0b..e09b09a603 100644 --- a/test/testdata/storage-class/vsphere.yaml +++ b/test/testdata/storage-class/vsphere.yaml @@ -5,7 +5,8 @@ metadata: annotations: storageclass.kubernetes.io/is-default-class: "false" parameters: - StoragePolicyName: "vSAN Default Storage Policy" + #StoragePolicyName: "vSAN Default Storage Policy" # This is used for TKGm environment. + svStorageClass: worker-storagepolicy # This is used for TKGs/uTKG environment. provisioner: csi.vsphere.vmware.com reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer \ No newline at end of file diff --git a/test/types.go b/test/types.go index 083fcdbbad..961eebe5dc 100644 --- a/test/types.go +++ b/test/types.go @@ -22,7 +22,7 @@ import ( "github.com/google/uuid" "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" - . "github.com/vmware-tanzu/velero/test/util/k8s" + "github.com/vmware-tanzu/velero/test/util/k8s" ) const StorageClassName = "e2e-storage-class" @@ -35,12 +35,20 @@ const Azure = "azure" const AzureCSI = "azure-csi" const AwsCSI = "aws-csi" const AWS = "aws" -const Gcp = "gcp" +const GCP = "gcp" const Vsphere = "vsphere" +const CSI = "csi" const UploaderTypeRestic = "restic" -var PublicCloudProviders = []string{AWS, Azure, Gcp, Vsphere} +const ( + KubeSystemNamespace = "kube-system" + VSphereCSIControllerNamespace = "vmware-system-csi" + VeleroVSphereSecretName = "velero-vsphere-config-secret" + VeleroVSphereConfigMapName = "velero-vsphere-plugin-config" +) + +var PublicCloudProviders = []string{AWS, Azure, GCP, Vsphere} var LocalCloudProviders = []string{Kind, VanillaZFS} var CloudProviders = append(PublicCloudProviders, LocalCloudProviders...) @@ -87,9 +95,9 @@ type VeleroConfig struct { GCFrequency string DefaultClusterContext string StandbyClusterContext string - ClientToInstallVelero *TestClient - DefaultClient *TestClient - StandbyClient *TestClient + ClientToInstallVelero *k8s.TestClient + DefaultClient *k8s.TestClient + StandbyClient *k8s.TestClient ClusterToInstallVelero string DefaultClusterName string StandbyClusterName string diff --git a/test/util/providers/common.go b/test/util/providers/common.go index 99441d268e..46e42894ef 100644 --- a/test/util/providers/common.go +++ b/test/util/providers/common.go @@ -78,7 +78,7 @@ func getProvider(cloudProvider string) (ObjectsInStorage, error) { case AWS, Vsphere: aws := AWSStorage("") s = &aws - case Gcp: + case GCP: gcs := GCSStorage("") s = &gcs case Azure: diff --git a/test/util/velero/install.go b/test/util/velero/install.go index e067ea8792..81204a9863 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -39,14 +39,9 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" velerexec "github.com/vmware-tanzu/velero/pkg/util/exec" - . "github.com/vmware-tanzu/velero/test" - . "github.com/vmware-tanzu/velero/test/util/eks" - . "github.com/vmware-tanzu/velero/test/util/k8s" -) - -const ( - KubeSystemNamespace = "kube-system" - VSphereCSIControllerNamespace = "vmware-system-csi" + "github.com/vmware-tanzu/velero/test" + eksutil "github.com/vmware-tanzu/velero/test/util/eks" + "github.com/vmware-tanzu/velero/test/util/k8s" ) // we provide more install options other than the standard install.InstallOptions in E2E test @@ -58,7 +53,7 @@ type installOptions struct { WithoutDisableInformerCacheParam bool } -func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluster bool) error { +func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyCluster bool) error { fmt.Printf("Velero install %s\n", time.Now().Format("2006-01-02 15:04:05")) // veleroCfg struct including a set of BSL params and a set of additional BSL params, @@ -77,7 +72,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste veleroCfg.CloudProvider = veleroCfg.StandbyClusterCloudProvider } - if slices.Contains(PublicCloudProviders, veleroCfg.CloudProvider) { + if slices.Contains(test.PublicCloudProviders, veleroCfg.CloudProvider) { fmt.Println("For public cloud platforms, object store plugin provider will be set as cloud provider") // If ObjectStoreProvider is not provided, then using the value same as CloudProvider if veleroCfg.ObjectStoreProvider == "" { @@ -93,20 +88,29 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste if err != nil { return errors.WithMessage(err, "Failed to get provider plugins") } - err = EnsureClusterExists(ctx) + err = k8s.EnsureClusterExists(ctx) if err != nil { return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists") } // TODO - handle this better - if veleroCfg.CloudProvider == Vsphere { + if veleroCfg.CloudProvider == test.Vsphere { // We overrider the ObjectStoreProvider here for vSphere because we want to use the aws plugin for the // backup, but needed to pick up the provider plugins earlier. vSphere plugin no longer needs a Volume // Snapshot location specified if veleroCfg.ObjectStoreProvider == "" { - veleroCfg.ObjectStoreProvider = AWS + veleroCfg.ObjectStoreProvider = test.AWS + } + + if err := cleanVSpherePluginConfig( + veleroCfg.ClientToInstallVelero.ClientGo, + veleroCfg.VeleroNamespace, + test.VeleroVSphereSecretName, + test.VeleroVSphereConfigMapName, + ); err != nil { + return errors.WithMessagef(err, "Failed to clear up vsphere plugin config %s namespace", veleroCfg.VeleroNamespace) } - if err := configvSpherePlugin(veleroCfg); err != nil { + if err := generateVSpherePlugin(veleroCfg); err != nil { return errors.WithMessagef(err, "Failed to config vsphere plugin") } } @@ -118,11 +122,11 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste // For AWS IRSA credential test, AWS IAM service account is required, so if ServiceAccountName and EKSPolicyARN // are both provided, we assume IRSA test is running, otherwise skip this IAM service account creation part. - if veleroCfg.CloudProvider == AWS && veleroInstallOptions.ServiceAccountName != "" { + if veleroCfg.CloudProvider == test.AWS && veleroInstallOptions.ServiceAccountName != "" { if veleroCfg.EKSPolicyARN == "" { return errors.New("Please provide EKSPolicyARN for IRSA test.") } - _, err = GetNamespace(ctx, *veleroCfg.ClientToInstallVelero, veleroCfg.VeleroNamespace) + _, err = k8s.GetNamespace(ctx, *veleroCfg.ClientToInstallVelero, veleroCfg.VeleroNamespace) // We should uninstall Velero for a new service account creation. if !apierrors.IsNotFound(err) { if err := VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace); err != nil { @@ -130,19 +134,19 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste } } // If velero namespace does not exist, we should create it for service account creation - if err := KubectlCreateNamespace(ctx, veleroCfg.VeleroNamespace); err != nil { + if err := k8s.KubectlCreateNamespace(ctx, veleroCfg.VeleroNamespace); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Velero", veleroCfg.VeleroNamespace) } - if err := KubectlDeleteClusterRoleBinding(ctx, "velero-cluster-role"); err != nil { + if err := k8s.KubectlDeleteClusterRoleBinding(ctx, "velero-cluster-role"); err != nil { return errors.Wrapf(err, "Failed to delete clusterrolebinding %s to %s namespace", "velero-cluster-role", veleroCfg.VeleroNamespace) } - if err := KubectlCreateClusterRoleBinding(ctx, "velero-cluster-role", "cluster-admin", veleroCfg.VeleroNamespace, veleroInstallOptions.ServiceAccountName); err != nil { + if err := k8s.KubectlCreateClusterRoleBinding(ctx, "velero-cluster-role", "cluster-admin", veleroCfg.VeleroNamespace, veleroInstallOptions.ServiceAccountName); err != nil { return errors.Wrapf(err, "Failed to create clusterrolebinding %s to %s namespace", "velero-cluster-role", veleroCfg.VeleroNamespace) } - if err := KubectlDeleteIAMServiceAcount(ctx, veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.ClusterToInstallVelero); err != nil { + if err := eksutil.KubectlDeleteIAMServiceAcount(ctx, veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.ClusterToInstallVelero); err != nil { return errors.Wrapf(err, "Failed to delete service account %s to %s namespace", veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace) } - if err := EksctlCreateIAMServiceAcount(ctx, veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.EKSPolicyARN, veleroCfg.ClusterToInstallVelero); err != nil { + if err := eksutil.EksctlCreateIAMServiceAcount(ctx, veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.EKSPolicyARN, veleroCfg.ClusterToInstallVelero); err != nil { return errors.Wrapf(err, "Failed to create service account %s to %s namespace", veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace) } } @@ -163,45 +167,79 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste return nil } -// configvSpherePlugin refers to https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/v1.3.0/docs/vanilla.md -func configvSpherePlugin(veleroCfg *VeleroConfig) error { +// generateVSpherePlugin refers to +// https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/v1.3.0/docs/vanilla.md +func generateVSpherePlugin(veleroCfg *test.VeleroConfig) error { cli := veleroCfg.ClientToInstallVelero - var err error - vsphereSecret := "velero-vsphere-config-secret" - configmaptName := "velero-vsphere-plugin-config" - if err := clearupvSpherePluginConfig(cli.ClientGo, veleroCfg.VeleroNamespace, vsphereSecret, configmaptName); err != nil { - return errors.WithMessagef(err, "Failed to clear up vsphere plugin config %s namespace", veleroCfg.VeleroNamespace) - } - if err := CreateNamespace(context.Background(), *cli, veleroCfg.VeleroNamespace); err != nil { - return errors.WithMessagef(err, "Failed to create Velero %s namespace", veleroCfg.VeleroNamespace) + + if err := k8s.CreateNamespace( + context.Background(), + *cli, + veleroCfg.VeleroNamespace, + ); err != nil { + return errors.WithMessagef( + err, + "Failed to create Velero %s namespace", + veleroCfg.VeleroNamespace, + ) } + + clusterFlavor := "VANILLA" + if err := createVCCredentialSecret(cli.ClientGo, veleroCfg.VeleroNamespace); err != nil { - return errors.WithMessagef(err, "Failed to create virtual center credential secret in %s namespace", veleroCfg.VeleroNamespace) - } - if err := WaitForSecretsComplete(cli.ClientGo, veleroCfg.VeleroNamespace, vsphereSecret); err != nil { - return errors.Wrap(err, "Failed to ensure velero-vsphere-config-secret secret completion in namespace kube-system") - } - _, err = CreateConfigMap(cli.ClientGo, veleroCfg.VeleroNamespace, configmaptName, map[string]string{ - "cluster_flavor": "VANILLA", - "vsphere_secret_name": vsphereSecret, - "vsphere_secret_namespace": veleroCfg.VeleroNamespace, - }, nil) - if err != nil { - return errors.WithMessagef(err, "Failed to create velero-vsphere-plugin-config configmap in %s namespace", veleroCfg.VeleroNamespace) + // For TKGs/uTKG the VC secret is not supposed to exist. + if apierrors.IsNotFound(err) { + clusterFlavor = "GUEST" + } else { + return errors.WithMessagef( + err, + "Failed to create virtual center credential secret in %s namespace", + veleroCfg.VeleroNamespace, + ) + } } - err = WaitForConfigMapComplete(cli.ClientGo, veleroCfg.VeleroNamespace, configmaptName) + _, err := k8s.CreateConfigMap( + cli.ClientGo, + veleroCfg.VeleroNamespace, + test.VeleroVSphereConfigMapName, + nil, + map[string]string{ + "cluster_flavor": clusterFlavor, + "vsphere_secret_name": test.VeleroVSphereSecretName, + "vsphere_secret_namespace": veleroCfg.VeleroNamespace, + }, + ) if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to ensure configmap %s completion in namespace: %s", configmaptName, veleroCfg.VeleroNamespace)) + return errors.WithMessagef( + err, + "Failed to create velero-vsphere-plugin-config ConfigMap in %s namespace", + veleroCfg.VeleroNamespace, + ) + } + + if err := k8s.WaitForConfigMapComplete( + cli.ClientGo, + veleroCfg.VeleroNamespace, + test.VeleroVSphereConfigMapName, + ); err != nil { + return errors.Wrap( + err, + fmt.Sprintf("Failed to ensure ConfigMap %s completion in namespace: %s", + test.VeleroVSphereConfigMapName, + veleroCfg.VeleroNamespace, + ), + ) } + return nil } -func clearupvSpherePluginConfig(c clientset.Interface, ns, secretName, configMapName string) error { +func cleanVSpherePluginConfig(c clientset.Interface, ns, secretName, configMapName string) error { //clear secret - _, err := GetSecret(c, ns, secretName) + _, err := k8s.GetSecret(c, ns, secretName) if err == nil { //exist - if err := WaitForSecretDelete(c, ns, secretName); err != nil { + if err := k8s.WaitForSecretDelete(c, ns, secretName); err != nil { return errors.WithMessagef(err, "Failed to clear up vsphere plugin secret in %s namespace", ns) } } else if !apierrors.IsNotFound(err) { @@ -209,9 +247,9 @@ func clearupvSpherePluginConfig(c clientset.Interface, ns, secretName, configMap } //clear configmap - _, err = GetConfigmap(c, ns, configMapName) + _, err = k8s.GetConfigmap(c, ns, configMapName) if err == nil { - if err := WaitForConfigmapDelete(c, ns, configMapName); err != nil { + if err := k8s.WaitForConfigmapDelete(c, ns, configMapName); err != nil { return errors.WithMessagef(err, "Failed to clear up vsphere plugin configmap in %s namespace", ns) } } else if !apierrors.IsNotFound(err) { @@ -282,10 +320,10 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options if len(options.Features) > 0 { args = append(args, "--features", options.Features) - if !strings.EqualFold(cloudProvider, Vsphere) && strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots { + if !strings.EqualFold(cloudProvider, test.Vsphere) && strings.EqualFold(options.Features, test.FeatureCSI) && options.UseVolumeSnapshots { // https://github.com/openebs/zfs-localpv/blob/develop/docs/snapshot.md fmt.Printf("Start to install %s VolumeSnapshotClass ... \n", cloudProvider) - if err := KubectlApplyByFile(ctx, fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", cloudProvider)); err != nil { + if err := k8s.KubectlApplyByFile(ctx, fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", cloudProvider)); err != nil { fmt.Println("Fail to install VolumeSnapshotClass when CSI feature is enabled: ", err) return err } @@ -454,11 +492,11 @@ func patchResources(resources *unstructured.UnstructuredList, namespace string, i++ } else if options.VeleroServerDebugMode && resource.GetKind() == "Deployment" && resource.GetName() == "velero" { - deployJsonStr, err := json.Marshal(resource.Object) + deployJSONStr, err := json.Marshal(resource.Object) if err != nil { return errors.Wrapf(err, "failed to marshal velero deployment") } - if err := json.Unmarshal(deployJsonStr, &deploy); err != nil { + if err := json.Unmarshal(deployJSONStr, &deploy); err != nil { return errors.Wrapf(err, "failed to unmarshal velero deployment") } veleroDeployIndex := -1 @@ -565,7 +603,7 @@ func waitVeleroReady(ctx context.Context, namespace string, useNodeAgent bool) e return nil } -func IsVeleroReady(ctx context.Context, veleroCfg *VeleroConfig) (bool, error) { +func IsVeleroReady(ctx context.Context, veleroCfg *test.VeleroConfig) (bool, error) { namespace := veleroCfg.VeleroNamespace useNodeAgent := veleroCfg.UseNodeAgent if useNodeAgent { @@ -599,7 +637,7 @@ func IsVeleroReady(ctx context.Context, veleroCfg *VeleroConfig) (bool, error) { } // Check BSL with poll - err = wait.PollUntilContextTimeout(ctx, PollInterval, time.Minute, true, func(ctx context.Context) (bool, error) { + err = wait.PollUntilContextTimeout(ctx, k8s.PollInterval, time.Minute, true, func(ctx context.Context) (bool, error) { return checkBSL(ctx, veleroCfg) == nil, nil }) if err != nil { @@ -608,7 +646,7 @@ func IsVeleroReady(ctx context.Context, veleroCfg *VeleroConfig) (bool, error) { return true, nil } -func checkBSL(ctx context.Context, veleroCfg *VeleroConfig) error { +func checkBSL(ctx context.Context, veleroCfg *test.VeleroConfig) error { namespace := veleroCfg.VeleroNamespace stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "get", "bsl", "default", "-o", "json", "-n", namespace)) @@ -626,7 +664,7 @@ func checkBSL(ctx context.Context, veleroCfg *VeleroConfig) error { return nil } -func PrepareVelero(ctx context.Context, caseName string, veleroCfg VeleroConfig) error { +func PrepareVelero(ctx context.Context, caseName string, veleroCfg test.VeleroConfig) error { ready, err := IsVeleroReady(context.Background(), &veleroCfg) if err != nil { fmt.Printf("error in checking velero status with %v", err) @@ -653,17 +691,20 @@ func VeleroUninstall(ctx context.Context, cli, namespace string) error { return nil } -// createVCCredentialSecret refer to https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/v1.3.0/docs/vanilla.md +// createVCCredentialSecret refer to +// https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/v1.3.0/docs/vanilla.md func createVCCredentialSecret(c clientset.Interface, veleroNamespace string) error { secret, err := getVCCredentialSecret(c) if err != nil { return err } + vsphereCfg, exist := secret.Data["csi-vsphere.conf"] if !exist { return errors.New("failed to retrieve csi-vsphere config") } - se := &corev1.Secret{ + + vsphereSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "velero-vsphere-config-secret", Namespace: veleroNamespace, @@ -671,17 +712,36 @@ func createVCCredentialSecret(c clientset.Interface, veleroNamespace string) err Type: corev1.SecretTypeOpaque, Data: map[string][]byte{"csi-vsphere.conf": vsphereCfg}, } - _, err = c.CoreV1().Secrets(veleroNamespace).Create(context.TODO(), se, metav1.CreateOptions{}) - return err + _, err = c.CoreV1().Secrets(veleroNamespace).Create( + context.TODO(), + vsphereSecret, + metav1.CreateOptions{}, + ) + if err != nil { + return err + } + + if err := k8s.WaitForSecretsComplete( + c, + veleroNamespace, + test.VeleroVSphereSecretName, + ); err != nil { + return errors.Wrap( + err, + "Failed to ensure velero-vsphere-config-secret secret completion in namespace kube-system", + ) + } + + return nil } // Reference https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/main/docs/vanilla.md#create-vc-credential-secret // Read secret from kube-system namespace first, if not found, try with vmware-system-csi. func getVCCredentialSecret(c clientset.Interface) (secret *corev1.Secret, err error) { - secret, err = GetSecret(c, KubeSystemNamespace, "vsphere-config-secret") + secret, err = k8s.GetSecret(c, test.KubeSystemNamespace, "vsphere-config-secret") if err != nil { if apierrors.IsNotFound(err) { - secret, err = GetSecret(c, VSphereCSIControllerNamespace, "vsphere-config-secret") + secret, err = k8s.GetSecret(c, test.VSphereCSIControllerNamespace, "vsphere-config-secret") } } return diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index bedfa25d1c..cec0d61869 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -54,77 +54,56 @@ const BackupObjectsPrefix = "backups" const RestoreObjectsPrefix = "restores" const PluginsObjectsPrefix = "plugins" -var pluginsMatrix = map[string]map[string][]string{ - "v1.7": { - "aws": {"velero/velero-plugin-for-aws:v1.3.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.3.0"}, - "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.3.0"}, - "csi": {"velero/velero-plugin-for-csi:v0.2.0"}, - }, - "v1.8": { - "aws": {"velero/velero-plugin-for-aws:v1.4.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.4.0"}, - "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.4.0"}, - "csi": {"velero/velero-plugin-for-csi:v0.2.0"}, - }, - "v1.9": { - "aws": {"velero/velero-plugin-for-aws:v1.5.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.5.0"}, - "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.4.0"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.5.0"}, - "csi": {"velero/velero-plugin-for-csi:v0.3.0"}, - }, +var PluginsMatrix = map[string]map[string][]string{ "v1.10": { - "aws": {"velero/velero-plugin-for-aws:v1.6.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.6.0"}, - "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.6.0"}, - "csi": {"velero/velero-plugin-for-csi:v0.4.0"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.6.0"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.6.0"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.6.0"}, + "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.4.0"}, }, "v1.11": { - "aws": {"velero/velero-plugin-for-aws:v1.7.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.7.0"}, - "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.7.0"}, - "csi": {"velero/velero-plugin-for-csi:v0.5.0"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.7.0"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.7.0"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.7.0"}, + "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.5.0"}, }, "v1.12": { - "aws": {"velero/velero-plugin-for-aws:v1.8.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.8.0"}, - "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.8.0"}, - "csi": {"velero/velero-plugin-for-csi:v0.6.0"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.8.0"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.8.0"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.8.0"}, + "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.6.0"}, }, "v1.13": { - "aws": {"velero/velero-plugin-for-aws:v1.9.2"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.9.2"}, - "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.9.2"}, - "csi": {"velero/velero-plugin-for-csi:v0.7.1"}, - "datamover": {"velero/velero-plugin-for-aws:v1.9.2"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.9.2"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.9.2"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.9.2"}, + "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.7.1"}, + "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.9.2"}, }, "v1.14": { - "aws": {"velero/velero-plugin-for-aws:v1.10.1"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.10.1"}, - "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.10.1"}, - "datamover": {"velero/velero-plugin-for-aws:v1.10.1"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.10.1"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.10.1"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.10.1"}, + "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.10.1"}, }, "v1.15": { - "aws": {"velero/velero-plugin-for-aws:v1.11.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.11.0"}, - "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.11.0"}, - "datamover": {"velero/velero-plugin-for-aws:v1.11.0"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.11.0"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.11.0"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.11.0"}, + "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.11.0"}, }, "main": { - "aws": {"velero/velero-plugin-for-aws:main"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:main"}, - "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"}, - "gcp": {"velero/velero-plugin-for-gcp:main"}, - "datamover": {"velero/velero-plugin-for-aws:main"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:main"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:main"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:main"}, + "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:main"}, }, } @@ -132,10 +111,10 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider string, nee var cloudMap map[string][]string arr := strings.Split(version, ".") if len(arr) >= 3 { - cloudMap = pluginsMatrix[arr[0]+"."+arr[1]] + cloudMap = PluginsMatrix[arr[0]+"."+arr[1]] } if len(cloudMap) == 0 { - cloudMap = pluginsMatrix["main"] + cloudMap = PluginsMatrix["main"] if len(cloudMap) == 0 { return nil, errors.Errorf("fail to get plugins by version: main") }