From 251826a0ff83a6d9cf6cce02682d5f72a297e681 Mon Sep 17 00:00:00 2001 From: apattewar-px Date: Fri, 23 Aug 2024 15:12:33 +0530 Subject: [PATCH 01/11] Data integrity for AsyncDR --- .../k8s/specs/fio-async-dr/fio-pvc.yaml | 11 + .../scheduler/k8s/specs/fio-async-dr/fio.yaml | 33 ++ .../fio-async-dr/pxd/px-storage-class.yaml | 10 + tests/basic/async_dr_test.go | 286 +++++++++++++++++- 4 files changed, 337 insertions(+), 3 deletions(-) create mode 100644 drivers/scheduler/k8s/specs/fio-async-dr/fio-pvc.yaml create mode 100644 drivers/scheduler/k8s/specs/fio-async-dr/fio.yaml create mode 100644 drivers/scheduler/k8s/specs/fio-async-dr/pxd/px-storage-class.yaml diff --git a/drivers/scheduler/k8s/specs/fio-async-dr/fio-pvc.yaml b/drivers/scheduler/k8s/specs/fio-async-dr/fio-pvc.yaml new file mode 100644 index 000000000..a7d0e9c63 --- /dev/null +++ b/drivers/scheduler/k8s/specs/fio-async-dr/fio-pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: fio-pvc-sv4 +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 50Gi + storageClassName: portworx-sc \ No newline at end of file diff --git a/drivers/scheduler/k8s/specs/fio-async-dr/fio.yaml b/drivers/scheduler/k8s/specs/fio-async-dr/fio.yaml new file mode 100644 index 000000000..ef6a97f0f --- /dev/null +++ b/drivers/scheduler/k8s/specs/fio-async-dr/fio.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fio-write-test-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: fio-write-test + template: + metadata: + labels: + app: fio-write-test + spec: + restartPolicy: Always + containers: + - name: fio-write-test-container + image: xridge/fio + command: ["/bin/sh", "-c"] + args: + - | + fio --blocksize=32k --directory=/data --filename=test \ + --ioengine=libaio --readwrite=write --size=5120M --name=test \ + --verify=meta --do_verify=1 --verify_pattern=0xDeadBeef --direct=1 \ + --gtod_reduce=1 --iodepth=32 --rate_iops=4000 --randrepeat=1 \ + --disable_lat=0 + volumeMounts: + - name: fio-data + mountPath: /data + volumes: + - name: fio-data + persistentVolumeClaim: + claimName: fio-pvc \ No newline at end of file diff --git a/drivers/scheduler/k8s/specs/fio-async-dr/pxd/px-storage-class.yaml b/drivers/scheduler/k8s/specs/fio-async-dr/pxd/px-storage-class.yaml new file mode 100644 index 000000000..c3d1f6875 --- /dev/null +++ b/drivers/scheduler/k8s/specs/fio-async-dr/pxd/px-storage-class.yaml @@ -0,0 +1,10 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: portworx-sc +parameters: + repl: "2" + sharedv4: "true" +provisioner: pxd.portworx.com +reclaimPolicy: Delete +volumeBindingMode: Immediate \ No newline at end of file diff --git a/tests/basic/async_dr_test.go b/tests/basic/async_dr_test.go index 1b3395fdf..6b837274a 100644 --- a/tests/basic/async_dr_test.go +++ b/tests/basic/async_dr_test.go @@ -56,6 +56,7 @@ const ( var ( kubeConfigWritten bool + //k8sCore = core.Instance() ) type failoverFailbackParam struct { @@ -71,6 +72,118 @@ type failoverFailbackParam struct { contexts []*scheduler.Context } +var _ = Describe("{MigrateDeploymentWithIntegrityCheck}", func() { + testrailID = 50803 + // testrailID corresponds to: https://portworx.testrail.net/index.php?/cases/view/50803 + BeforeEach(func() { + if !kubeConfigWritten { + // Write kubeconfig files after reading from the config maps created by torpedo deploy script + WriteKubeconfigToFiles() + kubeConfigWritten = true + } + wantAllAfterSuiteActions = false + }) + JustBeforeEach(func() { + StartTorpedoTest("MigrateDeployment", "Migration of application to destination cluster", nil, testrailID) + runID = testrailuttils.AddRunsToMilestone(testrailID) + }) + var ( + contexts []*scheduler.Context + migrationNamespaces []string + taskNamePrefix = "async-dr-mig" + allMigrations []*storkapi.Migration + includeResourcesFlag = true + startApplicationsFlag = false + ) + + It("has to deploy app, create cluster pair, migrate app", func() { + Step("Deploy applications", func() { + + err := SetSourceKubeConfig() + log.FailOnError(err, "Switching context to source cluster failed") + // Schedule applications + for i := 0; i < Inst().GlobalScaleFactor; i++ { + taskName := fmt.Sprintf("%s-%d", taskNamePrefix, i) + log.Infof("Task name %s\n", taskName) + appContexts := ScheduleApplications("fio-async-dr") + contexts = append(contexts, appContexts...) + ValidateApplications(contexts) + + for _, ctx := range appContexts { + // Override default App readiness time out of 5 mins with 10 mins + //What is appReadinessTimeout? + ctx.ReadinessTimeout = appReadinessTimeout + namespace := GetAppNamespace(ctx, taskName) + migrationNamespaces = append(migrationNamespaces, namespace) + } + Step("Create cluster pair between source and destination clusters", func() { + // Set cluster context to cluster where torpedo is running + ScheduleValidateClusterPair(appContexts[0], false, true, defaultClusterPairDir, false) + }) + } + + log.Infof("Migration Namespaces: %v", migrationNamespaces) + + }) + + time.Sleep(5 * time.Minute) + log.Info("Start migration") + + for i, currMigNamespace := range migrationNamespaces { + migrationName := migrationKey + fmt.Sprintf("%d", i) + currMig, err := CreateMigration(migrationName, currMigNamespace, defaultClusterPairName, currMigNamespace, &includeResourcesFlag, &startApplicationsFlag) + Expect(err).NotTo(HaveOccurred(), + fmt.Sprintf("failed to create migration: %s in namespace %s. Error: [%v]", + migrationKey, currMigNamespace, err)) + allMigrations = append(allMigrations, currMig) + } + + for _, mig := range allMigrations { + err := storkops.Instance().ValidateMigration(mig.Name, mig.Namespace, migrationRetryTimeout, migrationRetryInterval) + Expect(err).NotTo(HaveOccurred(), + fmt.Sprintf("failed to validate migration: %s in namespace %s. Error: [%v]", + mig.Name, mig.Namespace, err)) + } + + log.InfoD("Start volume only migration") + includeResourcesFlag = false + for i, currMigNamespace := range migrationNamespaces { + migrationName := migrationKey + "volumeonly-" + fmt.Sprintf("%d", i) + currMig, createMigErr := CreateMigration(migrationName, currMigNamespace, defaultClusterPairName, currMigNamespace, &includeResourcesFlag, &startApplicationsFlag) + allMigrations = append(allMigrations, currMig) + log.FailOnError(createMigErr, "Failed to create %s migration in %s namespace", migrationName, currMigNamespace) + err := storkops.Instance().ValidateMigration(currMig.Name, currMig.Namespace, migrationRetryTimeout, migrationRetryInterval) + dash.VerifyFatal(err, nil, "Migration successful?") + resp, getMigErr := storkops.Instance().GetMigration(currMig.Name, currMig.Namespace) + dash.VerifyFatal(getMigErr, nil, "Received migration response?") + dash.VerifyFatal(resp.Status.Summary.NumberOfMigratedResources == 0, true, "Validate no resources migrated") + } + + Step("teardown all applications on source cluster before switching context to destination cluster", func() { + for _, ctx := range contexts { + TearDownContext(ctx, map[string]bool{ + SkipClusterScopedObjects: true, + scheduler.OptionsWaitForResourceLeakCleanup: true, + scheduler.OptionsWaitForDestroy: true, + }) + } + }) + + Step("teardown migrations", func() { + for _, mig := range allMigrations { + err := DeleteAndWaitForMigrationDeletion(mig.Name, mig.Namespace) + Expect(err).NotTo(HaveOccurred(), + fmt.Sprintf("failed to delete migration: %s in namespace %s. Error: [%v]", + mig.Name, mig.Namespace, err)) + } + }) + }) + JustAfterEach(func() { + defer EndTorpedoTest() + AfterEachTest(contexts, testrailID, runID) + }) +}) + // This test performs basic test of starting an application, creating cluster pair, // and migrating application to the destination clsuter var _ = Describe("{MigrateDeployment}", func() { @@ -109,6 +222,7 @@ var _ = Describe("{MigrateDeployment}", func() { appContexts := ScheduleApplications(taskName) contexts = append(contexts, appContexts...) ValidateApplications(contexts) + for _, ctx := range appContexts { // Override default App readiness time out of 5 mins with 10 mins ctx.ReadinessTimeout = appReadinessTimeout @@ -287,6 +401,52 @@ var _ = Describe("{MigrateDeploymentMetroAsync}", func() { }) }) +// Data integrity by Arpit +var _ = Describe("{StorkctlPerformFailoverFailbackDefaultAsyncSingleWithDataIntegrityVerification}", func() { + testrailID = 296255 + // testrailID corresponds to: https://portworx.testrail.net/index.php?/cases/view/296255 + BeforeEach(func() { + if !kubeConfigWritten { + // Write kubeconfig files after reading from the config maps created by torpedo deploy script + WriteKubeconfigToFiles() + kubeConfigWritten = true + } + wantAllAfterSuiteActions = false + }) + JustBeforeEach(func() { + StartTorpedoTest("StorkctlPerformFailoverFailbackDefaultAsyncSingle", "Failover and Failback using storkctl on async cluster for single NS", nil, testrailID) + runID = testrailuttils.AddRunsToMilestone(testrailID) + }) + + It("has to deploy app, create cluster pair, migrate app and do failover/failback", func() { + Step("Deploy app, Create cluster pair, Migrate app and Do failover/failback", func() { + validateFailoverFailbackWithDataIntegrity("asyncdr", "asyncdr-failover-failback", true, false, false, false) + + }) + }) + + //// Add logic to verify data integrity + var contexts []*scheduler.Context + stepLog := "Schedule fio app" + It(stepLog, func() { + log.InfoD(stepLog) + contexts = make([]*scheduler.Context, 0) + + for i := 0; i < Inst().GlobalScaleFactor; i++ { + contexts = append(contexts, ScheduleApplications(fmt.Sprintf("fio-async-dr-%d", i))...) + } + + ValidateApplications(contexts) + + stepLog = "Choosing a single Storage Node randomly and performing SV Motion on it" + + }) + JustAfterEach(func() { + defer EndTorpedoTest() + AfterEachTest(contexts, testrailID, runID) + }) +}) + var _ = Describe("{StorkctlPerformFailoverFailbackDefaultAsyncSingle}", func() { testrailID = 296255 // testrailID corresponds to: https://portworx.testrail.net/index.php?/cases/view/296255 @@ -676,7 +836,7 @@ var _ = Describe("{UpgradeVolumeDriverDuringAppBkpRestore}", func() { backupName = "storkbackup-" + time.Now().Format("15h03m05s") taskNamePrefix = "appbkprest-upgradepx" defaultNs = "kube-system" - timeout = 10 * time.Minute + timeout = 10 * time.Minute ) bkpNs, contexts := initialSetupApps(taskNamePrefix, true) storageNodes := node.GetStorageNodes() @@ -806,7 +966,7 @@ var _ = Describe("{UpgradeVolumeDriverDuringAsyncDrMigration}", func() { kubeConfigPath[cluster], err = GetCustomClusterConfigPath(cluster) log.FailOnError(err, "Getting error while fetching path for %v cluster, error is %v", cluster, err) } - + var migrationSchedName string var schdPol *storkapi.SchedulePolicy cpName := defaultClusterPairName + time.Now().Format("15h03m05s") @@ -994,6 +1154,126 @@ func upgradePX(upgradeHop string, storageNodes []node.Node) (string, string, int return upgradeStatus, updatedPXVersion, durationInMins } +func validateFailoverFailbackWithDataIntegrity(clusterType, taskNamePrefix string, single, skipSourceOp, includeNs, excludeNs bool) { + defaultNs := "kube-system" + migrationNamespaces, contexts := initialSetupApps(taskNamePrefix, single) + + // Approach without Fio to verify + // 1.Get volumes + // 2. Identify node where volume exists + // 3. With the help of ip of node and path , either mount or not to get md5sum + // 4. Post failover, repeat above 3 steps + // 5. Compare md5 sum obtained in the last pre and post migration + + // Approach with Fio to verify + // Run Fio app + // Generate dataset + // Run migrationSchedule + // Run failover on source + // Post failover run fio command in read mode + // Verify no err + + migNamespaces := strings.Join(migrationNamespaces, ",") + kubeConfigPathSrc, err := GetCustomClusterConfigPath(asyncdr.FirstCluster) + log.FailOnError(err, "Failed to get source configPath: %v", err) + kubeConfigPathDest, err := GetCustomClusterConfigPath(asyncdr.SecondCluster) + log.FailOnError(err, "Failed to get destination configPath: %v", err) + if single { + defaultNs = migrationNamespaces[0] + migNamespaces = defaultNs + } + extraArgs := map[string]string{ + "namespaces": migNamespaces, + "kubeconfig": kubeConfigPathSrc, + } + log.Infof("Creating clusterpair between first and second cluster") + cpName := defaultClusterPairName + time.Now().Format("15h03m05s") + if clusterType == "asyncdr" { + err = ScheduleBidirectionalClusterPair(cpName, defaultNs, "", storkapi.BackupLocationType(defaultBackupLocation), defaultSecret, "async-dr", asyncdr.FirstCluster, asyncdr.SecondCluster) + } else { + err = ScheduleBidirectionalClusterPair(cpName, defaultNs, "", "", "", "sync-dr", asyncdr.FirstCluster, asyncdr.SecondCluster) + } + log.FailOnError(err, "Failed creating bidirectional cluster pair") + log.Infof("Start migration schedule and perform failover") + migrationSchedName := migrationSchedKey + time.Now().Format("15h03m05s") + createMigSchdAndValidateMigration(migrationSchedName, cpName, defaultNs, kubeConfigPathSrc, extraArgs) + err = SetCustomKubeConfig(asyncdr.SecondCluster) + log.FailOnError(err, "Switching context to second cluster failed") + extraArgsFailoverFailback := map[string]string{ + "kubeconfig": kubeConfigPathDest, + } + if includeNs { + extraArgsFailoverFailback["include-namespaces"] = migrationNamespaces[0] + } + if excludeNs { + extraArgsFailoverFailback["exclude-namespaces"] = migrationNamespaces[0] + } + failoverParam := failoverFailbackParam{ + action: "failover", + failoverOrFailbackNs: defaultNs, + migrationSchedName: migrationSchedName, + configPath: kubeConfigPathDest, + single: single, + skipSourceOp: skipSourceOp, + includeNs: includeNs, + excludeNs: excludeNs, + extraArgsFailoverFailback: extraArgsFailoverFailback, + contexts: contexts, + } + performFailoverFailback(failoverParam) + + pod, err := k8sCore.GetPodByName("fio-async-dr", migNamespaces) + cmd := []string{"fio", "--blocksize=32k", "--directory=/data", "--filename=test", "--ioengine=libaio", "--readwrite=read", "--size=5120M", "--name=test", "--verify=meta", "--do_verify=1", "--verify_pattern=0xDeadBeef", "--direct=1", "--randrepeat=1", "--output=fio-log_read.txt"} + output, err := core.Instance().RunCommandInPod(cmd, pod.Name, "compute", pod.Namespace) + log.FailOnError(err, "More about error %s", output) + + if skipSourceOp { + err = hardSetConfig(kubeConfigPathSrc) + log.FailOnError(err, "Error setting source config: %v", err) + for _, ctx := range contexts { + waitForPodsToBeRunning(ctx, false) + } + } else { + err = hardSetConfig(kubeConfigPathDest) + log.FailOnError(err, "Error setting destination config: %v", err) + extraArgs["kubeconfig"] = kubeConfigPathDest + newMigSched := migrationSchedName + "-rev" + if includeNs { + extraArgs["namespaces"] = migrationNamespaces[0] + } + if excludeNs { + extraArgs["namespaces"] = strings.Join(migrationNamespaces[1:], ",") + extraArgsFailoverFailback["exclude-namespaces"] = migrationNamespaces[1] + } + createMigSchdAndValidateMigration(newMigSched, cpName, defaultNs, kubeConfigPathDest, extraArgs) + failoverback := failoverFailbackParam{ + action: "failback", + failoverOrFailbackNs: defaultNs, + migrationSchedName: newMigSched, + configPath: kubeConfigPathDest, + single: single, + skipSourceOp: false, + includeNs: includeNs, + excludeNs: excludeNs, + extraArgsFailoverFailback: extraArgsFailoverFailback, + contexts: contexts, + } + performFailoverFailback(failoverback) + } + err = asyncdr.WaitForNamespaceDeletion(migrationNamespaces) + if err != nil { + log.Infof("Failed to delete namespaces: %v", err) + } + err = hardSetConfig(kubeConfigPathDest) + if err != nil { + log.Infof("Failed to se dest kubeconfig for NS deletion on dest: %v", err) + } + err = asyncdr.WaitForNamespaceDeletion(migrationNamespaces) + if err != nil { + log.Infof("Failed to delete namespaces: %v", err) + } +} + func validateFailoverFailback(clusterType, taskNamePrefix string, single, skipSourceOp, includeNs, excludeNs bool) { defaultNs := "kube-system" migrationNamespaces, contexts := initialSetupApps(taskNamePrefix, single) @@ -1596,4 +1876,4 @@ func extractActionName(output, action string) string { getStatusCommand = strings.TrimSuffix(getStatusCommand, "`") getStatusCmdArgs := strings.Split(getStatusCommand, " ") return getStatusCmdArgs[3] -} \ No newline at end of file +} From 232d40f39e48bde3ed2cf88c012f01c4be979515 Mon Sep 17 00:00:00 2001 From: apattewar-px Date: Fri, 23 Aug 2024 18:32:15 +0530 Subject: [PATCH 02/11] Data integrity for AsyncDR --- drivers/scheduler/k8s/specs/fio-async-dr/fio-pvc.yaml | 4 ++-- tests/basic/async_dr_test.go | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/scheduler/k8s/specs/fio-async-dr/fio-pvc.yaml b/drivers/scheduler/k8s/specs/fio-async-dr/fio-pvc.yaml index a7d0e9c63..cec3ead75 100644 --- a/drivers/scheduler/k8s/specs/fio-async-dr/fio-pvc.yaml +++ b/drivers/scheduler/k8s/specs/fio-async-dr/fio-pvc.yaml @@ -1,11 +1,11 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: fio-pvc-sv4 + name: fio-pvc spec: accessModes: - ReadWriteMany resources: requests: - storage: 50Gi + storage: 5Gi storageClassName: portworx-sc \ No newline at end of file diff --git a/tests/basic/async_dr_test.go b/tests/basic/async_dr_test.go index 6b837274a..ff6e70c86 100644 --- a/tests/basic/async_dr_test.go +++ b/tests/basic/async_dr_test.go @@ -1223,8 +1223,10 @@ func validateFailoverFailbackWithDataIntegrity(clusterType, taskNamePrefix strin performFailoverFailback(failoverParam) pod, err := k8sCore.GetPodByName("fio-async-dr", migNamespaces) - cmd := []string{"fio", "--blocksize=32k", "--directory=/data", "--filename=test", "--ioengine=libaio", "--readwrite=read", "--size=5120M", "--name=test", "--verify=meta", "--do_verify=1", "--verify_pattern=0xDeadBeef", "--direct=1", "--randrepeat=1", "--output=fio-log_read.txt"} + log.Infof(pod.Name) + cmd := []string{"fio", "--blocksize=32k", "--directory=/data", "--filename=test", "--ioengine=libaio", "--readwrite=read", "--size=5120M", "--name=test", "--verify=meta", "--do_verify=1", "--verify_pattern=0xDeadBeef", "--direct=1", "--randrepeat=1", "--output=fio-log_read.txt;", "cat", "fio-log_read.txt"} output, err := core.Instance().RunCommandInPod(cmd, pod.Name, "compute", pod.Namespace) + log.Infof(output) log.FailOnError(err, "More about error %s", output) if skipSourceOp { From 9f1be233850482448595986a61b905ff271590ff Mon Sep 17 00:00:00 2001 From: apattewar-px Date: Fri, 23 Aug 2024 19:45:13 +0530 Subject: [PATCH 03/11] Data integrity for AsyncDR --- tests/basic/async_dr_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/basic/async_dr_test.go b/tests/basic/async_dr_test.go index ff6e70c86..43bef8662 100644 --- a/tests/basic/async_dr_test.go +++ b/tests/basic/async_dr_test.go @@ -1174,6 +1174,11 @@ func validateFailoverFailbackWithDataIntegrity(clusterType, taskNamePrefix strin // Verify no err migNamespaces := strings.Join(migrationNamespaces, ",") + + pod, err := k8sCore.GetPodByName(contexts[0].App.Key, migNamespaces) + log.FailOnError(err, "Failed to get pod name for fio app") + log.Infof(pod.Name) + kubeConfigPathSrc, err := GetCustomClusterConfigPath(asyncdr.FirstCluster) log.FailOnError(err, "Failed to get source configPath: %v", err) kubeConfigPathDest, err := GetCustomClusterConfigPath(asyncdr.SecondCluster) @@ -1221,9 +1226,6 @@ func validateFailoverFailbackWithDataIntegrity(clusterType, taskNamePrefix strin contexts: contexts, } performFailoverFailback(failoverParam) - - pod, err := k8sCore.GetPodByName("fio-async-dr", migNamespaces) - log.Infof(pod.Name) cmd := []string{"fio", "--blocksize=32k", "--directory=/data", "--filename=test", "--ioengine=libaio", "--readwrite=read", "--size=5120M", "--name=test", "--verify=meta", "--do_verify=1", "--verify_pattern=0xDeadBeef", "--direct=1", "--randrepeat=1", "--output=fio-log_read.txt;", "cat", "fio-log_read.txt"} output, err := core.Instance().RunCommandInPod(cmd, pod.Name, "compute", pod.Namespace) log.Infof(output) From 4b283bc5c6e5dda35719e6abddf9221836832a2d Mon Sep 17 00:00:00 2001 From: apattewar-px Date: Fri, 23 Aug 2024 20:04:15 +0530 Subject: [PATCH 04/11] Data integrity for AsyncDR --- tests/basic/async_dr_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/basic/async_dr_test.go b/tests/basic/async_dr_test.go index 43bef8662..e06437fda 100644 --- a/tests/basic/async_dr_test.go +++ b/tests/basic/async_dr_test.go @@ -1175,7 +1175,6 @@ func validateFailoverFailbackWithDataIntegrity(clusterType, taskNamePrefix strin migNamespaces := strings.Join(migrationNamespaces, ",") - pod, err := k8sCore.GetPodByName(contexts[0].App.Key, migNamespaces) log.FailOnError(err, "Failed to get pod name for fio app") log.Infof(pod.Name) @@ -1226,8 +1225,13 @@ func validateFailoverFailbackWithDataIntegrity(clusterType, taskNamePrefix strin contexts: contexts, } performFailoverFailback(failoverParam) + + podList, _ := core.Instance().GetPods(migNamespaces, nil) + log.Infof(podList.Items[0].Name) + firstPod := podList.Items[0].Name + //pod, err := k8sCore.GetPodByName(podlist, migNamespaces) cmd := []string{"fio", "--blocksize=32k", "--directory=/data", "--filename=test", "--ioengine=libaio", "--readwrite=read", "--size=5120M", "--name=test", "--verify=meta", "--do_verify=1", "--verify_pattern=0xDeadBeef", "--direct=1", "--randrepeat=1", "--output=fio-log_read.txt;", "cat", "fio-log_read.txt"} - output, err := core.Instance().RunCommandInPod(cmd, pod.Name, "compute", pod.Namespace) + output, err := core.Instance().RunCommandInPod(cmd, firstPod, "compute", migNamespaces) log.Infof(output) log.FailOnError(err, "More about error %s", output) From c65f8d96f911a8703171f48900c0acf5b5b8d449 Mon Sep 17 00:00:00 2001 From: apattewar-px Date: Mon, 26 Aug 2024 00:39:02 +0530 Subject: [PATCH 05/11] Async DR with data integrity check --- tests/basic/async_dr_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/basic/async_dr_test.go b/tests/basic/async_dr_test.go index e06437fda..da74a8e90 100644 --- a/tests/basic/async_dr_test.go +++ b/tests/basic/async_dr_test.go @@ -1175,9 +1175,6 @@ func validateFailoverFailbackWithDataIntegrity(clusterType, taskNamePrefix strin migNamespaces := strings.Join(migrationNamespaces, ",") - log.FailOnError(err, "Failed to get pod name for fio app") - log.Infof(pod.Name) - kubeConfigPathSrc, err := GetCustomClusterConfigPath(asyncdr.FirstCluster) log.FailOnError(err, "Failed to get source configPath: %v", err) kubeConfigPathDest, err := GetCustomClusterConfigPath(asyncdr.SecondCluster) From 56ddefab96c57dc6cc38c8418c836df3bc695068 Mon Sep 17 00:00:00 2001 From: apattewar-px Date: Mon, 26 Aug 2024 20:22:57 +0530 Subject: [PATCH 06/11] Data integrity for AsyncDR --- tests/basic/async_dr_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/basic/async_dr_test.go b/tests/basic/async_dr_test.go index da74a8e90..996e06393 100644 --- a/tests/basic/async_dr_test.go +++ b/tests/basic/async_dr_test.go @@ -1227,8 +1227,8 @@ func validateFailoverFailbackWithDataIntegrity(clusterType, taskNamePrefix strin log.Infof(podList.Items[0].Name) firstPod := podList.Items[0].Name //pod, err := k8sCore.GetPodByName(podlist, migNamespaces) - cmd := []string{"fio", "--blocksize=32k", "--directory=/data", "--filename=test", "--ioengine=libaio", "--readwrite=read", "--size=5120M", "--name=test", "--verify=meta", "--do_verify=1", "--verify_pattern=0xDeadBeef", "--direct=1", "--randrepeat=1", "--output=fio-log_read.txt;", "cat", "fio-log_read.txt"} - output, err := core.Instance().RunCommandInPod(cmd, firstPod, "compute", migNamespaces) + cmd := []string{"fio", "--blocksize=32k", "--directory=/data", "--filename=test", "--ioengine=libaio", "--readwrite=read", "--size=5120M", "--name=test", "--verify=meta", "--do_verify=1", "--verify_pattern=0xDeadBeef", "--direct=1", "--randrepeat=1", "--output=fio-log_read.txt;", "cat", "fio-log_read.txt;", "sleep 900"} + output, err := core.Instance().RunCommandInPod(cmd, firstPod, "", migNamespaces) log.Infof(output) log.FailOnError(err, "More about error %s", output) From 12262c416c19596415774e0e7945aff709c95032 Mon Sep 17 00:00:00 2001 From: apattewar-px Date: Mon, 26 Aug 2024 20:38:59 +0530 Subject: [PATCH 07/11] Data integrity for AsyncDR --- tests/basic/async_dr_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/basic/async_dr_test.go b/tests/basic/async_dr_test.go index 996e06393..32dc4dd83 100644 --- a/tests/basic/async_dr_test.go +++ b/tests/basic/async_dr_test.go @@ -1227,7 +1227,7 @@ func validateFailoverFailbackWithDataIntegrity(clusterType, taskNamePrefix strin log.Infof(podList.Items[0].Name) firstPod := podList.Items[0].Name //pod, err := k8sCore.GetPodByName(podlist, migNamespaces) - cmd := []string{"fio", "--blocksize=32k", "--directory=/data", "--filename=test", "--ioengine=libaio", "--readwrite=read", "--size=5120M", "--name=test", "--verify=meta", "--do_verify=1", "--verify_pattern=0xDeadBeef", "--direct=1", "--randrepeat=1", "--output=fio-log_read.txt;", "cat", "fio-log_read.txt;", "sleep 900"} + cmd := []string{"fio", "--blocksize=32k", "--directory=/data", "--filename=test", "--ioengine=libaio", "--readwrite=read", "--size=5120M", "--name=test", "--verify=meta", "--do_verify=1", "--verify_pattern=0xDeadBeef", "--direct=1", "--randrepeat=1", "--output=fio-log_read.txt"} output, err := core.Instance().RunCommandInPod(cmd, firstPod, "", migNamespaces) log.Infof(output) log.FailOnError(err, "More about error %s", output) From 2bcc8d2ef64b1ba12387717924d4a54673f0e9cc Mon Sep 17 00:00:00 2001 From: apattewar-px Date: Mon, 26 Aug 2024 21:20:43 +0530 Subject: [PATCH 08/11] Data integrity for AsyncDR --- tests/basic/async_dr_test.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/basic/async_dr_test.go b/tests/basic/async_dr_test.go index 32dc4dd83..7658b3a97 100644 --- a/tests/basic/async_dr_test.go +++ b/tests/basic/async_dr_test.go @@ -1226,11 +1226,18 @@ func validateFailoverFailbackWithDataIntegrity(clusterType, taskNamePrefix strin podList, _ := core.Instance().GetPods(migNamespaces, nil) log.Infof(podList.Items[0].Name) firstPod := podList.Items[0].Name - //pod, err := k8sCore.GetPodByName(podlist, migNamespaces) - cmd := []string{"fio", "--blocksize=32k", "--directory=/data", "--filename=test", "--ioengine=libaio", "--readwrite=read", "--size=5120M", "--name=test", "--verify=meta", "--do_verify=1", "--verify_pattern=0xDeadBeef", "--direct=1", "--randrepeat=1", "--output=fio-log_read.txt"} - output, err := core.Instance().RunCommandInPod(cmd, firstPod, "", migNamespaces) - log.Infof(output) - log.FailOnError(err, "More about error %s", output) + + //Generate read result file + cmd1 := []string{"fio", "--blocksize=32k", "--directory=/data", "--filename=test", "--ioengine=libaio", "--readwrite=read", "--size=5120M", "--name=test", "--verify=meta", "--do_verify=1", "--verify_pattern=0xDeadBeef", "--direct=1", "--randrepeat=1", "--output=fio-log_read.txt"} + output1, err := core.Instance().RunCommandInPod(cmd1, firstPod, "", migNamespaces) + log.Infof(output1) + log.FailOnError(err, "More about error %s", output1) + + //Check for err after comparison + cmd2 := []string{"egrep", "-i", "err", "fio-log_read.txt"} + output2, err := core.Instance().RunCommandInPod(cmd2, firstPod, "", migNamespaces) + log.Infof(output2) + log.FailOnError(err, "More about error %s", output2) if skipSourceOp { err = hardSetConfig(kubeConfigPathSrc) From 6313d352eb27ce82eaad1ba183a5eeb85340ac86 Mon Sep 17 00:00:00 2001 From: apattewar-px Date: Tue, 27 Aug 2024 10:30:35 +0530 Subject: [PATCH 09/11] Data integrity for AsyncDR --- tests/basic/async_dr_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/basic/async_dr_test.go b/tests/basic/async_dr_test.go index 7658b3a97..fea25c905 100644 --- a/tests/basic/async_dr_test.go +++ b/tests/basic/async_dr_test.go @@ -1229,15 +1229,16 @@ func validateFailoverFailbackWithDataIntegrity(clusterType, taskNamePrefix strin //Generate read result file cmd1 := []string{"fio", "--blocksize=32k", "--directory=/data", "--filename=test", "--ioengine=libaio", "--readwrite=read", "--size=5120M", "--name=test", "--verify=meta", "--do_verify=1", "--verify_pattern=0xDeadBeef", "--direct=1", "--randrepeat=1", "--output=fio-log_read.txt"} + //log.Infof(cmd1) output1, err := core.Instance().RunCommandInPod(cmd1, firstPod, "", migNamespaces) log.Infof(output1) - log.FailOnError(err, "More about error %s", output1) + log.FailOnError(err, "More about error %s", err) //Check for err after comparison - cmd2 := []string{"egrep", "-i", "err", "fio-log_read.txt"} + cmd2 := []string{"egrep", "-i", "err", "/data/fio-log_read.txt"} output2, err := core.Instance().RunCommandInPod(cmd2, firstPod, "", migNamespaces) log.Infof(output2) - log.FailOnError(err, "More about error %s", output2) + log.FailOnError(err, "More about error %s", err) if skipSourceOp { err = hardSetConfig(kubeConfigPathSrc) From fcde10cddf52a04a38b52ee41553758f3c378c13 Mon Sep 17 00:00:00 2001 From: apattewar-px Date: Tue, 27 Aug 2024 14:49:58 +0530 Subject: [PATCH 10/11] Data integrity for AsyncDR --- tests/basic/async_dr_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/basic/async_dr_test.go b/tests/basic/async_dr_test.go index fea25c905..fa7f2dcec 100644 --- a/tests/basic/async_dr_test.go +++ b/tests/basic/async_dr_test.go @@ -1228,6 +1228,7 @@ func validateFailoverFailbackWithDataIntegrity(clusterType, taskNamePrefix strin firstPod := podList.Items[0].Name //Generate read result file + //Data integrity code cmd1 := []string{"fio", "--blocksize=32k", "--directory=/data", "--filename=test", "--ioengine=libaio", "--readwrite=read", "--size=5120M", "--name=test", "--verify=meta", "--do_verify=1", "--verify_pattern=0xDeadBeef", "--direct=1", "--randrepeat=1", "--output=fio-log_read.txt"} //log.Infof(cmd1) output1, err := core.Instance().RunCommandInPod(cmd1, firstPod, "", migNamespaces) @@ -1235,7 +1236,7 @@ func validateFailoverFailbackWithDataIntegrity(clusterType, taskNamePrefix strin log.FailOnError(err, "More about error %s", err) //Check for err after comparison - cmd2 := []string{"egrep", "-i", "err", "/data/fio-log_read.txt"} + cmd2 := []string{"egrep", "err=[0-9]+", "/data/fio-log_read.txt"} output2, err := core.Instance().RunCommandInPod(cmd2, firstPod, "", migNamespaces) log.Infof(output2) log.FailOnError(err, "More about error %s", err) From 4578e7f0d225104e864f2ed103132ad014ba46d5 Mon Sep 17 00:00:00 2001 From: apattewar-px Date: Tue, 27 Aug 2024 15:33:23 +0530 Subject: [PATCH 11/11] Data integrity for AsyncDR --- drivers/scheduler/k8s/specs/fio-async-dr/fio-pvc.yaml | 2 +- drivers/scheduler/k8s/specs/fio-async-dr/fio.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/scheduler/k8s/specs/fio-async-dr/fio-pvc.yaml b/drivers/scheduler/k8s/specs/fio-async-dr/fio-pvc.yaml index cec3ead75..f77b4f403 100644 --- a/drivers/scheduler/k8s/specs/fio-async-dr/fio-pvc.yaml +++ b/drivers/scheduler/k8s/specs/fio-async-dr/fio-pvc.yaml @@ -7,5 +7,5 @@ spec: - ReadWriteMany resources: requests: - storage: 5Gi + storage: 20Gi storageClassName: portworx-sc \ No newline at end of file diff --git a/drivers/scheduler/k8s/specs/fio-async-dr/fio.yaml b/drivers/scheduler/k8s/specs/fio-async-dr/fio.yaml index ef6a97f0f..496a8b1f2 100644 --- a/drivers/scheduler/k8s/specs/fio-async-dr/fio.yaml +++ b/drivers/scheduler/k8s/specs/fio-async-dr/fio.yaml @@ -20,7 +20,7 @@ spec: args: - | fio --blocksize=32k --directory=/data --filename=test \ - --ioengine=libaio --readwrite=write --size=5120M --name=test \ + --ioengine=libaio --readwrite=write --size=512M --name=test \ --verify=meta --do_verify=1 --verify_pattern=0xDeadBeef --direct=1 \ --gtod_reduce=1 --iodepth=32 --rate_iops=4000 --randrepeat=1 \ --disable_lat=0