From 5f071d6fd0e52c3ee19f2eb195151e906c291a05 Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sat, 26 Aug 2023 16:23:30 +0800 Subject: [PATCH 01/13] for NodeUnstage/NodePublish/NodeUnpublish, also need to retry on Failed volume Signed-off-by: Shi, Crane --- pkg/node/node.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/node/node.go b/pkg/node/node.go index 0f30f6576..315b037ce 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -353,7 +353,9 @@ func (s *CSINodeService) NodeUnstageVolume(ctx context.Context, req *csi.NodeUns } currStatus := volumeCR.Spec.CSIStatus - if currStatus == apiV1.Created { + if currStatus == apiV1.Failed { + ll.Warningf("Volume status: %s. Need to retry.", currStatus) + } else if currStatus == apiV1.Created { ll.Info("Volume has been already unstaged") return &csi.NodeUnstageVolumeResponse{}, nil } else if currStatus != apiV1.VolumeReady { @@ -468,8 +470,9 @@ func (s *CSINodeService) NodePublishVolume(ctx context.Context, req *csi.NodePub } currStatus := volumeCR.Spec.CSIStatus - // if currStatus not in [VolumeReady, Published] - if currStatus != apiV1.VolumeReady && currStatus != apiV1.Published { + if currStatus == apiV1.Failed { + ll.Warningf("Volume status: %s. Need to retry.", currStatus) + } else if currStatus != apiV1.VolumeReady && currStatus != apiV1.Published { msg := fmt.Sprintf("current volume CR status - %s, expected to be in [%s, %s]", currStatus, apiV1.VolumeReady, apiV1.Published) ll.Error(msg) @@ -554,8 +557,9 @@ func (s *CSINodeService) NodeUnpublishVolume(ctx context.Context, req *csi.NodeU } currStatus := volumeCR.Spec.CSIStatus - // if currStatus not in [VolumeReady, Published] - if currStatus != apiV1.VolumeReady && currStatus != apiV1.Published { + if currStatus == apiV1.Failed { + ll.Warningf("Volume status: %s. Need to retry.", currStatus) + } else if currStatus != apiV1.VolumeReady && currStatus != apiV1.Published { msg := fmt.Sprintf("current volume CR status - %s, expected to be in [%s, %s]", currStatus, apiV1.VolumeReady, apiV1.Published) ll.Error(msg) From 2e6ab7f29483dc87a864112ac7f23ebafab82d4e Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sat, 26 Aug 2023 16:42:45 +0800 Subject: [PATCH 02/13] fix lint error Signed-off-by: Shi, Crane --- pkg/node/node.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/node/node.go b/pkg/node/node.go index 315b037ce..f4bba5783 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -353,12 +353,15 @@ func (s *CSINodeService) NodeUnstageVolume(ctx context.Context, req *csi.NodeUns } currStatus := volumeCR.Spec.CSIStatus - if currStatus == apiV1.Failed { + switch currStatus { + case apiV1.Failed: ll.Warningf("Volume status: %s. Need to retry.", currStatus) - } else if currStatus == apiV1.Created { + case apiV1.Created: ll.Info("Volume has been already unstaged") return &csi.NodeUnstageVolumeResponse{}, nil - } else if currStatus != apiV1.VolumeReady { + case apiV1.VolumeReady: + ll.Infof("Expected volume status: %s", currStatus) + default: msg := fmt.Sprintf("current volume CR status - %s, expected to be in [%s, %s]", currStatus, apiV1.Created, apiV1.VolumeReady) ll.Error(msg) From fdc5970ece28f2917b1733d54176a1427767d5d7 Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sun, 27 Aug 2023 02:14:36 +0800 Subject: [PATCH 03/13] fix UT Signed-off-by: Shi, Crane --- pkg/node/node_test.go | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go index ad4e6e7cb..37bccc5ec 100644 --- a/pkg/node/node_test.go +++ b/pkg/node/node_test.go @@ -101,7 +101,9 @@ var _ = Describe("CSINodeService NodePublish()", func() { Expect(err).To(BeNil()) Expect(volumeCR.Spec.Owners[0]).To(Equal(testPod1Name)) - // publish again such volume + // publish again such volume in Failed Status + volumeCR.Spec.CSIStatus = apiV1.Failed + err = node.k8sClient.UpdateCR(testCtx, volumeCR) resp, err = node.NodePublishVolume(testCtx, req) Expect(resp).NotTo(BeNil()) Expect(err).To(BeNil()) @@ -111,6 +113,7 @@ var _ = Describe("CSINodeService NodePublish()", func() { err = node.k8sClient.ReadCR(testCtx, testV1ID, "", volumeCR) Expect(err).To(BeNil()) Expect(len(volumeCR.Spec.Owners)).To(Equal(1)) + Expect(volumeCR.Spec.CSIStatus).To(Equal(apiV1.Published)) }) }) @@ -157,19 +160,6 @@ var _ = Describe("CSINodeService NodePublish()", func() { Expect(err).NotTo(BeNil()) Expect(err.Error()).To(ContainSubstring("Staging Path missing in request")) }) - It("Should fail, because Volume has failed status", func() { - req := getNodePublishRequest(testV1ID, targetPath, *testVolumeCap) - vol1 := &vcrd.Volume{} - err := node.k8sClient.ReadCR(testCtx, testVolume1.Id, testNs, vol1) - Expect(err).To(BeNil()) - vol1.Spec.CSIStatus = apiV1.Failed - err = node.k8sClient.UpdateCR(testCtx, vol1) - Expect(err).To(BeNil()) - - resp, err := node.NodePublishVolume(testCtx, req) - Expect(resp).To(BeNil()) - Expect(err).NotTo(BeNil()) - }) It("Should fail, because of volume CR isn't exist", func() { req := getNodePublishRequest(testV1ID, targetPath, *testVolumeCap) err := node.k8sClient.DeleteCR(testCtx, &testVolumeCR1) @@ -506,6 +496,18 @@ var _ = Describe("CSINodeService NodeUnStage()", func() { err = node.k8sClient.ReadCR(testCtx, testV1ID, "", volumeCR) Expect(err).To(BeNil()) Expect(volumeCR.Spec.CSIStatus).To(Equal(apiV1.Created)) + + // retry unstage on failed volume + volumeCR.Spec.CSIStatus = apiV1.Failed + err = node.k8sClient.UpdateCR(testCtx, volumeCR) + Expect(err).To(BeNil()) + resp, err = node.NodeUnstageVolume(testCtx, req) + Expect(resp).NotTo(BeNil()) + Expect(err).To(BeNil()) + + err = node.k8sClient.ReadCR(testCtx, testV1ID, "", volumeCR) + Expect(err).To(BeNil()) + Expect(volumeCR.Spec.CSIStatus).To(Equal(apiV1.Created)) }) }) From a50a7f1259d7a1635f1419be0157b5cf4fcee2c6 Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sun, 27 Aug 2023 02:17:28 +0800 Subject: [PATCH 04/13] add UT Signed-off-by: Shi, Crane --- pkg/node/node_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go index 37bccc5ec..0e2c05dc0 100644 --- a/pkg/node/node_test.go +++ b/pkg/node/node_test.go @@ -160,6 +160,19 @@ var _ = Describe("CSINodeService NodePublish()", func() { Expect(err).NotTo(BeNil()) Expect(err.Error()).To(ContainSubstring("Staging Path missing in request")) }) + It("Should fail, because Volume has failed status", func() { + req := getNodePublishRequest(testV1ID, targetPath, *testVolumeCap) + vol1 := &vcrd.Volume{} + err := node.k8sClient.ReadCR(testCtx, testVolume1.Id, testNs, vol1) + Expect(err).To(BeNil()) + vol1.Spec.CSIStatus = apiV1.Failed + err = node.k8sClient.UpdateCR(testCtx, vol1) + Expect(err).To(BeNil()) + + resp, err := node.NodePublishVolume(testCtx, req) + Expect(resp).To(BeNil()) + Expect(err).NotTo(BeNil()) + }) It("Should fail, because of volume CR isn't exist", func() { req := getNodePublishRequest(testV1ID, targetPath, *testVolumeCap) err := node.k8sClient.DeleteCR(testCtx, &testVolumeCR1) From bfb1d983c09e2c4fdb9a6f4d3c3aa8b2946fd1ae Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sun, 27 Aug 2023 02:29:07 +0800 Subject: [PATCH 05/13] add UT Signed-off-by: Shi, Crane --- pkg/node/node_test.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go index 0e2c05dc0..0099757b1 100644 --- a/pkg/node/node_test.go +++ b/pkg/node/node_test.go @@ -160,12 +160,12 @@ var _ = Describe("CSINodeService NodePublish()", func() { Expect(err).NotTo(BeNil()) Expect(err.Error()).To(ContainSubstring("Staging Path missing in request")) }) - It("Should fail, because Volume has failed status", func() { + It("Should fail, because Volume has unexpected status", func() { req := getNodePublishRequest(testV1ID, targetPath, *testVolumeCap) vol1 := &vcrd.Volume{} err := node.k8sClient.ReadCR(testCtx, testVolume1.Id, testNs, vol1) Expect(err).To(BeNil()) - vol1.Spec.CSIStatus = apiV1.Failed + vol1.Spec.CSIStatus = apiV1.Creating err = node.k8sClient.UpdateCR(testCtx, vol1) Expect(err).To(BeNil()) @@ -367,6 +367,19 @@ var _ = Describe("CSINodeService NodeUnPublish()", func() { Expect(err).To(BeNil()) Expect(volumeCR.Spec.CSIStatus).To(Equal(apiV1.VolumeReady)) Expect(volumeCR.Spec.Owners).To(BeNil()) + + // unpublish again on failed volume + volumeCR.Spec.CSIStatus = apiV1.Failed + err = node.k8sClient.UpdateCR(testCtx, volumeCR) + Expect(err).To(BeNil()) + + resp, err = node.NodeUnpublishVolume(testCtx, req) + Expect(resp).NotTo(BeNil()) + Expect(err).To(BeNil()) + + err = node.k8sClient.ReadCR(testCtx, testV1ID, "", volumeCR) + Expect(err).To(BeNil()) + Expect(volumeCR.Spec.CSIStatus).To(Equal(apiV1.VolumeReady)) }) It("Should unpublish volume and don't change volume CR status", func() { req := getNodeUnpublishRequest(testV1ID, targetPath1) @@ -579,12 +592,12 @@ var _ = Describe("CSINodeService NodeUnStage()", func() { Expect(volumeCR.Spec.CSIStatus).To(Equal(apiV1.Failed)) }) - It("Should failed, because Volume has failed status", func() { + It("Should failed, because Volume has unexpected status", func() { req := getNodeUnstageRequest(testV1ID, targetPath) vol1 := &vcrd.Volume{} err := node.k8sClient.ReadCR(testCtx, testVolume1.Id, testNs, vol1) Expect(err).To(BeNil()) - vol1.Spec.CSIStatus = apiV1.Failed + vol1.Spec.CSIStatus = apiV1.Creating err = node.k8sClient.UpdateCR(testCtx, vol1) Expect(err).To(BeNil()) From 1a77d4399d3ad3fb2d86ee11f938d082a3807dba Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sun, 27 Aug 2023 04:05:08 +0800 Subject: [PATCH 06/13] Redo NodeStageVolume if srcPath is not mounted Signed-off-by: Shi, Crane --- pkg/node/node.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pkg/node/node.go b/pkg/node/node.go index f4bba5783..e49c6cc50 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -494,6 +494,27 @@ func (s *CSINodeService) NodePublishVolume(ctx context.Context, req *csi.NodePub resp, errToReturn = nil, fmt.Errorf("failed to publish volume: fake attach error %s", err.Error()) } } else { + // will check whether srcPath is mounted, if not, need to redo NodeStageVolume + srcMounted, err := s.fsOps.IsMounted(srcPath) + if err != nil { + errMsg := fmt.Sprintf("execute IsMounted on %s with error: %s", srcPath, err.Error()) + ll.Error(errMsg) + return nil, fmt.Errorf("failed to publish volume: %s", errMsg) + } + if !srcMounted { + nodeStageReq := &csi.NodeStageVolumeRequest{ + VolumeId: volumeID, + StagingTargetPath: stagePath, + VolumeCapability: req.GetVolumeCapability(), + } + nodeStageResp, err := s.NodeStageVolume(ctx, nodeStageReq) + if nodeStageResp == nil && err != nil { + errMsg := fmt.Sprintf("redo NodeStageVolume on volume %s with error: %s", volumeID, err.Error()) + ll.Error(errMsg) + return nil, fmt.Errorf("failed to publish volume: %s", errMsg) + } + } + _, isBlock := req.GetVolumeCapability().GetAccessType().(*csi.VolumeCapability_Block) if err := s.fsOps.PrepareAndPerformMount(srcPath, dstPath, isBlock, !isBlock, mountOptions...); err != nil { ll.Errorf("Unable to mount volume: %v", err) From d36e785019829224b11c1594795c046ed3ce5abe Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sun, 27 Aug 2023 04:12:38 +0800 Subject: [PATCH 07/13] correct stagingTargetPath Signed-off-by: Shi, Crane --- pkg/node/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/node/node.go b/pkg/node/node.go index e49c6cc50..29ac4c9a5 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -504,7 +504,7 @@ func (s *CSINodeService) NodePublishVolume(ctx context.Context, req *csi.NodePub if !srcMounted { nodeStageReq := &csi.NodeStageVolumeRequest{ VolumeId: volumeID, - StagingTargetPath: stagePath, + StagingTargetPath: req.GetStagingTargetPath(), VolumeCapability: req.GetVolumeCapability(), } nodeStageResp, err := s.NodeStageVolume(ctx, nodeStageReq) From 4376b109d14acddec5793e637ce07b5581dd5803 Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sun, 27 Aug 2023 15:30:03 +0800 Subject: [PATCH 08/13] add warning log for redo NodeStageVolume Signed-off-by: Shi, Crane --- pkg/node/node.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/node/node.go b/pkg/node/node.go index 29ac4c9a5..f9512004a 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -502,6 +502,7 @@ func (s *CSINodeService) NodePublishVolume(ctx context.Context, req *csi.NodePub return nil, fmt.Errorf("failed to publish volume: %s", errMsg) } if !srcMounted { + ll.Warnf("staging path %s is not mounted! need to redo NodeStageVolume!", srcPath) nodeStageReq := &csi.NodeStageVolumeRequest{ VolumeId: volumeID, StagingTargetPath: req.GetStagingTargetPath(), From c835b96a5d94936800c8a5f63652de39af834234 Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sun, 27 Aug 2023 15:37:04 +0800 Subject: [PATCH 09/13] fix that, in multi-pod case, successful unpublish volume cannot resume failed volume status Signed-off-by: Shi, Crane --- pkg/node/node.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/node/node.go b/pkg/node/node.go index f9512004a..134fc3553 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -631,6 +631,9 @@ func (s *CSINodeService) NodeUnpublishVolume(ctx context.Context, req *csi.NodeU volumeCR.Spec.Owners = owners if len(volumeCR.Spec.Owners) == 0 { volumeCR.Spec.CSIStatus = apiV1.VolumeReady + } else { + // ensure the Published status of volume in the successful processing + volumeCR.Spec.CSIStatus = apiV1.Published } if updateErr := s.k8sClient.UpdateCR(ctxWithID, volumeCR); updateErr != nil { ll.Errorf("Unable to set volume CR status to VolumeReady: %v", updateErr) From a3abfb9beda0c8a447b8daf618313836b8428308 Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sun, 27 Aug 2023 16:29:22 +0800 Subject: [PATCH 10/13] unlock volume to redo NodeStageVolume if applicable in NodePublishVolume Signed-off-by: Shi, Crane --- pkg/node/node.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/node/node.go b/pkg/node/node.go index 134fc3553..bbfa9bbc6 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -508,7 +508,19 @@ func (s *CSINodeService) NodePublishVolume(ctx context.Context, req *csi.NodePub StagingTargetPath: req.GetStagingTargetPath(), VolumeCapability: req.GetVolumeCapability(), } + + // unlock volume to redo NodeStageVolume + err := s.volMu.UnlockKey(req.GetVolumeId()) + if err != nil { + errMsg := fmt.Sprintf("unlock volume %s to redo NodeStageVolume with error: %s", volumeID, err.Error()) + ll.Error(errMsg) + return nil, fmt.Errorf("failed to publish volume: %s", errMsg) + } nodeStageResp, err := s.NodeStageVolume(ctx, nodeStageReq) + + // re-lock the volume to proceed NodePublishVolume + s.volMu.LockKey(req.GetVolumeId()) + if nodeStageResp == nil && err != nil { errMsg := fmt.Sprintf("redo NodeStageVolume on volume %s with error: %s", volumeID, err.Error()) ll.Error(errMsg) From af81d9afb8c68b77112d7acc856091214707c4ed Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sun, 27 Aug 2023 17:26:07 +0800 Subject: [PATCH 11/13] fix issue of outdated volume Signed-off-by: Shi, Crane --- pkg/node/node.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/node/node.go b/pkg/node/node.go index bbfa9bbc6..eff682ca8 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -526,6 +526,12 @@ func (s *CSINodeService) NodePublishVolume(ctx context.Context, req *csi.NodePub ll.Error(errMsg) return nil, fmt.Errorf("failed to publish volume: %s", errMsg) } + + // update the content of volume + volumeCR, err = s.crHelper.GetVolumeByID(volumeID) + if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("unable to get updated volume %s", volumeID)) + } } _, isBlock := req.GetVolumeCapability().GetAccessType().(*csi.VolumeCapability_Block) From 7ecd9539fb8981fcc671cd0a23bf0340c9e887d9 Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Mon, 28 Aug 2023 22:50:08 +0800 Subject: [PATCH 12/13] fix and add UT Signed-off-by: Shi, Crane --- pkg/node/node_test.go | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go index 0099757b1..1c06635b7 100644 --- a/pkg/node/node_test.go +++ b/pkg/node/node_test.go @@ -87,9 +87,9 @@ var _ = Describe("CSINodeService NodePublish()", func() { req := getNodePublishRequest(testV1ID, targetPath, *testVolumeCap) req.VolumeContext[util.PodNameKey] = testPod1Name - fsOps.On("PrepareAndPerformMount", - path.Join(req.GetStagingTargetPath(), stagingFileName), req.GetTargetPath(), false, true). - Return(nil) + srcPath := path.Join(req.GetStagingTargetPath(), stagingFileName) + fsOps.On("PrepareAndPerformMount", srcPath, req.GetTargetPath(), false, true).Return(nil) + fsOps.On("IsMounted", srcPath).Return(true, nil) resp, err := node.NodePublishVolume(testCtx, req) Expect(resp).NotTo(BeNil()) @@ -187,15 +187,27 @@ var _ = Describe("CSINodeService NodePublish()", func() { It("Should fail, because of PrepareAndPerformMount failed", func() { req := getNodePublishRequest(testV1ID, targetPath, *testVolumeCap) - fsOps.On("PrepareAndPerformMount", - path.Join(req.GetStagingTargetPath(), stagingFileName), req.GetTargetPath(), false, true). - Return(errors.New("error mount")) + srcPath := path.Join(req.GetStagingTargetPath(), stagingFileName) + fsOps.On("PrepareAndPerformMount", srcPath, req.GetTargetPath(), false, true).Return(errors.New("error mount")) + fsOps.On("IsMounted", srcPath).Return(true, nil) resp, err := node.NodePublishVolume(testCtx, req) Expect(resp).To(BeNil()) Expect(err).NotTo(BeNil()) Expect(err.Error()).To(ContainSubstring("mount error")) }) + It("Should fail, because the check on whether stagingPath is mounted failed", func() { + req := getNodePublishRequest(testV1ID, targetPath, *testVolumeCap) + + srcPath := path.Join(req.GetStagingTargetPath(), stagingFileName) + errMsg := fmt.Sprintf("unable to check whether %s is mounted", srcPath) + fsOps.On("IsMounted", srcPath).Return(false, errors.New(errMsg)) + + resp, err := node.NodePublishVolume(testCtx, req) + Expect(resp).To(BeNil()) + Expect(err).NotTo(BeNil()) + Expect(err.Error()).To(ContainSubstring(errMsg)) + }) }) }) From 330960f099ce5c8b565251940f5d071313497ddf Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Mon, 28 Aug 2023 23:50:51 +0800 Subject: [PATCH 13/13] add UT Signed-off-by: Shi, Crane --- pkg/node/node_test.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go index 1c06635b7..9a06aadaf 100644 --- a/pkg/node/node_test.go +++ b/pkg/node/node_test.go @@ -115,6 +115,28 @@ var _ = Describe("CSINodeService NodePublish()", func() { Expect(len(volumeCR.Spec.Owners)).To(Equal(1)) Expect(volumeCR.Spec.CSIStatus).To(Equal(apiV1.Published)) }) + It("Should publish volume successfully with stagingPath unmounted", func() { + req := getNodePublishRequest(testV1ID, targetPath, *testVolumeCap) + req.VolumeContext[util.PodNameKey] = testPod1Name + + partitionPath := "/partition/path/for/volume1" + stagingPath := path.Join(req.GetStagingTargetPath(), stagingFileName) + prov.On("GetVolumePath", &testVolume1).Return(partitionPath, nil) + fsOps.On("PrepareAndPerformMount", partitionPath, stagingPath, true, false).Return(nil) + + fsOps.On("PrepareAndPerformMount", stagingPath, req.GetTargetPath(), false, true).Return(nil) + fsOps.On("IsMounted", stagingPath).Return(false, nil) + + resp, err := node.NodePublishVolume(testCtx, req) + Expect(resp).NotTo(BeNil()) + Expect(err).To(BeNil()) + + // check owner appearance + volumeCR := &vcrd.Volume{} + err = node.k8sClient.ReadCR(testCtx, testV1ID, "", volumeCR) + Expect(err).To(BeNil()) + Expect(volumeCR.Spec.Owners[0]).To(Equal(testPod1Name)) + }) }) Context("NodePublish() failure", func() { @@ -208,6 +230,23 @@ var _ = Describe("CSINodeService NodePublish()", func() { Expect(err).NotTo(BeNil()) Expect(err.Error()).To(ContainSubstring(errMsg)) }) + It("Should fail, because redoing NodeStageVolume when stagingPath unmounted failed", func() { + req := getNodePublishRequest(testV1ID, targetPath, *testVolumeCap) + req.VolumeContext[util.PodNameKey] = testPod1Name + + partitionPath := "/partition/path/for/volume1" + stagingPath := path.Join(req.GetStagingTargetPath(), stagingFileName) + prov.On("GetVolumePath", &testVolume1).Return(partitionPath, nil) + fsOps.On("PrepareAndPerformMount", partitionPath, stagingPath, true, false).Return(errors.New("mount error")) + fsOps.On("IsMounted", stagingPath).Return(false, nil) + + redoErr := fmt.Sprintf("redo NodeStageVolume on volume %s with error", testV1ID) + + resp, err := node.NodePublishVolume(testCtx, req) + Expect(resp).To(BeNil()) + Expect(err).NotTo(BeNil()) + Expect(err.Error()).To(ContainSubstring(redoErr)) + }) }) })