fix: add retry for kill pod

Signed-off-by: henrywangx <[email protected]>

cmd/argoexec/commands/wait.go

@@ -4,9 +4,13 @@ import (
 	"context"
 	"time"
 
+	errorsutil "github.com/argoproj/argo-workflows/v3/util/errors"
 	"github.com/argoproj/pkg/stats"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/argoproj/argo-workflows/v3/workflow/executor"
 )
 
 func NewWaitCommand() *cobra.Command {
@@ -31,7 +35,21 @@ func waitContainer(ctx context.Context) error {
 	stats.StartStatsTicker(5 * time.Minute)
 
 	defer func() {
-		if err := wfExecutor.KillSidecars(ctx); err != nil {
+		// Killing sidecar containers
+		retryCnt := 0
+		err := wait.ExponentialBackoff(executor.ExecutorRetry, func() (bool, error) {
+			err := wfExecutor.KillSidecars(ctx)
+			if err == nil {
+				return true, nil
+			}
+			if errorsutil.IsTransientErr(err) {
+				log.WithError(err).WithField("retryCnt", retryCnt).Warn("fail to kill sidecar")
+				retryCnt++
+				return false, nil
+			}
+			return false, err
+		})
+		if err != nil {
 			wfExecutor.AddError(err)
 		}
 	}()