fix: add retry for kill pod

Signed-off-by: henrywangx <[email protected]>

cmd/argoexec/commands/wait.go

@@ -7,6 +7,10 @@ import (
 	"github.com/argoproj/pkg/stats"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	errorsutil "github.com/argoproj/argo-workflows/v3/util/errors"
+	"github.com/argoproj/argo-workflows/v3/workflow/executor"
 )
 
 func NewWaitCommand() *cobra.Command {
@@ -31,7 +35,21 @@ func waitContainer(ctx context.Context) error {
 	stats.StartStatsTicker(5 * time.Minute)
 
 	defer func() {
-		if err := wfExecutor.KillSidecars(ctx); err != nil {
+		// Killing sidecar containers
+		retryCnt := 0
+		err := wait.ExponentialBackoff(executor.ExecutorRetry, func() (bool, error) {
+			err := wfExecutor.KillSidecars(ctx)
+			if err == nil {
+				return true, nil
+			}
+			if errorsutil.IsTransientErr(err) {
+				log.WithError(err).WithField("retryCnt", retryCnt).Warn("fail to kill sidecar")
+				retryCnt++
+				return false, nil
+			}
+			return false, err
+		})
+		if err != nil {
 			wfExecutor.AddError(err)
 		}
 	}()

workflow/executor/k8sapi/client.go

@@ -20,7 +20,7 @@ import (
 	errorsutil "github.com/argoproj/argo-workflows/v3/util/errors"
 	waitutil "github.com/argoproj/argo-workflows/v3/util/wait"
 	"github.com/argoproj/argo-workflows/v3/workflow/common"
-	execcommon "github.com/argoproj/argo-workflows/v3/workflow/executor/common"
+	"github.com/argoproj/argo-workflows/v3/workflow/executor"
 )
 
 type k8sAPIClient struct {
@@ -110,11 +110,17 @@ func (c *k8sAPIClient) GetContainerStatuses(ctx context.Context) (*corev1.Pod, [
 
 func (c *k8sAPIClient) KillContainer(pod *corev1.Pod, container *corev1.ContainerStatus, sig syscall.Signal) error {
 	command := []string{"/bin/sh", "-c", fmt.Sprintf("kill -%d 1", sig)}
-	exec, err := common.ExecPodContainer(c.config, c.namespace, c.podName, container.Name, true, true, command...)
-	if err != nil {
-		return err
-	}
-	_, _, err = common.GetExecutorOutput(exec)
+	err := wait.ExponentialBackoff(executor.ExecutorRetry, func() (bool, error) {
+		exec, err := common.ExecPodContainer(c.config, c.namespace, c.podName, container.Name, true, true, command...)
+		if err != nil {
+			return false, nil
+		}
+		_, _, err = common.GetExecutorOutput(exec)
+		if err != nil {
+			return false, err
+		}
+		return true, nil
+	})
 	return err
 }