fix: retries for lookup entrypoint/template. Fixes: #11441

workflow/controller/workflowpod.go

@@ -15,6 +15,7 @@ import (
 	apiv1 "k8s.io/api/core/v1"
 	apierr "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/utils/ptr"
 
 	"github.com/argoproj/argo-workflows/v3/errors"
@@ -23,6 +24,7 @@ import (
 	"github.com/argoproj/argo-workflows/v3/util/deprecation"
 	errorsutil "github.com/argoproj/argo-workflows/v3/util/errors"
 	"github.com/argoproj/argo-workflows/v3/util/intstr"
+	"github.com/argoproj/argo-workflows/v3/util/retry"
 	"github.com/argoproj/argo-workflows/v3/util/template"
 	"github.com/argoproj/argo-workflows/v3/workflow/common"
 	"github.com/argoproj/argo-workflows/v3/workflow/controller/entrypoint"
@@ -381,12 +383,26 @@ func (woc *wfOperationCtx) createWorkflowPod(ctx context.Context, nodeName strin
 		pod.Spec = *patchedPodSpec
 	}
 
+	var x *entrypoint.Image
+
 	for i, c := range pod.Spec.Containers {
 		if c.Name != common.WaitContainerName {
 			// https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#notes
 			if len(c.Command) == 0 {
-				x, err := woc.controller.entrypoint.Lookup(ctx, c.Image, entrypoint.Options{
-					Namespace: woc.wf.Namespace, ServiceAccountName: woc.execWf.Spec.ServiceAccountName, ImagePullSecrets: woc.execWf.Spec.ImagePullSecrets,
+				err := wait.ExponentialBackoff(retry.DefaultRetry, func() (bool, error) {
+					var lookupErr error
+					x, lookupErr = woc.controller.entrypoint.Lookup(ctx, c.Image, entrypoint.Options{
+						Namespace:          woc.wf.Namespace,
+						ServiceAccountName: woc.execWf.Spec.ServiceAccountName,
+						ImagePullSecrets:   woc.execWf.Spec.ImagePullSecrets,
+					})
+					if lookupErr != nil {
+						if errorsutil.IsTransientErr(lookupErr) {
+							return false, nil
+						}
+						return true, lookupErr
+					}
+					return true, nil
 				})
 				if err != nil {
 					return nil, fmt.Errorf("failed to look-up entrypoint/cmd for image %q, you must either explicitly specify the command, or list the image's command in the index: https://argo-workflows.readthedocs.io/en/latest/workflow-executors/#emissary-emissary: %w", c.Image, err)

workflow/templateresolution/context.go

@@ -7,10 +7,13 @@ import (
 	log "github.com/sirupsen/logrus"
 	apierr "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 
 	"github.com/argoproj/argo-workflows/v3/errors"
 	wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
 	typed "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
+	errorsutil "github.com/argoproj/argo-workflows/v3/util/errors"
+	"github.com/argoproj/argo-workflows/v3/util/retry"
 	"github.com/argoproj/argo-workflows/v3/workflow/common"
 )
 
@@ -149,11 +152,46 @@ func (ctx *Context) GetTemplate(h wfv1.TemplateReferenceHolder) (*wfv1.Template,
 	ctx.log.Debug("Getting the template")
 	if x := h.GetTemplate(); x != nil {
 		return x, nil
-	} else if x := h.GetTemplateRef(); x != nil {
-		return ctx.GetTemplateFromRef(x)
-	} else if x := h.GetTemplateName(); x != "" {
-		return ctx.GetTemplateByName(x)
 	}
+
+	if x := h.GetTemplateRef(); x != nil {
+		var tmpl *wfv1.Template
+		err := wait.ExponentialBackoff(retry.DefaultRetry, func() (bool, error) {
+			var getRefErr error
+			tmpl, getRefErr = ctx.GetTemplateFromRef(x)
+			if getRefErr != nil {
+				if errorsutil.IsTransientErr(getRefErr) {
+					return false, nil
+				}
+				return true, getRefErr
+			}
+			return true, nil
+		})
+		if err != nil {
+			return nil, err
+		}
+		return tmpl, nil
+	}
+
+	if x := h.GetTemplateName(); x != "" {
+		var tmpl *wfv1.Template
+		err := wait.ExponentialBackoff(retry.DefaultRetry, func() (bool, error) {
+			var getNameErr error
+			tmpl, getNameErr = ctx.GetTemplateByName(x)
+			if getNameErr != nil {
+				if errorsutil.IsTransientErr(getNameErr) {
+					return false, nil
+				}
+				return true, getNameErr
+			}
+			return true, nil
+		})
+		if err != nil {
+			return nil, err
+		}
+		return tmpl, nil
+	}
+
 	return nil, errors.Errorf(errors.CodeInternal, "failed to get a template")
 }
 
@@ -166,7 +204,7 @@ func (ctx *Context) GetTemplateScope() string {
 	return string(ctx.tmplBase.GetResourceScope()) + "/" + ctx.tmplBase.GetName()
 }
 
-// ResolveTemplate digs into referenes and returns a merged template.
+// ResolveTemplate digs into references and returns a merged template.
 // This method is the public start point of template resolution.
 func (ctx *Context) ResolveTemplate(tmplHolder wfv1.TemplateReferenceHolder) (*Context, *wfv1.Template, bool, error) {
 	return ctx.resolveTemplateImpl(tmplHolder)