diff --git a/workflow/controller/workflowpod.go b/workflow/controller/workflowpod.go index f793060d1fcf..97c6676754b5 100644 --- a/workflow/controller/workflowpod.go +++ b/workflow/controller/workflowpod.go @@ -15,6 +15,7 @@ import ( apiv1 "k8s.io/api/core/v1" apierr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/ptr" "github.com/argoproj/argo-workflows/v3/errors" @@ -23,6 +24,7 @@ import ( "github.com/argoproj/argo-workflows/v3/util/deprecation" errorsutil "github.com/argoproj/argo-workflows/v3/util/errors" "github.com/argoproj/argo-workflows/v3/util/intstr" + "github.com/argoproj/argo-workflows/v3/util/retry" "github.com/argoproj/argo-workflows/v3/util/template" "github.com/argoproj/argo-workflows/v3/workflow/common" "github.com/argoproj/argo-workflows/v3/workflow/controller/entrypoint" @@ -381,12 +383,26 @@ func (woc *wfOperationCtx) createWorkflowPod(ctx context.Context, nodeName strin pod.Spec = *patchedPodSpec } + var x *entrypoint.Image + for i, c := range pod.Spec.Containers { if c.Name != common.WaitContainerName { // https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#notes if len(c.Command) == 0 { - x, err := woc.controller.entrypoint.Lookup(ctx, c.Image, entrypoint.Options{ - Namespace: woc.wf.Namespace, ServiceAccountName: woc.execWf.Spec.ServiceAccountName, ImagePullSecrets: woc.execWf.Spec.ImagePullSecrets, + err := wait.ExponentialBackoff(retry.DefaultRetry, func() (bool, error) { + var lookupErr error + x, lookupErr = woc.controller.entrypoint.Lookup(ctx, c.Image, entrypoint.Options{ + Namespace: woc.wf.Namespace, + ServiceAccountName: woc.execWf.Spec.ServiceAccountName, + ImagePullSecrets: woc.execWf.Spec.ImagePullSecrets, + }) + if lookupErr != nil { + if errorsutil.IsTransientErr(lookupErr) { + return false, nil + } + return true, lookupErr + } + return true, nil }) if err != nil { return nil, fmt.Errorf("failed to look-up entrypoint/cmd for image %q, you must either explicitly specify the command, or list the image's command in the index: https://argo-workflows.readthedocs.io/en/latest/workflow-executors/#emissary-emissary: %w", c.Image, err) diff --git a/workflow/templateresolution/context.go b/workflow/templateresolution/context.go index 131864685887..4137d45f762c 100644 --- a/workflow/templateresolution/context.go +++ b/workflow/templateresolution/context.go @@ -7,10 +7,13 @@ import ( log "github.com/sirupsen/logrus" apierr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "github.com/argoproj/argo-workflows/v3/errors" wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" typed "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1" + errorsutil "github.com/argoproj/argo-workflows/v3/util/errors" + "github.com/argoproj/argo-workflows/v3/util/retry" "github.com/argoproj/argo-workflows/v3/workflow/common" ) @@ -149,11 +152,46 @@ func (ctx *Context) GetTemplate(h wfv1.TemplateReferenceHolder) (*wfv1.Template, ctx.log.Debug("Getting the template") if x := h.GetTemplate(); x != nil { return x, nil - } else if x := h.GetTemplateRef(); x != nil { - return ctx.GetTemplateFromRef(x) - } else if x := h.GetTemplateName(); x != "" { - return ctx.GetTemplateByName(x) } + + if x := h.GetTemplateRef(); x != nil { + var tmpl *wfv1.Template + err := wait.ExponentialBackoff(retry.DefaultRetry, func() (bool, error) { + var getRefErr error + tmpl, getRefErr = ctx.GetTemplateFromRef(x) + if getRefErr != nil { + if errorsutil.IsTransientErr(getRefErr) { + return false, nil + } + return true, getRefErr + } + return true, nil + }) + if err != nil { + return nil, err + } + return tmpl, nil + } + + if x := h.GetTemplateName(); x != "" { + var tmpl *wfv1.Template + err := wait.ExponentialBackoff(retry.DefaultRetry, func() (bool, error) { + var getNameErr error + tmpl, getNameErr = ctx.GetTemplateByName(x) + if getNameErr != nil { + if errorsutil.IsTransientErr(getNameErr) { + return false, nil + } + return true, getNameErr + } + return true, nil + }) + if err != nil { + return nil, err + } + return tmpl, nil + } + return nil, errors.Errorf(errors.CodeInternal, "failed to get a template") } @@ -166,7 +204,7 @@ func (ctx *Context) GetTemplateScope() string { return string(ctx.tmplBase.GetResourceScope()) + "/" + ctx.tmplBase.GetName() } -// ResolveTemplate digs into referenes and returns a merged template. +// ResolveTemplate digs into references and returns a merged template. // This method is the public start point of template resolution. func (ctx *Context) ResolveTemplate(tmplHolder wfv1.TemplateReferenceHolder) (*Context, *wfv1.Template, bool, error) { return ctx.resolveTemplateImpl(tmplHolder)