docs: describe containerset retrystrategy and verify it works. Fixes: #11502

docs/container-set-template.md

@@ -116,3 +116,26 @@ Example B: Lopsided requests, e.g. `a -> b` where `a` is cheap and `b` is expens
 Can you see the problem here? `a` only has small requests, but the container set will use the  total of all requests. So it's as if you're using all that GPU for 10h. This will be expensive.
 
 Solution: do not use container set when you have lopsided requests.
+
+## Container Set Retries
+
+> v3.3 and after
+
+Container Set Retry policies describes how to retry a container nodes in the container set if it fails.
+
+Number of retries(default 0) and sleep duration between retries(default 0s, instant retry) can be set.
+
+The container won't retry if it's unable to locate the command.
+
+Here is an example of a Container Set Template with `retryStrategy`:
+
+```yaml
+      containerSet:
+        containers:
+          - name: retry-containerset
+            image: alpine:latest
+            retryStrategy:
+              limit: "3"
+            command: [ sh, -c ]
+            args: [ "echo intentional failure; exit 1" ]
+```

docs/fields.md

@@ -2411,7 +2411,7 @@ _No description available_
 | Field Name | Field Type | Description   |
 |:----------:|:----------:|---------------|
 |`containers`|`Array<`[`ContainerNode`](#containernode)`>`|_No description available_|
-|`retryStrategy`|[`ContainerSetRetryStrategy`](#containersetretrystrategy)|RetryStrategy describes how to retry a container nodes in the container set if it fails. Nbr of retries(default 0) and sleep duration between retries(default 0s, instant retry) can be set.|
+|`retryStrategy`|[`ContainerSetRetryStrategy`](#containersetretrystrategy)|RetryStrategy describes how to retry a container nodes in the container set if it fails. Nbr of retries(default 0) and sleep duration between retries(default 0s, instant retry) can be set. The container won't retry if it's unable to locate the command.|
 |`volumeMounts`|`Array<`[`VolumeMount`](#volumemount)`>`|_No description available_|
 
 ## DAGTemplate

pkg/apis/workflow/v1alpha1/container_set_template_types.go

@@ -14,6 +14,7 @@ type ContainerSetTemplate struct {
 	VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty" protobuf:"bytes,3,rep,name=volumeMounts"`
 	// RetryStrategy describes how to retry a container nodes in the container set if it fails.
 	// Nbr of retries(default 0) and sleep duration between retries(default 0s, instant retry) can be set.
+	// The container won't retry if it's unable to locate the command.
 	RetryStrategy *ContainerSetRetryStrategy `json:"retryStrategy,omitempty" protobuf:"bytes,5,opt,name=retryStrategy"`
 }
 

test/e2e/cli_test.go

@@ -1733,6 +1733,62 @@ func (s *CLISuite) TestPluginStruct() {
 		})
 }
 
+func (s *CLISuite) TestWorkflowTemplateWithRetryStrategyInContainerSet() {
+	var name string
+	s.Given().
+		WorkflowTemplate("@testdata/workflow-template-with-containerset.yaml").
+		Workflow(`
+metadata:
+  generateName: workflow-template-containerset-
+spec:
+  workflowTemplateRef:
+    name: containerset-with-retrystrategy
+`).
+		When().
+		CreateWorkflowTemplates().
+		SubmitWorkflow().
+		WaitForWorkflow(fixtures.ToBeFailed).
+		Then().
+		ExpectWorkflow(func(t *testing.T, metadata *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
+			assert.Equal(t, status.Phase, wfv1.WorkflowFailed)
+			name = metadata.Name
+		})
+	// Success, no need retry
+	s.Run("ContainerLogs", func() {
+		s.Given().
+			RunCli([]string{"logs", name, name, "-c", "c1"}, func(t *testing.T, output string, err error) {
+				if assert.NoError(t, err) {
+					count := strings.Count(output, "capturing logs")
+					assert.Equal(t, 1, count)
+					assert.Contains(t, output, "hi")
+				}
+			})
+	})
+	// Command err. No retry logic is entered.
+	s.Run("ContainerLogs", func() {
+		s.Given().
+			RunCli([]string{"logs", name, name, "-c", "c2"}, func(t *testing.T, output string, err error) {
+				if assert.NoError(t, err) {
+					count := strings.Count(output, "capturing logs")
+					assert.Equal(t, 0, count)
+					assert.Contains(t, output, "executable file not found in $PATH")
+				}
+			})
+	})
+	// Retry when err.
+	s.Run("ContainerLogs", func() {
+		s.Given().
+			RunCli([]string{"logs", name, name, "-c", "c3"}, func(t *testing.T, output string, err error) {
+				if assert.NoError(t, err) {
+					count := strings.Count(output, "capturing logs")
+					assert.Equal(t, 2, count)
+					countFailureInfo := strings.Count(output, "intentional failure")
+					assert.Equal(t, 2, countFailureInfo)
+				}
+			})
+	})
+}
+
 func TestCLISuite(t *testing.T) {
 	suite.Run(t, new(CLISuite))
 }

test/e2e/testdata/workflow-template-with-containerset.yaml

@@ -0,0 +1,32 @@
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: containerset-with-retrystrategy
+  annotations:
+    workflows.argoproj.io/description: |
+      This workflow template is used to create a workflow with containerset.
+spec:
+  entrypoint: test
+  templates:
+    - name: test
+      containerSet:
+        retryStrategy:
+          retries: "2"
+        containers:
+          - name: c1
+            image: python:alpine3.6
+            command:
+              - python
+              - -c
+            args:
+              - |
+                print("hi")
+          - name: c2
+            image: python:alpine3.6
+            command:
+              - invalid
+              - command
+          - name: c3
+            image: alpine:latest
+            command: [ sh, -c ]
+            args: [ "echo intentional failure; exit 1" ]