Merge branch 'main' into addInvalidConnectionRetry

argoproj · Sep 9, 2024 · 18da18e · 18da18e
2 parents 145e0b1 + ed3b8ce
commit 18da18e
Show file tree

Hide file tree

Showing 136 changed files with 3,022 additions and 2,801 deletions.
diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml
@@ -196,33 +196,46 @@ jobs:
         include:
           - test: test-executor
             profile: minimal
+            useApi: false
           - test: test-corefunctional
             profile: minimal
+            useApi: false
           - test: test-functional
             profile: minimal
+            useApi: false
           - test: test-api
             profile: mysql
+            useApi: true
           - test: test-cli
             profile: mysql
+            useApi: true
           - test: test-cron
             profile: minimal
+            useApi: false
           - test: test-examples
             profile: minimal
+            useApi: false
           - test: test-plugins
             profile: plugins
+            useApi: false
           - test: test-java-sdk
             profile: minimal
+            useApi: true
           - test: test-python-sdk
             profile: minimal
+            useApi: true
           - test: test-executor
             install_k3s_version: v1.28.11+k3s1
             profile: minimal
+            useApi: false
           - test: test-corefunctional
             install_k3s_version: v1.28.11+k3s1
             profile: minimal
+            useApi: false
           - test: test-functional
             install_k3s_version: v1.28.11+k3s1
             profile: minimal
+            useApi: false
     steps:
       - name: Install socat (needed by Kubernetes)
         run: sudo apt-get -y install socat
@@ -281,17 +294,17 @@ jobs:
         run: make controller kit STATIC_FILES=false
       - name: Build CLI
         run: make cli STATIC_FILES=false
-        if: ${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}}
+        if: ${{matrix.useApi}}
       - name: Start controller/API
         run: |
           make start PROFILE=${{matrix.profile}} \
             AUTH_MODE=client STATIC_FILES=false \
             LOG_LEVEL=info \
-            API=${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}} \
+            API=${{matrix.useApi}} \
             UI=false \
             POD_STATUS_CAPTURE_FINALIZER=true > /tmp/argo.log 2>&1 &
       - name: Wait for controller to be up
-        run: make wait API=${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}}
+        run: make wait API=${{matrix.useApi}}
         timeout-minutes: 5
       - name: Run tests ${{matrix.test}}
         run: make ${{matrix.test}} E2E_SUITE_TIMEOUT=20m STATIC_FILES=false

diff --git a/.spelling b/.spelling
@@ -185,6 +185,7 @@ memoizing
 metadata
 minikube
 mutex
+mutexes
 namespace
 namespaces
 natively
@@ -205,6 +206,7 @@ sandboxed
 shortcodes
 stateful
 stderr
+symlinks
 temporality
 triaged
 un-reconciled

diff --git a/Dockerfile.windows b/Dockerfile.windows
@@ -50,7 +50,7 @@ ARG GIT_TREE_STATE
 WORKDIR C:/Users/ContainerAdministrator/go/src/github.com/argoproj/argo-workflows
 COPY . .
 # run in git bash for all the shell commands in Makefile to work
-RUN bash -c 'make dist/argoexec GIT_COMMIT=${GIT_COMMIT} GIT_TAG=${GIT_TAG} GIT_TREE_STATE=${GIT_TREE_STATE}'
+RUN bash -c 'make dist/argoexec GIT_COMMIT=${GIT_COMMIT} GIT_TAG=${GIT_TAG} GIT_TREE_STATE=${GIT_TREE_STATE} HACK_PKG_FILES_AS_PKGS=true'
 
 ####################################################################################################
 # argoexec

diff --git a/Makefile b/Makefile
@@ -107,9 +107,19 @@ ifndef $(GOPATH)
 endif
 
 # -- file lists
-ARGOEXEC_PKG_FILES        := $(shell go list -f '{{ join .Deps "\n" }}' ./cmd/argoexec/ |  grep 'argoproj/argo-workflows/v3/' | xargs go list -f '{{ range $$file := .GoFiles }}{{ print $$.ImportPath "/" $$file "\n" }}{{ end }}' | cut -c 39-)
-CLI_PKG_FILES             := $(shell go list -f '{{ join .Deps "\n" }}' ./cmd/argo/ |  grep 'argoproj/argo-workflows/v3/' | xargs go list -f '{{ range $$file := .GoFiles }}{{ print $$.ImportPath "/" $$file "\n" }}{{ end }}' | cut -c 39-)
-CONTROLLER_PKG_FILES      := $(shell go list -f '{{ join .Deps "\n" }}' ./cmd/workflow-controller/ |  grep 'argoproj/argo-workflows/v3/' | xargs go list -f '{{ range $$file := .GoFiles }}{{ print $$.ImportPath "/" $$file "\n" }}{{ end }}' | cut -c 39-)
+HACK_PKG_FILES_AS_PKGS ?= false
+ifeq ($(HACK_PKG_FILES_AS_PKGS),false)
+	ARGOEXEC_PKG_FILES        := $(shell go list -f '{{ join .Deps "\n" }}' ./cmd/argoexec/ |  grep 'argoproj/argo-workflows/v3/' | xargs go list -f '{{ range $$file := .GoFiles }}{{ print $$.ImportPath "/" $$file "\n" }}{{ end }}' | cut -c 39-)
+	CLI_PKG_FILES             := $(shell go list -f '{{ join .Deps "\n" }}' ./cmd/argo/ |  grep 'argoproj/argo-workflows/v3/' | xargs go list -f '{{ range $$file := .GoFiles }}{{ print $$.ImportPath "/" $$file "\n" }}{{ end }}' | cut -c 39-)
+	CONTROLLER_PKG_FILES      := $(shell go list -f '{{ join .Deps "\n" }}' ./cmd/workflow-controller/ |  grep 'argoproj/argo-workflows/v3/' | xargs go list -f '{{ range $$file := .GoFiles }}{{ print $$.ImportPath "/" $$file "\n" }}{{ end }}' | cut -c 39-)
+else
+# Building argoexec on windows cannot rebuild the openapi, we need to fall back to the old
+# behaviour where we fake dependencies and therefore don't rebuild
+	ARGOEXEC_PKG_FILES    := $(shell echo cmd/argoexec            && go list -f '{{ join .Deps "\n" }}' ./cmd/argoexec/            | grep 'argoproj/argo-workflows/v3/' | cut -c 39-)
+	CLI_PKG_FILES         := $(shell echo cmd/argo                && go list -f '{{ join .Deps "\n" }}' ./cmd/argo/                | grep 'argoproj/argo-workflows/v3/' | cut -c 39-)
+	CONTROLLER_PKG_FILES  := $(shell echo cmd/workflow-controller && go list -f '{{ join .Deps "\n" }}' ./cmd/workflow-controller/ | grep 'argoproj/argo-workflows/v3/' | cut -c 39-)
+endif
+
 TYPES := $(shell find pkg/apis/workflow/v1alpha1 -type f -name '*.go' -not -name openapi_generated.go -not -name '*generated*' -not -name '*test.go')
 CRDS := $(shell find manifests/base/crds -type f -name 'argoproj.io_*.yaml')
 SWAGGER_FILES := pkg/apiclient/_.primary.swagger.json \

diff --git a/api/jsonschema/schema.json b/api/jsonschema/schema.json
diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
diff --git a/docs/environment-variables.md b/docs/environment-variables.md
@@ -29,7 +29,6 @@ This document outlines environment variables that can be used to customize behav
 | `CRON_SYNC_PERIOD`                       | `time.Duration`     | `10s`                                                                                       | How often to sync cron workflows.                                                                                                                                                                                                                                        |
 | `DEFAULT_REQUEUE_TIME`                   | `time.Duration`     | `10s`                                                                                       | The re-queue time for the rate limiter of the workflow queue.                                                                                                                                                                                                            |
 | `DISABLE_MAX_RECURSION`                  | `bool`              | `false`                                                                                     | Set to true to disable the recursion preventer, which will stop a workflow running which has called into a child template 100 times                                                                                                                                      |
-| `POD_ABSENT_TIMEOUT`                     | `time.Duration`     | `2m`                                                                                        | The time used to determine if pod absence should imply node failure                                                                                                                                                                                                      |
 | `EXPRESSION_TEMPLATES`                   | `bool`              | `true`                                                                                      | Escape hatch to disable expression templates.                                                                                                                                                                                                                            |
 | `EVENT_AGGREGATION_WITH_ANNOTATIONS`     | `bool`              | `false`                                                                                     | Whether event annotations will be used when aggregating events.                                                                                                                                                                                                          |
 | `GZIP_IMPLEMENTATION`                    | `string`            | `PGZip`                                                                                     | The implementation of compression/decompression. Currently only "`PGZip`" and "`GZip`" are supported.                                                                                                                                                                    |
@@ -46,6 +45,7 @@ This document outlines environment variables that can be used to customize behav
 | `OPERATION_DURATION_METRIC_BUCKET_COUNT` | `int`               | `6`                                                                                         | The number of buckets to collect the metric for the operation duration.                                                                                                                                                                                                  |
 | `POD_NAMES`                              | `string`            | `v2`                                                                                        | Whether to have pod names contain the template name (v2) or be the node id (v1) - should be set the same for Argo Server.                                                                                                                                                |
 | `RECENTLY_STARTED_POD_DURATION`          | `time.Duration`     | `10s`                                                                                       | The duration of a pod before the pod is considered to be recently started.                                                                                                                                                                                               |
+| `RECENTLY_DELETED_POD_DURATION`          | `time.Duration`     | `10s`                                                                                       | The duration of a pod before the pod is considered to be recently deleted.                                                                                                                                                                                               |
 | `RETRY_BACKOFF_DURATION`                 | `time.Duration`     | `10ms`                                                                                      | The retry back-off duration when retrying API calls.                                                                                                                                                                                                                     |
 | `RETRY_BACKOFF_FACTOR`                   | `float`             | `2.0`                                                                                       | The retry back-off factor when retrying API calls.                                                                                                                                                                                                                       |
 | `RETRY_BACKOFF_STEPS`                    | `int`               | `5`                                                                                         | The retry back-off steps when retrying API calls.                                                                                                                                                                                                                        |

diff --git a/docs/fields.md b/docs/fields.md
@@ -1564,7 +1564,7 @@ PodGC describes how to delete completed pods as they complete
 ### Fields
 | Field Name | Field Type | Description   |
 |:----------:|:----------:|---------------|
-|`deleteDelayDuration`|[`Duration`](#duration)|DeleteDelayDuration specifies the duration before pods in the GC queue get deleted.|
+|`deleteDelayDuration`|`string`|DeleteDelayDuration specifies the duration before pods in the GC queue get deleted.|
 |`labelSelector`|[`LabelSelector`](#labelselector)|LabelSelector is the label selector to check if the pods match the labels before being added to the pod GC queue.|
 |`strategy`|`string`|Strategy is the strategy to use. One of "OnPodCompletion", "OnPodSuccess", "OnWorkflowCompletion", "OnWorkflowSuccess". If unset, does not delete Pods|
 
@@ -5147,23 +5147,6 @@ ObjectReference contains enough information to let you inspect or modify the ref
 |`resourceVersion`|`string`|Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency|
 |`uid`|`string`|UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids|
 
-## Duration
-
-Duration is a wrapper around time.Duration which supports correct marshaling to YAML and JSON. In particular, it marshals into strings, which can be used as map keys in json.
-
-<details markdown>
-<summary>Examples with this field (click to open)</summary>
-
-- [`pod-gc-strategy-with-label-selector.yaml`](https://github.com/argoproj/argo-workflows/blob/main/examples/pod-gc-strategy-with-label-selector.yaml)
-
-- [`pod-gc-strategy.yaml`](https://github.com/argoproj/argo-workflows/blob/main/examples/pod-gc-strategy.yaml)
-</details>
-
-### Fields
-| Field Name | Field Type | Description   |
-|:----------:|:----------:|---------------|
-|`duration`|`string`|_No description available_|
-
 ## LabelSelector
 
 A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.

diff --git a/docs/metrics.md b/docs/metrics.md
@@ -336,10 +336,11 @@ The rate of this shows how busy that area of the controller is.
 
 Queues:
 
-- `workflow_queue`: the queue of Workflow updates from the cluster
 - `cron_wf_queue`: the queue of CronWorkflow updates from the cluster
-- `workflow_ttl_queue`: workflows which are queued for deletion due to age
 - `pod_cleanup_queue`: pods which are queued for deletion
+- `workflow_queue`: the queue of Workflow updates from the cluster
+- `workflow_ttl_queue`: workflows which are queued for deletion due to age
+- `workflow_archive_queue`: workflows which are queued for archiving
 
 This and associated metrics are all directly sourced from the [client-go workqueue metrics](https://godocs.io/k8s.io/client-go/util/workqueue)
 

diff --git a/docs/parallelism.md b/docs/parallelism.md
@@ -0,0 +1,43 @@
+# Limiting parallelism
+
+You can restrict the number of parallel workflow executions.
+
+## Controller-level
+
+You can limit the total number of parallel workflow executions in the [workflow controller ConfigMap](workflow-controller-configmap.yaml):
+
+```yaml
+data:
+  parallelism: "10"
+```
+
+You can also limit the total number of parallel workflow executions in a single namespace:
+
+```yaml
+data:
+  namespaceParallelism: "4"
+```
+
+!!! Note
+    Workflows that are executing but restricted from running more nodes due to other mechanisms will still count toward parallelism limits.
+
+### Priority
+
+You can set a `priority` on workflows:
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: priority-
+spec:
+  priority: 3
+  # ...
+```
+
+Workflows that have not started due to Controller-level parallelism will be queued: workflows with higher priority numbers will start before lower priority ones.
+The default is `priority: 0`.
+
+## Synchronization
+
+You can also use [mutexes, semaphores, and parallelism](synchronization.md) to control the parallel execution of workflows and templates.