Support retries and connection settings

changelog/65.added.md

@@ -0,0 +1 @@
+Added support for retry logic and specific connection settings in `vault:client`

docs/ref/configuration.md

@@ -184,6 +184,110 @@ The time in seconds to cache tokens/SecretIDs for. Defaults to `ttl`,
 which caches the secret for as long as it is valid, unless a new configuration
 is requested from the master.
 
+:::{vconf} client
+:::
+### `client`
+Configures Vault API client behavior. By default,
+the client retries requests with a backoff strategy,
+unless the response includes a `Retry-After` header, which is respected.
+Connection errors as well as responses with the status codes
+`412`, `429`, `500`, `502`, `503`, `504` are retried.
+
+:::{vconf} client:connect_timeout
+:::
+#### connect_timeout
+The number of seconds to wait for a connection to be established.
+Defaults to `9.2`.
+
+:::{vconf} client:read_timeout
+:::
+#### read_timeout
+The number of seconds to wait between packets sent by the server.
+Defaults to `30`.
+
+:::{vconf} client:max_retries
+:::
+#### max_retries
+The maximum number of retries (not including the initial request) before
+raising an exception. Set this to `0` to disable retry behavior.
+Defaults to `5`. Maximum: `10`.
+
+:::{vconf} client:backoff_factor
+:::
+#### backoff_factor
+A backoff factor (in seconds) to use between retry attempts when applying
+the backoff strategy (based on the Fibonacci sequence).
+The first retry is always executed immediately, unless
+the status code was HTTP 429. Defaults to `0.1`. Maximum: `3.0`
+
+:::{hint}
+The effective sleep time before the nth retry (for n > 1) is given by:
+
+  > {backoff_factor} * {(n+2)th Fibonacci number}
+
+The default values thus result in the following sleep times (in seconds),
+without accounting for {vconf}`backoff_jitter <client:backoff_jitter>`:
+
+  > [initial request] 0 [1st] 0.2 [2nd] 0.3 [3rd] 0.5 [4th] 0.8 [5th]
+
+If the response had a status code of HTTP 429 and did not include
+the `Retry-After` header, the immediate retry is skipped:
+
+  > [initial request] 0.2 [1st] 0.3 [2nd] 0.5 [3rd] 0.8 [4th] 1.3 [5th]
+
+:::
+
+:::{vconf} client:backoff_max
+:::
+#### backoff_max
+A cap for the effective sleep time between retries.
+Defaults to `10.0`. Maximum: `60.0`.
+
+:::{vconf} client:backoff_jitter
+:::
+#### backoff_jitter
+A maximum number of seconds to randomize the effective sleep time
+between retries by. Defaults to `0.2`. Maximum: `5.0`
+
+:::{vconf} client:retry_post
+:::
+#### retry_post
+Whether to retry requests that are potentially non-idempotent (`POST`, `PATCH`). Defaults to `False`.
+
+:::{note}
+HTTP 429 responses are always retried, regardless of HTTP verb.
+:::
+
+:::{vconf} client:retry_statuses
+:::
+#### retry_statuses
+A list of HTTP status codes which should be retried.
+Defaults to `[412, 500, 502, 503, 504]`.
+
+:::{note}
+HTTP 429 is always retried, regardless of HTTP verb and whether it is present
+in this list. It is recommended to ensure the `Retry-After` header is sent by Vault to optimize the spent resources.
+See {vconf}`respect_retry_after <client:respect_retry_after>` for details.
+:::
+
+:::{vconf} client:respect_retry_after
+:::
+#### respect_retry_after
+Whether to respect the `Retry-After` header sent by Vault, usually when a
+rate limit has been hit. Defaults to `True`.
+
+:::{hint}
+This header is not sent by default and must be enabled explicitly
+via [enable_rate_limit_response_headers](https://developer.hashicorp.com/vault/api-docs/system/quotas-config#enable_rate_limit_response_headers).
+:::
+
+:::{vconf} client:retry_after_max
+:::
+#### retry_after_max
+When {vconf}`respect_retry_after <client:respect_retry_after>` is True, limit
+the maximum amount of seconds the client will sleep before retrying. Set this to `null`/`None`
+to disable this behavior. Defaults to `60`.
+
 :::{vconf} server
 :::
 ### `server`
@@ -439,6 +543,22 @@ vault:
     config: 3600
     kv_metadata: connection
     secret: ttl
+  client:
+    max_retries: 5
+    connect_timeout: 9.2
+    read_timeout: 30
+    backoff_factor: 0.1
+    backoff_max: 10
+    backoff_jitter: 0.2
+    retry_post: false
+    retry_statuses:
+      - 412
+      - 500
+      - 502
+      - 503
+      - 504
+    respect_retry_after: true
+    retry_after_max: 60
   config_location: <variable, depends on running scope>
   issue:
     allow_minion_override_params: false

src/saltext/vault/runners/vault.py

@@ -324,6 +324,7 @@ def get_config(
                 "token_lifecycle": _config("auth:token_lifecycle"),
             },
             "cache": _config("cache"),
+            "client": _config("client"),
             "server": _config("server"),
             "wrap_info_nested": [],
         }

src/saltext/vault/utils/vault/__init__.py

@@ -4,15 +4,6 @@
 
 import logging
 
-import salt.cache
-import salt.crypt
-import salt.exceptions
-import salt.utils.data
-import salt.utils.dictupdate
-import salt.utils.json
-import salt.utils.versions
-
-import saltext.vault.utils.vault.helpers as hlp
 from saltext.vault.utils.vault.auth import InvalidVaultSecretId
 from saltext.vault.utils.vault.auth import InvalidVaultToken
 from saltext.vault.utils.vault.auth import LocalVaultSecretId
@@ -24,6 +15,7 @@
 from saltext.vault.utils.vault.exceptions import VaultNotFoundError
 from saltext.vault.utils.vault.exceptions import VaultPermissionDeniedError
 from saltext.vault.utils.vault.exceptions import VaultPreconditionFailedError
+from saltext.vault.utils.vault.exceptions import VaultRateLimitExceededError
 from saltext.vault.utils.vault.exceptions import VaultServerError
 from saltext.vault.utils.vault.exceptions import VaultUnavailableError
 from saltext.vault.utils.vault.exceptions import VaultUnsupportedOperationError