feat(api): redact secrets in logs. Fixes #8685

docs/environment-variables.md

@@ -22,6 +22,7 @@ most users. Environment variables may be removed at any time.
 | `ARGO_AGENT_PATCH_RATE`                | `time.Duration`     | `DEFAULT_REQUEUE_TIME`                                                                      | Rate that the Argo Agent will patch the workflow task-set.                                                                                                                                                                                                               |
 | `ARGO_AGENT_CPU_LIMIT`                 | `resource.Quantity` | `100m`                                                                                      | CPU resource limit for the agent.                                                                                                                                                                                                                                        |
 | `ARGO_AGENT_MEMORY_LIMIT`              | `resource.Quantity` | `256m`                                                                                      | Memory resource limit for the agent.                                                                                                                                                                                                                                     |
+| `ARGO_REDACT_POD_LOGS`                 | `bool`              | `false`                                                                                      | Whether to redact pod logs to hide/mask secrets.                                                                                                                                                                                                                        |
 | `BUBBLE_ENTRY_TEMPLATE_ERR`            | `bool`              | `true`                                                                                      | Whether to bubble up template errors to workflow.                                                                                                                                                                                                                        |
 | `CACHE_GC_PERIOD`                      | `time.Duration`     | `0s`                                                                                        | How often to perform memoization cache GC, which is disabled by default and can be enabled by providing a non-zero duration.                                                                                                                                             |
 | `CACHE_GC_AFTER_NOT_HIT_DURATION`      | `time.Duration`     | `30s`                                                                                       | When a memoization cache has not been hit after this duration, it will be deleted.                                                                                                                                                                                       |

test/e2e/argo_server_test.go

@@ -1975,3 +1975,75 @@ func (s *ArgoServerSuite) TestRateLimitHeader() {
 func TestArgoServerSuite(t *testing.T) {
 	suite.Run(t, new(ArgoServerSuite))
 }
+
+func (s *ArgoServerSuite) TestWorkflowLogRedaction() {
+	nsName := fixtures.Namespace
+	// create secret if not present
+	secretName := "test-secret"
+	secretData := map[string][]byte{
+		"testpassword": []byte("S00perS3cretPa55word"),
+	}
+	secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName}, Data: secretData}
+	ctx := context.Background()
+	s.Run("CreateSecret", func() {
+		_, e := s.KubeClient.CoreV1().Secrets(nsName).Create(ctx, secret, metav1.CreateOptions{})
+		assert.NoError(s.T(), e)
+	})
+	defer func() {
+		// Clean up created secret
+		_ = s.KubeClient.CoreV1().Secrets(nsName).Delete(ctx, secretName, metav1.DeleteOptions{})
+	}()
+
+	var name string
+	s.Given().
+		Workflow("@smoke/workflow-with-secrets.yaml").
+		When().
+		SubmitWorkflow().
+		WaitForWorkflow(fixtures.ToStart).
+		Then().
+		ExpectWorkflow(func(t *testing.T, metadata *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
+			name = metadata.Name
+		})
+
+	// lets check the logs
+	for _, tt := range []struct {
+		name string
+		path string
+	}{
+		{"PodLogs", "/" + name + "/log?logOptions.container=main"},
+		{"WorkflowLogs", "/log?podName=" + name + "&logOptions.container=main"},
+	} {
+		s.Run(tt.name, func() {
+			s.stream("/api/v1/workflows/argo/"+name+tt.path, func(t *testing.T, line string) (done bool) {
+				if strings.Contains(line, "data: ") {
+					assert.Contains(t, line, "secret from env: S00perS3cretPa55word")
+					return true
+				}
+				return false
+			})
+		})
+	}
+
+	// set pod log redaction to true
+	_ = os.Setenv("ARGO_REDACT_POD_LOGS", "true")
+	defer func() { _ = os.Unsetenv("ARGO_REDACT_POD_LOGS") }()
+
+	// lets check the logs
+	for _, tt := range []struct {
+		name string
+		path string
+	}{
+		{"PodLogs", "/" + name + "/log?logOptions.container=main"},
+		{"WorkflowLogs", "/log?podName=" + name + "&logOptions.container=main"},
+	} {
+		s.Run(tt.name, func() {
+			s.stream("/api/v1/workflows/argo/"+name+tt.path, func(t *testing.T, line string) (done bool) {
+				if strings.Contains(line, "data: ") {
+					assert.Contains(t, line, "secret from env: [ redacted ]")
+					return true
+				}
+				return false
+			})
+		})
+	}
+}

test/e2e/smoke/workflow-with-secrets.yaml

@@ -0,0 +1,17 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: secrets-
+spec:
+  entrypoint: print-secret
+  templates:
+  - name: print-secret
+    container:
+      image: argoproj/argosay:v2
+      args: [echo, "secret from env: $MYSECRETPASSWORD"]
+      env:
+      - name: MYSECRETPASSWORD
+        valueFrom:
+          secretKeyRef:
+            name: test-secret
+            key: testpassword

util/logs/workflow-logger.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"os"
 	"regexp"
 	"sort"
 	"strings"
@@ -80,6 +81,14 @@ func WorkflowLogs(ctx context.Context, wfClient versioned.Interface, kubeClient
 
 	var podListOptions metav1.ListOptions
 
+	// get env variable for pod logs redaction
+	enablePodLogRedaction := os.Getenv("ARGO_REDACT_POD_LOGS")
+	// get secrets for redaction
+	secrets, err := kubeClient.CoreV1().Secrets(req.GetNamespace()).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		logCtx.WithField("err", err).Debugln("error in listing secrets")
+	}
+
 	// we add selector if cli specify the pod selector when using logs
 	if req.GetSelector() != "" {
 		podListOptions = metav1.ListOptions{LabelSelector: common.LabelKeyWorkflow + "=" + req.GetName() + "," + req.GetSelector()}
@@ -165,6 +174,17 @@ func WorkflowLogs(ctx context.Context, wfClient versioned.Interface, kubeClient
 						}
 						if rx.MatchString(content) { // this means we filter the lines in the server, but will still incur the cost of retrieving them from Kubernetes
 							logCtx.WithFields(log.Fields{"timestamp": timestamp, "content": content}).Debug("Log line")
+
+							// log redaction for secrets
+							if secrets != nil && enablePodLogRedaction == "true" {
+								for _, s := range secrets.Items {
+									for _, v := range s.Data {
+										if strings.Contains(content, string(v)) {
+											content = strings.Replace(content, string(v), "[ redacted ]", -1)
+										}
+									}
+								}
+							}
 							unsortedEntries <- logEntry{podName: podName, content: content, timestamp: timestamp}
 						}
 					}