add redact

.golangci.yml

@@ -20,6 +20,7 @@ linters-settings:
       - octalLiteral
       - whyNoLint
       - wrapperFunc
+      - hugeParam
   gocyclo:
     min-complexity: 20
   golint:

.teller.example.yml

@@ -16,6 +16,7 @@ providers:
     env:
       FOO_BAR:
         path: ~/my-dot-env.env
+        redact_with: "**FOOBAR**" # optional
 
   # # requires an API key in: HEROKU_API_KEY (you can fetch from ~/.netrc)
   # heroku:

README.md

@@ -119,7 +119,11 @@ Use this one liner from now on:
 $ docker run --rm -it --env-file <(teller env) alpine sh
 ```
 
-## Scan for hardcoded secrets
+## Scan for secrets
+
+Teller can help you fight secret sprawl and hard coded secrets, as well as be the best productivity tool for working with your vault.
+
+It can also integrate into your CI and serve as a shift-left security tool for your DevSecOps pipeline.
 
 Look for your vault-kept secrets in your code by running:
 
@@ -136,17 +140,50 @@ run: teller scan --silent
 It will break your build if it finds something (returns exit code `1`).
 
 
-Use Teller for productively and securely running process and you get this for free -- nothing to configure. If you have data that you're bringing that you're sure isn't sensitive, flag it in your `teller.yml`:
+Use Teller for productively and securely running your processes and you get this for free -- nothing to configure. If you have data that you're bringing that you're sure isn't sensitive, flag it in your `teller.yml`:
 
 ```
 dotenv:
   env:
     FOO:
       path: ~/my-dot-env.env
-      not_sensitive: true
+      severity: none # will skip scanning. possible values: high | medium | low | none
+```
+
+By default we treat all entries as sensitive, with value `high`.
+
+
+## Redact secrets from process outputs, logs, and files
+
+You can use `teller` as a redaction tool across your infrastructure, and run processes while redacting their output as well as clean up logs and live tails of logs.
+
+Run a process and redact its output in real time:
+
+```
+$ teller run --redact -- your-process arg1 arg2
+```
+
+Pipe any process output, tail or logs into teller to redact those, live:
+
+```
+$ cat some.log | teller redact
+```
+
+It should also work with `tail -f`:
+
+```
+$ tail -f /var/log/apache.log | teller redact
 ```
 
-By default we treat all entries as sensitive.
+
+Finally, if you've got some files you want to redact, you can do that too:
+
+```
+$ teller --in dirty.csv --out clean.csv
+```
+
+If you omit `--in` Teller will take `stdin`, and if you omit `--out` Teller will output to `stdout`.
+
 
 ## Populate templates
 
@@ -217,6 +254,8 @@ env:
     path: ... # path to value or mapping
     field: <key> # optional: use if path contains a k/v dict
     decrypt: true | false # optional: use if provider supports encryption at the value side
+    severity: high | medium | low | none # optional: used for secret scanning, default is high. 'none' means not a secret 
+    redact_with: "**XXX**" # optional: used as a placeholder swapping the secret with it. default is "**REDACTED**"
   VAR2:
     path: ...
 ```

main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"io"
 	"os"
 
 	"github.com/alecthomas/kong"
@@ -10,12 +11,15 @@ import (
 
 var CLI struct {
 	Config string `short:"c" help:"Path to teller.yml"`
-	Run    struct {
-		Cmd []string `arg name:"cmd" help:"Command to execute"`
+
+	Run struct {
+		Redact bool     `optional name:"redact" help:"Redact output of the child process"`
+		Cmd    []string `arg name:"cmd" help:"Command to execute"`
 	} `cmd help:"Run a command"`
 
 	Version struct {
 	} `cmd short:"v" help:"Teller version"`
+
 	New struct {
 	} `cmd help:"Create a new teller configuration file"`
 
@@ -33,6 +37,11 @@ var CLI struct {
 		OutFile      string `arg name:"out_file" help:"Output file"`
 	} `cmd help:"Inject vars into a template file"`
 
+	Redact struct {
+		In  string `optional name:"in" help:"Input file"`
+		Out string `optional name:"out" help:"Output file"`
+	} `cmd help:"Scans your codebase for sensitive keys"`
+
 	Scan struct {
 		Path   string `arg optional name:"path" help:"Scan root, default: '.'"`
 		Silent bool   `optional name:"silent" help:"No text, just exit code"`
@@ -45,6 +54,7 @@ var (
 	date    = "unknown"
 )
 
+//nolint
 func main() {
 	ctx := kong.Parse(&CLI)
 
@@ -83,7 +93,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	teller := pkg.NewTeller(tlrfile, CLI.Run.Cmd)
+	teller := pkg.NewTeller(tlrfile, CLI.Run.Cmd, CLI.Run.Redact)
 	err = teller.Collect()
 	if err != nil {
 		fmt.Printf("Error: %v", err)
@@ -99,6 +109,37 @@ func main() {
 		}
 		teller.Exec()
 
+	case "redact":
+		// redact (stdin)
+		// redact --in FILE --out FOUT
+		// redact --in FILE (stdout)
+		var fin io.Reader = os.Stdin
+		var fout io.Writer = os.Stdout
+
+		if CLI.Redact.In != "" {
+			f, err := os.Open(CLI.Redact.In)
+			if err != nil {
+				fmt.Printf("Error: %v\n", err)
+				os.Exit(1)
+			}
+			fin = f
+		}
+
+		if CLI.Redact.Out != "" {
+			f, err := os.Create(CLI.Redact.Out)
+			if err != nil {
+				fmt.Printf("Error: %v\n", err)
+				os.Exit(1)
+			}
+
+			fout = f
+		}
+
+		if err := teller.RedactLines(fin, fout); err != nil {
+			fmt.Printf("Error: %v\n", err)
+			os.Exit(1)
+		}
+
 	case "sh":
 		fmt.Print(teller.ExportEnv())
 

pkg/core/types.go

@@ -10,13 +10,14 @@ const (
 )
 
 type KeyPath struct {
-	Env      string            `yaml:"env,omitempty"`
-	Path     string            `yaml:"path"`
-	Field    string            `yaml:"field,omitempty"`
-	Remap    map[string]string `yaml:"remap,omitempty"`
-	Decrypt  bool              `yaml:"decrypt,omitempty"`
-	Optional bool              `yaml:"optional,omitempty"`
-	Severity Severity          `yaml:"severity,omitempty" default:"high"`
+	Env        string            `yaml:"env,omitempty"`
+	Path       string            `yaml:"path"`
+	Field      string            `yaml:"field,omitempty"`
+	Remap      map[string]string `yaml:"remap,omitempty"`
+	Decrypt    bool              `yaml:"decrypt,omitempty"`
+	Optional   bool              `yaml:"optional,omitempty"`
+	Severity   Severity          `yaml:"severity,omitempty" default:"high"`
+	RedactWith string            `yaml:"redact_with,omitempty" default:"**REDACTED**"`
 }
 type WizardAnswers struct {
 	Project      string
@@ -50,12 +51,19 @@ func (a EntriesByKey) Len() int           { return len(a) }
 func (a EntriesByKey) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
 func (a EntriesByKey) Less(i, j int) bool { return a[i].Key > a[j].Key }
 
+type EntriesByValueSize []EnvEntry
+
+func (a EntriesByValueSize) Len() int           { return len(a) }
+func (a EntriesByValueSize) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a EntriesByValueSize) Less(i, j int) bool { return len(a[i].Value) > len(a[j].Value) }
+
 type EnvEntry struct {
 	Key          string
 	Value        string
 	Provider     string
 	ResolvedPath string
 	Severity     Severity
+	RedactWith   string
 }
 type EnvEntryLookup struct {
 	Entries []EnvEntry

pkg/redactor.go

@@ -0,0 +1,30 @@
+package pkg
+
+import (
+	"sort"
+	"strings"
+
+	"github.com/spectralops/teller/pkg/core"
+)
+
+type Redactor struct {
+	Entries []core.EnvEntry
+}
+
+func NewRedactor(entries []core.EnvEntry) *Redactor {
+	return &Redactor{
+		Entries: entries,
+	}
+}
+
+func (r *Redactor) Redact(s string) string {
+	redacted := s
+	entries := append([]core.EnvEntry(nil), r.Entries...)
+
+	sort.Sort(core.EntriesByValueSize(entries))
+	for _, ent := range entries {
+		redacted = strings.ReplaceAll(redacted, ent.Value, ent.RedactWith)
+	}
+
+	return redacted
+}

pkg/redactor_test.go

@@ -0,0 +1,112 @@
+package pkg
+
+import (
+	"testing"
+
+	"github.com/alecthomas/assert"
+	"github.com/spectralops/teller/pkg/core"
+)
+
+func TestRedactorOverlap(t *testing.T) {
+
+	// in this case we dont want '123' to appear in the clear after all redactions are made.
+	// it can happen if the smaller secret get replaced first because both
+	// secrets overlap. we need to ensure the wider secrets always get
+	// replaced first.
+
+	entries := []core.EnvEntry{
+		{
+			Provider:     "test",
+			ResolvedPath: "/some/path",
+			Key:          "OTHER_KEY",
+			Value:        "hello",
+			RedactWith:   "**OTHER_KEY**",
+		},
+		{
+			Provider:     "test",
+			ResolvedPath: "/some/path",
+			Key:          "SOME_KEY",
+			Value:        "hello123",
+			RedactWith:   "**SOME_KEY**",
+		},
+	}
+	redactor := Redactor{Entries: entries}
+	s := `
+	func Foobar(){
+		secret := "hello"
+		callService(secret, "hello123")
+		// hello, hello123
+	}
+	`
+	sr := `
+	func Foobar(){
+		secret := "**OTHER_KEY**"
+		callService(secret, "**SOME_KEY**")
+		// **OTHER_KEY**, **SOME_KEY**
+	}
+	`
+
+	assert.Equal(t, redactor.Redact(s), sr)
+}
+func TestRedactorMultiple(t *testing.T) {
+
+	entries := []core.EnvEntry{
+		{
+			Provider:     "test",
+			ResolvedPath: "/some/path",
+			Key:          "SOME_KEY",
+			Value:        "shazam",
+			RedactWith:   "**SOME_KEY**",
+		},
+		{
+			Provider:     "test",
+			ResolvedPath: "/some/path",
+			Key:          "OTHER_KEY",
+			Value:        "loot",
+			RedactWith:   "**OTHER_KEY**",
+		},
+	}
+	redactor := Redactor{Entries: entries}
+	s := `
+	func Foobar(){
+		secret := "loot"
+		callService(secret, "shazam")
+	}
+	`
+	sr := `
+	func Foobar(){
+		secret := "**OTHER_KEY**"
+		callService(secret, "**SOME_KEY**")
+	}
+	`
+
+	assert.Equal(t, redactor.Redact(s), sr)
+}
+
+func TestRedactor(t *testing.T) {
+
+	entries := []core.EnvEntry{
+		{
+			Provider:     "test",
+			ResolvedPath: "/some/path",
+			Key:          "SOME_KEY",
+			Value:        "shazam",
+			RedactWith:   "**NOPE**",
+		},
+	}
+	redactor := Redactor{Entries: entries}
+	s := `
+	func Foobar(){
+		secret := "shazam"
+		callService(secret, "shazam")
+	}
+	`
+	sr := `
+	func Foobar(){
+		secret := "**NOPE**"
+		callService(secret, "**NOPE**")
+	}
+	`
+
+	assert.Equal(t, redactor.Redact(s), sr)
+}