diff --git a/.golangci.yml b/.golangci.yml index c75d0ad8..06ca2b97 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -20,6 +20,7 @@ linters-settings: - octalLiteral - whyNoLint - wrapperFunc + - hugeParam gocyclo: min-complexity: 20 golint: diff --git a/.teller.example.yml b/.teller.example.yml index b0018e07..35d97810 100644 --- a/.teller.example.yml +++ b/.teller.example.yml @@ -16,6 +16,7 @@ providers: env: FOO_BAR: path: ~/my-dot-env.env + redact_with: "**FOOBAR**" # optional # # requires an API key in: HEROKU_API_KEY (you can fetch from ~/.netrc) # heroku: diff --git a/README.md b/README.md index aa99b3d9..ff1ea53f 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,11 @@ Use this one liner from now on: $ docker run --rm -it --env-file <(teller env) alpine sh ``` -## Scan for hardcoded secrets +## Scan for secrets + +Teller can help you fight secret sprawl and hard coded secrets, as well as be the best productivity tool for working with your vault. + +It can also integrate into your CI and serve as a shift-left security tool for your DevSecOps pipeline. Look for your vault-kept secrets in your code by running: @@ -136,17 +140,50 @@ run: teller scan --silent It will break your build if it finds something (returns exit code `1`). -Use Teller for productively and securely running process and you get this for free -- nothing to configure. If you have data that you're bringing that you're sure isn't sensitive, flag it in your `teller.yml`: +Use Teller for productively and securely running your processes and you get this for free -- nothing to configure. If you have data that you're bringing that you're sure isn't sensitive, flag it in your `teller.yml`: ``` dotenv: env: FOO: path: ~/my-dot-env.env - not_sensitive: true + severity: none # will skip scanning. possible values: high | medium | low | none +``` + +By default we treat all entries as sensitive, with value `high`. + + +## Redact secrets from process outputs, logs, and files + +You can use `teller` as a redaction tool across your infrastructure, and run processes while redacting their output as well as clean up logs and live tails of logs. + +Run a process and redact its output in real time: + +``` +$ teller run --redact -- your-process arg1 arg2 +``` + +Pipe any process output, tail or logs into teller to redact those, live: + +``` +$ cat some.log | teller redact +``` + +It should also work with `tail -f`: + +``` +$ tail -f /var/log/apache.log | teller redact ``` -By default we treat all entries as sensitive. + +Finally, if you've got some files you want to redact, you can do that too: + +``` +$ teller --in dirty.csv --out clean.csv +``` + +If you omit `--in` Teller will take `stdin`, and if you omit `--out` Teller will output to `stdout`. + ## Populate templates @@ -217,6 +254,8 @@ env: path: ... # path to value or mapping field: # optional: use if path contains a k/v dict decrypt: true | false # optional: use if provider supports encryption at the value side + severity: high | medium | low | none # optional: used for secret scanning, default is high. 'none' means not a secret + redact_with: "**XXX**" # optional: used as a placeholder swapping the secret with it. default is "**REDACTED**" VAR2: path: ... ``` diff --git a/main.go b/main.go index 576b2d0d..68ab9226 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "io" "os" "github.com/alecthomas/kong" @@ -10,12 +11,15 @@ import ( var CLI struct { Config string `short:"c" help:"Path to teller.yml"` - Run struct { - Cmd []string `arg name:"cmd" help:"Command to execute"` + + Run struct { + Redact bool `optional name:"redact" help:"Redact output of the child process"` + Cmd []string `arg name:"cmd" help:"Command to execute"` } `cmd help:"Run a command"` Version struct { } `cmd short:"v" help:"Teller version"` + New struct { } `cmd help:"Create a new teller configuration file"` @@ -33,6 +37,11 @@ var CLI struct { OutFile string `arg name:"out_file" help:"Output file"` } `cmd help:"Inject vars into a template file"` + Redact struct { + In string `optional name:"in" help:"Input file"` + Out string `optional name:"out" help:"Output file"` + } `cmd help:"Scans your codebase for sensitive keys"` + Scan struct { Path string `arg optional name:"path" help:"Scan root, default: '.'"` Silent bool `optional name:"silent" help:"No text, just exit code"` @@ -45,6 +54,7 @@ var ( date = "unknown" ) +//nolint func main() { ctx := kong.Parse(&CLI) @@ -83,7 +93,7 @@ func main() { os.Exit(1) } - teller := pkg.NewTeller(tlrfile, CLI.Run.Cmd) + teller := pkg.NewTeller(tlrfile, CLI.Run.Cmd, CLI.Run.Redact) err = teller.Collect() if err != nil { fmt.Printf("Error: %v", err) @@ -99,6 +109,37 @@ func main() { } teller.Exec() + case "redact": + // redact (stdin) + // redact --in FILE --out FOUT + // redact --in FILE (stdout) + var fin io.Reader = os.Stdin + var fout io.Writer = os.Stdout + + if CLI.Redact.In != "" { + f, err := os.Open(CLI.Redact.In) + if err != nil { + fmt.Printf("Error: %v\n", err) + os.Exit(1) + } + fin = f + } + + if CLI.Redact.Out != "" { + f, err := os.Create(CLI.Redact.Out) + if err != nil { + fmt.Printf("Error: %v\n", err) + os.Exit(1) + } + + fout = f + } + + if err := teller.RedactLines(fin, fout); err != nil { + fmt.Printf("Error: %v\n", err) + os.Exit(1) + } + case "sh": fmt.Print(teller.ExportEnv()) diff --git a/pkg/core/types.go b/pkg/core/types.go index 40322d0b..2cb1e187 100644 --- a/pkg/core/types.go +++ b/pkg/core/types.go @@ -10,13 +10,14 @@ const ( ) type KeyPath struct { - Env string `yaml:"env,omitempty"` - Path string `yaml:"path"` - Field string `yaml:"field,omitempty"` - Remap map[string]string `yaml:"remap,omitempty"` - Decrypt bool `yaml:"decrypt,omitempty"` - Optional bool `yaml:"optional,omitempty"` - Severity Severity `yaml:"severity,omitempty" default:"high"` + Env string `yaml:"env,omitempty"` + Path string `yaml:"path"` + Field string `yaml:"field,omitempty"` + Remap map[string]string `yaml:"remap,omitempty"` + Decrypt bool `yaml:"decrypt,omitempty"` + Optional bool `yaml:"optional,omitempty"` + Severity Severity `yaml:"severity,omitempty" default:"high"` + RedactWith string `yaml:"redact_with,omitempty" default:"**REDACTED**"` } type WizardAnswers struct { Project string @@ -50,12 +51,19 @@ func (a EntriesByKey) Len() int { return len(a) } func (a EntriesByKey) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a EntriesByKey) Less(i, j int) bool { return a[i].Key > a[j].Key } +type EntriesByValueSize []EnvEntry + +func (a EntriesByValueSize) Len() int { return len(a) } +func (a EntriesByValueSize) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a EntriesByValueSize) Less(i, j int) bool { return len(a[i].Value) > len(a[j].Value) } + type EnvEntry struct { Key string Value string Provider string ResolvedPath string Severity Severity + RedactWith string } type EnvEntryLookup struct { Entries []EnvEntry diff --git a/pkg/redactor.go b/pkg/redactor.go new file mode 100644 index 00000000..557cff23 --- /dev/null +++ b/pkg/redactor.go @@ -0,0 +1,30 @@ +package pkg + +import ( + "sort" + "strings" + + "github.com/spectralops/teller/pkg/core" +) + +type Redactor struct { + Entries []core.EnvEntry +} + +func NewRedactor(entries []core.EnvEntry) *Redactor { + return &Redactor{ + Entries: entries, + } +} + +func (r *Redactor) Redact(s string) string { + redacted := s + entries := append([]core.EnvEntry(nil), r.Entries...) + + sort.Sort(core.EntriesByValueSize(entries)) + for _, ent := range entries { + redacted = strings.ReplaceAll(redacted, ent.Value, ent.RedactWith) + } + + return redacted +} diff --git a/pkg/redactor_test.go b/pkg/redactor_test.go new file mode 100644 index 00000000..6f74cf83 --- /dev/null +++ b/pkg/redactor_test.go @@ -0,0 +1,112 @@ +package pkg + +import ( + "testing" + + "github.com/alecthomas/assert" + "github.com/spectralops/teller/pkg/core" +) + +func TestRedactorOverlap(t *testing.T) { + + // in this case we dont want '123' to appear in the clear after all redactions are made. + // it can happen if the smaller secret get replaced first because both + // secrets overlap. we need to ensure the wider secrets always get + // replaced first. + + entries := []core.EnvEntry{ + { + Provider: "test", + ResolvedPath: "/some/path", + Key: "OTHER_KEY", + Value: "hello", + RedactWith: "**OTHER_KEY**", + }, + { + Provider: "test", + ResolvedPath: "/some/path", + Key: "SOME_KEY", + Value: "hello123", + RedactWith: "**SOME_KEY**", + }, + } + redactor := Redactor{Entries: entries} + s := ` + func Foobar(){ + secret := "hello" + callService(secret, "hello123") + // hello, hello123 + } + ` + sr := ` + func Foobar(){ + secret := "**OTHER_KEY**" + callService(secret, "**SOME_KEY**") + // **OTHER_KEY**, **SOME_KEY** + } + ` + + assert.Equal(t, redactor.Redact(s), sr) +} +func TestRedactorMultiple(t *testing.T) { + + entries := []core.EnvEntry{ + { + Provider: "test", + ResolvedPath: "/some/path", + Key: "SOME_KEY", + Value: "shazam", + RedactWith: "**SOME_KEY**", + }, + { + Provider: "test", + ResolvedPath: "/some/path", + Key: "OTHER_KEY", + Value: "loot", + RedactWith: "**OTHER_KEY**", + }, + } + redactor := Redactor{Entries: entries} + s := ` + func Foobar(){ + secret := "loot" + callService(secret, "shazam") + } + ` + sr := ` + func Foobar(){ + secret := "**OTHER_KEY**" + callService(secret, "**SOME_KEY**") + } + ` + + assert.Equal(t, redactor.Redact(s), sr) +} + +func TestRedactor(t *testing.T) { + + entries := []core.EnvEntry{ + { + Provider: "test", + ResolvedPath: "/some/path", + Key: "SOME_KEY", + Value: "shazam", + RedactWith: "**NOPE**", + }, + } + redactor := Redactor{Entries: entries} + s := ` + func Foobar(){ + secret := "shazam" + callService(secret, "shazam") + } + ` + sr := ` + func Foobar(){ + secret := "**NOPE**" + callService(secret, "**NOPE**") + } + ` + + assert.Equal(t, redactor.Redact(s), sr) +} diff --git a/pkg/teller.go b/pkg/teller.go index 6c435189..085c2a49 100644 --- a/pkg/teller.go +++ b/pkg/teller.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "fmt" + "io" "io/ioutil" "os" "os/exec" @@ -24,6 +25,7 @@ import ( // Entries - when loaded, these contains the mapped entries. Load them with Collect() // Templating - Teller's templating options. type Teller struct { + Redact bool Cmd []string Config *TellerFile Porcelain *Porcelain @@ -31,17 +33,20 @@ type Teller struct { Providers Providers Entries []core.EnvEntry Templating *Templating + Redactor *Redactor } // Create a new Teller instance, using a tellerfile, and a command to execute (if any) -func NewTeller(tlrfile *TellerFile, cmd []string) *Teller { +func NewTeller(tlrfile *TellerFile, cmd []string, redact bool) *Teller { return &Teller{ + Redact: redact, Config: tlrfile, Cmd: cmd, Providers: &BuiltinProviders{}, Populate: core.NewPopulate(tlrfile.Opts), Porcelain: &Porcelain{Out: os.Stdout}, Templating: &Templating{}, + Redactor: &Redactor{}, } } @@ -52,7 +57,7 @@ func bail(e error) { } // execute a command, and take care to sanitize the child process environment (conditionally) -func (tl *Teller) execCmd(cmd string, cmdArgs []string) error { +func (tl *Teller) execCmd(cmd string, cmdArgs []string, withRedaction bool) error { command := exec.Command(cmd, cmdArgs...) if !tl.Config.CarryEnv { command.Env = funk.Map(tl.Entries, func(ent interface{}) string { @@ -66,6 +71,13 @@ func (tl *Teller) execCmd(cmd string, cmdArgs []string) error { os.Setenv(b.Key, b.Value) } } + if withRedaction { + out, err := command.CombinedOutput() + redacted := tl.Redactor.Redact(string(out)) + os.Stdout.Write([]byte(redacted)) + return err + } + command.Stdin = os.Stdin command.Stdout = os.Stdout command.Stderr = os.Stderr @@ -130,6 +142,27 @@ func (tl *Teller) SetupNewProject(fname string) error { return nil } +// Execute a command with teller. This requires all entries to be loaded beforehand with Collect() +func (tl *Teller) RedactLines(r io.Reader, w io.Writer) error { + scanner := bufio.NewScanner(r) + //nolint + buf := make([]byte, 0, 64*1024) + //nolint + scanner.Buffer(buf, 10*1024*1024) // 10MB lines correlating to 10MB files max (bundles?) + + for scanner.Scan() { + if _, err := fmt.Fprintln(w, tl.Redactor.Redact(string(scanner.Bytes()))); err != nil { + return err + } + } + + if err := scanner.Err(); err != nil { + return err + } + + return nil +} + // Execute a command with teller. This requires all entries to be loaded beforehand with Collect() func (tl *Teller) Exec() { tl.Porcelain.PrintContext(tl.Config.Project, tl.Config.LoadedFrom) @@ -142,7 +175,7 @@ func (tl *Teller) Exec() { } } - err := tl.execCmd(tl.Cmd[0], tl.Cmd[1:]) + err := tl.execCmd(tl.Cmd[0], tl.Cmd[1:], tl.Redact) if err != nil { bail(err) } @@ -270,6 +303,12 @@ func updateParams(ent *core.EnvEntry, from *core.KeyPath) { } else { ent.Severity = from.Severity } + + if from.RedactWith == "" { + ent.RedactWith = "**REDACTED**" + } else { + ent.RedactWith = from.RedactWith + } } // The main "load all variables from all providers" logic. Walks over all definitions in the tellerfile @@ -321,5 +360,6 @@ func (tl *Teller) Collect() error { sort.Sort(core.EntriesByKey(entries)) tl.Entries = entries + tl.Redactor = NewRedactor(entries) return nil }