From ad9b6556630118ba73b5112101cec2ab6cb57db2 Mon Sep 17 00:00:00 2001 From: Dotan Nahum Date: Fri, 7 May 2021 14:38:30 +0300 Subject: [PATCH 1/3] prepare --- .gitignore | 2 ++ pkg/core/types.go | 7 +++++++ pkg/providers/heroku.go | 7 +++++++ pkg/teller.go | 27 ++++++++++++++++++--------- pkg/tellerfile.go | 14 ++++++++------ 5 files changed, 42 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index f06384f6..506d19ce 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ dist/ teller .vscode/ node_modules/ +todo.txt +.teller.writecase.yml diff --git a/pkg/core/types.go b/pkg/core/types.go index 2cb1e187..22411b47 100644 --- a/pkg/core/types.go +++ b/pkg/core/types.go @@ -18,6 +18,8 @@ type KeyPath struct { Optional bool `yaml:"optional,omitempty"` Severity Severity `yaml:"severity,omitempty" default:"high"` RedactWith string `yaml:"redact_with,omitempty" default:"**REDACTED**"` + IsSource bool `yaml:"source,omitempty"` + Handle string `yaml:"handle,omitempty"` } type WizardAnswers struct { Project string @@ -33,6 +35,8 @@ func (k *KeyPath) WithEnv(env string) KeyPath { Field: k.Field, Decrypt: k.Decrypt, Optional: k.Optional, + IsSource: k.IsSource, + Handle: k.Handle, } } func (k *KeyPath) SwitchPath(path string) KeyPath { @@ -42,6 +46,8 @@ func (k *KeyPath) SwitchPath(path string) KeyPath { Env: k.Env, Decrypt: k.Decrypt, Optional: k.Optional, + IsSource: k.IsSource, + Handle: k.Handle, } } @@ -64,6 +70,7 @@ type EnvEntry struct { ResolvedPath string Severity Severity RedactWith string + Handle string } type EnvEntryLookup struct { Entries []EnvEntry diff --git a/pkg/providers/heroku.go b/pkg/providers/heroku.go index bf3bf9c6..6a8b6910 100644 --- a/pkg/providers/heroku.go +++ b/pkg/providers/heroku.go @@ -21,6 +21,7 @@ func NewHeroku() (core.Provider, error) { heroku.DefaultTransport.BearerToken = os.Getenv("HEROKU_API_KEY") svc := heroku.NewService(heroku.DefaultClient) + //svc.ConfigVarUpdate() return &Heroku{client: svc}, nil } @@ -77,3 +78,9 @@ func (h *Heroku) Get(p core.KeyPath) (*core.EnvEntry, error) { func (h *Heroku) getSecret(kp core.KeyPath) (heroku.ConfigVarInfoForAppResult, error) { return h.client.ConfigVarInfoForApp(context.TODO(), kp.Path) } + +/* +func (h *Heroku) setSecret(kp core.KeyPath) (heroku.ConfigVarInfoForAppResult, error) { + h.client. +} +*/ diff --git a/pkg/teller.go b/pkg/teller.go index 085c2a49..22033aa5 100644 --- a/pkg/teller.go +++ b/pkg/teller.go @@ -311,22 +311,18 @@ func updateParams(ent *core.EnvEntry, from *core.KeyPath) { } } -// The main "load all variables from all providers" logic. Walks over all definitions in the tellerfile -// and then: fetches, converts, creates a new EnvEntry. We're also mapping the sensitivity aspects of it. -// Note that for a similarly named entry - last one wins. -func (tl *Teller) Collect() error { - t := tl.Config +func (tl *Teller) CollectFromProviderMap(ps *ProvidersMap) ([]core.EnvEntry, error) { entries := []core.EnvEntry{} - for pname, conf := range t.Providers { + for pname, conf := range *ps { p, err := tl.Providers.GetProvider(pname) if err != nil { - return err + return nil, err } if conf.EnvMapping != nil { es, err := p.GetMapping(tl.Populate.KeyPath(*conf.EnvMapping)) if err != nil { - return err + return nil, err } // optionally remap environment variables synced from the provider @@ -347,7 +343,7 @@ func (tl *Teller) Collect() error { if v.Optional { continue } else { - return err + return nil, err } } else { //nolint @@ -359,6 +355,19 @@ func (tl *Teller) Collect() error { } sort.Sort(core.EntriesByKey(entries)) + return entries, nil +} + +// The main "load all variables from all providers" logic. Walks over all definitions in the tellerfile +// and then: fetches, converts, creates a new EnvEntry. We're also mapping the sensitivity aspects of it. +// Note that for a similarly named entry - last one wins. +func (tl *Teller) Collect() error { + t := tl.Config + entries, err := tl.CollectFromProviderMap(&t.Providers) + if err != nil { + return err + } + tl.Entries = entries tl.Redactor = NewRedactor(entries) return nil diff --git a/pkg/tellerfile.go b/pkg/tellerfile.go index 4c7a8bad..fad25427 100644 --- a/pkg/tellerfile.go +++ b/pkg/tellerfile.go @@ -7,13 +7,15 @@ import ( "gopkg.in/yaml.v2" ) +type ProvidersMap map[string]MappingConfig type TellerFile struct { - Opts map[string]string `yaml:"opts,omitempty"` - Confirm string `yaml:"confirm,omitempty"` - Project string `yaml:"project,omitempty"` - CarryEnv bool `yaml:"carry_env,omitempty"` - Providers map[string]MappingConfig `yaml:"providers,omitempty"` - LoadedFrom string + Opts map[string]string `yaml:"opts,omitempty"` + Confirm string `yaml:"confirm,omitempty"` + Project string `yaml:"project,omitempty"` + CarryEnv bool `yaml:"carry_env,omitempty"` + Providers ProvidersMap `yaml:"providers,omitempty"` + Environments map[string]ProvidersMap `yaml:"providers,omitempty"` + LoadedFrom string } type MappingConfig struct { From 1eb99a4346f175b1a044766a125ef538bd228fc3 Mon Sep 17 00:00:00 2001 From: Dotan Nahum Date: Fri, 7 May 2021 17:50:12 +0300 Subject: [PATCH 2/3] implement drift, basics of syncing --- .gitignore | 1 + Makefile | 6 +++- README.md | 44 +++++++++++++++++++----- main.go | 13 ++++++++ pkg/core/types.go | 43 ++++++++++++++++-------- pkg/porcelain.go | 29 +++++++++++++--- pkg/porcelain_test.go | 38 ++++++++++++++++++++- pkg/providers/heroku.go | 7 ---- pkg/redactor.go | 3 +- pkg/teller.go | 74 ++++++++++++++++++++++++++++++++++++----- pkg/teller_test.go | 33 ++++++++++++++++-- pkg/tellerfile.go | 14 ++++---- 12 files changed, 252 insertions(+), 53 deletions(-) diff --git a/.gitignore b/.gitignore index 506d19ce..5bbd55f1 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ teller node_modules/ todo.txt .teller.writecase.yml +coverage.out diff --git a/Makefile b/Makefile index 5e0ca5e6..38954399 100644 --- a/Makefile +++ b/Makefile @@ -25,4 +25,8 @@ deps: release: goreleaser --rm-dist -.PHONY: deps setup-mac release readme lint mocks +coverage: + go test ./pkg/... -coverprofile=coverage.out + go tool cover -func=coverage.out + +.PHONY: deps setup-mac release readme lint mocks coverage diff --git a/README.md b/README.md index 25dbba7a..6718572f 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ Behind the scenes: `teller` fetched the correct variables, placed those (and _ju # Features -## Running subprocesses +## :running: Running subprocesses Manually exporting and setting up environment variables for running a process with demo-like / production-like set up? @@ -108,7 +108,7 @@ Using `teller` and a `.teller.yml` file that exposes nothing to the prying eyes, $ teller run -- your-process arg1 arg2... --switch1 ... ``` -## Inspecting variables +## :mag_right: Inspecting variables This will output the current variables `teller` picks up. Only first 2 letters will be shown from each, of course. @@ -117,7 +117,7 @@ This will output the current variables `teller` picks up. Only first 2 letters w $ teller show ``` -## Local shell population +## :tv: Local shell population Hardcoding secrets into your shell scripts and dotfiles? @@ -129,7 +129,7 @@ In this case, this is what you should add: eval "$(teller sh)" ``` -## Easy Docker environment +## :whale: Easy Docker environment Tired of grabbing all kinds of variables, setting those up, and worried about these appearing in your shell history as well? @@ -139,7 +139,7 @@ Use this one liner from now on: $ docker run --rm -it --env-file <(teller env) alpine sh ``` -## Scan for secrets +## :warning: Scan for secrets Teller can help you fight secret sprawl and hard coded secrets, as well as be the best productivity tool for working with your vault. @@ -173,7 +173,7 @@ dotenv: By default we treat all entries as sensitive, with value `high`. -## Redact secrets from process outputs, logs, and files +## :recycle: Redact secrets from process outputs, logs, and files You can use `teller` as a redaction tool across your infrastructure, and run processes while redacting their output as well as clean up logs and live tails of logs. @@ -204,8 +204,36 @@ $ teller --in dirty.csv --out clean.csv If you omit `--in` Teller will take `stdin`, and if you omit `--out` Teller will output to `stdout`. +## :beetle: Detect secrets and value drift -## Populate templates +You can detect _secret drift_ by comparing values from different providers against each other. It might be that you want to pin a set of keys in different providers to always be the same value; when they aren't -- that means you have a drift. + +For this, you first need to label values as `source` and couple with the appropriate sink as `sink` (use same label for both to couple them). Then, source keys will be compared against other keys in your configuration: + +```yaml +providers: + dotenv: + env_sync: + path: ~/my-dot-env.env + source: s1 + dotenv2: + kind: dotenv + env_sync: + path: ~/other-dot-env.env + sink: s1 +``` + +And run + +``` +$ teller drift dotenv dotenv2 -c your-config.yml +``` + +![](https://user-images.githubusercontent.com/83390/117453797-07512380-af4e-11eb-949e-cc875e854fad.png) + + + +## :scroll: Populate templates Have a kickstarter project you want to populate quickly with some variables (not secrets though!)? @@ -231,7 +259,7 @@ Will get you, assuming `FOO_BAR=Spock`: Hello, Spock! ``` -## Prompts and options +## :white_check_mark: Prompts and options There are a few options that you can use: diff --git a/main.go b/main.go index 68ab9226..ec8c6702 100644 --- a/main.go +++ b/main.go @@ -46,6 +46,10 @@ var CLI struct { Path string `arg optional name:"path" help:"Scan root, default: '.'"` Silent bool `optional name:"silent" help:"No text, just exit code"` } `cmd help:"Scans your codebase for sensitive keys"` + + Drift struct { + Providers []string `arg optional name:"providers" help:"A list of providers to check for drift"` + } `cmd help:"Scans your codebase for sensitive keys"` } var ( @@ -109,6 +113,15 @@ func main() { } teller.Exec() + case "drift ": + fallthrough + case "drift": + drifts := teller.Drift(CLI.Drift.Providers) + if len(drifts) > 0 { + teller.Porcelain.PrintDrift(drifts) + os.Exit(1) + } + case "redact": // redact (stdin) // redact --in FILE --out FOUT diff --git a/pkg/core/types.go b/pkg/core/types.go index 22411b47..e701b0d4 100644 --- a/pkg/core/types.go +++ b/pkg/core/types.go @@ -18,8 +18,8 @@ type KeyPath struct { Optional bool `yaml:"optional,omitempty"` Severity Severity `yaml:"severity,omitempty" default:"high"` RedactWith string `yaml:"redact_with,omitempty" default:"**REDACTED**"` - IsSource bool `yaml:"source,omitempty"` - Handle string `yaml:"handle,omitempty"` + Source string `yaml:"source,omitempty"` + Sink string `yaml:"sink,omitempty"` } type WizardAnswers struct { Project string @@ -35,8 +35,8 @@ func (k *KeyPath) WithEnv(env string) KeyPath { Field: k.Field, Decrypt: k.Decrypt, Optional: k.Optional, - IsSource: k.IsSource, - Handle: k.Handle, + Source: k.Source, + Sink: k.Sink, } } func (k *KeyPath) SwitchPath(path string) KeyPath { @@ -46,11 +46,17 @@ func (k *KeyPath) SwitchPath(path string) KeyPath { Env: k.Env, Decrypt: k.Decrypt, Optional: k.Optional, - IsSource: k.IsSource, - Handle: k.Handle, + Source: k.Source, + Sink: k.Sink, } } +type DriftedEntriesBySource []DriftedEntry + +func (a DriftedEntriesBySource) Len() int { return len(a) } +func (a DriftedEntriesBySource) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a DriftedEntriesBySource) Less(i, j int) bool { return a[i].Source.Source < a[j].Source.Source } + type EntriesByKey []EnvEntry func (a EntriesByKey) Len() int { return len(a) } @@ -66,27 +72,35 @@ func (a EntriesByValueSize) Less(i, j int) bool { return len(a[i].Value) > len(a type EnvEntry struct { Key string Value string + ProviderName string Provider string ResolvedPath string Severity Severity RedactWith string - Handle string + Source string + Sink string +} +type DriftedEntry struct { + Diff string + Source EnvEntry + Target EnvEntry } type EnvEntryLookup struct { Entries []EnvEntry } -func (e *EnvEntryLookup) EnvBy(key, provider, path, dflt string) string { - for _, e := range e.Entries { +func (ee *EnvEntryLookup) EnvBy(key, provider, path, dflt string) string { + for i := range ee.Entries { + e := ee.Entries[i] if e.Key == key && e.Provider == provider && e.ResolvedPath == path { return e.Value } - } return dflt } -func (e *EnvEntryLookup) EnvByKey(key, dflt string) string { - for _, e := range e.Entries { +func (ee *EnvEntryLookup) EnvByKey(key, dflt string) string { + for i := range ee.Entries { + e := ee.Entries[i] if e.Key == key { return e.Value } @@ -95,8 +109,9 @@ func (e *EnvEntryLookup) EnvByKey(key, dflt string) string { return dflt } -func (e *EnvEntryLookup) EnvByKeyAndProvider(key, provider, dflt string) string { - for _, e := range e.Entries { +func (ee *EnvEntryLookup) EnvByKeyAndProvider(key, provider, dflt string) string { + for i := range ee.Entries { + e := ee.Entries[i] if e.Key == key && e.Provider == provider { return e.Value } diff --git a/pkg/porcelain.go b/pkg/porcelain.go index 8e6a3e0a..5659e101 100644 --- a/pkg/porcelain.go +++ b/pkg/porcelain.go @@ -105,12 +105,13 @@ func (p *Porcelain) PrintEntries(entries []core.EnvEntry) { green := color.New(color.FgGreen).SprintFunc() red := color.New(color.FgRed).SprintFunc() - for _, v := range entries { + for i := range entries { + v := entries[i] ep := ellipsis.Shorten(v.ResolvedPath, 30) if v.Value == "" { - fmt.Fprintf(&buf, "[%s %s %s] %s\n", yellow(v.Provider), gray(ep), red("missing"), green(v.Key)) + fmt.Fprintf(&buf, "[%s %s %s] %s\n", yellow(v.ProviderName), gray(ep), red("missing"), green(v.Key)) } else { - fmt.Fprintf(&buf, "[%s %s] %s %s %s\n", yellow(v.Provider), gray(ep), green(v.Key), gray("="), maskedValue(v.Value)) + fmt.Fprintf(&buf, "[%s %s] %s %s %s\n", yellow(v.ProviderName), gray(ep), green(v.Key), gray("="), maskedValue(v.Value)) } } @@ -137,7 +138,7 @@ func (p *Porcelain) PrintMatches(matches []core.Match) { if m.Entry.Severity == core.Medium { sevcolor = yellow } - fmt.Printf("[%s] %s (%s,%s): found match for %s/%s (%s)\n", sevcolor(m.Entry.Severity), green(m.Path), yellow(m.LineNumber), yellow(m.MatchIndex), gray(m.Entry.Provider), red(m.Entry.Key), gray(maskedValue(m.Entry.Value))) + fmt.Printf("[%s] %s (%s,%s): found match for %s/%s (%s)\n", sevcolor(m.Entry.Severity), green(m.Path), yellow(m.LineNumber), yellow(m.MatchIndex), gray(m.Entry.ProviderName), red(m.Entry.Key), gray(maskedValue(m.Entry.Value))) } } @@ -150,3 +151,23 @@ func (p *Porcelain) PrintMatchSummary(findings []core.Match, entries []core.EnvE fmt.Printf("Scanning for %v entries: found %v matches in %v\n", yellow(len(entries)), goodbad(len(findings)), goodbad(elapsed)) } + +func (p *Porcelain) PrintDrift(drifts []core.DriftedEntry) { + green := color.New(color.FgGreen).SprintFunc() + gray := color.New(color.FgHiBlack).SprintFunc() + red := color.New(color.FgRed).SprintFunc() + + if len(drifts) > 0 { + fmt.Fprintf(p.Out, "Drifts detected: %v\n\n", len(drifts)) + + for i := range drifts { + d := drifts[i] + if d.Diff == "changed" { + fmt.Fprintf(p.Out, "%v [%v] %v %v %v != %v %v %v\n", d.Diff, d.Source.Source, green(d.Source.ProviderName), green(d.Source.Key), gray(maskedValue(d.Source.Value)), red(d.Target.ProviderName), red(d.Target.Key), gray(maskedValue(d.Target.Value))) + } else { + fmt.Fprintf(p.Out, "%v [%v] %v %v %v ??\n", d.Diff, d.Source.Source, green(d.Source.ProviderName), green(d.Source.Key), gray(maskedValue(d.Source.Value))) + } + } + } + +} diff --git a/pkg/porcelain_test.go b/pkg/porcelain_test.go index 77e5a385..7e3b0e8c 100644 --- a/pkg/porcelain_test.go +++ b/pkg/porcelain_test.go @@ -22,8 +22,44 @@ func TestPorcelainNonInteractive(t *testing.T) { b.Reset() p.PrintEntries([]core.EnvEntry{ - {Key: "k", Value: "v", Provider: "test-provider", ResolvedPath: "path/kv"}, + {Key: "k", Value: "v", Provider: "test-provider", ProviderName: "test-provider", ResolvedPath: "path/kv"}, }) assert.Equal(t, b.String(), "[test-provider path/kv] k = v*****\n") b.Reset() } + +func TestPorcelainPrintDrift(t *testing.T) { + var b bytes.Buffer + p := Porcelain{ + Out: &b, + } + p.PrintDrift([]core.DriftedEntry{ + { + Diff: "changed", + Source: core.EnvEntry{ + + Source: "s1", Key: "k", Value: "v", Provider: "test-provider", ProviderName: "test-provider", ResolvedPath: "path/kv", + }, + + Target: core.EnvEntry{ + + Sink: "s1", Key: "k", Value: "x", Provider: "test-provider", ProviderName: "test-provider", ResolvedPath: "path/kv", + }, + }, + { + Diff: "changed", + Source: core.EnvEntry{ + Source: "s2", Key: "k2", Value: "1", Provider: "test-provider", ProviderName: "test-provider", ResolvedPath: "path/kv", + }, + + Target: core.EnvEntry{ + Sink: "s2", Key: "k2", Value: "2", Provider: "test-provider", ProviderName: "test-provider", ResolvedPath: "path/kv", + }, + }, + }) + assert.Equal(t, b.String(), `Drifts detected: 2 + +changed [s1] test-provider k v***** != test-provider k x***** +changed [s2] test-provider k2 1***** != test-provider k2 2***** +`) +} diff --git a/pkg/providers/heroku.go b/pkg/providers/heroku.go index 6a8b6910..bf3bf9c6 100644 --- a/pkg/providers/heroku.go +++ b/pkg/providers/heroku.go @@ -21,7 +21,6 @@ func NewHeroku() (core.Provider, error) { heroku.DefaultTransport.BearerToken = os.Getenv("HEROKU_API_KEY") svc := heroku.NewService(heroku.DefaultClient) - //svc.ConfigVarUpdate() return &Heroku{client: svc}, nil } @@ -78,9 +77,3 @@ func (h *Heroku) Get(p core.KeyPath) (*core.EnvEntry, error) { func (h *Heroku) getSecret(kp core.KeyPath) (heroku.ConfigVarInfoForAppResult, error) { return h.client.ConfigVarInfoForApp(context.TODO(), kp.Path) } - -/* -func (h *Heroku) setSecret(kp core.KeyPath) (heroku.ConfigVarInfoForAppResult, error) { - h.client. -} -*/ diff --git a/pkg/redactor.go b/pkg/redactor.go index 557cff23..0011a3f4 100644 --- a/pkg/redactor.go +++ b/pkg/redactor.go @@ -22,7 +22,8 @@ func (r *Redactor) Redact(s string) string { entries := append([]core.EnvEntry(nil), r.Entries...) sort.Sort(core.EntriesByValueSize(entries)) - for _, ent := range entries { + for i := range entries { + ent := entries[i] redacted = strings.ReplaceAll(redacted, ent.Value, ent.RedactWith) } diff --git a/pkg/teller.go b/pkg/teller.go index 22033aa5..bfd620b8 100644 --- a/pkg/teller.go +++ b/pkg/teller.go @@ -67,7 +67,8 @@ func (tl *Teller) execCmd(cmd string, cmdArgs []string, withRedaction bool) erro command.Env = append(command.Env, funk.Map([]string{"USER", "HOME", "PATH"}, func(k string) string { return fmt.Sprintf("%s=%s", k, os.Getenv(k)) }).([]string)...) } else { - for _, b := range tl.Entries { + for i := range tl.Entries { + b := tl.Entries[i] os.Setenv(b.Key, b.Value) } } @@ -95,7 +96,8 @@ func (tl *Teller) ExportEnv() string { var b bytes.Buffer fmt.Fprintf(&b, "#!/bin/sh\n") - for _, v := range tl.Entries { + for i := range tl.Entries { + v := tl.Entries[i] fmt.Fprintf(&b, "export %s=%s\n", v.Key, v.Value) } return b.String() @@ -105,7 +107,8 @@ func (tl *Teller) ExportEnv() string { func (tl *Teller) ExportDotenv() string { var b bytes.Buffer - for _, v := range tl.Entries { + for i := range tl.Entries { + v := tl.Entries[i] fmt.Fprintf(&b, "%s=%s\n", v.Key, v.Value) } return b.String() @@ -214,7 +217,8 @@ func checkForMatches(path string, entries []core.EnvEntry) ([]core.Match, error) } linestr := string(line) - for _, ent := range entries { + for i := range entries { + ent := entries[i] if ent.Value == "" || ent.Severity == core.None { continue } @@ -297,7 +301,11 @@ func (tl *Teller) TemplateFile(from, to string) error { return nil } -func updateParams(ent *core.EnvEntry, from *core.KeyPath) { +func updateParams(ent *core.EnvEntry, from *core.KeyPath, pname string) { + ent.ProviderName = pname + ent.Source = from.Source + ent.Sink = from.Sink + if from.Severity == "" { ent.Severity = core.High } else { @@ -315,6 +323,12 @@ func (tl *Teller) CollectFromProviderMap(ps *ProvidersMap) ([]core.EnvEntry, err entries := []core.EnvEntry{} for pname, conf := range *ps { p, err := tl.Providers.GetProvider(pname) + if err != nil { + // ok, maybe same provider, with 'kind'? + p, err = tl.Providers.GetProvider(conf.Kind) + } + + // still no provider? bail. if err != nil { return nil, err } @@ -325,18 +339,20 @@ func (tl *Teller) CollectFromProviderMap(ps *ProvidersMap) ([]core.EnvEntry, err return nil, err } - // optionally remap environment variables synced from the provider + //nolint for k, v := range es { + // optionally remap environment variables synced from the provider if val, ok := conf.EnvMapping.Remap[v.Key]; ok { es[k].Key = val - updateParams(&es[k], conf.EnvMapping) } + updateParams(&es[k], conf.EnvMapping, pname) } entries = append(entries, es...) } if conf.Env != nil { + //nolint for k, v := range *conf.Env { ent, err := p.Get(tl.Populate.KeyPath(v.WithEnv(k))) if err != nil { @@ -347,7 +363,7 @@ func (tl *Teller) CollectFromProviderMap(ps *ProvidersMap) ([]core.EnvEntry, err } } else { //nolint - updateParams(ent, &v) + updateParams(ent, &v, pname) entries = append(entries, *ent) } } @@ -372,3 +388,45 @@ func (tl *Teller) Collect() error { tl.Redactor = NewRedactor(entries) return nil } + +func (tl *Teller) Drift(providerNames []string) []core.DriftedEntry { + sources := map[string]core.EnvEntry{} + targets := map[string][]core.EnvEntry{} + filtering := len(providerNames) > 0 + for i := range tl.Entries { + ent := tl.Entries[i] + if filtering && !funk.ContainsString(providerNames, ent.ProviderName) { + continue + } + if ent.Source != "" { + sources[ent.Source+":"+ent.Key] = ent + } else if ent.Sink != "" { + k := ent.Sink + ":" + ent.Key + ents := targets[k] + if ents == nil { + targets[k] = []core.EnvEntry{ent} + } else { + targets[k] = append(ents, ent) + } + } + } + + drifts := []core.DriftedEntry{} + + //nolint + for sk, source := range sources { + ents := targets[sk] + if ents == nil { + drifts = append(drifts, core.DriftedEntry{Diff: "missing", Source: source}) + } + + for _, e := range ents { + if e.Value != source.Value { + drifts = append(drifts, core.DriftedEntry{Diff: "changed", Source: source, Target: e}) + } + } + } + + sort.Sort(core.DriftedEntriesBySource(drifts)) + return drifts +} diff --git a/pkg/teller_test.go b/pkg/teller_test.go index 01104b19..f0c39374 100644 --- a/pkg/teller_test.go +++ b/pkg/teller_test.go @@ -52,6 +52,7 @@ func (im *InMemProvider) GetMapping(p core.KeyPath) ([]core.EnvEntry, error) { Value: v, ResolvedPath: p.Path, Provider: im.Name(), + ProviderName: im.Name(), }) } sort.Sort(core.EntriesByKey(entries)) @@ -67,6 +68,7 @@ func (im *InMemProvider) Get(p core.KeyPath) (*core.EnvEntry, error) { Value: s, ResolvedPath: p.Path, Provider: im.Name(), + ProviderName: im.Name(), }, nil } @@ -81,7 +83,7 @@ func TestTellerExports(t *testing.T) { tl = Teller{ Entries: []core.EnvEntry{ - {Key: "k", Value: "v", Provider: "test-provider", ResolvedPath: "path/kv"}, + {Key: "k", Value: "v", Provider: "test-provider", ProviderName: "test-provider", ResolvedPath: "path/kv"}, }, } @@ -216,10 +218,37 @@ func TestTellerPorcelainNonInteractive(t *testing.T) { b.Reset() tl.Entries = append(tl.Entries, core.EnvEntry{ - Key: "k", Value: "v", Provider: "test-provider", ResolvedPath: "path/kv", + Key: "k", Value: "v", Provider: "test-provider", ProviderName: "test-provider", ResolvedPath: "path/kv", }) tl.PrintEnvKeys() assert.Equal(t, b.String(), "-*- teller: loaded variables for test-project using nowhere -*-\n\n[test-provider path/kv] k = v*****\n") } + +func TestTellerDrift(t *testing.T) { + tl := Teller{ + Entries: []core.EnvEntry{}, + Providers: &BuiltinProviders{}, + } + + tl = Teller{ + Entries: []core.EnvEntry{ + {Key: "k", Value: "v", Source: "s1", Provider: "test-provider", ProviderName: "test-provider", ResolvedPath: "path/kv"}, + {Key: "k", Value: "v", Sink: "s1", Provider: "test-provider", ProviderName: "test-provider2", ResolvedPath: "path/kv"}, + {Key: "kX", Value: "vx", Source: "s1", Provider: "test-provider", ProviderName: "test-provider", ResolvedPath: "path/kv"}, + {Key: "kX", Value: "CHANGED", Sink: "s1", Provider: "test-provider", ProviderName: "test-provider2", ResolvedPath: "path/kv"}, + + // these do not have sink/source + {Key: "k--", Value: "00", Provider: "test-provider", ProviderName: "test-provider", ResolvedPath: "path/kv"}, + {Key: "k--", Value: "11", Provider: "test-provider", ProviderName: "test-provider2", ResolvedPath: "path/kv"}, + }, + } + + drifts := tl.Drift([]string{}) + + assert.Equal(t, len(drifts), 1) + d := drifts[0] + assert.Equal(t, d.Source.Value, "vx") + assert.Equal(t, d.Target.Value, "CHANGED") +} diff --git a/pkg/tellerfile.go b/pkg/tellerfile.go index fad25427..daf62c43 100644 --- a/pkg/tellerfile.go +++ b/pkg/tellerfile.go @@ -9,16 +9,16 @@ import ( type ProvidersMap map[string]MappingConfig type TellerFile struct { - Opts map[string]string `yaml:"opts,omitempty"` - Confirm string `yaml:"confirm,omitempty"` - Project string `yaml:"project,omitempty"` - CarryEnv bool `yaml:"carry_env,omitempty"` - Providers ProvidersMap `yaml:"providers,omitempty"` - Environments map[string]ProvidersMap `yaml:"providers,omitempty"` - LoadedFrom string + Opts map[string]string `yaml:"opts,omitempty"` + Confirm string `yaml:"confirm,omitempty"` + Project string `yaml:"project,omitempty"` + CarryEnv bool `yaml:"carry_env,omitempty"` + Providers ProvidersMap `yaml:"providers,omitempty"` + LoadedFrom string } type MappingConfig struct { + Kind string `yaml:"kind,omitempty"` EnvMapping *core.KeyPath `yaml:"env_sync,omitempty"` Env *map[string]core.KeyPath `yaml:"env,omitempty"` } From a36443801a1d0289597ee43d9f7ca57aee4b4dfc Mon Sep 17 00:00:00 2001 From: Dotan Nahum Date: Fri, 7 May 2021 17:57:45 +0300 Subject: [PATCH 3/3] small fixes --- main.go | 2 +- pkg/core/types.go | 4 ++-- pkg/teller_test.go | 5 ----- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/main.go b/main.go index ec8c6702..a0d319f4 100644 --- a/main.go +++ b/main.go @@ -49,7 +49,7 @@ var CLI struct { Drift struct { Providers []string `arg optional name:"providers" help:"A list of providers to check for drift"` - } `cmd help:"Scans your codebase for sensitive keys"` + } `cmd help:"Detect secret and value drift between providers"` } var ( diff --git a/pkg/core/types.go b/pkg/core/types.go index e701b0d4..cf11772e 100644 --- a/pkg/core/types.go +++ b/pkg/core/types.go @@ -92,7 +92,7 @@ type EnvEntryLookup struct { func (ee *EnvEntryLookup) EnvBy(key, provider, path, dflt string) string { for i := range ee.Entries { e := ee.Entries[i] - if e.Key == key && e.Provider == provider && e.ResolvedPath == path { + if e.Key == key && e.ProviderName == provider && e.ResolvedPath == path { return e.Value } } @@ -112,7 +112,7 @@ func (ee *EnvEntryLookup) EnvByKey(key, dflt string) string { func (ee *EnvEntryLookup) EnvByKeyAndProvider(key, provider, dflt string) string { for i := range ee.Entries { e := ee.Entries[i] - if e.Key == key && e.Provider == provider { + if e.Key == key && e.ProviderName == provider { return e.Value } diff --git a/pkg/teller_test.go b/pkg/teller_test.go index f0c39374..705e4ef3 100644 --- a/pkg/teller_test.go +++ b/pkg/teller_test.go @@ -228,11 +228,6 @@ func TestTellerPorcelainNonInteractive(t *testing.T) { func TestTellerDrift(t *testing.T) { tl := Teller{ - Entries: []core.EnvEntry{}, - Providers: &BuiltinProviders{}, - } - - tl = Teller{ Entries: []core.EnvEntry{ {Key: "k", Value: "v", Source: "s1", Provider: "test-provider", ProviderName: "test-provider", ResolvedPath: "path/kv"}, {Key: "k", Value: "v", Sink: "s1", Provider: "test-provider", ProviderName: "test-provider2", ResolvedPath: "path/kv"},