Merge pull request #16 from SpectralOps/implement-write

First stage of implementing write -- add drift

.gitignore

@@ -3,3 +3,6 @@ dist/
 teller
 .vscode/
 node_modules/
+todo.txt
+.teller.writecase.yml
+coverage.out

Makefile

@@ -25,4 +25,8 @@ deps:
 release:
 	goreleaser --rm-dist
 
-.PHONY: deps setup-mac release readme lint mocks
+coverage:
+	go test ./pkg/... -coverprofile=coverage.out
+	go tool cover -func=coverage.out
+
+.PHONY: deps setup-mac release readme lint mocks coverage

README.md

@@ -95,7 +95,7 @@ Behind the scenes: `teller` fetched the correct variables, placed those (and _ju
 
 # Features
 
-## Running subprocesses
+## :running: Running subprocesses
 
 Manually exporting and setting up environment variables for running a process with demo-like / production-like set up?
 
@@ -108,7 +108,7 @@ Using `teller` and a `.teller.yml` file that exposes nothing to the prying eyes,
 $ teller run -- your-process arg1 arg2... --switch1 ...
 ```
 
-## Inspecting variables
+## :mag_right: Inspecting variables
 
 This will output the current variables `teller` picks up. Only first 2 letters will be shown from each, of course.
 
@@ -117,7 +117,7 @@ This will output the current variables `teller` picks up. Only first 2 letters w
 $ teller show
 ```
 
-## Local shell population
+## :tv: Local shell population
 
 Hardcoding secrets into your shell scripts and dotfiles?
 
@@ -129,7 +129,7 @@ In this case, this is what you should add:
 eval "$(teller sh)"
 ```
 
-## Easy Docker environment
+## :whale: Easy Docker environment
 
 Tired of grabbing all kinds of variables, setting those up, and worried about these appearing in your shell history as well?
 
@@ -139,7 +139,7 @@ Use this one liner from now on:
 $ docker run --rm -it --env-file <(teller env) alpine sh
 ```
 
-## Scan for secrets
+## :warning:  Scan for secrets
 
 Teller can help you fight secret sprawl and hard coded secrets, as well as be the best productivity tool for working with your vault.
 
@@ -173,7 +173,7 @@ dotenv:
 By default we treat all entries as sensitive, with value `high`.
 
 
-## Redact secrets from process outputs, logs, and files
+## :recycle: Redact secrets from process outputs, logs, and files
 
 You can use `teller` as a redaction tool across your infrastructure, and run processes while redacting their output as well as clean up logs and live tails of logs.
 
@@ -204,8 +204,36 @@ $ teller --in dirty.csv --out clean.csv
 
 If you omit `--in` Teller will take `stdin`, and if you omit `--out` Teller will output to `stdout`.
 
+## :beetle: Detect secrets and value drift
 
-## Populate templates
+You can detect _secret drift_ by comparing values from different providers against each other. It might be that you want to pin a set of keys in different providers to always be the same value; when they aren't -- that means you have a drift.
+
+For this, you first need to label values as `source` and couple with the appropriate sink as `sink` (use same label for both to couple them). Then, source keys will be compared against other keys in your configuration:
+
+```yaml
+providers:
+  dotenv:
+    env_sync:
+      path: ~/my-dot-env.env
+      source: s1
+  dotenv2:
+    kind: dotenv
+    env_sync:
+      path: ~/other-dot-env.env
+      sink: s1
+```
+
+And run
+
+```
+$ teller drift dotenv dotenv2 -c your-config.yml
+```
+
+![](https://user-images.githubusercontent.com/83390/117453797-07512380-af4e-11eb-949e-cc875e854fad.png)
+
+
+
+## :scroll: Populate templates
 
 Have a kickstarter project you want to populate quickly with some variables (not secrets though!)?
 
@@ -231,7 +259,7 @@ Will get you, assuming `FOO_BAR=Spock`:
 Hello, Spock!
 ```
 
-## Prompts and options
+## :white_check_mark: Prompts and options
 
 There are a few options that you can use:
 

main.go

@@ -46,6 +46,10 @@ var CLI struct {
 		Path   string `arg optional name:"path" help:"Scan root, default: '.'"`
 		Silent bool   `optional name:"silent" help:"No text, just exit code"`
 	} `cmd help:"Scans your codebase for sensitive keys"`
+
+	Drift struct {
+		Providers []string `arg optional name:"providers" help:"A list of providers to check for drift"`
+	} `cmd help:"Detect secret and value drift between providers"`
 }
 
 var (
@@ -109,6 +113,15 @@ func main() {
 		}
 		teller.Exec()
 
+	case "drift <providers>":
+		fallthrough
+	case "drift":
+		drifts := teller.Drift(CLI.Drift.Providers)
+		if len(drifts) > 0 {
+			teller.Porcelain.PrintDrift(drifts)
+			os.Exit(1)
+		}
+
 	case "redact":
 		// redact (stdin)
 		// redact --in FILE --out FOUT

pkg/core/types.go

@@ -18,6 +18,8 @@ type KeyPath struct {
 	Optional   bool              `yaml:"optional,omitempty"`
 	Severity   Severity          `yaml:"severity,omitempty" default:"high"`
 	RedactWith string            `yaml:"redact_with,omitempty" default:"**REDACTED**"`
+	Source     string            `yaml:"source,omitempty"`
+	Sink       string            `yaml:"sink,omitempty"`
 }
 type WizardAnswers struct {
 	Project      string
@@ -33,6 +35,8 @@ func (k *KeyPath) WithEnv(env string) KeyPath {
 		Field:    k.Field,
 		Decrypt:  k.Decrypt,
 		Optional: k.Optional,
+		Source:   k.Source,
+		Sink:     k.Sink,
 	}
 }
 func (k *KeyPath) SwitchPath(path string) KeyPath {
@@ -42,9 +46,17 @@ func (k *KeyPath) SwitchPath(path string) KeyPath {
 		Env:      k.Env,
 		Decrypt:  k.Decrypt,
 		Optional: k.Optional,
+		Source:   k.Source,
+		Sink:     k.Sink,
 	}
 }
 
+type DriftedEntriesBySource []DriftedEntry
+
+func (a DriftedEntriesBySource) Len() int           { return len(a) }
+func (a DriftedEntriesBySource) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a DriftedEntriesBySource) Less(i, j int) bool { return a[i].Source.Source < a[j].Source.Source }
+
 type EntriesByKey []EnvEntry
 
 func (a EntriesByKey) Len() int           { return len(a) }
@@ -60,26 +72,35 @@ func (a EntriesByValueSize) Less(i, j int) bool { return len(a[i].Value) > len(a
 type EnvEntry struct {
 	Key          string
 	Value        string
+	ProviderName string
 	Provider     string
 	ResolvedPath string
 	Severity     Severity
 	RedactWith   string
+	Source       string
+	Sink         string
+}
+type DriftedEntry struct {
+	Diff   string
+	Source EnvEntry
+	Target EnvEntry
 }
 type EnvEntryLookup struct {
 	Entries []EnvEntry
 }
 
-func (e *EnvEntryLookup) EnvBy(key, provider, path, dflt string) string {
-	for _, e := range e.Entries {
-		if e.Key == key && e.Provider == provider && e.ResolvedPath == path {
+func (ee *EnvEntryLookup) EnvBy(key, provider, path, dflt string) string {
+	for i := range ee.Entries {
+		e := ee.Entries[i]
+		if e.Key == key && e.ProviderName == provider && e.ResolvedPath == path {
 			return e.Value
 		}
-
 	}
 	return dflt
 }
-func (e *EnvEntryLookup) EnvByKey(key, dflt string) string {
-	for _, e := range e.Entries {
+func (ee *EnvEntryLookup) EnvByKey(key, dflt string) string {
+	for i := range ee.Entries {
+		e := ee.Entries[i]
 		if e.Key == key {
 			return e.Value
 		}
@@ -88,9 +109,10 @@ func (e *EnvEntryLookup) EnvByKey(key, dflt string) string {
 	return dflt
 }
 
-func (e *EnvEntryLookup) EnvByKeyAndProvider(key, provider, dflt string) string {
-	for _, e := range e.Entries {
-		if e.Key == key && e.Provider == provider {
+func (ee *EnvEntryLookup) EnvByKeyAndProvider(key, provider, dflt string) string {
+	for i := range ee.Entries {
+		e := ee.Entries[i]
+		if e.Key == key && e.ProviderName == provider {
 			return e.Value
 		}
 

pkg/porcelain.go

@@ -105,12 +105,13 @@ func (p *Porcelain) PrintEntries(entries []core.EnvEntry) {
 	green := color.New(color.FgGreen).SprintFunc()
 	red := color.New(color.FgRed).SprintFunc()
 
-	for _, v := range entries {
+	for i := range entries {
+		v := entries[i]
 		ep := ellipsis.Shorten(v.ResolvedPath, 30)
 		if v.Value == "" {
-			fmt.Fprintf(&buf, "[%s %s %s] %s\n", yellow(v.Provider), gray(ep), red("missing"), green(v.Key))
+			fmt.Fprintf(&buf, "[%s %s %s] %s\n", yellow(v.ProviderName), gray(ep), red("missing"), green(v.Key))
 		} else {
-			fmt.Fprintf(&buf, "[%s %s] %s %s %s\n", yellow(v.Provider), gray(ep), green(v.Key), gray("="), maskedValue(v.Value))
+			fmt.Fprintf(&buf, "[%s %s] %s %s %s\n", yellow(v.ProviderName), gray(ep), green(v.Key), gray("="), maskedValue(v.Value))
 		}
 	}
 
@@ -137,7 +138,7 @@ func (p *Porcelain) PrintMatches(matches []core.Match) {
 		if m.Entry.Severity == core.Medium {
 			sevcolor = yellow
 		}
-		fmt.Printf("[%s] %s (%s,%s): found match for %s/%s (%s)\n", sevcolor(m.Entry.Severity), green(m.Path), yellow(m.LineNumber), yellow(m.MatchIndex), gray(m.Entry.Provider), red(m.Entry.Key), gray(maskedValue(m.Entry.Value)))
+		fmt.Printf("[%s] %s (%s,%s): found match for %s/%s (%s)\n", sevcolor(m.Entry.Severity), green(m.Path), yellow(m.LineNumber), yellow(m.MatchIndex), gray(m.Entry.ProviderName), red(m.Entry.Key), gray(maskedValue(m.Entry.Value)))
 	}
 }
 
@@ -150,3 +151,23 @@ func (p *Porcelain) PrintMatchSummary(findings []core.Match, entries []core.EnvE
 
 	fmt.Printf("Scanning for %v entries: found %v matches in %v\n", yellow(len(entries)), goodbad(len(findings)), goodbad(elapsed))
 }
+
+func (p *Porcelain) PrintDrift(drifts []core.DriftedEntry) {
+	green := color.New(color.FgGreen).SprintFunc()
+	gray := color.New(color.FgHiBlack).SprintFunc()
+	red := color.New(color.FgRed).SprintFunc()
+
+	if len(drifts) > 0 {
+		fmt.Fprintf(p.Out, "Drifts detected: %v\n\n", len(drifts))
+
+		for i := range drifts {
+			d := drifts[i]
+			if d.Diff == "changed" {
+				fmt.Fprintf(p.Out, "%v [%v] %v %v %v != %v %v %v\n", d.Diff, d.Source.Source, green(d.Source.ProviderName), green(d.Source.Key), gray(maskedValue(d.Source.Value)), red(d.Target.ProviderName), red(d.Target.Key), gray(maskedValue(d.Target.Value)))
+			} else {
+				fmt.Fprintf(p.Out, "%v [%v] %v %v %v ??\n", d.Diff, d.Source.Source, green(d.Source.ProviderName), green(d.Source.Key), gray(maskedValue(d.Source.Value)))
+			}
+		}
+	}
+
+}

pkg/porcelain_test.go

@@ -22,8 +22,44 @@ func TestPorcelainNonInteractive(t *testing.T) {
 	b.Reset()
 
 	p.PrintEntries([]core.EnvEntry{
-		{Key: "k", Value: "v", Provider: "test-provider", ResolvedPath: "path/kv"},
+		{Key: "k", Value: "v", Provider: "test-provider", ProviderName: "test-provider", ResolvedPath: "path/kv"},
 	})
 	assert.Equal(t, b.String(), "[test-provider path/kv] k = v*****\n")
 	b.Reset()
 }
+
+func TestPorcelainPrintDrift(t *testing.T) {
+	var b bytes.Buffer
+	p := Porcelain{
+		Out: &b,
+	}
+	p.PrintDrift([]core.DriftedEntry{
+		{
+			Diff: "changed",
+			Source: core.EnvEntry{
+
+				Source: "s1", Key: "k", Value: "v", Provider: "test-provider", ProviderName: "test-provider", ResolvedPath: "path/kv",
+			},
+
+			Target: core.EnvEntry{
+
+				Sink: "s1", Key: "k", Value: "x", Provider: "test-provider", ProviderName: "test-provider", ResolvedPath: "path/kv",
+			},
+		},
+		{
+			Diff: "changed",
+			Source: core.EnvEntry{
+				Source: "s2", Key: "k2", Value: "1", Provider: "test-provider", ProviderName: "test-provider", ResolvedPath: "path/kv",
+			},
+
+			Target: core.EnvEntry{
+				Sink: "s2", Key: "k2", Value: "2", Provider: "test-provider", ProviderName: "test-provider", ResolvedPath: "path/kv",
+			},
+		},
+	})
+	assert.Equal(t, b.String(), `Drifts detected: 2
+
+changed [s1] test-provider k v***** != test-provider k x*****
+changed [s2] test-provider k2 1***** != test-provider k2 2*****
+`)
+}

pkg/redactor.go

@@ -22,7 +22,8 @@ func (r *Redactor) Redact(s string) string {
 	entries := append([]core.EnvEntry(nil), r.Entries...)
 
 	sort.Sort(core.EntriesByValueSize(entries))
-	for _, ent := range entries {
+	for i := range entries {
+		ent := entries[i]
 		redacted = strings.ReplaceAll(redacted, ent.Value, ent.RedactWith)
 	}