Merge pull request #80 from tooploox/bilus/secrets-init

Add `oya secrets init` command

Oyafile

@@ -23,7 +23,7 @@ install: |
 test.Doc: Run tests.
 test: |
   set -e
-  go test -v ./... && \
+  go test -v ./...
   godog $@
 
 test.Doc: Run installer tests.

cmd/internal/secrets.go

@@ -1,13 +1,51 @@
 package internal
 
 import (
+	"encoding/json"
+	"fmt"
 	"io"
 	"os"
 
 	"github.com/pkg/errors"
 	"github.com/tooploox/oya/pkg/secrets"
 )
 
+var ErrUnsupportedType = errors.New("Unsupported type")
+
+func SecretsInit(typ, email, name, desc, format string, stdout, stderr io.Writer) error {
+	if typ != "pgp" {
+		return ErrUnsupportedType
+	}
+
+	keyPair, err := secrets.Init(email, name, desc)
+	if err != nil {
+		return err
+	}
+
+	if err = secrets.GeneratePGPSopsYaml(keyPair); err != nil {
+		return err
+	}
+
+	if err = secrets.ImportPGPKeypair(keyPair); err != nil {
+		return err
+	}
+
+	if format == "json" {
+		b, err := json.MarshalIndent(keyPair, "", "  ")
+		if err != nil {
+			return err
+		}
+		stdout.Write(b)
+	} else {
+		fmt.Fprintf(stdout, "Generated a new PGP key (%q).\n", email)
+		fmt.Fprintf(stdout, "Fingerprint: %v\n", keyPair.Fingerprint)
+		fmt.Fprintf(stdout, "Imported the generated PGP key into GPG.\n")
+		fmt.Fprintf(stdout, "Generated .sops.yaml referencing the new key.\n")
+	}
+
+	return nil
+}
+
 func SecretsView(path string, stdout, stderr io.Writer) error {
 	output, found, err := secrets.Decrypt(path)
 	if err != nil {

cmd/secrets.go

@@ -12,6 +12,36 @@ var secretsCmd = &cobra.Command{
 	Short: "Manage secrets in Oyafile.secrets",
 }
 
+var secretsInitCmd = &cobra.Command{
+	Use:          "init",
+	Short:        "Initialize secret management",
+	SilenceUsage: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		typ, err := cmd.Flags().GetString("type")
+		if err != nil {
+			return err
+		}
+		email, err := cmd.Flags().GetString("email")
+		if err != nil {
+			return err
+		}
+		name, err := cmd.Flags().GetString("name")
+		if err != nil {
+			return err
+		}
+		desc, err := cmd.Flags().GetString("description")
+		if err != nil {
+			return err
+		}
+		format, err := cmd.Flags().GetString("format")
+		if err != nil {
+			return err
+		}
+		return internal.SecretsInit(typ, email, name, desc, format,
+			cmd.OutOrStdout(), cmd.OutOrStderr())
+	},
+}
+
 var secretsViewCmd = &cobra.Command{
 	Use:          "view",
 	Short:        "View secrets",
@@ -53,6 +83,12 @@ var secretsEncryptCmd = &cobra.Command{
 }
 
 func init() {
+	secretsInitCmd.Flags().StringP("type", "t", "pgp", "Key st TODO")
+	secretsInitCmd.Flags().StringP("email", "e", "", "Email address to use to generate the key pair")
+	secretsInitCmd.Flags().StringP("name", "n", "", "Name to use to generate the key pair")
+	secretsInitCmd.Flags().StringP("description", "d", "", "Key pair description")
+	secretsInitCmd.Flags().StringP("format", "f", "text", "Output format (text/json)")
+	secretsCmd.AddCommand(secretsInitCmd)
 	secretsCmd.AddCommand(secretsViewCmd)
 	secretsCmd.AddCommand(secretsEditCmd)
 	secretsCmd.AddCommand(secretsEncryptCmd)

features/run.feature

@@ -337,7 +337,6 @@ Scenario: Command in task exits with non-zero code when set -e is in effect
   And the command exit code is 27
 
 # https://github.com/mvdan/sh/issues/404
-@current
 Scenario: set -e behaves correctly in conditionals etc.
   Given file ./Oyafile containing
     """

features/secrets.feature

@@ -40,12 +40,15 @@ Scenario: Encrypts secrets file
     SECRETPHRASE
     """
 
-@bug
 Scenario: Views secrets file
   Given file ./secrets.oya containing
     """
     foo: SECRETPHRASE
     """
+  Then file ./secrets.oya contains
+    """
+    foo: SECRETPHRASE
+    """
   And I run "oya secrets encrypt secrets.oya"
   Then the command succeeds
   When I run "oya secrets view secrets.oya"
@@ -85,3 +88,30 @@ Scenario: It correctly merges secrets
   peach
 
   """
+
+Scenario: It can quickly generate and import PGP key
+  Given file ./Oyafile containing
+    """
+    Project: Secrets
+    all: |
+      echo ${Oya[foo.bar]}
+      echo ${Oya[foo.baz]}
+    """
+  And file ./secrets2.oya containing
+    """
+    foo:
+      bar: banana
+      baz: peach
+    """
+  And the SOPS_PGP_FP environment variable set to ""
+  When I run "oya secrets init --name 'Oya test key' --email '[email protected]'"
+  And I run "oya secrets encrypt secrets2.oya"
+  And I run "oya run all"
+  Then the command succeeds
+  And the command outputs
+  """
+  banana
+  peach
+
+  """
+  And secrets2.oya is encrypted using PGP key in .sops.yaml

go.mod

@@ -68,6 +68,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/mozilla-services/yaml v0.0.0-20180922153656-28ffe5d0cafb // indirect
+	github.com/mozilla/mig v0.0.0-20190703170622-33eefe9c974e
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/openzipkin/zipkin-go v0.1.3 // indirect
 	github.com/pelletier/go-toml v1.4.0

go.sum

@@ -256,6 +256,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mozilla-services/yaml v0.0.0-20180922153656-28ffe5d0cafb/go.mod h1:Is/Ucts/yU/mWyGR8yELRoO46mejouKsJfQLAIfTR18=
+github.com/mozilla/mig v0.0.0-20190703170622-33eefe9c974e h1:nV1ImdtNbFnwKPky77b7kx1+6AzDjJ+H4XjoxLxw/yU=
+github.com/mozilla/mig v0.0.0-20190703170622-33eefe9c974e/go.mod h1:2c03209qVdj62h4aUMg8KfnbIGC83caKZB8/4F/7YV4=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=

oya_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/tooploox/oya/cmd"
+	"github.com/tooploox/oya/pkg/secrets"
 )
 
 const sopsPgpKey = "317D 6971 DD80 4501 A6B8  65B9 0F1F D46E 2E8C 7202"
@@ -42,8 +43,11 @@ func (c *SuiteContext) MustSetUp() {
 }
 
 func (c *SuiteContext) MustTearDown() {
-	err := os.RemoveAll(c.projectDir)
-	if err != nil {
+	if err := removePGPKeys(c.projectDir); err != nil {
+		panic(err)
+	}
+
+	if err := os.RemoveAll(c.projectDir); err != nil {
 		panic(err)
 	}
 }
@@ -59,6 +63,27 @@ func setEnv(projectDir string) {
 	}
 }
 
+// removePGPKeys removes PGP keys based on fingerprints(s) in .sops.yaml, NOT sopsPgpKey ^.
+func removePGPKeys(projectDir string) error {
+	if err := os.Chdir(projectDir); err != nil {
+		return err
+	}
+	sops, err := secrets.LoadPGPSopsYaml()
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+
+		return err
+	}
+	fingerprints := make([]string, 0)
+
+	for _, rule := range sops.CreationRules {
+		fingerprints = append(fingerprints, strings.Split(rule.PGP, ",")...)
+	}
+	return secrets.RemovePGPKeypairs(fingerprints)
+}
+
 // overrideOyaCmd overrides `oya` command used by $Tasks in templates
 // to run oya tasks.
 // It builds oya to a temporary directory and use it to launch Oya in scripts.
@@ -156,7 +181,7 @@ func (c *SuiteContext) execute(command string) error {
 	cmd.ResetFlags()
 
 	oldArgs := os.Args
-	os.Args = strings.Fields(command)
+	os.Args = parseCommand(command)
 	defer func() {
 		os.Args = oldArgs
 	}()
@@ -165,6 +190,20 @@ func (c *SuiteContext) execute(command string) error {
 	return nil
 }
 
+func parseCommand(command string) []string {
+	argv := make([]string, 0)
+	r := regexp.MustCompile(`([^\s"']+)|"([^"]*)"|'([^']*)'`)
+	matches := r.FindAllStringSubmatch(command, -1)
+	for _, match := range matches {
+		for _, group := range match[1:] {
+			if group != "" {
+				argv = append(argv, group)
+			}
+		}
+	}
+	return argv
+}
+
 func (c *SuiteContext) iRunOya(command string) error {
 	return c.execute("oya " + command)
 }
@@ -230,6 +269,25 @@ func (c *SuiteContext) theCommandExitCodeIs(expectedExitCode int) error {
 	return nil
 }
 
+func (c *SuiteContext) oyafileIsEncryptedUsingKeyInSopsyaml(oyafilePath string) error {
+	sops, err := secrets.LoadPGPSopsYaml()
+	if err != nil {
+		return err
+	}
+	contents, err := ioutil.ReadFile(oyafilePath)
+	if err != nil {
+		return err
+	}
+	for _, rule := range sops.CreationRules {
+		fingerprint := rule.PGP
+		if strings.Contains(string(contents), fingerprint) {
+			return nil
+		}
+	}
+
+	return errors.Errorf("%q not encrypted using key is .sops.yaml", oyafilePath)
+}
+
 func FeatureContext(s *godog.Suite) {
 	c := SuiteContext{}
 	s.Step(`^I'm in project dir$`, c.iAmInProjectDir)
@@ -249,7 +307,7 @@ func FeatureContext(s *godog.Suite) {
 	s.Step(`^the command outputs text matching$`, c.theCommandOutputsTextMatching)
 	s.Step(`^the command exit code is (.+)$`, c.theCommandExitCodeIs)
 	s.Step(`^the ([^ ]+) environment variable set to "([^"]*)"$`, c.environmentVariableSet)
-
+	s.Step(`^([^ ]+) is encrypted using PGP key in .sops.yaml$`, c.oyafileIsEncryptedUsingKeyInSopsyaml)
 	s.BeforeScenario(func(interface{}) { c.MustSetUp() })
 	s.AfterScenario(func(interface{}, error) { c.MustTearDown() })
 }

pkg/secrets/pgp.go

@@ -0,0 +1,81 @@
+package secrets
+
+import (
+	"io/ioutil"
+	"os/exec"
+
+	mig "github.com/mozilla/mig/pgp"
+	"gopkg.in/yaml.v2"
+)
+
+func generatePGPKeyPair(email, name, desc string) (KeyPair, error) {
+	pubkey, privkey, fp, err := mig.GenerateKeyPair(name, desc, email)
+	if err != nil {
+		return KeyPair{}, err
+	}
+	return KeyPair{
+		Public:      string(pubkey),
+		Private:     string(privkey),
+		Fingerprint: string(fp),
+	}, nil
+}
+
+type SopsYaml struct {
+	CreationRules []CreationRule `yaml:"creation_rules"`
+}
+
+type CreationRule struct {
+	PGP string `yaml:"pgp"`
+}
+
+func GeneratePGPSopsYaml(keyPair KeyPair) error {
+	sops := SopsYaml{
+		CreationRules: []CreationRule{
+			{PGP: keyPair.Fingerprint},
+		},
+	}
+
+	content, err := yaml.Marshal(sops)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(".sops.yaml", content, 0644)
+}
+
+func LoadPGPSopsYaml() (SopsYaml, error) {
+	contents, err := ioutil.ReadFile(".sops.yaml")
+	if err != nil {
+		return SopsYaml{}, err
+	}
+
+	var sops SopsYaml
+	return sops, yaml.Unmarshal(contents, &sops)
+}
+
+func ImportPGPKeypair(keyPair KeyPair) error {
+	cmd := exec.Command("gpg", "--import")
+	in, err := cmd.StdinPipe()
+	if err != nil {
+		return err
+	}
+	if err = cmd.Start(); err != nil {
+		return err
+	}
+	if _, err := in.Write(([]byte)(keyPair.Private)); err != nil {
+		return err
+	}
+	in.Close()
+	return cmd.Wait()
+}
+
+func RemovePGPKeypairs(fingerprints []string) error {
+	for _, fingerprint := range fingerprints {
+		if err := exec.Command("gpg", "--batch", "--yes", "--delete-secret-keys", fingerprint).Run(); err != nil {
+			return err
+		}
+		if err := exec.Command("gpg", "--batch", "--yes", "--delete-key", fingerprint).Run(); err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/secrets/secrets.go

@@ -7,6 +7,16 @@ import (
 	"os/exec"
 )
 
+type KeyPair struct {
+	Public      string `json:"public_key"`
+	Private     string `json:"private_key"`
+	Fingerprint string `json:"fingerprint"`
+}
+
+func Init(email, name, desc string) (KeyPair, error) {
+	return generatePGPKeyPair(email, name, desc)
+}
+
 func Decrypt(path string) ([]byte, bool, error) {
 	if ok, err := isSopsFile(path); !ok || err != nil {
 		return nil, false, err