From 6c67edf1419066fb7e75d487b06b4e43f1646060 Mon Sep 17 00:00:00 2001 From: Guy Levintal <39405738+guylev008@users.noreply.github.com> Date: Mon, 16 May 2022 20:28:52 +0300 Subject: [PATCH] Add sync support for gsm (#105) Add sync support for gsm --- README.md | 8 +- go.mod | 2 +- pkg/providers/google_secretmanager.go | 90 +++++++++++++++++-- pkg/providers/google_secretmanager_test.go | 25 ++++++ .../google_secretmanager_mock.go | 66 +++++++++++++- 5 files changed, 178 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index d6e05549..40f629e5 100644 --- a/README.md +++ b/README.md @@ -691,11 +691,12 @@ You should populate `GOOGLE_APPLICATION_CREDENTIALS=account.json` in your enviro ### Features -* Sync - `no` +* Sync - `yes` * Mapping - `yes` -* Modes - `read`, [write: accepting PR](https://github.com/spectralops/teller) +* Modes - `read+write+delete` * Key format * `env` - path based, needs to include a version + * `env_sync` - your project's path (gets the secrets latest version), when using --sync a new secret version will be created * `decrypt` - available in this provider, will use KMS automatically @@ -703,6 +704,9 @@ You should populate `GOOGLE_APPLICATION_CREDENTIALS=account.json` in your enviro ```yaml google_secretmanager: + env_sync: + # secrets version is not relevant here since we are getting the latest version + path: projects/44882 env: MG_KEY: # need to supply the relevant version (versions/1) diff --git a/go.mod b/go.mod index 798b4edd..12babf3c 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( go.etcd.io/etcd/pkg/v3 v3.5.0-alpha.0 golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 + google.golang.org/api v0.40.0 google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b @@ -151,7 +152,6 @@ require ( golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect - google.golang.org/api v0.40.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/grpc v1.35.0 // indirect google.golang.org/protobuf v1.27.1 // indirect diff --git a/pkg/providers/google_secretmanager.go b/pkg/providers/google_secretmanager.go index 65b58630..67b9a793 100644 --- a/pkg/providers/google_secretmanager.go +++ b/pkg/providers/google_secretmanager.go @@ -3,6 +3,9 @@ package providers import ( "context" "fmt" + "regexp" + "sort" + "strings" secretmanager "cloud.google.com/go/secretmanager/apiv1" secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1" @@ -10,10 +13,14 @@ import ( "github.com/googleapis/gax-go/v2" "github.com/spectralops/teller/pkg/core" "github.com/spectralops/teller/pkg/logging" + "google.golang.org/api/iterator" ) type GoogleSMClient interface { AccessSecretVersion(ctx context.Context, req *secretmanagerpb.AccessSecretVersionRequest, opts ...gax.CallOption) (*secretmanagerpb.AccessSecretVersionResponse, error) + DestroySecretVersion(ctx context.Context, req *secretmanagerpb.DestroySecretVersionRequest, opts ...gax.CallOption) (*secretmanagerpb.SecretVersion, error) + ListSecrets(ctx context.Context, in *secretmanagerpb.ListSecretsRequest, opts ...gax.CallOption) *secretmanager.SecretIterator + AddSecretVersion(ctx context.Context, req *secretmanagerpb.AddSecretVersionRequest, opts ...gax.CallOption) (*secretmanagerpb.SecretVersion, error) } type GoogleSecretManager struct { client GoogleSMClient @@ -33,18 +40,46 @@ func (a *GoogleSecretManager) Name() string { } func (a *GoogleSecretManager) Put(p core.KeyPath, val string) error { - return fmt.Errorf("provider %q does not implement write yet", a.Name()) + reg := regexp.MustCompile(`(?i)/versions/\d+$`) + res := reg.ReplaceAllString(p.Path, "") + return a.addSecret(res, val) } func (a *GoogleSecretManager) PutMapping(p core.KeyPath, m map[string]string) error { - return fmt.Errorf("provider %q does not implement write yet", a.Name()) + for k, v := range m { + path := fmt.Sprintf("%v/secrets/%v", p.Path, k) + err := a.addSecret(path, v) + if err != nil { + return err + } + } + + return nil } func (a *GoogleSecretManager) GetMapping(kp core.KeyPath) ([]core.EnvEntry, error) { - return nil, fmt.Errorf("does not support full env sync (path: %s)", kp.Path) + secrets, err := a.getSecrets(kp.Path) + if err != nil { + return nil, err + } + + entries := []core.EnvEntry{} + + for _, val := range secrets { + path := fmt.Sprintf("%s/%s", val, "versions/latest") + secretVal, err := a.getSecret(path) + if err != nil { + return nil, err + } + key := strings.TrimPrefix(val, kp.Path) + entries = append(entries, kp.FoundWithKey(key, secretVal)) + } + sort.Sort(core.EntriesByKey(entries)) + + return entries, nil } func (a *GoogleSecretManager) Get(p core.KeyPath) (*core.EnvEntry, error) { - secret, err := a.getSecret(p) + secret, err := a.getSecret(p.Path) if err != nil { return nil, err } @@ -54,16 +89,16 @@ func (a *GoogleSecretManager) Get(p core.KeyPath) (*core.EnvEntry, error) { } func (a *GoogleSecretManager) Delete(kp core.KeyPath) error { - return fmt.Errorf("%s does not implement delete yet", a.Name()) + return a.deleteSecret(kp.Path) } func (a *GoogleSecretManager) DeleteMapping(kp core.KeyPath) error { return fmt.Errorf("%s does not implement delete yet", a.Name()) } -func (a *GoogleSecretManager) getSecret(kp core.KeyPath) (string, error) { +func (a *GoogleSecretManager) getSecret(path string) (string, error) { r := secretmanagerpb.AccessSecretVersionRequest{ - Name: kp.Path, + Name: path, } a.logger.WithField("path", r.Name).Debug("get secret") @@ -73,3 +108,44 @@ func (a *GoogleSecretManager) getSecret(kp core.KeyPath) (string, error) { } return string(secret.Payload.Data), nil } + +func (a *GoogleSecretManager) deleteSecret(path string) error { + req := &secretmanagerpb.DestroySecretVersionRequest{ + Name: path, + } + _, err := a.client.DestroySecretVersion(context.TODO(), req) + return err +} + +func (a *GoogleSecretManager) addSecret(path, val string) error { + req := &secretmanagerpb.AddSecretVersionRequest{ + Parent: path, + Payload: &secretmanagerpb.SecretPayload{ + Data: []byte(val), + }, + } + + _, err := a.client.AddSecretVersion(context.TODO(), req) + return err +} + +func (a *GoogleSecretManager) getSecrets(path string) ([]string, error) { + req := &secretmanagerpb.ListSecretsRequest{ + Parent: path, + } + entries := []string{} + + it := a.client.ListSecrets(context.TODO(), req) + for { + resp, err := it.Next() + if err == iterator.Done { + break + } + + if err != nil { + return nil, err + } + entries = append(entries, resp.Name) + } + return entries, nil +} diff --git a/pkg/providers/google_secretmanager_test.go b/pkg/providers/google_secretmanager_test.go index 5909eb09..67c062c3 100644 --- a/pkg/providers/google_secretmanager_test.go +++ b/pkg/providers/google_secretmanager_test.go @@ -4,6 +4,7 @@ import ( "errors" "testing" + secretmanager "cloud.google.com/go/secretmanager/apiv1" "github.com/alecthomas/assert" "github.com/golang/mock/gomock" secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1" @@ -24,10 +25,34 @@ func TestGoogleSM(t *testing.T) { out := &secretmanagerpb.AccessSecretVersionResponse{ Payload: sec, } + outDelete := &secretmanagerpb.SecretVersion{ + Name: string(sec.Data), + } + outAdd := &secretmanagerpb.SecretVersion{ + Name: string(sec.Data), + } + outList := &secretmanager.SecretIterator{ + Response: string(sec.Data), + } in := secretmanagerpb.AccessSecretVersionRequest{ Name: path, } + inDelete := secretmanagerpb.DestroySecretVersionRequest{ + Name: path, + } + inList := secretmanagerpb.ListSecretsRequest{ + Parent: path, + } + inAdd := secretmanagerpb.AddSecretVersionRequest{ + Parent: path, + Payload: &secretmanagerpb.SecretPayload{ + Data: []byte("some value"), + }, + } client.EXPECT().AccessSecretVersion(gomock.Any(), gomock.Eq(&in)).Return(out, nil).AnyTimes() + client.EXPECT().DestroySecretVersion(gomock.Any(), gomock.Eq(&inDelete)).Return(outDelete, nil).AnyTimes() + client.EXPECT().ListSecrets(gomock.Any(), gomock.Eq(&inList)).Return(outList).AnyTimes() + client.EXPECT().AddSecretVersion(gomock.Any(), gomock.Eq(&inAdd)).Return(outAdd, nil).AnyTimes() s := GoogleSecretManager{ client: client, logger: GetTestLogger(), diff --git a/pkg/providers/mock_providers/google_secretmanager_mock.go b/pkg/providers/mock_providers/google_secretmanager_mock.go index a171790f..a78b3c56 100644 --- a/pkg/providers/mock_providers/google_secretmanager_mock.go +++ b/pkg/providers/mock_providers/google_secretmanager_mock.go @@ -8,9 +8,10 @@ import ( context "context" reflect "reflect" + secretmanager "cloud.google.com/go/secretmanager/apiv1" gomock "github.com/golang/mock/gomock" gax "github.com/googleapis/gax-go/v2" - secretmanager "google.golang.org/genproto/googleapis/cloud/secretmanager/v1" + secretmanager0 "google.golang.org/genproto/googleapis/cloud/secretmanager/v1" ) // MockGoogleSMClient is a mock of GoogleSMClient interface. @@ -37,14 +38,14 @@ func (m *MockGoogleSMClient) EXPECT() *MockGoogleSMClientMockRecorder { } // AccessSecretVersion mocks base method. -func (m *MockGoogleSMClient) AccessSecretVersion(ctx context.Context, req *secretmanager.AccessSecretVersionRequest, opts ...gax.CallOption) (*secretmanager.AccessSecretVersionResponse, error) { +func (m *MockGoogleSMClient) AccessSecretVersion(ctx context.Context, req *secretmanager0.AccessSecretVersionRequest, opts ...gax.CallOption) (*secretmanager0.AccessSecretVersionResponse, error) { m.ctrl.T.Helper() varargs := []interface{}{ctx, req} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "AccessSecretVersion", varargs...) - ret0, _ := ret[0].(*secretmanager.AccessSecretVersionResponse) + ret0, _ := ret[0].(*secretmanager0.AccessSecretVersionResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -55,3 +56,62 @@ func (mr *MockGoogleSMClientMockRecorder) AccessSecretVersion(ctx, req interface varargs := append([]interface{}{ctx, req}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AccessSecretVersion", reflect.TypeOf((*MockGoogleSMClient)(nil).AccessSecretVersion), varargs...) } + +// AddSecretVersion mocks base method. +func (m *MockGoogleSMClient) AddSecretVersion(ctx context.Context, req *secretmanager0.AddSecretVersionRequest, opts ...gax.CallOption) (*secretmanager0.SecretVersion, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, req} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "AddSecretVersion", varargs...) + ret0, _ := ret[0].(*secretmanager0.SecretVersion) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AddSecretVersion indicates an expected call of AddSecretVersion. +func (mr *MockGoogleSMClientMockRecorder) AddSecretVersion(ctx, req interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, req}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddSecretVersion", reflect.TypeOf((*MockGoogleSMClient)(nil).AddSecretVersion), varargs...) +} + +// DestroySecretVersion mocks base method. +func (m *MockGoogleSMClient) DestroySecretVersion(ctx context.Context, req *secretmanager0.DestroySecretVersionRequest, opts ...gax.CallOption) (*secretmanager0.SecretVersion, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, req} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DestroySecretVersion", varargs...) + ret0, _ := ret[0].(*secretmanager0.SecretVersion) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DestroySecretVersion indicates an expected call of DestroySecretVersion. +func (mr *MockGoogleSMClientMockRecorder) DestroySecretVersion(ctx, req interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, req}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DestroySecretVersion", reflect.TypeOf((*MockGoogleSMClient)(nil).DestroySecretVersion), varargs...) +} + +// ListSecrets mocks base method. +func (m *MockGoogleSMClient) ListSecrets(ctx context.Context, in *secretmanager0.ListSecretsRequest, opts ...gax.CallOption) *secretmanager.SecretIterator { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListSecrets", varargs...) + ret0, _ := ret[0].(*secretmanager.SecretIterator) + return ret0 +} + +// ListSecrets indicates an expected call of ListSecrets. +func (mr *MockGoogleSMClientMockRecorder) ListSecrets(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListSecrets", reflect.TypeOf((*MockGoogleSMClient)(nil).ListSecrets), varargs...) +}