diff --git a/.travis.yml b/.travis.yml index 3573c1a..194dc2f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ before_install: - export PATH="$GOPATH/bin:$PATH" - docker run -d -p 2379:2379 quay.io/coreos/etcd /usr/local/bin/etcd -advertise-client-urls http://0.0.0.0:2379 -listen-client-urls http://0.0.0.0:2379 - docker run -d -p 8500:8500 --name consul consul - - docker run -d -p 8200:8200 --cap-add=IPC_LOCK -e 'VAULT_DEV_ROOT_TOKEN_ID=root' -e 'VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200' vault + - docker run -d -p 8200:8200 --cap-add=IPC_LOCK -e 'VAULT_DEV_ROOT_TOKEN_ID=root' -e 'VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200' vault:0.9.6 env: - VAULT_ADDR=http://127.0.0.1:8200 diff --git a/Gopkg.lock b/Gopkg.lock index d275a98..d9597d4 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -11,8 +11,8 @@ "mvcc/mvccpb", "pkg/types" ] - revision = "c9d46ab3799b7f2174268e75f72d01e6d6aac953" - version = "v3.3.2" + revision = "e348b1aedd9167360c466ae98f7343d3e22281f8" + version = "v3.3.3" [[projects]] name = "github.com/davecgh/go-spew" @@ -29,8 +29,8 @@ [[projects]] name = "github.com/go-yaml/yaml" packages = ["."] - revision = "7f97868eec74b32b0982dd158a51a446d1da7eb5" - version = "v2.1.1" + revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" + version = "v2.2.1" [[projects]] name = "github.com/gogo/protobuf" @@ -63,8 +63,8 @@ [[projects]] name = "github.com/hashicorp/consul" packages = ["api"] - revision = "9a494b5fb9c86180a5702e29c485df1507a47198" - version = "v1.0.6" + revision = "fb848fc48818f58690db09d14640513aa6bf3c02" + version = "v1.0.7" [[projects]] branch = "master" @@ -104,7 +104,7 @@ "json/scanner", "json/token" ] - revision = "f40e974e75af4e271d97ce0fc917af5898ae7bda" + revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168" [[projects]] name = "github.com/hashicorp/serf" @@ -174,6 +174,7 @@ name = "golang.org/x/net" packages = [ "context", + "http/httpguts", "http2", "http2/hpack", "idna", @@ -181,7 +182,7 @@ "lex/httplex", "trace" ] - revision = "6078986fec03a1dcc236c34816c71b0e05018fda" + revision = "d41e8174641f662c5a2d1c7a5f9e828788eb8706" [[projects]] name = "golang.org/x/text" @@ -208,7 +209,7 @@ branch = "master" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] - revision = "f8c8703595236ae70fdf8789ecb656ea0bcdcf46" + revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200" [[projects]] name = "google.golang.org/grpc" @@ -238,12 +239,12 @@ "tap", "transport" ] - revision = "8e4536a86ab602859c20df5ebfd0bd4228d08655" - version = "v1.10.0" + revision = "d11072e7ca9811b1100b80ca0269ac831f06d024" + version = "v1.11.3" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "d7e2815c5a5540ba66d586045277848589a2145bb8ada25465a1d217d2f43a85" + inputs-digest = "dd119391aa68184558f6ef082ab5f3155d9b87ec9e987c896a34bb9dcabe1bc0" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 9382ee3..f9feb64 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -20,7 +20,7 @@ [[constraint]] name = "github.com/hashicorp/vault" - version = "0.9.3" + version = "0.9.0" [prune] go-tests = true diff --git a/backend/consul/consul.go b/backend/consul/consul.go index 4328b4c..8b2e5c1 100644 --- a/backend/consul/consul.go +++ b/backend/consul/consul.go @@ -3,6 +3,7 @@ package consul import ( "context" "path" + "strings" "github.com/hashicorp/consul/api" "github.com/heetch/confita/backend" @@ -10,20 +11,38 @@ import ( // Backend loads keys from Consul. type Backend struct { - client *api.Client - prefix string + client *api.Client + prefix string + prefetch bool + cache map[string][]byte } // NewBackend creates a configuration loader that loads from Consul. -func NewBackend(client *api.Client, prefix string) *Backend { - return &Backend{ +func NewBackend(client *api.Client, opts ...Option) *Backend { + b := Backend{ client: client, - prefix: prefix, } + + for _, opt := range opts { + opt(&b) + } + + return &b } // Get loads the given key from Consul. func (b *Backend) Get(ctx context.Context, key string) ([]byte, error) { + if b.cache == nil && b.prefetch { + err := b.fetchTree(ctx) + if err != nil { + return nil, err + } + } + + if b.cache != nil { + return b.fromCache(ctx, key) + } + var opt api.QueryOptions kv, _, err := b.client.KV().Get(path.Join(b.prefix, key), opt.WithContext(ctx)) @@ -38,7 +57,52 @@ func (b *Backend) Get(ctx context.Context, key string) ([]byte, error) { return kv.Value, nil } +func (b *Backend) fetchTree(ctx context.Context) error { + var opt api.QueryOptions + + list, _, err := b.client.KV().List(b.prefix, opt.WithContext(ctx)) + if err != nil { + return err + } + + b.cache = make(map[string][]byte) + + for _, kv := range list { + b.cache[strings.TrimLeft(kv.Key, b.prefix+"/")] = kv.Value + } + + return nil +} + +func (b *Backend) fromCache(ctx context.Context, key string) ([]byte, error) { + v, ok := b.cache[key] + if !ok { + return nil, backend.ErrNotFound + } + + return v, nil +} + // Name returns the name of the backend. func (b *Backend) Name() string { return "consul" } + +// Option is used to configure the Consul backend. +type Option func(*Backend) + +// WithPrefix is used to specify the prefix to prepend on every keys. +func WithPrefix(prefix string) Option { + return func(b *Backend) { + b.prefix = prefix + } +} + +// WithPrefetch is used to prefetch the entire tree and cache it to +// avoid further round trips. If the WithPrefix option is used, will fetch +// the tree under the specified prefix. +func WithPrefetch() Option { + return func(b *Backend) { + b.prefetch = true + } +} diff --git a/backend/consul/consul_test.go b/backend/consul/consul_test.go index 5f43b4d..c42544d 100644 --- a/backend/consul/consul_test.go +++ b/backend/consul/consul_test.go @@ -16,7 +16,7 @@ func TestConsulBackend(t *testing.T) { require.NoError(t, err) defer client.KV().DeleteTree(prefix, nil) - b := NewBackend(client, prefix) + b := NewBackend(client, WithPrefix(prefix)) t.Run("NotFound", func(t *testing.T) { _, err = b.Get(context.Background(), "something that doesn't exist") @@ -42,3 +42,45 @@ func TestConsulBackend(t *testing.T) { require.Error(t, err) }) } + +func TestConsulBackendWithPrefetch(t *testing.T) { + prefix := "confita-tests" + + client, err := api.NewClient(api.DefaultConfig()) + require.NoError(t, err) + defer client.KV().DeleteTree(prefix, nil) + + b := NewBackend(client, WithPrefix(prefix), WithPrefetch()) + + t.Run("OK", func(t *testing.T) { + _, err = client.KV().Put(&api.KVPair{Key: prefix + "/key1", Value: []byte("value1")}, nil) + require.NoError(t, err) + + _, err = client.KV().Put(&api.KVPair{Key: prefix + "/key2", Value: []byte("value2")}, nil) + require.NoError(t, err) + + _, err = client.KV().Put(&api.KVPair{Key: prefix + "/key3", Value: []byte("value3")}, nil) + require.NoError(t, err) + + val, err := b.Get(context.Background(), "key1") + require.NoError(t, err) + + // deleting the tree + client.KV().DeleteTree(prefix, nil) + + // WithPrefetch should have prefetched all the keys + // they should be available even if the tree has been removed. + val, err = b.Get(context.Background(), "key1") + require.NoError(t, err) + require.Equal(t, []byte("value1"), val) + + val, err = b.Get(context.Background(), "key2") + require.NoError(t, err) + require.Equal(t, []byte("value2"), val) + + val, err = b.Get(context.Background(), "key3") + require.NoError(t, err) + require.Equal(t, []byte("value3"), val) + }) + +} diff --git a/backend/etcd/etcd.go b/backend/etcd/etcd.go index ef04db3..c47c203 100644 --- a/backend/etcd/etcd.go +++ b/backend/etcd/etcd.go @@ -3,6 +3,7 @@ package etcd import ( "context" "path" + "strings" "github.com/coreos/etcd/clientv3" "github.com/heetch/confita/backend" @@ -10,20 +11,38 @@ import ( // Backend loads keys from etcd. type Backend struct { - client *clientv3.Client - prefix string + client *clientv3.Client + prefix string + prefetch bool + cache map[string][]byte } // NewBackend creates a configuration loader that loads from etcd. -func NewBackend(client *clientv3.Client, prefix string) *Backend { - return &Backend{ +func NewBackend(client *clientv3.Client, opts ...Option) *Backend { + b := Backend{ client: client, - prefix: prefix, } + + for _, opt := range opts { + opt(&b) + } + + return &b } // Get loads the given key from etcd. func (b *Backend) Get(ctx context.Context, key string) ([]byte, error) { + if b.cache == nil && b.prefetch { + err := b.fetchTree(ctx) + if err != nil { + return nil, err + } + } + + if b.cache != nil { + return b.fromCache(ctx, key) + } + resp, err := b.client.Get(ctx, path.Join(b.prefix, key)) if err != nil { return nil, err @@ -36,7 +55,50 @@ func (b *Backend) Get(ctx context.Context, key string) ([]byte, error) { return resp.Kvs[0].Value, nil } +func (b *Backend) fetchTree(ctx context.Context) error { + resp, err := b.client.KV.Get(ctx, b.prefix, clientv3.WithPrefix()) + if err != nil { + return err + } + + b.cache = make(map[string][]byte) + + for _, kv := range resp.Kvs { + b.cache[strings.TrimLeft(string(kv.Key), b.prefix+"/")] = kv.Value + } + + return nil +} + +func (b *Backend) fromCache(ctx context.Context, key string) ([]byte, error) { + v, ok := b.cache[key] + if !ok { + return nil, backend.ErrNotFound + } + + return v, nil +} + // Name returns the name of the backend. func (b *Backend) Name() string { return "etcd" } + +// Option is used to configure the Consul backend. +type Option func(*Backend) + +// WithPrefix is used to specify the prefix to prepend on every keys. +func WithPrefix(prefix string) Option { + return func(b *Backend) { + b.prefix = prefix + } +} + +// WithPrefetch is used to prefetch the entire tree and cache it to +// avoid further round trips. If the WithPrefix option is used, will fetch +// the tree under the specified prefix. +func WithPrefetch() Option { + return func(b *Backend) { + b.prefetch = true + } +} diff --git a/backend/etcd/etcd_test.go b/backend/etcd/etcd_test.go index 76ddc17..cce3641 100644 --- a/backend/etcd/etcd_test.go +++ b/backend/etcd/etcd_test.go @@ -16,7 +16,55 @@ func TestEtcdBackend(t *testing.T) { require.NoError(t, err) defer client.Close() - b := NewBackend(client, "prefix") + b := NewBackend(client, WithPrefix("prefix")) _, err = b.Get(context.Background(), "something that doesn't exist") require.Equal(t, backend.ErrNotFound, err) } + +func TestEtcdBackendWithPrefetch(t *testing.T) { + client, err := clientv3.New(clientv3.Config{ + Endpoints: []string{"localhost:2379"}, + }) + require.NoError(t, err) + defer client.Close() + + prefix := "confita-tests" + + ctx := context.Background() + + defer client.KV.Delete(ctx, prefix, clientv3.WithPrefix()) + + b := NewBackend(client, WithPrefix(prefix), WithPrefetch()) + + t.Run("OK", func(t *testing.T) { + _, err = client.KV.Put(ctx, prefix+"/key1", "value1") + require.NoError(t, err) + + _, err = client.KV.Put(ctx, prefix+"/key2", "value2") + require.NoError(t, err) + + _, err = client.KV.Put(ctx, prefix+"/key3", "value3") + require.NoError(t, err) + + val, err := b.Get(ctx, "key1") + require.NoError(t, err) + + // deleting the tree + client.KV.Delete(ctx, prefix, clientv3.WithPrefix()) + + // WithPrefetch should have prefetched all the keys + // they should be available even if the tree has been removed. + val, err = b.Get(ctx, "key1") + require.NoError(t, err) + require.Equal(t, []byte("value1"), val) + + val, err = b.Get(ctx, "key2") + require.NoError(t, err) + require.Equal(t, []byte("value2"), val) + + val, err = b.Get(ctx, "key3") + require.NoError(t, err) + require.Equal(t, []byte("value3"), val) + }) + +} diff --git a/backend/vault/vault.go b/backend/vault/vault.go index 0178582..9103c61 100644 --- a/backend/vault/vault.go +++ b/backend/vault/vault.go @@ -12,28 +12,34 @@ import ( type Backend struct { client *api.Logical path string + secret *api.Secret } // NewBackend creates a configuration loader that loads from Vault -func NewBackend(c *api.Logical, p string) *Backend { +// all the keys from the given path and holds them in memory. +func NewBackend(client *api.Logical, path string) *Backend { return &Backend{ - client: c, - path: p, + client: client, + path: path, } } -// Get loads the given key from Vault +// Get loads the given key from Vault. func (b *Backend) Get(ctx context.Context, key string) ([]byte, error) { - secret, err := b.client.Read(b.path) - if err != nil { - return nil, err - } + var err error + + if b.secret == nil { + b.secret, err = b.client.Read(b.path) + if err != nil { + return nil, err + } - if secret == nil { - return nil, fmt.Errorf("secret not found at the following path: %s", b.path) + if b.secret == nil { + return nil, fmt.Errorf("secret not found at the following path: %s", b.path) + } } - if v, ok := secret.Data[key]; ok { + if v, ok := b.secret.Data[key]; ok { return []byte(v.(string)), nil } diff --git a/backend/vault/vault_test.go b/backend/vault/vault_test.go index 35be31a..a133185 100644 --- a/backend/vault/vault_test.go +++ b/backend/vault/vault_test.go @@ -2,6 +2,7 @@ package vault import ( "context" + "os" "testing" "github.com/hashicorp/vault/api" @@ -11,6 +12,7 @@ import ( ) func TestVaultBackend(t *testing.T) { + os.Setenv("VAULT_ADDR", "http://127.0.0.1:8200") client, err := api.NewClient(api.DefaultConfig()) require.NoError(t, err) @@ -18,14 +20,18 @@ func TestVaultBackend(t *testing.T) { c := client.Logical() path := "secret/test" - b := NewBackend(c, path) + + defer c.Delete(path) t.Run("SecretPathNotFound", func(t *testing.T) { + b := NewBackend(c, path) _, err := b.Get(context.Background(), "foo") require.EqualError(t, err, "secret not found at the following path: secret/test") }) t.Run("OK", func(t *testing.T) { + b := NewBackend(c, path) + _, err = c.Write(path, map[string]interface{}{ "foo": "bar", @@ -43,6 +49,7 @@ func TestVaultBackend(t *testing.T) { }) t.Run("NotFound", func(t *testing.T) { + b := NewBackend(c, path) _, err := b.Get(context.Background(), "badKey") require.EqualError(t, err, backend.ErrNotFound.Error()) }) diff --git a/config.go b/config.go index 014bea1..377de53 100644 --- a/config.go +++ b/config.go @@ -36,6 +36,14 @@ func NewLoader(backends ...backend.Backend) *Loader { return &l } +type fieldConfig struct { + Name string + Key string + Value *reflect.Value + Required bool + Backend string +} + // Load analyses all the fields of the given struct for a "config" tag and queries each backend // in order for the corresponding key. The given context can be used for timeout and cancelation. func (l *Loader) Load(ctx context.Context, to interface{}) error { @@ -52,10 +60,15 @@ func (l *Loader) Load(ctx context.Context, to interface{}) error { } ref = ref.Elem() - return l.parseStruct(ctx, &ref) + + fields := l.parseStruct(&ref) + + return l.resolve(ctx, fields) } -func (l *Loader) parseStruct(ctx context.Context, ref *reflect.Value) error { +func (l *Loader) parseStruct(ref *reflect.Value) []*fieldConfig { + var list []*fieldConfig + t := ref.Type() numFields := ref.NumField() @@ -82,50 +95,53 @@ func (l *Loader) parseStruct(ctx context.Context, ref *reflect.Value) error { // if struct or *struct, parse recursively switch { case typ.Kind() == reflect.Struct: - err := l.parseStruct(ctx, &value) - if err != nil { - return err - } - + list = append(list, l.parseStruct(&value)...) continue case typ.Kind() == reflect.Ptr: if value.Type().Elem().Kind() == reflect.Struct && !value.IsNil() { value = value.Elem() - - err := l.parseStruct(ctx, &value) - if err != nil { - return err - } - + list = append(list, l.parseStruct(&value)...) continue } } + // empty tag or no tag, skip the field if tag == "" { continue } - key := tag - var required bool - var bcknd string + f := fieldConfig{ + Name: field.Name, + Key: tag, + Value: &value, + } if idx := strings.Index(tag, ","); idx != -1 { - key = tag[:idx] + f.Key = tag[:idx] opts := strings.Split(tag[idx+1:], ",") - for _, o := range opts { - if o == "required" { - required = true + for _, opt := range opts { + if opt == "required" { + f.Required = true } - if strings.HasPrefix(o, "backend=") { - bcknd = o[len("backend="):] + if strings.HasPrefix(opt, "backend=") { + f.Backend = opt[len("backend="):] } } } + list = append(list, &f) + } + + return list +} + +func (l *Loader) resolve(ctx context.Context, fields []*fieldConfig) error { + for _, f := range fields { var found bool var backendFound bool + for _, b := range l.backends { select { case <-ctx.Done(): @@ -133,13 +149,14 @@ func (l *Loader) parseStruct(ctx context.Context, ref *reflect.Value) error { default: } - if bcknd != "" && bcknd != b.Name() { + if f.Backend != "" && f.Backend != b.Name() { continue } + backendFound = true if u, ok := b.(backend.ValueUnmarshaler); ok { - err := u.UnmarshalValue(ctx, key, value.Addr().Interface()) + err := u.UnmarshalValue(ctx, f.Key, f.Value.Addr().Interface()) if err != nil && err != backend.ErrNotFound { return err } @@ -147,7 +164,7 @@ func (l *Loader) parseStruct(ctx context.Context, ref *reflect.Value) error { continue } - raw, err := b.Get(ctx, key) + raw, err := b.Get(ctx, f.Key) if err != nil { if err == backend.ErrNotFound { continue @@ -156,7 +173,7 @@ func (l *Loader) parseStruct(ctx context.Context, ref *reflect.Value) error { return err } - err = convert(string(raw), &value) + err = convert(string(raw), f.Value) if err != nil { return err } @@ -165,12 +182,12 @@ func (l *Loader) parseStruct(ctx context.Context, ref *reflect.Value) error { break } - if bcknd != "" && !backendFound { - return fmt.Errorf("the backend: '%s' is not supported", bcknd) + if f.Backend != "" && !backendFound { + return fmt.Errorf("the backend: '%s' is not supported", f.Backend) } - if required && !found { - return fmt.Errorf("required key '%s' for field '%s' not found", key, field.Name) + if f.Required && !found { + return fmt.Errorf("required key '%s' for field '%s' not found", f.Key, f.Name) } }