diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 385c76fb30..b06baf2819 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -675,6 +675,9 @@ func TestLoad(t *testing.T) { cfg.Storage = StorageConfig{ Type: GitStorageType, Git: &Git{ + Backend: GitBackend{ + Type: GitBackendMemory, + }, Repository: "https://github.com/flipt-io/flipt.git", Ref: "production", RefType: GitRefTypeStatic, @@ -817,6 +820,9 @@ func TestLoad(t *testing.T) { cfg.Storage = StorageConfig{ Type: GitStorageType, Git: &Git{ + Backend: GitBackend{ + Type: GitBackendMemory, + }, Ref: "main", RefType: GitRefTypeStatic, Repository: "git@github.com:foo/bar.git", @@ -834,6 +840,9 @@ func TestLoad(t *testing.T) { cfg.Storage = StorageConfig{ Type: GitStorageType, Git: &Git{ + Backend: GitBackend{ + Type: GitBackendMemory, + }, Ref: "main", RefType: GitRefTypeStatic, Repository: "git@github.com:foo/bar.git", @@ -852,6 +861,9 @@ func TestLoad(t *testing.T) { cfg.Storage = StorageConfig{ Type: GitStorageType, Git: &Git{ + Backend: GitBackend{ + Type: GitBackendMemory, + }, Ref: "main", RefType: GitRefTypeSemver, Repository: "git@github.com:foo/bar.git", @@ -862,6 +874,27 @@ func TestLoad(t *testing.T) { return cfg }, }, + { + name: "git config provided with ref_type", + path: "./testdata/storage/git_provided_with_backend_type.yml", + expected: func() *Config { + cfg := Default() + cfg.Storage = StorageConfig{ + Type: GitStorageType, + Git: &Git{ + Backend: GitBackend{ + Type: GitBackendFilesystem, + Path: "/path/to/gitdir", + }, + Ref: "main", + RefType: GitRefTypeStatic, + Repository: "git@github.com:foo/bar.git", + PollInterval: 30 * time.Second, + }, + } + return cfg + }, + }, { name: "git repository not provided", path: "./testdata/storage/invalid_git_repo_not_specified.yml", @@ -900,6 +933,9 @@ func TestLoad(t *testing.T) { cfg.Storage = StorageConfig{ Type: GitStorageType, Git: &Git{ + Backend: GitBackend{ + Type: GitBackendMemory, + }, Ref: "main", RefType: GitRefTypeStatic, Repository: "git@github.com:foo/bar.git", diff --git a/internal/config/storage.go b/internal/config/storage.go index ce6e70ba8c..71b7f3cb3c 100644 --- a/internal/config/storage.go +++ b/internal/config/storage.go @@ -48,6 +48,7 @@ func (c *StorageConfig) setDefaults(v *viper.Viper) error { case string(LocalStorageType): v.SetDefault("storage.local.path", ".") case string(GitStorageType): + v.SetDefault("storage.git.backend.type", "memory") v.SetDefault("storage.git.ref", "main") v.SetDefault("storage.git.ref_type", "static") v.SetDefault("storage.git.poll_interval", "30s") @@ -164,11 +165,12 @@ const ( // Git contains configuration for referencing a git repository. type Git struct { Repository string `json:"repository,omitempty" mapstructure:"repository" yaml:"repository,omitempty"` + Backend GitBackend `json:"backend,omitempty" mapstructure:"backend" yaml:"backend,omitempty"` Ref string `json:"ref,omitempty" mapstructure:"ref" yaml:"ref,omitempty"` RefType GitRefType `json:"refType,omitempty" mapstructure:"ref_type" yaml:"ref_type,omitempty"` Directory string `json:"directory,omitempty" mapstructure:"directory" yaml:"directory,omitempty"` - CaCertBytes string `json:"-" mapstructure:"ca_cert_bytes" yaml:"-" ` - CaCertPath string `json:"-" mapstructure:"ca_cert_path" yaml:"-" ` + CaCertBytes string `json:"-" mapstructure:"ca_cert_bytes" yaml:"-"` + CaCertPath string `json:"-" mapstructure:"ca_cert_path" yaml:"-"` InsecureSkipTLS bool `json:"-" mapstructure:"insecure_skip_tls" yaml:"-"` PollInterval time.Duration `json:"pollInterval,omitempty" mapstructure:"poll_interval" yaml:"poll_interval,omitempty"` Authentication Authentication `json:"-" mapstructure:"authentication,omitempty" yaml:"-"` @@ -186,6 +188,18 @@ func (g *Git) validate() error { return nil } +type GitBackendType string + +const ( + GitBackendMemory = GitBackendType("memory") + GitBackendFilesystem = GitBackendType("filesystem") +) + +type GitBackend struct { + Type GitBackendType `json:"type,omitempty" mapstructure:"type" yaml:"type,omitempty"` + Path string `json:"path,omitempty" mapstructure:"path" yaml:"path,omitempty"` +} + // Object contains configuration of readonly object storage. type Object struct { Type ObjectSubStorageType `json:"type,omitempty" mapstructure:"type" yaml:"type,omitempty"` diff --git a/internal/config/testdata/storage/git_provided_with_backend_type.yml b/internal/config/testdata/storage/git_provided_with_backend_type.yml new file mode 100644 index 0000000000..ed1c21902f --- /dev/null +++ b/internal/config/testdata/storage/git_provided_with_backend_type.yml @@ -0,0 +1,7 @@ +storage: + type: git + git: + repository: "git@github.com:foo/bar.git" + backend: + type: "filesystem" + path: "/path/to/gitdir" diff --git a/internal/storage/fs/git/store.go b/internal/storage/fs/git/store.go index eeb729cfe2..a00901c3cf 100644 --- a/internal/storage/fs/git/store.go +++ b/internal/storage/fs/git/store.go @@ -8,10 +8,14 @@ import ( "slices" "sync" + "github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/transport" + gitstorage "github.com/go-git/go-git/v5/storage" + "github.com/go-git/go-git/v5/storage/filesystem" "github.com/go-git/go-git/v5/storage/memory" "go.flipt.io/flipt/internal/containers" "go.flipt.io/flipt/internal/gitfs" @@ -35,6 +39,7 @@ type SnapshotStore struct { *storagefs.Poller logger *zap.Logger + storage gitstorage.Storer url string baseRef string refTypeTag bool @@ -110,12 +115,23 @@ func WithDirectory(directory string) containers.Option[SnapshotStore] { } } +// WithFilesystemStorage configures the Git repository to clone into +// the local filesystem, instead of the default which is in-memory. +// The provided path is location for the dotgit folder. +func WithFilesystemStorage(path string) containers.Option[SnapshotStore] { + return func(ss *SnapshotStore) { + fs := osfs.New(path) + ss.storage = filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) + } +} + // NewSnapshotStore constructs and configures a Store. // The store uses the connection and credential details provided to build // fs.FS implementations around a target git repository. func NewSnapshotStore(ctx context.Context, logger *zap.Logger, url string, opts ...containers.Option[SnapshotStore]) (_ *SnapshotStore, err error) { store := &SnapshotStore{ logger: logger.With(zap.String("repository", url)), + storage: memory.NewStorage(), url: url, baseRef: "main", referenceResolver: staticResolver(), @@ -147,7 +163,7 @@ func NewSnapshotStore(ctx context.Context, logger *zap.Logger, url string, opts cloneOpts.SingleBranch = true } - store.repo, err = git.Clone(memory.NewStorage(), nil, cloneOpts) + store.repo, err = git.Clone(store.storage, nil, cloneOpts) if err != nil { return nil, fmt.Errorf("performing initial clone: %w", err) } @@ -158,7 +174,7 @@ func NewSnapshotStore(ctx context.Context, logger *zap.Logger, url string, opts } } else { // fetch single reference - store.repo, err = git.InitWithOptions(memory.NewStorage(), nil, git.InitOptions{ + store.repo, err = git.InitWithOptions(store.storage, nil, git.InitOptions{ DefaultBranch: plumbing.Main, }) if err != nil { diff --git a/internal/storage/fs/git/store_test.go b/internal/storage/fs/git/store_test.go index d9e7cfdc2f..21c69ad069 100644 --- a/internal/storage/fs/git/store_test.go +++ b/internal/storage/fs/git/store_test.go @@ -121,6 +121,86 @@ flags: })) } +func Test_Store_View_WithFilesystemStorage(t *testing.T) { + ch := make(chan struct{}) + store, skip := testStore(t, gitRepoURL, + WithFilesystemStorage(t.TempDir()), + WithPollOptions( + fs.WithInterval(time.Second), + fs.WithNotify(t, func(modified bool) { + if modified { + close(ch) + } + }), + )) + if skip { + return + } + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + // pull repo + workdir := memfs.New() + repo, err := git.Clone(memory.NewStorage(), workdir, &git.CloneOptions{ + Auth: &http.BasicAuth{Username: "root", Password: "password"}, + URL: gitRepoURL, + RemoteName: "origin", + ReferenceName: plumbing.NewBranchReferenceName("main"), + }) + require.NoError(t, err) + + tree, err := repo.Worktree() + require.NoError(t, err) + + require.NoError(t, tree.Checkout(&git.CheckoutOptions{ + Branch: "refs/heads/main", + })) + + // update features.yml + fi, err := workdir.OpenFile("features.yml", os.O_TRUNC|os.O_RDWR, os.ModePerm) + require.NoError(t, err) + + updated := []byte(`namespace: production +flags: + - key: foo + name: Foo`) + + _, err = fi.Write(updated) + require.NoError(t, err) + require.NoError(t, fi.Close()) + + // commit changes + _, err = tree.Commit("chore: update features.yml", &git.CommitOptions{ + All: true, + Author: &object.Signature{Email: "dev@flipt.io", Name: "dev"}, + }) + require.NoError(t, err) + + // push new commit + require.NoError(t, repo.Push(&git.PushOptions{ + Auth: &http.BasicAuth{Username: "root", Password: "password"}, + RemoteName: "origin", + })) + + // wait until the snapshot is updated or + // we timeout + select { + case <-ch: + case <-time.After(time.Minute): + t.Fatal("timed out waiting for snapshot") + } + + require.NoError(t, err) + + t.Log("received new snapshot") + + require.NoError(t, store.View(ctx, "", func(s storage.ReadOnlyStore) error { + _, err = s.GetFlag(ctx, storage.NewResource("production", "foo")) + return err + })) +} + func Test_Store_View_WithRevision(t *testing.T) { ch := make(chan struct{}) store, skip := testStore(t, gitRepoURL, WithPollOptions( diff --git a/internal/storage/fs/store/store.go b/internal/storage/fs/store/store.go index edcb86ce0a..3073cf46d2 100644 --- a/internal/storage/fs/store/store.go +++ b/internal/storage/fs/store/store.go @@ -32,6 +32,7 @@ import ( func NewStore(ctx context.Context, logger *zap.Logger, cfg *config.Config) (_ storage.Store, err error) { switch cfg.Storage.Type { case config.GitStorageType: + storage := cfg.Storage.Git opts := []containers.Option[git.SnapshotStore]{ git.WithRef(cfg.Storage.Git.Ref), git.WithPollOptions( @@ -41,21 +42,37 @@ func NewStore(ctx context.Context, logger *zap.Logger, cfg *config.Config) (_ st git.WithDirectory(cfg.Storage.Git.Directory), } - if cfg.Storage.Git.RefType == config.GitRefTypeSemver { + if storage.RefType == config.GitRefTypeSemver { opts = append(opts, git.WithSemverResolver()) } - if cfg.Storage.Git.CaCertBytes != "" { - opts = append(opts, git.WithCABundle([]byte(cfg.Storage.Git.CaCertBytes))) - } else if cfg.Storage.Git.CaCertPath != "" { - if bytes, err := os.ReadFile(cfg.Storage.Git.CaCertPath); err == nil { + switch storage.Backend.Type { + case config.GitBackendFilesystem: + path := storage.Backend.Path + if path == "" { + path, err = os.MkdirTemp(os.TempDir(), "flipt-git-*") + if err != nil { + return nil, fmt.Errorf("making tempory directory for git storage: %w", err) + } + } + + opts = append(opts, git.WithFilesystemStorage(path)) + logger = logger.With(zap.String("git_storage_type", "filesystem"), zap.String("git_storage_path", path)) + case config.GitBackendMemory: + logger = logger.With(zap.String("git_storage_type", "memory")) + } + + if storage.CaCertBytes != "" { + opts = append(opts, git.WithCABundle([]byte(storage.CaCertBytes))) + } else if storage.CaCertPath != "" { + if bytes, err := os.ReadFile(storage.CaCertPath); err == nil { opts = append(opts, git.WithCABundle(bytes)) } else { return nil, err } } - auth := cfg.Storage.Git.Authentication + auth := storage.Authentication switch { case auth.BasicAuth != nil: opts = append(opts, git.WithAuth(&http.BasicAuth{ @@ -95,7 +112,7 @@ func NewStore(ctx context.Context, logger *zap.Logger, cfg *config.Config) (_ st opts = append(opts, git.WithAuth(method)) } - snapStore, err := git.NewSnapshotStore(ctx, logger, cfg.Storage.Git.Repository, opts...) + snapStore, err := git.NewSnapshotStore(ctx, logger, storage.Repository, opts...) if err != nil { return nil, err }