feat(fs/git): add support for cloning target repository to the local filesystem

internal/config/config_test.go

@@ -675,6 +675,9 @@ func TestLoad(t *testing.T) {
 				cfg.Storage = StorageConfig{
 					Type: GitStorageType,
 					Git: &Git{
+						Backend: GitBackend{
+							Type: GotBackendMemory,
+						},
 						Repository:   "https://github.com/flipt-io/flipt.git",
 						Ref:          "production",
 						RefType:      GitRefTypeStatic,
@@ -817,6 +820,9 @@ func TestLoad(t *testing.T) {
 				cfg.Storage = StorageConfig{
 					Type: GitStorageType,
 					Git: &Git{
+						Backend: GitBackend{
+							Type: GotBackendMemory,
+						},
 						Ref:          "main",
 						RefType:      GitRefTypeStatic,
 						Repository:   "[email protected]:foo/bar.git",
@@ -834,6 +840,9 @@ func TestLoad(t *testing.T) {
 				cfg.Storage = StorageConfig{
 					Type: GitStorageType,
 					Git: &Git{
+						Backend: GitBackend{
+							Type: GotBackendMemory,
+						},
 						Ref:          "main",
 						RefType:      GitRefTypeStatic,
 						Repository:   "[email protected]:foo/bar.git",
@@ -852,6 +861,9 @@ func TestLoad(t *testing.T) {
 				cfg.Storage = StorageConfig{
 					Type: GitStorageType,
 					Git: &Git{
+						Backend: GitBackend{
+							Type: GotBackendMemory,
+						},
 						Ref:          "main",
 						RefType:      GitRefTypeSemver,
 						Repository:   "[email protected]:foo/bar.git",
@@ -862,6 +874,27 @@ func TestLoad(t *testing.T) {
 				return cfg
 			},
 		},
+		{
+			name: "git config provided with ref_type",
+			path: "./testdata/storage/git_provided_with_backend_type.yml",
+			expected: func() *Config {
+				cfg := Default()
+				cfg.Storage = StorageConfig{
+					Type: GitStorageType,
+					Git: &Git{
+						Backend: GitBackend{
+							Type: GitBackendFilesystem,
+							Path: "/path/to/gitdir",
+						},
+						Ref:          "main",
+						RefType:      GitRefTypeStatic,
+						Repository:   "[email protected]:foo/bar.git",
+						PollInterval: 30 * time.Second,
+					},
+				}
+				return cfg
+			},
+		},
 		{
 			name:    "git repository not provided",
 			path:    "./testdata/storage/invalid_git_repo_not_specified.yml",
@@ -900,6 +933,9 @@ func TestLoad(t *testing.T) {
 				cfg.Storage = StorageConfig{
 					Type: GitStorageType,
 					Git: &Git{
+						Backend: GitBackend{
+							Type: GotBackendMemory,
+						},
 						Ref:          "main",
 						RefType:      GitRefTypeStatic,
 						Repository:   "[email protected]:foo/bar.git",

internal/config/storage.go

@@ -48,6 +48,7 @@ func (c *StorageConfig) setDefaults(v *viper.Viper) error {
 	case string(LocalStorageType):
 		v.SetDefault("storage.local.path", ".")
 	case string(GitStorageType):
+		v.SetDefault("storage.git.backend.type", "memory")
 		v.SetDefault("storage.git.ref", "main")
 		v.SetDefault("storage.git.ref_type", "static")
 		v.SetDefault("storage.git.poll_interval", "30s")
@@ -164,11 +165,12 @@ const (
 // Git contains configuration for referencing a git repository.
 type Git struct {
 	Repository      string         `json:"repository,omitempty" mapstructure:"repository" yaml:"repository,omitempty"`
+	Backend         GitBackend     `json:"backend,omitempty" mapstructure:"backend" yaml:"backend,omitempty"`
 	Ref             string         `json:"ref,omitempty" mapstructure:"ref" yaml:"ref,omitempty"`
 	RefType         GitRefType     `json:"refType,omitempty" mapstructure:"ref_type" yaml:"ref_type,omitempty"`
 	Directory       string         `json:"directory,omitempty" mapstructure:"directory" yaml:"directory,omitempty"`
-	CaCertBytes     string         `json:"-" mapstructure:"ca_cert_bytes" yaml:"-" `
-	CaCertPath      string         `json:"-" mapstructure:"ca_cert_path" yaml:"-" `
+	CaCertBytes     string         `json:"-" mapstructure:"ca_cert_bytes" yaml:"-"`
+	CaCertPath      string         `json:"-" mapstructure:"ca_cert_path" yaml:"-"`
 	InsecureSkipTLS bool           `json:"-" mapstructure:"insecure_skip_tls" yaml:"-"`
 	PollInterval    time.Duration  `json:"pollInterval,omitempty" mapstructure:"poll_interval" yaml:"poll_interval,omitempty"`
 	Authentication  Authentication `json:"-" mapstructure:"authentication,omitempty" yaml:"-"`
@@ -186,6 +188,18 @@ func (g *Git) validate() error {
 	return nil
 }
 
+type GitBackendType string
+
+const (
+	GotBackendMemory     = GitBackendType("memory")
+	GitBackendFilesystem = GitBackendType("filesystem")
+)
+
+type GitBackend struct {
+	Type GitBackendType `json:"type,omitempty" mapstructure:"type" yaml:"type,omitempty"`
+	Path string         `json:"path,omitempty" mapstructure:"path" yaml:"path,omitempty"`
+}
+
 // Object contains configuration of readonly object storage.
 type Object struct {
 	Type   ObjectSubStorageType `json:"type,omitempty" mapstructure:"type" yaml:"type,omitempty"`

internal/config/testdata/storage/git_provided_with_backend_type.yml

@@ -0,0 +1,7 @@
+storage:
+  type: git
+  git:
+    repository: "[email protected]:foo/bar.git"
+    backend:
+      type: "filesystem"
+      path: "/path/to/gitdir"

internal/storage/fs/git/store.go

@@ -8,10 +8,14 @@ import (
 	"slices"
 	"sync"
 
+	"github.com/go-git/go-billy/v5/osfs"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/cache"
 	"github.com/go-git/go-git/v5/plumbing/transport"
+	gitstorage "github.com/go-git/go-git/v5/storage"
+	"github.com/go-git/go-git/v5/storage/filesystem"
 	"github.com/go-git/go-git/v5/storage/memory"
 	"go.flipt.io/flipt/internal/containers"
 	"go.flipt.io/flipt/internal/gitfs"
@@ -35,6 +39,7 @@ type SnapshotStore struct {
 	*storagefs.Poller
 
 	logger            *zap.Logger
+	storage           gitstorage.Storer
 	url               string
 	baseRef           string
 	refTypeTag        bool
@@ -110,12 +115,23 @@ func WithDirectory(directory string) containers.Option[SnapshotStore] {
 	}
 }
 
+// WithFilesystemStorage configures the Git repository to clone into
+// the local filesystem, instead of the default which is in-memory.
+// The provided path is location for the dotgit folder.
+func WithFilesystemStorage(path string) containers.Option[SnapshotStore] {
+	return func(ss *SnapshotStore) {
+		fs := osfs.New(path)
+		ss.storage = filesystem.NewStorage(fs, cache.NewObjectLRUDefault())
+	}
+}
+
 // NewSnapshotStore constructs and configures a Store.
 // The store uses the connection and credential details provided to build
 // fs.FS implementations around a target git repository.
 func NewSnapshotStore(ctx context.Context, logger *zap.Logger, url string, opts ...containers.Option[SnapshotStore]) (_ *SnapshotStore, err error) {
 	store := &SnapshotStore{
 		logger:            logger.With(zap.String("repository", url)),
+		storage:           memory.NewStorage(),
 		url:               url,
 		baseRef:           "main",
 		referenceResolver: staticResolver(),
@@ -147,7 +163,7 @@ func NewSnapshotStore(ctx context.Context, logger *zap.Logger, url string, opts
 			cloneOpts.SingleBranch = true
 		}
 
-		store.repo, err = git.Clone(memory.NewStorage(), nil, cloneOpts)
+		store.repo, err = git.Clone(store.storage, nil, cloneOpts)
 		if err != nil {
 			return nil, fmt.Errorf("performing initial clone: %w", err)
 		}
@@ -158,7 +174,7 @@ func NewSnapshotStore(ctx context.Context, logger *zap.Logger, url string, opts
 		}
 	} else {
 		// fetch single reference
-		store.repo, err = git.InitWithOptions(memory.NewStorage(), nil, git.InitOptions{
+		store.repo, err = git.InitWithOptions(store.storage, nil, git.InitOptions{
 			DefaultBranch: plumbing.Main,
 		})
 		if err != nil {

internal/storage/fs/git/store_test.go

@@ -121,6 +121,86 @@ flags:
 	}))
 }
 
+func Test_Store_View_WithFilesystemStorage(t *testing.T) {
+	ch := make(chan struct{})
+	store, skip := testStore(t, gitRepoURL,
+		WithFilesystemStorage(t.TempDir()),
+		WithPollOptions(
+			fs.WithInterval(time.Second),
+			fs.WithNotify(t, func(modified bool) {
+				if modified {
+					close(ch)
+				}
+			}),
+		))
+	if skip {
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+
+	// pull repo
+	workdir := memfs.New()
+	repo, err := git.Clone(memory.NewStorage(), workdir, &git.CloneOptions{
+		Auth:          &http.BasicAuth{Username: "root", Password: "password"},
+		URL:           gitRepoURL,
+		RemoteName:    "origin",
+		ReferenceName: plumbing.NewBranchReferenceName("main"),
+	})
+	require.NoError(t, err)
+
+	tree, err := repo.Worktree()
+	require.NoError(t, err)
+
+	require.NoError(t, tree.Checkout(&git.CheckoutOptions{
+		Branch: "refs/heads/main",
+	}))
+
+	// update features.yml
+	fi, err := workdir.OpenFile("features.yml", os.O_TRUNC|os.O_RDWR, os.ModePerm)
+	require.NoError(t, err)
+
+	updated := []byte(`namespace: production
+flags:
+    - key: foo
+      name: Foo`)
+
+	_, err = fi.Write(updated)
+	require.NoError(t, err)
+	require.NoError(t, fi.Close())
+
+	// commit changes
+	_, err = tree.Commit("chore: update features.yml", &git.CommitOptions{
+		All:    true,
+		Author: &object.Signature{Email: "[email protected]", Name: "dev"},
+	})
+	require.NoError(t, err)
+
+	// push new commit
+	require.NoError(t, repo.Push(&git.PushOptions{
+		Auth:       &http.BasicAuth{Username: "root", Password: "password"},
+		RemoteName: "origin",
+	}))
+
+	// wait until the snapshot is updated or
+	// we timeout
+	select {
+	case <-ch:
+	case <-time.After(time.Minute):
+		t.Fatal("timed out waiting for snapshot")
+	}
+
+	require.NoError(t, err)
+
+	t.Log("received new snapshot")
+
+	require.NoError(t, store.View(ctx, "", func(s storage.ReadOnlyStore) error {
+		_, err = s.GetFlag(ctx, storage.NewResource("production", "foo"))
+		return err
+	}))
+}
+
 func Test_Store_View_WithRevision(t *testing.T) {
 	ch := make(chan struct{})
 	store, skip := testStore(t, gitRepoURL, WithPollOptions(

internal/storage/fs/store/store.go

@@ -32,6 +32,7 @@ import (
 func NewStore(ctx context.Context, logger *zap.Logger, cfg *config.Config) (_ storage.Store, err error) {
 	switch cfg.Storage.Type {
 	case config.GitStorageType:
+		storage := cfg.Storage.Git
 		opts := []containers.Option[git.SnapshotStore]{
 			git.WithRef(cfg.Storage.Git.Ref),
 			git.WithPollOptions(
@@ -41,21 +42,37 @@ func NewStore(ctx context.Context, logger *zap.Logger, cfg *config.Config) (_ st
 			git.WithDirectory(cfg.Storage.Git.Directory),
 		}
 
-		if cfg.Storage.Git.RefType == config.GitRefTypeSemver {
+		if storage.RefType == config.GitRefTypeSemver {
 			opts = append(opts, git.WithSemverResolver())
 		}
 
-		if cfg.Storage.Git.CaCertBytes != "" {
-			opts = append(opts, git.WithCABundle([]byte(cfg.Storage.Git.CaCertBytes)))
-		} else if cfg.Storage.Git.CaCertPath != "" {
-			if bytes, err := os.ReadFile(cfg.Storage.Git.CaCertPath); err == nil {
+		switch storage.Backend.Type {
+		case config.GitBackendFilesystem:
+			path := storage.Backend.Path
+			if path == "" {
+				path, err = os.MkdirTemp(os.TempDir(), "flipt-git-*")
+				if err != nil {
+					return nil, fmt.Errorf("making tempory directory for git storage: %w", err)
+				}
+			}
+
+			opts = append(opts, git.WithFilesystemStorage(path))
+			logger = logger.With(zap.String("git_storage_type", "filesystem"), zap.String("git_storage_path", path))
+		case config.GotBackendMemory:
+			logger = logger.With(zap.String("git_storage_type", "memory"))
+		}
+
+		if storage.CaCertBytes != "" {
+			opts = append(opts, git.WithCABundle([]byte(storage.CaCertBytes)))
+		} else if storage.CaCertPath != "" {
+			if bytes, err := os.ReadFile(storage.CaCertPath); err == nil {
 				opts = append(opts, git.WithCABundle(bytes))
 			} else {
 				return nil, err
 			}
 		}
 
-		auth := cfg.Storage.Git.Authentication
+		auth := storage.Authentication
 		switch {
 		case auth.BasicAuth != nil:
 			opts = append(opts, git.WithAuth(&http.BasicAuth{
@@ -95,7 +112,7 @@ func NewStore(ctx context.Context, logger *zap.Logger, cfg *config.Config) (_ st
 			opts = append(opts, git.WithAuth(method))
 		}
 
-		snapStore, err := git.NewSnapshotStore(ctx, logger, cfg.Storage.Git.Repository, opts...)
+		snapStore, err := git.NewSnapshotStore(ctx, logger, storage.Repository, opts...)
 		if err != nil {
 			return nil, err
 		}