Skip to content

Commit

Permalink
Merge branch 'main' into authz
Browse files Browse the repository at this point in the history
  • Loading branch information
markphelps authored May 21, 2024
2 parents ab1b48a + 68ea081 commit 31afa47
Show file tree
Hide file tree
Showing 9 changed files with 280 additions and 174 deletions.
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,12 @@
This format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [v1.42.1](https://github.com/flipt-io/flipt/releases/tag/v1.42.1) - 2024-05-21

### Changed

- Optimized the Git backend to only fetch explicitly required references (#3100)

## [v1.42.0](https://github.com/flipt-io/flipt/releases/tag/v1.42.0) - 2024-05-15

### Added
Expand Down
1 change: 1 addition & 0 deletions go.work.sum
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,7 @@ github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWH
github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM=
github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM=
github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
github.com/cockroachdb/apd/v2 v2.0.2/go.mod h1:DDxRlzC2lo3/vSlmSoS7JkqbbrARPuFOGr0B9pvN3Gw=
Expand Down
12 changes: 6 additions & 6 deletions internal/storage/fs/git/reference_resolvers.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@ import (
"github.com/go-git/go-git/v5/plumbing"
)

// ReferenceResolver is a function type used to describe reference resolver functions.
type ReferenceResolver func(repo *git.Repository, ref string) (plumbing.Hash, error)
// referenceResolver is a function type used to describe reference resolver functions.
type referenceResolver func(repo *git.Repository, ref string) (plumbing.Hash, error)

// StaticResolver is a resolver which just resolve static references.
func StaticResolver() ReferenceResolver {
// staticResolver is a resolver which just resolve static references.
func staticResolver() referenceResolver {
return func(repo *git.Repository, ref string) (plumbing.Hash, error) {
if plumbing.IsHash(ref) {
return plumbing.NewHash(ref), nil
Expand All @@ -27,8 +27,8 @@ func StaticResolver() ReferenceResolver {
}
}

// SemverResolver is a resolver which resolver semantic versioning references for tags.
func SemverResolver() ReferenceResolver {
// semverResolver is a resolver which resolver semantic versioning references for tags.
func semverResolver() referenceResolver {
return func(repo *git.Repository, ref string) (plumbing.Hash, error) {
constraint, err := semver.NewConstraint(ref)
if err != nil {
Expand Down
10 changes: 5 additions & 5 deletions internal/storage/fs/git/reference_resolvers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import (
func TestStaticResolver(t *testing.T) {
t.Run("should resolve static references correctly", func(t *testing.T) {
repo := newGitRepo(t)
resolver := StaticResolver()
resolver := staticResolver()

commitHash := repo.createCommit(t)
resolvedHash, err := resolver(repo.repo, "main")
Expand All @@ -38,7 +38,7 @@ func TestStaticResolver(t *testing.T) {
func TestSemverResolver(t *testing.T) {
t.Run("should resolve semver tags correctly when the reference is a constraint", func(t *testing.T) {
repo := newGitRepo(t)
resolver := SemverResolver()
resolver := semverResolver()
constraint := "v0.1.*"

commitHash := repo.createCommit(t)
Expand All @@ -61,7 +61,7 @@ func TestSemverResolver(t *testing.T) {

t.Run("should resolve semver tags correctly when the reference is not a constraint", func(t *testing.T) {
repo := newGitRepo(t)
resolver := SemverResolver()
resolver := semverResolver()

commitHash := repo.createCommit(t)
repo.createTag(t, "v0.1.0", commitHash)
Expand All @@ -74,7 +74,7 @@ func TestSemverResolver(t *testing.T) {

t.Run("should resolve semver tags correctly when there is non compliant semver tags", func(t *testing.T) {
repo := newGitRepo(t)
resolver := SemverResolver()
resolver := semverResolver()

commitHash := repo.createCommit(t)
repo.createTag(t, "non-semver-tag", commitHash)
Expand All @@ -90,7 +90,7 @@ func TestSemverResolver(t *testing.T) {

t.Run("should return an error when no matching tag was found", func(t *testing.T) {
repo := newGitRepo(t)
resolver := SemverResolver()
resolver := semverResolver()

commitHash := repo.createCommit(t)
repo.createTag(t, "v0.1.0", commitHash)
Expand Down
109 changes: 84 additions & 25 deletions internal/storage/fs/git/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ package git
import (
"context"
"errors"
"fmt"
"io/fs"
"slices"
"sync"

"github.com/go-git/go-git/v5"
Expand Down Expand Up @@ -35,7 +37,8 @@ type SnapshotStore struct {
logger *zap.Logger
url string
baseRef string
referenceResolver ReferenceResolver
refTypeTag bool
referenceResolver referenceResolver
directory string
auth transport.AuthMethod
insecureSkipTLS bool
Expand All @@ -58,10 +61,11 @@ func WithRef(ref string) containers.Option[SnapshotStore] {
}
}

// WithRefResolver configures how the reference will be resolved for the repository.
func WithRefResolver(resolver ReferenceResolver) containers.Option[SnapshotStore] {
// WithSemverResolver configures how the reference will be resolved for the repository.
func WithSemverResolver() containers.Option[SnapshotStore] {
return func(s *SnapshotStore) {
s.referenceResolver = resolver
s.refTypeTag = true
s.referenceResolver = semverResolver()
}
}

Expand Down Expand Up @@ -114,7 +118,7 @@ func NewSnapshotStore(ctx context.Context, logger *zap.Logger, url string, opts
logger: logger.With(zap.String("repository", url)),
url: url,
baseRef: "main",
referenceResolver: StaticResolver(),
referenceResolver: staticResolver(),
}
containers.ApplyAll(store, opts...)

Expand All @@ -125,19 +129,60 @@ func NewSnapshotStore(ctx context.Context, logger *zap.Logger, url string, opts
return nil, err
}

store.repo, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
Auth: store.auth,
URL: store.url,
CABundle: store.caBundle,
InsecureSkipTLS: store.insecureSkipTLS,
})
if err != nil {
return nil, err
}
if !plumbing.IsHash(store.baseRef) {
// if the base ref is not an explicit SHA then
// attempt to clone either the explicit branch
// or all references for tag based semver
cloneOpts := &git.CloneOptions{
Auth: store.auth,
URL: store.url,
CABundle: store.caBundle,
InsecureSkipTLS: store.insecureSkipTLS,
}

// do an initial fetch to setup remote tracking branches
if _, err := store.fetch(ctx); err != nil {
return nil, err
// if our reference is a branch type then we can assume it exists
// and attempt to only clone from this branch initially
if !store.refTypeTag {
cloneOpts.ReferenceName = plumbing.NewBranchReferenceName(store.baseRef)
cloneOpts.SingleBranch = true
}

store.repo, err = git.Clone(memory.NewStorage(), nil, cloneOpts)
if err != nil {
return nil, fmt.Errorf("performing initial clone: %w", err)
}

// do an initial fetch to setup remote tracking branches
if _, err := store.fetch(ctx, []string{store.baseRef}); err != nil {
return nil, fmt.Errorf("performing initial fetch: %w", err)
}
} else {
// fetch single reference
store.repo, err = git.InitWithOptions(memory.NewStorage(), nil, git.InitOptions{
DefaultBranch: plumbing.Main,
})
if err != nil {
return nil, err
}

if _, err = store.repo.CreateRemote(&config.RemoteConfig{
Name: "origin",
URLs: []string{store.url},
}); err != nil {
return nil, err
}

if err := store.repo.FetchContext(ctx, &git.FetchOptions{
Auth: store.auth,
CABundle: store.caBundle,
InsecureSkipTLS: store.insecureSkipTLS,
Depth: 1,
RefSpecs: []config.RefSpec{
config.RefSpec(fmt.Sprintf("%[1]s:%[1]s", store.baseRef)),
},
}); err != nil {
return nil, err
}
}

// fetch base ref snapshot at-least once before returning store
Expand Down Expand Up @@ -180,8 +225,13 @@ func (s *SnapshotStore) View(ctx context.Context, storeRef storage.Reference, fn
return fn(snap)
}

refs := s.snaps.References()
if !slices.Contains(refs, ref) {
refs = append(refs, ref)
}

// force attempt a fetch to get the latest references
if _, err := s.fetch(ctx); err != nil {
if _, err := s.fetch(ctx, refs); err != nil {
return err
}

Expand All @@ -202,7 +252,7 @@ func (s *SnapshotStore) View(ctx context.Context, storeRef storage.Reference, fn
// HEAD updates to a new revision, it builds a snapshot and updates it
// on the store.
func (s *SnapshotStore) update(ctx context.Context) (bool, error) {
if updated, err := s.fetch(ctx); !(err == nil && updated) {
if updated, err := s.fetch(ctx, s.snaps.References()); !(err == nil && updated) {
// either nothing updated or err != nil
return updated, err
}
Expand All @@ -223,16 +273,25 @@ func (s *SnapshotStore) update(ctx context.Context) (bool, error) {
return true, errors.Join(errs...)
}

func (s *SnapshotStore) fetch(ctx context.Context) (bool, error) {
func (s *SnapshotStore) fetch(ctx context.Context, heads []string) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()

refSpecs := []config.RefSpec{}

if s.refTypeTag {
refSpecs = append(refSpecs, "+refs/tags/*:refs/tags/*")
}

for _, head := range heads {
refSpecs = append(refSpecs,
config.RefSpec(fmt.Sprintf("+refs/heads/%[1]s:refs/heads/%[1]s", head)),
)
}

if err := s.repo.FetchContext(ctx, &git.FetchOptions{
Auth: s.auth,
RefSpecs: []config.RefSpec{
"+refs/heads/*:refs/heads/*",
"+refs/tags/*:refs/tags/*",
},
Auth: s.auth,
RefSpecs: refSpecs,
}); err != nil {
if !errors.Is(err, git.NoErrAlreadyUpToDate) {
return false, err
Expand Down
70 changes: 56 additions & 14 deletions internal/storage/fs/git/store_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ flags:
require.NoError(t, fi.Close())

// commit changes
_, err = tree.Commit("chore: update features.yml", &git.CommitOptions{
_, err = tree.Commit("chore: update features.yml add foo and bar", &git.CommitOptions{
All: true,
Author: &object.Signature{Email: "[email protected]", Name: "dev"},
})
Expand All @@ -185,18 +185,6 @@ flags:
RefSpecs: []config.RefSpec{"refs/heads/new-branch:refs/heads/new-branch"},
}))

// wait until the snapshot is updated or
// we timeout
select {
case <-ch:
case <-time.After(time.Minute):
t.Fatal("timed out waiting for snapshot")
}

require.NoError(t, err)

t.Log("received new snapshot")

require.NoError(t, store.View(ctx, "", func(s storage.ReadOnlyStore) error {
_, err := s.GetFlag(ctx, storage.NewResource("production", "bar"))
require.Error(t, err, "flag should not be found in default revision")
Expand All @@ -209,11 +197,65 @@ flags:
return nil
}))

// should be able to fetch flag from previously unfetched reference
require.NoError(t, store.View(ctx, "new-branch", func(s storage.ReadOnlyStore) error {
_, err := s.GetFlag(ctx, storage.NewResource("production", "bar"))
require.NoError(t, err, "flag should be present on new-branch")
return nil
}))

// flag bar should not yet be present
require.NoError(t, store.View(ctx, "new-branch", func(s storage.ReadOnlyStore) error {
_, err := s.GetFlag(ctx, storage.NewResource("production", "baz"))
require.Error(t, err, "flag should not be found in explicitly named new-branch revision")
return nil
}))

// update features.yml, now with the bar flag
fi, err = workdir.OpenFile("features.yml", os.O_TRUNC|os.O_RDWR, os.ModePerm)
require.NoError(t, err)

updated = []byte(`namespace: production
flags:
- key: foo
name: Foo
- key: bar
name: Bar
- key: baz
name: Baz`)

_, err = fi.Write(updated)
require.NoError(t, err)
require.NoError(t, fi.Close())

// commit changes
_, err = tree.Commit("chore: update features.yml add baz", &git.CommitOptions{
All: true,
Author: &object.Signature{Email: "[email protected]", Name: "dev"},
})
require.NoError(t, err)

// push new commit
require.NoError(t, repo.Push(&git.PushOptions{
Auth: &http.BasicAuth{Username: "root", Password: "password"},
RemoteName: "origin",
RefSpecs: []config.RefSpec{"refs/heads/new-branch:refs/heads/new-branch"},
}))

// we should expect to see a modified event now because
// the new reference should be tracked
select {
case <-ch:
case <-time.After(time.Minute):
t.Fatal("timed out waiting for fetch")
}

// should be able to fetch flag bar now that it has been pushed
require.NoError(t, store.View(ctx, "new-branch", func(s storage.ReadOnlyStore) error {
_, err := s.GetFlag(ctx, storage.NewResource("production", "baz"))
require.NoError(t, err, "flag should be present on new-branch")
return nil
}))
}

func Test_Store_View_WithSemverRevision(t *testing.T) {
Expand All @@ -232,7 +274,7 @@ func Test_Store_View_WithSemverRevision(t *testing.T) {
ch := make(chan struct{})
store, skip := testStore(t, gitRepoURL,
WithRef("v0.1.*"),
WithRefResolver(SemverResolver()),
WithSemverResolver(),
WithPollOptions(
fs.WithInterval(time.Second),
fs.WithNotify(t, func(modified bool) {
Expand Down
10 changes: 4 additions & 6 deletions internal/storage/fs/store/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,21 +32,19 @@ import (
func NewStore(ctx context.Context, logger *zap.Logger, cfg *config.Config) (_ storage.Store, err error) {
switch cfg.Storage.Type {
case config.GitStorageType:
refResolver := git.StaticResolver()
if cfg.Storage.Git.RefType == config.GitRefTypeSemver {
refResolver = git.SemverResolver()
}

opts := []containers.Option[git.SnapshotStore]{
git.WithRef(cfg.Storage.Git.Ref),
git.WithRefResolver(refResolver),
git.WithPollOptions(
storagefs.WithInterval(cfg.Storage.Git.PollInterval),
),
git.WithInsecureTLS(cfg.Storage.Git.InsecureSkipTLS),
git.WithDirectory(cfg.Storage.Git.Directory),
}

if cfg.Storage.Git.RefType == config.GitRefTypeSemver {
opts = append(opts, git.WithSemverResolver())
}

if cfg.Storage.Git.CaCertBytes != "" {
opts = append(opts, git.WithCABundle([]byte(cfg.Storage.Git.CaCertBytes)))
} else if cfg.Storage.Git.CaCertPath != "" {
Expand Down
Loading

0 comments on commit 31afa47

Please sign in to comment.