From 421d20599e4e11a9a0e22559ed99817fac3c3365 Mon Sep 17 00:00:00 2001 From: George MacRorie Date: Tue, 21 May 2024 12:35:11 +0100 Subject: [PATCH] perf(fs/git): fetch single branch only Signed-off-by: George MacRorie --- .../storage/fs/git/reference_resolvers.go | 12 +-- .../fs/git/reference_resolvers_test.go | 10 +-- internal/storage/fs/git/store.go | 86 ++++++++++++++----- internal/storage/fs/git/store_test.go | 2 +- internal/storage/fs/store/store.go | 11 ++- 5 files changed, 82 insertions(+), 39 deletions(-) diff --git a/internal/storage/fs/git/reference_resolvers.go b/internal/storage/fs/git/reference_resolvers.go index c9580316a3..68f1107014 100644 --- a/internal/storage/fs/git/reference_resolvers.go +++ b/internal/storage/fs/git/reference_resolvers.go @@ -8,11 +8,11 @@ import ( "github.com/go-git/go-git/v5/plumbing" ) -// ReferenceResolver is a function type used to describe reference resolver functions. -type ReferenceResolver func(repo *git.Repository, ref string) (plumbing.Hash, error) +// referenceResolver is a function type used to describe reference resolver functions. +type referenceResolver func(repo *git.Repository, ref string) (plumbing.Hash, error) -// StaticResolver is a resolver which just resolve static references. -func StaticResolver() ReferenceResolver { +// staticResolver is a resolver which just resolve static references. +func staticResolver() referenceResolver { return func(repo *git.Repository, ref string) (plumbing.Hash, error) { if plumbing.IsHash(ref) { return plumbing.NewHash(ref), nil @@ -27,8 +27,8 @@ func StaticResolver() ReferenceResolver { } } -// SemverResolver is a resolver which resolver semantic versioning references for tags. -func SemverResolver() ReferenceResolver { +// semverResolver is a resolver which resolver semantic versioning references for tags. +func semverResolver() referenceResolver { return func(repo *git.Repository, ref string) (plumbing.Hash, error) { constraint, err := semver.NewConstraint(ref) if err != nil { diff --git a/internal/storage/fs/git/reference_resolvers_test.go b/internal/storage/fs/git/reference_resolvers_test.go index bebf0d222e..07cb83bf4d 100644 --- a/internal/storage/fs/git/reference_resolvers_test.go +++ b/internal/storage/fs/git/reference_resolvers_test.go @@ -17,7 +17,7 @@ import ( func TestStaticResolver(t *testing.T) { t.Run("should resolve static references correctly", func(t *testing.T) { repo := newGitRepo(t) - resolver := StaticResolver() + resolver := staticResolver() commitHash := repo.createCommit(t) resolvedHash, err := resolver(repo.repo, "main") @@ -38,7 +38,7 @@ func TestStaticResolver(t *testing.T) { func TestSemverResolver(t *testing.T) { t.Run("should resolve semver tags correctly when the reference is a constraint", func(t *testing.T) { repo := newGitRepo(t) - resolver := SemverResolver() + resolver := semverResolver() constraint := "v0.1.*" commitHash := repo.createCommit(t) @@ -61,7 +61,7 @@ func TestSemverResolver(t *testing.T) { t.Run("should resolve semver tags correctly when the reference is not a constraint", func(t *testing.T) { repo := newGitRepo(t) - resolver := SemverResolver() + resolver := semverResolver() commitHash := repo.createCommit(t) repo.createTag(t, "v0.1.0", commitHash) @@ -74,7 +74,7 @@ func TestSemverResolver(t *testing.T) { t.Run("should resolve semver tags correctly when there is non compliant semver tags", func(t *testing.T) { repo := newGitRepo(t) - resolver := SemverResolver() + resolver := semverResolver() commitHash := repo.createCommit(t) repo.createTag(t, "non-semver-tag", commitHash) @@ -90,7 +90,7 @@ func TestSemverResolver(t *testing.T) { t.Run("should return an error when no matching tag was found", func(t *testing.T) { repo := newGitRepo(t) - resolver := SemverResolver() + resolver := semverResolver() commitHash := repo.createCommit(t) repo.createTag(t, "v0.1.0", commitHash) diff --git a/internal/storage/fs/git/store.go b/internal/storage/fs/git/store.go index 88d9ced70b..a8f2a67175 100644 --- a/internal/storage/fs/git/store.go +++ b/internal/storage/fs/git/store.go @@ -37,7 +37,8 @@ type SnapshotStore struct { logger *zap.Logger url string baseRef string - referenceResolver ReferenceResolver + refTypeTag bool + referenceResolver referenceResolver directory string auth transport.AuthMethod insecureSkipTLS bool @@ -60,10 +61,11 @@ func WithRef(ref string) containers.Option[SnapshotStore] { } } -// WithRefResolver configures how the reference will be resolved for the repository. -func WithRefResolver(resolver ReferenceResolver) containers.Option[SnapshotStore] { +// WithSemverResolver configures how the reference will be resolved for the repository. +func WithSemverResolver() containers.Option[SnapshotStore] { return func(s *SnapshotStore) { - s.referenceResolver = resolver + s.refTypeTag = true + s.referenceResolver = semverResolver() } } @@ -116,7 +118,7 @@ func NewSnapshotStore(ctx context.Context, logger *zap.Logger, url string, opts logger: logger.With(zap.String("repository", url)), url: url, baseRef: "main", - referenceResolver: StaticResolver(), + referenceResolver: staticResolver(), } containers.ApplyAll(store, opts...) @@ -127,19 +129,58 @@ func NewSnapshotStore(ctx context.Context, logger *zap.Logger, url string, opts return nil, err } - store.repo, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{ - Auth: store.auth, - URL: store.url, - CABundle: store.caBundle, - InsecureSkipTLS: store.insecureSkipTLS, - }) - if err != nil { - return nil, err - } + if !plumbing.IsHash(store.baseRef) { + // if the base ref is not an explicit SHA then + // attempt to clone either the explicit branch + // or all references for tag based semver + cloneOpts := &git.CloneOptions{ + Auth: store.auth, + URL: store.url, + CABundle: store.caBundle, + InsecureSkipTLS: store.insecureSkipTLS, + } - // do an initial fetch to setup remote tracking branches - if _, err := store.fetch(ctx, store.snaps.References()); err != nil { - return nil, err + // if our reference is a branch type then we can assume it exists + // and attempt to only clone from this branch initially + if !store.refTypeTag { + cloneOpts.ReferenceName = plumbing.NewBranchReferenceName(store.baseRef) + cloneOpts.SingleBranch = true + } + + store.repo, err = git.Clone(memory.NewStorage(), nil, cloneOpts) + if err != nil { + return nil, err + } + + // do an initial fetch to setup remote tracking branches + if _, err := store.fetch(ctx, store.snaps.References()); err != nil { + return nil, err + } + } else { + // fetch single reference + store.repo, err = git.InitWithOptions(memory.NewStorage(), nil, git.InitOptions{ + DefaultBranch: plumbing.Main, + }) + if err != nil { + return nil, err + } + + if _, err = store.repo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{store.url}, + }); err != nil { + return nil, err + } + + if err := store.repo.FetchContext(ctx, &git.FetchOptions{ + Auth: store.auth, + Depth: 1, + RefSpecs: []config.RefSpec{ + config.RefSpec(fmt.Sprintf("%[1]s:%[1]s", store.baseRef)), + }, + }); err != nil { + return nil, err + } } // fetch base ref snapshot at-least once before returning store @@ -234,8 +275,10 @@ func (s *SnapshotStore) fetch(ctx context.Context, heads []string) (bool, error) s.mu.Lock() defer s.mu.Unlock() - refSpecs := []config.RefSpec{ - "+refs/tags/*:refs/tags/*", + refSpecs := []config.RefSpec{} + + if s.refTypeTag { + refSpecs = append(refSpecs, "+refs/tags/*:refs/tags/*") } for _, head := range heads { @@ -245,8 +288,9 @@ func (s *SnapshotStore) fetch(ctx context.Context, heads []string) (bool, error) } if err := s.repo.FetchContext(ctx, &git.FetchOptions{ - Auth: s.auth, - RefSpecs: refSpecs, + Auth: s.auth, + RemoteURL: s.url, + RefSpecs: refSpecs, }); err != nil { if !errors.Is(err, git.NoErrAlreadyUpToDate) { return false, err diff --git a/internal/storage/fs/git/store_test.go b/internal/storage/fs/git/store_test.go index 8758371f10..d9e7cfdc2f 100644 --- a/internal/storage/fs/git/store_test.go +++ b/internal/storage/fs/git/store_test.go @@ -274,7 +274,7 @@ func Test_Store_View_WithSemverRevision(t *testing.T) { ch := make(chan struct{}) store, skip := testStore(t, gitRepoURL, WithRef("v0.1.*"), - WithRefResolver(SemverResolver()), + WithSemverResolver(), WithPollOptions( fs.WithInterval(time.Second), fs.WithNotify(t, func(modified bool) { diff --git a/internal/storage/fs/store/store.go b/internal/storage/fs/store/store.go index f76d6a1ed2..83cbc0583a 100644 --- a/internal/storage/fs/store/store.go +++ b/internal/storage/fs/store/store.go @@ -32,14 +32,9 @@ import ( func NewStore(ctx context.Context, logger *zap.Logger, cfg *config.Config) (_ storage.Store, err error) { switch cfg.Storage.Type { case config.GitStorageType: - refResolver := git.StaticResolver() - if cfg.Storage.Git.RefType == config.GitRefTypeSemver { - refResolver = git.SemverResolver() - } - opts := []containers.Option[git.SnapshotStore]{ git.WithRef(cfg.Storage.Git.Ref), - git.WithRefResolver(refResolver), + git.WithSemverResolver(), git.WithPollOptions( storagefs.WithInterval(cfg.Storage.Git.PollInterval), ), @@ -47,6 +42,10 @@ func NewStore(ctx context.Context, logger *zap.Logger, cfg *config.Config) (_ st git.WithDirectory(cfg.Storage.Git.Directory), } + if cfg.Storage.Git.RefType == config.GitRefTypeSemver { + opts = append(opts, git.WithSemverResolver()) + } + if cfg.Storage.Git.CaCertBytes != "" { opts = append(opts, git.WithCABundle([]byte(cfg.Storage.Git.CaCertBytes))) } else if cfg.Storage.Git.CaCertPath != "" {