From 260ebf98faabafce973429fb2592e956a9a624ba Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 10 Sep 2024 16:50:29 +0100 Subject: [PATCH 1/7] gittest: expose git server and allow committing from it --- git/git_test.go | 16 +++++++------- integration/integration_test.go | 2 +- testutil/gittest/gittest.go | 39 +++++++++++++++++++++++++++------ 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/git/git_test.go b/git/git_test.go index e7a58f9..65fd7d5 100644 --- a/git/git_test.go +++ b/git/git_test.go @@ -83,7 +83,7 @@ func TestCloneRepo(t *testing.T) { // We do not overwrite a repo if one is already present. t.Run("AlreadyCloned", func(t *testing.T) { srvFS := memfs.New() - _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) + _ = gittest.NewRepo(t, srvFS).Commit(gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) authMW := mwtest.BasicAuthMW(tc.srvUsername, tc.srvPassword) srv := httptest.NewServer(authMW(gittest.NewServer(srvFS))) clientFS := memfs.New() @@ -102,7 +102,7 @@ func TestCloneRepo(t *testing.T) { t.Run("BasicAuth", func(t *testing.T) { t.Parallel() srvFS := memfs.New() - _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) + _ = gittest.NewRepo(t, srvFS).Commit(gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) authMW := mwtest.BasicAuthMW(tc.srvUsername, tc.srvPassword) srv := httptest.NewServer(authMW(gittest.NewServer(srvFS))) clientFS := memfs.New() @@ -135,7 +135,7 @@ func TestCloneRepo(t *testing.T) { t.Run("InURL", func(t *testing.T) { t.Parallel() srvFS := memfs.New() - _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) + _ = gittest.NewRepo(t, srvFS).Commit(gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) authMW := mwtest.BasicAuthMW(tc.srvUsername, tc.srvPassword) srv := httptest.NewServer(authMW(gittest.NewServer(srvFS))) @@ -173,7 +173,7 @@ func TestShallowCloneRepo(t *testing.T) { t.Run("NotEmpty", func(t *testing.T) { t.Parallel() srvFS := memfs.New() - _ = gittest.NewRepo(t, srvFS, + _ = gittest.NewRepo(t, srvFS).Commit( gittest.Commit(t, "README.md", "Hello, world!", "Many wow!"), gittest.Commit(t, "foo", "bar!", "Such commit!"), gittest.Commit(t, "baz", "qux", "V nice!"), @@ -207,7 +207,7 @@ func TestShallowCloneRepo(t *testing.T) { t.Parallel() srvFS := memfs.New() - _ = gittest.NewRepo(t, srvFS, + _ = gittest.NewRepo(t, srvFS).Commit( gittest.Commit(t, "README.md", "Hello, world!", "Many wow!"), gittest.Commit(t, "foo", "bar!", "Such commit!"), gittest.Commit(t, "baz", "qux", "V nice!"), @@ -244,7 +244,7 @@ func TestCloneRepoSSH(t *testing.T) { tmpDir := t.TempDir() srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) - _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) + _ = gittest.NewRepo(t, srvFS).Commit(gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) key := randKeygen(t) tr := gittest.NewServerSSH(t, srvFS, key.PublicKey()) gitURL := tr.String() @@ -275,7 +275,7 @@ func TestCloneRepoSSH(t *testing.T) { tmpDir := t.TempDir() srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) - _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) + _ = gittest.NewRepo(t, srvFS).Commit(gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) key := randKeygen(t) tr := gittest.NewServerSSH(t, srvFS, key.PublicKey()) gitURL := tr.String() @@ -306,7 +306,7 @@ func TestCloneRepoSSH(t *testing.T) { tmpDir := t.TempDir() srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) - _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) + _ = gittest.NewRepo(t, srvFS).Commit(gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) key := randKeygen(t) tr := gittest.NewServerSSH(t, srvFS, key.PublicKey()) gitURL := tr.String() diff --git a/integration/integration_test.go b/integration/integration_test.go index 29b5e8a..24de9ac 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -1682,7 +1682,7 @@ RUN echo $FOO > /root/foo.txt RUN date --utc > /root/date.txt `, testImageAlpine) - newServer := func(dockerfile string) *httptest.Server { + newServer := func(dockerfile string) *gittest.GitServer { return gittest.CreateGitServer(t, gittest.Options{ Files: map[string]string{"Dockerfile": dockerfile}, }) diff --git a/testutil/gittest/gittest.go b/testutil/gittest/gittest.go index f3d5f1d..2cee527 100644 --- a/testutil/gittest/gittest.go +++ b/testutil/gittest/gittest.go @@ -39,9 +39,14 @@ type Options struct { TLS bool } +type GitServer struct { + *httptest.Server + Repo *GitRepo +} + // CreateGitServer creates a git repository with an in-memory filesystem // and serves it over HTTP using a httptest.Server. -func CreateGitServer(t *testing.T, opts Options) *httptest.Server { +func CreateGitServer(t *testing.T, opts Options) *GitServer { t.Helper() if opts.AuthMW == nil { opts.AuthMW = mwtest.BasicAuthMW(opts.Username, opts.Password) @@ -51,11 +56,19 @@ func CreateGitServer(t *testing.T, opts Options) *httptest.Server { commits = append(commits, Commit(t, path, content, "my test commit")) } fs := memfs.New() - _ = NewRepo(t, fs, commits...) + repo := NewRepo(t, fs).Commit(commits...) + var srv *httptest.Server if opts.TLS { - return httptest.NewTLSServer(opts.AuthMW(NewServer(fs))) + srv = httptest.NewTLSServer(opts.AuthMW(NewServer(fs))) + // return httptest.NewTLSServer(opts.AuthMW(NewServer(fs))) + } else { + srv = httptest.NewServer(opts.AuthMW(NewServer(fs))) + } + // return httptest.NewServer(opts.AuthMW(NewServer(fs))) + return &GitServer{ + Server: srv, + Repo: repo, } - return httptest.NewServer(opts.AuthMW(NewServer(fs))) } // NewServer returns a http.Handler that serves a git repository. @@ -251,8 +264,13 @@ func Commit(t *testing.T, path, content, msg string) CommitFunc { } } +type GitRepo struct { + repo *git.Repository + fs billy.Filesystem +} + // NewRepo returns a new Git repository. -func NewRepo(t *testing.T, fs billy.Filesystem, commits ...CommitFunc) *git.Repository { +func NewRepo(t *testing.T, fs billy.Filesystem) *GitRepo { t.Helper() storage := filesystem.NewStorage(fs, cache.NewObjectLRU(cache.DefaultMaxSize)) repo, err := git.Init(storage, fs) @@ -263,10 +281,17 @@ func NewRepo(t *testing.T, fs billy.Filesystem, commits ...CommitFunc) *git.Repo err = storage.SetReference(h) require.NoError(t, err) + return &GitRepo{ + repo: repo, + fs: fs, + } +} + +func (g *GitRepo) Commit(commits ...CommitFunc) *GitRepo { for _, commit := range commits { - commit(fs, repo) + commit(g.fs, g.repo) } - return repo + return g } // WriteFile writes a file to the filesystem. From 5a2d6f52060b4eb55961dc624df92777e467f447 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 10 Sep 2024 20:47:31 +0100 Subject: [PATCH 2/7] gittest: export members of GitRepo --- testutil/gittest/gittest.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/testutil/gittest/gittest.go b/testutil/gittest/gittest.go index 2cee527..396579c 100644 --- a/testutil/gittest/gittest.go +++ b/testutil/gittest/gittest.go @@ -259,14 +259,15 @@ func Commit(t *testing.T, path, content, msg string) CommitFunc { }, }) require.NoError(t, err) - _, err = repo.CommitObject(commit) + obj, err := repo.CommitObject(commit) require.NoError(t, err) + t.Logf("commited object %s", obj.Hash.String()) } } type GitRepo struct { - repo *git.Repository - fs billy.Filesystem + Repo *git.Repository + FS billy.Filesystem } // NewRepo returns a new Git repository. @@ -282,14 +283,14 @@ func NewRepo(t *testing.T, fs billy.Filesystem) *GitRepo { require.NoError(t, err) return &GitRepo{ - repo: repo, - fs: fs, + Repo: repo, + FS: fs, } } func (g *GitRepo) Commit(commits ...CommitFunc) *GitRepo { for _, commit := range commits { - commit(g.fs, g.repo) + commit(g.FS, g.Repo) } return g } @@ -297,7 +298,7 @@ func (g *GitRepo) Commit(commits ...CommitFunc) *GitRepo { // WriteFile writes a file to the filesystem. func WriteFile(t *testing.T, fs billy.Filesystem, path, content string) { t.Helper() - file, err := fs.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644) + file, err := fs.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0o644) require.NoError(t, err) _, err = file.Write([]byte(content)) require.NoError(t, err) From ec3fc5ba4d328cb4af2c00d6247b541ed417b7c2 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 10 Sep 2024 20:47:51 +0100 Subject: [PATCH 3/7] git: add failing test for FetchAfterClone --- git/git_test.go | 55 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/git/git_test.go b/git/git_test.go index 65fd7d5..0772d73 100644 --- a/git/git_test.go +++ b/git/git_test.go @@ -10,7 +10,9 @@ import ( "os" "path/filepath" "regexp" + "strings" "testing" + "time" "github.com/coder/envbuilder/git" "github.com/coder/envbuilder/options" @@ -234,6 +236,59 @@ func TestShallowCloneRepo(t *testing.T) { }) } +func TestFetchAfterClone(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + // setup a git repo + srvDir := t.TempDir() + srvFS := osfs.New(srvDir, osfs.WithChrootOS()) + repo := gittest.NewRepo(t, srvFS) + repo.Commit(gittest.Commit(t, "README.md", "Hello, worldd!", "initial commit")) + headBefore, err := repo.Repo.Head() + require.NoError(t, err) + srv := httptest.NewServer(gittest.NewServer(srvFS)) + + // clone to a tempdir + clientDir := t.TempDir() + clientFS := osfs.New(clientDir, osfs.WithChrootOS()) + cloned, err := git.CloneRepo(ctx, git.CloneRepoOptions{ + Path: "/repo", + RepoURL: srv.URL, + Storage: clientFS, + }) + + require.NoError(t, err) + require.True(t, cloned) + + // add some commits on the server + repo.Commit(gittest.Commit(t, "README.md", "Hello, world!", "fix typo")) + // ensure state + bs, err := os.ReadFile(filepath.Join(srvDir, "README.md")) + require.NoError(t, err) + require.Equal(t, "Hello, world!", strings.TrimSpace(string(bs))) + headAfter, err := repo.Repo.Head() + require.NoError(t, err) + require.NotEqual(t, headBefore.Hash(), headAfter.Hash()) + + // run CloneRepo again + clonedAgain, err := git.CloneRepo(ctx, git.CloneRepoOptions{ + Path: "/repo", + RepoURL: srv.URL, + Storage: clientFS, + }) + require.NoError(t, err) + require.True(t, clonedAgain) + + // Inspect the cloned repo and check last commit + headFile, err := clientFS.Open(filepath.Join(".git/refs/heads/main")) + require.NoError(t, err) + var sb strings.Builder + _, err = io.Copy(&sb, headFile) + require.NoError(t, err) + require.Equal(t, headAfter, sb.String()) +} + func TestCloneRepoSSH(t *testing.T) { t.Parallel() From 47030e1e9b2a3f2e53402f0a5081fcb2f723fb8c Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 11 Sep 2024 14:21:19 +0100 Subject: [PATCH 4/7] git: log head of cloned repo --- envbuilder.go | 2 +- git/git.go | 12 +++++++++++- git/git_test.go | 25 ++++++------------------- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 3df3562..b8802f3 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -164,7 +164,7 @@ func Run(ctx context.Context, opts options.Options) error { fallbackErr = git.ShallowCloneRepo(ctx, logStage, cloneOpts) if fallbackErr == nil { - endStage("📦 Cloned repository!") + endStage("📦 Cloned repository") buildTimeWorkspaceFolder = cloneOpts.Path } else { opts.Logger(log.LevelError, "Failed to clone repository for remote repo mode: %s", fallbackErr.Error()) diff --git a/git/git.go b/git/git.go index 7d132c3..7227ee0 100644 --- a/git/git.go +++ b/git/git.go @@ -9,6 +9,7 @@ import ( "os" "strings" + "github.com/coder/envbuilder/log" "github.com/coder/envbuilder/options" giturls "github.com/chainguard-dev/git-urls" @@ -30,6 +31,7 @@ import ( type CloneRepoOptions struct { Path string Storage billy.Filesystem + Logger log.Func RepoURL string RepoAuth transport.AuthMethod @@ -45,6 +47,7 @@ type CloneRepoOptions struct { // If a repository is already initialized at the given path, it will not // be cloned again. // +// The string returned is the hash of the repository HEAD. // The bool returned states whether the repository was cloned or not. func CloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOptions) (bool, error) { parsed, err := giturls.Parse(opts.RepoURL) @@ -104,10 +107,13 @@ func CloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOpt return false, fmt.Errorf("open %q: %w", opts.RepoURL, err) } if repo != nil { + if head, err := repo.Head(); err == nil && head != nil { + logf("existing repo HEAD: %s", head.Hash().String()) + } return false, nil } - _, err = git.CloneContext(ctx, gitStorage, fs, &git.CloneOptions{ + repo, err = git.CloneContext(ctx, gitStorage, fs, &git.CloneOptions{ URL: parsed.String(), Auth: opts.RepoAuth, Progress: opts.Progress, @@ -124,6 +130,9 @@ func CloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOpt if err != nil { return false, fmt.Errorf("clone %q: %w", opts.RepoURL, err) } + if head, err := repo.Head(); err == nil && head != nil { + logf("cloned repo HEAD: %s", head.Hash().String()) + } return true, nil } @@ -131,6 +140,7 @@ func CloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOpt // with a depth of 1. If the destination folder exists and is not empty, the // clone will not be performed. // +// The string returned is the hash of the repository HEAD. // The bool returned states whether the repository was cloned or not. func ShallowCloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOptions) error { opts.Depth = 1 diff --git a/git/git_test.go b/git/git_test.go index 0772d73..19033ee 100644 --- a/git/git_test.go +++ b/git/git_test.go @@ -10,7 +10,6 @@ import ( "os" "path/filepath" "regexp" - "strings" "testing" "time" @@ -238,6 +237,7 @@ func TestShallowCloneRepo(t *testing.T) { func TestFetchAfterClone(t *testing.T) { t.Parallel() + t.Skip() ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() // setup a git repo @@ -245,48 +245,35 @@ func TestFetchAfterClone(t *testing.T) { srvFS := osfs.New(srvDir, osfs.WithChrootOS()) repo := gittest.NewRepo(t, srvFS) repo.Commit(gittest.Commit(t, "README.md", "Hello, worldd!", "initial commit")) - headBefore, err := repo.Repo.Head() - require.NoError(t, err) srv := httptest.NewServer(gittest.NewServer(srvFS)) // clone to a tempdir clientDir := t.TempDir() clientFS := osfs.New(clientDir, osfs.WithChrootOS()) - cloned, err := git.CloneRepo(ctx, git.CloneRepoOptions{ + cloned, err := git.CloneRepo(ctx, t.Logf, git.CloneRepoOptions{ Path: "/repo", RepoURL: srv.URL, Storage: clientFS, }) - require.NoError(t, err) require.True(t, cloned) // add some commits on the server repo.Commit(gittest.Commit(t, "README.md", "Hello, world!", "fix typo")) - // ensure state - bs, err := os.ReadFile(filepath.Join(srvDir, "README.md")) - require.NoError(t, err) - require.Equal(t, "Hello, world!", strings.TrimSpace(string(bs))) - headAfter, err := repo.Repo.Head() - require.NoError(t, err) - require.NotEqual(t, headBefore.Hash(), headAfter.Hash()) // run CloneRepo again - clonedAgain, err := git.CloneRepo(ctx, git.CloneRepoOptions{ + clonedAgain, err := git.CloneRepo(ctx, t.Logf, git.CloneRepoOptions{ Path: "/repo", RepoURL: srv.URL, Storage: clientFS, }) require.NoError(t, err) require.True(t, clonedAgain) - - // Inspect the cloned repo and check last commit - headFile, err := clientFS.Open(filepath.Join(".git/refs/heads/main")) + contentAfter, err := clientFS.Open("/repo/README.md") require.NoError(t, err) - var sb strings.Builder - _, err = io.Copy(&sb, headFile) + content, err := io.ReadAll(contentAfter) require.NoError(t, err) - require.Equal(t, headAfter, sb.String()) + require.Equal(t, "Hello, worldd!", string(content), "expected client repo to be updated after fetch") } func TestCloneRepoSSH(t *testing.T) { From 644e1983ac89b55c7ef2d80877ca305fea0be2db Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 12 Sep 2024 21:56:45 +0100 Subject: [PATCH 5/7] first stab at fast-forward, need to fix tests --- git/git.go | 51 ++++++++++++++++++++++++++++++------- git/git_test.go | 16 +++++++----- testutil/gittest/gittest.go | 2 ++ 3 files changed, 53 insertions(+), 16 deletions(-) diff --git a/git/git.go b/git/git.go index 7227ee0..5210e9a 100644 --- a/git/git.go +++ b/git/git.go @@ -55,6 +55,7 @@ func CloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOpt return false, fmt.Errorf("parse url %q: %w", opts.RepoURL, err) } logf("Parsed Git URL as %q", parsed.Redacted()) + opts.RepoURL = parsed.String() if parsed.Hostname() == "dev.azure.com" { // Azure DevOps requires capabilities multi_ack / multi_ack_detailed, // which are not fully implemented and by default are included in @@ -100,19 +101,17 @@ func CloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOpt gitStorage := filesystem.NewStorage(gitDir, cache.NewObjectLRU(cache.DefaultMaxSize*10)) fsStorage := filesystem.NewStorage(fs, cache.NewObjectLRU(cache.DefaultMaxSize*10)) repo, err := git.Open(fsStorage, gitDir) - if errors.Is(err, git.ErrRepositoryNotExists) { - err = nil + if err == nil { + // Repo exists, so fast-forward it. + return fastForwardRepo(ctx, logf, opts, repo, reference) } - if err != nil { + + // Something went wrong opening the repo. + if !errors.Is(err, git.ErrRepositoryNotExists) { return false, fmt.Errorf("open %q: %w", opts.RepoURL, err) } - if repo != nil { - if head, err := repo.Head(); err == nil && head != nil { - logf("existing repo HEAD: %s", head.Hash().String()) - } - return false, nil - } + // Repo does not exist, so clone it. repo, err = git.CloneContext(ctx, gitStorage, fs, &git.CloneOptions{ URL: parsed.String(), Auth: opts.RepoAuth, @@ -136,6 +135,40 @@ func CloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOpt return true, nil } +func fastForwardRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOptions, repo *git.Repository, referenceName string) (bool, error) { + if head, err := repo.Head(); err == nil && head != nil { + logf("existing repo HEAD: %s", head.Hash().String()) + } + wt, err := repo.Worktree() + if err != nil { + return false, fmt.Errorf("worktree: %w", err) + } + err = wt.PullContext(ctx, &git.PullOptions{ + RemoteName: "", // use default remote + ReferenceName: plumbing.ReferenceName(referenceName), + SingleBranch: opts.SingleBranch, + Depth: opts.Depth, + Auth: opts.RepoAuth, + Progress: opts.Progress, + Force: false, + InsecureSkipTLS: opts.Insecure, + CABundle: opts.CABundle, + ProxyOptions: opts.ProxyOptions, + }) + if err == nil { + if head, err := repo.Head(); err == nil && head != nil { + logf("fast-forwarded to %s", head.Hash().String()) + } + return true, nil + } + if errors.Is(err, git.NoErrAlreadyUpToDate) { + logf("existing repo already up-to-date") + return false, nil + } + logf("failed to fast-forward: %s", err.Error()) + return false, err +} + // ShallowCloneRepo will clone the repository at the given URL into the given path // with a depth of 1. If the destination folder exists and is not empty, the // clone will not be performed. diff --git a/git/git_test.go b/git/git_test.go index 19033ee..ef1d454 100644 --- a/git/git_test.go +++ b/git/git_test.go @@ -11,7 +11,6 @@ import ( "path/filepath" "regexp" "testing" - "time" "github.com/coder/envbuilder/git" "github.com/coder/envbuilder/options" @@ -39,6 +38,7 @@ func TestCloneRepo(t *testing.T) { mungeURL func(*string) expectError string expectClone bool + prepFS func(billy.Filesystem) }{ { name: "no auth", @@ -237,19 +237,21 @@ func TestShallowCloneRepo(t *testing.T) { func TestFetchAfterClone(t *testing.T) { t.Parallel() - t.Skip() - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() + + ctx := context.Background() + // ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + // defer cancel() + // setup a git repo srvDir := t.TempDir() - srvFS := osfs.New(srvDir, osfs.WithChrootOS()) + srvFS := osfs.New(srvDir) repo := gittest.NewRepo(t, srvFS) repo.Commit(gittest.Commit(t, "README.md", "Hello, worldd!", "initial commit")) srv := httptest.NewServer(gittest.NewServer(srvFS)) // clone to a tempdir clientDir := t.TempDir() - clientFS := osfs.New(clientDir, osfs.WithChrootOS()) + clientFS := osfs.New(clientDir) cloned, err := git.CloneRepo(ctx, t.Logf, git.CloneRepoOptions{ Path: "/repo", RepoURL: srv.URL, @@ -273,7 +275,7 @@ func TestFetchAfterClone(t *testing.T) { require.NoError(t, err) content, err := io.ReadAll(contentAfter) require.NoError(t, err) - require.Equal(t, "Hello, worldd!", string(content), "expected client repo to be updated after fetch") + require.Equal(t, "Hello, world!", string(content), "expected client repo to be updated after fetch") } func TestCloneRepoSSH(t *testing.T) { diff --git a/testutil/gittest/gittest.go b/testutil/gittest/gittest.go index 396579c..ee8becb 100644 --- a/testutil/gittest/gittest.go +++ b/testutil/gittest/gittest.go @@ -262,6 +262,8 @@ func Commit(t *testing.T, path, content, msg string) CommitFunc { obj, err := repo.CommitObject(commit) require.NoError(t, err) t.Logf("commited object %s", obj.Hash.String()) + err = repo.Storer.SetReference(plumbing.NewHashReference("refs/heads/main", obj.Hash)) + require.NoError(t, err) } } From 697f9c4f9ebf995b6a6485f597bb55301a22a1c2 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 13 Sep 2024 12:36:02 +0100 Subject: [PATCH 6/7] fetch instead of pull, try to detect different remote --- git/git.go | 111 +++++++++++++++++++++++++++++++++++++----------- git/git_test.go | 19 +++++++-- 2 files changed, 102 insertions(+), 28 deletions(-) diff --git a/git/git.go b/git/git.go index 5210e9a..1c80fe9 100644 --- a/git/git.go +++ b/git/git.go @@ -15,6 +15,7 @@ import ( giturls "github.com/chainguard-dev/git-urls" "github.com/go-git/go-billy/v5" "github.com/go-git/go-git/v5" + gitconfig "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/protocol/packp/capability" @@ -99,8 +100,7 @@ func CloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOpt return false, fmt.Errorf("chroot .git: %w", err) } gitStorage := filesystem.NewStorage(gitDir, cache.NewObjectLRU(cache.DefaultMaxSize*10)) - fsStorage := filesystem.NewStorage(fs, cache.NewObjectLRU(cache.DefaultMaxSize*10)) - repo, err := git.Open(fsStorage, gitDir) + repo, err := git.Open(gitStorage, fs) if err == nil { // Repo exists, so fast-forward it. return fastForwardRepo(ctx, logf, opts, repo, reference) @@ -129,44 +129,107 @@ func CloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOpt if err != nil { return false, fmt.Errorf("clone %q: %w", opts.RepoURL, err) } - if head, err := repo.Head(); err == nil && head != nil { - logf("cloned repo HEAD: %s", head.Hash().String()) + if head, err := repoHead(repo); err != nil { + logf("failed to get repo HEAD: %s", err) + } else { + logf("cloned repo at %s", head) } return true, nil } func fastForwardRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOptions, repo *git.Repository, referenceName string) (bool, error) { - if head, err := repo.Head(); err == nil && head != nil { - logf("existing repo HEAD: %s", head.Hash().String()) + if referenceName == "" { + referenceName = "refs/heads/main" } - wt, err := repo.Worktree() + ref := plumbing.ReferenceName(referenceName) + headBefore, err := repoHead(repo) if err != nil { - return false, fmt.Errorf("worktree: %w", err) + return false, err } - err = wt.PullContext(ctx, &git.PullOptions{ - RemoteName: "", // use default remote - ReferenceName: plumbing.ReferenceName(referenceName), - SingleBranch: opts.SingleBranch, - Depth: opts.Depth, + logf("existing repo HEAD: %s", headBefore) + + remote, err := repo.Remote("origin") + if err != nil { + return false, fmt.Errorf("remote: %w", err) + } + + if len(remote.Config().URLs) == 0 { + logf("remote %q has no URLs configured!", remote.String()) + return false, nil + } + + // If the remote URL differs, stop! We don't want to accidentally + // fetch from a different remote. + for _, u := range remote.Config().URLs { + if u != opts.RepoURL { + logf("remote %q has URL %q, expected %q", remote.String(), u, opts.RepoURL) + return false, nil + } + } + + var refSpecs []gitconfig.RefSpec + if referenceName != "" { + // git checkout -b --track / + refSpecs = []gitconfig.RefSpec{ + gitconfig.RefSpec(fmt.Sprintf("%s:%s", referenceName, referenceName)), + } + } + + if err := remote.FetchContext(ctx, &git.FetchOptions{ Auth: opts.RepoAuth, - Progress: opts.Progress, + CABundle: opts.CABundle, + Depth: opts.Depth, Force: false, InsecureSkipTLS: opts.Insecure, - CABundle: opts.CABundle, + Progress: opts.Progress, ProxyOptions: opts.ProxyOptions, - }) - if err == nil { - if head, err := repo.Head(); err == nil && head != nil { - logf("fast-forwarded to %s", head.Hash().String()) + RefSpecs: refSpecs, + RemoteName: "origin", + }); err != nil { + if err != git.NoErrAlreadyUpToDate { + return false, fmt.Errorf("fetch changes from remote: %w", err) } - return true, nil + logf("repository is already up-to-date") + } + + // Now that we've fetched changes from the remote, + // attempt to check out the requested reference. + wt, err := repo.Worktree() + if err != nil { + return false, fmt.Errorf("worktree: %w", err) } - if errors.Is(err, git.NoErrAlreadyUpToDate) { - logf("existing repo already up-to-date") + st, err := wt.Status() + if err != nil { + return false, fmt.Errorf("status: %w", err) + } + // If the working tree is dirty, assume that the user wishes to + // test local changes. Skip the checkout. + if !st.IsClean() { + logf("working tree is dirty, skipping checkout") return false, nil } - logf("failed to fast-forward: %s", err.Error()) - return false, err + if err := wt.Checkout(&git.CheckoutOptions{ + Branch: ref, + // Note: we already checked that the working tree is clean, but I'd rather + // err on the side of caution. + Keep: true, + }); err != nil { + return false, fmt.Errorf("checkout branch %s: %w", ref.String(), err) + } + headAfter, err := repoHead(repo) + if err != nil { + return false, fmt.Errorf("check repo HEAD: %w", err) + } + logf("checked out %s", headAfter) + return headBefore != headAfter, nil +} + +func repoHead(repo *git.Repository) (string, error) { + head, err := repo.Head() + if err != nil { + return "", err + } + return head.Hash().String(), nil } // ShallowCloneRepo will clone the repository at the given URL into the given path diff --git a/git/git_test.go b/git/git_test.go index ef1d454..2880a0c 100644 --- a/git/git_test.go +++ b/git/git_test.go @@ -22,6 +22,7 @@ import ( "github.com/go-git/go-billy/v5/osfs" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" gossh "golang.org/x/crypto/ssh" ) @@ -89,14 +90,24 @@ func TestCloneRepo(t *testing.T) { srv := httptest.NewServer(authMW(gittest.NewServer(srvFS))) clientFS := memfs.New() // A repo already exists! - _ = gittest.NewRepo(t, clientFS) + _ = gittest.NewRepo(t, clientFS).Commit(gittest.Commit(t, "WRITEME.md", "I'm already here!", "Wow!")) cloned, err := git.CloneRepo(context.Background(), t.Logf, git.CloneRepoOptions{ - Path: "/", + Path: "/workspace", RepoURL: srv.URL, Storage: clientFS, + RepoAuth: &githttp.BasicAuth{ + Username: tc.username, + Password: tc.password, + }, }) - require.NoError(t, err) - require.False(t, cloned) + if tc.expectError != "" { + assert.ErrorContains(t, err, tc.expectError) + } + assert.False(t, cloned) + + // Ensure we do not overwrite the existing repo. + readme := mustRead(t, clientFS, "/workspace/WRITEME.md") + assert.Equal(t, "I'm already here!", readme) }) // Basic Auth From a786a59f383f96b015e271a74730915085acc02b Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Mon, 18 Nov 2024 11:10:05 +0000 Subject: [PATCH 7/7] fix(integration): fix unit tests --- integration/integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/integration_test.go b/integration/integration_test.go index 937abe3..c15c778 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -431,7 +431,7 @@ func TestGitSSHAuth(t *testing.T) { tmpDir := t.TempDir() srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) - _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "Dockerfile", "FROM "+testImageAlpine, "Initial commit")) + _ = gittest.NewRepo(t, srvFS).Commit(gittest.Commit(t, "Dockerfile", "FROM "+testImageAlpine, "Initial commit")) tr := gittest.NewServerSSH(t, srvFS, signer.PublicKey()) _, err = runEnvbuilder(t, runOpts{env: []string{ @@ -456,7 +456,7 @@ func TestGitSSHAuth(t *testing.T) { tmpDir := t.TempDir() srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) - _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "Dockerfile", "FROM "+testImageAlpine, "Initial commit")) + _ = gittest.NewRepo(t, srvFS).Commit(gittest.Commit(t, "Dockerfile", "FROM "+testImageAlpine, "Initial commit")) tr := gittest.NewServerSSH(t, srvFS, signer.PublicKey()) _, err = runEnvbuilder(t, runOpts{env: []string{