diff --git a/envbuilder.go b/envbuilder.go index 37a97bb..e7df546 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -246,7 +246,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro fallbackErr = git.ShallowCloneRepo(ctx, logStage, cloneOpts) if fallbackErr == nil { - endStage("📦 Cloned repository!") + endStage("📦 Cloned repository") buildTimeWorkspaceFolder = cloneOpts.Path } else { opts.Logger(log.LevelError, "Failed to clone repository for remote repo mode: %s", fallbackErr.Error()) diff --git a/git/git.go b/git/git.go index f37b968..ca8faca 100644 --- a/git/git.go +++ b/git/git.go @@ -10,11 +10,13 @@ import ( "os" "strings" + "github.com/coder/envbuilder/log" "github.com/coder/envbuilder/options" giturls "github.com/chainguard-dev/git-urls" "github.com/go-git/go-billy/v5" "github.com/go-git/go-git/v5" + gitconfig "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/protocol/packp/capability" @@ -31,6 +33,7 @@ import ( type CloneRepoOptions struct { Path string Storage billy.Filesystem + Logger log.Func RepoURL string RepoAuth transport.AuthMethod @@ -46,6 +49,7 @@ type CloneRepoOptions struct { // If a repository is already initialized at the given path, it will not // be cloned again. // +// The string returned is the hash of the repository HEAD. // The bool returned states whether the repository was cloned or not. func CloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOptions) (bool, error) { parsed, err := giturls.Parse(opts.RepoURL) @@ -53,6 +57,7 @@ func CloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOpt return false, fmt.Errorf("parse url %q: %w", opts.RepoURL, err) } logf("Parsed Git URL as %q", parsed.Redacted()) + opts.RepoURL = parsed.String() if parsed.Hostname() == "dev.azure.com" { // Azure DevOps requires capabilities multi_ack / multi_ack_detailed, // which are not fully implemented and by default are included in @@ -96,19 +101,19 @@ func CloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOpt return false, fmt.Errorf("chroot .git: %w", err) } gitStorage := filesystem.NewStorage(gitDir, cache.NewObjectLRU(cache.DefaultMaxSize*10)) - fsStorage := filesystem.NewStorage(fs, cache.NewObjectLRU(cache.DefaultMaxSize*10)) - repo, err := git.Open(fsStorage, gitDir) - if errors.Is(err, git.ErrRepositoryNotExists) { - err = nil + repo, err := git.Open(gitStorage, fs) + if err == nil { + // Repo exists, so fast-forward it. + return fastForwardRepo(ctx, logf, opts, repo, reference) } - if err != nil { + + // Something went wrong opening the repo. + if !errors.Is(err, git.ErrRepositoryNotExists) { return false, fmt.Errorf("open %q: %w", opts.RepoURL, err) } - if repo != nil { - return false, nil - } - _, err = git.CloneContext(ctx, gitStorage, fs, &git.CloneOptions{ + // Repo does not exist, so clone it. + repo, err = git.CloneContext(ctx, gitStorage, fs, &git.CloneOptions{ URL: parsed.String(), Auth: opts.RepoAuth, Progress: opts.Progress, @@ -125,13 +130,114 @@ func CloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOpt if err != nil { return false, fmt.Errorf("clone %q: %w", opts.RepoURL, err) } + if head, err := repoHead(repo); err != nil { + logf("failed to get repo HEAD: %s", err) + } else { + logf("cloned repo at %s", head) + } return true, nil } +func fastForwardRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOptions, repo *git.Repository, referenceName string) (bool, error) { + if referenceName == "" { + referenceName = "refs/heads/main" + } + ref := plumbing.ReferenceName(referenceName) + headBefore, err := repoHead(repo) + if err != nil { + return false, err + } + logf("existing repo HEAD: %s", headBefore) + + remote, err := repo.Remote("origin") + if err != nil { + return false, fmt.Errorf("remote: %w", err) + } + + if len(remote.Config().URLs) == 0 { + logf("remote %q has no URLs configured!", remote.String()) + return false, nil + } + + // If the remote URL differs, stop! We don't want to accidentally + // fetch from a different remote. + for _, u := range remote.Config().URLs { + if u != opts.RepoURL { + logf("remote %q has URL %q, expected %q", remote.String(), u, opts.RepoURL) + return false, nil + } + } + + var refSpecs []gitconfig.RefSpec + if referenceName != "" { + // git checkout -b --track / + refSpecs = []gitconfig.RefSpec{ + gitconfig.RefSpec(fmt.Sprintf("%s:%s", referenceName, referenceName)), + } + } + + if err := remote.FetchContext(ctx, &git.FetchOptions{ + Auth: opts.RepoAuth, + CABundle: opts.CABundle, + Depth: opts.Depth, + Force: false, + InsecureSkipTLS: opts.Insecure, + Progress: opts.Progress, + ProxyOptions: opts.ProxyOptions, + RefSpecs: refSpecs, + RemoteName: "origin", + }); err != nil { + if err != git.NoErrAlreadyUpToDate { + return false, fmt.Errorf("fetch changes from remote: %w", err) + } + logf("repository is already up-to-date") + } + + // Now that we've fetched changes from the remote, + // attempt to check out the requested reference. + wt, err := repo.Worktree() + if err != nil { + return false, fmt.Errorf("worktree: %w", err) + } + st, err := wt.Status() + if err != nil { + return false, fmt.Errorf("status: %w", err) + } + // If the working tree is dirty, assume that the user wishes to + // test local changes. Skip the checkout. + if !st.IsClean() { + logf("working tree is dirty, skipping checkout") + return false, nil + } + if err := wt.Checkout(&git.CheckoutOptions{ + Branch: ref, + // Note: we already checked that the working tree is clean, but I'd rather + // err on the side of caution. + Keep: true, + }); err != nil { + return false, fmt.Errorf("checkout branch %s: %w", ref.String(), err) + } + headAfter, err := repoHead(repo) + if err != nil { + return false, fmt.Errorf("check repo HEAD: %w", err) + } + logf("checked out %s", headAfter) + return headBefore != headAfter, nil +} + +func repoHead(repo *git.Repository) (string, error) { + head, err := repo.Head() + if err != nil { + return "", err + } + return head.Hash().String(), nil +} + // ShallowCloneRepo will clone the repository at the given URL into the given path // with a depth of 1. If the destination folder exists and is not empty, the // clone will not be performed. // +// The string returned is the hash of the repository HEAD. // The bool returned states whether the repository was cloned or not. func ShallowCloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOptions) error { opts.Depth = 1 diff --git a/git/git_test.go b/git/git_test.go index 0da5a16..ef21646 100644 --- a/git/git_test.go +++ b/git/git_test.go @@ -23,6 +23,7 @@ import ( "github.com/go-git/go-billy/v5/osfs" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" gossh "golang.org/x/crypto/ssh" ) @@ -39,6 +40,7 @@ func TestCloneRepo(t *testing.T) { mungeURL func(*string) expectError string expectClone bool + prepFS func(billy.Filesystem) }{ { name: "no auth", @@ -84,26 +86,36 @@ func TestCloneRepo(t *testing.T) { // We do not overwrite a repo if one is already present. t.Run("AlreadyCloned", func(t *testing.T) { srvFS := memfs.New() - _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) + _ = gittest.NewRepo(t, srvFS).Commit(gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) authMW := mwtest.BasicAuthMW(tc.srvUsername, tc.srvPassword) srv := httptest.NewServer(authMW(gittest.NewServer(srvFS))) clientFS := memfs.New() // A repo already exists! - _ = gittest.NewRepo(t, clientFS) + _ = gittest.NewRepo(t, clientFS).Commit(gittest.Commit(t, "WRITEME.md", "I'm already here!", "Wow!")) cloned, err := git.CloneRepo(context.Background(), t.Logf, git.CloneRepoOptions{ - Path: "/", + Path: "/workspace", RepoURL: srv.URL, Storage: clientFS, + RepoAuth: &githttp.BasicAuth{ + Username: tc.username, + Password: tc.password, + }, }) - require.NoError(t, err) - require.False(t, cloned) + if tc.expectError != "" { + assert.ErrorContains(t, err, tc.expectError) + } + assert.False(t, cloned) + + // Ensure we do not overwrite the existing repo. + readme := mustRead(t, clientFS, "/workspace/WRITEME.md") + assert.Equal(t, "I'm already here!", readme) }) // Basic Auth t.Run("BasicAuth", func(t *testing.T) { t.Parallel() srvFS := memfs.New() - _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) + _ = gittest.NewRepo(t, srvFS).Commit(gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) authMW := mwtest.BasicAuthMW(tc.srvUsername, tc.srvPassword) srv := httptest.NewServer(authMW(gittest.NewServer(srvFS))) clientFS := memfs.New() @@ -136,7 +148,7 @@ func TestCloneRepo(t *testing.T) { t.Run("InURL", func(t *testing.T) { t.Parallel() srvFS := memfs.New() - _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) + _ = gittest.NewRepo(t, srvFS).Commit(gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) authMW := mwtest.BasicAuthMW(tc.srvUsername, tc.srvPassword) srv := httptest.NewServer(authMW(gittest.NewServer(srvFS))) @@ -174,7 +186,7 @@ func TestShallowCloneRepo(t *testing.T) { t.Run("NotEmpty", func(t *testing.T) { t.Parallel() srvFS := memfs.New() - _ = gittest.NewRepo(t, srvFS, + _ = gittest.NewRepo(t, srvFS).Commit( gittest.Commit(t, "README.md", "Hello, world!", "Many wow!"), gittest.Commit(t, "foo", "bar!", "Such commit!"), gittest.Commit(t, "baz", "qux", "V nice!"), @@ -208,7 +220,7 @@ func TestShallowCloneRepo(t *testing.T) { t.Parallel() srvFS := memfs.New() - _ = gittest.NewRepo(t, srvFS, + _ = gittest.NewRepo(t, srvFS).Commit( gittest.Commit(t, "README.md", "Hello, world!", "Many wow!"), gittest.Commit(t, "foo", "bar!", "Such commit!"), gittest.Commit(t, "baz", "qux", "V nice!"), @@ -235,6 +247,49 @@ func TestShallowCloneRepo(t *testing.T) { }) } +func TestFetchAfterClone(t *testing.T) { + t.Parallel() + + ctx := context.Background() + // ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + // defer cancel() + + // setup a git repo + srvDir := t.TempDir() + srvFS := osfs.New(srvDir) + repo := gittest.NewRepo(t, srvFS) + repo.Commit(gittest.Commit(t, "README.md", "Hello, worldd!", "initial commit")) + srv := httptest.NewServer(gittest.NewServer(srvFS)) + + // clone to a tempdir + clientDir := t.TempDir() + clientFS := osfs.New(clientDir) + cloned, err := git.CloneRepo(ctx, t.Logf, git.CloneRepoOptions{ + Path: "/repo", + RepoURL: srv.URL, + Storage: clientFS, + }) + require.NoError(t, err) + require.True(t, cloned) + + // add some commits on the server + repo.Commit(gittest.Commit(t, "README.md", "Hello, world!", "fix typo")) + + // run CloneRepo again + clonedAgain, err := git.CloneRepo(ctx, t.Logf, git.CloneRepoOptions{ + Path: "/repo", + RepoURL: srv.URL, + Storage: clientFS, + }) + require.NoError(t, err) + require.True(t, clonedAgain) + contentAfter, err := clientFS.Open("/repo/README.md") + require.NoError(t, err) + content, err := io.ReadAll(contentAfter) + require.NoError(t, err) + require.Equal(t, "Hello, world!", string(content), "expected client repo to be updated after fetch") +} + func TestCloneRepoSSH(t *testing.T) { t.Parallel() @@ -245,7 +300,7 @@ func TestCloneRepoSSH(t *testing.T) { tmpDir := t.TempDir() srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) - _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) + _ = gittest.NewRepo(t, srvFS).Commit(gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) key := randKeygen(t) tr := gittest.NewServerSSH(t, srvFS, key.PublicKey()) gitURL := tr.String() @@ -276,7 +331,7 @@ func TestCloneRepoSSH(t *testing.T) { tmpDir := t.TempDir() srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) - _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) + _ = gittest.NewRepo(t, srvFS).Commit(gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) key := randKeygen(t) tr := gittest.NewServerSSH(t, srvFS, key.PublicKey()) gitURL := tr.String() @@ -307,7 +362,7 @@ func TestCloneRepoSSH(t *testing.T) { tmpDir := t.TempDir() srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) - _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) + _ = gittest.NewRepo(t, srvFS).Commit(gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) key := randKeygen(t) tr := gittest.NewServerSSH(t, srvFS, key.PublicKey()) gitURL := tr.String() diff --git a/integration/integration_test.go b/integration/integration_test.go index 2c12b97..c15c778 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -431,7 +431,7 @@ func TestGitSSHAuth(t *testing.T) { tmpDir := t.TempDir() srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) - _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "Dockerfile", "FROM "+testImageAlpine, "Initial commit")) + _ = gittest.NewRepo(t, srvFS).Commit(gittest.Commit(t, "Dockerfile", "FROM "+testImageAlpine, "Initial commit")) tr := gittest.NewServerSSH(t, srvFS, signer.PublicKey()) _, err = runEnvbuilder(t, runOpts{env: []string{ @@ -456,7 +456,7 @@ func TestGitSSHAuth(t *testing.T) { tmpDir := t.TempDir() srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) - _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "Dockerfile", "FROM "+testImageAlpine, "Initial commit")) + _ = gittest.NewRepo(t, srvFS).Commit(gittest.Commit(t, "Dockerfile", "FROM "+testImageAlpine, "Initial commit")) tr := gittest.NewServerSSH(t, srvFS, signer.PublicKey()) _, err = runEnvbuilder(t, runOpts{env: []string{ @@ -2085,7 +2085,7 @@ RUN echo $FOO > /root/foo.txt RUN date --utc > /root/date.txt `, testImageAlpine) - newServer := func(dockerfile string) *httptest.Server { + newServer := func(dockerfile string) *gittest.GitServer { return gittest.CreateGitServer(t, gittest.Options{ Files: map[string]string{"Dockerfile": dockerfile}, }) diff --git a/testutil/gittest/gittest.go b/testutil/gittest/gittest.go index f3d5f1d..ee8becb 100644 --- a/testutil/gittest/gittest.go +++ b/testutil/gittest/gittest.go @@ -39,9 +39,14 @@ type Options struct { TLS bool } +type GitServer struct { + *httptest.Server + Repo *GitRepo +} + // CreateGitServer creates a git repository with an in-memory filesystem // and serves it over HTTP using a httptest.Server. -func CreateGitServer(t *testing.T, opts Options) *httptest.Server { +func CreateGitServer(t *testing.T, opts Options) *GitServer { t.Helper() if opts.AuthMW == nil { opts.AuthMW = mwtest.BasicAuthMW(opts.Username, opts.Password) @@ -51,11 +56,19 @@ func CreateGitServer(t *testing.T, opts Options) *httptest.Server { commits = append(commits, Commit(t, path, content, "my test commit")) } fs := memfs.New() - _ = NewRepo(t, fs, commits...) + repo := NewRepo(t, fs).Commit(commits...) + var srv *httptest.Server if opts.TLS { - return httptest.NewTLSServer(opts.AuthMW(NewServer(fs))) + srv = httptest.NewTLSServer(opts.AuthMW(NewServer(fs))) + // return httptest.NewTLSServer(opts.AuthMW(NewServer(fs))) + } else { + srv = httptest.NewServer(opts.AuthMW(NewServer(fs))) + } + // return httptest.NewServer(opts.AuthMW(NewServer(fs))) + return &GitServer{ + Server: srv, + Repo: repo, } - return httptest.NewServer(opts.AuthMW(NewServer(fs))) } // NewServer returns a http.Handler that serves a git repository. @@ -246,13 +259,21 @@ func Commit(t *testing.T, path, content, msg string) CommitFunc { }, }) require.NoError(t, err) - _, err = repo.CommitObject(commit) + obj, err := repo.CommitObject(commit) + require.NoError(t, err) + t.Logf("commited object %s", obj.Hash.String()) + err = repo.Storer.SetReference(plumbing.NewHashReference("refs/heads/main", obj.Hash)) require.NoError(t, err) } } +type GitRepo struct { + Repo *git.Repository + FS billy.Filesystem +} + // NewRepo returns a new Git repository. -func NewRepo(t *testing.T, fs billy.Filesystem, commits ...CommitFunc) *git.Repository { +func NewRepo(t *testing.T, fs billy.Filesystem) *GitRepo { t.Helper() storage := filesystem.NewStorage(fs, cache.NewObjectLRU(cache.DefaultMaxSize)) repo, err := git.Init(storage, fs) @@ -263,16 +284,23 @@ func NewRepo(t *testing.T, fs billy.Filesystem, commits ...CommitFunc) *git.Repo err = storage.SetReference(h) require.NoError(t, err) + return &GitRepo{ + Repo: repo, + FS: fs, + } +} + +func (g *GitRepo) Commit(commits ...CommitFunc) *GitRepo { for _, commit := range commits { - commit(fs, repo) + commit(g.FS, g.Repo) } - return repo + return g } // WriteFile writes a file to the filesystem. func WriteFile(t *testing.T, fs billy.Filesystem, path, content string) { t.Helper() - file, err := fs.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644) + file, err := fs.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0o644) require.NoError(t, err) _, err = file.Write([]byte(content)) require.NoError(t, err)