git: Use blobless clone instead of shallow clones

Shallow clones are destructive to the history of the repo, and cannot be used in situations where we build from a git source and want to do cherry-picks from that same git source (for instance in binary repos like linux-firmware).

To fix this use blobless clones instead which gets the full history but only downloads blobs on demand. This is slightly less space efficient but works with essentially all git commands. This also lets us simplify the clone and fetch logic somewhat.

Signed-off-by: Reilly Brogan <[email protected]>

builder/source/git.go

@@ -72,50 +72,40 @@ func (g *GitSource) submodules() error {
 
 	cmd.Dir = g.ClonePath
 	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
+	cmd.Stderr = os.Stdout
 
 	return cmd.Run()
 }
 
-// clone shallow clones an upstream git repository to the local disk.
-func clone(uri, path, ref string) error {
-	var cmd *exec.Cmd
-
-	// Check if the reference is a commit
-	if len(ref) == 40 {
-		// Init a new repository at the checkout path
-		initCmd := exec.Command("git", "init", path)
-
-		if err := initCmd.Run(); err != nil {
-			return err
-		}
+// For some reason git blobless clones create pack files with 600 permissions. These break future operations
+// as those files cannot be read by non-root users. Fix those permissions so things work as they should.
+func (g *GitSource) fixPermissions() error {
+	cmd := exec.Command("bash", "-c", "chmod +r .git/objects/pack/*.promisor")
 
-		// Set the default remote to the upstream URI
-		addRemoteCmd := exec.Command("git", "remote", "add", "origin", uri)
-		addRemoteCmd.Dir = path
+	cmd.Dir = g.ClonePath
 
-		if err := addRemoteCmd.Run(); err != nil {
-			return err
-		}
+	return cmd.Run()
+}
 
-		// Shallow fetch the reference we want
-		fetchCmd := exec.Command("git", "fetch", "--depth", "1", "origin", ref)
-		fetchCmd.Dir = path
+// clone shallow clones an upstream git repository to the local disk.
+func clone(uri, path, ref string) error {
+	var cmd *exec.Cmd
 
-		if err := fetchCmd.Run(); err != nil {
-			return err
-		}
+	// Create a blobless clone without checking out a ref
+	initCmd := exec.Command("git", "clone", "--filter=blob:none", "--no-checkout", uri, path)
+	initCmd.Stdout = os.Stdout
+	initCmd.Stderr = os.Stdout
 
-		// Set the next command to run to checkout the head
-		cmd = exec.Command("git", "checkout", "FETCH_HEAD")
-		cmd.Dir = path
-	} else {
-		// Not a git commit, so shallow clone the repo at the reference
-		cmd = exec.Command("git", "clone", "--depth", "1", "--branch", ref, uri, path)
+	if err := initCmd.Run(); err != nil {
+		return err
 	}
 
+	// Checkout the ref we want
+	cmd = exec.Command("git", "switch", "--discard-changes", "--recurse-submodules", "--detach", ref)
+	cmd.Dir = path
+
 	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
+	cmd.Stderr = os.Stdout
 
 	return cmd.Run()
 }
@@ -125,27 +115,17 @@ func clone(uri, path, ref string) error {
 func reset(path, ref string) error {
 	fetchArgs := []string{
 		"fetch",
-		"--depth",
-		"1",
 		"origin",
 	}
 
-	// We have to add the tag keyword if the ref is a tag, otherwise
-	// git won't actually fetch the tag
-	if len(ref) != 40 {
-		fetchArgs = append(fetchArgs, "tag")
-	}
-
-	fetchArgs = append(fetchArgs, ref)
-
 	fetchCmd := exec.Command("git", fetchArgs...)
 	fetchCmd.Dir = path
 
 	if err := fetchCmd.Run(); err != nil {
 		return err
 	}
 
-	resetCmd := exec.Command("git", "reset", "--hard", ref)
+	resetCmd := exec.Command("git", "switch", "--discard-changes", "--recurse-submodules", "--detach", ref)
 	resetCmd.Dir = path
 
 	return resetCmd.Run()
@@ -167,7 +147,12 @@ func (g *GitSource) Fetch() error {
 	}
 
 	// Check out submodules
-	return g.submodules()
+	err := g.submodules()
+	if err != nil {
+		return err
+	}
+
+	return g.fixPermissions()
 }
 
 // IsFetched will check if we have the ref available, if not it will return