Add flag to set symlink following behavior for uploading directory (#591

)

We would like to be able to use upload_dir in a situation that makes heavy use of symlinks. However, the previous implementation did not respect symlinks and instead resulted in resolving the symlink to the actual path. This is problematic because it means that duplicated data was going to be recreated on a download.

This is solved by introducing a symlink_behavior argument that accepts unspecified (default), preserve, and resolve which map to the SymlinkBehaviorType enum. This allows the caller to specify how to handle symlinks and ensure that later downloads match exactly what the upload was. This also appears to reduce the data uploaded.

go/pkg/command/command.go

@@ -71,6 +71,17 @@ func (s SymlinkBehaviorType) String() string {
 	return fmt.Sprintf("InvalidSymlinkBehaviorType(%d)", s)
 }
 
+func SymlinkBehaviorFromString(s string) SymlinkBehaviorType {
+	switch s {
+	case "resolve", "ResolveSymlink":
+		return ResolveSymlink
+	case "preserve", "PreserveSymlink":
+		return PreserveSymlink
+	default:
+		return UnspecifiedSymlinkBehavior
+	}
+}
+
 // InputExclusion represents inputs to be excluded from being considered for command execution.
 type InputExclusion struct {
 	// Required: the path regular expression to match for exclusion.

go/pkg/tool/embeddedtool.go

@@ -11,24 +11,27 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/bazelbuild/remote-apis-sdks/go/pkg/command"
 	"github.com/bazelbuild/remote-apis-sdks/go/pkg/outerr"
 
 	log "github.com/golang/glog"
 )
 
 var (
-	inputDigest  string
-	pathPrefix   string
-	overwrite    bool
-	actionRoot   string
-	execAttempts int
-	jsonOutput   string
+	inputDigest     string
+	pathPrefix      string
+	symlinkBehavior string
+	overwrite       bool
+	actionRoot      string
+	execAttempts    int
+	jsonOutput      string
 )
 
 // RegisterFlags registers the flags necessary for the embedded tool to work.
 func RegisterFlags() {
 	flag.StringVar(&inputDigest, "digest", "", "Digest in <digest/size_bytes> format. This flag should not be provided if action_root is set.")
 	flag.StringVar(&pathPrefix, "path", "", "Path to which outputs should be downloaded to.")
+	flag.StringVar(&symlinkBehavior, "symlink_behavior", "unspecified", "For upload_dir: how to handle symlinks. One of 'unspecified', 'resolve', 'preserve'.")
 	flag.BoolVar(&overwrite, "overwrite", false, "Overwrite the output path if it already exist.")
 	flag.StringVar(&actionRoot, "action_root", "", "For execute_action: the root of the action spec, containing ac.textproto (Action proto), cmd.textproto (Command proto), and input/ (root of the input tree). This flag should not be provided if digest is set.")
 	flag.IntVar(&execAttempts, "exec_attempts", 10, "For check_determinism: the number of times to remotely execute the action and check for mismatches.")
@@ -121,7 +124,7 @@ var RemoteToolOperations = map[OpType]func(ctx context.Context, c *Client){
 		}
 	},
 	uploadDir: func(ctx context.Context, c *Client) {
-		us, err := c.UploadDirectory(ctx, getPathFlag())
+		us, err := c.UploadDirectory(ctx, getPathFlag(), command.SymlinkBehaviorFromString(symlinkBehavior))
 		if jsonOutput != "" {
 			js, _ := json.MarshalIndent(us, "", "  ")
 			if jsonOutput == "-" {

go/pkg/tool/tool.go

@@ -338,9 +338,9 @@ type UploadStats struct {
 }
 
 // UploadDirectory uploads a directory from the specified path as a Merkle-tree to the remote cache.
-func (c *Client) UploadDirectory(ctx context.Context, path string) (*UploadStats, error) {
+func (c *Client) UploadDirectory(ctx context.Context, path string, symlinkBehavior command.SymlinkBehaviorType) (*UploadStats, error) {
 	log.Infof("Computing Merkle tree rooted at %s", path)
-	root, blobs, stats, err := c.GrpcClient.ComputeMerkleTree(ctx, path, "", "", &command.InputSpec{Inputs: []string{"."}}, filemetadata.NewNoopCache())
+	root, blobs, stats, err := c.GrpcClient.ComputeMerkleTree(ctx, path, "", "", &command.InputSpec{Inputs: []string{"."}, SymlinkBehavior: symlinkBehavior}, filemetadata.NewNoopCache())
 	if err != nil {
 		return &UploadStats{Error: err.Error()}, err
 	}