Get input output list with symlinks

go/pkg/fakes/server.go

@@ -28,7 +28,7 @@ import (
 	repb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
 	bsgrpc "google.golang.org/genproto/googleapis/bytestream"
 	bspb "google.golang.org/genproto/googleapis/bytestream"
-	anypb "google.golang.org/protobuf/types/known/anypb"
+	"google.golang.org/protobuf/types/known/anypb"
 	dpb "google.golang.org/protobuf/types/known/durationpb"
 	tspb "google.golang.org/protobuf/types/known/timestamppb"
 )
@@ -307,6 +307,26 @@ func (f *InputFile) apply(ac *repb.ActionResult, s *Server, execRoot string) err
 	return nil
 }
 
+// InputSymlink (Path -> Target) to be made available to the fake action.
+type InputSymlink struct {
+	Path          string // newname
+	TargetContent string
+	Target        string // oldname
+}
+
+// Apply puts the target file in the fake CAS and create a symlink in OS.
+func (ins *InputSymlink) apply(ac *repb.ActionResult, s *Server, execRoot string) error {
+	inf := InputFile{Path: ins.Target, Contents: ins.TargetContent}
+	if err := inf.apply(ac, s, execRoot); err != nil {
+		return err
+	}
+	// create a symlink from old name (target) to new name (path)
+	if err := os.Symlink(filepath.Join(execRoot, ins.Target), filepath.Join(execRoot, ins.Path)); err != nil {
+		return fmt.Errorf("error creating symlink: %w", err)
+	}
+	return nil
+}
+
 // OutputFile is to be added as an output of the fake action.
 type OutputFile struct {
 	Path     string

go/pkg/tool/tool.go

@@ -706,7 +706,7 @@ func (c *Client) formatAction(ctx context.Context, actionProto *repb.Action, res
 	}
 	showActionRes.WriteString("\nInputs\n======\n")
 	log.Infof("Fetching input tree from input root digest..")
-	inpTree, _, err := c.getInputTree(ctx, actionProto.GetInputRootDigest())
+	inpTree, _, _, err := c.getInputTree(ctx, actionProto.GetInputRootDigest())
 	if err != nil {
 		showActionRes.WriteString("Failed to fetch input tree:\n")
 		showActionRes.WriteString(err.Error())
@@ -773,7 +773,7 @@ func (c *Client) getOutputs(ctx context.Context, actionRes *repb.ActionResult) (
 			return "", err
 		}
 
-		outputs, _, err := c.flattenTree(ctx, outDirTree)
+		outputs, _, _, err := c.flattenTree(ctx, outDirTree)
 		if err != nil {
 			return "", err
 		}
@@ -784,41 +784,41 @@ func (c *Client) getOutputs(ctx context.Context, actionRes *repb.ActionResult) (
 	return res.String(), nil
 }
 
-func (c *Client) getInputTree(ctx context.Context, root *repb.Digest) (string, []string, error) {
+func (c *Client) getInputTree(ctx context.Context, root *repb.Digest) (string, map[string]string, map[string]string, error) {
 	var res bytes.Buffer
 
 	dg, err := digest.NewFromProto(root)
 	if err != nil {
-		return "", nil, err
+		return "", nil, nil, fmt.Errorf("failed generate Digest object from proto: %v", err)
 	}
 	res.WriteString(fmt.Sprintf("[Root directory digest: %v]", dg))
 
 	dirs, err := c.GrpcClient.GetDirectoryTree(ctx, root)
 	if err != nil {
-		return "", nil, err
+		return "", nil, nil, fmt.Errorf("failed to get dir tree: %v", err)
 	}
 	if len(dirs) == 0 {
-		return "", nil, fmt.Errorf("Empty directories returned by GetTree for %v", dg)
+		return "", nil, nil, fmt.Errorf("empty directories returned by GetTree for %v", dg)
 	}
 	t := &repb.Tree{
 		Root:     dirs[0],
 		Children: dirs,
 	}
-	inputs, paths, err := c.flattenTree(ctx, t)
+	inputs, paths, symlinks, err := c.flattenTree(ctx, t)
 	if err != nil {
-		return "", nil, err
+		return "", nil, nil, err
 	}
 	res.WriteString("\n")
 	res.WriteString(inputs)
 
-	return res.String(), paths, nil
+	return res.String(), paths, symlinks, nil
 }
 
-func (c *Client) flattenTree(ctx context.Context, t *repb.Tree) (string, []string, error) {
+func (c *Client) flattenTree(ctx context.Context, t *repb.Tree) (string, map[string]string, map[string]string, error) {
 	var res bytes.Buffer
 	outputs, err := c.GrpcClient.FlattenTree(t, "")
 	if err != nil {
-		return "", nil, err
+		return "", nil, nil, fmt.Errorf("failed falt tree: %v", err)
 	}
 	// Sort the values by path.
 	paths := make([]string, 0, len(outputs))
@@ -830,8 +830,11 @@ func (c *Client) flattenTree(ctx context.Context, t *repb.Tree) (string, []strin
 		paths = append(paths, path)
 	}
 	sort.Strings(paths)
+	pathToDgs := make(map[string]string, len(paths))
+	symToDgs := map[string]string{}
 	for _, path := range paths {
 		output := outputs[path]
+		dg := output.Digest
 		var np string
 		if output.NodeProperties != nil {
 			np = fmt.Sprintf(" [Node properties: %v]", prototext.MarshalOptions{Multiline: false}.Format(output.NodeProperties))
@@ -840,11 +843,18 @@ func (c *Client) flattenTree(ctx context.Context, t *repb.Tree) (string, []strin
 			res.WriteString(fmt.Sprintf("%v: [Directory digest: %v]%s\n", path, output.Digest, np))
 		} else if output.SymlinkTarget != "" {
 			res.WriteString(fmt.Sprintf("%v: [Symlink digest: %v, Symlink Target: %v]%s\n", path, output.Digest, output.SymlinkTarget, np))
+			path = path + "->" + output.SymlinkTarget
+			if o, ok := outputs[output.SymlinkTarget]; ok {
+				dg = o.Digest
+			}
+			symToDgs[path] = fmt.Sprintf("%v", dg)
+			continue
 		} else {
 			res.WriteString(fmt.Sprintf("%v: [File digest: %v]%s\n", path, output.Digest, np))
 		}
+		pathToDgs[path] = fmt.Sprintf("%v", dg)
 	}
-	return res.String(), paths, nil
+	return res.String(), pathToDgs, symToDgs, nil
 }
 
 func (c *Client) getActionResult(ctx context.Context, actionDigest string) (*repb.ActionResult, error) {
@@ -862,3 +872,88 @@ func (c *Client) getActionResult(ctx context.Context, actionDigest string) (*rep
 	}
 	return resPb, nil
 }
+
+// FlatActionIO is a collection of input root Digest, input paths
+// (with value as digests), and output files/dirs (with value as digests) of an
+// action. For a symlink, the key is `<path>-><target>`, the value is the dg of
+// the target file.
+type FlatActionIO struct {
+	RootDg             string
+	InputPaths         map[string]string
+	InputPathSymlinks  map[string]string
+	OutputFiles        map[string]string
+	OutputDirs         map[string]string
+	OutputFileSymlinks map[string]string
+	OutputDirSymlinks  map[string]string
+}
+
+// FlattenActionIO returns the Inputs and Outputs of an action Digest.
+func (c *Client) FlattenActionIO(ctx context.Context, actionDigest string) (*FlatActionIO, error) {
+	acDg, err := digest.NewFromString(actionDigest)
+	if err != nil {
+		return nil, fmt.Errorf("error creating action digest: %w", err)
+	}
+	actionProto := &repb.Action{}
+
+	if _, err := c.GrpcClient.ReadProto(ctx, acDg, actionProto); err != nil {
+		return nil, fmt.Errorf("error reading from proto: %w", err)
+	}
+	rootDg, err := digest.NewFromProto(actionProto.GetInputRootDigest())
+	if err != nil {
+		return nil, fmt.Errorf("error getting input root digest: %w", err)
+	}
+	_, inputPaths, inputSymlinks, err := c.getInputTree(ctx, actionProto.GetInputRootDigest())
+	if err != nil {
+		return nil, err
+	}
+	// If getActionResult failed (say, error in checking action cache:
+	// http://shortn/_dMVjMYPOZE), leave the output as empty, log the warning
+	// message, and proceed with the inputs' information.
+	resPb, err := c.getActionResult(ctx, actionDigest)
+	if err != nil {
+		log.Warningf("Error in getting action result for digest %v: %v\n", actionDigest, err)
+	}
+
+	outputFiles := map[string]string{}
+	outputDirs := map[string]string{}
+	outputFileSymlinks := map[string]string{}
+	outputDirSymlinks := map[string]string{}
+	for _, f := range resPb.GetOutputFiles() {
+		if f != nil {
+			dg, err := digest.NewFromProto(f.GetDigest())
+			if err != nil {
+				log.Errorf("error creating Digest from proto %v: %w", f.GetDigest(), err)
+			}
+			outputFiles[f.GetPath()] = fmt.Sprintf("%v", dg)
+		}
+	}
+	for _, d := range resPb.GetOutputDirectories() {
+		if d != nil {
+			dg, err := digest.NewFromProto(d.GetTreeDigest())
+			if err != nil {
+				log.Errorf("error creating Digest from proto %v: %w", d.GetTreeDigest(), err)
+			}
+			outputDirs[d.GetPath()] = fmt.Sprintf("%v", dg)
+		}
+	}
+	for _, fs := range resPb.GetOutputFileSymlinks() {
+		if fs != nil {
+			outputFileSymlinks[fs.GetPath()+"->"+fs.GetTarget()] = outputFiles[fs.GetTarget()]
+		}
+	}
+	for _, ds := range resPb.GetOutputDirectorySymlinks() {
+		if ds != nil {
+			outputDirSymlinks[ds.GetPath()+"->"+ds.GetTarget()] = outputDirs[ds.GetTarget()]
+		}
+	}
+
+	return &FlatActionIO{
+		RootDg:             fmt.Sprintf("%v", rootDg),
+		InputPaths:         inputPaths,
+		InputPathSymlinks:  inputSymlinks,
+		OutputFiles:        outputFiles,
+		OutputDirs:         outputDirs,
+		OutputFileSymlinks: outputFileSymlinks,
+		OutputDirSymlinks:  outputDirSymlinks,
+	}, nil
+}

go/pkg/tool/tool_test.go

@@ -166,7 +166,7 @@ func TestTool_DownloadAction(t *testing.T) {
 		OutputFiles: []string{"a/b/out"},
 		Platform: &repb.Platform{
 			Properties: []*repb.Platform_Property{
-				&repb.Platform_Property{
+				{
 					Name:  "container-image",
 					Value: "foo",
 				},
@@ -444,3 +444,62 @@ func TestTool_UploadBlob(t *testing.T) {
 		t.Fatalf("Expected 1 write for blob '%v', got %v", dg.String(), cas.BlobWrites(dg))
 	}
 }
+
+func TestTool_FlattenActionIO(t *testing.T) {
+	e, cleanup := fakes.NewTestEnv(t)
+	defer cleanup()
+	cmd := &command.Command{
+		Args:     []string{"tool"},
+		ExecRoot: e.ExecRoot,
+		InputSpec: &command.InputSpec{
+			Inputs:          []string{"foo.c", "bar.c"},
+			SymlinkBehavior: command.PreserveSymlink,
+		},
+	}
+	opt := command.DefaultExecutionOptions()
+	_, acDg, _, _ := e.Set(
+		cmd,
+		opt,
+		&command.Result{Status: command.CacheHitResultStatus},
+		&fakes.InputFile{Path: "foo.c", Contents: "foo"},
+		&fakes.InputSymlink{Path: "bar.c", TargetContent: "bar", Target: "previous_dir/old_target_file"},
+		&fakes.OutputFile{Path: "a/b/out", Contents: "foo"},
+		&fakes.OutputSymlink{Path: "a/b/sl", Target: "a/b/out"},
+	)
+	toolClient := &Client{GrpcClient: e.Client.GrpcClient}
+	tmpDir := t.TempDir()
+	io, err := toolClient.FlattenActionIO(context.Background(), acDg.String())
+	if err != nil {
+		t.Fatalf("DownloadActionResult(%v,%v) failed: %v", acDg.String(), tmpDir, err)
+	}
+
+	getInputRootDg := io.RootDg
+	wantInputRootDg := "ea520b417ef2887249b4ea2d24ff64f5d7d6c419eb1a0ce2067c39ece5c37b56/206"
+	if diff := cmp.Diff(wantInputRootDg, getInputRootDg); diff != "" {
+		t.Errorf("FlattenActionIO returned diff in Inputs' list: (-want +got)\n%s", diff)
+	}
+
+	getInputPaths := io.InputPaths
+	wantInputsPaths := map[string]string{"foo.c": "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae/3",
+		"previous_dir/old_target_file": "fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9/3"}
+	if diff := cmp.Diff(wantInputsPaths, getInputPaths); diff != "" {
+		t.Errorf("FlattenActionIO returned diff in InputPaths: (-want +got)\n%s", diff)
+	}
+	getInputPathSymlinks := io.InputPathSymlinks
+	wantInputPathSymlinks := map[string]string{"bar.c->previous_dir/old_target_file": "fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9/3"}
+	if diff := cmp.Diff(wantInputPathSymlinks, getInputPathSymlinks); diff != "" {
+		t.Errorf("FlattenActionIO returned diff in InputPathSymlinks: (-want +got)\n%s", diff)
+	}
+
+	getOutputFiles := io.OutputFiles
+	getOutputFileSymlinks := io.OutputFileSymlinks
+
+	wantOutputFiles := map[string]string{"a/b/out": "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae/3"}
+	wantOutputFileSymlinks := map[string]string{"a/b/sl->a/b/out": "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae/3"}
+	if diff := cmp.Diff(wantOutputFiles, getOutputFiles); diff != "" {
+		t.Errorf("FlattenActionIO returned diff in Output Files: (-want +got)\n%s", diff)
+	}
+	if diff := cmp.Diff(wantOutputFileSymlinks, getOutputFileSymlinks); diff != "" {
+		t.Errorf("FlattenActionIO returned diff in Output File Symlinks: (-want +got)\n%s", diff)
+	}
+}