Merge pull request #5519 from pmengelbert/pmengelbert/add_llb_symlinks/1

Add FileActionSymlink and `llb.Symlink`

client/client_test.go

@@ -222,6 +222,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
 	testLayerLimitOnMounts,
 	testFrontendVerifyPlatforms,
 	testRunValidExitCodes,
+	testFileOpSymlink,
 }
 
 func TestIntegration(t *testing.T) {
@@ -2426,6 +2427,95 @@ func testOCILayoutPlatformSource(t *testing.T, sb integration.Sandbox) {
 	}
 }
 
+func testFileOpSymlink(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+
+	const (
+		fileOwner = 7777
+		fileGroup = 8888
+		linkOwner = 1111
+		linkGroup = 2222
+
+		dummyTimestamp = 42
+	)
+
+	dummyTime := time.Unix(dummyTimestamp, 0)
+
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	st := llb.Scratch().
+		File(llb.Mkdir("/foo", 0700).Mkfile("bar", 0600, []byte("contents"), llb.ChownOpt{
+			User: &llb.UserOpt{
+				UID: fileOwner,
+			},
+			Group: &llb.UserOpt{
+				UID: fileGroup,
+			},
+		})).
+		File(llb.Symlink("bar", "/baz", llb.WithCreatedTime(dummyTime), llb.ChownOpt{
+			User: &llb.UserOpt{
+				UID: linkOwner,
+			},
+			Group: &llb.UserOpt{
+				UID: linkGroup,
+			},
+		}))
+
+	def, err := st.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	destDir := t.TempDir()
+
+	out := filepath.Join(destDir, "out.tar")
+	outW, err := os.Create(out)
+	require.NoError(t, err)
+	defer outW.Close()
+
+	_, err = c.Solve(sb.Context(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:   ExporterTar,
+				Output: fixedWriteCloser(outW),
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := os.ReadFile(out)
+	require.NoError(t, err)
+	m, err := testutil.ReadTarToMap(dt, false)
+	require.NoError(t, err)
+
+	entry, ok := m["bar"]
+	require.True(t, ok)
+
+	dt = entry.Data
+	header := entry.Header
+	require.NoError(t, err)
+
+	require.Equal(t, []byte("contents"), dt)
+	require.Equal(t, fileOwner, header.Uid)
+	require.Equal(t, fileGroup, header.Gid)
+
+	entry, ok = m["baz"]
+	require.Equal(t, true, ok)
+
+	header = entry.Header
+	// ensure it is a symlink
+	require.Equal(t, tar.TypeSymlink, rune(header.Typeflag))
+	// ensure it is a symlink to the proper location
+	require.Equal(t, "bar", header.Linkname)
+
+	// make sure it was chowned properly
+	require.Equal(t, linkOwner, header.Uid)
+	require.Equal(t, linkGroup, header.Gid)
+
+	// ensure it was timestamped properly
+	require.Equal(t, dummyTime, header.ModTime)
+}
+
 func testFileOpRmWildcard(t *testing.T, sb integration.Sandbox) {
 	requiresLinux(t)
 	c, err := New(sb.Context(), sb.Address())

client/llb/fileop.go

@@ -85,6 +85,13 @@ func (fa *FileAction) Mkfile(p string, m os.FileMode, dt []byte, opt ...MkfileOp
 	return a
 }
 
+// Symlink creates a symlink at `newpath` that points to `oldpath`
+func (fa *FileAction) Symlink(oldpath, newpath string, opt ...SymlinkOption) *FileAction {
+	a := Symlink(oldpath, newpath, opt...)
+	a.prev = fa
+	return a
+}
+
 func (fa *FileAction) Rm(p string, opt ...RmOption) *FileAction {
 	a := Rm(p, opt...)
 	a.prev = fa
@@ -193,6 +200,7 @@ type ChownOption interface {
 	MkdirOption
 	MkfileOption
 	CopyOption
+	SymlinkOption
 }
 
 type mkdirOptionFunc func(*MkdirInfo)
@@ -290,6 +298,10 @@ func (co ChownOpt) SetCopyOption(mi *CopyInfo) {
 	mi.ChownOpt = &co
 }
 
+func (co ChownOpt) SetSymlinkOption(si *SymlinkInfo) {
+	si.ChownOpt = &co
+}
+
 func (co *ChownOpt) marshal(base pb.InputIndex) *pb.ChownOpt {
 	if co == nil {
 		return nil
@@ -337,6 +349,57 @@ func Mkfile(p string, m os.FileMode, dt []byte, opts ...MkfileOption) *FileActio
 	}
 }
 
+// SymlinkInfo is the modifiable options used to create symlinks
+type SymlinkInfo struct {
+	ChownOpt    *ChownOpt
+	CreatedTime *time.Time
+}
+
+func (si *SymlinkInfo) SetSymlinkOption(si2 *SymlinkInfo) {
+	*si2 = *si
+}
+
+type SymlinkOption interface {
+	SetSymlinkOption(*SymlinkInfo)
+}
+
+// Symlink creates a symlink at `newpath` that points to `oldpath`
+func Symlink(oldpath, newpath string, opts ...SymlinkOption) *FileAction {
+	var si SymlinkInfo
+	for _, o := range opts {
+		o.SetSymlinkOption(&si)
+	}
+
+	return &FileAction{
+		action: &fileActionSymlink{
+			oldpath: oldpath,
+			newpath: newpath,
+			info:    si,
+		},
+	}
+}
+
+type fileActionSymlink struct {
+	oldpath string
+	newpath string
+	info    SymlinkInfo
+}
+
+func (s *fileActionSymlink) addCaps(f *FileOp) {
+	addCap(&f.constraints, pb.CapFileSymlinkCreate)
+}
+
+func (s *fileActionSymlink) toProtoAction(_ context.Context, _ string, base pb.InputIndex) (pb.IsFileAction, error) {
+	return &pb.FileAction_Symlink{
+		Symlink: &pb.FileActionSymlink{
+			Oldpath:   s.oldpath,
+			Newpath:   s.newpath,
+			Owner:     s.info.ChownOpt.marshal(base),
+			Timestamp: marshalTime(s.info.CreatedTime),
+		},
+	}, nil
+}
+
 type MkfileOption interface {
 	SetMkfileOption(*MkfileInfo)
 }
@@ -606,6 +669,10 @@ func (c CreatedTime) SetMkfileOption(mi *MkfileInfo) {
 	mi.CreatedTime = (*time.Time)(&c)
 }
 
+func (c CreatedTime) SetSymlinkOption(si *SymlinkInfo) {
+	si.CreatedTime = (*time.Time)(&c)
+}
+
 func (c CreatedTime) SetCopyOption(mi *CopyInfo) {
 	mi.CreatedTime = (*time.Time)(&c)
 }

client/llb/fileop_test.go

@@ -179,6 +179,67 @@ func TestFileMkfile(t *testing.T) {
 	require.Equal(t, int64(-1), mkdir.Timestamp)
 }
 
+func TestFileSymlink(t *testing.T) {
+	t.Parallel()
+
+	st := Image("foo").Dir("/src").File(
+		Mkdir("dir", 0o755).
+			Symlink("/src/dir", "/srcdir").
+			Mkfile("/srcdir/file", 0700, []byte("asdfjkl;")).
+			Symlink("dir/file", "/srcdirfile").
+			Mkdir("/src/dir/subdir", 0o755).
+			Symlink("/src/dir/subdir", "/src/dir/subdir/nested"))
+
+	const numOps = 2
+	const numActions = 6
+	def, err := st.Marshal(context.TODO())
+
+	require.NoError(t, err)
+
+	m, arr := parseDef(t, def.Def)
+	require.Equal(t, numOps+1, len(arr))
+
+	dgst, idx := last(t, arr)
+	require.Equal(t, 0, idx)
+	require.Equal(t, m[dgst], arr[numOps-1])
+
+	fileOpNode := arr[1]
+	fileOp := fileOpNode.Op.(*pb.Op_File).File
+	require.Equal(t, numActions, len(fileOp.Actions))
+	require.Equal(t, 1, len(fileOpNode.Inputs))
+	require.Equal(t, m[fileOpNode.Inputs[0].Digest], arr[0])
+	require.Equal(t, 0, int(fileOpNode.Inputs[0].Index))
+
+	symlinkTests := []*pb.FileActionSymlink{
+		nil,
+		{Oldpath: "/src/dir", Newpath: "/srcdir"},
+		nil,
+		{Oldpath: "dir/file", Newpath: "/srcdirfile"},
+		nil,
+		{Oldpath: "/src/dir/subdir", Newpath: "/src/dir/subdir/nested"},
+	}
+
+	for i := 0; i < numActions; i++ {
+		expectedOutput := -1
+		if i == numActions-1 {
+			expectedOutput = 0
+		}
+
+		require.Equal(t, int(fileOp.Actions[i].Input), i)
+		require.Equal(t, -1, int(fileOp.Actions[i].SecondaryInput))
+		require.Equal(t, expectedOutput, int(fileOp.Actions[i].Output))
+
+		if symlinkTests[i] == nil {
+			continue
+		}
+
+		symlink := fileOp.Actions[i].Action.(*pb.FileAction_Symlink).Symlink
+
+		require.Equal(t, symlink.Oldpath, symlinkTests[i].Oldpath)
+		require.Equal(t, symlink.Newpath, symlinkTests[i].Newpath)
+	}
+}
+
 func TestFileRm(t *testing.T) {
 	t.Parallel()
 

cmd/buildctl/debug/dumpllb.go

@@ -130,6 +130,8 @@ func attr(dgst digest.Digest, op *pb.Op) (string, string) {
 				name = fmt.Sprintf("mkdir{path=%s}", act.Mkdir.Path)
 			case *pb.FileAction_Rm:
 				name = fmt.Sprintf("rm{path=%s}", act.Rm.Path)
+			case *pb.FileAction_Symlink:
+				name = fmt.Sprintf("symlink{oldpath=%s, newpath=%s}", act.Symlink.Oldpath, act.Symlink.Newpath)
 			}
 
 			names = append(names, name)

solver/llbsolver/file/backend.go

@@ -67,6 +67,40 @@ func mkdir(d string, action *pb.FileActionMkDir, user *copy.User, idmap *idtools
 	return nil
 }
 
+func symlink(d string, action *pb.FileActionSymlink, user *copy.User, idmap *idtools.IdentityMapping) (err error) {
+	defer func() {
+		var osErr *os.PathError
+		if errors.As(err, &osErr) {
+			// remove system root from error path if present
+			osErr.Path = strings.TrimPrefix(osErr.Path, d)
+		}
+	}()
+
+	newpath, err := fs.RootPath(d, filepath.Join("/", action.Newpath))
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	ch, err := mapUserToChowner(user, idmap)
+	if err != nil {
+		return err
+	}
+
+	if err := os.Symlink(action.Oldpath, newpath); err != nil {
+		return errors.WithStack(err)
+	}
+
+	if err := copy.Chown(newpath, nil, ch); err != nil {
+		return errors.WithStack(err)
+	}
+
+	if err := copy.Utimes(newpath, timestampToTime(action.Timestamp)); err != nil {
+		return errors.WithStack(err)
+	}
+
+	return nil
+}
+
 func mkfile(d string, action *pb.FileActionMkFile, user *copy.User, idmap *idtools.IdentityMapping) (err error) {
 	defer func() {
 		var osErr *os.PathError
@@ -304,6 +338,27 @@ func (fb *Backend) Mkfile(ctx context.Context, m, user, group fileoptypes.Mount,
 	return mkfile(dir, action, u, mnt.m.IdentityMapping())
 }
 
+func (fb *Backend) Symlink(ctx context.Context, m, user, group fileoptypes.Mount, action *pb.FileActionSymlink) error {
+	mnt, ok := m.(*Mount)
+	if !ok {
+		return errors.Errorf("invalid mount type %T", m)
+	}
+
+	lm := snapshot.LocalMounter(mnt.m)
+	dir, err := lm.Mount()
+	if err != nil {
+		return err
+	}
+	defer lm.Unmount()
+
+	u, err := fb.readUserWrapper(action.Owner, user, group)
+	if err != nil {
+		return err
+	}
+
+	return symlink(dir, action, u, mnt.m.IdentityMapping())
+}
+
 func (fb *Backend) Rm(ctx context.Context, m fileoptypes.Mount, action *pb.FileActionRm) error {
 	mnt, ok := m.(*Mount)
 	if !ok {

solver/llbsolver/ops/file.go

@@ -84,6 +84,13 @@ func (f *fileOp) CacheMap(ctx context.Context, g session.Group, index int) (*sol
 			if err != nil {
 				return nil, false, err
 			}
+		case *pb.FileAction_Symlink:
+			p := a.Symlink.CloneVT()
+			markInvalid(action.Input)
+			dt, err = json.Marshal(p)
+			if err != nil {
+				return nil, false, err
+			}
 		case *pb.FileAction_Rm:
 			p := a.Rm.CloneVT()
 			markInvalid(action.Input)
@@ -586,6 +593,14 @@ func (s *FileOpSolver) getInput(ctx context.Context, idx int, inputs []fileoptyp
 			if err := s.b.Mkdir(ctx, inpMount, user, group, a.Mkdir); err != nil {
 				return input{}, err
 			}
+		case *pb.FileAction_Symlink:
+			user, group, err := loadOwner(ctx, a.Symlink.Owner)
+			if err != nil {
+				return input{}, err
+			}
+			if err := s.b.Symlink(ctx, inpMount, user, group, a.Symlink); err != nil {
+				return input{}, err
+			}
 		case *pb.FileAction_Mkfile:
 			user, group, err := loadOwner(ctx, a.Mkfile.Owner)
 			if err != nil {