Local disk CAS

go/pkg/client/BUILD.bazel

@@ -25,6 +25,7 @@ go_library(
         "//go/pkg/command",
         "//go/pkg/contextmd",
         "//go/pkg/digest",
+        "//go/pkg/diskcas",
         "//go/pkg/filemetadata",
         "//go/pkg/io/impath",
         "//go/pkg/io/walker",

go/pkg/client/cas_download.go

@@ -102,6 +102,17 @@ func (c *Client) DownloadOutputs(ctx context.Context, outs map[string]*TreeOutpu
 			symlinks = append(symlinks, out)
 			continue
 		}
+		if c.diskCas != nil {
+			absPath := out.Path
+			if !filepath.IsAbs(absPath) {
+				absPath = filepath.Join(outDir, absPath)
+			}
+			if c.diskCas.Load(out.Digest, absPath) {
+				fullStats.Requested += out.Digest.Size
+				fullStats.Cached += out.Digest.Size
+				continue
+			}
+		}
 		if _, ok := downloads[out.Digest]; ok {
 			copies = append(copies, out)
 			// All copies are effectivelly cached
@@ -130,6 +141,11 @@ func (c *Client) DownloadOutputs(ctx context.Context, outs map[string]*TreeOutpu
 		if err := cache.Update(absPath, md); err != nil {
 			return fullStats, err
 		}
+		if c.diskCas != nil {
+			if err := c.diskCas.Store(output.Digest, absPath); err != nil {
+				return fullStats, err
+			}
+		}
 	}
 	for _, out := range copies {
 		perm := c.RegularMode

go/pkg/client/client.go

@@ -19,6 +19,7 @@ import (
 	"github.com/bazelbuild/remote-apis-sdks/go/pkg/casng"
 	"github.com/bazelbuild/remote-apis-sdks/go/pkg/chunker"
 	"github.com/bazelbuild/remote-apis-sdks/go/pkg/digest"
+	"github.com/bazelbuild/remote-apis-sdks/go/pkg/diskcas"
 	"github.com/bazelbuild/remote-apis-sdks/go/pkg/retry"
 	"github.com/bazelbuild/remote-apis-sdks/go/pkg/uploadinfo"
 	"github.com/pkg/errors"
@@ -188,6 +189,7 @@ type Client struct {
 	uploadOnce          sync.Once
 	downloadOnce        sync.Once
 	useBatchCompression UseBatchCompression
+	diskCas             *diskcas.DiskCas
 }
 
 const (
@@ -333,6 +335,20 @@ func (o *TreeSymlinkOpts) Apply(c *Client) {
 	c.TreeSymlinkOpts = o
 }
 
+type DiskCasOpts struct {
+	Context       context.Context
+	Path          string
+	MaxCapacityGb float64
+}
+
+// Apply sets the client's TreeSymlinkOpts.
+func (o *DiskCasOpts) Apply(c *Client) {
+	if o.Path != "" {
+		capBytes := uint64(o.MaxCapacityGb * 1024 * 1024 * 1024)
+		c.diskCas = diskcas.New(o.Context, o.Path, capBytes)
+	}
+}
+
 // MaxBatchDigests is maximum amount of digests to batch in upload and download operations.
 type MaxBatchDigests int
 

go/pkg/diskcas/BUILD.bazel

@@ -0,0 +1,29 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "diskcas",
+    srcs = [
+        "atim_darwin.go",
+        "atim_linux.go",
+        "atim_windows.go",
+        "diskcas.go",
+    ],
+    importpath = "github.com/bazelbuild/remote-apis-sdks/go/pkg/diskcas",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//go/pkg/digest",
+         "@com_github_golang_glog//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "diskcas_test",
+    srcs = ["diskcas_test.go"],
+    embed = [":diskcas"],
+    deps = [
+        "//go/pkg/digest",
+        "//go/pkg/testutil",
+        "@com_github_pborman_uuid//:go_default_library",
+        "@org_golang_x_sync//errgroup:go_default_library",
+    ],
+)

go/pkg/diskcas/atim_darwin.go

@@ -0,0 +1,16 @@
+// Utility to get the last accessed time on Darwin.
+package diskcas
+
+import (
+	"os"
+	"syscall"
+	"time"
+)
+
+func GetLastAccessTime(path string) (time.Time, error) {
+	info, err := os.Stat(path)
+	if err != nil {
+		return time.Time{}, err
+	}
+	return time.Unix(info.Sys().(*syscall.Stat_t).Atimespec.Unix()), nil
+}

go/pkg/diskcas/atim_linux.go

@@ -0,0 +1,16 @@
+// Utility to get the last accessed time on Linux.
+package diskcas
+
+import (
+	"os"
+	"syscall"
+	"time"
+)
+
+func GetLastAccessTime(path string) (time.Time, error) {
+	info, err := os.Stat(path)
+	if err != nil {
+		return time.Time{}, err
+	}
+	return time.Unix(info.Sys().(*syscall.Stat_t).Atim.Unix()), nil
+}

go/pkg/diskcas/atim_windows.go

@@ -0,0 +1,18 @@
+// Utility to get the last accessed time on Windows.
+package diskcas
+
+import (
+	"os"
+	"syscall"
+	"time"
+)
+
+// This will return correct values only if `fsutil behavior set disablelastaccess 0` is set.
+// Tracking of last access time is disabled by default on Windows.
+func GetLastAccessTime(path string) (time.Time, error) {
+	info, err := os.Stat(path)
+	if err != nil {
+		return time.Time{}, err
+	}
+	return time.Unix(0, info.Sys().(*syscall.Win32FileAttributeData).LastAccessTime.Nanoseconds()), nil
+}