Make cache ro

.github/workflows/release.yml

@@ -175,4 +175,4 @@ jobs:
 
       - name: Cleanup .heph
         run: |
-          rm -rf .heph/cache/test
+          sudo rm -rf .heph/cache/test

backend/go/BUILD

@@ -18,7 +18,7 @@ go_toolchain_gosh = group(
     deps = ["go.sh"],
 )
 
-def go_toolchain(name, version, architectures = []):
+def go_toolchain(name, version, architectures = [], runtime_env = {}, **kwargs):
     run = [
         "./$SRC_INSTALL '{}'".format(version),
         "mv $SRC_GO $OUT_GO",
@@ -52,7 +52,11 @@ def go_toolchain(name, version, architectures = []):
         },
         cache = heph.cache(history = 1),
         support_files = "go",
-        transitive = heph.target_spec(runtime_env = {"GO_OUTDIR": "$(outdir)"}),
+        runtime_env = {"GO_SHARED": "$(shared_stage_dir)"},
+        transitive = heph.target_spec(
+            runtime_env = runtime_env | {"GO_OUTDIR": "$(outdir)", "GO_SHARED": "$(shared_stage_dir)"},
+            **kwargs,
+        ),
     )
 
 godeps = target(
@@ -91,7 +95,7 @@ def go_install(name, pkg, version, bin_name):
         name = name,
         run = [
             "export GOBIN=$(pwd)/gobin",
-            "go install -modcacherw {}@{}".format(pkg, version),
+            "go install {}@{}".format(pkg, version),
         ],
         tools = [go],
         out = "gobin/" + bin_name,
@@ -301,7 +305,7 @@ def go_mod_download(name, path, version):
         name = name,
         run = [
             "echo module heph_ignore > go.mod", # stops go reading the main go.mod, and downloading all of those too
-            "go mod download -modcacherw -json {}@{} | tee mod.json".format(path, version),
+            "go mod download -json {}@{} | tee mod.json".format(path, version),
             "rm go.mod",
             """export MOD_DIR=$(cat mod.json | awk -F\\" '/"Dir": / { print $4 }') && cp -r "$MOD_DIR/." .""",
         ],

backend/go/go.sh

@@ -1,9 +1,8 @@
 #!/bin/bash
 
-export GOPATH=${GOPATH:-$GO_OUTDIR/gopath}
-export GOCACHE=${GOCACHE:-$GO_OUTDIR/gocache}
+export GOPATH=${GOPATH:-$GO_SHARED/path}
+export GOCACHE=${GOCACHE:-$GO_SHARED/cache}
 export GOROOT=$GO_OUTDIR/go
-export GOFLAGS="-modcacherw -buildvcs=false"
 export CGO_ENABLED=${CGO_ENABLED:-0}
 
 set -u

backend/node/BUILD

@@ -56,7 +56,7 @@ def node_toolchain(name, version):
             "ARCH": get_arch(),
         },
         support_files = ["./node"],
-        transitive = heph.target_spec(runtime_env = {"NODE_OUTDIR": "$(outdir)"}),
+        transitive = heph.target_spec(runtime_env = {"NODE_OUTDIR": "$(outdir)", "NODE_SHARED": "$(shared_stage_dir)"}),
     )
 
 def yarn_toolchain(name, version, node):

backend/node/npm.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-export npm_config_cache=$NODE_OUTDIR/nodecache
+export npm_config_cache=$NODE_SHARED/nodecache
 
 exec $NODE_OUTDIR/node/bin/node $NODE_OUTDIR/node/lib/node_modules/npm/bin/npm-cli.js "$@"

backend/node/yarn.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-export YARN_CACHE_FOLDER=${YARN_CACHE_FOLDER:-$YARN_OUTDIR/yarncache}
+export YARN_CACHE_FOLDER=${YARN_CACHE_FOLDER:-$NODE_SHARED/yarncache}
 
 exec $YARN_OUTDIR/yarn/bin/yarn "$@"

bootstrap/bootstrap.go

@@ -274,7 +274,7 @@ func Boot(ctx context.Context, opts BootOpts) (Bootstrap, error) {
 func BootScheduler(ctx context.Context, bs Bootstrap) (*scheduler.Scheduler, error) {
 	fins := &finalizers.Finalizers{}
 
-	localCache, err := lcache.NewState(bs.Root, bs.Pool, bs.Graph.Targets(), bs.Observability, fins, bs.Config.Engine.GC, bs.Config.Engine.ParallelCaching)
+	localCache, err := lcache.NewState(bs.Root, bs.Pool, bs.Graph.Targets(), bs.Observability, fins, bs.Config.Engine.GC, bs.Config.Engine.ParallelCaching, bs.Config.Engine.CacheRW)
 	if err != nil {
 		return nil, err
 	}

bootstrap/functions.go

@@ -65,6 +65,26 @@ func QueryFunctions(
 
 			return ltarget.OutExpansionRoot().Join(t.Package.Path).Abs(), nil
 		},
+		"shared_stage_dir": func(expr exprs.Expr) (string, error) {
+			t, err := getTarget(expr)
+			if err != nil {
+				return "", err
+			}
+
+			universe, err := g.DAG().GetParents(t.Target)
+			if err != nil {
+				return "", err
+			}
+			universe = append(universe, t)
+
+			ltarget := localCache.Metas.Find(t)
+
+			if !graph.Contains(universe, t.Addr) {
+				return "", fmt.Errorf("cannot get shared stage dir of %v", t.Addr)
+			}
+
+			return ltarget.SharedStageRoot().Abs(), nil
+		},
 		"hash_input": func(expr exprs.Expr) (string, error) {
 			t, err := getTarget(expr)
 			if err != nil {

config/config.go

@@ -33,6 +33,7 @@ type Config struct {
 	} `yaml:"cloud"`
 	Engine struct {
 		GC              bool `yaml:"gc"`
+		CacheRW         bool `yaml:"cache_rw"`
 		CacheHints      bool `yaml:"cache_hints"`
 		GitCacheHints   bool `yaml:"git_cache_hints"`
 		InstallTools    bool `yaml:"install_tools"`

config/file_config.go

@@ -16,6 +16,7 @@ type FileConfig struct {
 	} `yaml:"cloud"`
 	Engine struct {
 		GC              *bool `yaml:"gc"`
+		CacheRW         *bool `yaml:"cache_rw"`
 		CacheHints      *bool `yaml:"cache_hints"`
 		GitCacheHints   *bool `yaml:"git_cache_hints"`
 		InstallTools    *bool `yaml:"install_tools"`
@@ -77,6 +78,10 @@ func (fc FileConfig) ApplyTo(c Config) Config {
 		c.Engine.GC = *fc.Engine.GC
 	}
 
+	if fc.Engine.CacheRW != nil {
+		c.Engine.CacheRW = *fc.Engine.CacheRW
+	}
+
 	if fc.Engine.CacheHints != nil {
 		c.Engine.CacheHints = *fc.Engine.CacheHints
 	}

lcache/cache_gc.go

@@ -6,6 +6,7 @@ import (
 	"github.com/hephbuild/heph/graph"
 	"github.com/hephbuild/heph/log/log"
 	"github.com/hephbuild/heph/utils/ads"
+	"github.com/hephbuild/heph/utils/xfs"
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -85,6 +86,8 @@ func (e *LocalCacheState) runGc(targets []*graph.Target, targetDirs []string, fl
 		if !ok {
 			flog("Not part of graph or not cached, delete")
 			if !dryrun {
+				xfs.MakeDirsReadWrite(dir)
+
 				err := os.RemoveAll(dir)
 				if err != nil {
 					log.Error(err)
@@ -135,6 +138,8 @@ func (e *LocalCacheState) runGc(targets []*graph.Target, targetDirs []string, fl
 		if len(entries) == 0 {
 			flog("Nothing left, delete")
 			if !dryrun {
+				xfs.MakeDirsReadWrite(dir)
+
 				err := os.RemoveAll(dir)
 				if err != nil {
 					log.Error(err)
@@ -180,6 +185,8 @@ func (e *LocalCacheState) runGc(targets []*graph.Target, targetDirs []string, fl
 			elog(entry, false)
 
 			if !dryrun {
+				xfs.MakeDirsReadWrite(entry.HashPath)
+
 				err := os.RemoveAll(entry.HashPath)
 				if err != nil {
 					log.Error(err)

lcache/lcache.go

@@ -44,11 +44,12 @@ type LocalCacheState struct {
 	EnableGC        bool
 	ParallelCaching bool
 	Pool            *worker2.Engine
+	CacheRW         bool
 }
 
 const LatestDir = "latest"
 
-func NewState(root *hroot.State, pool *worker2.Engine, targets *graph.Targets, obs *observability.Observability, finalizers *finalizers.Finalizers, gc, parallelCaching bool) (*LocalCacheState, error) {
+func NewState(root *hroot.State, pool *worker2.Engine, targets *graph.Targets, obs *observability.Observability, finalizers *finalizers.Finalizers, gc, parallelCaching, cacheRw bool) (*LocalCacheState, error) {
 	cachePath := root.Home.Join("cache")
 	loc, err := vfssimple.NewLocation("file://" + cachePath.Abs() + "/")
 	if err != nil {
@@ -64,6 +65,7 @@ func NewState(root *hroot.State, pool *worker2.Engine, targets *graph.Targets, o
 		Finalizers:      finalizers,
 		EnableGC:        gc,
 		ParallelCaching: parallelCaching,
+		CacheRW:         cacheRw,
 		Pool:            pool,
 		Metas: NewTargetMetas(func(k targetMetaKey) *Target {
 			gtarget := targets.Find(k.addr)
@@ -80,6 +82,7 @@ func NewState(root *hroot.State, pool *worker2.Engine, targets *graph.Targets, o
 				cacheHashOutput:            &maps.Map[string, string]{},
 				cacheHashInputPathsModtime: nil,
 				expandLock:                 locks.NewFlock(gtarget.Addr+" (expand)", lockPath(root, gtarget, "expand")),
+				sharedStageRoot:            root.Home.Join("shared_stage", gtarget.Package.Path, gtarget.Name),
 			}
 
 			ts := t.Spec()
@@ -402,6 +405,7 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou
 	type OutDirMeta struct {
 		Version int
 		Outputs []string
+		CacheRW bool
 	}
 	version := 1
 
@@ -416,11 +420,13 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou
 		}
 
 		var currentMeta OutDirMeta
+		currentMeta.CacheRW = true // Legacy behavior
 		_ = json.Unmarshal(b, &currentMeta)
 		if currentMeta.Version != version || !ads.ContainsAll(currentMeta.Outputs, outputs) {
 			shouldExpand = true
-		}
-		if currentMeta.Version != version {
+		} else if currentMeta.Version != version {
+			shouldCleanExpand = true
+		} else if currentMeta.CacheRW != e.CacheRW {
 			shouldCleanExpand = true
 		}
 	}
@@ -443,6 +449,8 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou
 			return outDir, err
 		}
 
+		xfs.MakeDirsReadWrite(outDir.Abs())
+
 		untarDedup := sets.NewStringSet(0)
 
 		for _, name := range outputs {
@@ -497,6 +505,10 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou
 		if err != nil {
 			return outDir, fmt.Errorf("write outdir meta: %w", err)
 		}
+
+		if !e.CacheRW {
+			xfs.MakeDirsReadOnly(outDir.Abs())
+		}
 	}
 
 	e.Metas.Find(target).outExpansionRoot = outDir
@@ -536,8 +548,11 @@ func (e *LocalCacheState) Target(ctx context.Context, target graph.Targeter, o T
 }
 
 func (e *LocalCacheState) CleanTarget(target specs.Specer, async bool) error {
-	cacheDir := e.cacheDirForHash(target, "")
-	err := xfs.DeleteDir(cacheDir.Abs(), async)
+	cacheDir := e.cacheDirForHash(target, "").Abs()
+
+	xfs.MakeDirsReadWrite(cacheDir)
+
+	err := xfs.DeleteDir(cacheDir, async)
 	if err != nil {
 		return err
 	}

lcache/target.go

@@ -6,6 +6,7 @@ import (
 	"github.com/hephbuild/heph/utils/locks"
 	"github.com/hephbuild/heph/utils/maps"
 	"github.com/hephbuild/heph/utils/xfs"
+	"os"
 	"sync"
 	"time"
 )
@@ -30,6 +31,7 @@ type Target struct {
 	cacheHashOutputTargetMutex maps.KMutex
 	cacheHashOutput            *maps.Map[string, string]
 	cacheHashInputPathsModtime map[string]time.Time
+	sharedStageRoot            xfs.Path
 }
 
 func (t *Target) String() string {
@@ -71,3 +73,8 @@ func (t *Target) ActualRestoreCacheFiles() xfs.RelPaths {
 
 	return t.actualRestoreCacheFiles
 }
+
+func (t *Target) SharedStageRoot() xfs.Path {
+	_ = os.MkdirAll(t.sharedStageRoot.Abs(), os.ModePerm)
+	return t.sharedStageRoot
+}

targetrun/run.go

@@ -304,6 +304,9 @@ func (e *Runner) Run(ctx context.Context, rr Request, iocfg sandbox.IOConfig, tr
 				}()
 
 				status.Emit(ctx, tgt.TargetStatus(target, "Clearing sandbox..."))
+
+				xfs.MakeDirsReadWrite(rtarget.SandboxRoot.Abs())
+
 				err = xfs.DeleteDir(rtarget.SandboxRoot.Abs(), false)
 				if err != nil {
 					return fmt.Errorf("clear sandbox: %w", err)

test/e2e/lib/cache.go

@@ -14,7 +14,7 @@ func RmCache() error {
 		return err
 	}
 
-	return os.RemoveAll(cache)
+	return RemoveAll(cache)
 }
 
 func TargetCacheRoot(tgt string, elems ...string) (string, error) {

test/e2e/lib/fs.go

@@ -2,7 +2,9 @@ package lib
 
 import (
 	"bytes"
+	"io/fs"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 )
@@ -48,3 +50,19 @@ func ReplaceFile(p, old, new string) error {
 
 	return os.WriteFile(p, nb, os.ModePerm)
 }
+
+// From https://github.com/golang/go/blob/3c72dd513c30df60c0624360e98a77c4ae7ca7c8/src/cmd/go/internal/modfetch/fetch.go
+
+func RemoveAll(dir string) error {
+	// Module cache has 0555 directories; make them writable in order to remove content.
+	filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error {
+		if err != nil {
+			return nil // ignore errors walking in file system
+		}
+		if info.IsDir() {
+			os.Chmod(path, 0777)
+		}
+		return nil
+	})
+	return os.RemoveAll(dir)
+}

test/e2e/lib/setup.go

@@ -14,7 +14,7 @@ func RmSandbox() error {
 		return err
 	}
 
-	return os.RemoveAll(filepath.Join(cache, "sandbox"))
+	return RemoveAll(filepath.Join(cache, "sandbox"))
 }
 
 func PrintConfig() error {

test/e2e/roots/cache-local/e2e_main.go

@@ -2,14 +2,13 @@ package main
 
 import (
 	. "e2e/lib"
-	"os"
 	"path/filepath"
 )
 
 // This is a sanity test that running a target works which has broken cached locally
 func main() {
 	cache := MustV(TempDir())
-	defer os.RemoveAll(cache)
+	defer RemoveAll(cache)
 
 	Must(ReplaceFile(".hephconfig.local", "<URI>", "file://"+cache+"/"))
 
@@ -20,7 +19,7 @@ func main() {
 	Must(ValidateCache("//:hello", []string{""}, false, true, false))
 
 	cacheRoot := MustV(TargetCacheRoot("//:hello"))
-	Must(os.Remove(filepath.Join(cacheRoot, "out_.tar.gz")))
+	Must(RemoveAll(filepath.Join(cacheRoot, "out_.tar.gz")))
 
 	// Test zero cache run
 	Must(Run("//:hello"))

test/e2e/roots/deps/e2e_main.change_hello1.go

@@ -2,15 +2,14 @@ package main
 
 import (
 	. "e2e/lib"
-	"os"
 	"path/filepath"
 )
 
 // This tests that changing when running a dependency A, changing a dependency C of a
 // dependency B does trigger a rerun when the output of B is not the same
 func main() {
 	tmp := MustV(TempDir())
-	defer os.RemoveAll(tmp)
+	defer RemoveAll(tmp)
 
 	Must(ReplaceFile(".hephconfig.local", "<TMP>", tmp))