Implement smart gen

backend/go/BUILD

@@ -104,6 +104,20 @@ def go_install(name, pkg, version, bin_name):
 def go_mod_gen_addr(pkg):
     return heph.canonicalize(pkg + ":_go_mod_gen")
 
+def _find_go_work():
+    parts = heph.path.split(heph.pkg.dir())
+
+    prefix = "."
+    for i in range(0, len(parts)+1):
+        p = prefix+"/go.work"
+
+        if len(glob(p)) > 0:
+            return p
+
+        prefix += "/.."
+
+    fail("unable to locate work file from "+heph.pkg.dir())
+
 def go_mod(
     mod_pkgs = [],
     cfg = {},
@@ -149,6 +163,11 @@ def go_mod(
 
     godeps_deps = glob("**/*.{mod,sum}") + [src] + mod_pkg_srcs + mod_pkg_modsums
 
+    if work:
+        p = _find_go_work()
+        godeps_deps += [p]
+        godeps_deps += glob(p+".sum")
+
     imports = target(
         name = "_go_mod_gen_imports",
         run = "godeps imports > $OUT",
@@ -159,12 +178,45 @@ def go_mod(
         env = goenv,
     )
 
-    mod_gen_kwargs = {}
-    if work:
-        mod_gen_kwargs = {
-            "sandbox": False,
-            "out_in_sandbox": True,
-        }
+    mod_pkgs_gen = [
+        ":_std_pkgs_*",
+        "go_lib",
+        "go_build_bin",
+        "test",
+        "thirdparty",
+        "go-test",
+        "//thirdparty/go/**:_go_mod_download*"
+    ]
+
+    for pkg in mod_pkgs+['//thirdparty/go']:
+        mod_pkgs_gen += [
+            pkg+"/**:_go_lib*",
+        ]
+
+    # Add labels defined in cfg
+    for pkg in cfg.values():
+        if "test" in pkg:
+            if "run" in pkg["test"]:
+                if "labels" in pkg["test"]["run"]:
+                    labels = pkg["test"]["run"]["labels"]
+                    if type(labels) != "list":
+                        labels = [labels]
+                    mod_pkgs_gen += labels
+
+    mod_pkgs_gen += [
+        heph.pkg.addr()+"/**:_go_lib*",
+        heph.pkg.addr()+"/**:go_bin#build*",
+        heph.pkg.addr()+"/**:_go_test_lib*",
+        heph.pkg.addr()+"/**:_go_xtest_lib*",
+        heph.pkg.addr()+"/**:_go_testmain_lib*",
+        heph.pkg.addr()+"/**:_go_xtestmain_lib*",
+        heph.pkg.addr()+"/**:_go_gen_testmain*",
+        heph.pkg.addr()+"/**:_go_gen_xtestmain*",
+        heph.pkg.addr()+"/**:_go_test#build*",
+        heph.pkg.addr()+"/**:_go_xtest#build*",
+        heph.pkg.addr()+"/**:go_test*",
+        heph.pkg.addr()+"/**:go_xtest*",
+    ]
 
     target(
         name = "_go_mod_gen",
@@ -180,9 +232,8 @@ def go_mod(
             "GOOS": get_os(),
             "GOARCH": get_arch(),
         } | goenv,
-        gen = True,
+        gen = mod_pkgs_gen,
         gen_deps_meta = True,
-        **mod_gen_kwargs,
     )
 
     target(
@@ -218,9 +269,6 @@ def go_mod(
         cache = False,
     )
 
-def go_work():
-    group(name = "_go_work", deps = "go.work")
-
 def _go_gen_importcfg():
     xargs = "xargs " + ("-S" if get_os() == "darwin" else "-s") + " 100000"
 
@@ -261,6 +309,7 @@ def go_mod_download(name, path, version):
         tools = [go],
         out = ["."],
         labels = ["thirdparty"],
+        env = {"GOWORK": "off"},
     )
 
 def go_library(
@@ -279,6 +328,8 @@ def go_library(
     src_dep = None,
     gen_embed = False,
 ):
+    pvt_name = name if name[0] == "_" else "_"+name
+
     p = dir + "/" if dir else ""
 
     if not import_path:
@@ -297,7 +348,7 @@ def go_library(
 
     if len(s_files) > 0:
         abi = target(
-            name = "_" + name + "#abi",
+            name = pvt_name + "#abi",
             deps = src_dep if src_dep else s_files,
             run = [
                 "eval $(go env)",
@@ -316,7 +367,7 @@ def go_library(
             },
         )
         lib = go_library(
-            name = "_" + name + "#lib",
+            name = pvt_name + "#lib",
             import_path = import_path,
             src_dep = src_dep,
             complete = False,
@@ -333,7 +384,7 @@ def go_library(
         asm = []
         for s_file in s_files:
             t = target(
-                name = "_" + name + "#asm_" + heph.path.base(s_file),
+                name = pvt_name + "#asm_" + heph.path.base(s_file),
                 deps = {
                     "lib": lib + "|a",
                     "hdr": lib + "|h",
@@ -407,7 +458,7 @@ def go_library(
         embed_deps["libs"] = [] # remove libs from deps
 
         embed_cfg = target(
-            name = "_" + name + "#embedcfg",
+            name = pvt_name + "#embedcfg",
             run = "godeps embed $SRC_SRC > $OUT",
             deps = embed_deps,
             out = p + pkg_name + ".embedcfg",

bootstrap/config.go

@@ -17,6 +17,7 @@ func BuildConfig(root *hroot.State, profiles []string) (*config.Config, error) {
 	cfg.Engine.GC = true
 	cfg.Engine.CacheHints = true
 	cfg.Engine.ParallelCaching = true
+	cfg.Engine.SmartGen = true
 	cfg.CacheOrder = config.CacheOrderLatency
 	cfg.BuildFiles.Patterns = []string{"**/{BUILD,*.BUILD}"}
 	cfg.Platforms = map[string]config.Platform{

bootstrap/jaeger.go

@@ -38,9 +38,11 @@ func setupJaeger(fins *finalizers.Finalizers, obs *observability.Observability,
 
 	obs.RegisterHook(hook)
 
+	ctx := context.Background()
+
 	fins.Register(func() {
-		_ = pr.ForceFlush(context.Background())
-		_ = pr.Shutdown(context.Background())
+		_ = pr.ForceFlush(ctx)
+		_ = pr.Shutdown(ctx)
 	})
 
 	return nil

bootstrap/rrs.go

@@ -9,25 +9,28 @@ import (
 	"github.com/hephbuild/heph/scheduler"
 	"github.com/hephbuild/heph/specs"
 	"github.com/hephbuild/heph/targetrun"
+	"github.com/hephbuild/heph/utils/ads"
+	"github.com/hephbuild/heph/utils/sets"
 	"github.com/hephbuild/heph/worker/poolwait"
 )
 
 var errHasExprDep = errors.New("has expr, bailing out")
 
-func generateRRs(ctx context.Context, g *graph.State, m specs.Matcher, args []string, bailOutOnExpr bool, opts targetrun.RequestOpts) (targetrun.Requests, error) {
+func generateRRs(ctx context.Context, g *graph.State, m specs.Matcher, args []string, opts targetrun.RequestOpts, bailOutOnExpr bool) (targetrun.Requests, error) {
 	targets, err := g.Targets().Filter(m)
 	if err != nil {
 		return nil, err
 	}
 
-	check := func(target *graph.Target) error {
-		if bailOutOnExpr {
+	check := func(target *graph.Target) error { return nil }
+	if bailOutOnExpr {
+		check = func(target *graph.Target) error {
 			if len(target.Spec().Deps.Exprs) > 0 {
 				return fmt.Errorf("%v: %w", target.Addr, errHasExprDep)
 			}
-		}
 
-		return nil
+			return nil
+		}
 	}
 
 	rrs := make(targetrun.Requests, 0, targets.Len())
@@ -59,56 +62,142 @@ func generateRRs(ctx context.Context, g *graph.State, m specs.Matcher, args []st
 		rrs = append(rrs, rr)
 	}
 
-	ancs, err := g.DAG().GetOrderedAncestors(targets.Slice(), true)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, anc := range ancs {
-		err := check(anc)
+	if bailOutOnExpr {
+		ancs, err := g.DAG().GetOrderedAncestors(targets.Slice(), true)
 		if err != nil {
 			return nil, err
 		}
+
+		for _, anc := range ancs {
+			err := check(anc)
+			if err != nil {
+				return nil, err
+			}
+		}
 	}
 
 	return rrs, nil
 }
 
-func GenerateRRs(ctx context.Context, e *scheduler.Scheduler, m specs.Matcher, targs []string, opts targetrun.RequestOpts, plain bool) (targetrun.Requests, error) {
-	if specs.IsMatcherExplicit(m) {
-		rrs, err := generateRRs(ctx, e.Graph, m, targs, true, opts)
+func RunAllGen(ctx context.Context, e *scheduler.Scheduler, plain bool) error {
+	err := RunGen(ctx, e, plain, func() (func(gent *graph.Target) bool, error) {
+		return func(gent *graph.Target) bool {
+			return true
+		}, nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func RunGen(ctx context.Context, e *scheduler.Scheduler, plain bool, filterFactory func() (func(gent *graph.Target) bool, error)) error {
+	allGenTargets := sets.NewStringSet(0)
+
+	for i := 0; ; i++ {
+		filter, err := filterFactory()
 		if err != nil {
-			if !(errors.Is(err, errHasExprDep) || errors.Is(err, specs.TargetNotFoundErr{})) {
-				return nil, err
+			return err
+		}
+
+		genTargets := ads.Filter(e.Graph.GeneratedTargets(), func(gent *graph.Target) bool {
+			if allGenTargets.Has(gent.Addr) {
+				// Already ran gen
+				return false
 			}
-			log.Debugf("generateRRs: %v", err)
-		} else {
-			return rrs, nil
+
+			return filter(gent)
+		})
+
+		if len(genTargets) == 0 {
+			break
 		}
-	}
 
-	err := runGen(ctx, e, "", false, plain)
-	if err != nil {
-		return nil, err
+		for _, target := range genTargets {
+			allGenTargets.Add(target.Addr)
+			log.Debugf("RG: GEN: %v", target.Addr)
+		}
+
+		// Run those gen targets
+		deps, err := e.ScheduleGenPass(ctx, genTargets)
+		if err != nil {
+			return err
+		}
+
+		err = poolwait.Wait(ctx, fmt.Sprintf("Gen run %v", i), e.Pool, deps, plain)
+		if err != nil {
+			return err
+		}
 	}
 
-	return generateRRs(ctx, e.Graph, m, targs, false, opts)
+	return nil
 }
 
-func runGen(ctx context.Context, e *scheduler.Scheduler, poolName string, linkAll, plain bool) error {
-	deps, err := e.ScheduleGenPass(ctx, linkAll)
-	if err != nil {
-		return err
-	}
+func GenerateRRs(ctx context.Context, e *scheduler.Scheduler, m specs.Matcher, targs []string, opts targetrun.RequestOpts, plain bool) (targetrun.Requests, error) {
+	if !e.Config.Engine.SmartGen {
+		if specs.IsMatcherExplicit(m) {
+			rrs, err := generateRRs(ctx, e.Graph, m, targs, opts, true)
+			if err != nil {
+				if !(errors.Is(err, errHasExprDep) || errors.Is(err, specs.TargetNotFoundErr{})) {
+					return nil, err
+				}
+				log.Debugf("generateRRs: %v", err)
+			} else {
+				return rrs, nil
+			}
 
-	if poolName == "" {
-		poolName = "PreRun gen"
+			for _, target := range e.Graph.Targets().Slice() {
+				target.ResetLinking()
+			}
+		}
 	}
 
-	err = poolwait.Wait(ctx, poolName, e.Pool, deps, plain)
+	err := RunGen(ctx, e, plain, func() (func(gent *graph.Target) bool, error) {
+		if !e.Config.Engine.SmartGen {
+			return func(gent *graph.Target) bool {
+				return true
+			}, nil
+		}
+
+		targets, err := e.Graph.Targets().Filter(m)
+		if err != nil {
+			if !errors.Is(err, specs.TargetNotFoundErr{}) {
+				return nil, err
+			}
+		}
+
+		var requiredMatchers = m
+		if targets != nil {
+			ms, err := e.Graph.RequiredMatchers(targets.Slice())
+			if err != nil {
+				return nil, err
+			}
+
+			requiredMatchers = specs.OrNodeFactory(requiredMatchers, ms)
+		}
+
+		requiredMatchersSimpl := requiredMatchers.Simplify()
+
+		log.Debug("GRR:  M:", requiredMatchers.String())
+		if requiredMatchers.String() != requiredMatchersSimpl.String() {
+			log.Debug("GRR: MS:", requiredMatchersSimpl.String())
+		}
+
+		return func(gent *graph.Target) bool {
+			gm := specs.KindMatcher{}
+			for _, m := range gent.Gen {
+				gm.Add(m)
+			}
+
+			r := specs.Intersects(gm, requiredMatchersSimpl)
+
+			return r.Bool()
+		}, nil
+	})
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	return nil
+	return generateRRs(ctx, e.Graph, m, targs, opts, false)
 }