base: Support automatic pulling of missing packages

xbstrap/base.py

@@ -401,6 +401,10 @@ def tool_archives_url(self):
             return None
         return self._root_yml["repositories"].get("tool_archives", None)
 
+    @property
+    def auto_pull(self):
+        return self._site_yml.get("auto_pull", False)
+
     @property
     def use_xbps(self):
         if "pkg_management" not in self._site_yml:
@@ -2996,6 +3000,8 @@ def __init__(self, cfg):
         self._settings = None
         self._sysroots = dict()  # Maps sysroot IDs to TemporaryDirectories
         self.build_scope = None
+        self.use_auto_scope = False
+        self.pull_out_of_scope = False
         self.dry_run = False
         self.check = False
         self.update = False
@@ -3009,6 +3015,10 @@ def __init__(self, cfg):
         self.isolate_sysroots = False
         self.progress_file = None
 
+        if self.cfg.auto_pull:
+            self.use_auto_scope = True
+            self.pull_out_of_scope = True
+
         if cfg.container_runtime == "cbuildrt":
             self.isolate_sysroots = True
 
@@ -3140,7 +3150,12 @@ def add_task_dependencies(s):
 
         elif action == Action.INSTALL_PKG:
             if self.build_scope is not None and subject not in self.build_scope:
-                item.build_edges.add(PlanKey(Action.WANT_PKG, subject))
+                if self.pull_out_of_scope:
+                    if not self._cfg.use_xbps:
+                        raise RuntimeError("Need xbps to pull packages that are out of scope")
+                    item.build_edges.add(PlanKey(Action.PULL_PKG_PACK, subject))
+                else:
+                    item.build_edges.add(PlanKey(Action.WANT_PKG, subject))
             elif self._cfg.use_xbps:
                 item.build_edges.add(PlanKey(Action.PACK_PKG, subject))
             else:
@@ -3341,31 +3356,74 @@ def is_outdated(item, dep_item):
                             item.outdated = True
                             activate(item.key)
 
-    def compute_plan(self, no_ordering=False, no_activation=False):
-        self._do_materialization()
-        if no_ordering:
-            return
-        self._do_ordering()
-        if no_activation:
+    # Automatically restricts the build scope to local packages,
+    # i.e., packages that are explicitly requested and packages that have existing build dirs.
+    def _compute_auto_scope(self):
+        if self.build_scope is not None:
             return
-        self._do_activation()
 
-    def materialized_steps(self):
-        return self._items.keys()
+        scope = set()
 
-    def run_plan(self):
+        # Add all tools/packages that we explicitly want to configure/build.
+        for action, subject in self.wanted:
+            if action in {
+                Action.CONFIGURE_TOOL,
+                Action.COMPILE_TOOL_STAGE,
+                Action.CONFIGURE_PKG,
+                Action.BUILD_PKG,
+                Action.REPRODUCE_BUILD_PKG,
+            }:
+                scope.add(subject)
+
+        # Add all tools that have a build directory.
+        try:
+            tool_dirs = os.listdir(self.cfg.tool_build_dir)
+        except FileNotFoundError:
+            tool_dirs = []
+        tool_dict = {tool.name: tool for tool in self.cfg.all_tools()}
+        for name in tool_dirs:
+            tool = tool_dict.get(name)
+            if tool is not None:
+                scope.add(tool)
+
+        # Add all packages that have a build directory.
+        try:
+            pkg_dirs = os.listdir(self.cfg.pkg_build_dir)
+        except FileNotFoundError:
+            pkg_dirs = []
+        pkg_dict = {pkg.name: pkg for pkg in self.cfg.all_pkgs()}
+        for name in pkg_dirs:
+            pkg = pkg_dict.get(name)
+            if pkg is not None:
+                scope.add(pkg)
+
+        self.build_scope = scope
+
+    def compute_plan(self, no_ordering=False, no_activation=False):
         self._settings = ItemSettings()
         if self.update:
             self._settings.check_remotes = 1
         if self.update and self.paranoid:
             self._settings.check_remotes = 2
         self._settings.reset = self.reset
 
-        # Compute the plan.
+        if self.use_auto_scope:
+            self._compute_auto_scope()
+
         self._do_materialization()
+        if no_ordering:
+            return
         self._do_ordering()
+        if no_activation:
+            return
         self._do_activation()
 
+    def materialized_steps(self):
+        return self._items.keys()
+
+    def run_plan(self):
+        self.compute_plan()
+
         # Run the plan.
         scheduled = [item for item in self._order if item.active]