Merge remote-tracking branch 'origin/master' into 508-templates-alt

crates/spk-build/src/build/binary.rs

@@ -20,7 +20,7 @@ use spk_schema::foundation::option_map::OptionMap;
 use spk_schema::foundation::spec_ops::{ComponentOps, PackageMutOps, PackageOps};
 use spk_schema::foundation::version::VERSION_SEP;
 use spk_schema::ident::{PkgRequest, PreReleasePolicy, RangeIdent, RequestedBy};
-use spk_schema::{ComponentSpecList, DeprecateMut, Ident, Package};
+use spk_schema::{ComponentFileMatchMode, ComponentSpecList, DeprecateMut, Ident, Package};
 use spk_solve::graph::Graph;
 use spk_solve::solution::Solution;
 use spk_solve::{BoxedResolverCallback, DefaultResolver, ResolverCallback, Solver};
@@ -622,6 +622,7 @@ fn split_manifest_by_component(
     manifest: &spfs::tracking::Manifest,
     components: &ComponentSpecList,
 ) -> Result<HashMap<Component, spfs::tracking::Manifest>> {
+    let mut seen = HashSet::new();
     let mut manifests = HashMap::with_capacity(components.len());
     for component in components.iter() {
         let mut component_manifest = spfs::tracking::Manifest::default();
@@ -647,7 +648,10 @@ fn split_manifest_by_component(
                 .files
                 .matches(&node.path.to_path("/"), node.entry.is_dir())
             {
-                relevant_paths.extend(path_and_parents(node.path.to_owned()));
+                let is_new_file = seen.insert(node.path.to_owned());
+                if matches!(component.file_match_mode, ComponentFileMatchMode::All) || is_new_file {
+                    relevant_paths.extend(path_and_parents(node.path.to_owned()));
+                }
             }
         }
         for node in manifest.walk() {

crates/spk-cli/cmd-test/src/test/build.rs

@@ -120,14 +120,14 @@ impl<'a> PackageBuildTester<'a> {
 
         let mut solver = Solver::default();
         solver.set_binary_only(true);
-        for request in self.additional_requirements.drain(..) {
-            solver.add_request(request)
-        }
         solver.update_options(self.options.clone());
         for repo in self.repos.iter().cloned() {
             solver.add_repository(repo);
         }
         solver.configure_for_build_environment(&self.recipe)?;
+        for request in self.additional_requirements.drain(..) {
+            solver.add_request(request)
+        }
         let mut runtime = solver.run();
         let solution = self.build_resolver.solve(&mut runtime).await;
         self.last_solve_graph = runtime.graph();

crates/spk-cli/cmd-test/src/test/install.rs

@@ -92,9 +92,6 @@ impl<'a> PackageInstallTester<'a> {
 
         let mut solver = Solver::default();
         solver.set_binary_only(true);
-        for request in self.additional_requirements.drain(..) {
-            solver.add_request(request)
-        }
         solver.update_options(self.options.clone());
         for repo in self.repos.iter().cloned() {
             solver.add_repository(repo);
@@ -106,6 +103,9 @@ impl<'a> PackageInstallTester<'a> {
             .with_pin(None)
             .with_compat(None);
         solver.add_request(request.into());
+        for request in self.additional_requirements.drain(..) {
+            solver.add_request(request)
+        }
 
         let mut runtime = solver.run();
         let solution = self.env_resolver.solve(&mut runtime).await;

crates/spk-cli/cmd-test/src/test/sources.rs

@@ -96,9 +96,6 @@ impl<'a> PackageSourceTester<'a> {
 
         let mut solver = Solver::default();
         solver.set_binary_only(true);
-        for request in self.additional_requirements.drain(..) {
-            solver.add_request(request)
-        }
         solver.update_options(self.options.clone());
         for repo in self.repos.iter().cloned() {
             solver.add_repository(repo);
@@ -117,6 +114,10 @@ impl<'a> PackageSourceTester<'a> {
             solver.add_request(request.into());
         }
 
+        for request in self.additional_requirements.drain(..) {
+            solver.add_request(request)
+        }
+
         let mut runtime = solver.run();
         let solution = self.env_resolver.solve(&mut runtime).await;
         self.last_solve_graph = runtime.graph();

crates/spk-cli/common/src/flags.rs

@@ -645,6 +645,11 @@ pub struct DecisionFormatterSettings {
     /// displayed in solve stats reports
     #[clap(long, env = "SPK_MAX_FREQUENT_ERRORS", default_value_t = 15)]
     pub max_frequent_errors: usize,
+
+    /// Display a visualization of the solver progress if the solve takes longer
+    /// than a few seconds.
+    #[clap(long)]
+    pub status_bar: bool,
 }
 
 impl DecisionFormatterSettings {
@@ -659,15 +664,26 @@ impl DecisionFormatterSettings {
     /// case some extra configuration might be needed before calling
     /// build.
     pub fn get_formatter_builder(&self, verbosity: u32) -> DecisionFormatterBuilder {
-        DecisionFormatterBuilder::new()
+        let mut builder = DecisionFormatterBuilder::new();
+        builder
             .with_verbosity(verbosity)
             .with_time_and_stats(self.time)
-            .with_verbosity_increase_every(self.increase_verbosity)
+            .with_verbosity_increase_every({
+                // If using the status bar, don't automatically increase
+                // verbosity. The extra verbosity decreases the solver speed
+                // significantly.
+                if self.status_bar {
+                    0
+                } else {
+                    self.increase_verbosity
+                }
+            })
             .with_timeout(self.timeout)
             .with_solution(self.show_solution)
             .with_long_solves_threshold(self.long_solves)
             .with_max_frequent_errors(self.max_frequent_errors)
-            .clone()
+            .with_status_bar(self.status_bar);
+        builder
     }
 }
 

crates/spk-cli/group1/src/cmd_bake.rs

@@ -6,12 +6,13 @@ use std::collections::HashMap;
 
 use clap::Args;
 use futures::TryFutureExt;
-use itertools::Itertools;
 use serde::Serialize;
+use spfs::Digest;
 use spk_cli_common::{current_env, flags, CommandArgs, Error, Result, Run};
 use spk_schema::foundation::spec_ops::PackageOps;
 use spk_schema::ident::RequestedBy;
 use spk_solve::solution::{PackageSource, SolvedRequest};
+use spk_solve::Component;
 
 // Constants for the valid output formats
 const LAYER_FORMAT: &str = "layers";
@@ -59,6 +60,8 @@ struct BakeLayer {
     spfs_layer: String,
     #[serde(default)]
     spk_package: String,
+    #[serde(default)]
+    spk_component: String,
     #[serde(default, skip_serializing_if = "String::is_empty")]
     spk_requester: String,
     #[serde(default, skip_serializing_if = "String::is_empty")]
@@ -67,6 +70,7 @@ struct BakeLayer {
 
 const EMPTY_TAG: &str = "";
 const UNKNOWN_PACKAGE: &str = "";
+const UNKNOWN_COMPONENT: &str = "";
 
 #[async_trait::async_trait]
 impl Run for Bake {
@@ -123,14 +127,17 @@ impl CommandArgs for Bake {
 }
 
 impl Bake {
-    /// Get the spfs layer for a resolved request from it's source
+    /// Get the spfs layers for a resolved request from its source
     /// repo, if possible. This returns a SkipEmbedded error if the
     /// resolved request is an embedded package. These can be skipped
     /// for the purposes of the Bake command. It returns a String
     /// message error if the request is for a src package, which the
     /// Bake command can do nothing with.
-    fn get_spfs_layer(&self, resolved: &SolvedRequest) -> Result<String> {
-        let spfs_layer = match &resolved.source {
+    fn get_spfs_component_layers(
+        &self,
+        resolved: &SolvedRequest,
+    ) -> Result<HashMap<Component, Digest>> {
+        let spfs_layers = match &resolved.source {
             PackageSource::Embedded => {
                 // Embedded builds are provided by another package
                 // in the solve. They don't have a layer of their
@@ -144,24 +151,10 @@ impl Bake {
             PackageSource::Repository {
                 repo: _,
                 components,
-            } => {
-                // Packages published before component support was
-                // added will have 'run:' and 'build:' components that
-                // point to the same layer, so the unique() call is
-                // used to reduce those to a single entry.
-                components
-                    .values()
-                    .map(ToString::to_string)
-                    .unique()
-                    .collect::<Vec<String>>()
-                    .join(", ")
-            }
+            } => components.clone(),
         };
 
-        // TODO: the join(", ") above can turn multiple layers into a
-        // single string blob that probably won't work well for
-        // component supporting spk commands
-        Ok(spfs_layer)
+        Ok(spfs_layers)
     }
 
     /// Get the layers from the active stack. These are digests for
@@ -179,17 +172,32 @@ impl Bake {
         let items = solution.items();
 
         // Get the layer(s) for the packages from their source repos
-        let mut layers_to_packages = HashMap::new();
+        let mut layers_to_packages: HashMap<Digest, (String, String)> = HashMap::new();
         for resolved in items {
-            let spfs_layer = match self.get_spfs_layer(&resolved) {
-                Ok(layer) => layer,
+            let spfs_layers = match self.get_spfs_component_layers(&resolved) {
+                Ok(layers) => layers,
                 Err(Error::SkipEmbedded) => continue,
                 Err(e) => return Err(e.into()),
             };
 
-            // Store in a map so they can be matched up with the
-            // layers in the runtime environment in the next loop.
-            layers_to_packages.insert(spfs_layer, resolved.spec.ident().to_string());
+            // Store in a map keyed by the layer so they can be
+            // matched up with the layers in the runtime environment
+            // in the next loop. The component and package ident need
+            // to be kept together as well.
+            for (component, layer) in spfs_layers.iter() {
+                let mut component_label = format!("{}", component.clone());
+                if layers_to_packages.contains_key(layer) {
+                    // Add the component name to the existing entry
+                    // because this layer provides more than one
+                    // component of the package.
+                    if let Some((_p, c)) = layers_to_packages.get(layer) {
+                        component_label = format!("{},{}", c, component_label);
+                    }
+                }
+
+                layers_to_packages
+                    .insert(*layer, (resolved.spec.ident().to_string(), component_label));
+            }
         }
 
         // Keep the runtime stack order with the first layer at the
@@ -203,23 +211,25 @@ impl Bake {
         // merging for overlay fs mount commands.
         let mut layers: Vec<BakeLayer> = Vec::with_capacity(runtime.status.stack.len());
         for layer in runtime.status.stack.iter() {
-            let spk_package = match layers_to_packages.get(&layer.to_string()) {
-                Some(p) => p.to_string(),
-                None => UNKNOWN_PACKAGE.to_string(),
+            let (spk_package, component) = match layers_to_packages.get(layer) {
+                Some((p, c)) => (p.to_string(), c.clone()),
+                None => (UNKNOWN_PACKAGE.to_string(), UNKNOWN_COMPONENT.to_string()),
             };
 
             // There's no "requested by" or "spfs tag" information in
             // an active runtime, yet.
             // TODO: store this info in an active runtime, from the
             // solve that made it, so it can be properly accessed here.
             let requested_by = RequestedBy::CurrentEnvironment.to_string();
+
             // TODO: need to expose spfs's repository's find_aliases()
             // or find_tags() in spk to get the tag from a digest
             let spfs_tag = EMPTY_TAG.to_string();
 
             layers.push(BakeLayer {
                 spfs_layer: layer.to_string(),
                 spk_package,
+                spk_component: component,
                 spk_requester: requested_by,
                 spfs_tag,
             });
@@ -255,8 +265,8 @@ impl Bake {
 
         let mut stack: Vec<BakeLayer> = Vec::with_capacity(items.len());
         for resolved in items {
-            let spfs_layer = match self.get_spfs_layer(&resolved) {
-                Ok(layer) => layer,
+            let spfs_layers = match self.get_spfs_component_layers(&resolved) {
+                Ok(layers) => layers,
                 Err(Error::SkipEmbedded) => continue,
                 Err(e) => return Err(e.into()),
             };
@@ -274,12 +284,15 @@ impl Bake {
             // find_aliases()/find_tags() in spk to get this from a digest
             let spfs_tag = EMPTY_TAG.to_string();
 
-            stack.push(BakeLayer {
-                spfs_layer,
-                spk_package: resolved.spec.ident().to_string(),
-                spk_requester: requested_by.join(", "),
-                spfs_tag,
-            });
+            for (component, layer) in spfs_layers.iter() {
+                stack.push(BakeLayer {
+                    spfs_layer: layer.to_string(),
+                    spk_package: resolved.spec.ident().to_string(),
+                    spk_component: component.to_string(),
+                    spk_requester: requested_by.join(", "),
+                    spfs_tag: spfs_tag.clone(),
+                });
+            }
         }
         Ok(stack)
     }

crates/spk-schema/src/component_spec.rs

@@ -13,6 +13,17 @@ use crate::Result;
 #[path = "./component_spec_test.rs"]
 mod component_spec_test;
 
+/// Control how files are filtered between components.
+#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
+pub enum ComponentFileMatchMode {
+    /// Matching files are always included.
+    #[default]
+    All,
+    /// Matching files are only included if they haven't already been matched
+    /// by a previously defined component.
+    Remaining,
+}
+
 /// Defines a named package component.
 #[derive(Clone, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
 pub struct ComponentSpec {
@@ -25,6 +36,8 @@ pub struct ComponentSpec {
     pub requirements: super::RequirementsList,
     #[serde(default)]
     pub embedded: super::EmbeddedPackagesList,
+    #[serde(default)]
+    pub file_match_mode: ComponentFileMatchMode,
 }
 
 impl ComponentSpec {
@@ -39,6 +52,7 @@ impl ComponentSpec {
             files: Default::default(),
             requirements: Default::default(),
             embedded: Default::default(),
+            file_match_mode: Default::default(),
         })
     }
 
@@ -51,6 +65,7 @@ impl ComponentSpec {
             files: FileMatcher::all(),
             requirements: Default::default(),
             embedded: Default::default(),
+            file_match_mode: Default::default(),
         }
     }
 
@@ -63,6 +78,7 @@ impl ComponentSpec {
             files: FileMatcher::all(),
             requirements: Default::default(),
             embedded: Default::default(),
+            file_match_mode: Default::default(),
         }
     }
 }

crates/spk-schema/src/component_spec_list.rs

@@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize};
 
 use super::ComponentSpec;
 use crate::foundation::ident_component::Component;
+use crate::ComponentFileMatchMode;
 
 #[cfg(test)]
 #[path = "./component_spec_list_test.rs"]
@@ -136,9 +137,15 @@ impl<'de> Deserialize<'de> for ComponentSpecList {
                     ));
                 }
 
+                let mut using_exclusive_filter_mode = false;
+
                 // all referenced components must have been defined
                 // within the spec as well
                 for component in components.iter() {
+                    if matches!(component.file_match_mode, ComponentFileMatchMode::Remaining) {
+                        using_exclusive_filter_mode = true;
+                    }
+
                     for name in component.uses.iter() {
                         if !seen.contains(name) {
                             return Err(serde::de::Error::custom(format!(
@@ -149,7 +156,12 @@ impl<'de> Deserialize<'de> for ComponentSpecList {
                     }
                 }
 
-                components.sort_by(|a, b| a.name.cmp(&b.name));
+                // when using Exclusive filter mode, the order has meaning and
+                // the components order must be preserved
+                if !using_exclusive_filter_mode {
+                    components.sort_by(|a, b| a.name.cmp(&b.name));
+                }
+
                 Ok(ComponentSpecList(components))
             }
         }