diff --git a/Cargo.lock b/Cargo.lock index 215d165c5..0c5d5ce8c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -55,6 +55,12 @@ dependencies = [ "backtrace", ] +[[package]] +name = "anymap2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" + [[package]] name = "arrayref" version = "0.3.6" @@ -727,6 +733,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + [[package]] name = "dyn-clone" version = "1.0.6" @@ -1388,6 +1400,16 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "kstring" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3066350882a1cd6d950d055997f379ac37fd39f81cd4d8ed186032eb3c5747" +dependencies = [ + "serde", + "static_assertions", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -1412,6 +1434,59 @@ version = "0.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" +[[package]] +name = "liquid" +version = "0.26.1" +source = "git+https://github.com/rydrman/liquid-rust?branch=allow-nil-filter-entry#d258842cbe2f9a463e566be24a0eb9a21fb7c739" +dependencies = [ + "doc-comment", + "liquid-core", + "liquid-derive", + "liquid-lib", + "serde", +] + +[[package]] +name = "liquid-core" +version = "0.26.1" +source = "git+https://github.com/rydrman/liquid-rust?branch=allow-nil-filter-entry#d258842cbe2f9a463e566be24a0eb9a21fb7c739" +dependencies = [ + "anymap2", + "itertools", + "kstring", + "liquid-derive", + "num-traits", + "pest", + "pest_derive", + "regex", + "serde", + "time 0.3.11", +] + +[[package]] +name = "liquid-derive" +version = "0.26.1" +source = "git+https://github.com/rydrman/liquid-rust?branch=allow-nil-filter-entry#d258842cbe2f9a463e566be24a0eb9a21fb7c739" +dependencies = [ + "proc-macro2", + "proc-quote", + "syn", +] + +[[package]] +name = "liquid-lib" +version = "0.26.1" +source = "git+https://github.com/rydrman/liquid-rust?branch=allow-nil-filter-entry#d258842cbe2f9a463e566be24a0eb9a21fb7c739" +dependencies = [ + "itertools", + "liquid-core", + "once_cell", + "percent-encoding", + "regex", + "time 0.3.11", + "unicode-segmentation", +] + [[package]] name = "lock_api" version = "0.4.9" @@ -1989,6 +2064,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + [[package]] name = "proc-macro2" version = "1.0.52" @@ -1998,6 +2079,30 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-quote" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e84ab161de78c915302ca325a19bee6df272800e2ae1a43fe3ef430bab2a100" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "proc-quote-impl", + "quote", + "syn", +] + +[[package]] +name = "proc-quote-impl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb3ec628b063cdbcf316e06a8b8c1a541d28fa6c0a8eacd2bfb2b7f49e88aa0" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", +] + [[package]] name = "procfs" version = "0.13.2" @@ -3018,6 +3123,7 @@ dependencies = [ "spk-cmd-explain", "spk-cmd-install", "spk-cmd-make-binary", + "spk-cmd-make-recipe", "spk-cmd-make-source", "spk-cmd-render", "spk-cmd-repo", @@ -3277,6 +3383,19 @@ dependencies = [ "tracing", ] +[[package]] +name = "spk-cmd-make-recipe" +version = "0.36.0" +dependencies = [ + "anyhow", + "async-trait", + "clap 3.2.23", + "spk-cli-common", + "spk-schema", + "spk-schema-liquid", + "tracing", +] + [[package]] name = "spk-cmd-make-source" version = "0.36.0" @@ -3402,6 +3521,7 @@ dependencies = [ "spfs", "spk-schema-foundation", "spk-schema-ident", + "spk-schema-liquid", "spk-schema-validators", "sys-info", "tempfile", @@ -3458,6 +3578,22 @@ dependencies = [ "thiserror", ] +[[package]] +name = "spk-schema-liquid" +version = "0.36.0" +dependencies = [ + "format_serde_error", + "lazy_static", + "liquid", + "liquid-core", + "regex", + "rstest", + "serde", + "serde_json", + "spk-schema-foundation", + "tracing", +] + [[package]] name = "spk-schema-validators" version = "0.36.0" @@ -3847,8 +3983,15 @@ dependencies = [ "itoa 1.0.6", "libc", "num_threads", + "time-macros", ] +[[package]] +name = "time-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" + [[package]] name = "tinytemplate" version = "1.2.1" @@ -4243,6 +4386,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" + [[package]] name = "unicode-width" version = "0.1.9" diff --git a/Cargo.toml b/Cargo.toml index 10f78c9b9..f6ac3468a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "crates/spk-config", "crates/spk-launcher", "crates/spk-schema", + "crates/spk-schema/crates/*", "crates/spk-solve", "crates/spk-solve/crates/*", "crates/spk-storage", diff --git a/crates/spk-cli/cmd-make-recipe/Cargo.toml b/crates/spk-cli/cmd-make-recipe/Cargo.toml new file mode 100644 index 000000000..778725fe0 --- /dev/null +++ b/crates/spk-cli/cmd-make-recipe/Cargo.toml @@ -0,0 +1,14 @@ +[package] +authors = ["Ryan Bottriell "] +edition = "2021" +name = "spk-cmd-make-recipe" +version = "0.36.0" + +[dependencies] +anyhow = "1.0" +async-trait = "0.1" +clap = { version = "3.2", features = ["derive", "env"] } +spk-cli-common = { path = '../common' } +spk-schema = { path = '../../spk-schema' } +spk-schema-liquid = { path = '../../spk-schema/crates/liquid' } +tracing = "0.1.35" diff --git a/crates/spk-cli/cmd-make-recipe/src/cmd_make_recipe.rs b/crates/spk-cli/cmd-make-recipe/src/cmd_make_recipe.rs new file mode 100644 index 000000000..7770e2c1b --- /dev/null +++ b/crates/spk-cli/cmd-make-recipe/src/cmd_make_recipe.rs @@ -0,0 +1,76 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use std::sync::Arc; + +use anyhow::{Context, Result}; +use clap::Args; +use spk_cli_common::{flags, CommandArgs, Run}; +use spk_schema::foundation::format::FormatOptionMap; +use spk_schema::foundation::spec_ops::Named; +use spk_schema::{SpecTemplate, Template, TemplateExt}; + +/// Render a package spec template into a recipe +/// +/// This is done automatically when building packages, but can +/// be a useful debugging tool when writing package spec files. +#[derive(Args)] +#[clap(visible_aliases = &["mkrecipe", "mkr"])] +pub struct MakeRecipe { + #[clap(flatten)] + pub options: flags::Options, + + #[clap(short, long, global = true, parse(from_occurrences))] + pub verbose: u32, + + /// The package spec file to render + #[clap(name = "SPEC_FILE")] + pub package: Option, +} + +impl CommandArgs for MakeRecipe { + // The important positional arg for a make-recipe is the package + fn get_positional_args(&self) -> Vec { + match self.package.clone() { + Some(p) => vec![p], + None => vec![], + } + } +} + +#[async_trait::async_trait] +impl Run for MakeRecipe { + async fn run(&mut self) -> Result { + let options = self.options.get_options()?; + + let template = match flags::find_package_template(&self.package)? { + flags::FindPackageTemplateResult::NotFound(name) => { + Arc::new(SpecTemplate::from_file(name.as_ref())?) + } + res => { + let (_, template) = res.must_be_found(); + template + } + }; + + tracing::info!("rendering template for {}", template.name()); + tracing::info!("using options {}", options.format_option_map()); + let data = spk_schema::TemplateData::new(&options); + tracing::debug!("full template data: {data:#?}"); + let rendered = spk_schema_liquid::render_template(template.source(), &data) + .context("Failed to render template")?; + print!("{rendered}"); + + match template.render(&options) { + Err(err) => { + tracing::error!("This template did not render into a valid spec {err}"); + Ok(1) + } + Ok(_) => { + tracing::info!("Successfully rendered a valid spec"); + Ok(0) + } + } + } +} diff --git a/crates/spk-cli/cmd-make-recipe/src/lib.rs b/crates/spk-cli/cmd-make-recipe/src/lib.rs new file mode 100644 index 000000000..50aab51c5 --- /dev/null +++ b/crates/spk-cli/cmd-make-recipe/src/lib.rs @@ -0,0 +1,7 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +#![deny(unsafe_op_in_unsafe_fn)] + +pub mod cmd_make_recipe; diff --git a/crates/spk-cli/common/src/flags.rs b/crates/spk-cli/common/src/flags.rs index 6f021d42c..b94413260 100644 --- a/crates/spk-cli/common/src/flags.rs +++ b/crates/spk-cli/common/src/flags.rs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/imageworks/spk -use std::collections::HashMap; use std::convert::From; use std::str::FromStr; use std::sync::Arc; @@ -14,7 +13,7 @@ use solve::{DecisionFormatter, DecisionFormatterBuilder, MultiSolverKind}; use spk_schema::foundation::format::FormatIdent; use spk_schema::foundation::ident_build::Build; use spk_schema::foundation::ident_component::Component; -use spk_schema::foundation::name::{OptName, OptNameBuf}; +use spk_schema::foundation::name::OptName; use spk_schema::foundation::option_map::{host_options, OptionMap}; use spk_schema::foundation::spec_ops::Named; use spk_schema::foundation::version::CompatRule; @@ -210,9 +209,16 @@ pub struct Options { /// an equals sign or colon (--opt name=value --opt other:value). /// Additionally, many options can be specified at once in yaml /// or json format (--opt '{name: value, other: value}'). + /// + /// Options can also be given in a file via the --options-file/-f flag. If + /// given, --opt will supersede anything in the options file(s). #[clap(long = "opt", short)] pub options: Vec, + /// Specify build/resolve options from a json or yaml file (see --opt/-o) + #[clap(long)] + pub options_file: Vec, + /// Do not add the default options for the current host system #[clap(long)] pub no_host: bool, @@ -225,23 +231,20 @@ impl Options { false => host_options().context("Failed to compute options for current host")?, }; - for req in self.get_var_requests()? { - opts.insert(req.var, req.value); + for filename in self.options_file.iter() { + let reader = std::fs::File::open(filename) + .with_context(|| format!("Failed to open: {filename:?}"))?; + let options: OptionMap = serde_yaml::from_reader(reader) + .with_context(|| format!("Failed to parse as option mapping: {filename:?}"))?; + opts.extend(options); } - Ok(opts) - } - - pub fn get_var_requests(&self) -> Result> { - let mut requests = Vec::with_capacity(self.options.len()); for pair in self.options.iter() { let pair = pair.trim(); if pair.starts_with('{') { - let given: HashMap = serde_yaml::from_str(pair) + let given: OptionMap = serde_yaml::from_str(pair) .context("--opt value looked like yaml, but could not be parsed")?; - for (name, value) in given.into_iter() { - requests.push(VarRequest::new_with_value(name, value)); - } + opts.extend(given); continue; } @@ -253,9 +256,19 @@ impl Options { }) .and_then(|(name, value)| Ok((OptName::new(name)?, value)))?; - requests.push(VarRequest::new_with_value(name, value)); + opts.insert(name.to_owned(), value.to_string()); } - Ok(requests) + + Ok(opts) + } + + pub fn get_var_requests(&self) -> Result> { + Ok(self + .get_options()? + .into_iter() + .filter(|(_name, value)| !value.is_empty()) + .map(|(name, value)| VarRequest::new_with_value(name, value)) + .collect()) } } @@ -332,23 +345,7 @@ impl Requests { S: AsRef, { let mut out = Vec::::new(); - let var_requests = options.get_var_requests()?; - let mut options = match options.no_host { - true => OptionMap::default(), - false => host_options()?, - }; - // Insert var_requests, which includes requests specified on the command-line, - // into the map so that they can override values provided by host_options(). - for req in var_requests { - options.insert(req.var, req.value); - } - - for (name, value) in options.iter() { - if !value.is_empty() { - out.push(VarRequest::new_with_value(name.clone(), value).into()); - } - } - + let options = options.get_options()?; for r in requests.into_iter() { let r = r.as_ref(); if r.contains('@') { diff --git a/crates/spk-cli/common/src/flags_test.rs b/crates/spk-cli/common/src/flags_test.rs index a9a7da88b..3a6a7cad2 100644 --- a/crates/spk-cli/common/src/flags_test.rs +++ b/crates/spk-cli/common/src/flags_test.rs @@ -24,6 +24,7 @@ use spk_schema::foundation::option_map::OptionMap; fn test_option_flags_parsing(#[case] args: &[&str], #[case] expected: &[(&str, &str)]) { let options = super::Options { no_host: true, + options_file: Default::default(), options: args.iter().map(ToString::to_string).collect(), }; let actual = options.get_options().unwrap(); diff --git a/crates/spk-schema/Cargo.toml b/crates/spk-schema/Cargo.toml index c64e84d84..97456ba1b 100644 --- a/crates/spk-schema/Cargo.toml +++ b/crates/spk-schema/Cargo.toml @@ -24,6 +24,7 @@ spfs = { version = '0.34.6', path = "../spfs" } spk-schema-foundation = { path = "./crates/foundation" } spk-schema-ident = { path = "./crates/ident" } spk-schema-validators = { path = "./crates/validators" } +spk-schema-liquid = { path = "./crates/liquid" } sys-info = "0.9.0" tempfile = { workspace = true } thiserror = { workspace = true } diff --git a/crates/spk-schema/crates/foundation/src/option_map/mod.rs b/crates/spk-schema/crates/foundation/src/option_map/mod.rs index ff1664bde..b0a01f5b8 100644 --- a/crates/spk-schema/crates/foundation/src/option_map/mod.rs +++ b/crates/spk-schema/crates/foundation/src/option_map/mod.rs @@ -224,6 +224,38 @@ impl OptionMap { env.remove(&name); } } + + /// Create a yaml mapping from this map, un-flattening all package options. + /// + /// OptionMaps hold package-specific options under dot-notated keys, eg `python.abi`. + /// This function will split those options into sub-objects, creating a two-level + /// mapping instead. In the case where there is a value for both `python` and `python.abi` + /// the former will be dropped. + pub fn to_yaml_value_expanded(&self) -> serde_yaml::Mapping { + use serde_yaml::{Mapping, Value}; + let mut yaml = Mapping::default(); + for (key, value) in self.iter() { + let target = match key.namespace() { + Some(ns) => { + let ns = Value::String(ns.to_string()); + let ns_value = yaml + .entry(ns) + .or_insert_with(|| serde_yaml::Value::Mapping(Default::default())); + if ns_value.as_mapping().is_none() { + *ns_value = serde_yaml::Value::Mapping(Default::default()); + } + ns_value + .as_mapping_mut() + .expect("already validated that this is a mapping") + } + None => &mut yaml, + }; + let key = serde_yaml::Value::String(key.base_name().to_string()); + let value = serde_yaml::Value::String(value.to_string()); + target.insert(key, value); + } + yaml + } } impl<'de> Deserialize<'de> for OptionMap { diff --git a/crates/spk-schema/crates/liquid/Cargo.toml b/crates/spk-schema/crates/liquid/Cargo.toml new file mode 100644 index 000000000..e35388974 --- /dev/null +++ b/crates/spk-schema/crates/liquid/Cargo.toml @@ -0,0 +1,30 @@ +[package] +authors = ["Ryan Bottriell "] +edition = "2021" +name = "spk-schema-liquid" +version = "0.36.0" + +[features] +migration-to-components = ["spk-schema-foundation/migration-to-components"] + +[dependencies] +lazy_static = "1.4" +format_serde_error = {version = "0.3", default_features = false, features = ["colored"]} +regex = "1.6.0" +serde = "1.0" +serde_json = "1.0" +spk-schema-foundation = { path = "../foundation" } +tracing = "0.1.35" + +# this branch is needed until it or a similar solution +# is accepted to solve the broken use of the default filter +# eg: {{ does.not.exit | default: value }} +[dependencies.liquid] +git = "https://github.com/rydrman/liquid-rust" +branch = "allow-nil-filter-entry" +[dependencies.liquid-core] +git = "https://github.com/rydrman/liquid-rust" +branch = "allow-nil-filter-entry" + +[dev-dependencies] +rstest = "0.15.0" diff --git a/crates/spk-schema/crates/liquid/src/error.rs b/crates/spk-schema/crates/liquid/src/error.rs new file mode 100644 index 000000000..e845c6905 --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/error.rs @@ -0,0 +1,52 @@ +use std::error::Error; + +use lazy_static::lazy_static; +use regex::Regex; + +#[cfg(test)] +#[path = "./error_test.rs"] +mod error_test; + +#[derive(Debug)] +struct ParsedError { + message: String, +} + +impl std::fmt::Display for ParsedError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +impl Error for ParsedError {} + +pub fn to_error_types(err: liquid::Error) -> format_serde_error::ErrorTypes { + lazy_static! { + static ref RE: Regex = Regex::new(r"(?ms)liquid: --> (\d+):(\d+)\n.*^\s+= (.*)") + .expect("a valid regular expression"); + } + let mut message = err.to_string(); + let mut line = Option::::None; + let mut column = Option::::None; + if let Some(m) = RE.captures(&message) { + line = m.get(1).and_then(|line| line.as_str().parse().ok()); + column = m + .get(2) + .and_then(|column| column.as_str().parse().ok()) + // format_serde_error appears to use 0-based index for columns + // whereas the liquid crates uses a 1-based index + .map(|col: usize| col - 1); + message = m + .get(3) + .map(|msg| msg.as_str()) + .unwrap_or("Invalid Template") + .trim() + .to_string(); + } + let error = Box::new(ParsedError { message }); + format_serde_error::ErrorTypes::Custom { + error, + line, + column, + } +} diff --git a/crates/spk-schema/crates/liquid/src/error_test.rs b/crates/spk-schema/crates/liquid/src/error_test.rs new file mode 100644 index 000000000..7f02eb296 --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/error_test.rs @@ -0,0 +1,19 @@ +use rstest::rstest; +use serde_json::json; + +#[rstest] +fn test_error_position_extraction() { + // ensure that the source position of an error can be + // properly extracted and used for the returned serde_format_error + + format_serde_error::never_color(); + static TPL: &str = r#"{% default = data | replace ''%}"#; + let err = + crate::render_template(TPL, &json!({})).expect_err("expected template render to fail"); + let expected = r#" + 1 | {% default = data | replace ''%} + | ^ unexpected "="; expected Variable +"#; + let message = err.to_string(); + assert_eq!(message, expected); +} diff --git a/crates/spk-schema/crates/liquid/src/filter_compare_version.rs b/crates/spk-schema/crates/liquid/src/filter_compare_version.rs new file mode 100644 index 000000000..554f3221f --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/filter_compare_version.rs @@ -0,0 +1,73 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use std::str::FromStr; + +use liquid::model::Scalar; +use liquid::ValueView; +use liquid_core::{ + Display_filter, + Expression, + Filter, + FilterParameters, + FilterReflection, + FromFilterParameters, + ParseFilter, + Result, + Runtime, + Value, +}; +use spk_schema_foundation::version::Version; +use spk_schema_foundation::version_range::{Ranged, VersionFilter}; + +#[cfg(test)] +#[path = "./filter_compare_version_test.rs"] +mod filter_compare_version_test; + +#[derive(Debug, FilterParameters)] +struct CompareVersionArgs { + #[parameter(description = "The comparison operation to perform", arg_type = "str")] + operator: Expression, + #[parameter( + description = "The version to compare with, if not part of the operator string", + arg_type = "str" + )] + rhs: Option, +} + +#[derive(Clone, ParseFilter, FilterReflection)] +#[filter( + name = "compare_version", + description = "Compares one version to another using spk ordering semantics", + parameters(CompareVersionArgs), + parsed(CompareVersionFilter) +)] +pub struct CompareVersion; + +#[derive(Debug, FromFilterParameters, Display_filter)] +#[name = "compare_version"] +struct CompareVersionFilter { + #[parameters] + args: CompareVersionArgs, +} + +impl Filter for CompareVersionFilter { + fn evaluate(&self, input: &dyn ValueView, runtime: &dyn Runtime) -> Result { + let args = self.args.evaluate(runtime)?; + let input = input.as_scalar().ok_or_else(|| { + liquid::Error::with_msg("Expected a scalar value for filter 'compare_version'") + })?; + let lhs = Version::from_str(input.into_string().as_str()) + .map_err(|err| liquid::Error::with_msg(err.to_string()))?; + let range_str = match &args.rhs { + None => args.operator.to_kstr().to_string(), + Some(rhs) => format!("{}{}", args.operator, rhs), + }; + let range = VersionFilter::from_str(range_str.as_str()) + .map_err(|err| liquid::Error::with_msg(err.to_string()))?; + + let result = range.is_applicable(&lhs).is_ok(); + Ok(Value::Scalar(Scalar::new(result).to_owned())) + } +} diff --git a/crates/spk-schema/crates/liquid/src/filter_compare_version_test.rs b/crates/spk-schema/crates/liquid/src/filter_compare_version_test.rs new file mode 100644 index 000000000..35c7d794f --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/filter_compare_version_test.rs @@ -0,0 +1,35 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use rstest::rstest; +use serde_json::json; + +#[rstest] +fn test_template_rendering_version_range() { + // the compare_version helper should be useful in if blocks + // in order to render section based on version ranges + + let options = json!({}); + static TPL: &str = r#" +{% default version = "1.2.3" %} +{% assign use_new_download = version | compare_version: ">=1.0" %} +pkg: package/{{ version }} +sources: +{%- if use_new_download %} + - git: https://downloads.testing/package/v{{ version }} +{%- else %} + - git: https://olddownloads.testing/package/v{{ version }} +{%- endif %} +"#; + static EXPECTED: &str = r#" + + +pkg: package/1.2.3 +sources: + - git: https://downloads.testing/package/v1.2.3 +"#; + let rendered = + crate::render_template(TPL, &options).expect("template should not fail to render"); + assert_eq!(rendered, EXPECTED); +} diff --git a/crates/spk-schema/crates/liquid/src/filter_parse_version.rs b/crates/spk-schema/crates/liquid/src/filter_parse_version.rs new file mode 100644 index 000000000..0076b32bd --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/filter_parse_version.rs @@ -0,0 +1,82 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use std::ops::Deref; +use std::str::FromStr; + +use liquid::ValueView; +use liquid_core::runtime::StackFrame; +use liquid_core::{ + Display_filter, + Expression, + Filter, + FilterParameters, + FilterReflection, + FromFilterParameters, + ParseFilter, + Result, + Runtime, + Value, +}; +use spk_schema_foundation::version::Version; + +#[cfg(test)] +#[path = "./filter_parse_version_test.rs"] +mod filter_parse_version_test; + +#[derive(Debug, FilterParameters)] +struct ParseVersionArgs { + #[parameter(description = "An optional sub-component to access", arg_type = "str")] + path: Option, +} + +#[derive(Clone, ParseFilter, FilterReflection)] +#[filter( + name = "parse_version", + description = "Parses an spk version, outputting one or all components", + parameters(ParseVersionArgs), + parsed(ParseVersionFilter) +)] +pub struct ParseVersion; + +#[derive(Debug, FromFilterParameters, Display_filter)] +#[name = "parse_version"] +struct ParseVersionFilter { + #[parameters] + args: ParseVersionArgs, +} + +impl Filter for ParseVersionFilter { + fn evaluate(&self, input: &dyn ValueView, runtime: &dyn Runtime) -> Result { + let input = input.as_scalar().ok_or_else(|| { + liquid::Error::with_msg("Expected a scalar value for filter 'parse_version'") + })?; + let version = Version::from_str(input.into_string().as_str()) + .map_err(|err| liquid::Error::with_msg(err.to_string()))?; + + let data = liquid::object!({ + "major": version.major(), + "minor": version.minor(), + "patch": version.patch(), + "base": version.base(), + "parts": version.parts.parts, + "plus_epsilon": version.parts.plus_epsilon, + "pre": version.pre.deref(), + "post": version.post.deref(), + }); + + if let Some(path) = &self.args.path { + if matches!(path, Expression::Literal(..)) { + return Err(liquid::Error::with_msg( + "parse_version expected a path to evaluate, but found a literal value", + )); + } + let rt = StackFrame::new(runtime, data); + let value = path.evaluate(&rt)?; + Ok(value.into_owned()) + } else { + Ok(data.into()) + } + } +} diff --git a/crates/spk-schema/crates/liquid/src/filter_parse_version_test.rs b/crates/spk-schema/crates/liquid/src/filter_parse_version_test.rs new file mode 100644 index 000000000..38c71983e --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/filter_parse_version_test.rs @@ -0,0 +1,47 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use rstest::rstest; +use serde_json::json; + +#[rstest] +fn test_parse_version_access_basic() { + let options = json!({}); + static TPL: &str = r#"{{ "1.2.3.4.5-beta.1+r.0" | parse_version: minor }}"#; + static EXPECTED: &str = r#"2"#; + let rendered = + crate::render_template(TPL, &options).expect("template should not fail to render"); + assert_eq!(rendered, EXPECTED); +} + +#[rstest] +fn test_parse_version_access_block_params() { + let options = json!({"version": "1.2.3.4.5-beta.1+r.0"}); + static TPL: &str = r#" +{% assign v = version | parse_version %} +{{version}} +{{v.base}} +{{v.major}} +{{v.minor}} +{{v.patch}} +{{v.parts[3]}} +{{v.parts[4]}} +{{v.post.r}} +{{v.pre.beta}} +"#; + static EXPECTED: &str = r#" +1.2.3.4.5-beta.1+r.0 +1.2.3.4.5 +1 +2 +3 +4 +5 +0 +1 +"#; + let rendered = + crate::render_template(TPL, &options).expect("template should not fail to render"); + assert_eq!(rendered.trim(), EXPECTED.trim()); +} diff --git a/crates/spk-schema/crates/liquid/src/filter_replace_regex.rs b/crates/spk-schema/crates/liquid/src/filter_replace_regex.rs new file mode 100644 index 000000000..75b42b5d7 --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/filter_replace_regex.rs @@ -0,0 +1,64 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use liquid::ValueView; +use liquid_core::{ + Display_filter, + Expression, + Filter, + FilterParameters, + FilterReflection, + FromFilterParameters, + ParseFilter, + Result, + Runtime, + Value, +}; + +#[cfg(test)] +#[path = "./filter_replace_regex_test.rs"] +mod filter_replace_regex_test; + +#[derive(Debug, FilterParameters)] +struct ReplaceRegexArgs { + #[parameter(description = "The regular expression to search.", arg_type = "str")] + search: Expression, + #[parameter( + description = "The text to replace search results with. If not given, the filter will just delete search results. Capture groups can be substituted using `$`", + arg_type = "str" + )] + replace: Option, +} + +#[derive(Clone, ParseFilter, FilterReflection)] +#[filter( + name = "replace_re", + description = "Like `replace`, but searches using a regular expression.", + parameters(ReplaceRegexArgs), + parsed(ReplaceRegexFilter) +)] +pub struct ReplaceRegex; + +#[derive(Debug, FromFilterParameters, Display_filter)] +#[name = "replace_re"] +struct ReplaceRegexFilter { + #[parameters] + args: ReplaceRegexArgs, +} + +impl Filter for ReplaceRegexFilter { + fn evaluate(&self, input: &dyn ValueView, runtime: &dyn Runtime) -> Result { + let args = self.args.evaluate(runtime)?; + + let input = input.to_kstr(); + + let search = regex::Regex::new(&args.search) + .map_err(|err| liquid::Error::with_msg(err.to_string()))?; + let replace = args.replace.unwrap_or_else(|| "".into()); + + Ok(Value::scalar( + search.replace_all(&input, replace.as_str()).to_string(), + )) + } +} diff --git a/crates/spk-schema/crates/liquid/src/filter_replace_regex_test.rs b/crates/spk-schema/crates/liquid/src/filter_replace_regex_test.rs new file mode 100644 index 000000000..dc6aabdf2 --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/filter_replace_regex_test.rs @@ -0,0 +1,44 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use rstest::rstest; +use serde_json::json; + +#[rstest] +fn test_replace_regex_basic() { + let options = json!({}); + static TPL: &str = r#"{{ "1992-02-25" | replace_re: "(\d+)-(\d+)-(\d+)", "$3/$2/$1" }}"#; + static EXPECTED: &str = r#"25/02/1992"#; + let rendered = + crate::render_template(TPL, &options).expect("template should not fail to render"); + assert_eq!(rendered, EXPECTED); +} + +#[rstest] +fn test_replace_regex_empty() { + let options = json!({}); + static TPL: &str = r#"{{ "Hello, World!" | replace_re: "[A-Z]" }}"#; + static EXPECTED: &str = r#"ello, orld!"#; + let rendered = + crate::render_template(TPL, &options).expect("template should not fail to render"); + assert_eq!(rendered, EXPECTED); +} + +#[rstest] +fn test_replace_regex_compile_error() { + let options = json!({"version": "1.2.3.4.5-beta.1+r.0"}); + static TPL: &str = r#"{{ "something" | replace_re: "(some]" }}"#; + static EXPECTED_ERR: &str = r#" +liquid: regex parse error: + (some] + ^ +error: unclosed group +from: Filter error + with: + filter=replace_re : "(some]" + input="something" +"#; + let err = crate::render_template(TPL, &options).expect_err("template should fail on bad regex"); + assert_eq!(err.to_string().trim(), EXPECTED_ERR.trim()); +} diff --git a/crates/spk-schema/crates/liquid/src/lib.rs b/crates/spk-schema/crates/liquid/src/lib.rs new file mode 100644 index 000000000..c26ec9cb0 --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/lib.rs @@ -0,0 +1,43 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk +//! Defines the default configuration for processing spec file templates in spk + +mod error; +mod filter_compare_version; +mod filter_parse_version; +mod filter_replace_regex; +mod tag_default; + +pub use format_serde_error::SerdeError as Error; + +/// Build the default template parser for spk +/// +/// This parser has all configuration and extensions +/// needed for rendering spk spec templates. +pub fn default_parser() -> liquid::Parser { + let res = liquid::ParserBuilder::new() + .stdlib() + .tag(tag_default::DefaultTag) + .filter(filter_parse_version::ParseVersion) + .filter(filter_compare_version::CompareVersion) + .filter(filter_replace_regex::ReplaceRegex) + .build(); + debug_assert!(matches!(res, Ok(_)), "default template parser is valid"); + res.unwrap() +} + +/// Render a template with the default configuration +pub fn render_template(tpl: T, data: &D) -> Result +where + T: AsRef, + D: serde::Serialize, +{ + let tpl = tpl.as_ref(); + let map_err = + |err| format_serde_error::SerdeError::new(tpl.to_string(), error::to_error_types(err)); + let parser = default_parser(); + let template = parser.parse(tpl).map_err(map_err)?; + let globals = liquid::to_object(data).map_err(map_err)?; + template.render(&globals).map_err(map_err) +} diff --git a/crates/spk-schema/crates/liquid/src/tag_default.rs b/crates/spk-schema/crates/liquid/src/tag_default.rs new file mode 100644 index 000000000..5d0b713a5 --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/tag_default.rs @@ -0,0 +1,141 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use liquid::model::map::Entry; +use liquid::{Object, ValueView}; +use liquid_core::error::ResultLiquidExt; +use liquid_core::parser::FilterChain; +use liquid_core::runtime::Variable; +use liquid_core::{ + Language, + ParseTag, + Renderable, + Result, + Runtime, + TagReflection, + TagTokenIter, + Value, +}; + +#[cfg(test)] +#[path = "./tag_default_test.rs"] +mod tag_default_test; + +/// Allows for specifying default values for template variables +/// +/// Because we require that all variables to be filled in by default, +/// this allows "optional" template variables to exist by enabling +/// developers to set default values for variables that are not +/// otherwise passed in +#[derive(Clone, Copy)] +pub struct DefaultTag; + +impl TagReflection for DefaultTag { + fn tag(&self) -> &'static str { + "default" + } + + fn description(&self) -> &'static str { + "assign a variable a value if it doesn't already have one" + } +} + +impl ParseTag for DefaultTag { + fn parse( + &self, + mut arguments: TagTokenIter<'_>, + options: &Language, + ) -> Result> { + let dst = arguments + .expect_next("Variable expected.")? + .expect_variable() + .into_result()?; + + arguments + .expect_next("Assignment operator \"=\" expected.")? + .expect_str("=") + .into_result_custom_msg("Assignment operator \"=\" expected.")?; + + let src = arguments + .expect_next("FilterChain expected.")? + .expect_filter_chain(options) + .into_result()?; + + // no more arguments should be supplied, trying to supply them is an error + arguments.expect_nothing()?; + + Ok(Box::new(Default { dst, src })) + } + + fn reflection(&self) -> &dyn TagReflection { + self + } +} + +#[derive(Debug)] +struct Default { + dst: Variable, + src: FilterChain, +} + +impl Default { + fn trace(&self) -> String { + format!("{{% default {} = {}%}}", self.dst, self.src) + } +} + +impl Renderable for Default { + fn render_to(&self, _writer: &mut dyn std::io::Write, runtime: &dyn Runtime) -> Result<()> { + let value = self + .src + .evaluate(runtime) + .trace_with(|| self.trace().into())? + .into_owned(); + + let variable = self.dst.evaluate(runtime)?; + let mut path = variable.iter().collect::>().into_iter(); + let root = path.next().expect("at least one entry in path"); + + let mut current_pos = liquid::model::Path::with_index(root.clone()); + let type_err = |pos: &liquid::model::Path| { + liquid::Error::with_msg("Cannot set default") + .trace("Stepping into non-object") + .context("position", pos.to_string()) + .context("target", self.dst.to_string()) + }; + + let Some(last) = path.next_back() else { + if runtime.get(&[root.clone()]).is_err() { + runtime.set_global(root.to_kstr().into(), value); + } + return Ok(()); + }; + let mut data = runtime + .get(&[root.to_owned()]) + .map(|v| v.into_owned()) + .unwrap_or_else(|_| Value::Object(Object::new())); + let mut data_ref = &mut data; + for step in path { + data_ref = data_ref + .as_object_mut() + .ok_or_else(|| type_err(¤t_pos))? + .entry(step.to_kstr()) + .or_insert_with(|| Value::Object(Object::new())); + current_pos.push(step.to_owned()); + } + match data_ref + .as_object_mut() + .ok_or_else(|| type_err(¤t_pos))? + .entry(last.to_kstr()) + { + Entry::Occupied(_) => {} + Entry::Vacant(v) => { + v.insert(value); + runtime.set_global(root.to_kstr().into(), data); + } + } + + Ok(()) + } +} diff --git a/crates/spk-schema/crates/liquid/src/tag_default_test.rs b/crates/spk-schema/crates/liquid/src/tag_default_test.rs new file mode 100644 index 000000000..d2b910c73 --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/tag_default_test.rs @@ -0,0 +1,111 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use rstest::rstest; +use serde_json::json; + +#[rstest] +fn test_template_rendering_default_value() { + // because we require a value for all variables in the template + // a helper must be provided that allows for defining default values + // as a convenience + + let options = json!({"version": "2.3.4"}); + static TPL: &str = r#" +{% default name = "my-package" %} +{% default version = "ignore-me" %} +pkg: {{ name }}/{{ version }} +sources: + - git: https://downloads.testing/{{ name }}/v{{ version }} +"#; + static EXPECTED: &str = r#" + + +pkg: my-package/2.3.4 +sources: + - git: https://downloads.testing/my-package/v2.3.4 +"#; + let rendered = + crate::render_template(TPL, &options).expect("template should not fail to render"); + assert_eq!(rendered, EXPECTED); +} + +#[rstest] +fn test_template_rendering_defaults_many() { + // ensure that using multiple defaults does not + // cause them to interfere + + let options = json!({}); + static TPL: &str = r#" +{% default name = "my-package" %} +{% default version = "2.3.4" %} +pkg: {{ name }}/{{ version }} +sources: + - git: https://downloads.testing/{{ name }}/v{{ version }} +"#; + static EXPECTED: &str = r#" + + +pkg: my-package/2.3.4 +sources: + - git: https://downloads.testing/my-package/v2.3.4 +"#; + let rendered = + crate::render_template(TPL, &options).expect("template should not fail to render"); + assert_eq!(rendered, EXPECTED); +} + +#[rstest] +fn test_template_rendering_default_nested() { + // ensure that setting a nested default value + // works as expected + + let options = json!({ + "nested": { + "existing": "existing" + } + }); + static TPL: &str = r#" +{% default nested.existing = "ignored" %} +{% default nested.other = "something" %} +pkg: {{ nested.other }}-{{ nested.existing }} +"#; + static EXPECTED: &str = r#" + + +pkg: something-existing +"#; + let rendered = match crate::render_template(TPL, &options) { + Ok(r) => r, + Err(err) => { + println!("{err}"); + panic!("template should not fail to render"); + } + }; + assert_eq!(rendered, EXPECTED); +} + +#[rstest] +fn test_template_rendering_default_nested_not_object() { + // ensure that setting a nested default value + // works as expected + + let options = json!({ + "integer": 64, + }); + static TPL: &str = r#" +{% default integer.nested = "invalid" %} +"#; + let err = crate::render_template(TPL, &options) + .expect_err("Should fail when setting default under non-object"); + let expected = r#"liquid: Cannot set default +from: Stepping into non-object + with: + position=integer + target=integer["nested"] + +"#; + let message = err.to_string(); + assert_eq!(message, expected); +} diff --git a/crates/spk-schema/src/error.rs b/crates/spk-schema/src/error.rs index 14f6235ac..65dcec904 100644 --- a/crates/spk-schema/src/error.rs +++ b/crates/spk-schema/src/error.rs @@ -50,6 +50,8 @@ pub enum Error { #[error(transparent)] InvalidYaml(#[from] format_serde_error::SerdeError), + #[error(transparent)] + InvalidTemplate(format_serde_error::SerdeError), } impl Error { diff --git a/crates/spk-schema/src/lib.rs b/crates/spk-schema/src/lib.rs index 67d87fbb9..aef4a8c64 100644 --- a/crates/spk-schema/src/lib.rs +++ b/crates/spk-schema/src/lib.rs @@ -56,7 +56,7 @@ pub use spk_schema_foundation::{ FromYaml, }; pub use spk_schema_ident::{self as ident, AnyIdent, BuildIdent, Request, VersionIdent}; -pub use template::{Template, TemplateExt}; +pub use template::{Template, TemplateData, TemplateExt}; pub use test_spec::TestStage; pub use validation::{default_validators, ValidationSpec, Validator}; pub use variant::{Variant, VariantExt}; diff --git a/crates/spk-schema/src/option_test.rs b/crates/spk-schema/src/option_test.rs index 7dc7ff5c3..0b915d80e 100644 --- a/crates/spk-schema/src/option_test.rs +++ b/crates/spk-schema/src/option_test.rs @@ -41,7 +41,7 @@ fn test_var_opt_validation(#[case] spec: &str, #[case] value: &str, #[case] expe #[case("{var: my-var}", None)] #[case("{var: my-var/}", None)] // empty is mapped to none #[case("{static: static, var: my-var}", Some("static"))] // static instead of default -#[case("{static: static, var: my-var/default}", Some("static"))] // static supercedes default +#[case("{static: static, var: my-var/default}", Some("static"))] // static supersedes default fn test_var_opt_parse_value(#[case] spec: &str, #[case] expected: Option<&str>) { let opt = Opt::from_yaml(spec).unwrap().into_var().unwrap(); let actual = opt.get_value(None); diff --git a/crates/spk-schema/src/spec.rs b/crates/spk-schema/src/spec.rs index e2d6c30d2..7b2182fc5 100644 --- a/crates/spk-schema/src/spec.rs +++ b/crates/spk-schema/src/spec.rs @@ -107,6 +107,13 @@ pub struct SpecTemplate { template: String, } +impl SpecTemplate { + /// The complete source string for this template + pub fn source(&self) -> &str { + &self.template + } +} + impl Named for SpecTemplate { fn name(&self) -> &PkgName { &self.name @@ -120,8 +127,11 @@ impl Template for SpecTemplate { &self.file_path } - fn render(&self, _options: &OptionMap) -> Result { - Ok(SpecRecipe::from_yaml(&self.template)?) + fn render(&self, options: &OptionMap) -> Result { + let data = super::TemplateData::new(options); + let rendered = spk_schema_liquid::render_template(&self.template, &data) + .map_err(Error::InvalidTemplate)?; + Ok(SpecRecipe::from_yaml(rendered)?) } } diff --git a/crates/spk-schema/src/spec_test.rs b/crates/spk-schema/src/spec_test.rs index 0c4ae5b10..4ff4df60c 100644 --- a/crates/spk-schema/src/spec_test.rs +++ b/crates/spk-schema/src/spec_test.rs @@ -3,11 +3,14 @@ // https://github.com/imageworks/spk use rstest::rstest; +use spk_schema_foundation::name::PkgName; use spk_schema_foundation::option_map; use spk_schema_foundation::option_map::OptionMap; +use spk_schema_foundation::spec_ops::HasVersion; +use super::SpecTemplate; use crate::prelude::*; -use crate::recipe; +use crate::{recipe, Template}; #[rstest] fn test_resolve_options_empty_options() { @@ -304,3 +307,52 @@ fn test_get_build_requirements_pkg_in_variant_preserves_order() { index = Some(3) ); } + +#[rstest] +fn test_template_error_message() { + format_serde_error::never_color(); + static SPEC: &str = r#"pkg: my-package/{{ opt.version }} +sources: + - git: https://downloads.testing/my-package/v{{ opt.typo }} +"#; + let tpl = SpecTemplate { + name: PkgName::new("my-package").unwrap().to_owned(), + file_path: "my-package.spk.yaml".into(), + template: SPEC.to_string(), + }; + let options = option_map! {"version" => "1.0.0"}; + let err = tpl + .render(&options) + .expect_err("expect template rendering to fail"); + let expected = r#" +liquid: Unknown index + with: + variable=opt + requested index=typo + available indexes=version +"#; + let message = err.to_string(); + assert_eq!(message.trim(), expected.trim()); +} + +#[rstest] +fn test_template_namespace_options() { + // options can have namespace values separated by a `.` + // which can be annoying to access when the actual key + // still has a `.` in it, so validate that these values + // can instead be accessed as a sub-object using the + // dot-notation built into templating + + format_serde_error::never_color(); + static SPEC: &str = r#"pkg: mypackage/{{ opt.namespace.version }}"#; + let tpl = SpecTemplate { + name: PkgName::new("my-package").unwrap().to_owned(), + file_path: "my-package.spk.yaml".into(), + template: SPEC.to_string(), + }; + let options = option_map! {"namespace.version" => "1.0.0"}; + let recipe = tpl + .render(&options) + .expect("template should render with sub-object access"); + assert_eq!(recipe.version().to_string(), "1.0.0"); +} diff --git a/crates/spk-schema/src/template.rs b/crates/spk-schema/src/template.rs index 4eb967eaa..f8ac2956f 100644 --- a/crates/spk-schema/src/template.rs +++ b/crates/spk-schema/src/template.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/imageworks/spk +use std::collections::HashMap; use std::path::Path; use crate::foundation::option_map::OptionMap; @@ -24,3 +25,44 @@ pub trait TemplateExt: Template { /// Load this template from a file on disk fn from_file(path: &Path) -> Result; } + +/// The structured data that should be made available +/// when rendering spk templates into recipes +#[derive(serde::Serialize, Debug, Clone)] +pub struct TemplateData { + /// Information about the release of spk being used + spk: SpkInfo, + /// The option values for this template, expanded + /// from an option map so that namespaced options + /// like `python.abi` actual live under the `python` + /// field rather than as a field with a '.' in the name + opt: serde_yaml::Mapping, + /// Environment variable data for the current process + env: HashMap, +} + +/// The structured data that should be made available +/// when rendering spk templates into recipes +#[derive(serde::Serialize, Debug, Clone)] +struct SpkInfo { + version: &'static str, +} + +impl Default for SpkInfo { + fn default() -> Self { + Self { + version: env!("CARGO_PKG_VERSION"), + } + } +} + +impl TemplateData { + /// Create the set of templating data for the current process and options + pub fn new(options: &OptionMap) -> Self { + TemplateData { + spk: SpkInfo::default(), + opt: options.to_yaml_value_expanded(), + env: std::env::vars().collect(), + } + } +} diff --git a/crates/spk/Cargo.toml b/crates/spk/Cargo.toml index e1f16029f..53ec7e2b9 100644 --- a/crates/spk/Cargo.toml +++ b/crates/spk/Cargo.toml @@ -34,6 +34,7 @@ cli = [ "dep:spk-cmd-install", "dep:spk-cmd-make-binary", "dep:spk-cmd-make-source", + "dep:spk-cmd-make-recipe", "dep:spk-cmd-render", "dep:spk-cmd-repo", "dep:spk-cmd-test", @@ -61,6 +62,7 @@ spk-cmd-explain = { path = '../spk-cli/cmd-explain', optional = true } spk-cmd-install = { path = '../spk-cli/cmd-install', optional = true } spk-cmd-make-binary = { path = '../spk-cli/cmd-make-binary', optional = true } spk-cmd-make-source = { path = '../spk-cli/cmd-make-source', optional = true } +spk-cmd-make-recipe = { path = '../spk-cli/cmd-make-recipe', optional = true } spk-cmd-render = { path = '../spk-cli/cmd-render', optional = true } spk-cmd-repo = { path = '../spk-cli/cmd-repo', optional = true } spk-cmd-test = { path = '../spk-cli/cmd-test', optional = true } diff --git a/crates/spk/src/cli.rs b/crates/spk/src/cli.rs index 2412d5c23..16f896452 100644 --- a/crates/spk/src/cli.rs +++ b/crates/spk/src/cli.rs @@ -24,6 +24,7 @@ use spk_cmd_env::cmd_env; use spk_cmd_explain::cmd_explain; use spk_cmd_install::cmd_install; use spk_cmd_make_binary::cmd_make_binary; +use spk_cmd_make_recipe::cmd_make_recipe; use spk_cmd_make_source::cmd_make_source; use spk_cmd_render::cmd_render; use spk_cmd_repo::cmd_repo; @@ -156,6 +157,7 @@ pub enum Command { Ls(cmd_ls::Ls), MakeBinary(cmd_make_binary::MakeBinary), MakeSource(cmd_make_source::MakeSource), + MakeRecipe(cmd_make_recipe::MakeRecipe), New(cmd_new::New), #[clap(alias = "variant-count", hide = true)] NumVariants(cmd_num_variants::NumVariants), @@ -191,6 +193,7 @@ impl Run for Command { Command::Ls(cmd) => cmd.run().await, Command::MakeBinary(cmd) => cmd.run().await, Command::MakeSource(cmd) => cmd.run().await, + Command::MakeRecipe(cmd) => cmd.run().await, Command::New(cmd) => cmd.run().await, Command::NumVariants(cmd) => cmd.run().await, Command::Publish(cmd) => cmd.run().await, @@ -223,6 +226,7 @@ impl CommandArgs for Command { Command::Ls(cmd) => cmd.get_positional_args(), Command::MakeBinary(cmd) => cmd.get_positional_args(), Command::MakeSource(cmd) => cmd.get_positional_args(), + Command::MakeRecipe(cmd) => cmd.get_positional_args(), Command::New(cmd) => cmd.get_positional_args(), Command::NumVariants(cmd) => cmd.get_positional_args(), Command::Publish(cmd) => cmd.get_positional_args(), diff --git a/docs/usage.md b/docs/usage.md deleted file mode 100644 index e0e774f92..000000000 --- a/docs/usage.md +++ /dev/null @@ -1,100 +0,0 @@ -# Advanced Usage - -Additional command line workflows for more advanced users. - -## Tag Streams - -When tags are created in spfs, they are added to what is known as a _tag stream_. Tag streams are simply a historical record of that tag over time, keeping track of each change, when it was made, and by whom. Previous versions of a tag can be referenced using a tilde, where the most recent version of a tag is version `~0`; the previous version is version `~1`; the version before that is version `~2` etc... This notation can be used everywhere a tag can be used. This means that `spfs run my-tag` is the same as `spfs run my-tag~0`. All the available versions of a tag can be viewed using the `spfs log ` command, where you will see this notation used. - -```bash -spfs shell - -echo 1 > /spfs/message.txt -spfs commit layer --tag my-layer - -spfs edit -echo 2 > /spfs/message.txt -spfs commit layer --tag my-layer - -spfs edit -echo 3 > /spfs/message.txt -spfs commit layer --tag my-layer - -spfs log my-layer -# 6E5CA5XL3L my-layer rbottriell@wolf0254.spimageworks.com 2020-03-18 10:12 -# 6E5CA5XL3L my-layer~1 rbottriell@wolf0254.spimageworks.com 2020-03-18 10:11 -# XHHVG3NDGE my-layer~2 rbottriell@wolf0254.spimageworks.com 2020-03-18 10:11 -``` - -### Reverting a Tag - -Using a tag stream, we can revert to previous versions of a tag by simply re-tagging the older version as the latest one. To continue from the example above: - -```bash -spfs tag my-layer~2 my-layer - -spfs log my-layer -# XHHVG3NDGE my-layer rbottriell@wolf0254.spimageworks.com 2020-03-18 10:16 -# 6E5CA5XL3L my-layer~1 rbottriell@wolf0254.spimageworks.com 2020-03-18 10:12 -# JJ3MEJOYQ2 my-layer~2 rbottriell@wolf0254.spimageworks.com 2020-03-18 10:11 -# XHHVG3NDGE my-layer~3 rbottriell@wolf0254.spimageworks.com 2020-03-18 10:11 -``` - -:point_right: | If you want to see or update shared tags, remember to specify the remote repository for each command (eg: `spfs log my-layer -r origin`) -:---: | :--- - -## Diff Tool - -Any two spfs file system states can be compared using the `spfs diff` command. With no arguments, this command works much like the `git status` command, showing the current set of active changes that have not been committed (if you are in an spfs runtime). - -## - -It's easy enough to pull and mount an spfs file tree, but sometimes it's not ideal to have to localize or sync the entire thing just to get a little bit of information or check the contents of a key file. SpFS provides 2 commands which allow for easy introspection of committed data without the need to enter into the environment itself. - -- `spfs ls` can be used to list directory contents of a stored file tree -- `spfs cat` can be used to output the contents of a file stored in spfs - -```bash -spfs shell -mkdir -p /spfs/bin -echo "I am root" > /spfs/root.txt -touch /spfs/bin/command -spfs commit layer --tag simple-fs -# exit the spfs runtime -exit - -spfs info simple-fs -# layer: -# refs: YJGTUV2Y -> simple-fs -# manifest: EDAWAZUS - -spfs ls simple-fs -# bin -# root.txt - -spfs ls simple-fs bin -# command - -spfs cat simple-fs root.txt -# I am root -``` - -## Repository Cleaning - -Over time, an spfs repository can get quite large, as it retains data from long ago that may not be used anymore as well as containing data for committed platforms, layers and blobs that are not referenced in any tag. The `spfs clean` command can be used to remove such data, as well as to find and remove old data for past tag history which is no longer desired. By default, the clean command will only find and print information about things that would be removed, and must be explicitly told to delete data. - -:warning: | These commands can and will remove data, and should be used with great caution. -:---: | :--- - -```bash -spfs clean --help -``` - -Objects are considered to be attached, and unremovable if they are reachable from any version of any tag in the repository. The `--prune` flag and related options can be used to get rid of older tag versions based on age or number of versions before cleaning the repository. This is a good way to try and disconnect additional objects, create more data that can be cleaned. - -:point_right: | The pruning process will always prefer keeping a tag version over removing it when multiple keep/prune conditions apply to it. Check the default values for each setting if you expected more tags than were shown. -:---: | :--- - -## Temporary Filesystem Size - -The spfs runtime uses a temporary, in-memory filesystem, which means that large sets of changes can run out of space because of RAM limitations. The size of this filesystem can be overridden using the `SPFS_FILESYSTEM_TMPFS_SIZE` variable (eg `SPFS_FILESYSTEM_TMPFS_SIZE=10G`). Note that specifying values close to or larger than the available memory on the system may cause deadlocks or system instability. diff --git a/docs/use/spec.md b/docs/use/spec.md index 8e3c60af7..a8f56897c 100644 --- a/docs/use/spec.md +++ b/docs/use/spec.md @@ -415,3 +415,92 @@ tests: script: - pytest ``` + +### Spec File Templating + +SPK package spec files also support the [liquid](https://shopify.github.io/liquid/) templating language, so long as the spec file remains valid yaml. + +The templating is rendered when the yaml file is read from disk, and before it's processed any further (to start a build, run tests, etc.). This means that it cannot, for example, be involved in rendering different specs for different variants of the package (unless you define and orchestrate those variants through a separate build system). + +The data that's made available to the template takes the form: + +```yaml +spk: + version: "0.23.0" # the version of spk being used +opt: {} # a map of all build options specified (either host options or at the command line) +env: {} # a map of the current environment variables from the caller +``` + +One common templating use case is to allow your package spec to be reused to build many different versions, for example: + +```yaml +# {% default opt.version = "2.3.4" %} +pkg: my-package/{{ version }} +``` + +Which could then be invoked for different versions at the command line: + +```sh +spk build my-package.spk.yaml # builds the default 2.3.4 +spk build my-package.spk.yaml -o version=2.4.0 # builds 2.4.0 +``` + +#### Template Extensions + +In addition to the default tags and filters within the liquid language, spk provides a few additional ones to help package maintainers: + +##### Tags + +**default** + +The `default` tag can be used to more easily declare the default value for a variable. The following two statements are equivalent: + +```liquid +{% assign var = var | default: "2.3.4" %} +{% default var = "2.3.4" %} +``` + +Additionally, this tag can be used to set defaults in nested structures. Often, for options that may be provided at the command line. + +```liquid +{% default opt.version = "2.3.4" %} +``` + +##### Filters + +**compare_version** + +The `compare_version` allows for comparing spk versions using any of the [version comparison operators](/use/versioning). It takes one or two arguments, depending on the data that you have to give. In all cases, the arguments are concatenated together and parsed as a version range. For example, the following assignments to py_3 all end up checking the same statement. + +```liquid +{% assign is_py3 = python.version | compare_version: ">=3" %} +{% assign is_py3 = python.version | compare_version: ">=", 3 %} +{% assign three = 3 %} +{% assign is_py3 = python.version | compare_version: ">=", three %} +``` + +**parse_version** + +The `parse_version` filter breaks down an spk version into its components, either returning an object or a single field from it, for example: + +```liquid +{% assign v = "1.2.3.4-alpha.0+r.4" | parse_version %} +{{ v.base }} # 1.2.3.4 +{{ v.major }} # 1 +{{ v.minor }} # 2 +{{ v.patch }} # 3 +{{ v.parts[3] }} # 4 +{{ v.post.r }} # 4 +{{ v.pre.alpha }} # 0 +{{ "1.2.3.4-alpha.0+r.4" | parse_version: minor }} # 2 +``` + +**replace_re** + +The `replace_re` filter works like the built-in `replace` filter, except that it matches using a perl-style regular expression and allows group replacement in the output. These regular expressions do not support look-arounds or back-references. For example: + +```liquid +{% assign version = opt.version | default: "2.3.4" %} +{% assign major_minor = version | replace_re: "(\d+)\.(\d+).*", "$1.$2" %} +{{ major_minor }} # 2.3 +``` diff --git a/packages/cmake/cmake.spk.yaml b/packages/cmake/cmake.spk.yaml index 88952daaf..0f412c8d9 100644 --- a/packages/cmake/cmake.spk.yaml +++ b/packages/cmake/cmake.spk.yaml @@ -1,4 +1,5 @@ -pkg: cmake/3.18.2 +# {% default version = "3.18.2" %} +pkg: cmake/{{ version }} api: v0/package sources: @@ -7,8 +8,8 @@ sources: # the tar file. - path: ./ - script: - - export TARFILE=cmake-3.18.2-Linux-x86_64.tar.gz - - if [ ! -e ./$TARFILE ] ; then wget https://github.com/Kitware/CMake/releases/download/v3.18.2/$TARFILE ; fi + - export TARFILE=cmake-{{ version }}-Linux-x86_64.tar.gz + - if [ ! -e ./$TARFILE ] ; then wget https://github.com/Kitware/CMake/releases/download/v{{ version }}/$TARFILE ; fi build: options: @@ -17,7 +18,7 @@ build: script: - mkdir -p build; cd build - tar -xvf - ../cmake-3.18.2-Linux-x86_64.tar.gz + ../cmake-{{ version }}-Linux-x86_64.tar.gz --strip-components=1 --exclude=doc --exclude=Help diff --git a/packages/gnu/gcc/gcc48.spk.yaml b/packages/gnu/gcc/gcc48.spk.yaml index a31b5d836..21fcd0764 100644 --- a/packages/gnu/gcc/gcc48.spk.yaml +++ b/packages/gnu/gcc/gcc48.spk.yaml @@ -1,8 +1,9 @@ -pkg: gcc/4.8.5 +# {% assign version = opt.version | default: "4.8.5" %} +pkg: gcc/{{ version }} api: v0/package sources: - - tar: http://ftpmirror.gnu.org/gnu/gcc/gcc-4.8.5/gcc-4.8.5.tar.gz + - tar: http://ftpmirror.gnu.org/gnu/gcc/gcc-{{ version }}/gcc-{{ version }}.tar.gz - path: patch-gcc46-texi.diff build: @@ -24,10 +25,10 @@ build: - pkg: coreutils - pkg: binutils - pkg: make - - pkg: gcc/<=4.8.5 + - pkg: gcc/<={{ version }} script: - - patch -d gcc-4.8.5 -p0