From c2e15480915957090a7f07b7eccb1a089f5f95f6 Mon Sep 17 00:00:00 2001 From: Ryan Bottriell Date: Wed, 28 Sep 2022 08:39:39 -0700 Subject: [PATCH 01/12] Add initial integration of liquid template language Signed-off-by: Ryan Bottriell --- Cargo.lock | 150 ++++++++++++++++ Cargo.toml | 1 + crates/spk-cli/cmd-make-recipe/Cargo.toml | 14 ++ .../cmd-make-recipe/src/cmd_make_recipe.rs | 74 ++++++++ crates/spk-cli/cmd-make-recipe/src/lib.rs | 7 + crates/spk-schema/Cargo.toml | 1 + crates/spk-schema/crates/liquid/Cargo.toml | 19 ++ .../liquid/src/filter_compare_version.rs | 73 ++++++++ .../liquid/src/filter_compare_version_test.rs | 35 ++++ .../crates/liquid/src/filter_parse_version.rs | 82 +++++++++ .../liquid/src/filter_parse_version_test.rs | 47 +++++ crates/spk-schema/crates/liquid/src/lib.rs | 37 ++++ .../crates/liquid/src/tag_default.rs | 92 ++++++++++ .../crates/liquid/src/tag_default_test.rs | 57 ++++++ crates/spk-schema/src/error.rs | 2 + crates/spk-schema/src/spec.rs | 13 +- crates/spk-schema/src/spec_test.rs | 29 +++- crates/spk/Cargo.toml | 2 + crates/spk/src/cli.rs | 4 + packages/Makefile | 10 +- packages/cmake/cmake.spk.yaml | 9 +- packages/python/python.spk.yaml | 163 ++++++++++++++++++ 22 files changed, 912 insertions(+), 9 deletions(-) create mode 100644 crates/spk-cli/cmd-make-recipe/Cargo.toml create mode 100644 crates/spk-cli/cmd-make-recipe/src/cmd_make_recipe.rs create mode 100644 crates/spk-cli/cmd-make-recipe/src/lib.rs create mode 100644 crates/spk-schema/crates/liquid/Cargo.toml create mode 100644 crates/spk-schema/crates/liquid/src/filter_compare_version.rs create mode 100644 crates/spk-schema/crates/liquid/src/filter_compare_version_test.rs create mode 100644 crates/spk-schema/crates/liquid/src/filter_parse_version.rs create mode 100644 crates/spk-schema/crates/liquid/src/filter_parse_version_test.rs create mode 100644 crates/spk-schema/crates/liquid/src/lib.rs create mode 100644 crates/spk-schema/crates/liquid/src/tag_default.rs create mode 100644 crates/spk-schema/crates/liquid/src/tag_default_test.rs create mode 100644 packages/python/python.spk.yaml diff --git a/Cargo.lock b/Cargo.lock index 215d165c5b..0ff9cfb1e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -55,6 +55,12 @@ dependencies = [ "backtrace", ] +[[package]] +name = "anymap2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" + [[package]] name = "arrayref" version = "0.3.6" @@ -727,6 +733,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + [[package]] name = "dyn-clone" version = "1.0.6" @@ -1388,6 +1400,16 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "kstring" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3066350882a1cd6d950d055997f379ac37fd39f81cd4d8ed186032eb3c5747" +dependencies = [ + "serde", + "static_assertions", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -1412,6 +1434,63 @@ version = "0.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" +[[package]] +name = "liquid" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00f55b9db2305857de3b3ceaa0e75cb51a76aaec793875fe152e139cb8fed05c" +dependencies = [ + "doc-comment", + "liquid-core", + "liquid-derive", + "liquid-lib", + "serde", +] + +[[package]] +name = "liquid-core" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a93764837aeac37f14b74708cd88a44d82edfa9ad2b1bcd9a3b4d8802fdd9f98" +dependencies = [ + "anymap2", + "itertools", + "kstring", + "liquid-derive", + "num-traits", + "pest", + "pest_derive", + "regex", + "serde", + "time 0.3.11", +] + +[[package]] +name = "liquid-derive" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "926454345f103e8433833077acdbfaa7c3e4b90788d585a8358f02f0b8f5a469" +dependencies = [ + "proc-macro2", + "proc-quote", + "syn", +] + +[[package]] +name = "liquid-lib" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd06ca30ae026d26ee7fa8596f9590959e2d3726bc5a0f16a21ac4f050ec83c0" +dependencies = [ + "itertools", + "liquid-core", + "once_cell", + "percent-encoding", + "regex", + "time 0.3.11", + "unicode-segmentation", +] + [[package]] name = "lock_api" version = "0.4.9" @@ -1989,6 +2068,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + [[package]] name = "proc-macro2" version = "1.0.52" @@ -1998,6 +2083,30 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-quote" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e84ab161de78c915302ca325a19bee6df272800e2ae1a43fe3ef430bab2a100" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "proc-quote-impl", + "quote", + "syn", +] + +[[package]] +name = "proc-quote-impl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb3ec628b063cdbcf316e06a8b8c1a541d28fa6c0a8eacd2bfb2b7f49e88aa0" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", +] + [[package]] name = "procfs" version = "0.13.2" @@ -3018,6 +3127,7 @@ dependencies = [ "spk-cmd-explain", "spk-cmd-install", "spk-cmd-make-binary", + "spk-cmd-make-recipe", "spk-cmd-make-source", "spk-cmd-render", "spk-cmd-repo", @@ -3277,6 +3387,19 @@ dependencies = [ "tracing", ] +[[package]] +name = "spk-cmd-make-recipe" +version = "0.36.0" +dependencies = [ + "anyhow", + "async-trait", + "clap 3.2.23", + "spk-cli-common", + "spk-schema", + "spk-schema-liquid", + "tracing", +] + [[package]] name = "spk-cmd-make-source" version = "0.36.0" @@ -3402,6 +3525,7 @@ dependencies = [ "spfs", "spk-schema-foundation", "spk-schema-ident", + "spk-schema-liquid", "spk-schema-validators", "sys-info", "tempfile", @@ -3458,6 +3582,19 @@ dependencies = [ "thiserror", ] +[[package]] +name = "spk-schema-liquid" +version = "0.36.0" +dependencies = [ + "liquid", + "liquid-core", + "rstest", + "serde", + "serde_json", + "spk-schema-foundation", + "tracing", +] + [[package]] name = "spk-schema-validators" version = "0.36.0" @@ -3847,8 +3984,15 @@ dependencies = [ "itoa 1.0.6", "libc", "num_threads", + "time-macros", ] +[[package]] +name = "time-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" + [[package]] name = "tinytemplate" version = "1.2.1" @@ -4243,6 +4387,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" + [[package]] name = "unicode-width" version = "0.1.9" diff --git a/Cargo.toml b/Cargo.toml index 10f78c9b90..f6ac3468ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "crates/spk-config", "crates/spk-launcher", "crates/spk-schema", + "crates/spk-schema/crates/*", "crates/spk-solve", "crates/spk-solve/crates/*", "crates/spk-storage", diff --git a/crates/spk-cli/cmd-make-recipe/Cargo.toml b/crates/spk-cli/cmd-make-recipe/Cargo.toml new file mode 100644 index 0000000000..778725fe03 --- /dev/null +++ b/crates/spk-cli/cmd-make-recipe/Cargo.toml @@ -0,0 +1,14 @@ +[package] +authors = ["Ryan Bottriell "] +edition = "2021" +name = "spk-cmd-make-recipe" +version = "0.36.0" + +[dependencies] +anyhow = "1.0" +async-trait = "0.1" +clap = { version = "3.2", features = ["derive", "env"] } +spk-cli-common = { path = '../common' } +spk-schema = { path = '../../spk-schema' } +spk-schema-liquid = { path = '../../spk-schema/crates/liquid' } +tracing = "0.1.35" diff --git a/crates/spk-cli/cmd-make-recipe/src/cmd_make_recipe.rs b/crates/spk-cli/cmd-make-recipe/src/cmd_make_recipe.rs new file mode 100644 index 0000000000..1ee57bf8be --- /dev/null +++ b/crates/spk-cli/cmd-make-recipe/src/cmd_make_recipe.rs @@ -0,0 +1,74 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use std::sync::Arc; + +use anyhow::{Context, Result}; +use clap::Args; +use spk_cli_common::{flags, CommandArgs, Run}; +use spk_schema::foundation::format::FormatOptionMap; +use spk_schema::foundation::spec_ops::Named; +use spk_schema::{SpecTemplate, Template, TemplateExt}; + +/// Render a package spec template into a recipe +/// +/// This is done automatically when building packages, but can +/// be a useful debugging tool when writing package spec files. +#[derive(Args)] +#[clap(visible_aliases = &["mkrecipe", "mkr"])] +pub struct MakeRecipe { + #[clap(flatten)] + pub options: flags::Options, + + #[clap(short, long, global = true, parse(from_occurrences))] + pub verbose: u32, + + /// The package spec file to render + #[clap(name = "SPEC_FILE")] + pub package: Option, +} + +impl CommandArgs for MakeRecipe { + // The important positional arg for a make-recipe is the package + fn get_positional_args(&self) -> Vec { + match self.package.clone() { + Some(p) => vec![p], + None => vec![], + } + } +} + +#[async_trait::async_trait] +impl Run for MakeRecipe { + async fn run(&mut self) -> Result { + let options = self.options.get_options()?; + + let template = match flags::find_package_template(&self.package)? { + flags::FindPackageTemplateResult::NotFound(name) => { + Arc::new(SpecTemplate::from_file(name.as_ref())?) + } + res => { + let (_, template) = res.must_be_found(); + template + } + }; + + tracing::info!("rendering template for {}", template.name()); + tracing::info!("using options {}", options.format_option_map()); + let rendered = spk_schema_liquid::render_template(template.source(), &options) + .context("Failed to render template")?; + print!("{rendered}"); + + match template.render(&options) { + Err(err) => { + tracing::error!("This template did not render into a valid spec {err}"); + Ok(1) + } + Ok(_) => { + tracing::info!("Successfully rendered a valid spec"); + Ok(0) + } + } + } +} diff --git a/crates/spk-cli/cmd-make-recipe/src/lib.rs b/crates/spk-cli/cmd-make-recipe/src/lib.rs new file mode 100644 index 0000000000..50aab51c51 --- /dev/null +++ b/crates/spk-cli/cmd-make-recipe/src/lib.rs @@ -0,0 +1,7 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +#![deny(unsafe_op_in_unsafe_fn)] + +pub mod cmd_make_recipe; diff --git a/crates/spk-schema/Cargo.toml b/crates/spk-schema/Cargo.toml index c64e84d840..97456ba1b3 100644 --- a/crates/spk-schema/Cargo.toml +++ b/crates/spk-schema/Cargo.toml @@ -24,6 +24,7 @@ spfs = { version = '0.34.6', path = "../spfs" } spk-schema-foundation = { path = "./crates/foundation" } spk-schema-ident = { path = "./crates/ident" } spk-schema-validators = { path = "./crates/validators" } +spk-schema-liquid = { path = "./crates/liquid" } sys-info = "0.9.0" tempfile = { workspace = true } thiserror = { workspace = true } diff --git a/crates/spk-schema/crates/liquid/Cargo.toml b/crates/spk-schema/crates/liquid/Cargo.toml new file mode 100644 index 0000000000..cbda09db86 --- /dev/null +++ b/crates/spk-schema/crates/liquid/Cargo.toml @@ -0,0 +1,19 @@ +[package] +authors = ["Ryan Bottriell "] +edition = "2021" +name = "spk-schema-liquid" +version = "0.36.0" + +[features] +migration-to-components = ["spk-schema-foundation/migration-to-components"] + +[dependencies] +liquid = "0.26.0" +liquid-core = "0.26.0" +serde = "1.0" +serde_json = "1.0" +spk-schema-foundation = { path = "../foundation" } +tracing = "0.1.35" + +[dev-dependencies] +rstest = "0.15.0" diff --git a/crates/spk-schema/crates/liquid/src/filter_compare_version.rs b/crates/spk-schema/crates/liquid/src/filter_compare_version.rs new file mode 100644 index 0000000000..554f3221f5 --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/filter_compare_version.rs @@ -0,0 +1,73 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use std::str::FromStr; + +use liquid::model::Scalar; +use liquid::ValueView; +use liquid_core::{ + Display_filter, + Expression, + Filter, + FilterParameters, + FilterReflection, + FromFilterParameters, + ParseFilter, + Result, + Runtime, + Value, +}; +use spk_schema_foundation::version::Version; +use spk_schema_foundation::version_range::{Ranged, VersionFilter}; + +#[cfg(test)] +#[path = "./filter_compare_version_test.rs"] +mod filter_compare_version_test; + +#[derive(Debug, FilterParameters)] +struct CompareVersionArgs { + #[parameter(description = "The comparison operation to perform", arg_type = "str")] + operator: Expression, + #[parameter( + description = "The version to compare with, if not part of the operator string", + arg_type = "str" + )] + rhs: Option, +} + +#[derive(Clone, ParseFilter, FilterReflection)] +#[filter( + name = "compare_version", + description = "Compares one version to another using spk ordering semantics", + parameters(CompareVersionArgs), + parsed(CompareVersionFilter) +)] +pub struct CompareVersion; + +#[derive(Debug, FromFilterParameters, Display_filter)] +#[name = "compare_version"] +struct CompareVersionFilter { + #[parameters] + args: CompareVersionArgs, +} + +impl Filter for CompareVersionFilter { + fn evaluate(&self, input: &dyn ValueView, runtime: &dyn Runtime) -> Result { + let args = self.args.evaluate(runtime)?; + let input = input.as_scalar().ok_or_else(|| { + liquid::Error::with_msg("Expected a scalar value for filter 'compare_version'") + })?; + let lhs = Version::from_str(input.into_string().as_str()) + .map_err(|err| liquid::Error::with_msg(err.to_string()))?; + let range_str = match &args.rhs { + None => args.operator.to_kstr().to_string(), + Some(rhs) => format!("{}{}", args.operator, rhs), + }; + let range = VersionFilter::from_str(range_str.as_str()) + .map_err(|err| liquid::Error::with_msg(err.to_string()))?; + + let result = range.is_applicable(&lhs).is_ok(); + Ok(Value::Scalar(Scalar::new(result).to_owned())) + } +} diff --git a/crates/spk-schema/crates/liquid/src/filter_compare_version_test.rs b/crates/spk-schema/crates/liquid/src/filter_compare_version_test.rs new file mode 100644 index 0000000000..35c7d794f8 --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/filter_compare_version_test.rs @@ -0,0 +1,35 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use rstest::rstest; +use serde_json::json; + +#[rstest] +fn test_template_rendering_version_range() { + // the compare_version helper should be useful in if blocks + // in order to render section based on version ranges + + let options = json!({}); + static TPL: &str = r#" +{% default version = "1.2.3" %} +{% assign use_new_download = version | compare_version: ">=1.0" %} +pkg: package/{{ version }} +sources: +{%- if use_new_download %} + - git: https://downloads.testing/package/v{{ version }} +{%- else %} + - git: https://olddownloads.testing/package/v{{ version }} +{%- endif %} +"#; + static EXPECTED: &str = r#" + + +pkg: package/1.2.3 +sources: + - git: https://downloads.testing/package/v1.2.3 +"#; + let rendered = + crate::render_template(TPL, &options).expect("template should not fail to render"); + assert_eq!(rendered, EXPECTED); +} diff --git a/crates/spk-schema/crates/liquid/src/filter_parse_version.rs b/crates/spk-schema/crates/liquid/src/filter_parse_version.rs new file mode 100644 index 0000000000..0076b32bd8 --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/filter_parse_version.rs @@ -0,0 +1,82 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use std::ops::Deref; +use std::str::FromStr; + +use liquid::ValueView; +use liquid_core::runtime::StackFrame; +use liquid_core::{ + Display_filter, + Expression, + Filter, + FilterParameters, + FilterReflection, + FromFilterParameters, + ParseFilter, + Result, + Runtime, + Value, +}; +use spk_schema_foundation::version::Version; + +#[cfg(test)] +#[path = "./filter_parse_version_test.rs"] +mod filter_parse_version_test; + +#[derive(Debug, FilterParameters)] +struct ParseVersionArgs { + #[parameter(description = "An optional sub-component to access", arg_type = "str")] + path: Option, +} + +#[derive(Clone, ParseFilter, FilterReflection)] +#[filter( + name = "parse_version", + description = "Parses an spk version, outputting one or all components", + parameters(ParseVersionArgs), + parsed(ParseVersionFilter) +)] +pub struct ParseVersion; + +#[derive(Debug, FromFilterParameters, Display_filter)] +#[name = "parse_version"] +struct ParseVersionFilter { + #[parameters] + args: ParseVersionArgs, +} + +impl Filter for ParseVersionFilter { + fn evaluate(&self, input: &dyn ValueView, runtime: &dyn Runtime) -> Result { + let input = input.as_scalar().ok_or_else(|| { + liquid::Error::with_msg("Expected a scalar value for filter 'parse_version'") + })?; + let version = Version::from_str(input.into_string().as_str()) + .map_err(|err| liquid::Error::with_msg(err.to_string()))?; + + let data = liquid::object!({ + "major": version.major(), + "minor": version.minor(), + "patch": version.patch(), + "base": version.base(), + "parts": version.parts.parts, + "plus_epsilon": version.parts.plus_epsilon, + "pre": version.pre.deref(), + "post": version.post.deref(), + }); + + if let Some(path) = &self.args.path { + if matches!(path, Expression::Literal(..)) { + return Err(liquid::Error::with_msg( + "parse_version expected a path to evaluate, but found a literal value", + )); + } + let rt = StackFrame::new(runtime, data); + let value = path.evaluate(&rt)?; + Ok(value.into_owned()) + } else { + Ok(data.into()) + } + } +} diff --git a/crates/spk-schema/crates/liquid/src/filter_parse_version_test.rs b/crates/spk-schema/crates/liquid/src/filter_parse_version_test.rs new file mode 100644 index 0000000000..38c71983ee --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/filter_parse_version_test.rs @@ -0,0 +1,47 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use rstest::rstest; +use serde_json::json; + +#[rstest] +fn test_parse_version_access_basic() { + let options = json!({}); + static TPL: &str = r#"{{ "1.2.3.4.5-beta.1+r.0" | parse_version: minor }}"#; + static EXPECTED: &str = r#"2"#; + let rendered = + crate::render_template(TPL, &options).expect("template should not fail to render"); + assert_eq!(rendered, EXPECTED); +} + +#[rstest] +fn test_parse_version_access_block_params() { + let options = json!({"version": "1.2.3.4.5-beta.1+r.0"}); + static TPL: &str = r#" +{% assign v = version | parse_version %} +{{version}} +{{v.base}} +{{v.major}} +{{v.minor}} +{{v.patch}} +{{v.parts[3]}} +{{v.parts[4]}} +{{v.post.r}} +{{v.pre.beta}} +"#; + static EXPECTED: &str = r#" +1.2.3.4.5-beta.1+r.0 +1.2.3.4.5 +1 +2 +3 +4 +5 +0 +1 +"#; + let rendered = + crate::render_template(TPL, &options).expect("template should not fail to render"); + assert_eq!(rendered.trim(), EXPECTED.trim()); +} diff --git a/crates/spk-schema/crates/liquid/src/lib.rs b/crates/spk-schema/crates/liquid/src/lib.rs new file mode 100644 index 0000000000..17b9aec401 --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/lib.rs @@ -0,0 +1,37 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk +//! Defines the default configuration for processing spec file templates in spk + +mod filter_compare_version; +mod filter_parse_version; +mod tag_default; + +pub use liquid::Error; + +/// Build the default template parser for spk +/// +/// This parser has all configuration and extensions +/// needed for rendering spk spec templates. +pub fn default_parser() -> liquid::Parser { + let res = liquid::ParserBuilder::new() + .stdlib() + .tag(tag_default::DefaultTag) + .filter(filter_parse_version::ParseVersion) + .filter(filter_compare_version::CompareVersion) + .build(); + debug_assert!(matches!(res, Ok(_)), "default template parser is valid"); + res.unwrap() +} + +/// Render a template with the default configuration +pub fn render_template(tpl: T, data: &D) -> Result +where + T: AsRef, + D: serde::Serialize, +{ + let parser = default_parser(); + let template = parser.parse(tpl.as_ref())?; + let globals = liquid::to_object(data)?; + template.render(&globals) +} diff --git a/crates/spk-schema/crates/liquid/src/tag_default.rs b/crates/spk-schema/crates/liquid/src/tag_default.rs new file mode 100644 index 0000000000..887d578239 --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/tag_default.rs @@ -0,0 +1,92 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use liquid_core::error::ResultLiquidExt; +use liquid_core::parser::FilterChain; +use liquid_core::{Language, ParseTag, Renderable, Result, Runtime, TagReflection, TagTokenIter}; + +#[cfg(test)] +#[path = "./tag_default_test.rs"] +mod tag_default_test; + +/// Allows for specifying default values for template variables +/// +/// Because we require that all variables to be filled in by default, +/// this allows "optional" template variables to exist by enabling +/// developers to set default values for variables that are not +/// otherwise passed in +#[derive(Clone, Copy)] +pub struct DefaultTag; + +impl TagReflection for DefaultTag { + fn tag(&self) -> &'static str { + "default" + } + + fn description(&self) -> &'static str { + "assign a variable value if it doesn't already have one" + } +} + +impl ParseTag for DefaultTag { + fn parse( + &self, + mut arguments: TagTokenIter<'_>, + options: &Language, + ) -> Result> { + let dst = arguments + .expect_next("Identifier expected.")? + .expect_identifier() + .into_result()? + .to_string() + .into(); + + arguments + .expect_next("Assignment operator \"=\" expected.")? + .expect_str("=") + .into_result_custom_msg("Assignment operator \"=\" expected.")?; + + let src = arguments + .expect_next("FilterChain expected.")? + .expect_filter_chain(options) + .into_result()?; + + // no more arguments should be supplied, trying to supply them is an error + arguments.expect_nothing()?; + + Ok(Box::new(Default { dst, src })) + } + + fn reflection(&self) -> &dyn TagReflection { + self + } +} + +#[derive(Debug)] +struct Default { + dst: liquid_core::model::KString, + src: FilterChain, +} + +impl Default { + fn trace(&self) -> String { + format!("{{% default {} = {}%}}", self.dst, self.src) + } +} + +impl Renderable for Default { + fn render_to(&self, _writer: &mut dyn std::io::Write, runtime: &dyn Runtime) -> Result<()> { + let value = self + .src + .evaluate(runtime) + .trace_with(|| self.trace().into())? + .into_owned(); + + let name = self.dst.as_str().into(); + if runtime.try_get(&[name]).is_none() { + runtime.set_global(self.dst.clone(), value); + } + Ok(()) + } +} diff --git a/crates/spk-schema/crates/liquid/src/tag_default_test.rs b/crates/spk-schema/crates/liquid/src/tag_default_test.rs new file mode 100644 index 0000000000..3d6eedfc7b --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/tag_default_test.rs @@ -0,0 +1,57 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use rstest::rstest; +use serde_json::json; + +#[rstest] +fn test_template_rendering_default_value() { + // because we require a value for all variables in the template + // a helper must be provided that allows for defining default values + // as a convenience + + let options = json!({"version": "2.3.4"}); + static TPL: &str = r#" +{% default name = "my-package" %} +{% default version = "ignore-me" %} +pkg: {{ name }}/{{ version }} +sources: + - git: https://downloads.testing/{{ name }}/v{{ version }} +"#; + static EXPECTED: &str = r#" + + +pkg: my-package/2.3.4 +sources: + - git: https://downloads.testing/my-package/v2.3.4 +"#; + let rendered = + crate::render_template(TPL, &options).expect("template should not fail to render"); + assert_eq!(rendered, EXPECTED); +} + +#[rstest] +fn test_template_rendering_defaults_many() { + // ensure that using multiple defaults does not + // cause them to interfere + + let options = json!({}); + static TPL: &str = r#" +{% default name = "my-package" %} +{% default version = "2.3.4" %} +pkg: {{ name }}/{{ version }} +sources: + - git: https://downloads.testing/{{ name }}/v{{ version }} +"#; + static EXPECTED: &str = r#" + + +pkg: my-package/2.3.4 +sources: + - git: https://downloads.testing/my-package/v2.3.4 +"#; + let rendered = + crate::render_template(TPL, &options).expect("template should not fail to render"); + assert_eq!(rendered, EXPECTED); +} diff --git a/crates/spk-schema/src/error.rs b/crates/spk-schema/src/error.rs index 14f6235ac8..691adb18d9 100644 --- a/crates/spk-schema/src/error.rs +++ b/crates/spk-schema/src/error.rs @@ -50,6 +50,8 @@ pub enum Error { #[error(transparent)] InvalidYaml(#[from] format_serde_error::SerdeError), + #[error(transparent)] + InvalidTemplate(spk_schema_liquid::Error), } impl Error { diff --git a/crates/spk-schema/src/spec.rs b/crates/spk-schema/src/spec.rs index e2d6c30d25..c6dd257d1e 100644 --- a/crates/spk-schema/src/spec.rs +++ b/crates/spk-schema/src/spec.rs @@ -107,6 +107,13 @@ pub struct SpecTemplate { template: String, } +impl SpecTemplate { + /// The complete source string for this template + pub fn source(&self) -> &str { + &self.template + } +} + impl Named for SpecTemplate { fn name(&self) -> &PkgName { &self.name @@ -120,8 +127,10 @@ impl Template for SpecTemplate { &self.file_path } - fn render(&self, _options: &OptionMap) -> Result { - Ok(SpecRecipe::from_yaml(&self.template)?) + fn render(&self, options: &OptionMap) -> Result { + let rendered = spk_schema_liquid::render_template(&self.template, &options) + .map_err(Error::InvalidTemplate)?; + Ok(SpecRecipe::from_yaml(&rendered)?) } } diff --git a/crates/spk-schema/src/spec_test.rs b/crates/spk-schema/src/spec_test.rs index 0c4ae5b10a..d4628b3931 100644 --- a/crates/spk-schema/src/spec_test.rs +++ b/crates/spk-schema/src/spec_test.rs @@ -3,11 +3,13 @@ // https://github.com/imageworks/spk use rstest::rstest; +use spk_schema_foundation::name::PkgName; use spk_schema_foundation::option_map; use spk_schema_foundation::option_map::OptionMap; +use super::SpecTemplate; use crate::prelude::*; -use crate::recipe; +use crate::{recipe, Template}; #[rstest] fn test_resolve_options_empty_options() { @@ -304,3 +306,28 @@ fn test_get_build_requirements_pkg_in_variant_preserves_order() { index = Some(3) ); } + +#[rstest] +fn test_template_error_position() { + format_serde_error::never_color(); + static SPEC: &str = r#"pkg: mypackage/{{ version }} +sources: + - git: https://downloads.testing/mypackage/v{{ verison }} +"#; + let tpl = SpecTemplate { + name: PkgName::new("my-package").unwrap().to_owned(), + file_path: "my-package.spk.yaml".into(), + template: SPEC.to_string(), + }; + let options = option_map! {"version" => "1.0.0"}; + let err = tpl + .render(&options) + .expect_err("expect template rendering to fail"); + let expected = r#" +liquid: Unknown variable + with: + requested variable=verison +"#; + let message = err.to_string(); + assert_eq!(message.trim(), expected.trim()); +} diff --git a/crates/spk/Cargo.toml b/crates/spk/Cargo.toml index e1f16029fc..53ec7e2b92 100644 --- a/crates/spk/Cargo.toml +++ b/crates/spk/Cargo.toml @@ -34,6 +34,7 @@ cli = [ "dep:spk-cmd-install", "dep:spk-cmd-make-binary", "dep:spk-cmd-make-source", + "dep:spk-cmd-make-recipe", "dep:spk-cmd-render", "dep:spk-cmd-repo", "dep:spk-cmd-test", @@ -61,6 +62,7 @@ spk-cmd-explain = { path = '../spk-cli/cmd-explain', optional = true } spk-cmd-install = { path = '../spk-cli/cmd-install', optional = true } spk-cmd-make-binary = { path = '../spk-cli/cmd-make-binary', optional = true } spk-cmd-make-source = { path = '../spk-cli/cmd-make-source', optional = true } +spk-cmd-make-recipe = { path = '../spk-cli/cmd-make-recipe', optional = true } spk-cmd-render = { path = '../spk-cli/cmd-render', optional = true } spk-cmd-repo = { path = '../spk-cli/cmd-repo', optional = true } spk-cmd-test = { path = '../spk-cli/cmd-test', optional = true } diff --git a/crates/spk/src/cli.rs b/crates/spk/src/cli.rs index 2412d5c239..16f896452e 100644 --- a/crates/spk/src/cli.rs +++ b/crates/spk/src/cli.rs @@ -24,6 +24,7 @@ use spk_cmd_env::cmd_env; use spk_cmd_explain::cmd_explain; use spk_cmd_install::cmd_install; use spk_cmd_make_binary::cmd_make_binary; +use spk_cmd_make_recipe::cmd_make_recipe; use spk_cmd_make_source::cmd_make_source; use spk_cmd_render::cmd_render; use spk_cmd_repo::cmd_repo; @@ -156,6 +157,7 @@ pub enum Command { Ls(cmd_ls::Ls), MakeBinary(cmd_make_binary::MakeBinary), MakeSource(cmd_make_source::MakeSource), + MakeRecipe(cmd_make_recipe::MakeRecipe), New(cmd_new::New), #[clap(alias = "variant-count", hide = true)] NumVariants(cmd_num_variants::NumVariants), @@ -191,6 +193,7 @@ impl Run for Command { Command::Ls(cmd) => cmd.run().await, Command::MakeBinary(cmd) => cmd.run().await, Command::MakeSource(cmd) => cmd.run().await, + Command::MakeRecipe(cmd) => cmd.run().await, Command::New(cmd) => cmd.run().await, Command::NumVariants(cmd) => cmd.run().await, Command::Publish(cmd) => cmd.run().await, @@ -223,6 +226,7 @@ impl CommandArgs for Command { Command::Ls(cmd) => cmd.get_positional_args(), Command::MakeBinary(cmd) => cmd.get_positional_args(), Command::MakeSource(cmd) => cmd.get_positional_args(), + Command::MakeRecipe(cmd) => cmd.get_positional_args(), Command::New(cmd) => cmd.get_positional_args(), Command::NumVariants(cmd) => cmd.get_positional_args(), Command::Publish(cmd) => cmd.get_positional_args(), diff --git a/packages/Makefile b/packages/Makefile index 2eed365691..dcd14a3ebd 100644 --- a/packages/Makefile +++ b/packages/Makefile @@ -60,8 +60,14 @@ perl: gnu perl/perl.spk .PHONY: python python2 python3 python: python2 python3 -python2: bootstrap bzip2/bzip2.spk zlib/zlib.spk python/python2.spk -python3: bootstrap bzip2/bzip2.spk zlib/zlib.spk libffi/libffi.spk openssl/openssl.spk python/python3.spk +python2: bootstrap bzip2/bzip2.spk zlib/zlib.spk + spk info -r local python/2.7.5/src > /dev/null 2>&1 || spk make-source -v python/python.spk.yaml -o version=2.7.5 + spk mkb -r origin -v -o version=2.7.5 + spk export python/2.7.5 python27.spk.yaml +python3: bootstrap bzip2/bzip2.spk zlib/zlib.spk libffi/libffi.spk openssl/openssl.spk + spk info -r local python/3.7.3/src > /dev/null 2>&1 || spk make-source -v python/python.spk.yaml -o version=3.7.3 + spk mkb -r origin -v -o version=3.7.3 + spk export python/3.7.3 python27.spk.yaml .PHONY: ninja ninja: ninja/ninja.spk diff --git a/packages/cmake/cmake.spk.yaml b/packages/cmake/cmake.spk.yaml index 88952daaf9..0f412c8d97 100644 --- a/packages/cmake/cmake.spk.yaml +++ b/packages/cmake/cmake.spk.yaml @@ -1,4 +1,5 @@ -pkg: cmake/3.18.2 +# {% default version = "3.18.2" %} +pkg: cmake/{{ version }} api: v0/package sources: @@ -7,8 +8,8 @@ sources: # the tar file. - path: ./ - script: - - export TARFILE=cmake-3.18.2-Linux-x86_64.tar.gz - - if [ ! -e ./$TARFILE ] ; then wget https://github.com/Kitware/CMake/releases/download/v3.18.2/$TARFILE ; fi + - export TARFILE=cmake-{{ version }}-Linux-x86_64.tar.gz + - if [ ! -e ./$TARFILE ] ; then wget https://github.com/Kitware/CMake/releases/download/v{{ version }}/$TARFILE ; fi build: options: @@ -17,7 +18,7 @@ build: script: - mkdir -p build; cd build - tar -xvf - ../cmake-3.18.2-Linux-x86_64.tar.gz + ../cmake-{{ version }}-Linux-x86_64.tar.gz --strip-components=1 --exclude=doc --exclude=Help diff --git a/packages/python/python.spk.yaml b/packages/python/python.spk.yaml new file mode 100644 index 0000000000..529c9b6c00 --- /dev/null +++ b/packages/python/python.spk.yaml @@ -0,0 +1,163 @@ +# {% default version = "3.7.3" %} +# {% assign v = version | parse_version %} +# {% assign is_py3 = version | compare_version: ">=3" %} +pkg: python/{{ version }} +api: v0/package +sources: + - git: https://github.com/python/cpython + ref: v{{ version }} +build: + options: + - var: os + - var: arch + - var: centos + - pkg: gcc + - pkg: stdfs + - pkg: bzip2 + # {% if is_py3 %} + - pkg: ncurses + - pkg: binutils + - pkg: libffi + - pkg: openssl + - pkg: zlib/1.2 + # {% endif %} + - var: abi + default: "cp{{v.major}}{{v.minor}}m" + choices: [ + "cp{{v.major}}{{v.minor}}m", + "cp{{v.major}}{{v.minor}}dm", + # {% unless is_py3 %} + "cp{{v.major}}{{v.minor}}mu", + "cp{{v.major}}{{v.minor}}dmu", + # {% endunless %} + ] + inheritance: Strong + - var: debug + default: off + choices: [on, off] + - var: optimize + default: on + choices: [on, off] + variants: + - { gcc: 6.3, abi: "cp{{v.major}}{{v.minor}}m", debug: off } + script: + - | + case "$SPK_OPT_debug" in + on) + if ! [[ "$SPK_OPT_abi" =~ ^cp{{v.major}}{{v.minor}}.*d ]]; then + echo "Must use an abi with debug when building with debug enabled!" + exit 1 + fi + DEBUG="--with-pydebug" + ;; + off) + if [[ "$SPK_OPT_abi" =~ ^cp{{v.major}}{{v.minor}}.*d ]]; then + echo "Must not use an abi with debug when building with debug disabled!" + exit 1 + fi + DEBUG="" + ;; + *) + echo "Unsupported debug: $SPK_OPT_debug" + ;; + esac + # {% unless is_py3 %} + - | + case "$SPK_OPT_abi" in + cp27m) + UNICODE="--enable-unicode=ucs2" + ;; + cp27mu) + UNICODE="--enable-unicode=ucs4" + ;; + *) + echo "Unsupported abi: $SPK_OPT_abi" + ;; + esac + # on systems where python3 is the default, we can + # see syntax errors unless we ensure that 'python' runs python2 + - echo "#!/bin/bash" > /spfs/bin/python + - echo 'exec python2 "$@"' >> /spfs/bin/python + - chmod +x /spfs/bin/python + # {% endunless %} + # {% if is_py3 %} + - | + OPTIMIZE="" + if [[ "${SPK_OPT_optimize}" == "on" ]]; then + OPTIMIZE="--enable-optimiations" + fi + # {% endif %} + - > + ./configure + --prefix=${PREFIX} + CC=$CC + CXX=$CXX + LDFLAGS='-Wl,--rpath=/spfs/lib,-L/spfs/lib' + PKG_CONFIG_PATH=/spfs/share/pkgconfig:/spfs/lib/pkgconfig + CPPFLAGS='-I/spfs/include/ncurses' + --enable-shared + --with-ensurepip=no + {% if is_py3 %}"$OPTIMIZE"{% endif %} + {% unless is_py3 %}"$UNICODE"{% endunless %} + $DEBUG + - make -j$(nproc) + - make install + # remove test files that are just bloat + - find /spfs/lib/python* -name "test" -type d | xargs -r rm -rv + - find /spfs/lib/python* -name "*_test" -type d | xargs -r rm -rv + - "ln -sf python{{ version | parse_version: major }} /spfs/bin/python" + # python is best in spfs when pyc files are not used at all + - find /spfs -type f -name "*.pyc" | xargs rm + +tests: + - stage: install + script: + # Verify we built a python with the requested ABI + - python_abi=$(/spfs/bin/python -c 'import wheel.bdist_wheel; + print(wheel.bdist_wheel.get_abi_tag())') + - | + if [ "$python_abi" != "$SPK_OPT_abi" ]; then + echo "Python binary ABI does not match spk options: $python_abi != $SPK_OPT_abi" + exit 1 + fi + - stage: install + script: + # Verify bz2 support is available by importing and not getting a traceback + - test -z "$(/spfs/bin/python -c 'import bz2' 2>&1)" + # Verify zlib support is available by importing and not getting a traceback + - test -z "$(/spfs/bin/python -c 'import zlib' 2>&1)" + +install: + environment: + - set: PYTHONDONTWRITEBYTECODE + value: 1 + requirements: + - pkg: binutils + fromBuildEnv: Binary + - pkg: gcc + fromBuildEnv: Binary + include: IfAlreadyPresent + - pkg: stdfs + - pkg: libffi + - pkg: ncurses + - pkg: bzip2 + fromBuildEnv: Binary + - pkg: zlib + fromBuildEnv: Binary + - pkg: openssl + fromBuildEnv: Binary + components: + - name: run + files: + - /etc/ + - /bin/ + - /lib/ + - '!/lib/pkgconfig' + - name: build + uses: [run] + files: + - /include/ + - /lib/pkgconfig + - name: man + files: + - /share/man From 9fd957ec12fea18af857733e5e68755521074358 Mon Sep 17 00:00:00 2001 From: Ryan Bottriell Date: Sun, 2 Oct 2022 21:43:18 -0700 Subject: [PATCH 02/12] Add filter to perform replacements via regular expressions Signed-off-by: Ryan Bottriell --- Cargo.lock | 1 + crates/spk-schema/crates/liquid/Cargo.toml | 1 + .../crates/liquid/src/filter_replace_regex.rs | 64 +++++++++++++++++++ .../liquid/src/filter_replace_regex_test.rs | 44 +++++++++++++ crates/spk-schema/crates/liquid/src/lib.rs | 2 + 5 files changed, 112 insertions(+) create mode 100644 crates/spk-schema/crates/liquid/src/filter_replace_regex.rs create mode 100644 crates/spk-schema/crates/liquid/src/filter_replace_regex_test.rs diff --git a/Cargo.lock b/Cargo.lock index 0ff9cfb1e4..cc26a99a88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3588,6 +3588,7 @@ version = "0.36.0" dependencies = [ "liquid", "liquid-core", + "regex", "rstest", "serde", "serde_json", diff --git a/crates/spk-schema/crates/liquid/Cargo.toml b/crates/spk-schema/crates/liquid/Cargo.toml index cbda09db86..54c1679f00 100644 --- a/crates/spk-schema/crates/liquid/Cargo.toml +++ b/crates/spk-schema/crates/liquid/Cargo.toml @@ -10,6 +10,7 @@ migration-to-components = ["spk-schema-foundation/migration-to-components"] [dependencies] liquid = "0.26.0" liquid-core = "0.26.0" +regex = "1.6.0" serde = "1.0" serde_json = "1.0" spk-schema-foundation = { path = "../foundation" } diff --git a/crates/spk-schema/crates/liquid/src/filter_replace_regex.rs b/crates/spk-schema/crates/liquid/src/filter_replace_regex.rs new file mode 100644 index 0000000000..75b42b5d74 --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/filter_replace_regex.rs @@ -0,0 +1,64 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use liquid::ValueView; +use liquid_core::{ + Display_filter, + Expression, + Filter, + FilterParameters, + FilterReflection, + FromFilterParameters, + ParseFilter, + Result, + Runtime, + Value, +}; + +#[cfg(test)] +#[path = "./filter_replace_regex_test.rs"] +mod filter_replace_regex_test; + +#[derive(Debug, FilterParameters)] +struct ReplaceRegexArgs { + #[parameter(description = "The regular expression to search.", arg_type = "str")] + search: Expression, + #[parameter( + description = "The text to replace search results with. If not given, the filter will just delete search results. Capture groups can be substituted using `$`", + arg_type = "str" + )] + replace: Option, +} + +#[derive(Clone, ParseFilter, FilterReflection)] +#[filter( + name = "replace_re", + description = "Like `replace`, but searches using a regular expression.", + parameters(ReplaceRegexArgs), + parsed(ReplaceRegexFilter) +)] +pub struct ReplaceRegex; + +#[derive(Debug, FromFilterParameters, Display_filter)] +#[name = "replace_re"] +struct ReplaceRegexFilter { + #[parameters] + args: ReplaceRegexArgs, +} + +impl Filter for ReplaceRegexFilter { + fn evaluate(&self, input: &dyn ValueView, runtime: &dyn Runtime) -> Result { + let args = self.args.evaluate(runtime)?; + + let input = input.to_kstr(); + + let search = regex::Regex::new(&args.search) + .map_err(|err| liquid::Error::with_msg(err.to_string()))?; + let replace = args.replace.unwrap_or_else(|| "".into()); + + Ok(Value::scalar( + search.replace_all(&input, replace.as_str()).to_string(), + )) + } +} diff --git a/crates/spk-schema/crates/liquid/src/filter_replace_regex_test.rs b/crates/spk-schema/crates/liquid/src/filter_replace_regex_test.rs new file mode 100644 index 0000000000..dc6aabdf26 --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/filter_replace_regex_test.rs @@ -0,0 +1,44 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk + +use rstest::rstest; +use serde_json::json; + +#[rstest] +fn test_replace_regex_basic() { + let options = json!({}); + static TPL: &str = r#"{{ "1992-02-25" | replace_re: "(\d+)-(\d+)-(\d+)", "$3/$2/$1" }}"#; + static EXPECTED: &str = r#"25/02/1992"#; + let rendered = + crate::render_template(TPL, &options).expect("template should not fail to render"); + assert_eq!(rendered, EXPECTED); +} + +#[rstest] +fn test_replace_regex_empty() { + let options = json!({}); + static TPL: &str = r#"{{ "Hello, World!" | replace_re: "[A-Z]" }}"#; + static EXPECTED: &str = r#"ello, orld!"#; + let rendered = + crate::render_template(TPL, &options).expect("template should not fail to render"); + assert_eq!(rendered, EXPECTED); +} + +#[rstest] +fn test_replace_regex_compile_error() { + let options = json!({"version": "1.2.3.4.5-beta.1+r.0"}); + static TPL: &str = r#"{{ "something" | replace_re: "(some]" }}"#; + static EXPECTED_ERR: &str = r#" +liquid: regex parse error: + (some] + ^ +error: unclosed group +from: Filter error + with: + filter=replace_re : "(some]" + input="something" +"#; + let err = crate::render_template(TPL, &options).expect_err("template should fail on bad regex"); + assert_eq!(err.to_string().trim(), EXPECTED_ERR.trim()); +} diff --git a/crates/spk-schema/crates/liquid/src/lib.rs b/crates/spk-schema/crates/liquid/src/lib.rs index 17b9aec401..54a1da8226 100644 --- a/crates/spk-schema/crates/liquid/src/lib.rs +++ b/crates/spk-schema/crates/liquid/src/lib.rs @@ -5,6 +5,7 @@ mod filter_compare_version; mod filter_parse_version; +mod filter_replace_regex; mod tag_default; pub use liquid::Error; @@ -19,6 +20,7 @@ pub fn default_parser() -> liquid::Parser { .tag(tag_default::DefaultTag) .filter(filter_parse_version::ParseVersion) .filter(filter_compare_version::CompareVersion) + .filter(filter_replace_regex::ReplaceRegex) .build(); debug_assert!(matches!(res, Ok(_)), "default template parser is valid"); res.unwrap() From 7fb02e3d40596135bee3e58f82799d4f5c677851 Mon Sep 17 00:00:00 2001 From: Ryan Bottriell Date: Tue, 4 Oct 2022 22:10:54 -0700 Subject: [PATCH 03/12] Redivide python 2/3 and add version var to more packages Signed-off-by: Ryan Bottriell Signed-off-by: Ryan Bottriell --- packages/Makefile | 10 +- packages/gnu/gcc/gcc48.spk.yaml | 11 ++- packages/gnu/gcc/gcc63.spk.yaml | 9 +- packages/python/python.spk.yaml | 163 ------------------------------- packages/python/python2.spk.yaml | 26 ++--- packages/python/python3.spk.yaml | 16 +-- 6 files changed, 36 insertions(+), 199 deletions(-) delete mode 100644 packages/python/python.spk.yaml diff --git a/packages/Makefile b/packages/Makefile index dcd14a3ebd..2eed365691 100644 --- a/packages/Makefile +++ b/packages/Makefile @@ -60,14 +60,8 @@ perl: gnu perl/perl.spk .PHONY: python python2 python3 python: python2 python3 -python2: bootstrap bzip2/bzip2.spk zlib/zlib.spk - spk info -r local python/2.7.5/src > /dev/null 2>&1 || spk make-source -v python/python.spk.yaml -o version=2.7.5 - spk mkb -r origin -v -o version=2.7.5 - spk export python/2.7.5 python27.spk.yaml -python3: bootstrap bzip2/bzip2.spk zlib/zlib.spk libffi/libffi.spk openssl/openssl.spk - spk info -r local python/3.7.3/src > /dev/null 2>&1 || spk make-source -v python/python.spk.yaml -o version=3.7.3 - spk mkb -r origin -v -o version=3.7.3 - spk export python/3.7.3 python27.spk.yaml +python2: bootstrap bzip2/bzip2.spk zlib/zlib.spk python/python2.spk +python3: bootstrap bzip2/bzip2.spk zlib/zlib.spk libffi/libffi.spk openssl/openssl.spk python/python3.spk .PHONY: ninja ninja: ninja/ninja.spk diff --git a/packages/gnu/gcc/gcc48.spk.yaml b/packages/gnu/gcc/gcc48.spk.yaml index a31b5d8366..e5c10e38eb 100644 --- a/packages/gnu/gcc/gcc48.spk.yaml +++ b/packages/gnu/gcc/gcc48.spk.yaml @@ -1,8 +1,9 @@ -pkg: gcc/4.8.5 +# {{ default version = "4.8.5" }} +pkg: gcc/{{ version }} api: v0/package sources: - - tar: http://ftpmirror.gnu.org/gnu/gcc/gcc-4.8.5/gcc-4.8.5.tar.gz + - tar: http://ftpmirror.gnu.org/gnu/gcc/gcc-{{ version }}/gcc-{{ version }}.tar.gz - path: patch-gcc46-texi.diff build: @@ -24,10 +25,10 @@ build: - pkg: coreutils - pkg: binutils - pkg: make - - pkg: gcc/<=4.8.5 + - pkg: gcc/<={{ version }} script: - - patch -d gcc-4.8.5 -p0 =3" %} -pkg: python/{{ version }} -api: v0/package -sources: - - git: https://github.com/python/cpython - ref: v{{ version }} -build: - options: - - var: os - - var: arch - - var: centos - - pkg: gcc - - pkg: stdfs - - pkg: bzip2 - # {% if is_py3 %} - - pkg: ncurses - - pkg: binutils - - pkg: libffi - - pkg: openssl - - pkg: zlib/1.2 - # {% endif %} - - var: abi - default: "cp{{v.major}}{{v.minor}}m" - choices: [ - "cp{{v.major}}{{v.minor}}m", - "cp{{v.major}}{{v.minor}}dm", - # {% unless is_py3 %} - "cp{{v.major}}{{v.minor}}mu", - "cp{{v.major}}{{v.minor}}dmu", - # {% endunless %} - ] - inheritance: Strong - - var: debug - default: off - choices: [on, off] - - var: optimize - default: on - choices: [on, off] - variants: - - { gcc: 6.3, abi: "cp{{v.major}}{{v.minor}}m", debug: off } - script: - - | - case "$SPK_OPT_debug" in - on) - if ! [[ "$SPK_OPT_abi" =~ ^cp{{v.major}}{{v.minor}}.*d ]]; then - echo "Must use an abi with debug when building with debug enabled!" - exit 1 - fi - DEBUG="--with-pydebug" - ;; - off) - if [[ "$SPK_OPT_abi" =~ ^cp{{v.major}}{{v.minor}}.*d ]]; then - echo "Must not use an abi with debug when building with debug disabled!" - exit 1 - fi - DEBUG="" - ;; - *) - echo "Unsupported debug: $SPK_OPT_debug" - ;; - esac - # {% unless is_py3 %} - - | - case "$SPK_OPT_abi" in - cp27m) - UNICODE="--enable-unicode=ucs2" - ;; - cp27mu) - UNICODE="--enable-unicode=ucs4" - ;; - *) - echo "Unsupported abi: $SPK_OPT_abi" - ;; - esac - # on systems where python3 is the default, we can - # see syntax errors unless we ensure that 'python' runs python2 - - echo "#!/bin/bash" > /spfs/bin/python - - echo 'exec python2 "$@"' >> /spfs/bin/python - - chmod +x /spfs/bin/python - # {% endunless %} - # {% if is_py3 %} - - | - OPTIMIZE="" - if [[ "${SPK_OPT_optimize}" == "on" ]]; then - OPTIMIZE="--enable-optimiations" - fi - # {% endif %} - - > - ./configure - --prefix=${PREFIX} - CC=$CC - CXX=$CXX - LDFLAGS='-Wl,--rpath=/spfs/lib,-L/spfs/lib' - PKG_CONFIG_PATH=/spfs/share/pkgconfig:/spfs/lib/pkgconfig - CPPFLAGS='-I/spfs/include/ncurses' - --enable-shared - --with-ensurepip=no - {% if is_py3 %}"$OPTIMIZE"{% endif %} - {% unless is_py3 %}"$UNICODE"{% endunless %} - $DEBUG - - make -j$(nproc) - - make install - # remove test files that are just bloat - - find /spfs/lib/python* -name "test" -type d | xargs -r rm -rv - - find /spfs/lib/python* -name "*_test" -type d | xargs -r rm -rv - - "ln -sf python{{ version | parse_version: major }} /spfs/bin/python" - # python is best in spfs when pyc files are not used at all - - find /spfs -type f -name "*.pyc" | xargs rm - -tests: - - stage: install - script: - # Verify we built a python with the requested ABI - - python_abi=$(/spfs/bin/python -c 'import wheel.bdist_wheel; - print(wheel.bdist_wheel.get_abi_tag())') - - | - if [ "$python_abi" != "$SPK_OPT_abi" ]; then - echo "Python binary ABI does not match spk options: $python_abi != $SPK_OPT_abi" - exit 1 - fi - - stage: install - script: - # Verify bz2 support is available by importing and not getting a traceback - - test -z "$(/spfs/bin/python -c 'import bz2' 2>&1)" - # Verify zlib support is available by importing and not getting a traceback - - test -z "$(/spfs/bin/python -c 'import zlib' 2>&1)" - -install: - environment: - - set: PYTHONDONTWRITEBYTECODE - value: 1 - requirements: - - pkg: binutils - fromBuildEnv: Binary - - pkg: gcc - fromBuildEnv: Binary - include: IfAlreadyPresent - - pkg: stdfs - - pkg: libffi - - pkg: ncurses - - pkg: bzip2 - fromBuildEnv: Binary - - pkg: zlib - fromBuildEnv: Binary - - pkg: openssl - fromBuildEnv: Binary - components: - - name: run - files: - - /etc/ - - /bin/ - - /lib/ - - '!/lib/pkgconfig' - - name: build - uses: [run] - files: - - /include/ - - /lib/pkgconfig - - name: man - files: - - /share/man diff --git a/packages/python/python2.spk.yaml b/packages/python/python2.spk.yaml index b86beff2e7..5a6951d8ca 100644 --- a/packages/python/python2.spk.yaml +++ b/packages/python/python2.spk.yaml @@ -1,8 +1,10 @@ -pkg: python/2.7.5 +# {% default version = "2.7.5" %} +# {% assign cpXX = version | replace_re: "(\d+)\.(\d+).*", "cp$1$2" %} +pkg: python/{{ version }} api: v0/package sources: - git: https://github.com/python/cpython - ref: v2.7.5 + ref: v{{ version }} build: options: - var: os @@ -12,29 +14,29 @@ build: - pkg: stdfs - pkg: bzip2 - var: abi - default: cp27mu - choices: [cp27dm, cp27dmu, cp27m, cp27mu] + default: "{{cpXX}}mu" + choices: ["{{cpXX}}dm", "{{cpXX}}dmu", "{{cpXX}}m", "{{cpXX}}mu"] inheritance: Strong - var: debug default: off choices: [on, off] variants: - - { gcc: 4.8, abi: cp27m, debug: off } - - { gcc: 4.8, abi: cp27mu, debug: off } - - { gcc: 6.3, abi: cp27m, debug: off } - - { gcc: 6.3, abi: cp27mu, debug: off } + - { gcc: 4.8, abi: "{{cpXX}}m", debug: off } + - { gcc: 4.8, abi: "{{cpXX}}mu", debug: off } + - { gcc: 6.3, abi: "{{cpXX}}m", debug: off } + - { gcc: 6.3, abi: "{{cpXX}}mu", debug: off } script: - | case "$SPK_OPT_debug" in on) - if ! [[ "$SPK_OPT_abi" =~ ^cp27.*d ]]; then + if ! [[ "$SPK_OPT_abi" =~ ^{{cpXX}}.*d ]]; then echo "Must use an abi with debug when building with debug enabled!" exit 1 fi DEBUG="--with-pydebug" ;; off) - if [[ "$SPK_OPT_abi" =~ ^cp27.*d ]]; then + if [[ "$SPK_OPT_abi" =~ ^{{cpXX}}.*d ]]; then echo "Must not use an abi with debug when building with debug disabled!" exit 1 fi @@ -46,10 +48,10 @@ build: esac - | case "$SPK_OPT_abi" in - cp27m) + {{cpXX}}m) UNICODE="--enable-unicode=ucs2" ;; - cp27mu) + {{cpXX}}mu) UNICODE="--enable-unicode=ucs4" ;; *) diff --git a/packages/python/python3.spk.yaml b/packages/python/python3.spk.yaml index 3af78391fa..5287f3734c 100644 --- a/packages/python/python3.spk.yaml +++ b/packages/python/python3.spk.yaml @@ -1,8 +1,10 @@ -pkg: python/3.7.3 +# {% default version = "3.7.3" %} +# {% assign cpXX = version | replace_re: "(\d+)\.(\d+).*", "cp$1$2" %} +pkg: python/{{ version }} api: v0/package sources: - git: https://github.com/python/cpython - ref: v3.7.3 + ref: v{{ version }} build: options: - var: os @@ -17,26 +19,26 @@ build: - pkg: openssl - pkg: zlib/1.2 - var: abi - default: cp37m - choices: [cp37m, cp37dm] + default: "{{cpXX}}m" + choices: ["{{cpXX}}m", "{{cpXX}}dm"] inheritance: Strong - var: debug default: off choices: [on, off] variants: - - { gcc: 6.3, abi: cp37m, debug: off } + - { gcc: 6.3, abi: "{{cpXX}}m", debug: off } script: - | case "$SPK_OPT_debug" in on) - if ! [[ "$SPK_OPT_abi" =~ ^cp37.*d ]]; then + if ! [[ "$SPK_OPT_abi" =~ ^{{cpXX}}.*d ]]; then echo "Must use an abi with debug when building with debug enabled!" exit 1 fi DEBUG="--with-pydebug" ;; off) - if [[ "$SPK_OPT_abi" =~ ^cp37.*d ]]; then + if [[ "$SPK_OPT_abi" =~ ^{{cpXX}}.*d ]]; then echo "Must not use an abi with debug when building with debug disabled!" exit 1 fi From 6291dd8373bfcdad84102dd5d419fd2574b273db Mon Sep 17 00:00:00 2001 From: Ryan Bottriell Date: Wed, 5 Oct 2022 08:34:19 -0700 Subject: [PATCH 04/12] Add flag to load options from yaml/json files Additionally, refactor the way that options and requests are loaded to use a consistent method of layering and de-duplication. Signed-off-by: Ryan Bottriell Signed-off-by: Ryan Bottriell --- crates/spk-cli/common/src/flags.rs | 61 ++++++++++++------------- crates/spk-cli/common/src/flags_test.rs | 1 + crates/spk-schema/src/spec.rs | 2 +- 3 files changed, 31 insertions(+), 33 deletions(-) diff --git a/crates/spk-cli/common/src/flags.rs b/crates/spk-cli/common/src/flags.rs index 6f021d42cb..5eade8dc01 100644 --- a/crates/spk-cli/common/src/flags.rs +++ b/crates/spk-cli/common/src/flags.rs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/imageworks/spk -use std::collections::HashMap; use std::convert::From; use std::str::FromStr; use std::sync::Arc; @@ -14,7 +13,7 @@ use solve::{DecisionFormatter, DecisionFormatterBuilder, MultiSolverKind}; use spk_schema::foundation::format::FormatIdent; use spk_schema::foundation::ident_build::Build; use spk_schema::foundation::ident_component::Component; -use spk_schema::foundation::name::{OptName, OptNameBuf}; +use spk_schema::foundation::name::OptName; use spk_schema::foundation::option_map::{host_options, OptionMap}; use spk_schema::foundation::spec_ops::Named; use spk_schema::foundation::version::CompatRule; @@ -210,9 +209,16 @@ pub struct Options { /// an equals sign or colon (--opt name=value --opt other:value). /// Additionally, many options can be specified at once in yaml /// or json format (--opt '{name: value, other: value}'). + /// + /// Options can also be given in a file via the --options-file/-f flag. If + /// given, --opt will supercede anything in the options file(s). #[clap(long = "opt", short)] pub options: Vec, + /// Specify build/resolve options from a json or yaml file (see --opt/-o) + #[clap(long)] + pub options_file: Vec, + /// Do not add the default options for the current host system #[clap(long)] pub no_host: bool, @@ -225,23 +231,20 @@ impl Options { false => host_options().context("Failed to compute options for current host")?, }; - for req in self.get_var_requests()? { - opts.insert(req.var, req.value); + for filename in self.options_file.iter() { + let reader = + std::fs::File::open(filename).context(format!("Failed to open: {filename:?}"))?; + let options: OptionMap = serde_yaml::from_reader(reader) + .context(format!("Failed to parse as option mapping: {filename:?}"))?; + opts.extend(options); } - Ok(opts) - } - - pub fn get_var_requests(&self) -> Result> { - let mut requests = Vec::with_capacity(self.options.len()); for pair in self.options.iter() { let pair = pair.trim(); if pair.starts_with('{') { - let given: HashMap = serde_yaml::from_str(pair) + let given: OptionMap = serde_yaml::from_str(pair) .context("--opt value looked like yaml, but could not be parsed")?; - for (name, value) in given.into_iter() { - requests.push(VarRequest::new_with_value(name, value)); - } + opts.extend(given); continue; } @@ -253,9 +256,19 @@ impl Options { }) .and_then(|(name, value)| Ok((OptName::new(name)?, value)))?; - requests.push(VarRequest::new_with_value(name, value)); + opts.insert(name.to_owned(), value.to_string()); } - Ok(requests) + + Ok(opts) + } + + pub fn get_var_requests(&self) -> Result> { + Ok(self + .get_options()? + .into_iter() + .filter(|(_name, value)| !value.is_empty()) + .map(|(name, value)| VarRequest::new_with_value(name, value)) + .collect()) } } @@ -332,23 +345,7 @@ impl Requests { S: AsRef, { let mut out = Vec::::new(); - let var_requests = options.get_var_requests()?; - let mut options = match options.no_host { - true => OptionMap::default(), - false => host_options()?, - }; - // Insert var_requests, which includes requests specified on the command-line, - // into the map so that they can override values provided by host_options(). - for req in var_requests { - options.insert(req.var, req.value); - } - - for (name, value) in options.iter() { - if !value.is_empty() { - out.push(VarRequest::new_with_value(name.clone(), value).into()); - } - } - + let options = options.get_options()?; for r in requests.into_iter() { let r = r.as_ref(); if r.contains('@') { diff --git a/crates/spk-cli/common/src/flags_test.rs b/crates/spk-cli/common/src/flags_test.rs index a9a7da88bb..3a6a7cad26 100644 --- a/crates/spk-cli/common/src/flags_test.rs +++ b/crates/spk-cli/common/src/flags_test.rs @@ -24,6 +24,7 @@ use spk_schema::foundation::option_map::OptionMap; fn test_option_flags_parsing(#[case] args: &[&str], #[case] expected: &[(&str, &str)]) { let options = super::Options { no_host: true, + options_file: Default::default(), options: args.iter().map(ToString::to_string).collect(), }; let actual = options.get_options().unwrap(); diff --git a/crates/spk-schema/src/spec.rs b/crates/spk-schema/src/spec.rs index c6dd257d1e..445933982f 100644 --- a/crates/spk-schema/src/spec.rs +++ b/crates/spk-schema/src/spec.rs @@ -130,7 +130,7 @@ impl Template for SpecTemplate { fn render(&self, options: &OptionMap) -> Result { let rendered = spk_schema_liquid::render_template(&self.template, &options) .map_err(Error::InvalidTemplate)?; - Ok(SpecRecipe::from_yaml(&rendered)?) + Ok(SpecRecipe::from_yaml(rendered)?) } } From 386aa8ca8f923b306fd3ce74f44ac20741633c11 Mon Sep 17 00:00:00 2001 From: Ryan Bottriell Date: Wed, 5 Oct 2022 11:58:22 -0700 Subject: [PATCH 05/12] Ensure namspaced options can be accessed naturally in templates Signed-off-by: Ryan Bottriell --- .../crates/foundation/src/option_map/mod.rs | 32 +++++++++++++++++++ crates/spk-schema/src/spec.rs | 3 +- crates/spk-schema/src/spec_test.rs | 24 +++++++++++++- 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/crates/spk-schema/crates/foundation/src/option_map/mod.rs b/crates/spk-schema/crates/foundation/src/option_map/mod.rs index ff1664bde1..b0a01f5b80 100644 --- a/crates/spk-schema/crates/foundation/src/option_map/mod.rs +++ b/crates/spk-schema/crates/foundation/src/option_map/mod.rs @@ -224,6 +224,38 @@ impl OptionMap { env.remove(&name); } } + + /// Create a yaml mapping from this map, un-flattening all package options. + /// + /// OptionMaps hold package-specific options under dot-notated keys, eg `python.abi`. + /// This function will split those options into sub-objects, creating a two-level + /// mapping instead. In the case where there is a value for both `python` and `python.abi` + /// the former will be dropped. + pub fn to_yaml_value_expanded(&self) -> serde_yaml::Mapping { + use serde_yaml::{Mapping, Value}; + let mut yaml = Mapping::default(); + for (key, value) in self.iter() { + let target = match key.namespace() { + Some(ns) => { + let ns = Value::String(ns.to_string()); + let ns_value = yaml + .entry(ns) + .or_insert_with(|| serde_yaml::Value::Mapping(Default::default())); + if ns_value.as_mapping().is_none() { + *ns_value = serde_yaml::Value::Mapping(Default::default()); + } + ns_value + .as_mapping_mut() + .expect("already validated that this is a mapping") + } + None => &mut yaml, + }; + let key = serde_yaml::Value::String(key.base_name().to_string()); + let value = serde_yaml::Value::String(value.to_string()); + target.insert(key, value); + } + yaml + } } impl<'de> Deserialize<'de> for OptionMap { diff --git a/crates/spk-schema/src/spec.rs b/crates/spk-schema/src/spec.rs index 445933982f..d8a53c7401 100644 --- a/crates/spk-schema/src/spec.rs +++ b/crates/spk-schema/src/spec.rs @@ -128,7 +128,8 @@ impl Template for SpecTemplate { } fn render(&self, options: &OptionMap) -> Result { - let rendered = spk_schema_liquid::render_template(&self.template, &options) + let option_data = options.to_yaml_value_expanded(); + let rendered = spk_schema_liquid::render_template(&self.template, &option_data) .map_err(Error::InvalidTemplate)?; Ok(SpecRecipe::from_yaml(rendered)?) } diff --git a/crates/spk-schema/src/spec_test.rs b/crates/spk-schema/src/spec_test.rs index d4628b3931..f51e07ff74 100644 --- a/crates/spk-schema/src/spec_test.rs +++ b/crates/spk-schema/src/spec_test.rs @@ -308,7 +308,7 @@ fn test_get_build_requirements_pkg_in_variant_preserves_order() { } #[rstest] -fn test_template_error_position() { +fn test_template_error_message() { format_serde_error::never_color(); static SPEC: &str = r#"pkg: mypackage/{{ version }} sources: @@ -331,3 +331,25 @@ liquid: Unknown variable let message = err.to_string(); assert_eq!(message.trim(), expected.trim()); } + +#[rstest] +fn test_template_namespace_options() { + // options can have namespace values separated by a `.` + // which can be annoying to access when the actual key + // still has a `.` in it, so validate that these values + // can instead be accessed as a sub-object using the + // dot-notation built into templating + + format_serde_error::never_color(); + static SPEC: &str = r#"pkg: mypackage/{{ namespace.version }}"#; + let tpl = SpecTemplate { + name: PkgName::new("my-package").unwrap().to_owned(), + file_path: "my-package.spk.yaml".into(), + template: SPEC.to_string(), + }; + let options = option_map! {"namespace.version" => "1.0.0"}; + let recipe = tpl + .render(&options) + .expect("template should render with sub-object access"); + assert_eq!(recipe.version().to_string(), "1.0.0"); +} From ad312a0b05a4b8fded9caa8b95d03e8b88c5ced1 Mon Sep 17 00:00:00 2001 From: Ryan Bottriell Date: Wed, 5 Oct 2022 13:27:16 -0700 Subject: [PATCH 06/12] Add regular expression to extract error position from liquid Signed-off-by: Ryan Bottriell --- Cargo.lock | 2 + crates/spk-schema/crates/liquid/Cargo.toml | 2 + crates/spk-schema/crates/liquid/src/error.rs | 52 +++++++++++++++++++ .../crates/liquid/src/error_test.rs | 19 +++++++ crates/spk-schema/crates/liquid/src/lib.rs | 14 +++-- crates/spk-schema/src/error.rs | 2 +- 6 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 crates/spk-schema/crates/liquid/src/error.rs create mode 100644 crates/spk-schema/crates/liquid/src/error_test.rs diff --git a/Cargo.lock b/Cargo.lock index cc26a99a88..e9833d851b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3586,6 +3586,8 @@ dependencies = [ name = "spk-schema-liquid" version = "0.36.0" dependencies = [ + "format_serde_error", + "lazy_static", "liquid", "liquid-core", "regex", diff --git a/crates/spk-schema/crates/liquid/Cargo.toml b/crates/spk-schema/crates/liquid/Cargo.toml index 54c1679f00..a72760a9ae 100644 --- a/crates/spk-schema/crates/liquid/Cargo.toml +++ b/crates/spk-schema/crates/liquid/Cargo.toml @@ -8,8 +8,10 @@ version = "0.36.0" migration-to-components = ["spk-schema-foundation/migration-to-components"] [dependencies] +lazy_static = "1.4" liquid = "0.26.0" liquid-core = "0.26.0" +format_serde_error = {version = "0.3", default_features = false, features = ["colored"]} regex = "1.6.0" serde = "1.0" serde_json = "1.0" diff --git a/crates/spk-schema/crates/liquid/src/error.rs b/crates/spk-schema/crates/liquid/src/error.rs new file mode 100644 index 0000000000..e845c69057 --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/error.rs @@ -0,0 +1,52 @@ +use std::error::Error; + +use lazy_static::lazy_static; +use regex::Regex; + +#[cfg(test)] +#[path = "./error_test.rs"] +mod error_test; + +#[derive(Debug)] +struct ParsedError { + message: String, +} + +impl std::fmt::Display for ParsedError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +impl Error for ParsedError {} + +pub fn to_error_types(err: liquid::Error) -> format_serde_error::ErrorTypes { + lazy_static! { + static ref RE: Regex = Regex::new(r"(?ms)liquid: --> (\d+):(\d+)\n.*^\s+= (.*)") + .expect("a valid regular expression"); + } + let mut message = err.to_string(); + let mut line = Option::::None; + let mut column = Option::::None; + if let Some(m) = RE.captures(&message) { + line = m.get(1).and_then(|line| line.as_str().parse().ok()); + column = m + .get(2) + .and_then(|column| column.as_str().parse().ok()) + // format_serde_error appears to use 0-based index for columns + // whereas the liquid crates uses a 1-based index + .map(|col: usize| col - 1); + message = m + .get(3) + .map(|msg| msg.as_str()) + .unwrap_or("Invalid Template") + .trim() + .to_string(); + } + let error = Box::new(ParsedError { message }); + format_serde_error::ErrorTypes::Custom { + error, + line, + column, + } +} diff --git a/crates/spk-schema/crates/liquid/src/error_test.rs b/crates/spk-schema/crates/liquid/src/error_test.rs new file mode 100644 index 0000000000..9be829c9ba --- /dev/null +++ b/crates/spk-schema/crates/liquid/src/error_test.rs @@ -0,0 +1,19 @@ +use rstest::rstest; +use serde_json::json; + +#[rstest] +fn test_error_position_extraction() { + // ensure that the source position of an error can be + // properly extracted and used for the returned serde_format_error + + format_serde_error::never_color(); + static TPL: &str = r#"{% default = data | replace ''%}"#; + let err = + crate::render_template(TPL, &json!({})).expect_err("expected template render to fail"); + let expected = r#" + 1 | {% default = data | replace ''%} + | ^ unexpected "="; expected Identifier +"#; + let message = err.to_string(); + assert_eq!(message, expected); +} diff --git a/crates/spk-schema/crates/liquid/src/lib.rs b/crates/spk-schema/crates/liquid/src/lib.rs index 54a1da8226..c26ec9cb0a 100644 --- a/crates/spk-schema/crates/liquid/src/lib.rs +++ b/crates/spk-schema/crates/liquid/src/lib.rs @@ -3,12 +3,13 @@ // https://github.com/imageworks/spk //! Defines the default configuration for processing spec file templates in spk +mod error; mod filter_compare_version; mod filter_parse_version; mod filter_replace_regex; mod tag_default; -pub use liquid::Error; +pub use format_serde_error::SerdeError as Error; /// Build the default template parser for spk /// @@ -27,13 +28,16 @@ pub fn default_parser() -> liquid::Parser { } /// Render a template with the default configuration -pub fn render_template(tpl: T, data: &D) -> Result +pub fn render_template(tpl: T, data: &D) -> Result where T: AsRef, D: serde::Serialize, { + let tpl = tpl.as_ref(); + let map_err = + |err| format_serde_error::SerdeError::new(tpl.to_string(), error::to_error_types(err)); let parser = default_parser(); - let template = parser.parse(tpl.as_ref())?; - let globals = liquid::to_object(data)?; - template.render(&globals) + let template = parser.parse(tpl).map_err(map_err)?; + let globals = liquid::to_object(data).map_err(map_err)?; + template.render(&globals).map_err(map_err) } diff --git a/crates/spk-schema/src/error.rs b/crates/spk-schema/src/error.rs index 691adb18d9..65dcec904f 100644 --- a/crates/spk-schema/src/error.rs +++ b/crates/spk-schema/src/error.rs @@ -51,7 +51,7 @@ pub enum Error { #[error(transparent)] InvalidYaml(#[from] format_serde_error::SerdeError), #[error(transparent)] - InvalidTemplate(spk_schema_liquid::Error), + InvalidTemplate(format_serde_error::SerdeError), } impl Error { From 89739a04086acdbd8ad3ca0fb136eb685448211f Mon Sep 17 00:00:00 2001 From: Ryan Bottriell Date: Wed, 5 Oct 2022 16:09:04 -0700 Subject: [PATCH 07/12] Update templates to also get env and spk data Signed-off-by: Ryan Bottriell --- .../cmd-make-recipe/src/cmd_make_recipe.rs | 4 +- crates/spk-schema/src/lib.rs | 2 +- crates/spk-schema/src/spec.rs | 4 +- crates/spk-schema/src/spec_test.rs | 1 + crates/spk-schema/src/template.rs | 42 +++++++++++++++++++ packages/gnu/gcc/gcc48.spk.yaml | 2 +- packages/gnu/gcc/gcc63.spk.yaml | 2 +- packages/python/python2.spk.yaml | 5 ++- packages/python/python3.spk.yaml | 2 +- 9 files changed, 55 insertions(+), 9 deletions(-) diff --git a/crates/spk-cli/cmd-make-recipe/src/cmd_make_recipe.rs b/crates/spk-cli/cmd-make-recipe/src/cmd_make_recipe.rs index 1ee57bf8be..7770e2c1b6 100644 --- a/crates/spk-cli/cmd-make-recipe/src/cmd_make_recipe.rs +++ b/crates/spk-cli/cmd-make-recipe/src/cmd_make_recipe.rs @@ -56,7 +56,9 @@ impl Run for MakeRecipe { tracing::info!("rendering template for {}", template.name()); tracing::info!("using options {}", options.format_option_map()); - let rendered = spk_schema_liquid::render_template(template.source(), &options) + let data = spk_schema::TemplateData::new(&options); + tracing::debug!("full template data: {data:#?}"); + let rendered = spk_schema_liquid::render_template(template.source(), &data) .context("Failed to render template")?; print!("{rendered}"); diff --git a/crates/spk-schema/src/lib.rs b/crates/spk-schema/src/lib.rs index 67d87fbb93..aef4a8c64c 100644 --- a/crates/spk-schema/src/lib.rs +++ b/crates/spk-schema/src/lib.rs @@ -56,7 +56,7 @@ pub use spk_schema_foundation::{ FromYaml, }; pub use spk_schema_ident::{self as ident, AnyIdent, BuildIdent, Request, VersionIdent}; -pub use template::{Template, TemplateExt}; +pub use template::{Template, TemplateData, TemplateExt}; pub use test_spec::TestStage; pub use validation::{default_validators, ValidationSpec, Validator}; pub use variant::{Variant, VariantExt}; diff --git a/crates/spk-schema/src/spec.rs b/crates/spk-schema/src/spec.rs index d8a53c7401..7b2182fc51 100644 --- a/crates/spk-schema/src/spec.rs +++ b/crates/spk-schema/src/spec.rs @@ -128,8 +128,8 @@ impl Template for SpecTemplate { } fn render(&self, options: &OptionMap) -> Result { - let option_data = options.to_yaml_value_expanded(); - let rendered = spk_schema_liquid::render_template(&self.template, &option_data) + let data = super::TemplateData::new(options); + let rendered = spk_schema_liquid::render_template(&self.template, &data) .map_err(Error::InvalidTemplate)?; Ok(SpecRecipe::from_yaml(rendered)?) } diff --git a/crates/spk-schema/src/spec_test.rs b/crates/spk-schema/src/spec_test.rs index f51e07ff74..433b44af2e 100644 --- a/crates/spk-schema/src/spec_test.rs +++ b/crates/spk-schema/src/spec_test.rs @@ -6,6 +6,7 @@ use rstest::rstest; use spk_schema_foundation::name::PkgName; use spk_schema_foundation::option_map; use spk_schema_foundation::option_map::OptionMap; +use spk_schema_foundation::spec_ops::HasVersion; use super::SpecTemplate; use crate::prelude::*; diff --git a/crates/spk-schema/src/template.rs b/crates/spk-schema/src/template.rs index 4eb967eaa8..a22e9e8360 100644 --- a/crates/spk-schema/src/template.rs +++ b/crates/spk-schema/src/template.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/imageworks/spk +use std::collections::HashMap; use std::path::Path; use crate::foundation::option_map::OptionMap; @@ -24,3 +25,44 @@ pub trait TemplateExt: Template { /// Load this template from a file on disk fn from_file(path: &Path) -> Result; } + +/// The structured data that should be made available +/// when rendering spk templates into recipes +#[derive(serde::Serialize, Debug, Clone)] +pub struct TemplateData { + /// Information about the release of spk being used + spk: SpkInfo, + /// The option values for this template, expanded + /// from an option map so that namespaced options + /// like `python.abi` actual live under the `python` + /// field rather than as a field with a '.' in the name + opt: serde_yaml::Mapping, + /// Environment variable data for the current process + env: HashMap, +} + +/// The structured data that should be made available +/// when rendering spk templates into recipes +#[derive(serde::Serialize, Debug, Clone)] +struct SpkInfo { + version: &'static str, +} + +impl Default for SpkInfo { + fn default() -> Self { + Self { + version: env!("CARGO_PKG_VERSION"), + } + } +} + +impl TemplateData { + /// Create the set of templating data for the current process and options + pub fn new(options: &OptionMap) -> Self { + TemplateData { + spk: SpkInfo::default(), + opt: options.to_yaml_value_expanded(), + env: std::env::vars().into_iter().collect(), + } + } +} diff --git a/packages/gnu/gcc/gcc48.spk.yaml b/packages/gnu/gcc/gcc48.spk.yaml index e5c10e38eb..9423cd5da0 100644 --- a/packages/gnu/gcc/gcc48.spk.yaml +++ b/packages/gnu/gcc/gcc48.spk.yaml @@ -1,4 +1,4 @@ -# {{ default version = "4.8.5" }} +# {{ assign version = opt.version | default: "4.8.5" }} pkg: gcc/{{ version }} api: v0/package diff --git a/packages/gnu/gcc/gcc63.spk.yaml b/packages/gnu/gcc/gcc63.spk.yaml index 5d02be43c8..90983e522b 100644 --- a/packages/gnu/gcc/gcc63.spk.yaml +++ b/packages/gnu/gcc/gcc63.spk.yaml @@ -1,4 +1,4 @@ -# {{ default version = "{{ version }}" }} +# {% assign version = opt.version | default: "6.3.3" %} pkg: gcc/{{ version }} api: v0/package diff --git a/packages/python/python2.spk.yaml b/packages/python/python2.spk.yaml index 5a6951d8ca..e6f0623271 100644 --- a/packages/python/python2.spk.yaml +++ b/packages/python/python2.spk.yaml @@ -1,5 +1,6 @@ -# {% default version = "2.7.5" %} -# {% assign cpXX = version | replace_re: "(\d+)\.(\d+).*", "cp$1$2" %} +# {% assign version = env.VV | default: "2.7.5" %} +# {% assign version = opt.version | default: "2.7.5" %} +# {% assign cpXX = opt.version | replace_re: "(\d+)\.(\d+).*", "cp$1$2" %} pkg: python/{{ version }} api: v0/package sources: diff --git a/packages/python/python3.spk.yaml b/packages/python/python3.spk.yaml index 5287f3734c..8d53e9611c 100644 --- a/packages/python/python3.spk.yaml +++ b/packages/python/python3.spk.yaml @@ -1,4 +1,4 @@ -# {% default version = "3.7.3" %} +# {% assign version = opt.version | default: "3.7.3" %} # {% assign cpXX = version | replace_re: "(\d+)\.(\d+).*", "cp$1$2" %} pkg: python/{{ version }} api: v0/package From 6f08a9742166b92b6a1706a3b012241c0276b877 Mon Sep 17 00:00:00 2001 From: Ryan Bottriell Date: Wed, 5 Oct 2022 17:14:47 -0700 Subject: [PATCH 08/12] Add a section to the documentation on templating Signed-off-by: Ryan Bottriell --- docs/usage.md | 100 ----------------------------------------------- docs/use/spec.md | 83 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 100 deletions(-) delete mode 100644 docs/usage.md diff --git a/docs/usage.md b/docs/usage.md deleted file mode 100644 index e0e774f92a..0000000000 --- a/docs/usage.md +++ /dev/null @@ -1,100 +0,0 @@ -# Advanced Usage - -Additional command line workflows for more advanced users. - -## Tag Streams - -When tags are created in spfs, they are added to what is known as a _tag stream_. Tag streams are simply a historical record of that tag over time, keeping track of each change, when it was made, and by whom. Previous versions of a tag can be referenced using a tilde, where the most recent version of a tag is version `~0`; the previous version is version `~1`; the version before that is version `~2` etc... This notation can be used everywhere a tag can be used. This means that `spfs run my-tag` is the same as `spfs run my-tag~0`. All the available versions of a tag can be viewed using the `spfs log ` command, where you will see this notation used. - -```bash -spfs shell - -echo 1 > /spfs/message.txt -spfs commit layer --tag my-layer - -spfs edit -echo 2 > /spfs/message.txt -spfs commit layer --tag my-layer - -spfs edit -echo 3 > /spfs/message.txt -spfs commit layer --tag my-layer - -spfs log my-layer -# 6E5CA5XL3L my-layer rbottriell@wolf0254.spimageworks.com 2020-03-18 10:12 -# 6E5CA5XL3L my-layer~1 rbottriell@wolf0254.spimageworks.com 2020-03-18 10:11 -# XHHVG3NDGE my-layer~2 rbottriell@wolf0254.spimageworks.com 2020-03-18 10:11 -``` - -### Reverting a Tag - -Using a tag stream, we can revert to previous versions of a tag by simply re-tagging the older version as the latest one. To continue from the example above: - -```bash -spfs tag my-layer~2 my-layer - -spfs log my-layer -# XHHVG3NDGE my-layer rbottriell@wolf0254.spimageworks.com 2020-03-18 10:16 -# 6E5CA5XL3L my-layer~1 rbottriell@wolf0254.spimageworks.com 2020-03-18 10:12 -# JJ3MEJOYQ2 my-layer~2 rbottriell@wolf0254.spimageworks.com 2020-03-18 10:11 -# XHHVG3NDGE my-layer~3 rbottriell@wolf0254.spimageworks.com 2020-03-18 10:11 -``` - -:point_right: | If you want to see or update shared tags, remember to specify the remote repository for each command (eg: `spfs log my-layer -r origin`) -:---: | :--- - -## Diff Tool - -Any two spfs file system states can be compared using the `spfs diff` command. With no arguments, this command works much like the `git status` command, showing the current set of active changes that have not been committed (if you are in an spfs runtime). - -## - -It's easy enough to pull and mount an spfs file tree, but sometimes it's not ideal to have to localize or sync the entire thing just to get a little bit of information or check the contents of a key file. SpFS provides 2 commands which allow for easy introspection of committed data without the need to enter into the environment itself. - -- `spfs ls` can be used to list directory contents of a stored file tree -- `spfs cat` can be used to output the contents of a file stored in spfs - -```bash -spfs shell -mkdir -p /spfs/bin -echo "I am root" > /spfs/root.txt -touch /spfs/bin/command -spfs commit layer --tag simple-fs -# exit the spfs runtime -exit - -spfs info simple-fs -# layer: -# refs: YJGTUV2Y -> simple-fs -# manifest: EDAWAZUS - -spfs ls simple-fs -# bin -# root.txt - -spfs ls simple-fs bin -# command - -spfs cat simple-fs root.txt -# I am root -``` - -## Repository Cleaning - -Over time, an spfs repository can get quite large, as it retains data from long ago that may not be used anymore as well as containing data for committed platforms, layers and blobs that are not referenced in any tag. The `spfs clean` command can be used to remove such data, as well as to find and remove old data for past tag history which is no longer desired. By default, the clean command will only find and print information about things that would be removed, and must be explicitly told to delete data. - -:warning: | These commands can and will remove data, and should be used with great caution. -:---: | :--- - -```bash -spfs clean --help -``` - -Objects are considered to be attached, and unremovable if they are reachable from any version of any tag in the repository. The `--prune` flag and related options can be used to get rid of older tag versions based on age or number of versions before cleaning the repository. This is a good way to try and disconnect additional objects, create more data that can be cleaned. - -:point_right: | The pruning process will always prefer keeping a tag version over removing it when multiple keep/prune conditions apply to it. Check the default values for each setting if you expected more tags than were shown. -:---: | :--- - -## Temporary Filesystem Size - -The spfs runtime uses a temporary, in-memory filesystem, which means that large sets of changes can run out of space because of RAM limitations. The size of this filesystem can be overridden using the `SPFS_FILESYSTEM_TMPFS_SIZE` variable (eg `SPFS_FILESYSTEM_TMPFS_SIZE=10G`). Note that specifying values close to or larger than the available memory on the system may cause deadlocks or system instability. diff --git a/docs/use/spec.md b/docs/use/spec.md index 8e3c60af7b..435b303ce9 100644 --- a/docs/use/spec.md +++ b/docs/use/spec.md @@ -415,3 +415,86 @@ tests: script: - pytest ``` + +### Spec File Templating + +SPK package spec files also support the [liquid](https://shopify.github.io/liquid/) templating language, so long as the spec file remains valid yaml. + +The templating is rendered when the yaml file is read from disk, and before it's processed any farther (to start a build, run tests, etc.). This means that it cannot, for example, be involved in rendering different specs for different variants of the package (unless you define and orchestrate those variants through a separate build system). + +The data that's made available to the template takes the form: + +```yaml +spk: + version: "0.23.0" # the version of spk being used +opt: {} # a map of all build options specified (either host options or at the command line) +env: {} # a map of the current environment variables from the caller +``` + +One common templating use case is to allow your package spec to be reused to build many different versions, for example: + +```yaml +# {% default opt.version = "2.3.4" %} +pkg: my-package/{{ version }} +``` + +Which could then be invoked for different versions at the command line: + +```sh +spk build my-package.spk.yaml # builds the default 2.3.4 +spk build my-package.spk.yaml -o version=2.4.0 # builds 2.4.0 +``` + +#### Template Extensions + +In addition to the default tags and filters within the liquid language, spk provides a few additional ones to help package maintainers: + +##### Tags + +**default** + +The `default` tag can be used to more easily declare the default value for a variable. The following two statements are equivalent: + +```liquid +{% assign version = version | default: "2.3.4" %} +{% default version = "2.3.4" %} +``` + +##### Filters + +**compare_version** + +The `compare_version` allows for comparing spk versions using any of the [version comparison operators](/use/versioning). It takes one or two parameters, depending on the data that you have to give. In all cases, the parameters are concatenated together and parsed as a version range. For example, the following assignments to py_3 all end up checking the same statement. + +```liquid +{% assign is_py3 = python.version | compare_version: ">=3" %} +{% assign is_py3 = python.version | compare_version: ">=", 3 %} +{% assign three = 3 %} +{% assign is_py3 = python.version | compare_version: ">=", three %} +``` + +**parse_version** + +The `parse_version` filter breaks down an spk version into it's components, either returning an object or a single field from it, for example: + +```liquid +{% assign v = "1.2.3.4-alpha.0+r.4" | parse_version %} +{{ v.base }} # 1.2.3.4 +{{ v.major }} # 1 +{{ v.minor }} # 2 +{{ v.patch }} # 3 +{{ v.parts[3] }} # 4 +{{ v.post.r }} # 4 +{{ v.pre.alpha }} # 0 +{{ "1.2.3.4-alpha.0+r.4" | parse_version: minor }} # 2 +``` + +**replace_re** + +The `replace_re` filter works like the built-in `replace` filter, except that it matches using a perl-style regular expression and allows group replacement in the output. These regular expressions do not support look-arounds or back-references. For example: + +```liquid +{% default version = "2.3.4" %} +{% assign major_minor = version | replace_re: "(\d+)\.(\d+).*", "$1.$2" %} +{{ major_minor }} # 2.3 +``` From beb29e0a036b7e2e80aa49898a601639637a18be Mon Sep 17 00:00:00 2001 From: Ryan Bottriell Date: Tue, 15 Nov 2022 12:38:03 -0800 Subject: [PATCH 09/12] Fix failing schema tests after change to complex data structure Signed-off-by: Ryan Bottriell --- crates/spk-schema/src/spec_test.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/spk-schema/src/spec_test.rs b/crates/spk-schema/src/spec_test.rs index 433b44af2e..4ff4df60c6 100644 --- a/crates/spk-schema/src/spec_test.rs +++ b/crates/spk-schema/src/spec_test.rs @@ -311,9 +311,9 @@ fn test_get_build_requirements_pkg_in_variant_preserves_order() { #[rstest] fn test_template_error_message() { format_serde_error::never_color(); - static SPEC: &str = r#"pkg: mypackage/{{ version }} + static SPEC: &str = r#"pkg: my-package/{{ opt.version }} sources: - - git: https://downloads.testing/mypackage/v{{ verison }} + - git: https://downloads.testing/my-package/v{{ opt.typo }} "#; let tpl = SpecTemplate { name: PkgName::new("my-package").unwrap().to_owned(), @@ -325,9 +325,11 @@ sources: .render(&options) .expect_err("expect template rendering to fail"); let expected = r#" -liquid: Unknown variable +liquid: Unknown index with: - requested variable=verison + variable=opt + requested index=typo + available indexes=version "#; let message = err.to_string(); assert_eq!(message.trim(), expected.trim()); @@ -342,7 +344,7 @@ fn test_template_namespace_options() { // dot-notation built into templating format_serde_error::never_color(); - static SPEC: &str = r#"pkg: mypackage/{{ namespace.version }}"#; + static SPEC: &str = r#"pkg: mypackage/{{ opt.namespace.version }}"#; let tpl = SpecTemplate { name: PkgName::new("my-package").unwrap().to_owned(), file_path: "my-package.spk.yaml".into(), From c2c2dd8bfdc7112d519c1f9fa8785cfc34705ef0 Mon Sep 17 00:00:00 2001 From: Ryan Bottriell Date: Fri, 18 Nov 2022 21:05:47 -0800 Subject: [PATCH 10/12] Improve grammar and fix typos Signed-off-by: Ryan Bottriell Signed-off-by: Ryan Bottriell --- crates/spk-cli/common/src/flags.rs | 8 ++++---- crates/spk-schema/crates/liquid/src/tag_default.rs | 2 +- crates/spk-schema/src/option_test.rs | 2 +- docs/use/spec.md | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/spk-cli/common/src/flags.rs b/crates/spk-cli/common/src/flags.rs index 5eade8dc01..b94413260e 100644 --- a/crates/spk-cli/common/src/flags.rs +++ b/crates/spk-cli/common/src/flags.rs @@ -211,7 +211,7 @@ pub struct Options { /// or json format (--opt '{name: value, other: value}'). /// /// Options can also be given in a file via the --options-file/-f flag. If - /// given, --opt will supercede anything in the options file(s). + /// given, --opt will supersede anything in the options file(s). #[clap(long = "opt", short)] pub options: Vec, @@ -232,10 +232,10 @@ impl Options { }; for filename in self.options_file.iter() { - let reader = - std::fs::File::open(filename).context(format!("Failed to open: {filename:?}"))?; + let reader = std::fs::File::open(filename) + .with_context(|| format!("Failed to open: {filename:?}"))?; let options: OptionMap = serde_yaml::from_reader(reader) - .context(format!("Failed to parse as option mapping: {filename:?}"))?; + .with_context(|| format!("Failed to parse as option mapping: {filename:?}"))?; opts.extend(options); } diff --git a/crates/spk-schema/crates/liquid/src/tag_default.rs b/crates/spk-schema/crates/liquid/src/tag_default.rs index 887d578239..84cca9fced 100644 --- a/crates/spk-schema/crates/liquid/src/tag_default.rs +++ b/crates/spk-schema/crates/liquid/src/tag_default.rs @@ -25,7 +25,7 @@ impl TagReflection for DefaultTag { } fn description(&self) -> &'static str { - "assign a variable value if it doesn't already have one" + "assign a variable a value if it doesn't already have one" } } diff --git a/crates/spk-schema/src/option_test.rs b/crates/spk-schema/src/option_test.rs index 7dc7ff5c3d..0b915d80ef 100644 --- a/crates/spk-schema/src/option_test.rs +++ b/crates/spk-schema/src/option_test.rs @@ -41,7 +41,7 @@ fn test_var_opt_validation(#[case] spec: &str, #[case] value: &str, #[case] expe #[case("{var: my-var}", None)] #[case("{var: my-var/}", None)] // empty is mapped to none #[case("{static: static, var: my-var}", Some("static"))] // static instead of default -#[case("{static: static, var: my-var/default}", Some("static"))] // static supercedes default +#[case("{static: static, var: my-var/default}", Some("static"))] // static supersedes default fn test_var_opt_parse_value(#[case] spec: &str, #[case] expected: Option<&str>) { let opt = Opt::from_yaml(spec).unwrap().into_var().unwrap(); let actual = opt.get_value(None); diff --git a/docs/use/spec.md b/docs/use/spec.md index 435b303ce9..dd85d789c6 100644 --- a/docs/use/spec.md +++ b/docs/use/spec.md @@ -420,7 +420,7 @@ tests: SPK package spec files also support the [liquid](https://shopify.github.io/liquid/) templating language, so long as the spec file remains valid yaml. -The templating is rendered when the yaml file is read from disk, and before it's processed any farther (to start a build, run tests, etc.). This means that it cannot, for example, be involved in rendering different specs for different variants of the package (unless you define and orchestrate those variants through a separate build system). +The templating is rendered when the yaml file is read from disk, and before it's processed any further (to start a build, run tests, etc.). This means that it cannot, for example, be involved in rendering different specs for different variants of the package (unless you define and orchestrate those variants through a separate build system). The data that's made available to the template takes the form: @@ -464,7 +464,7 @@ The `default` tag can be used to more easily declare the default value for a var **compare_version** -The `compare_version` allows for comparing spk versions using any of the [version comparison operators](/use/versioning). It takes one or two parameters, depending on the data that you have to give. In all cases, the parameters are concatenated together and parsed as a version range. For example, the following assignments to py_3 all end up checking the same statement. +The `compare_version` allows for comparing spk versions using any of the [version comparison operators](/use/versioning). It takes one or two arguments, depending on the data that you have to give. In all cases, the arguments are concatenated together and parsed as a version range. For example, the following assignments to py_3 all end up checking the same statement. ```liquid {% assign is_py3 = python.version | compare_version: ">=3" %} @@ -475,7 +475,7 @@ The `compare_version` allows for comparing spk versions using any of the [versio **parse_version** -The `parse_version` filter breaks down an spk version into it's components, either returning an object or a single field from it, for example: +The `parse_version` filter breaks down an spk version into its components, either returning an object or a single field from it, for example: ```liquid {% assign v = "1.2.3.4-alpha.0+r.4" | parse_version %} From 61a891aa93272e780aa9bea18c7cb424f3637049 Mon Sep 17 00:00:00 2001 From: Ryan Bottriell Date: Sat, 8 Apr 2023 20:51:39 -0700 Subject: [PATCH 11/12] Update to branch with liquid bugfix Signed-off-by: Ryan Bottriell Signed-off-by: Ryan Bottriell --- Cargo.lock | 20 ++++++++------------ crates/spk-schema/crates/liquid/Cargo.toml | 12 ++++++++++-- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e9833d851b..0c5d5ce8cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1436,9 +1436,8 @@ checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" [[package]] name = "liquid" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f55b9db2305857de3b3ceaa0e75cb51a76aaec793875fe152e139cb8fed05c" +version = "0.26.1" +source = "git+https://github.com/rydrman/liquid-rust?branch=allow-nil-filter-entry#d258842cbe2f9a463e566be24a0eb9a21fb7c739" dependencies = [ "doc-comment", "liquid-core", @@ -1449,9 +1448,8 @@ dependencies = [ [[package]] name = "liquid-core" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a93764837aeac37f14b74708cd88a44d82edfa9ad2b1bcd9a3b4d8802fdd9f98" +version = "0.26.1" +source = "git+https://github.com/rydrman/liquid-rust?branch=allow-nil-filter-entry#d258842cbe2f9a463e566be24a0eb9a21fb7c739" dependencies = [ "anymap2", "itertools", @@ -1467,9 +1465,8 @@ dependencies = [ [[package]] name = "liquid-derive" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "926454345f103e8433833077acdbfaa7c3e4b90788d585a8358f02f0b8f5a469" +version = "0.26.1" +source = "git+https://github.com/rydrman/liquid-rust?branch=allow-nil-filter-entry#d258842cbe2f9a463e566be24a0eb9a21fb7c739" dependencies = [ "proc-macro2", "proc-quote", @@ -1478,9 +1475,8 @@ dependencies = [ [[package]] name = "liquid-lib" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd06ca30ae026d26ee7fa8596f9590959e2d3726bc5a0f16a21ac4f050ec83c0" +version = "0.26.1" +source = "git+https://github.com/rydrman/liquid-rust?branch=allow-nil-filter-entry#d258842cbe2f9a463e566be24a0eb9a21fb7c739" dependencies = [ "itertools", "liquid-core", diff --git a/crates/spk-schema/crates/liquid/Cargo.toml b/crates/spk-schema/crates/liquid/Cargo.toml index a72760a9ae..e353889749 100644 --- a/crates/spk-schema/crates/liquid/Cargo.toml +++ b/crates/spk-schema/crates/liquid/Cargo.toml @@ -9,8 +9,6 @@ migration-to-components = ["spk-schema-foundation/migration-to-components"] [dependencies] lazy_static = "1.4" -liquid = "0.26.0" -liquid-core = "0.26.0" format_serde_error = {version = "0.3", default_features = false, features = ["colored"]} regex = "1.6.0" serde = "1.0" @@ -18,5 +16,15 @@ serde_json = "1.0" spk-schema-foundation = { path = "../foundation" } tracing = "0.1.35" +# this branch is needed until it or a similar solution +# is accepted to solve the broken use of the default filter +# eg: {{ does.not.exit | default: value }} +[dependencies.liquid] +git = "https://github.com/rydrman/liquid-rust" +branch = "allow-nil-filter-entry" +[dependencies.liquid-core] +git = "https://github.com/rydrman/liquid-rust" +branch = "allow-nil-filter-entry" + [dev-dependencies] rstest = "0.15.0" From 3b10510dff3ccd2b894e137ade652b0747666d30 Mon Sep 17 00:00:00 2001 From: Ryan Bottriell Date: Sat, 8 Apr 2023 21:14:19 -0700 Subject: [PATCH 12/12] Allow default tag to assign nested values Signed-off-by: Ryan Bottriell Signed-off-by: Ryan Bottriell --- .../crates/liquid/src/error_test.rs | 2 +- .../crates/liquid/src/tag_default.rs | 69 ++++++++++++++++--- .../crates/liquid/src/tag_default_test.rs | 54 +++++++++++++++ crates/spk-schema/src/template.rs | 2 +- docs/use/spec.md | 12 +++- packages/gnu/gcc/gcc48.spk.yaml | 2 +- 6 files changed, 125 insertions(+), 16 deletions(-) diff --git a/crates/spk-schema/crates/liquid/src/error_test.rs b/crates/spk-schema/crates/liquid/src/error_test.rs index 9be829c9ba..7f02eb296f 100644 --- a/crates/spk-schema/crates/liquid/src/error_test.rs +++ b/crates/spk-schema/crates/liquid/src/error_test.rs @@ -12,7 +12,7 @@ fn test_error_position_extraction() { crate::render_template(TPL, &json!({})).expect_err("expected template render to fail"); let expected = r#" 1 | {% default = data | replace ''%} - | ^ unexpected "="; expected Identifier + | ^ unexpected "="; expected Variable "#; let message = err.to_string(); assert_eq!(message, expected); diff --git a/crates/spk-schema/crates/liquid/src/tag_default.rs b/crates/spk-schema/crates/liquid/src/tag_default.rs index 84cca9fced..5d0b713a50 100644 --- a/crates/spk-schema/crates/liquid/src/tag_default.rs +++ b/crates/spk-schema/crates/liquid/src/tag_default.rs @@ -2,9 +2,21 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/imageworks/spk +use liquid::model::map::Entry; +use liquid::{Object, ValueView}; use liquid_core::error::ResultLiquidExt; use liquid_core::parser::FilterChain; -use liquid_core::{Language, ParseTag, Renderable, Result, Runtime, TagReflection, TagTokenIter}; +use liquid_core::runtime::Variable; +use liquid_core::{ + Language, + ParseTag, + Renderable, + Result, + Runtime, + TagReflection, + TagTokenIter, + Value, +}; #[cfg(test)] #[path = "./tag_default_test.rs"] @@ -36,11 +48,9 @@ impl ParseTag for DefaultTag { options: &Language, ) -> Result> { let dst = arguments - .expect_next("Identifier expected.")? - .expect_identifier() - .into_result()? - .to_string() - .into(); + .expect_next("Variable expected.")? + .expect_variable() + .into_result()?; arguments .expect_next("Assignment operator \"=\" expected.")? @@ -65,7 +75,7 @@ impl ParseTag for DefaultTag { #[derive(Debug)] struct Default { - dst: liquid_core::model::KString, + dst: Variable, src: FilterChain, } @@ -83,10 +93,49 @@ impl Renderable for Default { .trace_with(|| self.trace().into())? .into_owned(); - let name = self.dst.as_str().into(); - if runtime.try_get(&[name]).is_none() { - runtime.set_global(self.dst.clone(), value); + let variable = self.dst.evaluate(runtime)?; + let mut path = variable.iter().collect::>().into_iter(); + let root = path.next().expect("at least one entry in path"); + + let mut current_pos = liquid::model::Path::with_index(root.clone()); + let type_err = |pos: &liquid::model::Path| { + liquid::Error::with_msg("Cannot set default") + .trace("Stepping into non-object") + .context("position", pos.to_string()) + .context("target", self.dst.to_string()) + }; + + let Some(last) = path.next_back() else { + if runtime.get(&[root.clone()]).is_err() { + runtime.set_global(root.to_kstr().into(), value); + } + return Ok(()); + }; + let mut data = runtime + .get(&[root.to_owned()]) + .map(|v| v.into_owned()) + .unwrap_or_else(|_| Value::Object(Object::new())); + let mut data_ref = &mut data; + for step in path { + data_ref = data_ref + .as_object_mut() + .ok_or_else(|| type_err(¤t_pos))? + .entry(step.to_kstr()) + .or_insert_with(|| Value::Object(Object::new())); + current_pos.push(step.to_owned()); } + match data_ref + .as_object_mut() + .ok_or_else(|| type_err(¤t_pos))? + .entry(last.to_kstr()) + { + Entry::Occupied(_) => {} + Entry::Vacant(v) => { + v.insert(value); + runtime.set_global(root.to_kstr().into(), data); + } + } + Ok(()) } } diff --git a/crates/spk-schema/crates/liquid/src/tag_default_test.rs b/crates/spk-schema/crates/liquid/src/tag_default_test.rs index 3d6eedfc7b..d2b910c732 100644 --- a/crates/spk-schema/crates/liquid/src/tag_default_test.rs +++ b/crates/spk-schema/crates/liquid/src/tag_default_test.rs @@ -55,3 +55,57 @@ sources: crate::render_template(TPL, &options).expect("template should not fail to render"); assert_eq!(rendered, EXPECTED); } + +#[rstest] +fn test_template_rendering_default_nested() { + // ensure that setting a nested default value + // works as expected + + let options = json!({ + "nested": { + "existing": "existing" + } + }); + static TPL: &str = r#" +{% default nested.existing = "ignored" %} +{% default nested.other = "something" %} +pkg: {{ nested.other }}-{{ nested.existing }} +"#; + static EXPECTED: &str = r#" + + +pkg: something-existing +"#; + let rendered = match crate::render_template(TPL, &options) { + Ok(r) => r, + Err(err) => { + println!("{err}"); + panic!("template should not fail to render"); + } + }; + assert_eq!(rendered, EXPECTED); +} + +#[rstest] +fn test_template_rendering_default_nested_not_object() { + // ensure that setting a nested default value + // works as expected + + let options = json!({ + "integer": 64, + }); + static TPL: &str = r#" +{% default integer.nested = "invalid" %} +"#; + let err = crate::render_template(TPL, &options) + .expect_err("Should fail when setting default under non-object"); + let expected = r#"liquid: Cannot set default +from: Stepping into non-object + with: + position=integer + target=integer["nested"] + +"#; + let message = err.to_string(); + assert_eq!(message, expected); +} diff --git a/crates/spk-schema/src/template.rs b/crates/spk-schema/src/template.rs index a22e9e8360..f8ac2956f6 100644 --- a/crates/spk-schema/src/template.rs +++ b/crates/spk-schema/src/template.rs @@ -62,7 +62,7 @@ impl TemplateData { TemplateData { spk: SpkInfo::default(), opt: options.to_yaml_value_expanded(), - env: std::env::vars().into_iter().collect(), + env: std::env::vars().collect(), } } } diff --git a/docs/use/spec.md b/docs/use/spec.md index dd85d789c6..a8f56897c6 100644 --- a/docs/use/spec.md +++ b/docs/use/spec.md @@ -456,8 +456,14 @@ In addition to the default tags and filters within the liquid language, spk prov The `default` tag can be used to more easily declare the default value for a variable. The following two statements are equivalent: ```liquid -{% assign version = version | default: "2.3.4" %} -{% default version = "2.3.4" %} +{% assign var = var | default: "2.3.4" %} +{% default var = "2.3.4" %} +``` + +Additionally, this tag can be used to set defaults in nested structures. Often, for options that may be provided at the command line. + +```liquid +{% default opt.version = "2.3.4" %} ``` ##### Filters @@ -494,7 +500,7 @@ The `parse_version` filter breaks down an spk version into its components, eithe The `replace_re` filter works like the built-in `replace` filter, except that it matches using a perl-style regular expression and allows group replacement in the output. These regular expressions do not support look-arounds or back-references. For example: ```liquid -{% default version = "2.3.4" %} +{% assign version = opt.version | default: "2.3.4" %} {% assign major_minor = version | replace_re: "(\d+)\.(\d+).*", "$1.$2" %} {{ major_minor }} # 2.3 ``` diff --git a/packages/gnu/gcc/gcc48.spk.yaml b/packages/gnu/gcc/gcc48.spk.yaml index 9423cd5da0..21fcd07645 100644 --- a/packages/gnu/gcc/gcc48.spk.yaml +++ b/packages/gnu/gcc/gcc48.spk.yaml @@ -1,4 +1,4 @@ -# {{ assign version = opt.version | default: "4.8.5" }} +# {% assign version = opt.version | default: "4.8.5" %} pkg: gcc/{{ version }} api: v0/package