diff --git a/Cargo.lock b/Cargo.lock index d569933..a76dc7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2951,6 +2951,7 @@ dependencies = [ "if-addrs 0.10.1", "log", "mdns-sd", + "regex", "reqwest", "serde", "serde_json", diff --git a/cli/src/main.rs b/cli/src/main.rs index 96bfb13..5211d4f 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -14,6 +14,7 @@ use humansize::{format_size, BINARY}; use owo_colors::OwoColorize; use prettytable::{row, Table}; use tokio::runtime::Runtime; +use utils::registry::{download_module, gen_manifest, PackageRegistry, PackageSpec}; use utils::structs::Manifest; use uuid::Uuid; @@ -70,6 +71,13 @@ pub enum Command { History, /// Liveness check: ping at least one node on the mesh. Ping, + /// Highly experimental: Pull a package from WAPM.io and store it in Serval Mesh + Pull { + /// The name of the software package, formatted as + /// [[protocol://]registry.tld/]author/packagename[.module][@version] + // TODO: parse directly via clap, see https://github.com/serval/serval-mesh/pull/43/files#r1139430044 + identifer: String, + }, } static SERVAL_NODE_URL: Mutex> = Mutex::new(None); @@ -142,7 +150,6 @@ fn upload_manifest(manifest_path: PathBuf) -> Result<()> { ]); } else { table.add_row(row!["Storing the WASM executable failed!"]); - table.add_row(row![format!("{} {}", response.status(), response.text()?)]); } println!("{table}"); @@ -289,6 +296,115 @@ fn blocking_maybe_discover_service_url( Ok(format!("http://{addr}:{port}")) } +/// Pull a Wasm package from a package manager, generate its manifest, and store it. +fn pull(identifier: String) -> Result<()> { + let Ok(pkg_spec) = PackageSpec::try_from(identifier.clone()) else { + return Err(anyhow!(format!( + "Failed to parse identifier `{}` into a package specification", + identifier.bold().blue() + ))); + }; + dbg!(&pkg_spec); + println!( + "📦 Identified package {}", + pkg_spec.profile_url().bold().blue() + ); + println!("🏷 Using module {}", pkg_spec.module.bold().blue()); + if pkg_spec.is_binary_cached() { + println!( + "✅ Binary for {} ({}) available locally.", + pkg_spec.fq_name().bold().green(), + pkg_spec.fq_digest() + ); + // Creating a temporary manifest file + let Ok(manifest_path) = gen_manifest(&pkg_spec) else { + return Err(anyhow!(format!( + "Failed to parse package specification `{}` into a manifest", + pkg_spec.fq_name().bold().blue() + ))); + }; + // Handing over to existing storage logic + upload_manifest(manifest_path)?; + } else { + println!( + "⌛️ Binary for {} not available locally, downloading...", + pkg_spec.fq_name().bold().blue() + ); + let mod_dl = download_module(&pkg_spec); + match mod_dl { + // This means the download function did not break. It does not mean that + // the executable was downloaded successfully... check HTTP status code. + Ok(status_code) => { + if status_code.is_success() { + println!( + "✅ Downloaded {} ({}) successfully.", + pkg_spec.fq_name().bold().green(), + pkg_spec.fq_digest() + ); + // Creating a temporary manifest file + let Ok(manifest_path) = gen_manifest(&pkg_spec) else { + return Err(anyhow!(format!( + "Failed to create a temporafy manifest for `{}`", + pkg_spec.fq_name().bold().blue() + ))); + }; + // Handing over to existing storage logic + upload_manifest(manifest_path)?; + } else if status_code.is_server_error() { + println!("🛑 Server error: {}", status_code); + println!(" There may be an issue with this package manager."); + } else if status_code.is_client_error() { + println!("🛑 Client error: {}", status_code); + if status_code == 404 { + println!(" Failed to download from {:?}", pkg_spec.download_urls()); + } + println!(); + // wapm.io does not aliast the "latest" tag, so the download will fail if latest is used. + // TODO: retrieve the latest version from wapm.io and insert it explicitly before + // downloading when the user specifies "latest". + if pkg_spec.version == "latest" && pkg_spec.registry == PackageRegistry::Wapm { + println!( + "💡 Please note that wapm.io does not properly alias the `{}` version tag.", + "latest".bold().yellow() + ); + println!(" You might want to look up the package and explicitly provide its most recent version:"); + println!(" \t{}", pkg_spec.profile_url()); + println!(); + } + // Currently, a 404 is very likely if a package only contains modules that have names other than + // the package name (which the module name defaults to if not provided). + // TODO: retrieve available modules and interactively ask which module should be downloaded + // Quick fix is to point this out to the user: + if pkg_spec.name == pkg_spec.module { + println!( + "💡 Please verify that this package actually contains a `{}` module", + pkg_spec.module.bold().yellow() + ); + println!(" by checking the MODULES section on its profile page:"); + println!(" \t{}", pkg_spec.profile_url()); + println!( + " If the module name differs from the package name, you can provide it as follows:" + ); + println!( + " \tcargo run -p serval -- pull {}/{}/{}.{}@{}", + pkg_spec.registry.domain(), + pkg_spec.author, + pkg_spec.name, + "".bold().yellow(), + pkg_spec.version, + ); + } + } else { + println!("😵‍💫 Something else happened. Status: {:?}", status_code); + } + } + // Something went horribly wrong. + Err(err) => println!("{:#?}", err), + } + } + Ok(()) +} + /// Parse command-line arguments and act. fn main() -> Result<()> { let args = Args::parse(); @@ -320,6 +436,7 @@ fn main() -> Result<()> { Command::Status { id } => status(id)?, Command::History => history()?, Command::Ping => ping()?, + Command::Pull { identifer } => pull(identifer)?, }; Ok(()) diff --git a/examples/serval-facts.toml b/examples/serval-facts.toml new file mode 100644 index 0000000..e627a59 --- /dev/null +++ b/examples/serval-facts.toml @@ -0,0 +1,6 @@ +name = "serval-facts" +namespace = "sh.serval" +version = "0.1" +binary = "serval-facts.wasm" +description = "Enjoy a selection of serval facts. Brought to you by https://github.com/serval/wasm-samples/tree/main/serval-facts" +required_extensions = [] \ No newline at end of file diff --git a/examples/serval-facts.wasm b/examples/serval-facts.wasm new file mode 100644 index 0000000..7add40a Binary files /dev/null and b/examples/serval-facts.wasm differ diff --git a/utils/Cargo.toml b/utils/Cargo.toml index 9c8dd41..23669a2 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -25,3 +25,5 @@ tokio = { workspace = true } toml = "0.7.0" uuid = { workspace = true } wasi-common = { workspace = true } +regex = "1.7.1" +sha256 = "1.1.2" diff --git a/utils/src/errors.rs b/utils/src/errors.rs index 2cff06f..062d0a1 100644 --- a/utils/src/errors.rs +++ b/utils/src/errors.rs @@ -70,6 +70,18 @@ pub enum ServalError { /// Translation for errors from ssri. #[error("ssri::Error: {0}")] SsriError(#[from] ssri::Error), + + /// The Package Registry is unknown. + #[error("unknown package registry`{0}`")] + PackageRegistryUnknownError(String), + + /// The Package Registry Manifest could not be constructed. + #[error("failed to parse registry manifest `{0}`")] + PackageRegistryManifestError(String), + + /// The Package Registry Manifest could not be constructed. + #[error("failed to download module from registry: {0}")] + PackageRegistryDownloadError(String), } use axum::http::StatusCode; diff --git a/utils/src/lib.rs b/utils/src/lib.rs index df36417..b1005fc 100644 --- a/utils/src/lib.rs +++ b/utils/src/lib.rs @@ -3,4 +3,5 @@ pub mod errors; pub mod futures; pub mod mdns; pub mod networking; +pub mod registry; pub mod structs; diff --git a/utils/src/registry.rs b/utils/src/registry.rs new file mode 100644 index 0000000..bf98b28 --- /dev/null +++ b/utils/src/registry.rs @@ -0,0 +1,322 @@ +//! Registry Module +//! +//! Serval supports downloading WebAssembly executables from package registries. +//! Downloaded packages are automatically stored to the Serval Mesh and can be +//! run just like any manually stored WebAssembly executable. + +use std::{fs::File, io::Write, path::PathBuf, str::FromStr, time::Duration}; + +use regex::Regex; +use reqwest::{blocking::Client, StatusCode}; +use sha256::digest; + +use crate::{errors::ServalError, structs::Manifest}; + +/// Package registry information, used to download executables and construct the Manifest. +#[derive(Debug, PartialEq, Clone)] +pub enum PackageRegistry { + Wapm, + Warg, +} + +impl FromStr for PackageRegistry { + type Err = ServalError; + + fn from_str(input: &str) -> Result { + match input { + "wapm.io" => Ok(PackageRegistry::Wapm), + "warg" => Ok(PackageRegistry::Warg), + _ => Err(ServalError::PackageRegistryUnknownError(input.to_string())), + } + } +} + +impl PackageRegistry { + pub fn namespace(&self) -> &str { + match self { + PackageRegistry::Wapm => "io.wapm", + PackageRegistry::Warg => "io.warg", + } + } + + pub fn domain(&self) -> &str { + match self { + PackageRegistry::Wapm => "wapm.io", + PackageRegistry::Warg => "warg.io", + } + } + + fn profile_url(&self, pkg: &PackageSpec) -> String { + match self { + PackageRegistry::Wapm => { + format!( + "https://wapm.io/{}/{}@{}", + pkg.author, pkg.name, pkg.version + ) + } + PackageRegistry::Warg => todo!(), + } + } + + fn fq_name(&self, pkg: &PackageSpec) -> String { + match self { + PackageRegistry::Wapm => { + format!( + "{}.{}.{}.{}@{}", + self.namespace(), + pkg.author, + pkg.name, + pkg.module, + pkg.version, + ) + } + PackageRegistry::Warg => todo!(), + } + } + + fn download_urls(&self, pkg: &PackageSpec) -> Vec { + match self { + PackageRegistry::Wapm => { + vec![ + // For some very stupid reason, wasm binaries can sit in multiple locations. Hopefully this is the full list: + format!("https://registry-cdn.wapm.io/contents/{}/{}/{}/{}.wasm", pkg.author, pkg.name, pkg.version, pkg.module), + format!("https://registry-cdn.wapm.io/contents/{}/{}/{}/target/wasm32-wasi/release/{}.wasm", pkg.author, pkg.name, pkg.version, pkg.module) + ] + } + PackageRegistry::Warg => todo!(), + } + } + // even cooler.... + //fn download(&self, pkg: &PackageSpec) -> Result { + // // do the work of downloading from this kind of registry + //} +} + +/// Specification for a registry package +#[derive(Debug, Clone, PartialEq)] +pub struct PackageSpec { + pub registry: PackageRegistry, + pub author: String, + pub name: String, + pub version: String, + pub module: String, +} + +impl PackageSpec { + pub fn profile_url(&self) -> String { + self.registry.profile_url(self) + } + + pub fn download_urls(&self) -> Vec { + self.registry.download_urls(self) + } + + pub fn fq_name(&self) -> String { + self.registry.fq_name(self) + } + + pub fn fq_digest(&self) -> String { + digest(self.fq_name()) + } + + pub fn namespace(&self) -> String { + format!("{}.{}", self.registry.namespace(), self.author) + } + + pub fn binary_path(&self) -> PathBuf { + PathBuf::from(format!("/tmp/{}.wasm", self.fq_digest())) + } + + pub fn manifest_path(&self) -> PathBuf { + PathBuf::from(format!("/tmp/{}.toml", self.fq_digest())) + } + + pub fn is_binary_cached(&self) -> bool { + self.binary_path().exists() + } +} + +/// Converts an identifier string to a `PackageSpec` +impl TryFrom for PackageSpec { + type Error = ServalError; + /** + This function matches a package specification string. + It supports a number of variants: + + Full URL to package in a supported registry: + ``` + # use utils::registry::PackageSpec; + let pkg_spec = PackageSpec::try_from(String::from("https://wapm.io/author/serval@version")).unwrap(); + # assert_eq!(pkg_spec, utils::registry::PackageSpec { + # registry: utils::registry::PackageRegistry::Wapm, + # author: "author".to_string(), + # name: "serval".to_string(), + # version: "version".to_string(), + # module: "serval".to_string(), + # }); + ``` + + Full URL to package in a supported registry, defaulting to latest version: + ``` + # use utils::registry::PackageSpec; + let pkg_spec = PackageSpec::try_from(String::from("https://wapm.io/author/tiger")).unwrap(); + # assert_eq!(pkg_spec, utils::registry::PackageSpec { + # registry: utils::registry::PackageRegistry::Wapm, + # author: "author".to_string(), + # name: "tiger".to_string(), + # version: "latest".to_string(), + # module: "tiger".to_string(), + # }); + ``` + + When providing a URL, the protocol is optional. This is also valid: + ``` + # use utils::registry::PackageSpec; + let pkg_spec = PackageSpec::try_from(String::from("wapm.io/author/lion@version")).unwrap(); + # assert_eq!(pkg_spec, utils::registry::PackageSpec { + # registry: utils::registry::PackageRegistry::Wapm, + # author: "author".to_string(), + # name: "lion".to_string(), + # version: "version".to_string(), + # module: "lion".to_string(), + # }); + # let pkg_spec = PackageSpec::try_from(String::from("wapm.io/author/cheetah")).unwrap(); + # assert_eq!(pkg_spec, utils::registry::PackageSpec { + # registry: utils::registry::PackageRegistry::Wapm, + # author: "author".to_string(), + # name: "cheetah".to_string(), + # version: "latest".to_string(), + # module: "cheetah".to_string(), + # }); + ``` + + When providing a simple author/package-style identifier, the default package + manager (currently [wapm.io](https://wapm.io) -- this will be made configurable) is used. + ``` + # use utils::registry::PackageSpec; + // provide specific version: + let pkg_spec = PackageSpec::try_from(String::from("author/panther@version")).unwrap(); + # assert_eq!(pkg_spec, utils::registry::PackageSpec { + # registry: utils::registry::PackageRegistry::Wapm, + # author: "author".to_string(), + # name: "panther".to_string(), + # version: "version".to_string(), + # module: "panther".to_string(), + # }); + // default to latest version: + let pkg_spec = PackageSpec::try_from(String::from("author/leopard")).unwrap(); + # assert_eq!(pkg_spec, utils::registry::PackageSpec { + # registry: utils::registry::PackageRegistry::Wapm, + # author: "author".to_string(), + # name: "leopard".to_string(), + # version: "latest".to_string(), + # module: "leopard".to_string(), + # }); + ``` + + In some cases, the actual Wasm module contained in a package has a different name than the + package, or a package may contain more than one module. + The package identifier defaults to a module name identical to the package name -- if a + different module should be used, it can be provided by appending it with a dot: + ``` + # use utils::registry::PackageSpec; + // provide specific version and module name: + let pkg_spec = PackageSpec::try_from(String::from("author/felis.catus@version")).unwrap(); + # assert_eq!(pkg_spec, utils::registry::PackageSpec { + # registry: utils::registry::PackageRegistry::Wapm, + # author: "author".to_string(), + # name: "felis".to_string(), + # version: "version".to_string(), + # module: "catus".to_string(), + # }); + // again, a missing version defaults to the latest version: + let pkg_spec = PackageSpec::try_from(String::from("author/felis.lybica")).unwrap(); + # assert_eq!(pkg_spec, utils::registry::PackageSpec { + # registry: utils::registry::PackageRegistry::Wapm, + # author: "author".to_string(), + # name: "felis".to_string(), + # version: "latest".to_string(), + # module: "lybica".to_string(), + # }); + ``` + */ + // TODO: The wapm.io package manager is currently the default package manager; this should be made configurable. + fn try_from(value: std::string::String) -> Result { + let re = Regex::new( + r"(?x) + (?:[a-z]+/{2})? # the protocol (optional, non-capturing) + (([a-z0-9.]+)(?:/))? # $1 package registry domain incl. trailing slash (optional, not used) + # $2 package registry domain w/o trailing slash (optional) + ([a-zA-Z0-9-]+) # $3 package author + (?:/) # slash (non-capturing) + ([a-zA-Z0-9-]+) # $4 package name + (?:(?:\.)([a-zA-Z0-9_]+))? # $5 module name (optional) + (?:(?:@)([a-zA-Z0-9.-]+))? # $6 package version (optional) + ", + ) + .unwrap(); + let cap = re.captures(&value).unwrap(); + // We attempt to extract the following capture groups: + // - the package registry domain without trailing slash ($2) + // - the package author ($3) + // - the package name ($4) + // - the package version ($6) + let (pkg_reg, pkg_auth, pkg_name, pkg_version) = ( + cap.get(2).map_or(PackageRegistry::Wapm, |m| { + PackageRegistry::from_str(m.as_str()).unwrap() + }), + String::from(cap.get(3).map(|m| m.as_str()).unwrap()), + String::from(cap.get(4).map(|m| m.as_str()).unwrap()), + String::from(cap.get(6).map_or("latest", |m| m.as_str())), + ); + // Collecting the module name ($5) separately as it needs to default to the package name + // if not provided. + let mod_name = cap + .get(5) + .map_or(pkg_name.clone(), |m| m.as_str().to_owned()); + Ok(PackageSpec { + author: pkg_auth, + name: pkg_name, + version: pkg_version, + registry: pkg_reg, + module: mod_name, + }) + } +} + +pub fn download_module(pkg_spec: &PackageSpec) -> Result { + let client = Client::builder() + .timeout(Duration::from_secs(360)) + .build() + .unwrap(); + let mut last_status: StatusCode = StatusCode::IM_A_TEAPOT; + for url in pkg_spec.download_urls() { + let response = client.get(url).send(); + match response { + Ok(r) => { + // println!("Ok: {:#?}", r); + let status = r.status(); + if r.status().is_success() { + let mut f = File::create(pkg_spec.binary_path())?; + f.write_all(&r.bytes().unwrap())?; + return Ok(status); + } else { + last_status = status; + } + } + _ => { + return Err(ServalError::PackageRegistryDownloadError( + "something went horribly wrong".to_string(), + )) + } + }; + } + Ok(last_status) +} + +pub fn gen_manifest(pkg_spec: &PackageSpec) -> Result { + let manifest = Manifest::from_packagespec(pkg_spec)?; + let mut f = File::create(pkg_spec.manifest_path())?; + f.write_all(toml::to_string(&manifest).unwrap().as_bytes())?; + Ok(pkg_spec.manifest_path()) +} diff --git a/utils/src/structs.rs b/utils/src/structs.rs index 964d812..247b82c 100644 --- a/utils/src/structs.rs +++ b/utils/src/structs.rs @@ -3,7 +3,7 @@ use std::{fmt::Display, path::PathBuf}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::errors::ServalError; +use crate::{errors::ServalError, registry::PackageSpec}; /// The results of running a WASM executable. #[derive(Debug)] @@ -45,6 +45,24 @@ impl Manifest { Ok(manifest) } + pub fn from_packagespec(pkg_spec: &PackageSpec) -> Result { + let mut name = pkg_spec.name.clone(); + // If the module name differs from the package name, surface the module name + // in the manifest to support installing multiple modules from the same package. + if pkg_spec.name != pkg_spec.module { + name = format!("{}.{}", name, pkg_spec.module); + } + let manifest = Manifest { + name, + namespace: pkg_spec.namespace(), + version: pkg_spec.version.clone(), + binary: pkg_spec.binary_path(), + description: pkg_spec.profile_url(), + required_extensions: Vec::new(), + }; + Ok(manifest) + } + pub fn binary(&self) -> &PathBuf { &self.binary }