From 3aed0281cab3e2b6965bff07a79b82b73aa218c8 Mon Sep 17 00:00:00 2001 From: bendn Date: Fri, 10 Mar 2023 11:37:54 +0700 Subject: [PATCH] clean everything up a bit (#27) --- Cargo.toml | 4 +- godot.lock | 25 +--- src/config_file.rs | 65 ++++---- src/main.rs | 84 ++++++----- src/package.rs | 330 +++++++++++++---------------------------- src/package/parsing.rs | 193 ++++++++++++++++++++++++ 6 files changed, 395 insertions(+), 306 deletions(-) create mode 100644 src/package/parsing.rs diff --git a/Cargo.toml b/Cargo.toml index 4455ee1..1465507 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "godot-package-manager" -version = "1.2.0" +version = "1.2.1" edition = "2021" authors = ["bendn "] description = "A package manager for godot" @@ -29,6 +29,8 @@ reqwest = { version = "0.11", features = [] } tokio = { version = "1", features = ["full"] } async-recursion = "1.0.2" futures = "0.3" +semver_rs = { version = "0.2", features = ["serde"] } +async-trait = "0.1.66" [dev-dependencies] glob = "0.3.0" diff --git a/godot.lock b/godot.lock index 895a832..bce530e 100644 --- a/godot.lock +++ b/godot.lock @@ -1,27 +1,14 @@ [ - { - "name": "@bendn/splitter", - "version": "1.0.6", - "integrity": "sha512-HT7q5qv6OEpX95e5r+kAsasoAvH0Mgf+aT4SdKQ18fyDIn1dW02WqbulF0AMwHufgRZkMf9SnQGiAq79P5ZIKQ==" - }, { "name": "@bendn/test", - "version": "2.0.10", - "integrity": "sha512-hyPGxDG8poa2ekmWr1BeTCUa7YaZYfhsN7jcLJ3q2cQVlowcTnzqmz4iV3t21QFyabE5R+rV+y6d5dAItrJeDw==" + "integrity": "sha512-hyPGxDG8poa2ekmWr1BeTCUa7YaZYfhsN7jcLJ3q2cQVlowcTnzqmz4iV3t21QFyabE5R+rV+y6d5dAItrJeDw==", + "tarball": "https://registry.npmjs.org/@bendn/test/-/test-2.0.10.tgz", + "version": "2.0.10" }, { "name": "@bendn/gdcli", - "version": "1.2.5", - "integrity": "sha512-/YOAd1+K4JlKvPTmpX8B7VWxGtFrxKq4R0A6u5qOaaVPK6uGsl4dGZaIHpxuqcurEcwPEOabkoShXKZaOXB0lw==" - }, - { - "name": "prettier", - "version": "2.8.4", - "integrity": "sha512-vIS4Rlc2FNh0BySk3Wkd6xmwxB0FpOndW5fisM5H8hsZSxU2VWVB5CWIkIjWvrHjIhxk2g3bfMKM87zNTrZddw==" - }, - { - "name": "stockfish", - "version": "15.0.0", - "integrity": "sha512-ze3vTgMrnCmMdtC8qpONBdXkSHWwv4Dx+GFV1V5TIi0qP1HF+Vwa4lArgqvHDooU8Mwfs1cgRzkW3SM8x/7TJg==" + "integrity": "sha512-/YOAd1+K4JlKvPTmpX8B7VWxGtFrxKq4R0A6u5qOaaVPK6uGsl4dGZaIHpxuqcurEcwPEOabkoShXKZaOXB0lw==", + "tarball": "https://registry.npmjs.org/@bendn/gdcli/-/gdcli-1.2.5.tgz", + "version": "1.2.5" } ] \ No newline at end of file diff --git a/src/config_file.rs b/src/config_file.rs index 7ddb1ad..db6c230 100644 --- a/src/config_file.rs +++ b/src/config_file.rs @@ -1,7 +1,8 @@ +use crate::package::parsing::IntoPackageList; use crate::package::Package; use anyhow::Result; use console::style; -use futures::stream::{self, StreamExt}; +use reqwest::Client; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -43,7 +44,7 @@ impl From for ParsedConfig { packages: from .packages .into_iter() - .map(|p| (p.name, p.version)) + .map(|p| (p.name, p.version.to_string())) .collect(), } } @@ -58,13 +59,12 @@ impl ParsedConfig { }) } - pub async fn into_configfile(self) -> ConfigFile { - let packages = stream::iter(self.packages.into_iter()) - .map(|(name, version)| async { Package::new(name, version).await.unwrap() }) - .buffer_unordered(crate::PARALLEL) - .collect::>() - .await; - ConfigFile { packages } + pub async fn into_configfile(self, client: Client) -> ConfigFile { + let mut packages = self.packages.into_package_list(client).await.unwrap(); + for mut p in &mut packages { + p.indirect = false + } + ConfigFile { packages: packages } } } @@ -80,7 +80,7 @@ impl ConfigFile { /// Creates a new [ConfigFile] from the given text /// Panics if the file cant be parsed as toml, hjson or yaml. - pub async fn new(contents: &String) -> Self { + pub async fn new(contents: &String, client: Client) -> Self { if contents.is_empty() { panic!("Empty CFG"); } @@ -88,16 +88,16 @@ impl ConfigFile { // definetly not going to backfire let mut cfg = if contents.as_bytes()[0] == b'{' { // json gets brute forced first so this isnt really needed - Self::parse(contents, ConfigType::JSON) + Self::parse(contents, ConfigType::JSON, client) .await .expect("Parsing CFG from JSON should work") } else if contents.len() > 3 && contents[..3] == *"---" { - Self::parse(contents, ConfigType::YAML) + Self::parse(contents, ConfigType::YAML, client) .await .expect("Parsing CFG from YAML should work") } else { for i in [ConfigType::JSON, ConfigType::YAML, ConfigType::TOML].into_iter() { - let res = Self::parse(contents, i).await; + let res = Self::parse(contents, i, client.clone()).await; // im sure theres some kind of idiomatic rust way to do this that i dont know of if res.is_ok() { @@ -118,16 +118,16 @@ impl ConfigFile { cfg } - pub async fn parse(txt: &str, t: ConfigType) -> Result { - Ok(ParsedConfig::parse(txt, t)?.into_configfile().await) + pub async fn parse(txt: &str, t: ConfigType, client: Client) -> Result { + Ok(ParsedConfig::parse(txt, t)?.into_configfile(client).await) } /// Creates a lockfile for this config file. /// note: Lockfiles are currently unused. - pub async fn lock(&mut self) -> String { + pub fn lock(&mut self) -> String { let mut pkgs = vec![]; - for mut p in self.collect() { - if p.is_installed() && p.get_manifest().await.is_ok() { + for p in self.collect() { + if p.is_installed() { pkgs.push(p) }; } @@ -140,7 +140,7 @@ impl ConfigFile { for p in pkgs { cb(p); if p.has_deps() { - inner(&mut p.dependencies, cb); + inner(&mut p.manifest.dependencies, cb); } } } @@ -168,10 +168,23 @@ mod tests { #[tokio::test] async fn parse() { let _t = crate::test_utils::mktemp(); + let c = Client::new(); let cfgs: [&mut ConfigFile; 3] = [ - &mut ConfigFile::new(&r#"dependencies: { "@bendn/test": 2.0.10 }"#.into()).await, // quoteless fails as a result of https://github.com/Canop/deser-hjson/issues/9 - &mut ConfigFile::new(&"dependencies:\n \"@bendn/test\": 2.0.10".into()).await, - &mut ConfigFile::new(&"[dependencies]\n\"@bendn/test\" = \"2.0.10\"".into()).await, + &mut ConfigFile::new( + &r#"dependencies: { "@bendn/test": 2.0.10 }"#.into(), + c.clone(), + ) + .await, // quoteless fails as a result of https://github.com/Canop/deser-hjson/issues/9 + &mut ConfigFile::new( + &"dependencies:\n \"@bendn/test\": 2.0.10".into(), + c.clone(), + ) + .await, + &mut ConfigFile::new( + &"[dependencies]\n\"@bendn/test\" = \"2.0.10\"".into(), + c.clone(), + ) + .await, ]; #[derive(Debug, Deserialize, Clone, Eq, PartialEq)] struct LockFileEntry { @@ -185,16 +198,16 @@ mod tests { for cfg in cfgs { assert_eq!(cfg.packages.len(), 1); assert_eq!(cfg.packages[0].to_string(), "@bendn/test@2.0.10"); - assert_eq!(cfg.packages[0].dependencies.len(), 1); + assert_eq!(cfg.packages[0].manifest.dependencies.len(), 1); assert_eq!( - cfg.packages[0].dependencies[0].to_string(), + cfg.packages[0].manifest.dependencies[0].to_string(), "@bendn/gdcli@1.2.5" ); for mut p in cfg.collect() { - p.download().await + p.download(c.clone()).await } assert_eq!( - serde_json::from_str::>(cfg.lock().await.as_str()).unwrap(), + serde_json::from_str::>(cfg.lock().as_str()).unwrap(), wanted_lockfile ); } diff --git a/src/main.rs b/src/main.rs index 20c11b1..ebd8d6c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,7 @@ mod package; mod theme; mod verbosity; -use crate::package::Package; +use crate::package::{parsing::IntoPackageList, Package}; use anyhow::Result; use async_recursion::async_recursion; use clap::{ColorChoice, Parser, Subcommand, ValueEnum}; @@ -12,7 +12,8 @@ use console::{self, Term}; use futures::stream::{self, StreamExt}; use indicatif::{HumanCount, HumanDuration, ProgressBar, ProgressIterator}; use lazy_static::lazy_static; -use package::ParsedPackage; +use package::parsing::ParsedPackage; +use reqwest::Client; use std::fs::{create_dir, read_dir, read_to_string, remove_dir, write}; use std::io::{stdin, Read}; use std::path::{Path, PathBuf}; @@ -145,7 +146,7 @@ async fn main() { ColorChoice::Never => set_colors(false), ColorChoice::Auto => set_colors(Term::stdout().is_term() && Term::stderr().is_term()), } - async fn get_cfg(path: PathBuf) -> ConfigFile { + async fn get_cfg(path: PathBuf, client: Client) -> ConfigFile { let mut contents = String::from(""); if path == Path::new("-") { let bytes = stdin() @@ -157,10 +158,10 @@ async fn main() { } else { contents = read_to_string(path).expect("Reading config file should be ok"); }; - ConfigFile::new(&contents).await + ConfigFile::new(&contents, client).await } - async fn lock(cfg: &mut ConfigFile, path: PathBuf) { - let lockfile = cfg.lock().await; + fn lock(cfg: &mut ConfigFile, path: PathBuf) { + let lockfile = cfg.lock(); if path == Path::new("-") { println!("{lockfile}"); } else { @@ -168,16 +169,17 @@ async fn main() { } } let _ = BEGIN.elapsed(); // needed to initialize the instant for whatever reason + let client = Client::new(); match args.action { Actions::Update => { - let c = &mut get_cfg(args.config_file).await; - update(c, true, args.verbosity).await; - lock(c, args.lock_file).await; + let c = &mut get_cfg(args.config_file, client.clone()).await; + update(c, true, args.verbosity, client.clone()).await; + lock(c, args.lock_file); } Actions::Purge => { - let c = &mut get_cfg(args.config_file).await; + let c = &mut get_cfg(args.config_file, client.clone()).await; purge(c, args.verbosity); - lock(c, args.lock_file).await; + lock(c, args.lock_file); } Actions::Tree { charset, @@ -186,29 +188,26 @@ async fn main() { } => println!( "{}", tree( - &mut get_cfg(args.config_file).await, // no locking needed + &mut get_cfg(args.config_file, client.clone()).await, // no locking needed charset, prefix, - print_tarballs + print_tarballs, + client ) .await ), Actions::Init { packages } => { - let buf = stream::iter(packages.into_iter()) - .map(|pp| async move { - let name = pp.to_string(); - pp.into_package() - .await - .unwrap_or_else(|_| panic!("Package {name} could not be parsed")) - }) - .buffer_unordered(4); - let packages = buf.collect::>().await; - init(packages).await.expect("Initializing cfg should be ok"); + init( + packages.into_package_list(client.clone()).await.unwrap(), + client, + ) + .await + .expect("Initializing cfg should be ok"); } } } -async fn update(cfg: &mut ConfigFile, modify: bool, v: Verbosity) { +async fn update(cfg: &mut ConfigFile, modify: bool, v: Verbosity, client: Client) { if !Path::new("./addons/").exists() { create_dir("./addons/").expect("Should be able to create addons folder"); } @@ -254,6 +253,7 @@ async fn update(cfg: &mut ConfigFile, modify: bool, v: Verbosity) { .map(|mut p| async { let p_name = p.to_string(); let tx = if bar_or_info { tx.clone() } else { None }; + let client = client.clone(); async move { if bar_or_info { tx.as_ref() @@ -261,9 +261,9 @@ async fn update(cfg: &mut ConfigFile, modify: bool, v: Verbosity) { .send(Status::Processing(p_name.clone())) .unwrap(); } - p.download().await; + p.download(client).await; if modify { - p.modify().unwrap(); + p.modify(); }; if bar_or_info { tx.unwrap().send(Status::Finished(p_name.clone())).unwrap(); @@ -394,6 +394,7 @@ async fn tree( charset: CharSet, prefix: PrefixType, print_tarballs: bool, + client: Client, ) -> String { let mut tree: String = if let Ok(s) = current_dir() { format!("{}\n", s.to_string_lossy()) @@ -417,6 +418,7 @@ async fn tree( print_tarballs, 0, &mut count, + client, ) .await; tree.push_str(format!("{} dependencies", HumanCount(count)).as_str()); @@ -432,6 +434,7 @@ async fn tree( print_tarballs: bool, depth: u32, count: &mut u64, + client: Client, ) { // the index is used to decide if the package is the last package, // so we can use a L instead of a T. @@ -453,12 +456,12 @@ async fn tree( ); if print_tarballs { tree.push(' '); - tree.push_str(p.get_manifest().await.unwrap().tarball.as_str()); + tree.push_str(p.manifest.tarball.as_str()); } tree.push('\n'); if p.has_deps() { iter( - &mut p.dependencies, + &mut p.manifest.dependencies, if prefix_type == PrefixType::Indent { tmp = format!("{prefix}{} ", if index != 0 { '│' } else { ' ' }); tmp.as_str() @@ -472,6 +475,7 @@ async fn tree( print_tarballs, depth + 1, count, + client.clone(), ) .await; } @@ -480,7 +484,7 @@ async fn tree( tree } -async fn init(mut packages: Vec) -> Result<()> { +async fn init(mut packages: Vec, client: Client) -> Result<()> { let mut c = ConfigFile::default(); if packages.is_empty() { let mut has_asked = false; @@ -497,7 +501,7 @@ async fn init(mut packages: Vec) -> Result<()> { has_asked = true; let p: ParsedPackage = putils::input("Package?")?; let p_name = p.to_string(); - let res = p.into_package().await; + let res = p.into_package(client.clone()).await; if let Err(e) = res { putils::fail(format!("{p_name} could not be parsed: {e}").as_str())?; just_failed = true; @@ -535,14 +539,21 @@ async fn init(mut packages: Vec) -> Result<()> { if putils::confirm("Would you like to view the dependency tree?", true)? { println!( "{}", - tree(&mut c, CharSet::UTF8, PrefixType::Indent, false).await + tree( + &mut c, + CharSet::UTF8, + PrefixType::Indent, + false, + client.clone() + ) + .await ); }; if !c.packages.is_empty() && putils::confirm("Would you like to install your new packages?", true)? { - update(&mut c, true, Verbosity::Normal).await; + update(&mut c, true, Verbosity::Normal, client.clone()).await; }; println!("Goodbye!"); Ok(()) @@ -583,9 +594,11 @@ mod test_utils { #[tokio::test] async fn gpm() { let _t = test_utils::mktemp(); + let c = Client::new(); let cfg_file = - &mut config_file::ConfigFile::new(&r#"packages: {"@bendn/test":2.0.10}"#.into()).await; - update(cfg_file, false, Verbosity::Verbose).await; + &mut config_file::ConfigFile::new(&r#"packages: {"@bendn/test":2.0.10}"#.into(), c.clone()) + .await; + update(cfg_file, false, Verbosity::Verbose, c.clone()).await; assert_eq!(test_utils::hashd("addons").join("|"), "1c2fd93634817a9e5f3f22427bb6b487520d48cf3cbf33e93614b055bcbd1329|8e77e3adf577d32c8bc98981f05d40b2eb303271da08bfa7e205d3f27e188bd7|a625595a71b159e33b3d1ee6c13bea9fc4372be426dd067186fe2e614ce76e3c|c5566e4fbea9cc6dbebd9366b09e523b20870b1d69dc812249fccd766ebce48e|c5566e4fbea9cc6dbebd9366b09e523b20870b1d69dc812249fccd766ebce48e|c850a9300388d6da1566c12a389927c3353bf931c4d6ea59b02beb302aac03ea|d060936e5f1e8b1f705066ade6d8c6de90435a91c51f122905a322251a181a5c|d711b57105906669572a0e53b8b726619e3a21463638aeda54e586a320ed0fc5|d794f3cee783779f50f37a53e1d46d9ebbc5ee7b37c36d7b6ee717773b6955cd|e4f9df20b366a114759282209ff14560401e316b0059c1746c979f478e363e87"); purge(cfg_file, Verbosity::Verbose); assert_eq!(test_utils::hashd("addons"), vec![] as Vec); @@ -594,7 +607,8 @@ async fn gpm() { cfg_file, crate::CharSet::UTF8, crate::PrefixType::Indent, - false + false, + c.clone(), ) .await .lines() diff --git a/src/package.rs b/src/package.rs index fa155df..1db6d43 100644 --- a/src/package.rs +++ b/src/package.rs @@ -1,10 +1,9 @@ -use crate::config_file::ConfigFile; use anyhow::{anyhow, Result}; -use async_recursion::async_recursion; use flate2::read::GzDecoder; use regex::{Captures, Regex}; -use serde::{Deserialize, Serialize}; -use serde_json::Value as JValue; +use reqwest::Client; +use semver_rs::Version; +use serde::Serialize; use sha1::{Digest, Sha1}; use std::fs::{create_dir_all, read_dir, read_to_string, remove_dir_all, write}; use std::io; @@ -13,6 +12,9 @@ use std::str::FromStr; use std::{collections::HashMap, fmt}; use tar::{Archive, EntryType::Directory}; +pub mod parsing; +use parsing::*; + const REGISTRY: &str = "https://registry.npmjs.org"; #[derive(Clone, Eq, Ord, PartialEq, PartialOrd, Default, Serialize, Debug)] @@ -23,169 +25,101 @@ const REGISTRY: &str = "https://registry.npmjs.org"; /// - removal pub struct Package { pub name: String, - pub version: String, #[serde(skip)] - pub dependencies: Vec, + pub version: Version, #[serde(skip)] pub indirect: bool, #[serde(flatten)] - pub manifest: Option, -} -#[derive(Default, Clone, Debug)] -pub struct ParsedPackage { - pub name: String, - pub version: Option, + pub manifest: Manifest, } -#[derive(Clone, Deserialize, Eq, Ord, PartialEq, PartialOrd, Default, Debug, Serialize)] +#[derive(Clone, Eq, Ord, PartialEq, PartialOrd, Default, Debug, Serialize)] pub struct Manifest { pub integrity: String, - #[serde(skip_serializing)] + #[serde(skip)] pub shasum: String, - #[serde(skip_serializing)] pub tarball: String, + #[serde(skip)] + pub dependencies: Vec, + version: String, } -impl ParsedPackage { - /// Turn into a [Package]. - pub async fn into_package(self) -> Result { - if self.version.is_some() { - Package::new(self.name, self.version.unwrap()).await - } else { - Package::new_no_version(self.name).await - } - } -} - -impl fmt::Display for ParsedPackage { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "{}@{}", - self.name, - self.version.as_ref().unwrap_or(&"latest".to_string()) - ) - } +const VERSION: &str = env!("CARGO_PKG_VERSION"); + +macro_rules! abbreviated_get { + ($url: expr, $client: expr) => { + $client + .get($url) + .header( + "User-Agent", + format!("gpm/{VERSION} (godot-package-manager/cli on GitHub)"), + ) + .header( + "Accept", + "application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8", + ) + .send() + .await + }; } -impl FromStr for ParsedPackage { - type Err = anyhow::Error; - - /// Supports 3 version syntax variations: `:`, `=`, `@`, if version not specified, will fetch latest. - /// see https://docs.npmjs.com/cli/v7/configuring-npm/package-json#name - fn from_str(s: &str) -> Result { - #[inline] - fn not_too_long(s: &str) -> bool { - s.len() < 214 - } - #[inline] - fn safe(s: &str) -> bool { - s.find(&[ - ' ', '<', '>', '[', ']', '{', '}', '|', '\\', '^', '%', ':', '=', - ]) - .is_none() - } - fn check(s: &str) -> Result<()> { - if not_too_long(s) && safe(s) { - Ok(()) - } else { - Err(anyhow!("Invalid package name")) - } - } - - fn split_p(s: &str, d: char) -> Result { - let Some((p, v)) = s.split_once(d) else { - check(s)?; - return Ok(ParsedPackage {name: s.to_string(), ..Default::default()}); - }; - check(p)?; - Ok(ParsedPackage { - name: p.to_string(), - version: Some(v.to_string()), - }) - } - if s.contains(':') { - // @bendn/gdcli:1.2.5 - return split_p(s, ':'); - } else if s.contains('=') { - // @bendn/gdcli=1.2.5 - return split_p(s, '='); - } else { - // @bendn/gdcli@1.2.5 - if s.as_bytes()[0] == b'@' { - let mut owned_s = s.to_string(); - owned_s.remove(0); - let Some((p, v)) = owned_s.split_once('@') else { - check(s)?; - return Ok(ParsedPackage {name: s.to_string(), ..Default::default()}); - }; - check(&format!("@{p}")[..])?; - return Ok(ParsedPackage { - name: format!("@{p}"), - version: Some(v.to_string()), - }); - } - return split_p(s, '@'); - }; - } +macro_rules! get { + ($url: expr, $client: expr) => { + $client + .get($url) + .header( + "User-Agent", + format!("gpm/{VERSION} (godot-package-manager/cli on GitHub)"), + ) + .send() + .await + }; } impl Package { #[inline] /// Does this package have dependencies? - pub fn has_deps(&self) -> bool { - !self.dependencies.is_empty() + pub fn has_deps(&mut self) -> bool { + !self.manifest.dependencies.is_empty() } /// Creates a new [Package] from a name and version. - /// Calls the Package::get_deps() function, so it will - /// try to access the fs, and if it fails, it will make - /// calls to cdn.jsdelivr.net to get the `package.json` file. - pub async fn new(name: String, version: String) -> Result { - let mut p = Package { - name, - version, + /// Makes network calls to get the manifest (which makes network calls to get dependency manifests) + pub async fn new(name: String, version: Version, client: Client) -> Result { + Ok(Self { + name: name.clone(), + version: version.clone(), + manifest: Self::get_manifest(client, name, version).await?, ..Default::default() - }; - p.get_deps().await?; - Ok(p) + }) } /// Create a package from a [str]. see also [ParsedPackage]. #[allow(dead_code)] // used for tests - pub async fn create_from_str(s: &str) -> Result { - ParsedPackage::from_str(s).unwrap().into_package().await + pub async fn create_from_str(s: &str, client: Client) -> Result { + ParsedPackage::from_str(s) + .unwrap() + .into_package(client) + .await } /// Creates a new [Package] from a name, gets the latest version from registry/name. - pub async fn new_no_version(name: String) -> Result { - let resp = reqwest::get(&format!("{REGISTRY}/{name}")) - .await? + pub async fn new_no_version(name: String, client: Client) -> Result { + let resp = abbreviated_get!(format!("{REGISTRY}/{name}/latest"), client.clone())? .text() .await?; if resp == "\"Not Found\"" { return Err(anyhow!("Package {name} was not found")); }; - let resp = serde_json::from_str::(&resp)?; - let v = resp - .get("dist-tags") - .ok_or(anyhow!("No dist tags!"))? - .get("latest") - .ok_or(anyhow!("No latest!"))? - .as_str() - .ok_or(anyhow!("Latest not string!"))?; - let mut p = Package::new(name, v.to_string()).await?; - p.manifest = serde_json::from_str( - resp.get("versions") - .ok_or(anyhow!("No versions!"))? - .get(v) - .ok_or(anyhow!("No latest version!"))? - .to_string() - .as_str(), - ) - .expect("Manifest"); - p.get_deps().await?; - Ok(p) + let resp = serde_json::from_str::(&resp)? + .into_manifest(client.clone()) + .await?; + Ok(Package { + name, + version: Version::new(&resp.version).parse()?, + manifest: resp, + ..Default::default() + }) } /// Returns wether this package is installed. @@ -202,10 +136,9 @@ impl Package { /// Installs this [Package] to a download directory, /// depending on wether this package is a direct dependency or not. - pub async fn download(&mut self) { + pub async fn download(&mut self, client: Client) { self.purge(); - let bytes = reqwest::get(&self.get_manifest().await.unwrap().tarball) - .await + let bytes = get!(&self.manifest.tarball, client) .expect("Tarball download should work") .bytes() .await @@ -216,8 +149,8 @@ impl Package { hasher.update(&bytes); const ERR: &str = "Tarball shasum should be a valid hex string"; assert_eq!( - self.get_manifest().await.unwrap().shasum, - format!("{:x}", hasher.finalize()), + &self.manifest.shasum, + &format!("{:x}", hasher.finalize()), "Tarball did not match checksum!" ); @@ -269,71 +202,21 @@ impl Package { .expect("Tarball should unpack"); } - /// Gets the [ConfigFile] for this [Package]. - /// Will attempt to read the `package.json` file, if this package is installed. - /// Else it will make network calls to `cdn.jsdelivr.net`. - #[async_recursion] - pub async fn get_config_file(&self) -> Result { - fn get(f: String) -> io::Result { - read_to_string(Path::new(&f).join("package.json")) - } - #[rustfmt::skip] - let c: Option = if let Ok(c) = get(self.indirect_download_dir()) { Some(c) } - else if let Ok(c) = get(self.download_dir()) { Some(c) } - else { None }; - if let Some(c) = c { - if let Ok(n) = ConfigFile::parse(&c, crate::config_file::ConfigType::JSON).await { - return Ok(n); - } - } - ConfigFile::parse( - &reqwest::get(&format!( - "https://cdn.jsdelivr.net/npm/{}@{}/package.json", - self.name, self.version, - )) - .await - .map_err(|_| { - anyhow!("Request to cdn.jsdelivr.net failed, package/version doesnt exist") - })? - .text() - .await?, - crate::config_file::ConfigType::JSON, - ) - .await - } - /// Gets the package manifest and puts it in `self.manfiest`. - pub async fn get_manifest(&mut self) -> Result<&Manifest> { - if self.manifest.is_some() { - return Ok(self.manifest.as_ref().unwrap()); - } - let resp = reqwest::get(&format!("{REGISTRY}/{}/{}", self.name, self.version)) - .await? + pub async fn get_manifest(client: Client, name: String, version: Version) -> Result { + let resp = abbreviated_get!(&format!("{REGISTRY}/{name}/{version}"), client.clone())? .text() .await?; if resp == "\"Not Found\"" { + return Err(anyhow!("Package {name}@{version} was not found",)); + } else if resp == format!("\"version not found: {version}\"") { return Err(anyhow!( - "Package {}@{} was not found", - self.name, - self.version - )); - } else if resp == format!("\"version not found: {}\"", self.version) { - return Err(anyhow!( - "Package {} exists, but version '{}' not found", - self.name, - self.version + "Package {name} exists, but version '{version}' was not found" )); } - let manifest = serde_json::from_str( - &serde_json::from_str::(resp.as_str()) - .unwrap() - .get("dist") - .unwrap() - .to_string(), - ) - .unwrap_or_else(|_| panic!("Unable to get manifest for package {self}")); - self.manifest = Some(manifest); - return Ok(self.manifest.as_ref().unwrap()); + serde_json::from_str::(&resp)? + .into_manifest(client) + .await } /// Returns the download directory for this package depending on wether it is indirect or not. @@ -354,16 +237,6 @@ impl Package { fn indirect_download_dir(&self) -> String { format!("./addons/__gpm_deps/{}/{}", self.name, self.version) } - - /// Gets the dependencies of this [Package], placing them in `self.dependencies`. - async fn get_deps(&mut self) -> Result<()> { - let cfg = self.get_config_file().await?; - cfg.packages.into_iter().for_each(|mut dep| { - dep.indirect = true; - self.dependencies.push(dep); - }); - Ok(()) - } } // package modification block @@ -462,7 +335,7 @@ impl Package { } /// Recursively modifies a directory. - fn recursive_modify(&self, dir: PathBuf, dep_map: &HashMap) -> io::Result<()> { + fn recursive_modify(&self, dir: PathBuf, dep_map: &HashMap) -> Result<()> { for entry in read_dir(&dir)? { let p = entry?; if p.path().is_dir() { @@ -496,33 +369,37 @@ impl Package { Ok(()) } - fn dep_map(&self) -> HashMap { + fn dep_map(&mut self) -> Result> { let mut dep_map = HashMap::::new(); - fn add(p: &Package, dep_map: &mut HashMap) { - let d = p.download_dir().strip_prefix("./").unwrap().to_string(); + fn add(p: &Package, dep_map: &mut HashMap) -> Result<()> { + let d = p + .download_dir() + .strip_prefix("./") + .ok_or(anyhow!("cant strip prefix!"))? + .to_string(); dep_map.insert(p.name.clone(), d.clone()); // unscoped (@ben/cli => cli) (for compat) if let Some((_, s)) = p.name.split_once('/') { dep_map.insert(s.into(), d); } + Ok(()) } - for pkg in &self.dependencies { - add(pkg, &mut dep_map); + for pkg in &self.manifest.dependencies { + add(pkg, &mut dep_map)?; } - add(self, &mut dep_map); - dep_map + add(self, &mut dep_map)?; + Ok(dep_map) } /// The catalyst for `recursive_modify`. - pub fn modify(&self) -> io::Result<()> { + pub fn modify(&mut self) { if !self.is_installed() { panic!("Attempting to modify a package that is not installed"); } - self.recursive_modify( - Path::new(&self.download_dir()).to_path_buf(), - &self.dep_map(), - ) + let map = &self.dep_map().unwrap(); + self.recursive_modify(Path::new(&self.download_dir()).to_path_buf(), map) + .unwrap(); } } @@ -540,10 +417,11 @@ mod tests { #[tokio::test] async fn download() { let _t = crate::test_utils::mktemp(); - let mut p = Package::create_from_str("@bendn/test:2.0.10") + let c = Client::new(); + let mut p = Package::create_from_str("@bendn/test:2.0.10", c.clone()) .await .unwrap(); - p.download().await; + p.download(c.clone()).await; assert_eq!( crate::test_utils::hashd(p.download_dir().as_str()), [ @@ -560,10 +438,11 @@ mod tests { async fn dep_map() { // no fs was touched in the making of this test assert_eq!( - Package::create_from_str("@bendn/test@2.0.10") + Package::create_from_str("@bendn/test@2.0.10", Client::new()) .await .unwrap() - .dep_map(), + .dep_map() + .unwrap(), HashMap::from([ ("test".into(), "addons/@bendn/test".into()), ("@bendn/test".into(), "addons/@bendn/test".into()), @@ -582,12 +461,13 @@ mod tests { #[tokio::test] async fn modify_load() { let _t = crate::test_utils::mktemp(); - let mut p = Package::create_from_str("@bendn/test=2.0.10") + let c = Client::new(); + let mut p = Package::create_from_str("@bendn/test=2.0.10", c.clone()) .await .unwrap(); - let dep_map = &p.dep_map(); + let dep_map = &p.dep_map().unwrap(); let cwd = Path::new("addons/@bendn/test").into(); - p.download().await; + p.download(c).await; p.indirect = false; assert_eq!( p.modify_load(Path::new("addons/test/main.gd"), cwd, dep_map) diff --git a/src/package/parsing.rs b/src/package/parsing.rs new file mode 100644 index 0000000..79b5c25 --- /dev/null +++ b/src/package/parsing.rs @@ -0,0 +1,193 @@ +use crate::package::{Manifest, Package}; +use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use futures::stream::{self, StreamExt}; +use reqwest::Client; +use semver_rs::Version; +use serde::Deserialize; +use std::{collections::HashMap, fmt}; + +macro_rules! parse_version { + ($ver: expr) => { + VersionType::Normal(Version::new($ver).parse()?) + }; +} + +#[derive(Clone, Debug)] +pub struct ParsedPackage { + pub name: String, + pub version: VersionType, +} + +#[derive(Clone, Debug)] +pub enum VersionType { + /// Normal version, just use it + Normal(Version), + /// Abstract version, figure it out later + Latest, +} + +impl fmt::Display for VersionType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}", + match self { + VersionType::Normal(v) => v.to_string(), + VersionType::Latest => "latest".to_string(), + } + ) + } +} + +impl ParsedPackage { + /// Turn into a [Package]. + pub async fn into_package(self, client: Client) -> Result { + match self.version { + VersionType::Normal(v) => Package::new(self.name, v, client).await, + VersionType::Latest => Package::new_no_version(self.name, client).await, + } + } +} + +impl fmt::Display for ParsedPackage { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}@{}", self.name, self.version) + } +} + +impl std::str::FromStr for ParsedPackage { + type Err = anyhow::Error; + + /// Supports 3 version syntax variations: `:`, `=`, `@`, if version not specified, will fetch latest. + /// see https://docs.npmjs.com/cli/v7/configuring-npm/package-json#name + fn from_str(s: &str) -> Result { + #[inline] + fn not_too_long(s: &str) -> bool { + s.len() < 214 + } + #[inline] + fn safe(s: &str) -> bool { + s.find(&[ + ' ', '<', '>', '[', ']', '{', '}', '|', '\\', '^', '%', ':', '=', + ]) + .is_none() + } + fn check(s: &str) -> Result<()> { + if not_too_long(s) && safe(s) { + Ok(()) + } else { + Err(anyhow!("Invalid package name")) + } + } + + fn split_p(s: &str, d: char) -> Result { + let Some((p, v)) = s.split_once(d) else { + check(s)?; + return Ok(ParsedPackage {name: s.to_string(), version: VersionType::Latest }); + }; + check(p)?; + Ok(ParsedPackage { + name: p.to_string(), + version: parse_version!(&v.to_string()), + }) + } + if s.contains(':') { + // @bendn/gdcli:1.2.5 + return split_p(s, ':'); + } else if s.contains('=') { + // @bendn/gdcli=1.2.5 + return split_p(s, '='); + } else { + // @bendn/gdcli@1.2.5 + if s.as_bytes()[0] == b'@' { + let mut owned_s = s.to_string(); + owned_s.remove(0); + let Some((p, v)) = owned_s.split_once('@') else { + check(s)?; + return Ok(ParsedPackage {name: s.to_string(), version: VersionType::Latest }); + }; + check(&format!("@{p}")[..])?; + return Ok(ParsedPackage { + name: format!("@{p}"), + version: parse_version!(&v.to_string()), + }); + } + return split_p(s, '@'); + }; + } +} + +#[derive(Clone, Default, Debug, Deserialize)] +pub struct ParsedManifest { + dist: ParsedManifestDist, + #[serde(default)] + dependencies: HashMap, + version: String, +} + +#[derive(Clone, Default, Debug, Deserialize)] +pub struct ParsedManifestDist { + pub integrity: String, + pub shasum: String, + pub tarball: String, +} + +impl ParsedManifest { + pub async fn into_manifest(self, client: Client) -> Result { + Ok(Manifest { + integrity: self.dist.integrity, + shasum: self.dist.shasum, + tarball: self.dist.tarball, + version: self.version, + dependencies: self.dependencies.into_package_list(client).await?, + }) + } +} + +#[async_trait] +pub trait IntoPackageList { + async fn into_package_list(self, client: Client) -> Result>; +} + +#[async_trait] +impl IntoPackageList for HashMap { + async fn into_package_list(self, client: Client) -> Result> { + let buf = stream::iter(self.into_iter()) + .map(|(name, version)| async { + let client = client.clone(); + async move { + Package::new(name, Version::new(&version).parse().unwrap(), client).await + } + .await + }) + .buffer_unordered(crate::PARALLEL); + let mut packages = vec![]; + for p in buf.collect::>>().await { + let mut p = p?; + p.indirect = true; + packages.push(p); + } + Ok(packages) + } +} + +#[async_trait] +impl IntoPackageList for Vec { + /// Fake result implementation + async fn into_package_list(self, client: Client) -> Result> { + let buf = stream::iter(self.into_iter()) + .map(|pp| async { + let client = client.clone(); + async move { + let name = pp.to_string(); + pp.into_package(client) + .await + .unwrap_or_else(|_| panic!("Package {name} could not be parsed")) + } + .await + }) + .buffer_unordered(4); + Ok(buf.collect::>().await) + } +}