diff --git a/README.md b/README.md index 89ffaaad..dfd6c75c 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ The following table provides a brief (incomplete) comparison: | `command` build spec | :white_check_mark: | :white_check_mark: | | custom build backends | :white_check_mark:[^1] | :white_check_mark: | | `rust-mlua` build spec | :white_check_mark: (builtin) | :white_check_mark: (external build backend) | -| install pre-built binary rocks | :x: (planned) | :white_check_mark: | +| install pre-built binary rocks | :white_check_mark: | :white_check_mark: | | parallel builds/installs | :white_check_mark: | :x: | | install multiple packages with a single command | :white_check_mark: | :x: | | install packages using version constraints | :white_check_mark: | :x: | diff --git a/nix/overlay.nix b/nix/overlay.nix index 7526be73..d3b57fb5 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -63,6 +63,7 @@ # disable vendored packages LIBGIT2_NO_VENDOR = 1; LIBSSH2_SYS_USE_PKG_CONFIG = 1; + ROCKS_SKIP_IMPURE_TESTS = 1; }; inherit buildType; diff --git a/rocks-bin/src/build.rs b/rocks-bin/src/build.rs index 6c821a80..5e887860 100644 --- a/rocks-bin/src/build.rs +++ b/rocks-bin/src/build.rs @@ -119,7 +119,7 @@ pub async fn build(data: Build, config: Config) -> Result<()> { .install() .await?; - build::Build::new(rockspec, &config, &progress.map(|p| p.new_bar())) + build::Build::new(&rockspec, &config, &progress.map(|p| p.new_bar())) .pin(pin) .behaviour(build_behaviour) .build() diff --git a/rocks-lib/resources/test/sample-tree/5.1/lock.json b/rocks-lib/resources/test/sample-tree/5.1/lock.json index 6a2f749e..c1b12794 100644 --- a/rocks-lib/resources/test/sample-tree/5.1/lock.json +++ b/rocks-lib/resources/test/sample-tree/5.1/lock.json @@ -9,7 +9,7 @@ "48ec344951668eca0e0a4ff284d804a11e4e709194df3191a72ed8fac89cf2e0" ], "constraint": null, - "source": "luarocks+https://luarocks.org/", + "source": "luarocks_rockspec+https://luarocks.org/", "hashes": { "rockspec": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=", "source": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" @@ -21,7 +21,7 @@ "pinned": false, "dependencies": [], "constraint": null, - "source": "luarocks+https://luarocks.org/", + "source": "luarocks_rockspec+https://luarocks.org/", "hashes": { "rockspec": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=", "source": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" @@ -35,7 +35,7 @@ "48ec344951668eca0e0a4ff284d804a11e4e709194df3191a72ed8fac89cf2e0" ], "constraint": null, - "source": "luarocks+https://luarocks.org/", + "source": "luarocks_rockspec+https://luarocks.org/", "hashes": { "rockspec": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=", "source": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" diff --git a/rocks-lib/resources/test/toml-edit-0.6.0-1.linux-x86_64.rock b/rocks-lib/resources/test/toml-edit-0.6.0-1.linux-x86_64.rock new file mode 100644 index 00000000..c6a13c31 Binary files /dev/null and b/rocks-lib/resources/test/toml-edit-0.6.0-1.linux-x86_64.rock differ diff --git a/rocks-lib/src/build/luarocks.rs b/rocks-lib/src/build/luarocks.rs index fd03e3bd..c962cc10 100644 --- a/rocks-lib/src/build/luarocks.rs +++ b/rocks-lib/src/build/luarocks.rs @@ -3,7 +3,7 @@ use std::{io, path::Path}; use crate::{ config::Config, lua_installation::LuaInstallation, - luarocks_installation::{ExecLuaRocksError, LuaRocksError, LuaRocksInstallation}, + luarocks::luarocks_installation::{ExecLuaRocksError, LuaRocksError, LuaRocksInstallation}, progress::{Progress, ProgressBar}, rockspec::Rockspec, tree::RockLayout, diff --git a/rocks-lib/src/build/mod.rs b/rocks-lib/src/build/mod.rs index 8b702e58..1b7653ca 100644 --- a/rocks-lib/src/build/mod.rs +++ b/rocks-lib/src/build/mod.rs @@ -38,7 +38,7 @@ pub mod variables; /// A rocks package builder, providing fine-grained control /// over how a package should be built. pub struct Build<'a> { - rockspec: Rockspec, + rockspec: &'a Rockspec, config: &'a Config, progress: &'a Progress, pin: PinnedState, @@ -50,7 +50,7 @@ pub struct Build<'a> { impl<'a> Build<'a> { /// Construct a new builder. pub fn new( - rockspec: Rockspec, + rockspec: &'a Rockspec, config: &'a Config, progress: &'a Progress, ) -> Self { @@ -263,7 +263,7 @@ async fn install( } async fn build( - rockspec: Rockspec, + rockspec: &Rockspec, pinned: PinnedState, constraint: LockConstraint, behaviour: BuildBehaviour, @@ -341,9 +341,9 @@ async fn build( None => temp_dir.path().into(), }; - run_build(&rockspec, &output_paths, &lua, config, &build_dir, progress).await?; + run_build(rockspec, &output_paths, &lua, config, &build_dir, progress).await?; - install(&rockspec, &tree, &output_paths, &lua, &build_dir, progress).await?; + install(rockspec, &tree, &output_paths, &lua, &build_dir, progress).await?; for directory in &rockspec.build.current_platform().copy_directories { recursive_copy_dir(&build_dir.join(directory), &output_paths.etc)?; diff --git a/rocks-lib/src/hash.rs b/rocks-lib/src/hash.rs index ff823295..d7d8cf3e 100644 --- a/rocks-lib/src/hash.rs +++ b/rocks-lib/src/hash.rs @@ -1,3 +1,4 @@ +use bytes::Bytes; use ssri::{Algorithm, Integrity, IntegrityOpts}; use std::fs::File; use std::io::{self, Read}; @@ -39,6 +40,14 @@ impl HasIntegrity for TempDir { } } +impl HasIntegrity for Bytes { + fn hash(&self) -> io::Result { + let mut integrity_opts = IntegrityOpts::new().algorithm(Algorithm::Sha256); + integrity_opts.input(self); + Ok(integrity_opts.result()) + } +} + fn hash_file(path: &Path, integrity_opts: &mut IntegrityOpts) -> io::Result<()> { let mut file = File::open(path)?; let mut buffer = Vec::new(); diff --git a/rocks-lib/src/lib.rs b/rocks-lib/src/lib.rs index 2b93eb3d..1a2eaacb 100644 --- a/rocks-lib/src/lib.rs +++ b/rocks-lib/src/lib.rs @@ -3,7 +3,7 @@ pub mod config; pub mod hash; pub mod lockfile; pub mod lua_installation; -pub mod luarocks_installation; +pub mod luarocks; pub mod manifest; pub mod operations; pub mod package; diff --git a/rocks-lib/src/lockfile/snapshots/rocks_lib__lockfile__tests__add_rocks.snap b/rocks-lib/src/lockfile/snapshots/rocks_lib__lockfile__tests__add_rocks.snap index 8cd6c42a..8c1739f3 100644 --- a/rocks-lib/src/lockfile/snapshots/rocks_lib__lockfile__tests__add_rocks.snap +++ b/rocks-lib/src/lockfile/snapshots/rocks_lib__lockfile__tests__add_rocks.snap @@ -12,7 +12,7 @@ expression: lockfile "pinned": false, "dependencies": [], "constraint": null, - "source": "luarocks+https://luarocks.org/", + "source": "luarocks_rockspec+https://luarocks.org/", "hashes": { "rockspec": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=", "source": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" @@ -52,7 +52,7 @@ expression: lockfile "48ec344951668eca0e0a4ff284d804a11e4e709194df3191a72ed8fac89cf2e0" ], "constraint": null, - "source": "luarocks+https://luarocks.org/", + "source": "luarocks_rockspec+https://luarocks.org/", "hashes": { "rockspec": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=", "source": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" @@ -66,7 +66,7 @@ expression: lockfile "48ec344951668eca0e0a4ff284d804a11e4e709194df3191a72ed8fac89cf2e0" ], "constraint": null, - "source": "luarocks+https://luarocks.org/", + "source": "luarocks_rockspec+https://luarocks.org/", "hashes": { "rockspec": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=", "source": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" diff --git a/rocks-lib/src/lockfile/snapshots/rocks_lib__lockfile__tests__parse_lockfile.snap b/rocks-lib/src/lockfile/snapshots/rocks_lib__lockfile__tests__parse_lockfile.snap index fffc3c6e..adbac26b 100644 --- a/rocks-lib/src/lockfile/snapshots/rocks_lib__lockfile__tests__parse_lockfile.snap +++ b/rocks-lib/src/lockfile/snapshots/rocks_lib__lockfile__tests__parse_lockfile.snap @@ -13,7 +13,7 @@ snapshot_kind: text "pinned": false, "dependencies": [], "constraint": null, - "source": "luarocks+https://luarocks.org/", + "source": "luarocks_rockspec+https://luarocks.org/", "hashes": { "rockspec": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=", "source": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" @@ -27,7 +27,7 @@ snapshot_kind: text "48ec344951668eca0e0a4ff284d804a11e4e709194df3191a72ed8fac89cf2e0" ], "constraint": null, - "source": "luarocks+https://luarocks.org/", + "source": "luarocks_rockspec+https://luarocks.org/", "hashes": { "rockspec": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=", "source": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" @@ -41,7 +41,7 @@ snapshot_kind: text "48ec344951668eca0e0a4ff284d804a11e4e709194df3191a72ed8fac89cf2e0" ], "constraint": null, - "source": "luarocks+https://luarocks.org/", + "source": "luarocks_rockspec+https://luarocks.org/", "hashes": { "rockspec": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=", "source": "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" diff --git a/rocks-lib/src/luarocks/install_binary_rock.rs b/rocks-lib/src/luarocks/install_binary_rock.rs new file mode 100644 index 00000000..91e6feb0 --- /dev/null +++ b/rocks-lib/src/luarocks/install_binary_rock.rs @@ -0,0 +1,212 @@ +use std::{ + collections::HashMap, + io::{self, Cursor}, + path::{Path, PathBuf}, +}; + +use bytes::Bytes; +use tempdir::TempDir; +use thiserror::Error; + +use crate::{ + build::{ + external_dependency::{ExternalDependencyError, ExternalDependencyInfo}, + BuildBehaviour, + }, + config::Config, + hash::HasIntegrity as _, + lockfile::{LocalPackage, LocalPackageHashes, LockConstraint, PinnedState}, + luarocks::rock_manifest::RockManifest, + package::PackageSpec, + progress::{Progress, ProgressBar}, + remote_package_source::RemotePackageSource, + rockspec::{LuaVersionError, Rockspec}, + tree::Tree, +}; + +use super::rock_manifest::RockManifestError; + +#[derive(Error, Debug)] +pub enum InstallBinaryRockError { + #[error("IO operation failed: {0}")] + Io(#[from] io::Error), + #[error(transparent)] + ExternalDependencyError(#[from] ExternalDependencyError), + #[error(transparent)] + LuaVersionError(#[from] LuaVersionError), + #[error("failed to unpack packed rock: {0}")] + Zip(#[from] zip::result::ZipError), + #[error("rock_manifest not found. Cannot install rock files that were packed using LuaRocks version 1")] + RockManifestNotFound, + #[error(transparent)] + RockManifestError(#[from] RockManifestError), +} + +pub(crate) struct BinaryRockInstall<'a> { + rockspec: &'a Rockspec, + rock_bytes: Bytes, + source: RemotePackageSource, + pin: PinnedState, + constraint: LockConstraint, + behaviour: BuildBehaviour, + config: &'a Config, + progress: &'a Progress, +} + +impl<'a> BinaryRockInstall<'a> { + pub(crate) fn new( + rockspec: &'a Rockspec, + source: RemotePackageSource, + rock_bytes: Bytes, + config: &'a Config, + progress: &'a Progress, + ) -> Self { + Self { + rockspec, + rock_bytes, + source, + config, + progress, + constraint: LockConstraint::default(), + behaviour: BuildBehaviour::default(), + pin: PinnedState::default(), + } + } + + pub(crate) fn pin(self, pin: PinnedState) -> Self { + Self { pin, ..self } + } + + pub(crate) fn constraint(self, constraint: LockConstraint) -> Self { + Self { constraint, ..self } + } + + pub(crate) fn behaviour(self, behaviour: BuildBehaviour) -> Self { + Self { behaviour, ..self } + } + + pub(crate) async fn install(self) -> Result { + let rockspec = self.rockspec; + self.progress.map(|p| { + p.set_message(format!( + "Unpacking and installing {}@{}...", + rockspec.package, rockspec.version + )) + }); + for (name, dep) in rockspec.external_dependencies.current_platform() { + let _ = ExternalDependencyInfo::detect(name, dep, self.config.external_deps())?; + } + + let lua_version = rockspec.lua_version_from_config(self.config)?; + + let tree = Tree::new(self.config.tree().clone(), lua_version.clone())?; + + let hashes = LocalPackageHashes { + rockspec: rockspec.hash()?, + source: self.rock_bytes.hash()?, + }; + let mut package = LocalPackage::from( + &PackageSpec::new(rockspec.package.clone(), rockspec.version.clone()), + self.constraint, + self.source, + hashes, + ); + package.spec.pinned = self.pin; + match tree.lockfile()?.get(&package.id()) { + Some(package) if self.behaviour == BuildBehaviour::NoForce => Ok(package.clone()), + _ => { + let unpack_dir = TempDir::new("rocks-bin-rock").unwrap().into_path(); + let cursor = Cursor::new(self.rock_bytes); + let mut zip = zip::ZipArchive::new(cursor)?; + zip.extract(&unpack_dir)?; + let rock_manifest_file = unpack_dir.join("rock_manifest"); + if !rock_manifest_file.is_file() { + return Err(InstallBinaryRockError::RockManifestNotFound); + } + let rock_manifest_content = std::fs::read_to_string(rock_manifest_file)?; + let output_paths = tree.rock(&package)?; + let rock_manifest = RockManifest::new(&rock_manifest_content)?; + install_manifest_entry( + &rock_manifest.lib, + &unpack_dir.join("lib"), + &output_paths.lib, + )?; + install_manifest_entry( + &rock_manifest.lua, + &unpack_dir.join("lua"), + &output_paths.src, + )?; + install_manifest_entry( + &rock_manifest.bin, + &unpack_dir.join("bin"), + &output_paths.bin, + )?; + install_manifest_entry( + &rock_manifest.doc, + &unpack_dir.join("doc"), + &output_paths.doc, + )?; + install_manifest_entry(&rock_manifest.root, &unpack_dir, &output_paths.etc)?; + Ok(package) + } + } + } +} + +fn install_manifest_entry( + entry: &HashMap, + src: &Path, + dest: &Path, +) -> Result<(), InstallBinaryRockError> { + for relative_src_path in entry.keys() { + let target = dest.join(relative_src_path); + std::fs::create_dir_all(target.parent().unwrap())?; + std::fs::copy(src.join(relative_src_path), target)?; + } + Ok(()) +} + +#[cfg(test)] +mod test { + + use crate::{ + config::ConfigBuilder, + operations::{unpack_rockspec, DownloadedPackedRockBytes}, + progress::MultiProgress, + }; + + use super::*; + #[tokio::test] + async fn install_binary_rock() { + if std::env::var("ROCKS_SKIP_IMPURE_TESTS").unwrap_or("0".into()) == "1" { + println!("Skipping impure test"); + return; + } + let content = std::fs::read("resources/test/toml-edit-0.6.0-1.linux-x86_64.rock").unwrap(); + let rock_bytes = Bytes::copy_from_slice(&content); + let rock = DownloadedPackedRockBytes { + name: "toml-edit".into(), + version: "0.6.0-1".parse().unwrap(), + bytes: rock_bytes, + file_name: "toml-edit-0.6.0-1.linux-x86_64.rock".into(), + }; + let rockspec = unpack_rockspec(&rock).await.unwrap(); + let dir = assert_fs::TempDir::new().unwrap(); + let config = ConfigBuilder::new() + .tree(Some(dir.to_path_buf())) + .build() + .unwrap(); + let progress = MultiProgress::new(); + let bar = progress.new_bar(); + BinaryRockInstall::new( + &rockspec, + RemotePackageSource::Test, + rock.bytes, + &config, + &Progress::Progress(bar), + ) + .install() + .await + .unwrap(); + } +} diff --git a/rocks-lib/src/luarocks_installation.rs b/rocks-lib/src/luarocks/luarocks_installation.rs similarity index 97% rename from rocks-lib/src/luarocks_installation.rs rename to rocks-lib/src/luarocks/luarocks_installation.rs index bc4371bc..a5f3756d 100644 --- a/rocks-lib/src/luarocks_installation.rs +++ b/rocks-lib/src/luarocks/luarocks_installation.rs @@ -104,7 +104,7 @@ impl LuaRocksInstallation { PackageReq::new("luarocks".into(), Some(LUAROCKS_VERSION.into())).unwrap(); if !self.tree.match_rocks(&luarocks_req)?.is_found() { let rockspec = Rockspec::new(LUAROCKS_ROCKSPEC).unwrap(); - let pkg = Build::new(rockspec, &self.config, progress) + let pkg = Build::new(&rockspec, &self.config, progress) .constraint(LockConstraint::Constrained( luarocks_req.version_req().clone(), )) @@ -165,12 +165,12 @@ impl LuaRocksInstallation { let bar = progress.map(|p| { p.add(ProgressBar::from(format!( "💻 Installing build dependency: {}", - install_spec.rockspec.package, + install_spec.downloaded_rock.rockspec().package, ))) }); let config = self.config.clone(); tokio::spawn(async move { - let rockspec = install_spec.rockspec; + let rockspec = install_spec.downloaded_rock.rockspec(); let pkg = Build::new(rockspec, &config, &bar) .constraint(install_spec.spec.constraint()) .behaviour(install_spec.build_behaviour) diff --git a/rocks-lib/src/luarocks/mod.rs b/rocks-lib/src/luarocks/mod.rs new file mode 100644 index 00000000..7a98cc5c --- /dev/null +++ b/rocks-lib/src/luarocks/mod.rs @@ -0,0 +1,12 @@ +pub mod install_binary_rock; +pub mod luarocks_installation; +pub mod rock_manifest; + +/// Retrieves the target compilation platform and returns it as a luarocks identifier. +pub(crate) fn current_platform_luarocks_identifier() -> String { + let platform = match std::env::consts::OS { + "macos" => "macosx", + p => p, + }; + format!("{}-{}", platform, std::env::consts::ARCH) +} diff --git a/rocks-lib/src/luarocks/rock_manifest.rs b/rocks-lib/src/luarocks/rock_manifest.rs new file mode 100644 index 00000000..a504ef7c --- /dev/null +++ b/rocks-lib/src/luarocks/rock_manifest.rs @@ -0,0 +1,122 @@ +use mlua::{FromLua, Lua, LuaSerdeExt as _, Table, Value}; +/// Compatibility layer/adapter for the luarocks client +use std::{collections::HashMap, path::PathBuf}; +use thiserror::Error; + +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct RockManifest { + pub lib: HashMap, + pub lua: HashMap, + pub bin: HashMap, + pub doc: HashMap, + pub root: HashMap, +} + +#[derive(Error, Debug)] +pub enum RockManifestError { + #[error("could not parse rock_manifest: {0}")] + MLua(#[from] mlua::Error), +} + +impl RockManifest { + pub fn new(rock_manifest_content: &str) -> Result { + let lua = Lua::new(); + lua.load(rock_manifest_content).exec()?; + let globals = lua.globals(); + let value = globals.get("rock_manifest")?; + Ok(Self::from_lua(value, &lua)?) + } +} + +impl FromLua for RockManifest { + fn from_lua(value: Value, lua: &Lua) -> mlua::Result { + match &value { + Value::Table(rock_manifest) => { + let lib = rock_manifest_entry_from_lua(rock_manifest, lua, "lib")?; + let lua_entry = rock_manifest_entry_from_lua(rock_manifest, lua, "lua")?; + let bin = rock_manifest_entry_from_lua(rock_manifest, lua, "bin")?; + let doc = rock_manifest_entry_from_lua(rock_manifest, lua, "doc")?; + let mut root = HashMap::new(); + rock_manifest.for_each(|key: String, value: Value| { + if let val @ Value::String(_) = value { + root.insert(PathBuf::from(key), String::from_lua(val, lua)?); + } + Ok(()) + })?; + Ok(Self { + lib, + lua: lua_entry, + bin, + doc, + root, + }) + } + Value::Nil => Ok(Self::default()), + val => Err(mlua::Error::DeserializeError(format!( + "Expected rock_manifest to be a table or nil, but got {}", + val.type_name() + ))), + } + } +} + +fn rock_manifest_entry_from_lua( + rock_manifest: &Table, + lua: &Lua, + key: &str, +) -> mlua::Result> { + if rock_manifest.contains_key(key)? { + lua.from_value(rock_manifest.get(key)?) + } else { + Ok(HashMap::default()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + pub async fn rock_manifest_from_lua() { + let rock_manifest_content = " +rock_manifest = { + doc = { + ['CHANGELOG.md'] = 'adbf3f997070946a5e61955d70bfadb2', + LICENSE = '6bcb3636a93bdb8304439a4ff57e979c', + ['README.md'] = '842bd0b364e36d982f02e22abee7742d' + }, + lib = { + ['toml_edit.so'] = '504d63aea7bb341a688ef28f1232fa9b' + }, + ['toml-edit-0.6.1-1.rockspec'] = 'fcdd3b0066632dec36cd5510e00bc55e' +} + "; + let rock_manifest = RockManifest::new(rock_manifest_content).unwrap(); + assert_eq!( + rock_manifest, + RockManifest { + lib: HashMap::from_iter(vec![( + "toml_edit.so".into(), + "504d63aea7bb341a688ef28f1232fa9b".into() + )]), + lua: HashMap::default(), + bin: HashMap::default(), + doc: HashMap::from_iter(vec![ + ( + "CHANGELOG.md".into(), + "adbf3f997070946a5e61955d70bfadb2".into() + ), + ("LICENSE".into(), "6bcb3636a93bdb8304439a4ff57e979c".into()), + ( + "README.md".into(), + "842bd0b364e36d982f02e22abee7742d".into() + ), + ]), + root: HashMap::from_iter(vec![( + "toml-edit-0.6.1-1.rockspec".into(), + "fcdd3b0066632dec36cd5510e00bc55e".into() + ),]), + } + ); + } +} diff --git a/rocks-lib/src/manifest/metadata.rs b/rocks-lib/src/manifest/metadata.rs deleted file mode 100644 index e69de29b..00000000 diff --git a/rocks-lib/src/manifest/mod.rs b/rocks-lib/src/manifest/mod.rs index ad1d84a4..8ecabdf0 100644 --- a/rocks-lib/src/manifest/mod.rs +++ b/rocks-lib/src/manifest/mod.rs @@ -1,12 +1,14 @@ use itertools::Itertools; use mlua::{Lua, LuaSerdeExt}; use reqwest::{header::ToStrError, Client}; -use std::collections::HashMap; use std::time::SystemTime; +use std::{cmp::Ordering, collections::HashMap}; use thiserror::Error; use tokio::{fs, io}; use url::Url; +use crate::luarocks; +use crate::package::{RemotePackageType, RemotePackageTypeFilterSpec}; use crate::{ config::{Config, LuaVersion}, package::{PackageName, PackageReq, PackageSpec, PackageVersion, RemotePackage}, @@ -95,9 +97,9 @@ async fn manifest_from_server( Ok(new_manifest) } -#[derive(Clone)] +#[derive(Clone, Debug)] pub(crate) struct ManifestMetadata { - pub repository: HashMap>>, + pub repository: HashMap>>, } impl<'de> serde::Deserialize<'de> for ManifestMetadata { @@ -106,7 +108,7 @@ impl<'de> serde::Deserialize<'de> for ManifestMetadata { D: serde::Deserializer<'de>, { let intermediate = IntermediateManifest::deserialize(deserializer)?; - Ok(from_intermediate(intermediate)) + Ok(Self::from_intermediate(intermediate)) } } @@ -130,7 +132,7 @@ impl ManifestMetadata { let intermediate = IntermediateManifest { repository: lua.from_value(lua.globals().get("repository")?)?, }; - let manifest = from_intermediate(intermediate); + let manifest = Self::from_intermediate(intermediate); Ok(manifest) } @@ -147,22 +149,71 @@ impl ManifestMetadata { self.repository[rock_name].keys().sorted().last() } - pub fn latest_match(&self, lua_package_req: &PackageReq) -> Option { + pub fn latest_match( + &self, + lua_package_req: &PackageReq, + filter: Option, + ) -> Option<(PackageSpec, RemotePackageType)> { + let filter = filter.unwrap_or_default(); if !self.has_rock(lua_package_req.name()) { return None; } - let version = self.repository[lua_package_req.name()] - .keys() - .sorted() - .rev() - .find(|version| lua_package_req.version_req().matches(version))?; - - Some(PackageSpec::new( - lua_package_req.name().to_owned(), - version.to_owned(), + let (version, rock_type) = self.repository[lua_package_req.name()] + .iter() + .filter(|(version, _)| lua_package_req.version_req().matches(version)) + .flat_map(|(version, rock_types)| { + rock_types.iter().filter_map(move |rock_type| { + let include = match rock_type { + RemotePackageType::Rockspec => filter.rockspec, + RemotePackageType::Src => filter.src, + RemotePackageType::Binary => filter.binary, + }; + if include { + Some((version, rock_type)) + } else { + None + } + }) + }) + .max_by( + |(version_a, type_a), (version_b, type_b)| match version_a.cmp(version_b) { + Ordering::Equal => type_a.cmp(type_b), + ordering => ordering, + }, + )?; + + Some(( + PackageSpec::new(lua_package_req.name().clone(), version.clone()), + rock_type.clone(), )) } + + /// Construct a `ManifestMetadata` from an intermediate representation, + /// silently skipping entries for versions we don't know how to parse. + fn from_intermediate(intermediate: IntermediateManifest) -> Self { + let repository = intermediate + .repository + .into_iter() + .map(|(name, package_map)| { + ( + name, + package_map + .into_iter() + .filter_map(|(version_str, entries)| { + let version = PackageVersion::parse(version_str.as_str()).ok()?; + let entries = entries + .into_iter() + .filter_map(|entry| RemotePackageType::try_from(entry).ok()) + .collect_vec(); + Some((version, entries)) + }) + .collect(), + ) + }) + .collect(); + Self { repository } + } } #[derive(Clone)] @@ -190,20 +241,53 @@ impl Manifest { pub fn metadata(&self) -> &ManifestMetadata { &self.metadata } - pub fn search(&self, package_req: &PackageReq) -> Option { + pub fn search( + &self, + package_req: &PackageReq, + filter: Option, + ) -> Option { if !self.metadata().has_rock(package_req.name()) { None } else { - Some(RemotePackage::new( - self.metadata().latest_match(package_req).unwrap(), - RemotePackageSource::LuarocksServer(self.server_url().clone()), - )) + let (package, package_type) = + self.metadata().latest_match(package_req, filter).unwrap(); + let remote_source = match package_type { + RemotePackageType::Rockspec => { + RemotePackageSource::LuarocksRockspec(self.server_url().clone()) + } + RemotePackageType::Src => { + RemotePackageSource::LuarocksSrcRock(self.server_url().clone()) + } + RemotePackageType::Binary => { + RemotePackageSource::LuarocksBinaryRock(self.server_url().clone()) + } + }; + Some(RemotePackage::new(package, remote_source)) + } + } +} + +struct UnsupportedArchitectureError; + +impl TryFrom for RemotePackageType { + type Error = UnsupportedArchitectureError; + fn try_from( + ManifestRockEntry { arch }: ManifestRockEntry, + ) -> Result { + match arch.as_str() { + "rockspec" => Ok(RemotePackageType::Rockspec), + "src" => Ok(RemotePackageType::Src), + "all" => Ok(RemotePackageType::Binary), + arch if arch == luarocks::current_platform_luarocks_identifier() => { + Ok(RemotePackageType::Binary) + } + _ => Err(UnsupportedArchitectureError), } } } #[derive(Clone, serde::Deserialize)] -pub struct ManifestRockEntry { +struct ManifestRockEntry { /// e.g. "linux-x86_64", "rockspec", "src", ... pub arch: String, } @@ -215,28 +299,6 @@ struct IntermediateManifest { repository: HashMap>>, } -/// Construct a `ManifestMetadata` from an intermediate representation, -/// silently skipping entries for versions we don't know how to parse. -fn from_intermediate(intermediate: IntermediateManifest) -> ManifestMetadata { - let repository = intermediate - .repository - .into_iter() - .map(|(name, package_map)| { - ( - name, - package_map - .into_iter() - .filter_map(|(version_str, entries)| { - let version = PackageVersion::parse(version_str.as_str()).ok()?; - Some((version, entries)) - }) - .collect(), - ) - }) - .collect(); - ManifestMetadata { repository } -} - #[cfg(test)] mod tests { use std::path::PathBuf; @@ -348,6 +410,6 @@ mod tests { let metadata = ManifestMetadata::new(&manifest).unwrap(); let package_req: PackageReq = "30log > 1.3.0".parse().unwrap(); - assert!(metadata.latest_match(&package_req).is_none()); + assert!(metadata.latest_match(&package_req, None).is_none()); } } diff --git a/rocks-lib/src/operations/download.rs b/rocks-lib/src/operations/download.rs index 6bdcb3f7..4d351c39 100644 --- a/rocks-lib/src/operations/download.rs +++ b/rocks-lib/src/operations/download.rs @@ -1,11 +1,17 @@ -use std::{io, path::PathBuf, string::FromUtf8Error}; +use std::{ + io::{self, Cursor, Read as _}, + path::PathBuf, + string::FromUtf8Error, +}; use bytes::Bytes; use thiserror::Error; +use url::Url; use crate::{ config::Config, - package::{PackageName, PackageReq, PackageVersion, RemotePackage}, + luarocks, + package::{PackageName, PackageReq, PackageSpec, PackageVersion, RemotePackageTypeFilterSpec}, progress::{Progress, ProgressBar}, remote_package_db::{RemotePackageDB, RemotePackageDBError, SearchError}, remote_package_source::RemotePackageSource, @@ -60,7 +66,7 @@ impl<'a> Download<'a> { pub async fn download_src_rock_to_file( self, destination_dir: Option, - ) -> Result { + ) -> Result { match self.package_db { Some(db) => { download_src_rock_to_file(self.package_req, destination_dir, db, self.progress) @@ -77,7 +83,7 @@ impl<'a> Download<'a> { /// Search for a `.src.rock` and download it to memory. pub async fn search_and_download_src_rock( self, - ) -> Result { + ) -> Result { match self.package_db { Some(db) => search_and_download_src_rock(self.package_req, db, self.progress).await, None => { @@ -86,26 +92,68 @@ impl<'a> Download<'a> { } } } + + pub(crate) async fn download_remote_rock( + self, + ) -> Result { + match self.package_db { + Some(db) => download_remote_rock(self.package_req, db, self.progress).await, + None => { + let db = RemotePackageDB::from_config(self.config).await?; + download_remote_rock(self.package_req, &db, self.progress).await + } + } + } } -pub struct DownloadedSrcRockBytes { +pub struct DownloadedPackedRockBytes { pub name: PackageName, pub version: PackageVersion, pub bytes: Bytes, pub file_name: String, } -pub struct DownloadedSrcRock { +pub struct DownloadedPackedRock { pub name: PackageName, pub version: PackageVersion, pub path: PathBuf, } +#[derive(Clone, Debug)] pub struct DownloadedRockspec { pub rockspec: Rockspec, pub(crate) source: RemotePackageSource, } +#[derive(Clone, Debug)] +pub(crate) enum RemoteRockDownload { + RockspecOnly { + rockspec_download: DownloadedRockspec, + }, + BinaryRock { + rockspec_download: DownloadedRockspec, + packed_rock: Bytes, + }, + SrcRock { + rockspec_download: DownloadedRockspec, + _src_rock: Bytes, + }, +} + +impl RemoteRockDownload { + pub fn rockspec(&self) -> &Rockspec { + match self { + Self::RockspecOnly { rockspec_download } + | Self::BinaryRock { + rockspec_download, .. + } + | Self::SrcRock { + rockspec_download, .. + } => &rockspec_download.rockspec, + } + } +} + #[derive(Error, Debug)] pub enum DownloadRockspecError { #[error("failed to download rockspec: {0}")] @@ -114,16 +162,92 @@ pub enum DownloadRockspecError { ResponseConversion(#[from] FromUtf8Error), #[error("error initialising remote package DB: {0}")] RemotePackageDB(#[from] RemotePackageDBError), + #[error(transparent)] + DownloadSrcRock(#[from] DownloadSrcRockError), } +/// Find and download a rockspec for a given package requirement async fn download_rockspec( package_req: &PackageReq, package_db: &RemotePackageDB, progress: &Progress, ) -> Result { - let package = package_db.find(package_req, progress)?; + let rockspec = match download_remote_rock(package_req, package_db, progress).await? { + RemoteRockDownload::RockspecOnly { + rockspec_download: rockspec, + } => rockspec, + RemoteRockDownload::BinaryRock { + rockspec_download: rockspec, + .. + } => rockspec, + RemoteRockDownload::SrcRock { + rockspec_download: rockspec, + .. + } => rockspec, + }; + Ok(rockspec) +} + +async fn download_remote_rock( + package_req: &PackageReq, + package_db: &RemotePackageDB, + progress: &Progress, +) -> Result { + let remote_package = package_db.find(package_req, None, progress)?; progress.map(|p| p.set_message(format!("📥 Downloading rockspec for {}", package_req))); - download_rockspec_impl(package).await + match &remote_package.source { + RemotePackageSource::LuarocksRockspec(url) => { + let package = &remote_package.package; + let rockspec_name = format!("{}-{}.rockspec", package.name(), package.version()); + let bytes = reqwest::get(format!("{}/{}", &url, rockspec_name)) + .await + .map_err(DownloadRockspecError::Request)? + .bytes() + .await + .map_err(DownloadRockspecError::Request)?; + let content = String::from_utf8(bytes.into())?; + let rockspec = DownloadedRockspec { + rockspec: Rockspec::new(&content)?, + source: remote_package.source, + }; + Ok(RemoteRockDownload::RockspecOnly { + rockspec_download: rockspec, + }) + } + RemotePackageSource::RockspecContent(content) => { + let rockspec = DownloadedRockspec { + rockspec: Rockspec::new(content)?, + source: remote_package.source, + }; + Ok(RemoteRockDownload::RockspecOnly { + rockspec_download: rockspec, + }) + } + RemotePackageSource::LuarocksBinaryRock(url) => { + let rock = download_binary_rock(&remote_package.package, url, progress).await?; + let rockspec = DownloadedRockspec { + rockspec: unpack_rockspec(&rock).await?, + source: remote_package.source, + }; + Ok(RemoteRockDownload::BinaryRock { + rockspec_download: rockspec, + packed_rock: rock.bytes, + }) + } + RemotePackageSource::LuarocksSrcRock(url) => { + let rock = download_src_rock(&remote_package.package, url, progress).await?; + let rockspec = DownloadedRockspec { + rockspec: unpack_rockspec(&rock).await?, + source: remote_package.source, + }; + Ok(RemoteRockDownload::SrcRock { + rockspec_download: rockspec, + _src_rock: rock.bytes, + }) + } + #[cfg(test)] + RemotePackageSource::Test => unimplemented!(), + } } #[derive(Error, Debug)] @@ -142,15 +266,29 @@ pub enum SearchAndDownloadError { Rockspec(#[from] RockspecError), #[error("error initialising remote package DB: {0}")] RemotePackageDB(#[from] RemotePackageDBError), + #[error("failed to read packed rock: {0}")] + Zip(#[from] zip::result::ZipError), + #[error("{0} not found in the source rock.")] + RockspecNotFoundInSrcRock(String), } async fn search_and_download_src_rock( package_req: &PackageReq, package_db: &RemotePackageDB, progress: &Progress, -) -> Result { - let package = package_db.find(package_req, progress)?; - Ok(download_src_rock(&package, progress).await?) +) -> Result { + let filter = Some(RemotePackageTypeFilterSpec { + rockspec: false, + binary: false, + src: true, + }); + let remote_package = package_db.find(package_req, filter, progress)?; + Ok(download_src_rock( + &remote_package.package, + unsafe { &remote_package.source.url() }, + progress, + ) + .await?) } #[derive(Error, Debug)] @@ -158,12 +296,25 @@ async fn search_and_download_src_rock( pub struct DownloadSrcRockError(#[from] reqwest::Error); pub(crate) async fn download_src_rock( - remote_package: &RemotePackage, + package: &PackageSpec, + server_url: &Url, progress: &Progress, -) -> Result { - progress.map(|p| p.set_message(format!("📥 Downloading {}", remote_package.package))); +) -> Result { + download_packed_rock_impl(package, server_url, "src.rock", progress).await +} - download_src_rock_impl(remote_package).await +pub(crate) async fn download_binary_rock( + package: &PackageSpec, + server_url: &Url, + progress: &Progress, +) -> Result { + download_packed_rock_impl( + package, + server_url, + format!("{}.rock", luarocks::current_platform_luarocks_identifier()).as_str(), + progress, + ) + .await } async fn download_src_rock_to_file( @@ -171,11 +322,11 @@ async fn download_src_rock_to_file( destination_dir: Option, package_db: &RemotePackageDB, progress: &Progress, -) -> Result { +) -> Result { progress.map(|p| p.set_message(format!("📥 Downloading {}", package_req))); let rock = search_and_download_src_rock(package_req, package_db, progress).await?; - let full_rock_name = full_rock_name(&rock.name, &rock.version); + let full_rock_name = mk_packed_rock_name(&rock.name, &rock.version, "src.rock"); tokio::fs::write( destination_dir .map(|dest| dest.join(&full_rock_name)) @@ -184,52 +335,34 @@ async fn download_src_rock_to_file( ) .await?; - Ok(DownloadedSrcRock { + Ok(DownloadedPackedRock { name: rock.name.to_owned(), version: rock.version.to_owned(), path: full_rock_name.into(), }) } -async fn download_rockspec_impl( - remote_package: RemotePackage, -) -> Result { - match &remote_package.source { - RemotePackageSource::LuarocksServer(url) => { - let package = &remote_package.package; - let rockspec_name = format!("{}-{}.rockspec", package.name(), package.version()); - let bytes = reqwest::get(format!("{}/{}", &url, rockspec_name)) - .await - .map_err(DownloadRockspecError::Request)? - .bytes() - .await - .map_err(DownloadRockspecError::Request)?; - let content = String::from_utf8(bytes.into())?; - Ok(DownloadedRockspec { - rockspec: Rockspec::new(&content)?, - source: remote_package.source, - }) - } - RemotePackageSource::RockspecContent(content) => Ok(DownloadedRockspec { - rockspec: Rockspec::new(content)?, - source: remote_package.source, - }), - #[cfg(test)] - RemotePackageSource::Test => unimplemented!(), - } -} - -async fn download_src_rock_impl( - remote_package: &RemotePackage, -) -> Result { - let package = &remote_package.package; - let full_rock_name = full_rock_name(package.name(), package.version()); +async fn download_packed_rock_impl( + package: &PackageSpec, + server_url: &Url, + ext: &str, + progress: &Progress, +) -> Result { + progress.map(|p| { + p.set_message(format!( + "📥 Downloading {}-{}.{}", + package.name(), + package.version(), + ext, + )) + }); + let full_rock_name = mk_packed_rock_name(package.name(), package.version(), ext); - let bytes = reqwest::get(format!("{}/{}", remote_package.source, full_rock_name)) + let bytes = reqwest::get(format!("{}/{}", server_url, full_rock_name)) .await? .bytes() .await?; - Ok(DownloadedSrcRockBytes { + Ok(DownloadedPackedRockBytes { name: package.name().clone(), version: package.version().clone(), bytes, @@ -237,6 +370,24 @@ async fn download_src_rock_impl( }) } -fn full_rock_name(name: &PackageName, version: &PackageVersion) -> String { - format!("{}-{}.src.rock", name, version) +fn mk_packed_rock_name(name: &PackageName, version: &PackageVersion, ext: &str) -> String { + format!("{}-{}.{}", name, version, ext) +} + +pub(crate) async fn unpack_rockspec( + rock: &DownloadedPackedRockBytes, +) -> Result { + let cursor = Cursor::new(&rock.bytes); + let rockspec_file_name = format!("{}-{}.rockspec", rock.name, rock.version); + let mut zip = zip::ZipArchive::new(cursor)?; + let rockspec_index = (0..zip.len()) + .find(|&i| zip.by_index(i).unwrap().name().eq(&rockspec_file_name)) + .ok_or(SearchAndDownloadError::RockspecNotFoundInSrcRock( + rockspec_file_name, + ))?; + let mut rockspec_file = zip.by_index(rockspec_index)?; + let mut content = String::new(); + rockspec_file.read_to_string(&mut content)?; + let rockspec = Rockspec::new(&content)?; + Ok(rockspec) } diff --git a/rocks-lib/src/operations/fetch.rs b/rocks-lib/src/operations/fetch.rs index 1a367bdf..7bf34beb 100644 --- a/rocks-lib/src/operations/fetch.rs +++ b/rocks-lib/src/operations/fetch.rs @@ -15,10 +15,8 @@ use thiserror::Error; use crate::config::Config; use crate::operations; use crate::package::PackageSpec; -use crate::package::RemotePackage; use crate::progress::Progress; use crate::progress::ProgressBar; -use crate::remote_package_source::RemotePackageSource; use crate::{rockspec::RockSource, rockspec::RockSourceSpec}; use super::DownloadSrcRockError; @@ -141,9 +139,7 @@ pub async fn fetch_src_rock( config: &Config, progress: &Progress, ) -> Result<(), FetchSrcRockError> { - let source = RemotePackageSource::LuarocksServer(config.server().clone()); - let remote_package = RemotePackage::new(package.clone(), source); - let src_rock = operations::download_src_rock(&remote_package, progress).await?; + let src_rock = operations::download_src_rock(package, config.server(), progress).await?; let cursor = Cursor::new(src_rock.bytes); let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type()); unpack( diff --git a/rocks-lib/src/operations/install.rs b/rocks-lib/src/operations/install.rs index f1bdc3a0..face5708 100644 --- a/rocks-lib/src/operations/install.rs +++ b/rocks-lib/src/operations/install.rs @@ -3,9 +3,13 @@ use std::{collections::HashMap, io, sync::Arc}; use crate::{ build::{Build, BuildBehaviour, BuildError}, config::{Config, LuaVersion, LuaVersionUnset}, - lockfile::{LocalPackage, LocalPackageId, Lockfile, PinnedState}, - luarocks_installation::{ - InstallBuildDependenciesError, LuaRocksError, LuaRocksInstallError, LuaRocksInstallation, + lockfile::{LocalPackage, LocalPackageId, LockConstraint, Lockfile, PinnedState}, + luarocks::{ + install_binary_rock::{BinaryRockInstall, InstallBinaryRockError}, + luarocks_installation::{ + InstallBuildDependenciesError, LuaRocksError, LuaRocksInstallError, + LuaRocksInstallation, + }, }, package::{PackageName, PackageReq}, progress::{MultiProgress, Progress, ProgressBar}, @@ -14,11 +18,14 @@ use crate::{ tree::Tree, }; +use bytes::Bytes; use futures::future::join_all; use itertools::Itertools; use thiserror::Error; -use super::{resolve::get_all_dependencies, SearchAndDownloadError}; +use super::{ + resolve::get_all_dependencies, DownloadedRockspec, RemoteRockDownload, SearchAndDownloadError, +}; /// A rocks package installer, providing fine-grained control /// over how packages should be installed. @@ -115,6 +122,8 @@ pub enum InstallError { BuildError(PackageName, BuildError), #[error("error initialising remote package DB: {0}")] RemotePackageDB(#[from] RemotePackageDBError), + #[error("failed to install pre-built rock {0}: {1}")] + InstallBinaryRockError(PackageName, InstallBinaryRockError), } async fn install( @@ -142,7 +151,6 @@ async fn install_impl( lockfile: &mut Lockfile, progress_arc: Arc>, ) -> Result, InstallError> { - let progress = Arc::clone(&progress_arc); let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); get_all_dependencies( @@ -164,38 +172,51 @@ async fn install_impl( let installed_packages = join_all(all_packages.clone().into_values().map(|install_spec| { let progress_arc = progress_arc.clone(); - let package = install_spec.rockspec.package.clone(); - - let bar = progress.map(|p| { - p.add(ProgressBar::from(format!( - "💻 Installing {}", - install_spec.rockspec.package, - ))) - }); + let downloaded_rock = install_spec.downloaded_rock; let config = config.clone(); tokio::spawn(async move { - let rockspec = install_spec.rockspec; + let rockspec = downloaded_rock.rockspec(); if let Some(BuildBackendSpec::LuaRock(build_backend)) = &rockspec.build.current_platform().build_backend { let luarocks = LuaRocksInstallation::new(&config)?; - luarocks.ensure_installed(&bar).await?; luarocks - .install_build_dependencies(build_backend, &rockspec, progress_arc) + .install_build_dependencies(build_backend, rockspec, progress_arc.clone()) .await?; } - let pkg = Build::new(rockspec, &config, &bar) - .pin(pin) - .constraint(install_spec.spec.constraint()) - .behaviour(install_spec.build_behaviour) - .source(install_spec.source) - .build() - .await - .map_err(|err| InstallError::BuildError(package, err))?; - - bar.map(|b| b.finish_and_clear()); + let pkg = match downloaded_rock { + RemoteRockDownload::RockspecOnly { rockspec_download } => { + install_rockspec( + rockspec_download, + install_spec.spec.constraint(), + install_spec.build_behaviour, + pin, + &config, + progress_arc, + ) + .await? + } + RemoteRockDownload::BinaryRock { + rockspec_download, + packed_rock, + } => { + install_binary_rock( + rockspec_download, + packed_rock, + install_spec.spec.constraint(), + install_spec.build_behaviour, + pin, + &config, + progress_arc, + ) + .await? + } + RemoteRockDownload::SrcRock { .. } => todo!( + "rocks does not yet support installing .src.rock packages without a rockspec" + ), + }; Ok::<_, InstallError>((pkg.id(), pkg)) }) @@ -225,3 +246,78 @@ async fn install_impl( Ok(installed_packages.into_values().collect_vec()) } + +async fn install_rockspec( + rockspec_download: DownloadedRockspec, + constraint: LockConstraint, + behaviour: BuildBehaviour, + pin: PinnedState, + config: &Config, + progress_arc: Arc>, +) -> Result { + let progress = Arc::clone(&progress_arc); + let rockspec = rockspec_download.rockspec; + let source = rockspec_download.source; + let package = rockspec.package.clone(); + let bar = progress.map(|p| p.add(ProgressBar::from(format!("💻 Installing {}", &package,)))); + + if let Some(BuildBackendSpec::LuaRock(build_backend)) = + &rockspec.build.current_platform().build_backend + { + let luarocks = LuaRocksInstallation::new(config)?; + luarocks.ensure_installed(&bar).await?; + luarocks + .install_build_dependencies(build_backend, &rockspec, progress_arc) + .await?; + } + + let pkg = Build::new(&rockspec, config, &bar) + .pin(pin) + .constraint(constraint) + .behaviour(behaviour) + .source(source) + .build() + .await + .map_err(|err| InstallError::BuildError(package, err))?; + + bar.map(|b| b.finish_and_clear()); + + Ok(pkg) +} + +async fn install_binary_rock( + rockspec_download: DownloadedRockspec, + packed_rock: Bytes, + constraint: LockConstraint, + behaviour: BuildBehaviour, + pin: PinnedState, + config: &Config, + progress_arc: Arc>, +) -> Result { + let progress = Arc::clone(&progress_arc); + let rockspec = rockspec_download.rockspec; + let package = rockspec.package.clone(); + let bar = progress.map(|p| { + p.add(ProgressBar::from(format!( + "💻 Installing {} (pre-built)", + &package, + ))) + }); + let pkg = BinaryRockInstall::new( + &rockspec, + rockspec_download.source, + packed_rock, + config, + &bar, + ) + .pin(pin) + .constraint(constraint) + .behaviour(behaviour) + .install() + .await + .map_err(|err| InstallError::InstallBinaryRockError(package, err))?; + + bar.map(|b| b.finish_and_clear()); + + Ok(pkg) +} diff --git a/rocks-lib/src/operations/resolve.rs b/rocks-lib/src/operations/resolve.rs index d5f5090b..542676f6 100644 --- a/rocks-lib/src/operations/resolve.rs +++ b/rocks-lib/src/operations/resolve.rs @@ -13,18 +13,15 @@ use crate::{ package::{PackageReq, PackageVersionReq}, progress::{MultiProgress, Progress}, remote_package_db::RemotePackageDB, - remote_package_source::RemotePackageSource, - rockspec::Rockspec, }; -use super::{Download, SearchAndDownloadError}; +use super::{Download, RemoteRockDownload, SearchAndDownloadError}; #[derive(Clone, Debug)] pub(crate) struct PackageInstallSpec { pub build_behaviour: BuildBehaviour, - pub rockspec: Rockspec, + pub downloaded_rock: RemoteRockDownload, pub spec: LocalPackageSpec, - pub source: RemotePackageSource, } #[async_recursion] @@ -54,11 +51,10 @@ pub(crate) async fn get_all_dependencies( tokio::spawn(async move { let bar = progress.map(|p| p.new_bar()); - let rockspec_download = Download::new(&package, &config, &bar) + let downloaded_rock = Download::new(&package, &config, &bar) .package_db(&package_db) - .download_rockspec() + .download_remote_rock() .await?; - let rockspec = rockspec_download.rockspec; let constraint = if *package.version_req() == PackageVersionReq::SemVer(VersionReq::STAR) { @@ -67,7 +63,8 @@ pub(crate) async fn get_all_dependencies( LockConstraint::Constrained(package.version_req().clone()) }; - let dependencies = rockspec + let dependencies = downloaded_rock + .rockspec() .dependencies .current_platform() .iter() @@ -86,6 +83,7 @@ pub(crate) async fn get_all_dependencies( ) .await?; + let rockspec = downloaded_rock.rockspec(); let local_spec = LocalPackageSpec::new( &rockspec.package, &rockspec.version, @@ -97,8 +95,7 @@ pub(crate) async fn get_all_dependencies( let install_spec = PackageInstallSpec { build_behaviour, spec: local_spec.clone(), - rockspec, - source: rockspec_download.source, + downloaded_rock, }; tx.send(install_spec).unwrap(); diff --git a/rocks-lib/src/operations/unpack.rs b/rocks-lib/src/operations/unpack.rs index 15f382df..2b3151df 100644 --- a/rocks-lib/src/operations/unpack.rs +++ b/rocks-lib/src/operations/unpack.rs @@ -43,7 +43,7 @@ mod tests { use super::*; #[tokio::test] - pub async fn unpack_rock() { + pub async fn test_unpack_src_rock() { let test_rock_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources") .join("test") diff --git a/rocks-lib/src/package/mod.rs b/rocks-lib/src/package/mod.rs index 125dba81..2fdc64c7 100644 --- a/rocks-lib/src/package/mod.rs +++ b/rocks-lib/src/package/mod.rs @@ -1,7 +1,7 @@ use itertools::Itertools; use mlua::FromLua; use serde::{de, Deserialize, Deserializer, Serialize}; -use std::{fmt::Display, str::FromStr}; +use std::{cmp::Ordering, fmt::Display, str::FromStr}; use thiserror::Error; mod outdated; @@ -72,6 +72,55 @@ impl RemotePackage { } } +#[derive(PartialEq, Eq, Hash, Clone, Debug)] +pub(crate) enum RemotePackageType { + Rockspec, + Src, + Binary, +} + +impl Ord for RemotePackageType { + fn cmp(&self, other: &Self) -> Ordering { + // Priority: binary > rockspec > src + match (self, other) { + (Self::Binary, Self::Binary) + | (Self::Rockspec, Self::Rockspec) + | (Self::Src, Self::Src) => Ordering::Equal, + + (Self::Binary, _) => Ordering::Greater, + (_, Self::Binary) => Ordering::Less, + (Self::Rockspec, Self::Src) => Ordering::Greater, + (Self::Src, Self::Rockspec) => Ordering::Less, + } + } +} + +impl PartialOrd for RemotePackageType { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[derive(Clone)] +pub struct RemotePackageTypeFilterSpec { + /// Include Rockspec + pub rockspec: bool, + /// Include Src + pub src: bool, + /// Include Binary + pub binary: bool, +} + +impl Default for RemotePackageTypeFilterSpec { + fn default() -> Self { + Self { + rockspec: true, + src: true, + binary: true, + } + } +} + #[derive(Error, Debug)] pub enum ParseRemotePackageError { #[error("unable to parse package {0}. expected format: `name@version`")] @@ -371,4 +420,21 @@ mod tests { let lua_utils = PackageSpec::parse("lua-utils.nvim".into(), "1.2-1".into()).unwrap(); assert!(!package_req.matches(&lua_utils)); } + + #[tokio::test] + pub async fn remote_package_type_priorities() { + let rock_types = vec![ + RemotePackageType::Binary, + RemotePackageType::Src, + RemotePackageType::Rockspec, + ]; + assert_eq!( + rock_types.into_iter().sorted().collect_vec(), + vec![ + RemotePackageType::Src, + RemotePackageType::Rockspec, + RemotePackageType::Binary, + ] + ); + } } diff --git a/rocks-lib/src/package/outdated.rs b/rocks-lib/src/package/outdated.rs index d9627f14..35119b16 100644 --- a/rocks-lib/src/package/outdated.rs +++ b/rocks-lib/src/package/outdated.rs @@ -44,7 +44,7 @@ impl PackageSpec { ) -> Result, RockConstraintUnsatisfied> { let latest_version = package_db - .latest_match(constraint) + .latest_match(constraint, None) .ok_or_else(|| RockConstraintUnsatisfied { name: self.name.clone(), constraint: constraint.version_req.clone(), diff --git a/rocks-lib/src/remote_package_db/mod.rs b/rocks-lib/src/remote_package_db/mod.rs index 94a7e822..dd2b59dc 100644 --- a/rocks-lib/src/remote_package_db/mod.rs +++ b/rocks-lib/src/remote_package_db/mod.rs @@ -1,7 +1,10 @@ use crate::{ config::Config, manifest::{Manifest, ManifestError}, - package::{PackageName, PackageReq, PackageSpec, PackageVersion, RemotePackage}, + package::{ + PackageName, PackageReq, PackageSpec, PackageVersion, RemotePackage, + RemotePackageTypeFilterSpec, + }, progress::{Progress, ProgressBar}, }; use itertools::Itertools as _; @@ -41,11 +44,12 @@ impl RemotePackageDB { pub(crate) fn find( &self, package_req: &PackageReq, + filter: Option, progress: &Progress, ) -> Result { let result = self.0.iter().find_map(|manifest| { progress.map(|p| p.set_message(format!("🔎 Searching {}", &manifest.server_url()))); - manifest.search(package_req) + manifest.search(package_req, filter.clone()) }); match result { Some(package) => Ok(package), @@ -88,10 +92,19 @@ impl RemotePackageDB { .last() } - pub fn latest_match(&self, package_req: &PackageReq) -> Option { + pub fn latest_match( + &self, + package_req: &PackageReq, + filter: Option, + ) -> Option { self.0 .iter() - .filter_map(|manifest| manifest.metadata().latest_match(package_req)) + .filter_map(|manifest| { + manifest + .metadata() + .latest_match(package_req, filter.clone()) + .map(|x| x.0) + }) .last() } } diff --git a/rocks-lib/src/remote_package_source/mod.rs b/rocks-lib/src/remote_package_source/mod.rs index 39b26b20..73e6478d 100644 --- a/rocks-lib/src/remote_package_source/mod.rs +++ b/rocks-lib/src/remote_package_source/mod.rs @@ -12,16 +12,39 @@ const PLUS: &str = "+"; /// The source of a remote package. #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] pub(crate) enum RemotePackageSource { - LuarocksServer(Url), + LuarocksRockspec(Url), + LuarocksSrcRock(Url), + LuarocksBinaryRock(Url), RockspecContent(String), #[cfg(test)] Test, } +impl RemotePackageSource { + pub(crate) unsafe fn url(self) -> Url { + match self { + Self::LuarocksRockspec(url) + | Self::LuarocksSrcRock(url) + | Self::LuarocksBinaryRock(url) => url, + Self::RockspecContent(_) => panic!("tried to get URL from RockspecContent"), + #[cfg(test)] + Self::Test => unimplemented!(), + } + } +} + impl Display for RemotePackageSource { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match &self { - RemotePackageSource::LuarocksServer(url) => format!("luarocks{}{}", PLUS, url).fmt(f), + RemotePackageSource::LuarocksRockspec(url) => { + format!("luarocks_rockspec{}{}", PLUS, url).fmt(f) + } + RemotePackageSource::LuarocksSrcRock(url) => { + format!("luarocks_src_rock{}{}", PLUS, url).fmt(f) + } + RemotePackageSource::LuarocksBinaryRock(url) => { + format!("luarocks_rock{}{}", PLUS, url).fmt(f) + } RemotePackageSource::RockspecContent(content) => { format!("rockspec{}{}", PLUS, content).fmt(f) } @@ -60,7 +83,15 @@ impl TryFrom for RemotePackageSource { if let Some(str) = value.get(pos + 1..) { let remote_source_type = value[..pos].into(); match remote_source_type { - "luarocks" => Ok(RemotePackageSource::LuarocksServer(Url::parse(str)?)), + "luarocks_rockspec" => { + Ok(RemotePackageSource::LuarocksRockspec(Url::parse(str)?)) + } + "luarocks_src_rock" => { + Ok(RemotePackageSource::LuarocksSrcRock(Url::parse(str)?)) + } + "luarocks_rock" => { + Ok(RemotePackageSource::LuarocksBinaryRock(Url::parse(str)?)) + } "rockspec" => Ok(RemotePackageSource::RockspecContent(str.into())), _ => Err(RemotePackageSourceError::UnknownRemoteSourceType( remote_source_type.into(), @@ -101,8 +132,14 @@ source = { #[test] fn luarocks_source_roundtrip() { - let source = - RemotePackageSource::LuarocksServer(Url::parse("https://luarocks.org/").unwrap()); + let url = Url::parse("https://luarocks.org/").unwrap(); + let source = RemotePackageSource::LuarocksRockspec(url.clone()); + let roundtripped = RemotePackageSource::try_from(format!("{}", source)).unwrap(); + assert_eq!(source, roundtripped); + let source = RemotePackageSource::LuarocksSrcRock(url.clone()); + let roundtripped = RemotePackageSource::try_from(format!("{}", source)).unwrap(); + assert_eq!(source, roundtripped); + let source = RemotePackageSource::LuarocksBinaryRock(url); let roundtripped = RemotePackageSource::try_from(format!("{}", source)).unwrap(); assert_eq!(source, roundtripped) } diff --git a/rocks-lib/src/rockspec/mod.rs b/rocks-lib/src/rockspec/mod.rs index 1950290f..12ac5d34 100644 --- a/rocks-lib/src/rockspec/mod.rs +++ b/rocks-lib/src/rockspec/mod.rs @@ -28,11 +28,11 @@ use crate::{ #[derive(Error, Debug)] pub enum RockspecError { - #[error(transparent)] + #[error("could not parse rockspec: {0}")] MLua(#[from] mlua::Error), #[error("{}copy_directories cannot contain the rockspec name", ._0.as_ref().map(|p| format!("{p}: ")).unwrap_or_default())] CopyDirectoriesContainRockspecName(Option), - #[error(transparent)] + #[error("could not parse rockspec: {0}")] LuaTable(#[from] LuaTableError), } diff --git a/rocks-lib/tests/build.rs b/rocks-lib/tests/build.rs index 9ba05669..6b10f309 100644 --- a/rocks-lib/tests/build.rs +++ b/rocks-lib/tests/build.rs @@ -25,7 +25,7 @@ async fn builtin_build() { let progress = MultiProgress::new(); let bar = progress.new_bar(); - Build::new(rockspec, &config, &Progress::Progress(bar)) + Build::new(&rockspec, &config, &Progress::Progress(bar)) .behaviour(Force) .build() .await @@ -50,7 +50,7 @@ async fn make_build() { let progress = MultiProgress::new(); let bar = progress.new_bar(); - Build::new(rockspec, &config, &Progress::Progress(bar)) + Build::new(&rockspec, &config, &Progress::Progress(bar)) .behaviour(Force) .build() .await @@ -87,7 +87,7 @@ async fn test_build_rockspec(rockspec_path: PathBuf) { let progress = MultiProgress::new(); let bar = progress.new_bar(); - Build::new(rockspec, &config, &Progress::Progress(bar)) + Build::new(&rockspec, &config, &Progress::Progress(bar)) .behaviour(Force) .build() .await diff --git a/rocks-lib/tests/luarocks_installation.rs b/rocks-lib/tests/luarocks_installation.rs index d3682c11..92f33ef8 100644 --- a/rocks-lib/tests/luarocks_installation.rs +++ b/rocks-lib/tests/luarocks_installation.rs @@ -8,7 +8,7 @@ use rocks_lib::progress::{MultiProgress, Progress, ProgressBar}; use rocks_lib::{ config::{ConfigBuilder, LuaVersion}, lua_installation::LuaInstallation, - luarocks_installation::LuaRocksInstallation, + luarocks::luarocks_installation::LuaRocksInstallation, }; #[tokio::test]