feat!: support installing from multiple servers (#285)

rocks-bin/src/build.rs

@@ -9,9 +9,9 @@ use rocks_lib::{
     build::BuildBehaviour,
     config::Config,
     lockfile::{LockConstraint::Unconstrained, PinnedState},
-    manifest::Manifest,
     package::{PackageName, PackageReq},
     progress::MultiProgress,
+    remote_package_db::RemotePackageDB,
     rockspec::Rockspec,
     tree::Tree,
 };
@@ -69,7 +69,7 @@ pub async fn build(data: Build, config: Config) -> Result<()> {
     let lua_version = rockspec.lua_version_from_config(&config)?;
 
     let tree = Tree::new(config.tree().clone(), lua_version)?;
-    let manifest = Manifest::from_config(config.server(), &config).await?;
+    let package_db = RemotePackageDB::from_config(&config).await?;
 
     let build_behaviour = match tree.has_rock_and(
         &PackageReq::new(
@@ -114,7 +114,7 @@ pub async fn build(data: Build, config: Config) -> Result<()> {
     rocks_lib::operations::install(
         dependencies_to_install,
         pin,
-        &manifest,
+        &package_db,
         &config,
         progress_arc,
     )

rocks-bin/src/download.rs

@@ -2,9 +2,9 @@ use clap::Args;
 use eyre::Result;
 use rocks_lib::{
     config::Config,
-    manifest::Manifest,
     package::PackageReq,
     progress::{MultiProgress, Progress},
+    remote_package_db::RemotePackageDB,
 };
 
 #[derive(Args)]
@@ -13,18 +13,13 @@ pub struct Download {
 }
 
 pub async fn download(dl_data: Download, config: Config) -> Result<()> {
-    let manifest = Manifest::from_config(config.server(), &config).await?;
+    let package_db = RemotePackageDB::from_config(&config).await?;
     let progress = MultiProgress::new();
     let bar = Progress::Progress(progress.new_bar());
 
-    let rock = rocks_lib::operations::download_to_file(
-        &dl_data.package_req,
-        None,
-        &manifest,
-        &config,
-        &bar,
-    )
-    .await?;
+    let rock =
+        rocks_lib::operations::download_to_file(&dl_data.package_req, None, &package_db, &bar)
+            .await?;
 
     bar.map(|b| {
         b.finish_with_message(format!(

rocks-bin/src/fetch.rs

@@ -3,8 +3,8 @@ use std::path::PathBuf;
 use eyre::Result;
 use rocks_lib::{
     config::Config,
-    manifest::Manifest,
     progress::{MultiProgress, Progress},
+    remote_package_db::RemotePackageDB,
 };
 
 use crate::unpack::UnpackRemote;
@@ -13,9 +13,9 @@ pub async fn fetch_remote(data: UnpackRemote, config: Config) -> Result<()> {
     let package_req = data.package_req;
     let progress = MultiProgress::new();
     let bar = Progress::Progress(progress.new_bar());
-    let manifest = Manifest::from_config(config.server(), &config).await?;
+    let package_db = RemotePackageDB::from_config(&config).await?;
     let rockspec =
-        rocks_lib::operations::download_rockspec(&package_req, &manifest, &config, &bar).await?;
+        rocks_lib::operations::download_rockspec(&package_req, &package_db, &bar).await?;
 
     let destination = data
         .path

rocks-bin/src/info.rs

@@ -2,10 +2,10 @@ use clap::Args;
 use eyre::Result;
 use rocks_lib::{
     config::{Config, LuaVersion},
-    manifest::Manifest,
     operations::download_rockspec,
     package::PackageReq,
     progress::{MultiProgress, Progress},
+    remote_package_db::RemotePackageDB,
     tree::Tree,
 };
 
@@ -17,12 +17,12 @@ pub struct Info {
 pub async fn info(data: Info, config: Config) -> Result<()> {
     // TODO(vhyrro): Add `Tree::from(&Config)`
     let tree = Tree::new(config.tree().clone(), LuaVersion::from(&config)?)?;
-    let manifest = Manifest::from_config(config.server(), &config).await?;
+    let package_db = RemotePackageDB::from_config(&config).await?;
 
     let progress = MultiProgress::new();
     let bar = Progress::Progress(progress.new_bar());
 
-    let rockspec = download_rockspec(&data.package, &manifest, &config, &bar).await?;
+    let rockspec = download_rockspec(&data.package, &package_db, &bar).await?;
 
     bar.map(|b| b.finish_and_clear());
 

rocks-bin/src/install.rs

@@ -5,9 +5,9 @@ use rocks_lib::{
     build::BuildBehaviour,
     config::{Config, LuaVersion},
     lockfile::PinnedState,
-    manifest::Manifest,
     package::PackageReq,
     progress::MultiProgress,
+    remote_package_db::RemotePackageDB,
     tree::Tree,
 };
 
@@ -54,11 +54,17 @@ pub async fn install(data: Install, config: Config) -> Result<()> {
         })
         .collect_vec();
 
-    let manifest = Manifest::from_config(config.server(), &config).await?;
+    let package_db = RemotePackageDB::from_config(&config).await?;
 
     // TODO(vhyrro): If the tree doesn't exist then error out.
-    rocks_lib::operations::install(packages, pin, &manifest, &config, MultiProgress::new_arc())
-        .await?;
+    rocks_lib::operations::install(
+        packages,
+        pin,
+        &package_db,
+        &config,
+        MultiProgress::new_arc(),
+    )
+    .await?;
 
     Ok(())
 }

rocks-bin/src/main.rs

@@ -43,10 +43,10 @@ pub struct Cli {
     #[arg(long, value_name = "server")]
     pub server: Option<String>,
 
-    /// Fetch rocks/rockspecs from this server only (overrides
-    /// any entries in the config file).
-    #[arg(long, value_name = "server")]
-    pub only_server: Option<String>,
+    /// Fetch rocks/rockspecs from this server in addition to the main server
+    /// (overrides any entries in the config file).
+    #[arg(long, value_name = "extra-server")]
+    pub extra_servers: Option<Vec<String>>,
 
     /// Restrict downloads to paths matching the given URL.
     #[arg(long, value_name = "url")]
@@ -165,7 +165,7 @@ async fn main() {
         .lua_dir(cli.lua_dir)
         .lua_version(cli.lua_version)
         .namespace(cli.namespace)
-        .only_server(cli.only_server)
+        .extra_servers(cli.extra_servers)
         .only_sources(cli.only_sources)
         .server(cli.server)
         .tree(cli.tree)

rocks-bin/src/outdated.rs

@@ -5,8 +5,8 @@ use eyre::Result;
 use itertools::Itertools;
 use rocks_lib::{
     config::{Config, LuaVersion},
-    manifest::Manifest,
     progress::{MultiProgress, ProgressBar},
+    remote_package_db::RemotePackageDB,
     tree::Tree,
 };
 use text_trees::{FormatCharacters, StringTreeNode, TreeFormatting};
@@ -25,7 +25,7 @@ pub async fn outdated(outdated_data: Outdated, config: Config) -> Result<()> {
 
     let tree = Tree::new(config.tree().clone(), LuaVersion::from(&config)?)?;
 
-    let manifest = Manifest::from_config(config.server(), &config).await?;
+    let package_db = RemotePackageDB::from_config(&config).await?;
 
     // NOTE: This will display all installed versions and each possible upgrade.
     // However, this should also take into account dependency constraints made by other rocks.
@@ -36,7 +36,7 @@ pub async fn outdated(outdated_data: Outdated, config: Config) -> Result<()> {
         .iter()
         .filter_map(|rock| {
             rock.to_package()
-                .has_update(manifest.metadata())
+                .has_update(&package_db)
                 .expect("TODO")
                 .map(|version| (rock, version))
         })

rocks-bin/src/remove.rs

@@ -2,9 +2,9 @@ use clap::Args;
 use eyre::Result;
 use rocks_lib::{
     config::{Config, LuaVersion},
-    manifest::Manifest,
     package::{PackageName, PackageSpec, PackageVersion},
     progress::{MultiProgress, Progress},
+    remote_package_db::RemotePackageDB,
     tree::Tree,
 };
 
@@ -17,14 +17,11 @@ pub struct Remove {
 }
 
 pub async fn remove(remove_args: Remove, config: Config) -> Result<()> {
-    let manifest = Manifest::from_config(config.server(), &config).await?;
+    let package_db = RemotePackageDB::from_config(&config).await?;
 
     let target_version = remove_args
         .version
-        .or(manifest
-            .metadata()
-            .latest_version(&remove_args.name)
-            .cloned())
+        .or(package_db.latest_version(&remove_args.name).cloned())
         .unwrap();
 
     let tree = Tree::new(config.tree().clone(), LuaVersion::from(&config)?)?;

rocks-bin/src/search.rs

@@ -7,9 +7,9 @@ use text_trees::{FormatCharacters, StringTreeNode, TreeFormatting};
 
 use rocks_lib::{
     config::Config,
-    manifest::Manifest,
     package::{PackageName, PackageReq, PackageVersion},
     progress::{MultiProgress, ProgressBar},
+    remote_package_db::RemotePackageDB,
 };
 
 #[derive(Args)]
@@ -30,39 +30,20 @@ pub async fn search(data: Search, config: Config) -> Result<()> {
 
     let formatting = TreeFormatting::dir_tree(FormatCharacters::box_chars());
 
-    let manifest = Manifest::from_config(config.server(), &config).await?;
+    let package_db = RemotePackageDB::from_config(&config).await?;
 
     let lua_package_req = data.lua_package_req;
 
-    let rock_to_version_map: HashMap<&PackageName, Vec<&PackageVersion>> = manifest
-        .metadata()
-        .repository
-        .iter()
-        .filter_map(|(name, elements)| {
-            if name
-                .to_string()
-                .contains(&lua_package_req.name().to_string())
-            {
-                Some((
-                    name,
-                    elements
-                        .keys()
-                        .filter(|version| lua_package_req.version_req().matches(version))
-                        .sorted_by(|a, b| Ord::cmp(b, a))
-                        .collect_vec(),
-                ))
-            } else {
-                None
-            }
-        })
-        .collect();
+    let result = package_db.search(&lua_package_req);
 
     bar.finish_and_clear();
 
     if data.porcelain {
+        let rock_to_version_map: HashMap<&PackageName, Vec<&PackageVersion>> =
+            HashMap::from_iter(result);
         println!("{}", serde_json::to_string(&rock_to_version_map)?);
     } else {
-        for (key, versions) in rock_to_version_map.into_iter().sorted() {
+        for (key, versions) in result.into_iter().sorted() {
             let mut tree = StringTreeNode::new(key.to_string().to_owned());
 
             for version in versions {

rocks-bin/src/test.rs

@@ -2,10 +2,10 @@ use clap::Args;
 use eyre::{OptionExt, Result};
 use rocks_lib::{
     config::Config,
-    manifest::Manifest,
     operations::{ensure_busted, ensure_dependencies, run_tests, TestEnv},
     progress::MultiProgress,
     project::Project,
+    remote_package_db::RemotePackageDB,
     tree::Tree,
 };
 
@@ -26,16 +26,16 @@ pub async fn test(test: Test, config: Config) -> Result<()> {
         Ok(lua_version) => Ok(lua_version),
         Err(_) => rockspec.test_lua_version().ok_or_eyre("lua version not set! Please provide a version through `--lua-version <ver>` or add it to your rockspec's dependencies"),
     }?;
-    let manifest = Manifest::from_config(config.server(), &config).await?;
+    let package_db = RemotePackageDB::from_config(&config).await?;
     let test_config = config.with_lua_version(lua_version);
     let tree = Tree::new(
         test_config.tree().clone(),
         test_config.lua_version().unwrap().clone(),
     )?;
     let progress = MultiProgress::new_arc();
     // TODO(#204): Only ensure busted if running with busted (e.g. a .busted directory exists)
-    ensure_busted(&tree, &manifest, &test_config, progress.clone()).await?;
-    ensure_dependencies(rockspec, &tree, &manifest, &test_config, progress).await?;
+    ensure_busted(&tree, &package_db, &test_config, progress.clone()).await?;
+    ensure_dependencies(rockspec, &tree, &package_db, &test_config, progress).await?;
     let test_args = test.test_args.unwrap_or_default();
     let test_env = if test.impure {
         TestEnv::Impure

rocks-bin/src/unpack.rs

@@ -4,9 +4,9 @@ use clap::Args;
 use eyre::Result;
 use rocks_lib::{
     config::Config,
-    manifest::Manifest,
     package::PackageReq,
     progress::{MultiProgress, Progress},
+    remote_package_db::RemotePackageDB,
 };
 
 #[derive(Args)]
@@ -49,12 +49,11 @@ and type `rocks make` to build.",
 
 pub async fn unpack_remote(data: UnpackRemote, config: Config) -> Result<()> {
     let package_req = data.package_req;
-    let manifest = Manifest::from_config(config.server(), &config).await?;
+    let package_db = RemotePackageDB::from_config(&config).await?;
     let progress = MultiProgress::new();
     let bar = Progress::Progress(progress.new_bar());
-    let rock =
-        rocks_lib::operations::search_and_download_src_rock(&package_req, &manifest, &config, &bar)
-            .await?;
+    let rock = rocks_lib::operations::search_and_download_src_rock(&package_req, &package_db, &bar)
+        .await?;
     let cursor = Cursor::new(rock.bytes);
 
     let destination = data

rocks-bin/src/update.rs

@@ -3,7 +3,8 @@ use eyre::Result;
 use rocks_lib::config::LuaVersion;
 use rocks_lib::lockfile::PinnedState;
 use rocks_lib::progress::{MultiProgress, ProgressBar};
-use rocks_lib::{config::Config, manifest::Manifest, operations, package::PackageReq, tree::Tree};
+use rocks_lib::remote_package_db::RemotePackageDB;
+use rocks_lib::{config::Config, operations, package::PackageReq, tree::Tree};
 
 #[derive(Args)]
 pub struct Update {}
@@ -16,7 +17,7 @@ pub async fn update(config: Config) -> Result<()> {
 
     let lockfile = tree.lockfile()?;
     let rocks = lockfile.rocks();
-    let manifest = Manifest::from_config(config.server(), &config).await?;
+    let package_db = RemotePackageDB::from_config(&config).await?;
 
     for package in rocks.values() {
         if package.pinned() == PinnedState::Unpinned {
@@ -26,7 +27,7 @@ pub async fn update(config: Config) -> Result<()> {
                     package.name().to_string(),
                     package.constraint().to_string_opt(),
                 )?,
-                &manifest,
+                &package_db,
                 &config,
                 progress.clone(),
             )