From 91562c2a805275d5d6bf6191d08feb2ffd579939 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 22 May 2024 18:11:54 +1000 Subject: [PATCH 1/9] VendorConfig::Compressed --- src/buck.rs | 41 +++++++++++++++++++++++++++++++++ src/buckify.rs | 61 ++++++++++++++++++++++++++++++++++++++++++------- src/cargo.rs | 2 +- src/config.rs | 62 ++++++++++++++++++++++++++++++++++++++------------ src/fixups.rs | 8 +++---- src/main.rs | 6 +++-- src/vendor.rs | 7 +++--- 7 files changed, 155 insertions(+), 32 deletions(-) diff --git a/src/buck.rs b/src/buck.rs index ab90a105..0705e2cf 100644 --- a/src/buck.rs +++ b/src/buck.rs @@ -448,6 +448,41 @@ impl Serialize for HttpArchive { } } +#[derive(Debug)] +pub struct CompressedCrate { + pub name: Name, + pub src: BuckPath, + pub sha256: String, + pub strip_prefix: String, + pub sub_targets: BTreeSet, + pub visibility: Visibility, + pub sort_key: Name, +} + +impl Serialize for CompressedCrate { + fn serialize(&self, ser: S) -> Result { + let Self { + name, + src, + sha256, + strip_prefix, + sub_targets, + visibility, + sort_key: _, + } = self; + let mut map = ser.serialize_map(None)?; + map.serialize_entry("name", name)?; + map.serialize_entry("src", src)?; + map.serialize_entry("sha256", sha256)?; + map.serialize_entry("strip_prefix", strip_prefix)?; + if !sub_targets.is_empty() { + map.serialize_entry("sub_targets", sub_targets)?; + } + map.serialize_entry("visibility", visibility)?; + map.end() + } +} + #[derive(Debug)] pub struct GitFetch { pub name: Name, @@ -1025,6 +1060,7 @@ impl Serialize for PrebuiltCxxLibrary { pub enum Rule { Alias(Alias), Filegroup(Filegroup), + ExtractArchive(CompressedCrate), HttpArchive(HttpArchive), GitFetch(GitFetch), Binary(RustBinary), @@ -1066,6 +1102,7 @@ fn rule_sort_key(rule: &Rule) -> impl Ord + '_ { // Make the alias rule come before the actual rule. Note that aliases // emitted by reindeer are always to a target within the same package. Rule::Alias(Alias { actual, .. }) => RuleSortKey::Other(actual, 0), + Rule::ExtractArchive(CompressedCrate { sort_key, .. }) => RuleSortKey::Other(sort_key, 1), Rule::HttpArchive(HttpArchive { sort_key, .. }) => RuleSortKey::Other(sort_key, 1), Rule::GitFetch(GitFetch { name, .. }) => RuleSortKey::GitFetch(name), Rule::Filegroup(_) @@ -1091,6 +1128,7 @@ impl Rule { Rule::Alias(Alias { name, .. }) | Rule::Filegroup(Filegroup { name, .. }) | Rule::HttpArchive(HttpArchive { name, .. }) + | Rule::ExtractArchive(CompressedCrate { name, .. }) | Rule::GitFetch(GitFetch { name, .. }) | Rule::Binary(RustBinary { common: @@ -1143,6 +1181,9 @@ impl Rule { Rule::Filegroup(filegroup) => { FunctionCall::new(&config.filegroup, filegroup).serialize(Serializer) } + Rule::ExtractArchive(compressed_crate) => { + FunctionCall::new(&config.extract_archive, compressed_crate).serialize(Serializer) + } Rule::HttpArchive(http_archive) => { FunctionCall::new(&config.http_archive, http_archive).serialize(Serializer) } diff --git a/src/buckify.rs b/src/buckify.rs index 83ebbc69..257dbce0 100644 --- a/src/buckify.rs +++ b/src/buckify.rs @@ -33,6 +33,7 @@ use crate::buck; use crate::buck::Alias; use crate::buck::BuckPath; use crate::buck::Common; +use crate::buck::CompressedCrate; use crate::buck::Filegroup; use crate::buck::GitFetch; use crate::buck::HttpArchive; @@ -56,6 +57,7 @@ use crate::cargo::Source; use crate::cargo::TargetReq; use crate::collection::SetOrMap; use crate::config::Config; +use crate::config::VendorConfig; use crate::fixups::ExportSources; use crate::fixups::Fixups; use crate::glob::Globs; @@ -219,7 +221,7 @@ fn generate_rules<'scope>( for rule in rules { let _ = rule_tx.send(Ok(rule)); } - if context.config.vendor.is_none() { + if context.config.vendor.is_not_source() { deps.push((pkg, TargetReq::Sources)); } } @@ -258,7 +260,13 @@ fn generate_nonvendored_sources_archive<'scope>( match &lockfile_package.source { Source::Local => Ok(None), - Source::CratesIo => generate_http_archive(context, pkg, lockfile_package).map(Some), + Source::CratesIo => match context.config.vendor { + VendorConfig::Off => generate_http_archive(context, pkg, lockfile_package).map(Some), + VendorConfig::Compressed => { + generate_extract_archive(context, pkg, lockfile_package).map(Some) + } + VendorConfig::Source(_) => unreachable!(), + }, Source::Git { repo, commit_hash, .. } => generate_git_fetch(repo, commit_hash).map(Some), @@ -272,6 +280,43 @@ fn generate_nonvendored_sources_archive<'scope>( } } +fn generate_extract_archive<'scope>( + context: &'scope RuleContext<'scope>, + pkg: &'scope Manifest, + lockfile_package: &LockfilePackage, +) -> anyhow::Result { + let sha256 = match &lockfile_package.checksum { + Some(checksum) => checksum.clone(), + None => { + // Dependencies from Source::CratesIo should almost certainly be + // associated with a checksum so a failure here is not expected. + bail!( + "No sha256 checksum available for \"{}\" {} in lockfile {}", + pkg.name, + pkg.version, + context.paths.lockfile_path.display(), + ); + } + }; + + // HACK: hardcoded index version. + // Need to copy these to a vendor/ directory too + let cache = ".cargo/registry/cache/index.crates.io-6f17d22bba15001f"; + + Ok(Rule::ExtractArchive(CompressedCrate { + name: Name(format!("{}-{}.crate", pkg.name, pkg.version)), + sha256, + src: BuckPath(PathBuf::from(format!( + "{cache}/{}-{}.crate", + pkg.name, pkg.version + ))), + strip_prefix: format!("{}-{}", pkg.name, pkg.version), + sub_targets: BTreeSet::new(), // populated later after all fixups are constructed + visibility: Visibility::Private, + sort_key: Name(format!("{}-{}", pkg.name, pkg.version)), + })) +} + fn generate_http_archive<'scope>( context: &'scope RuleContext<'scope>, pkg: &'scope Manifest, @@ -414,7 +459,7 @@ fn generate_target_rules<'scope>( let manifest_dir = pkg.manifest_dir(); let mapped_manifest_dir = - if context.config.vendor.is_some() || matches!(pkg.source, Source::Local) { + if context.config.vendor.is_source() || matches!(pkg.source, Source::Local) { relative_path(&paths.third_party_dir, manifest_dir) } else if let Source::Git { repo, .. } = &pkg.source { let git_fetch = short_name_for_git_repo(repo)?; @@ -428,7 +473,7 @@ fn generate_target_rules<'scope>( let edition = tgt.edition.unwrap_or(pkg.edition); let mut licenses = BTreeSet::new(); - if config.vendor.is_none() { + if !config.vendor.is_source() { // The `licenses` attribute takes `attrs.source()` which is the file // containing the custom license text. For `vendor = false` mode, we // don't have such a file on disk, and we don't have a Buck label either @@ -458,7 +503,7 @@ fn generate_target_rules<'scope>( // filename, or a list of globs. // If we're configured to get precise sources and we're using 2018+ edition source, then // parse the crate to see what files are actually used. - let mut srcs = if (config.vendor.is_some() || matches!(pkg.source, Source::Local)) + let mut srcs = if (config.vendor.is_source() || matches!(pkg.source, Source::Local)) && fixups.precise_srcs() && edition >= Edition::Rust2018 { @@ -504,7 +549,7 @@ fn generate_target_rules<'scope>( ) .context("rustc_flags")?; - if config.vendor.is_some() || matches!(pkg.source, Source::Local) { + if config.vendor.is_source() || matches!(pkg.source, Source::Local) { unzip_platform( config, &mut base, @@ -870,7 +915,7 @@ fn generate_target_rules<'scope>( // For non-disk sources (i.e. non-vendor mode git_fetch and // http_archive), `srcs` and `exclude` are ignored because // we can't look at the files to match globs. - let srcs = if config.vendor.is_some() || matches!(pkg.source, Source::Local) { + let srcs = if config.vendor.is_source() || matches!(pkg.source, Source::Local) { // e.g. {"src/lib.rs": "vendor/foo-1.0.0/src/lib.rs"} let mut globs = Globs::new(srcs, exclude).context("export sources")?; let srcs = globs @@ -985,7 +1030,7 @@ fn buckify_for_universe( // Fill in all http_archive rules with all the sub_targets which got // mentioned by fixups. - if config.vendor.is_none() { + if !config.vendor.is_source() { let mut need_subtargets = HashMap::>::new(); let mut insert = |subtarget_or_path: &SubtargetOrPath| { if let SubtargetOrPath::Subtarget(subtarget) = subtarget_or_path { diff --git a/src/cargo.rs b/src/cargo.rs index fc26d796..575fe24c 100644 --- a/src/cargo.rs +++ b/src/cargo.rs @@ -61,7 +61,7 @@ pub fn cargo_get_lockfile_and_metadata( let cargo_home; let lockfile; - if config.vendor.is_none() { + if config.vendor.is_not_source() { cargo_home = None; // Whether or not there is a Cargo.lock already, do not read it yet. diff --git a/src/config.rs b/src/config.rs index 5197af5c..ba50d91b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -77,11 +77,8 @@ pub struct Config { #[serde(default)] pub buck: BuckConfig, - #[serde( - default = "default_vendor_config", - deserialize_with = "deserialize_vendor_config" - )] - pub vendor: Option, + #[serde(default, deserialize_with = "deserialize_vendor_config")] + pub vendor: VendorConfig, #[serde(default)] pub audit: AuditConfig, @@ -131,6 +128,8 @@ pub struct BuckConfig { /// Rule name for http_archive #[serde(default)] pub http_archive: StringWithDefault, + #[serde(default)] + pub extract_archive: StringWithDefault, /// Rule name for git_fetch #[serde(default)] pub git_fetch: StringWithDefault, @@ -153,9 +152,26 @@ pub struct BuckConfig { pub buildscript_genrule: StringWithDefault, } +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub enum VendorConfig { + Off, + Compressed, + Source(VendorSourceConfig), +} +impl VendorConfig { + pub(crate) fn is_source(&self) -> bool { + matches!(self, Self::Source(_)) + } + + pub(crate) fn is_not_source(&self) -> bool { + !self.is_source() + } +} + #[derive(Debug, Default, Clone, Deserialize)] #[serde(deny_unknown_fields)] -pub struct VendorConfig { +pub struct VendorSourceConfig { /// List of .gitignore files to use to filter checksum files, relative to /// this config file. #[serde(default)] @@ -165,6 +181,12 @@ pub struct VendorConfig { pub checksum_exclude: HashSet, } +impl Default for VendorConfig { + fn default() -> Self { + VendorConfig::Source(Default::default()) + } +} + #[derive(Debug, Default, Clone, Deserialize)] #[serde(deny_unknown_fields)] pub struct AuditConfig { @@ -247,10 +269,6 @@ impl From for StringWithDefault { } } -fn default_vendor_config() -> Option { - Some(VendorConfig::default()) -} - fn default_platforms() -> HashMap { const DEFAULT_PLATFORMS_TOML: &str = include_str!("default_platforms.toml"); @@ -270,14 +288,14 @@ fn default_universes() -> BTreeMap { map } -fn deserialize_vendor_config<'de, D>(deserializer: D) -> Result, D::Error> +fn deserialize_vendor_config<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, { struct VendorConfigVisitor; impl<'de> Visitor<'de> for VendorConfigVisitor { - type Value = Option; + type Value = VendorConfig; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("[vendor] section, or `vendor = false`") @@ -289,14 +307,30 @@ where { // `vendor = true`: default configuration with vendoring. // `vendor = false`: do not vendor. - Ok(value.then(VendorConfig::default)) + Ok(if value { + VendorConfig::default() + } else { + VendorConfig::Off + }) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + if v == "compressed" { + Ok(VendorConfig::Compressed) + } else { + Err(E::custom("unknown vendor type")) + } } fn visit_map(self, map: M) -> Result where M: MapAccess<'de>, { - VendorConfig::deserialize(MapAccessDeserializer::new(map)).map(Some) + VendorSourceConfig::deserialize(MapAccessDeserializer::new(map)) + .map(VendorConfig::Source) } } diff --git a/src/fixups.rs b/src/fixups.rs index 1b129b56..ee44c4dd 100644 --- a/src/fixups.rs +++ b/src/fixups.rs @@ -166,7 +166,7 @@ impl<'meta> Fixups<'meta> { &self, relative_to_manifest_dir: &Path, ) -> anyhow::Result { - if self.config.vendor.is_some() || matches!(self.package.source, Source::Local) { + if self.config.vendor.is_source() || matches!(self.package.source, Source::Local) { // Path to vendored file looks like "vendor/foo-1.0.0/src/lib.rs" let manifest_dir = relative_path(&self.third_party_dir, self.manifest_dir); let path = manifest_dir.join(relative_to_manifest_dir); @@ -311,7 +311,7 @@ impl<'meta> Fixups<'meta> { }; for fix in fixes { - if self.config.vendor.is_none() { + if self.config.vendor.is_not_source() { if let Source::Git { repo, .. } = &self.package.source { // Cxx_library fixups only work if the sources are vendored // or from an http_archive. They do not work with sources @@ -875,7 +875,7 @@ impl<'meta> Fixups<'meta> { for cargo_env in config.cargo_env.iter() { let v = match cargo_env { CargoEnv::CARGO_MANIFEST_DIR => { - if self.config.vendor.is_some() + if self.config.vendor.is_source() || matches!(self.package.source, Source::Local) { StringOrPath::Path(BuckPath(relative_path( @@ -936,7 +936,7 @@ impl<'meta> Fixups<'meta> { // This function is only used in vendoring mode, so it's guaranteed that // manifest_dir is a subdirectory of third_party_dir. - assert!(self.config.vendor.is_some() || matches!(self.package.source, Source::Local)); + assert!(self.config.vendor.is_source() || matches!(self.package.source, Source::Local)); let manifest_rel = relative_path(&self.third_party_dir, self.manifest_dir); let srcs_globs: Vec = srcs diff --git a/src/main.rs b/src/main.rs index a22c9860..3cdf88eb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,6 +25,8 @@ use std::path::PathBuf; use clap::Parser; use clap::Subcommand; +use crate::config::VendorConfig; + mod audit_sec; mod buck; mod buckify; @@ -145,10 +147,10 @@ fn try_main() -> anyhow::Result<()> { } SubCommand::Buckify { stdout } => { - if config.vendor.is_some() && !vendor::is_vendored(&paths)? { + if config.vendor.is_source() && !vendor::is_vendored(&paths)? { // If you ran `reindeer buckify` without `reindeer vendor`, then // default to generating non-vendored targets. - config.vendor = None; + config.vendor = VendorConfig::Off; } buckify::buckify(&config, &args, &paths, *stdout)?; } diff --git a/src/vendor.rs b/src/vendor.rs index 3abde03f..bf823c84 100644 --- a/src/vendor.rs +++ b/src/vendor.rs @@ -21,6 +21,7 @@ use crate::buckify::relative_path; use crate::cargo; use crate::config::Config; use crate::config::VendorConfig; +use crate::config::VendorSourceConfig; use crate::remap::RemapConfig; use crate::Args; use crate::Paths; @@ -68,8 +69,8 @@ pub(crate) fn cargo_vendor( assert!(is_vendored(paths)?); } - if let Some(vendor_config) = &config.vendor { - filter_checksum_files(&paths.third_party_dir, vendordir, vendor_config)?; + if let VendorConfig::Source(source_config) = &config.vendor { + filter_checksum_files(&paths.third_party_dir, vendordir, source_config)?; } if audit_sec { @@ -115,7 +116,7 @@ pub(crate) fn is_vendored(paths: &Paths) -> anyhow::Result { fn filter_checksum_files( third_party_dir: &Path, vendordir: &Path, - config: &VendorConfig, + config: &VendorSourceConfig, ) -> anyhow::Result<()> { if config.checksum_exclude.is_empty() && config.gitignore_checksum_exclude.is_empty() { return Ok(()); From 2e11b0f465bd9527290f6e7dd34c8a177575d259 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 22 May 2024 19:27:28 +1000 Subject: [PATCH 2/9] implement `reindeer vendor` for VendorConfig::Compressed --- src/buckify.rs | 20 ++++++++----- src/vendor.rs | 77 ++++++++++++++++++++++++++++++++++---------------- 2 files changed, 65 insertions(+), 32 deletions(-) diff --git a/src/buckify.rs b/src/buckify.rs index 257dbce0..89304c98 100644 --- a/src/buckify.rs +++ b/src/buckify.rs @@ -299,15 +299,13 @@ fn generate_extract_archive<'scope>( } }; - // HACK: hardcoded index version. - // Need to copy these to a vendor/ directory too - let cache = ".cargo/registry/cache/index.crates.io-6f17d22bba15001f"; + let vendordir = "vendor"; Ok(Rule::ExtractArchive(CompressedCrate { name: Name(format!("{}-{}.crate", pkg.name, pkg.version)), sha256, src: BuckPath(PathBuf::from(format!( - "{cache}/{}-{}.crate", + "{vendordir}/{}-{}.crate", pkg.name, pkg.version ))), strip_prefix: format!("{}-{}", pkg.name, pkg.version), @@ -1071,10 +1069,18 @@ fn buckify_for_universe( rules = rules .into_iter() .map(|mut rule| { - if let Rule::HttpArchive(rule) = &mut rule { - if let Some(need_subtargets) = need_subtargets.remove(&rule.name) { - rule.sub_targets = need_subtargets; + match &mut rule { + Rule::HttpArchive(rule) => { + if let Some(need_subtargets) = need_subtargets.remove(&rule.name) { + rule.sub_targets = need_subtargets; + } } + Rule::ExtractArchive(rule) => { + if let Some(need_subtargets) = need_subtargets.remove(&rule.name) { + rule.sub_targets = need_subtargets; + } + } + _ => (), } rule }) diff --git a/src/vendor.rs b/src/vendor.rs index bf823c84..b6b49af8 100644 --- a/src/vendor.rs +++ b/src/vendor.rs @@ -42,35 +42,62 @@ pub(crate) fn cargo_vendor( ) -> anyhow::Result<()> { let vendordir = Path::new("vendor"); // relative to third_party_dir - let mut cmdline = vec![ - "vendor", - "--manifest-path", - paths.manifest_path.to_str().unwrap(), - vendordir.to_str().unwrap(), - "--versioned-dirs", - ]; - if no_delete { - cmdline.push("--no-delete"); - } + match &config.vendor { + VendorConfig::Compressed => { + let mut cmdline = vec![ + "local-registry", + "-s", + paths.lockfile_path.to_str().unwrap(), + vendordir.to_str().unwrap(), + ]; + if no_delete { + cmdline.push("--no-delete"); + } + log::info!("Running cargo {:?}", cmdline); + let mut cargoconfig = cargo::run_cargo( + config, + Some(&paths.cargo_home), + &paths.third_party_dir, + args, + &cmdline, + )?; + cargoconfig.splice(0..0, b"# ".iter().copied()); + fs::write(paths.cargo_home.join("config.toml"), &cargoconfig)?; + // if !cargoconfig.is_empty() { + // assert!(is_vendored(paths)?); + // } + } + VendorConfig::Source(source_config) => { + let mut cmdline = vec![ + "vendor", + "--manifest-path", + paths.manifest_path.to_str().unwrap(), + vendordir.to_str().unwrap(), + "--versioned-dirs", + ]; + if no_delete { + cmdline.push("--no-delete"); + } - fs::create_dir_all(&paths.cargo_home)?; + fs::create_dir_all(&paths.cargo_home)?; - log::info!("Running cargo {:?}", cmdline); - let cargoconfig = cargo::run_cargo( - config, - Some(&paths.cargo_home), - &paths.third_party_dir, - args, - &cmdline, - )?; + log::info!("Running cargo {:?}", cmdline); + let cargoconfig = cargo::run_cargo( + config, + Some(&paths.cargo_home), + &paths.third_party_dir, + args, + &cmdline, + )?; - fs::write(paths.cargo_home.join("config.toml"), &cargoconfig)?; - if !cargoconfig.is_empty() { - assert!(is_vendored(paths)?); - } + fs::write(paths.cargo_home.join("config.toml"), &cargoconfig)?; + if !cargoconfig.is_empty() { + assert!(is_vendored(paths)?); + } - if let VendorConfig::Source(source_config) = &config.vendor { - filter_checksum_files(&paths.third_party_dir, vendordir, source_config)?; + filter_checksum_files(&paths.third_party_dir, vendordir, source_config)?; + } + _ => (), } if audit_sec { From 7e814ef06c17eed40d643f1c6b5b8958ab2deb5f Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 22 May 2024 19:58:08 +1000 Subject: [PATCH 3/9] Use a relative path in the .cargo/config.toml --- src/vendor.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/vendor.rs b/src/vendor.rs index b6b49af8..de430e86 100644 --- a/src/vendor.rs +++ b/src/vendor.rs @@ -54,14 +54,23 @@ pub(crate) fn cargo_vendor( cmdline.push("--no-delete"); } log::info!("Running cargo {:?}", cmdline); - let mut cargoconfig = cargo::run_cargo( + let _ = cargo::run_cargo( config, Some(&paths.cargo_home), &paths.third_party_dir, args, &cmdline, )?; - cargoconfig.splice(0..0, b"# ".iter().copied()); + let cargoconfig = format!( + r#"[source.crates-io] +registry = 'sparse+https://index.crates.io/' +replace-with = 'local-registry' + +[source.local-registry] +local-registry = {vendordir:?} +"# + ); + // cargoconfig.splice(0..0, b"# ".iter().copied()); fs::write(paths.cargo_home.join("config.toml"), &cargoconfig)?; // if !cargoconfig.is_empty() { // assert!(is_vendored(paths)?); From 570b67c8ede388e7d2dd37592ddbf31f07c28b90 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 22 May 2024 20:10:09 +1000 Subject: [PATCH 4/9] Check is_vendored for compressed mode --- src/main.rs | 13 +++++++++---- src/remap.rs | 6 ++++-- src/vendor.rs | 24 ++++++++++++++++-------- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/src/main.rs b/src/main.rs index 3cdf88eb..492c47c0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -147,10 +147,15 @@ fn try_main() -> anyhow::Result<()> { } SubCommand::Buckify { stdout } => { - if config.vendor.is_source() && !vendor::is_vendored(&paths)? { - // If you ran `reindeer buckify` without `reindeer vendor`, then - // default to generating non-vendored targets. - config.vendor = VendorConfig::Off; + match &config.vendor { + VendorConfig::Compressed | VendorConfig::Source(..) + if !vendor::is_vendored(&config, &paths)? => + { + // If you ran `reindeer buckify` without `reindeer vendor`, then + // default to generating non-vendored targets. + config.vendor = VendorConfig::Off; + } + _ => {} } buckify::buckify(&config, &args, &paths, *stdout)?; } diff --git a/src/remap.rs b/src/remap.rs index e939aa4a..8ec88c5c 100644 --- a/src/remap.rs +++ b/src/remap.rs @@ -11,13 +11,13 @@ use std::path::PathBuf; use serde::Deserialize; use serde::Serialize; -#[derive(Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct RemapConfig { #[serde(rename = "source", default)] pub sources: Map, } -#[derive(Serialize, Deserialize, Default)] +#[derive(Debug, Serialize, Deserialize, Default)] pub struct RemapSource { pub directory: Option, pub git: Option, @@ -26,4 +26,6 @@ pub struct RemapSource { pub tag: Option, #[serde(rename = "replace-with")] pub replace_with: Option, + #[serde(rename = "local-registry")] + pub local_registry: Option, } diff --git a/src/vendor.rs b/src/vendor.rs index de430e86..eee227e1 100644 --- a/src/vendor.rs +++ b/src/vendor.rs @@ -70,11 +70,10 @@ replace-with = 'local-registry' local-registry = {vendordir:?} "# ); - // cargoconfig.splice(0..0, b"# ".iter().copied()); fs::write(paths.cargo_home.join("config.toml"), &cargoconfig)?; - // if !cargoconfig.is_empty() { - // assert!(is_vendored(paths)?); - // } + if !cargoconfig.is_empty() { + assert!(is_vendored(config, paths)?); + } } VendorConfig::Source(source_config) => { let mut cmdline = vec![ @@ -101,7 +100,7 @@ local-registry = {vendordir:?} fs::write(paths.cargo_home.join("config.toml"), &cargoconfig)?; if !cargoconfig.is_empty() { - assert!(is_vendored(paths)?); + assert!(is_vendored(config, paths)?); } filter_checksum_files(&paths.third_party_dir, vendordir, source_config)?; @@ -116,7 +115,7 @@ local-registry = {vendordir:?} Ok(()) } -pub(crate) fn is_vendored(paths: &Paths) -> anyhow::Result { +pub(crate) fn is_vendored(config: &Config, paths: &Paths) -> anyhow::Result { // .cargo/config.toml is Cargo's preferred name for the config, but .cargo/config // is the older name so it takes priority if present. let mut cargo_config_path = paths.cargo_home.join("config"); @@ -143,8 +142,17 @@ pub(crate) fn is_vendored(paths: &Paths) -> anyhow::Result { let remap_config: RemapConfig = toml::from_str(&content) .context(format!("Failed to parse {}", cargo_config_path.display()))?; - match remap_config.sources.get("vendored-sources") { - Some(vendored_sources) => Ok(vendored_sources.directory.is_some()), + let source_name = match config.vendor { + VendorConfig::Compressed => "local-registry", + VendorConfig::Source(_) => "vendored-sources", + _ => return Ok(false), + }; + match remap_config.sources.get(source_name) { + Some(source) => match config.vendor { + VendorConfig::Compressed => Ok(source.local_registry.is_some()), + VendorConfig::Source(_) => Ok(source.directory.is_some()), + VendorConfig::Off => Ok(false), + }, None => Ok(false), } } From d9eb1c334cd0a546c083e418730d98bb4d102e2f Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 22 May 2024 20:38:23 +1000 Subject: [PATCH 5/9] remove sha256 from extract archive rules --- src/buck.rs | 3 --- src/buckify.rs | 19 ++----------------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/src/buck.rs b/src/buck.rs index 0705e2cf..47c6cb76 100644 --- a/src/buck.rs +++ b/src/buck.rs @@ -452,7 +452,6 @@ impl Serialize for HttpArchive { pub struct CompressedCrate { pub name: Name, pub src: BuckPath, - pub sha256: String, pub strip_prefix: String, pub sub_targets: BTreeSet, pub visibility: Visibility, @@ -464,7 +463,6 @@ impl Serialize for CompressedCrate { let Self { name, src, - sha256, strip_prefix, sub_targets, visibility, @@ -473,7 +471,6 @@ impl Serialize for CompressedCrate { let mut map = ser.serialize_map(None)?; map.serialize_entry("name", name)?; map.serialize_entry("src", src)?; - map.serialize_entry("sha256", sha256)?; map.serialize_entry("strip_prefix", strip_prefix)?; if !sub_targets.is_empty() { map.serialize_entry("sub_targets", sub_targets)?; diff --git a/src/buckify.rs b/src/buckify.rs index 89304c98..32b182bd 100644 --- a/src/buckify.rs +++ b/src/buckify.rs @@ -281,29 +281,14 @@ fn generate_nonvendored_sources_archive<'scope>( } fn generate_extract_archive<'scope>( - context: &'scope RuleContext<'scope>, + _context: &'scope RuleContext<'scope>, pkg: &'scope Manifest, - lockfile_package: &LockfilePackage, + _lockfile_package: &LockfilePackage, ) -> anyhow::Result { - let sha256 = match &lockfile_package.checksum { - Some(checksum) => checksum.clone(), - None => { - // Dependencies from Source::CratesIo should almost certainly be - // associated with a checksum so a failure here is not expected. - bail!( - "No sha256 checksum available for \"{}\" {} in lockfile {}", - pkg.name, - pkg.version, - context.paths.lockfile_path.display(), - ); - } - }; - let vendordir = "vendor"; Ok(Rule::ExtractArchive(CompressedCrate { name: Name(format!("{}-{}.crate", pkg.name, pkg.version)), - sha256, src: BuckPath(PathBuf::from(format!( "{vendordir}/{}-{}.crate", pkg.name, pkg.version From 9e0af3a354af5104962615efb4be1e3da4a69374 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 22 May 2024 20:47:41 +1000 Subject: [PATCH 6/9] Rename VendorConfig::Compressed -> LocalRegistry --- src/buckify.rs | 2 +- src/config.rs | 6 +++--- src/main.rs | 2 +- src/vendor.rs | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/buckify.rs b/src/buckify.rs index 32b182bd..fc9a1549 100644 --- a/src/buckify.rs +++ b/src/buckify.rs @@ -262,7 +262,7 @@ fn generate_nonvendored_sources_archive<'scope>( Source::Local => Ok(None), Source::CratesIo => match context.config.vendor { VendorConfig::Off => generate_http_archive(context, pkg, lockfile_package).map(Some), - VendorConfig::Compressed => { + VendorConfig::LocalRegistry => { generate_extract_archive(context, pkg, lockfile_package).map(Some) } VendorConfig::Source(_) => unreachable!(), diff --git a/src/config.rs b/src/config.rs index ba50d91b..2fd500d8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -156,7 +156,7 @@ pub struct BuckConfig { #[serde(deny_unknown_fields)] pub enum VendorConfig { Off, - Compressed, + LocalRegistry, Source(VendorSourceConfig), } impl VendorConfig { @@ -318,8 +318,8 @@ where where E: serde::de::Error, { - if v == "compressed" { - Ok(VendorConfig::Compressed) + if v == "local-registry" { + Ok(VendorConfig::LocalRegistry) } else { Err(E::custom("unknown vendor type")) } diff --git a/src/main.rs b/src/main.rs index 492c47c0..96c4f5fc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -148,7 +148,7 @@ fn try_main() -> anyhow::Result<()> { SubCommand::Buckify { stdout } => { match &config.vendor { - VendorConfig::Compressed | VendorConfig::Source(..) + VendorConfig::LocalRegistry | VendorConfig::Source(..) if !vendor::is_vendored(&config, &paths)? => { // If you ran `reindeer buckify` without `reindeer vendor`, then diff --git a/src/vendor.rs b/src/vendor.rs index eee227e1..001b22f9 100644 --- a/src/vendor.rs +++ b/src/vendor.rs @@ -43,7 +43,7 @@ pub(crate) fn cargo_vendor( let vendordir = Path::new("vendor"); // relative to third_party_dir match &config.vendor { - VendorConfig::Compressed => { + VendorConfig::LocalRegistry => { let mut cmdline = vec![ "local-registry", "-s", @@ -143,13 +143,13 @@ pub(crate) fn is_vendored(config: &Config, paths: &Paths) -> anyhow::Result "local-registry", + VendorConfig::LocalRegistry => "local-registry", VendorConfig::Source(_) => "vendored-sources", _ => return Ok(false), }; match remap_config.sources.get(source_name) { Some(source) => match config.vendor { - VendorConfig::Compressed => Ok(source.local_registry.is_some()), + VendorConfig::LocalRegistry => Ok(source.local_registry.is_some()), VendorConfig::Source(_) => Ok(source.directory.is_some()), VendorConfig::Off => Ok(false), }, From 819b9836aa3c2b793c0edbcd31f57faa7526c59c Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 22 May 2024 21:20:56 +1000 Subject: [PATCH 7/9] rename CompressedCrate -> ExtractArchive --- src/buck.rs | 10 +++++----- src/buckify.rs | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/buck.rs b/src/buck.rs index 47c6cb76..daf14f0a 100644 --- a/src/buck.rs +++ b/src/buck.rs @@ -449,7 +449,7 @@ impl Serialize for HttpArchive { } #[derive(Debug)] -pub struct CompressedCrate { +pub struct ExtractArchive { pub name: Name, pub src: BuckPath, pub strip_prefix: String, @@ -458,7 +458,7 @@ pub struct CompressedCrate { pub sort_key: Name, } -impl Serialize for CompressedCrate { +impl Serialize for ExtractArchive { fn serialize(&self, ser: S) -> Result { let Self { name, @@ -1057,7 +1057,7 @@ impl Serialize for PrebuiltCxxLibrary { pub enum Rule { Alias(Alias), Filegroup(Filegroup), - ExtractArchive(CompressedCrate), + ExtractArchive(ExtractArchive), HttpArchive(HttpArchive), GitFetch(GitFetch), Binary(RustBinary), @@ -1099,7 +1099,7 @@ fn rule_sort_key(rule: &Rule) -> impl Ord + '_ { // Make the alias rule come before the actual rule. Note that aliases // emitted by reindeer are always to a target within the same package. Rule::Alias(Alias { actual, .. }) => RuleSortKey::Other(actual, 0), - Rule::ExtractArchive(CompressedCrate { sort_key, .. }) => RuleSortKey::Other(sort_key, 1), + Rule::ExtractArchive(ExtractArchive { sort_key, .. }) => RuleSortKey::Other(sort_key, 1), Rule::HttpArchive(HttpArchive { sort_key, .. }) => RuleSortKey::Other(sort_key, 1), Rule::GitFetch(GitFetch { name, .. }) => RuleSortKey::GitFetch(name), Rule::Filegroup(_) @@ -1125,7 +1125,7 @@ impl Rule { Rule::Alias(Alias { name, .. }) | Rule::Filegroup(Filegroup { name, .. }) | Rule::HttpArchive(HttpArchive { name, .. }) - | Rule::ExtractArchive(CompressedCrate { name, .. }) + | Rule::ExtractArchive(ExtractArchive { name, .. }) | Rule::GitFetch(GitFetch { name, .. }) | Rule::Binary(RustBinary { common: diff --git a/src/buckify.rs b/src/buckify.rs index fc9a1549..73e7bfbf 100644 --- a/src/buckify.rs +++ b/src/buckify.rs @@ -33,7 +33,7 @@ use crate::buck; use crate::buck::Alias; use crate::buck::BuckPath; use crate::buck::Common; -use crate::buck::CompressedCrate; +use crate::buck::ExtractArchive; use crate::buck::Filegroup; use crate::buck::GitFetch; use crate::buck::HttpArchive; @@ -287,7 +287,7 @@ fn generate_extract_archive<'scope>( ) -> anyhow::Result { let vendordir = "vendor"; - Ok(Rule::ExtractArchive(CompressedCrate { + Ok(Rule::ExtractArchive(ExtractArchive { name: Name(format!("{}-{}.crate", pkg.name, pkg.version)), src: BuckPath(PathBuf::from(format!( "{vendordir}/{}-{}.crate", From e1d2c5330fb116615bede1a3c51e8c40730a1dc6 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 22 May 2024 21:49:35 +1000 Subject: [PATCH 8/9] Vendor git dependencies as well in local-registry mode --- src/buckify.rs | 43 ++++++++++++++++++++++++++++++++++++++++--- src/vendor.rs | 1 + 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/buckify.rs b/src/buckify.rs index 73e7bfbf..33392847 100644 --- a/src/buckify.rs +++ b/src/buckify.rs @@ -269,7 +269,14 @@ fn generate_nonvendored_sources_archive<'scope>( }, Source::Git { repo, commit_hash, .. - } => generate_git_fetch(repo, commit_hash).map(Some), + } => match context.config.vendor { + VendorConfig::Off => generate_git_fetch(repo, commit_hash).map(Some), + VendorConfig::LocalRegistry => { + generate_extract_archive_git(context, pkg, lockfile_package, repo, commit_hash) + .map(Some) + } + VendorConfig::Source(_) => unreachable!(), + }, Source::Unrecognized(_) => { bail!( "`vendor = false` mode is supported only with exclusively crates.io and https git dependencies. \"{}\" {} is coming from some other source", @@ -286,7 +293,6 @@ fn generate_extract_archive<'scope>( _lockfile_package: &LockfilePackage, ) -> anyhow::Result { let vendordir = "vendor"; - Ok(Rule::ExtractArchive(ExtractArchive { name: Name(format!("{}-{}.crate", pkg.name, pkg.version)), src: BuckPath(PathBuf::from(format!( @@ -300,6 +306,32 @@ fn generate_extract_archive<'scope>( })) } +fn generate_extract_archive_git<'scope>( + _context: &'scope RuleContext<'scope>, + pkg: &'scope Manifest, + _lockfile_package: &LockfilePackage, + repo: &str, + _commit_hash: &str, +) -> anyhow::Result { + let vendordir = "vendor"; + let short_name = short_name_for_git_repo(repo)?; + Ok(Rule::ExtractArchive(ExtractArchive { + name: Name(format!("{}.git", short_name)), + // + // Would be nice if cargo local-registry wrote out a git-related name instead + // of `mycrate-0.1.0.crate`. However it does not. + // + src: BuckPath(PathBuf::from(format!( + "{vendordir}/{}-{}.crate", + pkg.name, pkg.version + ))), + strip_prefix: format!("{}-{}", pkg.name, pkg.version), + sub_targets: BTreeSet::new(), // populated later after all fixups are constructed + visibility: Visibility::Private, + sort_key: Name(format!("{}-{}", pkg.name, pkg.version)), + })) +} + fn generate_http_archive<'scope>( context: &'scope RuleContext<'scope>, pkg: &'scope Manifest, @@ -445,7 +477,12 @@ fn generate_target_rules<'scope>( if context.config.vendor.is_source() || matches!(pkg.source, Source::Local) { relative_path(&paths.third_party_dir, manifest_dir) } else if let Source::Git { repo, .. } = &pkg.source { - let git_fetch = short_name_for_git_repo(repo)?; + let mut git_fetch = short_name_for_git_repo(repo)?; + if matches!(context.config.vendor, VendorConfig::LocalRegistry) { + // Subtle difference -- git_fetch rules will strip the `.git` off the name + // when used as a dependency. extract_archive will not. + git_fetch += ".git"; + } let repository_root = find_repository_root(manifest_dir)?; let path_within_repo = relative_path(repository_root, manifest_dir); PathBuf::from(git_fetch).join(path_within_repo) diff --git a/src/vendor.rs b/src/vendor.rs index 001b22f9..1ed19f41 100644 --- a/src/vendor.rs +++ b/src/vendor.rs @@ -49,6 +49,7 @@ pub(crate) fn cargo_vendor( "-s", paths.lockfile_path.to_str().unwrap(), vendordir.to_str().unwrap(), + "--git", ]; if no_delete { cmdline.push("--no-delete"); From a7b0188ae08a1d1d21f0e42567bc32e90eb71c48 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Thu, 23 May 2024 15:53:16 +1000 Subject: [PATCH 9/9] Pretty hacky way to make LocalRegistry git deps work --- src/buckify.rs | 50 ++++++++++++++++---------------------------------- src/fixups.rs | 6 ++++++ 2 files changed, 22 insertions(+), 34 deletions(-) diff --git a/src/buckify.rs b/src/buckify.rs index 33392847..e9b29820 100644 --- a/src/buckify.rs +++ b/src/buckify.rs @@ -272,8 +272,7 @@ fn generate_nonvendored_sources_archive<'scope>( } => match context.config.vendor { VendorConfig::Off => generate_git_fetch(repo, commit_hash).map(Some), VendorConfig::LocalRegistry => { - generate_extract_archive_git(context, pkg, lockfile_package, repo, commit_hash) - .map(Some) + generate_extract_archive(context, pkg, lockfile_package).map(Some) } VendorConfig::Source(_) => unreachable!(), }, @@ -306,32 +305,6 @@ fn generate_extract_archive<'scope>( })) } -fn generate_extract_archive_git<'scope>( - _context: &'scope RuleContext<'scope>, - pkg: &'scope Manifest, - _lockfile_package: &LockfilePackage, - repo: &str, - _commit_hash: &str, -) -> anyhow::Result { - let vendordir = "vendor"; - let short_name = short_name_for_git_repo(repo)?; - Ok(Rule::ExtractArchive(ExtractArchive { - name: Name(format!("{}.git", short_name)), - // - // Would be nice if cargo local-registry wrote out a git-related name instead - // of `mycrate-0.1.0.crate`. However it does not. - // - src: BuckPath(PathBuf::from(format!( - "{vendordir}/{}-{}.crate", - pkg.name, pkg.version - ))), - strip_prefix: format!("{}-{}", pkg.name, pkg.version), - sub_targets: BTreeSet::new(), // populated later after all fixups are constructed - visibility: Visibility::Private, - sort_key: Name(format!("{}-{}", pkg.name, pkg.version)), - })) -} - fn generate_http_archive<'scope>( context: &'scope RuleContext<'scope>, pkg: &'scope Manifest, @@ -476,13 +449,10 @@ fn generate_target_rules<'scope>( let mapped_manifest_dir = if context.config.vendor.is_source() || matches!(pkg.source, Source::Local) { relative_path(&paths.third_party_dir, manifest_dir) + } else if let VendorConfig::LocalRegistry = context.config.vendor { + PathBuf::from(format!("{}-{}.crate", pkg.name, pkg.version)) } else if let Source::Git { repo, .. } = &pkg.source { - let mut git_fetch = short_name_for_git_repo(repo)?; - if matches!(context.config.vendor, VendorConfig::LocalRegistry) { - // Subtle difference -- git_fetch rules will strip the `.git` off the name - // when used as a dependency. extract_archive will not. - git_fetch += ".git"; - } + let git_fetch = short_name_for_git_repo(repo)?; let repository_root = find_repository_root(manifest_dir)?; let path_within_repo = relative_path(repository_root, manifest_dir); PathBuf::from(git_fetch).join(path_within_repo) @@ -581,6 +551,10 @@ fn generate_target_rules<'scope>( fixups.compute_srcs(srcs)?, ) .context("srcs")?; + } else if let VendorConfig::LocalRegistry = config.vendor { + let http_archive_target = format!(":{}-{}.crate", pkg.name, pkg.version); + base.srcs + .insert(BuckPath(PathBuf::from(http_archive_target))); } else if let Source::Git { repo, .. } = &pkg.source { let short_name = short_name_for_git_repo(repo)?; let git_fetch_target = format!(":{}.git", short_name); @@ -949,6 +923,14 @@ fn generate_target_rules<'scope>( globs.check_all_globs_used()?; } srcs + } else if let VendorConfig::LocalRegistry = config.vendor { + // e.g. {":foo-1.0.0.git": "foo-1.0.0"} + let http_archive_target = format!(":{}-{}.crate", pkg.name, pkg.version); + [( + BuckPath(mapped_manifest_dir.clone()), + SubtargetOrPath::Path(BuckPath(PathBuf::from(http_archive_target))), + )] + .into() } else if let Source::Git { repo, .. } = &pkg.source { // e.g. {":foo-123.git": "foo-123"} let short_name = short_name_for_git_repo(repo)?; diff --git a/src/fixups.rs b/src/fixups.rs index ee44c4dd..98c82697 100644 --- a/src/fixups.rs +++ b/src/fixups.rs @@ -42,6 +42,7 @@ use crate::cargo::NodeDepKind; use crate::cargo::Source; use crate::collection::SetOrMap; use crate::config::Config; +use crate::config::VendorConfig; use crate::glob::Globs; use crate::glob::SerializableGlobSet as GlobSet; use crate::glob::NO_EXCLUDE; @@ -882,6 +883,11 @@ impl<'meta> Fixups<'meta> { &self.third_party_dir, self.manifest_dir, ))) + } else if let VendorConfig::LocalRegistry = self.config.vendor { + StringOrPath::String(format!( + "{}-{}.crate", + self.package.name, self.package.version, + )) } else if let Source::Git { repo, .. } = &self.package.source { let short_name = short_name_for_git_repo(repo)?; StringOrPath::String(short_name.to_owned())