From 0dc709ed5a53c723f47fa1d10063e57bb50a63c8 Mon Sep 17 00:00:00 2001 From: Shadaj Laddad Date: Mon, 2 Dec 2024 12:44:19 -0800 Subject: [PATCH 1/2] chore(stageleft): use same hashing library everywhere (#1590) --- Cargo.lock | 23 ++--------------------- hydroflow_plus/src/deploy/trybuild.rs | 4 +--- stageleft_macro/Cargo.toml | 2 +- stageleft_macro/src/lib.rs | 4 +++- stageleft_tool/Cargo.toml | 2 +- stageleft_tool/src/lib.rs | 5 +++-- 6 files changed, 11 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 17b2e0470e4..396a800a5c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1325,12 +1325,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - [[package]] name = "home" version = "0.5.9" @@ -3227,19 +3221,6 @@ dependencies = [ "digest", ] -[[package]] -name = "sha256" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18278f6a914fa3070aa316493f7d2ddfb9ac86ebc06fa3b83bffda487e9065b0" -dependencies = [ - "async-trait", - "bytes", - "hex", - "sha2", - "tokio", -] - [[package]] name = "sharded-slab" version = "0.1.7" @@ -3364,7 +3345,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "sha256", + "sha2", "syn 2.0.75", ] @@ -3391,7 +3372,7 @@ version = "0.4.0" dependencies = [ "proc-macro2", "quote", - "sha256", + "sha2", "syn 2.0.75", "syn-inline-mod", ] diff --git a/hydroflow_plus/src/deploy/trybuild.rs b/hydroflow_plus/src/deploy/trybuild.rs index 1d075b1205a..7922e342b91 100644 --- a/hydroflow_plus/src/deploy/trybuild.rs +++ b/hydroflow_plus/src/deploy/trybuild.rs @@ -73,9 +73,7 @@ pub fn create_graph_trybuild( } }); - let mut hasher = Sha256::new(); - hasher.update(&source); - let hash = format!("{:X}", hasher.finalize()) + let hash = format!("{:X}", Sha256::digest(&source)) .chars() .take(8) .collect::(); diff --git a/stageleft_macro/Cargo.toml b/stageleft_macro/Cargo.toml index 8ef2802afba..b9f1b3fba3f 100644 --- a/stageleft_macro/Cargo.toml +++ b/stageleft_macro/Cargo.toml @@ -19,7 +19,7 @@ quote = "1.0.35" syn = { version = "2.0.46", features = [ "parsing", "extra-traits", "visit" ] } proc-macro2 = "1.0.74" proc-macro-crate = "1.0.0" -sha256 = "1.0.0" +sha2 = "0.10.0" [dev-dependencies] insta = "1.39" diff --git a/stageleft_macro/src/lib.rs b/stageleft_macro/src/lib.rs index 279a53a88c7..b36241554b9 100644 --- a/stageleft_macro/src/lib.rs +++ b/stageleft_macro/src/lib.rs @@ -1,5 +1,6 @@ use proc_macro2::{Punct, Spacing, Span, TokenStream}; use quote::{quote, quote_spanned, ToTokens}; +use sha2::{Digest, Sha256}; use syn::punctuated::Punctuated; use syn::spanned::Spanned; use syn::{AngleBracketedGenericArguments, Token, Type}; @@ -339,7 +340,7 @@ pub fn entry( .chars() .filter(|c| c.is_alphanumeric()) .collect::(); - let input_hash = "macro_".to_string() + &sha256::digest(input_contents); + let input_hash = "macro_".to_string() + &format!("{:X}", Sha256::digest(input_contents)); let input_hash_ident = syn::Ident::new(&input_hash, Span::call_site()); let input_hash_impl_ident = syn::Ident::new(&(input_hash + "_impl"), Span::call_site()); @@ -353,6 +354,7 @@ pub fn entry( #[cfg_attr(not(stageleft_macro), allow(unused))] #[cfg_attr(not(stageleft_macro), doc(hidden))] + #[allow(non_snake_case)] pub(crate) fn #input_hash_impl_ident(input: #root::internal::TokenStream) -> #root::internal::TokenStream { let input_parsed = #root::internal::syn::parse::Parser::parse( #root::internal::syn::punctuated::Punctuated::<#root::internal::syn::Expr, #root::internal::syn::Token![,]>::parse_terminated, diff --git a/stageleft_tool/Cargo.toml b/stageleft_tool/Cargo.toml index d8250cb5c3f..cca49aa24ca 100644 --- a/stageleft_tool/Cargo.toml +++ b/stageleft_tool/Cargo.toml @@ -18,4 +18,4 @@ syn-inline-mod = "0.6.0" quote = "1.0.35" syn = { version = "2.0.46", features = [ "parsing", "extra-traits", "visit" ] } proc-macro2 = "1.0.74" -sha256 = "1.0.0" +sha2 = "0.10.0" diff --git a/stageleft_tool/src/lib.rs b/stageleft_tool/src/lib.rs index 6cc232eeb6b..2af956a2e93 100644 --- a/stageleft_tool/src/lib.rs +++ b/stageleft_tool/src/lib.rs @@ -4,6 +4,7 @@ use std::{env, fs}; use proc_macro2::Span; use quote::ToTokens; +use sha2::{Digest, Sha256}; use syn::visit::Visit; use syn::visit_mut::VisitMut; use syn::{parse_quote, UsePath}; @@ -41,7 +42,7 @@ impl<'a> Visit<'a> for GenMacroVistor { .chars() .filter(|c| c.is_alphanumeric()) .collect::(); - let contents_hash = sha256::digest(contents); + let contents_hash = format!("{:X}", Sha256::digest(contents)); self.exported_macros .insert((contents_hash, cur_path.to_token_stream().to_string())); } @@ -72,7 +73,7 @@ pub fn gen_macro(staged_path: &Path, crate_name: &str) { let proc_macro_wrapper: syn::ItemFn = parse_quote!( #[proc_macro] - #[expect(unused_qualifications, reason = "generated code")] + #[expect(unused_qualifications, non_snake_case, reason = "generated code")] pub fn #underscored_path(input: ::proc_macro::TokenStream) -> ::proc_macro::TokenStream { let input = ::stageleft::internal::TokenStream::from(input); let out = #exported_from_parsed::#underscored_path_impl(input); From ef8d55ee70bfd755d132fc32fed7c0e89f06b5fd Mon Sep 17 00:00:00 2001 From: Shadaj Laddad Date: Mon, 2 Dec 2024 12:52:48 -0800 Subject: [PATCH 2/2] feat(hydroflow_plus_std): extract initial Hydroflow+ utilities into a standard library --- Cargo.lock | 14 ++++++++++ Cargo.toml | 1 + hydroflow_plus_std/Cargo.toml | 28 +++++++++++++++++++ hydroflow_plus_std/build.rs | 3 ++ hydroflow_plus_std/src/lib.rs | 13 +++++++++ .../src}/quorum.rs | 0 .../src}/request_response.rs | 0 hydroflow_plus_test/Cargo.toml | 1 + hydroflow_plus_test/src/cluster/mod.rs | 2 -- hydroflow_plus_test/src/cluster/paxos.rs | 5 ++-- .../src/cluster/paxos_bench.rs | 2 +- hydroflow_plus_test/src/cluster/two_pc.rs | 3 +- 12 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 hydroflow_plus_std/Cargo.toml create mode 100644 hydroflow_plus_std/build.rs create mode 100644 hydroflow_plus_std/src/lib.rs rename {hydroflow_plus_test/src/cluster => hydroflow_plus_std/src}/quorum.rs (100%) rename {hydroflow_plus_test/src/cluster => hydroflow_plus_std/src}/request_response.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 396a800a5c2..2ef983b989d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1607,6 +1607,19 @@ dependencies = [ "trybuild-internals-api", ] +[[package]] +name = "hydroflow_plus_std" +version = "0.1.0" +dependencies = [ + "async-ssh2-lite", + "ctor", + "hydro_deploy", + "hydroflow_plus", + "insta", + "stageleft", + "stageleft_tool", +] + [[package]] name = "hydroflow_plus_test" version = "0.0.0" @@ -1615,6 +1628,7 @@ dependencies = [ "futures", "hydro_deploy", "hydroflow_plus", + "hydroflow_plus_std", "insta", "rand", "serde", diff --git a/Cargo.toml b/Cargo.toml index 224bad27797..23491ccfe4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "hydroflow_lang", "hydroflow_macro", "hydroflow_plus", + "hydroflow_plus_std", "hydroflow_plus_test", "hydroflow_plus_test_local", "hydroflow_plus_test_local_macro", diff --git a/hydroflow_plus_std/Cargo.toml b/hydroflow_plus_std/Cargo.toml new file mode 100644 index 00000000000..0a5f61e9743 --- /dev/null +++ b/hydroflow_plus_std/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "hydroflow_plus_std" +publish = true +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +documentation = "https://docs.rs/hydroflow_plus_std/" +description = "Standard library of distributed systems building blocks for Hydroflow+" + +[lints] +workspace = true + +[lib] +path = "src/lib.rs" + +[dependencies] +hydroflow_plus = { path = "../hydroflow_plus", version = "^0.10.0", default-features = false } +stageleft = { path = "../stageleft", version = "^0.5.0" } + +[build-dependencies] +stageleft_tool = { path = "../stageleft_tool", version = "^0.4.0" } + +[dev-dependencies] +hydroflow_plus = { path = "../hydroflow_plus", version = "^0.10.0" } +insta = "1.39" +hydro_deploy = { path = "../hydro_deploy/core", version = "^0.10.0" } +async-ssh2-lite = { version = "0.5.0", features = ["vendored-openssl"] } +ctor = "0.2.8" diff --git a/hydroflow_plus_std/build.rs b/hydroflow_plus_std/build.rs new file mode 100644 index 00000000000..99775c3c7da --- /dev/null +++ b/hydroflow_plus_std/build.rs @@ -0,0 +1,3 @@ +fn main() { + stageleft_tool::gen_final!(); +} diff --git a/hydroflow_plus_std/src/lib.rs b/hydroflow_plus_std/src/lib.rs new file mode 100644 index 00000000000..fc5a2ab71fb --- /dev/null +++ b/hydroflow_plus_std/src/lib.rs @@ -0,0 +1,13 @@ +stageleft::stageleft_no_entry_crate!(); + +pub mod quorum; +pub mod request_response; + +#[stageleft::runtime] +#[cfg(test)] +mod tests { + #[ctor::ctor] + fn init() { + hydroflow_plus::deploy::init_test(); + } +} diff --git a/hydroflow_plus_test/src/cluster/quorum.rs b/hydroflow_plus_std/src/quorum.rs similarity index 100% rename from hydroflow_plus_test/src/cluster/quorum.rs rename to hydroflow_plus_std/src/quorum.rs diff --git a/hydroflow_plus_test/src/cluster/request_response.rs b/hydroflow_plus_std/src/request_response.rs similarity index 100% rename from hydroflow_plus_test/src/cluster/request_response.rs rename to hydroflow_plus_std/src/request_response.rs diff --git a/hydroflow_plus_test/Cargo.toml b/hydroflow_plus_test/Cargo.toml index 6d57e17ebbe..5f58809ef48 100644 --- a/hydroflow_plus_test/Cargo.toml +++ b/hydroflow_plus_test/Cargo.toml @@ -13,6 +13,7 @@ stageleft_devel = [] [dependencies] hydroflow_plus = { path = "../hydroflow_plus", version = "^0.10.0" } +hydroflow_plus_std = { path = "../hydroflow_plus_std", version = "^0.1.0" } tokio = { version = "1.29.0", features = [ "full" ] } stageleft = { path = "../stageleft", version = "^0.5.0" } rand = "0.8.0" diff --git a/hydroflow_plus_test/src/cluster/mod.rs b/hydroflow_plus_test/src/cluster/mod.rs index 88bcbe2d949..b4e5be9bb00 100644 --- a/hydroflow_plus_test/src/cluster/mod.rs +++ b/hydroflow_plus_test/src/cluster/mod.rs @@ -4,7 +4,5 @@ pub mod map_reduce; pub mod paxos; pub mod paxos_bench; pub mod paxos_kv; -pub mod quorum; -pub mod request_response; pub mod simple_cluster; pub mod two_pc; diff --git a/hydroflow_plus_test/src/cluster/paxos.rs b/hydroflow_plus_test/src/cluster/paxos.rs index af3a3ca04f3..66b28264aba 100644 --- a/hydroflow_plus_test/src/cluster/paxos.rs +++ b/hydroflow_plus_test/src/cluster/paxos.rs @@ -4,12 +4,11 @@ use std::hash::Hash; use std::time::Duration; use hydroflow_plus::*; +use hydroflow_plus_std::quorum::{collect_quorum, collect_quorum_with_response}; +use hydroflow_plus_std::request_response::join_responses; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; -use super::quorum::{collect_quorum, collect_quorum_with_response}; -use super::request_response::join_responses; - pub struct Proposer {} pub struct Acceptor {} diff --git a/hydroflow_plus_test/src/cluster/paxos_bench.rs b/hydroflow_plus_test/src/cluster/paxos_bench.rs index 4d87409fbde..7517bd2cc94 100644 --- a/hydroflow_plus_test/src/cluster/paxos_bench.rs +++ b/hydroflow_plus_test/src/cluster/paxos_bench.rs @@ -3,11 +3,11 @@ use std::rc::Rc; use std::time::Duration; use hydroflow_plus::*; +use hydroflow_plus_std::quorum::collect_quorum; use tokio::time::Instant; use super::paxos::{Acceptor, Ballot, Proposer}; use super::paxos_kv::{paxos_kv, KvPayload, Replica}; -use super::quorum::collect_quorum; pub struct Client {} diff --git a/hydroflow_plus_test/src/cluster/two_pc.rs b/hydroflow_plus_test/src/cluster/two_pc.rs index 4a5f8220cb6..429de9ab633 100644 --- a/hydroflow_plus_test/src/cluster/two_pc.rs +++ b/hydroflow_plus_test/src/cluster/two_pc.rs @@ -1,6 +1,5 @@ use hydroflow_plus::*; - -use super::quorum::collect_quorum; +use hydroflow_plus_std::quorum::collect_quorum; // if the variable start with p, that means current work is at the participant side. if start with c, at coordinator side. //