From cfe8ac959bd3e071fc6ebdc5e8068f06ee6d4817 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 21 Dec 2024 12:35:57 -0500 Subject: [PATCH 01/14] refactor to diesel async --- scylla-server/Cargo.lock | 239 ++++++++++++++++-- scylla-server/Cargo.toml | 3 +- .../src/controllers/data_controller.rs | 2 +- .../src/controllers/data_type_controller.rs | 2 +- .../controllers/file_insertion_controller.rs | 2 +- .../src/controllers/run_controller.rs | 6 +- scylla-server/src/db_handler.rs | 24 +- scylla-server/src/error.rs | 6 +- scylla-server/src/lib.rs | 4 +- scylla-server/src/main.rs | 58 +++-- scylla-server/src/services/data_service.rs | 15 +- .../src/services/data_type_service.rs | 9 +- scylla-server/src/services/run_service.rs | 21 +- scylla-server/tests/data_service_test.rs | 12 +- scylla-server/tests/data_type_service_test.rs | 11 +- scylla-server/tests/run_service_test.rs | 8 +- scylla-server/tests/test_utils.rs | 34 ++- 17 files changed, 348 insertions(+), 108 deletions(-) diff --git a/scylla-server/Cargo.lock b/scylla-server/Cargo.lock index 6f82b66c..aaad1555 100755 --- a/scylla-server/Cargo.lock +++ b/scylla-server/Cargo.lock @@ -315,6 +315,18 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bb8" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89aabfae550a5c44b43ab941844ffcd2e993cb6900b342debf59e9ea74acdb8" +dependencies = [ + "async-trait", + "futures-util", + "parking_lot", + "tokio", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -580,7 +592,21 @@ dependencies = [ "diesel_derives", "itoa", "pq-sys", - "r2d2", +] + +[[package]] +name = "diesel-async" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51a307ac00f7c23f526a04a77761a0519b9f0eb2838ebf5b905a58580095bdcb" +dependencies = [ + "async-trait", + "bb8", + "diesel", + "futures-util", + "scoped-futures", + "tokio", + "tokio-postgres", ] [[package]] @@ -624,6 +650,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -704,6 +731,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fastrand" version = "2.1.0" @@ -759,6 +792,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -898,6 +932,15 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "home" version = "0.5.9" @@ -1185,6 +1228,16 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d3c2fcf089c060eb333302d80c5f3ffa8297abecf220f788e4a09ef85f59420" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.7.4" @@ -1368,6 +1421,24 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.5" @@ -1406,6 +1477,35 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +[[package]] +name = "postgres-protocol" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acda0ebdebc28befa84bee35e651e4c5f09073d668c7aed4cf7e23c3cda84b23" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand", + "sha2", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f66ea23a2d0e5734297357705193335e0a957696f34bed2f2faefacb2fec336f" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1534,17 +1634,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "r2d2" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" -dependencies = [ - "log", - "parking_lot", - "scheduled-thread-pool", -] - [[package]] name = "rand" version = "0.8.5" @@ -1770,12 +1859,12 @@ dependencies = [ ] [[package]] -name = "scheduled-thread-pool" -version = "0.2.7" +name = "scoped-futures" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" +checksum = "1b24aae2d0636530f359e9d5ef0c04669d11c5e756699b27a6a6d845d8329091" dependencies = [ - "parking_lot", + "pin-project-lite", ] [[package]] @@ -1795,6 +1884,7 @@ dependencies = [ "clap", "console-subscriber", "diesel", + "diesel-async", "diesel_migrations", "dotenvy", "pq-sys", @@ -1925,6 +2015,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1943,6 +2044,12 @@ dependencies = [ "libc", ] +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "slab" version = "0.4.9" @@ -2001,6 +2108,17 @@ dependencies = [ "lock_api", ] +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -2089,6 +2207,21 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tinyvec" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.38.0" @@ -2130,6 +2263,32 @@ dependencies = [ "syn 2.0.67", ] +[[package]] +name = "tokio-postgres" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b5d3742945bc7d7f210693b0c58ae542c6fd47b17adbbda0885f3dcb34a6bdb" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + [[package]] name = "tokio-rustls" version = "0.25.0" @@ -2383,12 +2542,33 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-normalization" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" + [[package]] name = "untrusted" version = "0.9.0" @@ -2440,6 +2620,12 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.79" @@ -2494,6 +2680,16 @@ version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" +[[package]] +name = "web-sys" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "which" version = "4.4.2" @@ -2506,6 +2702,17 @@ dependencies = [ "rustix", ] +[[package]] +name = "whoami" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" +dependencies = [ + "redox_syscall", + "wasite", + "web-sys", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/scylla-server/Cargo.toml b/scylla-server/Cargo.toml index 3e795258..21a3ef86 100644 --- a/scylla-server/Cargo.toml +++ b/scylla-server/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" default-run = "scylla-server" [dependencies] -diesel = { version = "2.2.4", features = ["postgres", "r2d2", "chrono"] } +diesel = { version = "2.2.4", features = ["postgres", "chrono"] } pq-sys = { version = "0.6", features = ["bundled"] } dotenvy = "0.15" serde = "1.0.203" @@ -30,6 +30,7 @@ serde_json = "1.0.128" diesel_migrations = { version = "2.2.0", features = ["postgres"] } rangemap = "1.5.1" axum-macros = "0.4.2" +diesel-async = { version = "0.5.2", features = ["postgres", "bb8", "async-connection-wrapper", "sync-connection-wrapper"] } [features] top = ["dep:console-subscriber"] diff --git a/scylla-server/src/controllers/data_controller.rs b/scylla-server/src/controllers/data_controller.rs index a0330112..837fcf1a 100644 --- a/scylla-server/src/controllers/data_controller.rs +++ b/scylla-server/src/controllers/data_controller.rs @@ -13,7 +13,7 @@ pub async fn get_data( State(pool): State, Path((data_type_name, run_id)): Path<(String, i32)>, ) -> Result>, ScyllaError> { - let mut db = pool.get()?; + let mut db = pool.get().await?; let data = data_service::get_data(&mut db, data_type_name, run_id).await?; // map data to frontend data types according to the From func of the client struct diff --git a/scylla-server/src/controllers/data_type_controller.rs b/scylla-server/src/controllers/data_type_controller.rs index b2821796..ce958131 100644 --- a/scylla-server/src/controllers/data_type_controller.rs +++ b/scylla-server/src/controllers/data_type_controller.rs @@ -9,7 +9,7 @@ use crate::{ pub async fn get_all_data_types( State(pool): State, ) -> Result>, ScyllaError> { - let mut db = pool.get()?; + let mut db = pool.get().await?; let data_types = data_type_service::get_all_data_types(&mut db).await?; let transformed_data_types: Vec = diff --git a/scylla-server/src/controllers/file_insertion_controller.rs b/scylla-server/src/controllers/file_insertion_controller.rs index f067895c..f139523e 100644 --- a/scylla-server/src/controllers/file_insertion_controller.rs +++ b/scylla-server/src/controllers/file_insertion_controller.rs @@ -21,7 +21,7 @@ pub async fn insert_file( mut multipart: Multipart, ) -> Result { // create a run ID cache - let mut db = pool.get()?; + let mut db = pool.get().await?; debug!("Warming up run ID map!"); let mut run_iter = run_service::get_all_runs(&mut db) .await? diff --git a/scylla-server/src/controllers/run_controller.rs b/scylla-server/src/controllers/run_controller.rs index 8922dabe..bade8d37 100644 --- a/scylla-server/src/controllers/run_controller.rs +++ b/scylla-server/src/controllers/run_controller.rs @@ -13,7 +13,7 @@ use crate::{ pub async fn get_all_runs( State(pool): State, ) -> Result>, ScyllaError> { - let mut db = pool.get()?; + let mut db = pool.get().await?; let run_data = run_service::get_all_runs(&mut db).await?; let transformed_run_data: Vec = run_data.into_iter().map(PublicRun::from).collect(); @@ -26,7 +26,7 @@ pub async fn get_run_by_id( State(pool): State, Path(run_id): Path, ) -> Result, ScyllaError> { - let mut db = pool.get()?; + let mut db = pool.get().await?; let run_data = run_service::get_run_by_id(&mut db, run_id).await?; if run_data.is_none() { @@ -43,7 +43,7 @@ pub async fn get_run_by_id( /// create a new run with an auto-incremented ID /// note the new run must be updated so the channel passed in notifies the data processor to use the new run pub async fn new_run(State(pool): State) -> Result, ScyllaError> { - let mut db = pool.get()?; + let mut db = pool.get().await?; let run_data = run_service::create_run(&mut db, chrono::offset::Utc::now()).await?; crate::RUN_ID.store(run_data.id, Ordering::Relaxed); diff --git a/scylla-server/src/db_handler.rs b/scylla-server/src/db_handler.rs index 0ed9145b..fe482f5f 100644 --- a/scylla-server/src/db_handler.rs +++ b/scylla-server/src/db_handler.rs @@ -54,7 +54,7 @@ impl DbHandler { loop { tokio::select! { _ = cancel_token.cancelled() => { - let Ok(mut database) = pool.get() else { + let Ok(mut database) = pool.get().await else { warn!("Could not get connection for cleanup"); break; }; @@ -62,7 +62,7 @@ impl DbHandler { while let Some(final_msgs) = batch_queue.recv().await { info!("{} batches remaining!", batch_queue.len()+1); // do not spawn new tasks in this mode, see below comment for chunk_size math - let chunk_size = final_msgs.len() / ((final_msgs.len() / 16380) + 1); + let chunk_size = final_msgs.len() / ((final_msgs.len() / 8190) + 1); if chunk_size == 0 { warn!("Could not insert {} messages, chunk size zero!", final_msgs.len()); continue; @@ -70,7 +70,7 @@ impl DbHandler { for chunk in final_msgs.chunks(chunk_size).collect::>() { info!( "A cleanup chunk uploaded: {:?}", - data_service::add_many(&mut database, chunk.to_vec()) + data_service::add_many(&mut database, chunk.to_vec()).await ); } } @@ -81,7 +81,7 @@ impl DbHandler { // libpq has max 65535 params, therefore batch // max for batch is 65535/4 params per message, hence the below, rounded down with a margin for safety // TODO avoid this code batch uploading the remainder messages as a new batch, combine it with another safely - let chunk_size = msgs.len() / ((msgs.len() / 16380) + 1); + let chunk_size = msgs.len() / ((msgs.len() / 8190) + 1); if chunk_size == 0 { warn!("Could not insert {} messages, chunk size zero!", msgs.len()); continue; @@ -90,8 +90,7 @@ impl DbHandler { for chunk in msgs.chunks(chunk_size).collect::>() { let owned = chunk.to_vec(); let pool = pool.clone(); - tokio::task::spawn_blocking(move || { - DbHandler::batch_upload(owned, pool)}); + tokio::spawn(DbHandler::batch_upload(owned, pool)); } debug!( "DB send: {} of {}", @@ -121,13 +120,14 @@ impl DbHandler { } } - //#[instrument(level = Level::DEBUG, skip(msg, pool))] - fn batch_upload(msg: Vec, pool: PoolHandle) { - let Ok(mut database) = pool.get() else { + #[instrument(level = Level::DEBUG, skip(msg, pool))] + async fn batch_upload(msg: Vec, pool: PoolHandle) { + let Ok(mut database) = pool.get().await else { warn!("Could not get connection for batch upload!"); return; }; - match data_service::add_many(&mut database, msg) { + warn!("MSG LEN {}", msg.len()); + match data_service::add_many(&mut database, msg).await { Ok(count) => info!("Batch uploaded: {:?}", count), Err(err) => warn!("Error in batch upload: {:?}", err), } @@ -180,7 +180,7 @@ impl DbHandler { } if !self.datatype_list.contains(&msg.name) { - let Ok(mut database) = self.pool.get() else { + let Ok(mut database) = self.pool.get().await else { warn!("Could not get connection for dataType upsert"); return; }; @@ -201,7 +201,7 @@ impl DbHandler { // Check for GPS points, insert them into current run if available if msg.name == "TPU/GPS/Location" { debug!("Upserting run with location points!"); - let Ok(mut database) = self.pool.get() else { + let Ok(mut database) = self.pool.get().await else { warn!("Could not get connection for db points update"); return; }; diff --git a/scylla-server/src/error.rs b/scylla-server/src/error.rs index b228d1ed..e7bf009c 100644 --- a/scylla-server/src/error.rs +++ b/scylla-server/src/error.rs @@ -8,7 +8,7 @@ pub enum ScyllaError { /// Deseil error DbError(diesel::result::Error), /// Diesel db connection error, - ConnError(diesel::r2d2::PoolError), + ConnError(diesel_async::pooled_connection::bb8::RunError), /// An instruction was not encodable InvalidEncoding(String), /// Could not communicate to car @@ -23,8 +23,8 @@ impl From for ScyllaError { } } -impl From for ScyllaError { - fn from(error: diesel::r2d2::PoolError) -> Self { +impl From for ScyllaError { + fn from(error: diesel_async::pooled_connection::bb8::RunError) -> Self { ScyllaError::ConnError(error) } } diff --git a/scylla-server/src/lib.rs b/scylla-server/src/lib.rs index 7043a662..d2fa16d9 100644 --- a/scylla-server/src/lib.rs +++ b/scylla-server/src/lib.rs @@ -19,9 +19,9 @@ pub mod serverdata; pub mod transformers; /// The type descriptor of the database passed to the middlelayer through axum state -pub type Database = diesel::PgConnection; +pub type Database<'a> = diesel_async::pooled_connection::bb8::PooledConnection<'a, diesel_async::AsyncPgConnection>; -pub type PoolHandle = diesel::r2d2::Pool>; +pub type PoolHandle = diesel_async::pooled_connection::bb8::Pool; #[derive(clap::ValueEnum, Debug, PartialEq, Copy, Clone, Default)] #[clap(rename_all = "kebab_case")] diff --git a/scylla-server/src/main.rs b/scylla-server/src/main.rs index d2cf4b6e..8ebd142c 100755 --- a/scylla-server/src/main.rs +++ b/scylla-server/src/main.rs @@ -10,9 +10,10 @@ use axum::{ Extension, Router, }; use clap::Parser; -use diesel::{ - r2d2::{ConnectionManager, Pool}, - PgConnection, +use diesel_async::async_connection_wrapper::AsyncConnectionWrapper; +use diesel_async::{ + pooled_connection::{bb8::Pool, AsyncDieselConnectionManager}, + AsyncConnection, AsyncPgConnection, }; use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; use dotenvy::dotenv; @@ -24,7 +25,7 @@ use scylla_server::{ data_type_controller, file_insertion_controller, run_controller, }, services::run_service::{self}, - PoolHandle, RateLimitMode, + RateLimitMode, }; use scylla_server::{ db_handler, @@ -103,7 +104,7 @@ struct ScyllaArgs { const MIGRATIONS: EmbeddedMigrations = embed_migrations!(); #[tokio::main] -async fn main() { +async fn main() -> Result<(), Box> { let cli = ScyllaArgs::parse(); println!("Initializing scylla server..."); @@ -134,22 +135,31 @@ async fn main() { tracing::subscriber::set_global_default(subscriber).expect("Could not init tracing"); } - // create the database stuff - info!("Initializing database connections..."); dotenv().ok(); - let manager = ConnectionManager::::new(std::env::var("DATABASE_URL").unwrap()); - // Refer to the `r2d2` documentation for more methods to use - // when building a connection pool - let db: PoolHandle = Pool::builder() - .test_on_check_out(true) - .build(manager) - .expect("Could not build connection pool"); + let db_url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be specified"); - let mut conn = db.get().unwrap(); - conn.run_pending_migrations(MIGRATIONS) - .expect("Could not run migrations!"); + info!("Beginning DB migration w/ temporary connection..."); + // it is best to create a temporary unmanaged connection to run the migrations + // a completely new set of connections is created by the pool manager because it cannot understand an already established connection + let conn: AsyncPgConnection = AsyncPgConnection::establish(&db_url).await?; + let mut async_wrapper: AsyncConnectionWrapper = + AsyncConnectionWrapper::from(conn); + tokio::task::spawn_blocking(move || { + async_wrapper.run_pending_migrations(MIGRATIONS).unwrap(); + }) + .await?; info!("Successfully migrated DB!"); + info!("Initializing database connections..."); + let manager = AsyncDieselConnectionManager::::new(db_url); + let pool: Pool = Pool::builder() + .max_size(10) + .min_idle(Some(2)) + .max_lifetime(Some(Duration::from_secs(60 * 60 * 24))) + .idle_timeout(Some(Duration::from_secs(60 * 2))) + .build(manager) + .await?; + // create the socket stuff let (socket_layer, io) = SocketIo::builder() .max_buffer_size(4096) // TODO tune values @@ -175,14 +185,14 @@ async fn main() { let token = CancellationToken::new(); // spawn the database handler task_tracker.spawn( - db_handler::DbHandler::new(mqtt_receive, db.clone(), cli.batch_upsert_time * 1000) + db_handler::DbHandler::new(mqtt_receive, pool.clone(), cli.batch_upsert_time * 1000) .handling_loop(db_send.clone(), token.clone()), ); // spawn the database inserter, if we have it enabled if !cli.disable_data_upload { task_tracker.spawn(db_handler::DbHandler::batching_loop( db_receive, - db.clone(), + pool.clone(), token.clone(), )); } else { @@ -193,9 +203,10 @@ async fn main() { } // creates the initial run - let curr_run = run_service::create_run(&mut db.get().unwrap(), chrono::offset::Utc::now()) - .await - .expect("Could not create initial run!"); + let curr_run = + run_service::create_run(&mut pool.get().await.unwrap(), chrono::offset::Utc::now()) + .await + .expect("Could not create initial run!"); debug!("Configuring current run: {:?}", curr_run); RUN_ID.store(curr_run.id, Ordering::Relaxed); @@ -253,7 +264,7 @@ async fn main() { .layer(socket_layer), ) .layer(TraceLayer::new_for_http()) - .with_state(db.clone()); + .with_state(pool.clone()); let listener = tokio::net::TcpListener::bind("0.0.0.0:8000") .await @@ -280,4 +291,5 @@ async fn main() { info!("Received exit signal, shutting down!"); token.cancel(); task_tracker.wait().await; + Ok(()) } diff --git a/scylla-server/src/services/data_service.rs b/scylla-server/src/services/data_service.rs index 383ff51c..fcf0bf16 100644 --- a/scylla-server/src/services/data_service.rs +++ b/scylla-server/src/services/data_service.rs @@ -1,5 +1,6 @@ use crate::{models::Data, schema::data::dsl::*, ClientData, Database}; use diesel::prelude::*; +use diesel_async::RunQueryDsl; /// Get datapoints that mach criteria /// * `db` - The database connection to use @@ -7,12 +8,12 @@ use diesel::prelude::*; /// * `run_id` - The run id to filter the data /// returns: A result containing the data or the error propogated by the db pub async fn get_data( - db: &mut Database, + db: &mut Database<'_>, data_type_name: String, run_id: i32, ) -> Result, diesel::result::Error> { data.filter(runId.eq(run_id).and(dataTypeName.eq(data_type_name))) - .load(db) + .load(db).await } /// Adds a datapoint @@ -23,7 +24,7 @@ pub async fn get_data( /// * `rin_id` - The run id to assign the data point to, note this run must already exist! /// returns: A result containing the data or the QueryError propogated by the db pub async fn add_data( - db: &mut Database, + db: &mut Database<'_>, client_data: ClientData, ) -> Result { diesel::insert_into(data) @@ -37,15 +38,15 @@ pub async fn add_data( .map(|v| Some(*v as f64)) .collect::>()), )) - .get_result(db) + .get_result(db).await } /// Adds many datapoints via a batch insert, skips any data which conflicts with existing data /// * `db` - The database connection to use /// * `client_data` - A list of data to batch insert /// returns: A result containing the number of rows inserted or the QueryError propogated by the db -pub fn add_many( - db: &mut Database, +pub async fn add_many( + db: &mut Database<'_>, client_data: Vec, ) -> Result { diesel::insert_into(data) @@ -67,5 +68,5 @@ pub fn add_many( .collect::>(), ) .on_conflict_do_nothing() - .execute(db) + .execute(db).await } diff --git a/scylla-server/src/services/data_type_service.rs b/scylla-server/src/services/data_type_service.rs index 757e5543..684c00a8 100644 --- a/scylla-server/src/services/data_type_service.rs +++ b/scylla-server/src/services/data_type_service.rs @@ -1,11 +1,12 @@ use crate::{models::DataType, schema::dataType::dsl::*, Database}; use diesel::prelude::*; +use diesel_async::RunQueryDsl; /// Gets all datatypes /// * `d ` - The connection to the database /// returns: A result containing the data or the QueryError propogated by the db -pub async fn get_all_data_types(db: &mut Database) -> Result, diesel::result::Error> { - dataType.load(db) +pub async fn get_all_data_types(db: &mut Database<'_>) -> Result, diesel::result::Error> { + dataType.load(db).await } /// Upserts a datatype, either creating or updating one depending on its existence @@ -15,7 +16,7 @@ pub async fn get_all_data_types(db: &mut Database) -> Result, dies /// * `node_name` - The name of the node linked to the data type, must already exist! /// returns: A result containing the data or the QueryError propogated by the db pub async fn upsert_data_type( - db: &mut Database, + db: &mut Database<'_>, data_type_name: String, new_unit: String, node_name: String, @@ -31,5 +32,5 @@ pub async fn upsert_data_type( .do_update() // actually allows for the upsert ability .set(&val) .returning(DataType::as_returning()) - .get_result(db) + .get_result(db).await } diff --git a/scylla-server/src/services/run_service.rs b/scylla-server/src/services/run_service.rs index 7897bbb5..fa4ebf7d 100644 --- a/scylla-server/src/services/run_service.rs +++ b/scylla-server/src/services/run_service.rs @@ -1,12 +1,13 @@ use crate::{models::Run, schema::run::dsl::*, Database}; use chrono::{DateTime, Utc}; use diesel::prelude::*; +use diesel_async::RunQueryDsl; /// Gets all runs /// * `db` - The prisma client to make the call to /// returns: A result containing the data or the QueryError propogated by the db -pub async fn get_all_runs(db: &mut Database) -> Result, diesel::result::Error> { - run.order(id.asc()).get_results(db) +pub async fn get_all_runs(db: &mut Database<'_>) -> Result, diesel::result::Error> { + run.order(id.asc()).get_results(db).await } /// Gets a single run by its id @@ -14,10 +15,10 @@ pub async fn get_all_runs(db: &mut Database) -> Result, diesel::result: /// * `run_id` - The id of the run to search for /// returns: A result containing the data (or None if the `run_id` was not a valid run) or the QueryError propogated by the db pub async fn get_run_by_id( - db: &mut Database, + db: &mut Database<'_>, run_id: i32, ) -> Result, diesel::result::Error> { - run.find(run_id).first(db).optional() + run.find(run_id).first::(db).await.optional() } /// Creates a run @@ -25,12 +26,12 @@ pub async fn get_run_by_id( /// * `timestamp` - time when the run starts /// returns: A result containing the data or the QueryError propogated by the db pub async fn create_run( - db: &mut Database, + db: &mut Database<'_>, timestamp: DateTime, ) -> Result { diesel::insert_into(run) .values(time.eq(timestamp)) - .get_result(db) + .get_result(db).await } /// Creates a run with a given id @@ -39,13 +40,13 @@ pub async fn create_run( /// * `run_id` - The id of the run to create, must not already be in use! /// returns: A result containing the data or the QueryError propogated by the db pub async fn create_run_with_id( - db: &mut Database, + db: &mut Database<'_>, timestamp: DateTime, run_id: i32, ) -> Result { diesel::insert_into(run) .values((time.eq(timestamp), id.eq(run_id))) - .get_result(db) + .get_result(db).await } /// Updates a run with GPS points @@ -54,12 +55,12 @@ pub async fn create_run_with_id( /// * `lat` - The latitude /// * `long` - The longitude pub async fn update_run_with_coords( - db: &mut Database, + db: &mut Database<'_>, run_id: i32, lat: f64, long: f64, ) -> Result { diesel::update(run.filter(id.eq(run_id))) .set((latitude.eq(lat), longitude.eq(long))) - .get_result(db) + .get_result(db).await } diff --git a/scylla-server/tests/data_service_test.rs b/scylla-server/tests/data_service_test.rs index 2988049d..bf450707 100644 --- a/scylla-server/tests/data_service_test.rs +++ b/scylla-server/tests/data_service_test.rs @@ -13,7 +13,8 @@ const TEST_KEYWORD: &str = "test"; #[tokio::test] async fn test_data_service() -> Result<(), diesel::result::Error> { - let mut db = cleanup_and_prepare().await?; + let pool = cleanup_and_prepare().await.unwrap(); + let mut db = pool.get().await.unwrap(); run_service::create_run_with_id( &mut db, @@ -36,7 +37,8 @@ async fn test_data_service() -> Result<(), diesel::result::Error> { #[tokio::test] async fn test_data_add() -> Result<(), diesel::result::Error> { - let mut db = cleanup_and_prepare().await?; + let pool = cleanup_and_prepare().await.unwrap(); + let mut db = pool.get().await.unwrap(); // node_service::upsert_node(&db, TEST_KEYWORD.to_owned()).await?; data_type_service::upsert_data_type( @@ -78,7 +80,8 @@ async fn test_data_add() -> Result<(), diesel::result::Error> { #[tokio::test] async fn test_data_fetch_empty() -> Result<(), diesel::result::Error> { - let mut db = cleanup_and_prepare().await?; + let pool = cleanup_and_prepare().await.unwrap(); + let mut db = pool.get().await.unwrap(); // should be empty, nothing was added to run let data = data_service::get_data(&mut db, TEST_KEYWORD.to_owned(), 0).await?; @@ -90,7 +93,8 @@ async fn test_data_fetch_empty() -> Result<(), diesel::result::Error> { #[tokio::test] async fn test_data_no_prereqs() -> Result<(), diesel::result::Error> { - let mut db = cleanup_and_prepare().await?; + let pool = cleanup_and_prepare().await.unwrap(); + let mut db = pool.get().await.unwrap(); // should err out as data type name doesnt exist yet data_service::add_data( diff --git a/scylla-server/tests/data_type_service_test.rs b/scylla-server/tests/data_type_service_test.rs index 9de5c9ee..5a148999 100644 --- a/scylla-server/tests/data_type_service_test.rs +++ b/scylla-server/tests/data_type_service_test.rs @@ -3,8 +3,9 @@ mod test_utils; use diesel::{ query_dsl::methods::{FilterDsl, SelectDsl}, - ExpressionMethods, RunQueryDsl, SelectableHelper, + ExpressionMethods, SelectableHelper, }; +use diesel_async::RunQueryDsl; use scylla_server::{ models::DataType, schema::dataType, services::data_type_service, transformers::data_type_transformer::PublicDataType, @@ -13,7 +14,8 @@ use test_utils::cleanup_and_prepare; #[tokio::test] async fn test_get_all_datatypes() -> Result<(), diesel::result::Error> { - let mut db = cleanup_and_prepare().await?; + let pool = cleanup_and_prepare().await.unwrap(); + let mut db = pool.get().await.unwrap(); // ensure datatypes is empty assert!(data_type_service::get_all_data_types(&mut db) @@ -29,7 +31,8 @@ async fn test_datatype_create() -> Result<(), diesel::result::Error> { let unit: String = "testUnitCreation".to_owned(); let node_name: String = "testNode".to_owned(); - let mut db = cleanup_and_prepare().await?; + let pool = cleanup_and_prepare().await.unwrap(); + let mut db = pool.get().await.unwrap(); // make node // node_service::upsert_node(&mut db, node_name.clone()).await?; @@ -41,7 +44,7 @@ async fn test_datatype_create() -> Result<(), diesel::result::Error> { let data = dataType::table .filter(dataType::name.eq(data_type_name.clone())) .select(DataType::as_select()) - .get_result(&mut db)?; + .get_result(&mut db).await?; assert_eq!( PublicDataType::from(data), diff --git a/scylla-server/tests/run_service_test.rs b/scylla-server/tests/run_service_test.rs index f72a6707..2a5db7de 100644 --- a/scylla-server/tests/run_service_test.rs +++ b/scylla-server/tests/run_service_test.rs @@ -6,7 +6,8 @@ mod test_utils; #[tokio::test] async fn test_get_all_runs() -> Result<(), diesel::result::Error> { - let mut db = cleanup_and_prepare().await?; + let pool = cleanup_and_prepare().await.unwrap(); + let mut db = pool.get().await.unwrap(); // ensure runs is empty assert!(run_service::get_all_runs(&mut db).await?.is_empty()); @@ -16,8 +17,9 @@ async fn test_get_all_runs() -> Result<(), diesel::result::Error> { #[tokio::test] async fn test_get_run_by_id() -> Result<(), diesel::result::Error> { - let mut db = cleanup_and_prepare().await?; - + let pool = cleanup_and_prepare().await.unwrap(); + let mut db = pool.get().await.unwrap(); + // add a run let run_c = run_service::create_run(&mut db, chrono::DateTime::from_timestamp_millis(1).unwrap()) diff --git a/scylla-server/tests/test_utils.rs b/scylla-server/tests/test_utils.rs index 4cc750c5..bfb0966b 100644 --- a/scylla-server/tests/test_utils.rs +++ b/scylla-server/tests/test_utils.rs @@ -1,20 +1,28 @@ -use diesel::prelude::*; +use std::time::Duration; + use dotenvy::dotenv; -use scylla_server::{ - schema::{data, dataType, run}, - Database, -}; +use scylla_server::schema::{data, dataType, run}; +use diesel_async::{pooled_connection::{bb8::Pool, AsyncDieselConnectionManager}, AsyncPgConnection, RunQueryDsl}; + -pub async fn cleanup_and_prepare() -> Result { +pub async fn cleanup_and_prepare() -> Result, diesel::result::Error> { dotenv().ok(); - let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set"); - let mut client = PgConnection::establish(&database_url) - .unwrap_or_else(|_| panic!("Error connecting to {}", database_url)); + let db_url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be specified"); + //let mut config = ManagerConfig::default(); + let manager = AsyncDieselConnectionManager::::new(db_url); + let pool: Pool = Pool::builder() + .max_size(10) + .min_idle(Some(5)) + .max_lifetime(Some(Duration::from_secs(60 * 60 * 24))) + .idle_timeout(Some(Duration::from_secs(60 * 2))) + .build(manager) + .await.unwrap(); + let mut client = pool.get().await.unwrap(); - diesel::delete(data::table).execute(&mut client)?; - diesel::delete(dataType::table).execute(&mut client)?; - diesel::delete(run::table).execute(&mut client)?; + diesel::delete(data::table).execute(&mut client).await?; + diesel::delete(dataType::table).execute(&mut client).await?; + diesel::delete(run::table).execute(&mut client).await?; - Ok(client) + Ok(pool.clone()) } From 763cd8e41ab8d3d72832f43440b1a337b42d26d6 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 21 Dec 2024 14:34:14 -0500 Subject: [PATCH 02/14] remove 4 re-allocations by modifying DB and insertion layer --- .../2024-11-10-031516_create_all/up.sql | 2 +- scylla-server/src/db_handler.rs | 9 ++-- scylla-server/src/lib.rs | 17 ++++++- scylla-server/src/models.rs | 20 +++++++- scylla-server/src/mqtt_processor.rs | 12 ++--- scylla-server/src/schema.rs | 8 +++- scylla-server/src/services/data_service.rs | 47 ++++++------------- .../src/transformers/data_transformer.rs | 5 +- .../src/transformers/run_transformer.rs | 2 +- scylla-server/tests/data_service_test.rs | 2 +- 10 files changed, 71 insertions(+), 53 deletions(-) diff --git a/scylla-server/migrations/2024-11-10-031516_create_all/up.sql b/scylla-server/migrations/2024-11-10-031516_create_all/up.sql index beb38f90..67dcbd03 100644 --- a/scylla-server/migrations/2024-11-10-031516_create_all/up.sql +++ b/scylla-server/migrations/2024-11-10-031516_create_all/up.sql @@ -15,7 +15,7 @@ CREATE TABLE "run" ( -- CreateTable CREATE TABLE "data" ( - "values" DOUBLE PRECISION[], + "values" REAL [] NOT NULL check ("values" <> '{}' AND array_position("values", NULL) IS NULL), "dataTypeName" TEXT NOT NULL, "time" TIMESTAMPTZ NOT NULL, "runId" INTEGER NOT NULL, diff --git a/scylla-server/src/db_handler.rs b/scylla-server/src/db_handler.rs index fe482f5f..d56be3f9 100644 --- a/scylla-server/src/db_handler.rs +++ b/scylla-server/src/db_handler.rs @@ -126,7 +126,6 @@ impl DbHandler { warn!("Could not get connection for batch upload!"); return; }; - warn!("MSG LEN {}", msg.len()); match data_service::add_many(&mut database, msg).await { Ok(count) => info!("Batch uploaded: {:?}", count), Err(err) => warn!("Error in batch upload: {:?}", err), @@ -147,8 +146,8 @@ impl DbHandler { tokio::select! { _ = cancel_token.cancelled() => { debug!("Pushing final messages to queue"); - data_channel.send(self.data_queue.clone()).await.expect("Could not comm data to db thread, shutdown"); - self.data_queue.clear(); + data_channel.send(self.data_queue).await.expect("Could not comm data to db thread, shutdown"); + self.data_queue = vec![]; break; }, Some(msg) = self.receiver.recv() => { @@ -171,11 +170,11 @@ impl DbHandler { > Duration::from_millis(self.upload_interval) && !self.data_queue.is_empty() { + // mem::take allows us to assign the value of data queue vec::new() while maintaining the memory for data_channel ownership data_channel - .send(self.data_queue.clone()) + .send(std::mem::take(&mut self.data_queue)) .await .expect("Could not comm data to db thread"); - self.data_queue.clear(); self.last_time = tokio::time::Instant::now(); } diff --git a/scylla-server/src/lib.rs b/scylla-server/src/lib.rs index d2fa16d9..567fce1e 100644 --- a/scylla-server/src/lib.rs +++ b/scylla-server/src/lib.rs @@ -19,7 +19,8 @@ pub mod serverdata; pub mod transformers; /// The type descriptor of the database passed to the middlelayer through axum state -pub type Database<'a> = diesel_async::pooled_connection::bb8::PooledConnection<'a, diesel_async::AsyncPgConnection>; +pub type Database<'a> = + diesel_async::pooled_connection::bb8::PooledConnection<'a, diesel_async::AsyncPgConnection>; pub type PoolHandle = diesel_async::pooled_connection::bb8::Pool; @@ -56,3 +57,17 @@ pub struct ClientData { #[serde(skip_serializing)] pub node: String, } + +/// this is the main conversion code to insert data. +/// +/// it is essential this conversion does no re-allocate +impl Into for ClientData { + fn into(self) -> models::DataInsert { + models::DataInsert { + values: self.values, + dataTypeName: self.name, + time: self.timestamp, + runId: self.run_id, + } + } +} diff --git a/scylla-server/src/models.rs b/scylla-server/src/models.rs index 88f61a68..175f817a 100644 --- a/scylla-server/src/models.rs +++ b/scylla-server/src/models.rs @@ -2,13 +2,31 @@ use chrono::{DateTime, Utc}; use diesel::prelude::*; use serde::Serialize; +/// Use this struct when querying data #[derive(Queryable, Debug, Identifiable, Insertable, Selectable, Serialize, AsChangeset)] #[diesel(table_name = crate::schema::data)] #[diesel(belongs_to(DataType, foreign_key = dataTypeName))] #[diesel(check_for_backend(diesel::pg::Pg))] #[diesel(primary_key(dataTypeName, time))] pub struct Data { - pub values: Option>>, + pub values: Vec>, + pub dataTypeName: String, + pub time: DateTime, + pub runId: i32, +} + +/// Use this struct when inserting data. +/// +/// This struct is required because Diesel must infer Arrays as Array>, +/// and to query such types our signature for values must be Vec>, +/// but the overhead of mapping Vec to Vec> is non-negligible. +#[derive(Insertable)] +#[diesel(table_name = crate::schema::data)] +#[diesel(belongs_to(DataType, foreign_key = dataTypeName))] +#[diesel(check_for_backend(diesel::pg::Pg))] +#[diesel(primary_key(dataTypeName, time))] +pub struct DataInsert { + pub values: Vec, pub dataTypeName: String, pub time: DateTime, pub runId: i32, diff --git a/scylla-server/src/mqtt_processor.rs b/scylla-server/src/mqtt_processor.rs index 8509dc26..cfaf040d 100644 --- a/scylla-server/src/mqtt_processor.rs +++ b/scylla-server/src/mqtt_processor.rs @@ -144,8 +144,8 @@ impl MqttProcessor { None => continue }; latency_ringbuffer.push(chrono::offset::Utc::now() - msg.timestamp); - self.send_db_msg(msg.clone()).await; - self.send_socket_msg(msg, &mut upload_counter); + self.send_socket_msg(&msg, &mut upload_counter); + self.send_db_msg(msg).await; }, Err(msg) => trace!("Received mqtt error: {:?}", msg), _ => trace!("Received misc mqtt: {:?}", msg), @@ -165,7 +165,7 @@ impl MqttProcessor { timestamp: chrono::offset::Utc::now(), values: vec![sockets_cnt] }; - self.send_socket_msg(client_data, &mut upload_counter); + self.send_socket_msg(&client_data, &mut upload_counter); } _ = latency_interval.tick() => { // set latency to 0 if no messages are in buffer @@ -184,7 +184,7 @@ impl MqttProcessor { values: vec![avg_latency as f32] }; trace!("Latency update sending: {}", client_data.values.first().unwrap_or(&0.0f32)); - self.send_socket_msg(client_data, &mut upload_counter); + self.send_socket_msg(&client_data, &mut upload_counter); } } } @@ -310,14 +310,14 @@ impl MqttProcessor { /// Send a message to the channel, printing and IGNORING any error that may occur /// * `client_data` - The client data to send over the broadcast async fn send_db_msg(&self, client_data: ClientData) { - if let Err(err) = self.channel.send(client_data.clone()).await { + if let Err(err) = self.channel.send(client_data).await { warn!("Error sending through channel: {:?}", err); } } /// Sends a message to the socket, printing and IGNORING any error that may occur /// * `client_data` - The client data to send over the broadcast - fn send_socket_msg(&self, client_data: ClientData, upload_counter: &mut u8) { + fn send_socket_msg(&self, client_data: &ClientData, upload_counter: &mut u8) { *upload_counter = upload_counter.wrapping_add(1); if *upload_counter >= self.upload_ratio { match self.io.emit( diff --git a/scylla-server/src/schema.rs b/scylla-server/src/schema.rs index 95aea5dc..b8115048 100644 --- a/scylla-server/src/schema.rs +++ b/scylla-server/src/schema.rs @@ -2,7 +2,7 @@ diesel::table! { data (time, dataTypeName) { - values -> Nullable>>, + values -> Array>, dataTypeName -> Text, time -> Timestamptz, runId -> Int4, @@ -32,4 +32,8 @@ diesel::table! { diesel::joinable!(data -> dataType (dataTypeName)); diesel::joinable!(data -> run (runId)); -diesel::allow_tables_to_appear_in_same_query!(data, dataType, run,); +diesel::allow_tables_to_appear_in_same_query!( + data, + dataType, + run, +); diff --git a/scylla-server/src/services/data_service.rs b/scylla-server/src/services/data_service.rs index fcf0bf16..f1aea2c6 100644 --- a/scylla-server/src/services/data_service.rs +++ b/scylla-server/src/services/data_service.rs @@ -1,4 +1,8 @@ -use crate::{models::Data, schema::data::dsl::*, ClientData, Database}; +use crate::{ + models::{Data, DataInsert}, + schema::data::dsl::*, + ClientData, Database, +}; use diesel::prelude::*; use diesel_async::RunQueryDsl; @@ -13,7 +17,8 @@ pub async fn get_data( run_id: i32, ) -> Result, diesel::result::Error> { data.filter(runId.eq(run_id).and(dataTypeName.eq(data_type_name))) - .load(db).await + .load(db) + .await } /// Adds a datapoint @@ -28,23 +33,11 @@ pub async fn add_data( client_data: ClientData, ) -> Result { diesel::insert_into(data) - .values(( - dataTypeName.eq(client_data.name), - time.eq(client_data.timestamp), - runId.eq(client_data.run_id), - values.eq(client_data - .values - .iter() - .map(|v| Some(*v as f64)) - .collect::>()), - )) - .get_result(db).await + .values(Into::::into(client_data)) + .get_result(db) + .await } -/// Adds many datapoints via a batch insert, skips any data which conflicts with existing data -/// * `db` - The database connection to use -/// * `client_data` - A list of data to batch insert -/// returns: A result containing the number of rows inserted or the QueryError propogated by the db pub async fn add_many( db: &mut Database<'_>, client_data: Vec, @@ -52,21 +45,11 @@ pub async fn add_many( diesel::insert_into(data) .values( client_data - .iter() - .map(|single_client_data| { - ( - dataTypeName.eq(single_client_data.name.clone()), - time.eq(single_client_data.timestamp), - runId.eq(single_client_data.run_id), - values.eq(single_client_data - .values - .iter() - .map(|v| Some(*v as f64)) - .collect::>()), - ) - }) - .collect::>(), + .into_iter() + .map(|single_client_data| Into::::into(single_client_data)) + .collect::>(), ) .on_conflict_do_nothing() - .execute(db).await + .execute(db) + .await } diff --git a/scylla-server/src/transformers/data_transformer.rs b/scylla-server/src/transformers/data_transformer.rs index a7670ba4..a7b279ad 100644 --- a/scylla-server/src/transformers/data_transformer.rs +++ b/scylla-server/src/transformers/data_transformer.rs @@ -9,7 +9,7 @@ use crate::ClientData; pub struct PublicData { #[serde(rename = "time")] pub time_ms: i64, - pub values: Vec, + pub values: Vec, } // custom impls to avoid comparing values fields @@ -39,7 +39,6 @@ impl From for PublicData { PublicData { values: value .values - .unwrap_or_default() .into_iter() .flatten() .collect(), @@ -53,7 +52,7 @@ impl From for PublicData { fn from(value: ClientData) -> Self { PublicData { time_ms: value.timestamp.timestamp_millis(), - values: value.values.iter().map(|f| *f as f64).collect(), + values: value.values, } } } diff --git a/scylla-server/src/transformers/run_transformer.rs b/scylla-server/src/transformers/run_transformer.rs index bd7676d1..5ee2c3e6 100644 --- a/scylla-server/src/transformers/run_transformer.rs +++ b/scylla-server/src/transformers/run_transformer.rs @@ -18,7 +18,7 @@ impl From for PublicRun { PublicRun { id: value.id, location_name: value.locationName.unwrap_or_default(), - driver_name: value.driverName.clone().unwrap_or_default(), + driver_name: value.driverName.unwrap_or_default(), time_ms: value.time.timestamp_millis(), notes: value.notes, } diff --git a/scylla-server/tests/data_service_test.rs b/scylla-server/tests/data_service_test.rs index bf450707..62489836 100644 --- a/scylla-server/tests/data_service_test.rs +++ b/scylla-server/tests/data_service_test.rs @@ -71,7 +71,7 @@ async fn test_data_add() -> Result<(), diesel::result::Error> { PublicData::from(data), PublicData { time_ms: 1000, - values: vec![0f64] + values: vec![0f32] } ); From 54af3c64b350e613ce2dea32887c1f7e694bc5d0 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 21 Dec 2024 15:06:12 -0500 Subject: [PATCH 03/14] implement better chunking logic, fix clippy --- scylla-server/src/db_handler.rs | 56 ++++++++++++++----- scylla-server/src/lib.rs | 14 ++--- scylla-server/src/models.rs | 4 +- scylla-server/src/schema.rs | 6 +- scylla-server/src/services/data_service.rs | 2 +- .../src/services/data_type_service.rs | 7 ++- scylla-server/src/services/run_service.rs | 9 ++- .../src/transformers/data_transformer.rs | 6 +- scylla-server/tests/data_type_service_test.rs | 3 +- scylla-server/tests/run_service_test.rs | 2 +- scylla-server/tests/test_utils.rs | 9 ++- 11 files changed, 75 insertions(+), 43 deletions(-) diff --git a/scylla-server/src/db_handler.rs b/scylla-server/src/db_handler.rs index d56be3f9..587deee7 100644 --- a/scylla-server/src/db_handler.rs +++ b/scylla-server/src/db_handler.rs @@ -25,6 +25,35 @@ pub struct DbHandler { upload_interval: u64, } +/// Chunks a vec into roughly equal vectors all under size `max_chunk_size` +/// This precomputes vec capacity but does however call to_vec(), reallocating the slices +fn chunk_vec(input: Vec, max_chunk_size: usize) -> Vec> { + if max_chunk_size == 0 { + panic!("Maximum chunk size must be greater than zero"); + } + + let len = input.len(); + if len == 0 { + return Vec::new(); + } + + // Calculate the number of chunks + let num_chunks = (len + max_chunk_size - 1) / max_chunk_size; + + // Recompute a balanced chunk size + let chunk_size = usize::max(1, (len + num_chunks - 1) / num_chunks); + + let mut result = Vec::with_capacity(num_chunks); + let mut start = 0; + + while start < len { + let end = usize::min(start + chunk_size, len); + result.push(input[start..end].to_vec()); + start = end; + } + result +} + impl DbHandler { /// Make a new db handler /// * `recv` - the broadcast reciver of which clientdata will be sent @@ -62,15 +91,17 @@ impl DbHandler { while let Some(final_msgs) = batch_queue.recv().await { info!("{} batches remaining!", batch_queue.len()+1); // do not spawn new tasks in this mode, see below comment for chunk_size math - let chunk_size = final_msgs.len() / ((final_msgs.len() / 8190) + 1); - if chunk_size == 0 { - warn!("Could not insert {} messages, chunk size zero!", final_msgs.len()); + if final_msgs.is_empty() { + debug!("A batch of zero messages was sent!"); continue; } - for chunk in final_msgs.chunks(chunk_size).collect::>() { + let chunk_size = final_msgs.len() / ((final_msgs.len() / 8190) + 1); + let chunks = chunk_vec(final_msgs, chunk_size); + debug!("Batch uploading {} chunks in sequence", chunks.len()); + for chunk in chunks { info!( "A cleanup chunk uploaded: {:?}", - data_service::add_many(&mut database, chunk.to_vec()).await + data_service::add_many(&mut database, chunk).await ); } } @@ -81,16 +112,15 @@ impl DbHandler { // libpq has max 65535 params, therefore batch // max for batch is 65535/4 params per message, hence the below, rounded down with a margin for safety // TODO avoid this code batch uploading the remainder messages as a new batch, combine it with another safely - let chunk_size = msgs.len() / ((msgs.len() / 8190) + 1); - if chunk_size == 0 { - warn!("Could not insert {} messages, chunk size zero!", msgs.len()); + if msgs.is_empty() { + debug!("A batch of zero messages was sent!"); continue; } - debug!("Batch uploading {} chunks in parrallel", msgs.len() / chunk_size); - for chunk in msgs.chunks(chunk_size).collect::>() { - let owned = chunk.to_vec(); - let pool = pool.clone(); - tokio::spawn(DbHandler::batch_upload(owned, pool)); + let chunk_size = msgs.len() / ((msgs.len() / 8190) + 1); + let chunks = chunk_vec(msgs, chunk_size); + debug!("Batch uploading {} chunks in parrallel", chunks.len()); + for chunk in chunks { + tokio::spawn(DbHandler::batch_upload(chunk, pool.clone())); } debug!( "DB send: {} of {}", diff --git a/scylla-server/src/lib.rs b/scylla-server/src/lib.rs index 567fce1e..47cff620 100644 --- a/scylla-server/src/lib.rs +++ b/scylla-server/src/lib.rs @@ -59,15 +59,15 @@ pub struct ClientData { } /// this is the main conversion code to insert data. -/// +/// /// it is essential this conversion does no re-allocate -impl Into for ClientData { - fn into(self) -> models::DataInsert { +impl From for models::DataInsert { + fn from(val: ClientData) -> Self { models::DataInsert { - values: self.values, - dataTypeName: self.name, - time: self.timestamp, - runId: self.run_id, + values: val.values, + dataTypeName: val.name, + time: val.timestamp, + runId: val.run_id, } } } diff --git a/scylla-server/src/models.rs b/scylla-server/src/models.rs index 175f817a..0babfc30 100644 --- a/scylla-server/src/models.rs +++ b/scylla-server/src/models.rs @@ -16,9 +16,9 @@ pub struct Data { } /// Use this struct when inserting data. -/// +/// /// This struct is required because Diesel must infer Arrays as Array>, -/// and to query such types our signature for values must be Vec>, +/// and to query such types our signature for values must be Vec>, /// but the overhead of mapping Vec to Vec> is non-negligible. #[derive(Insertable)] #[diesel(table_name = crate::schema::data)] diff --git a/scylla-server/src/schema.rs b/scylla-server/src/schema.rs index b8115048..fb98e761 100644 --- a/scylla-server/src/schema.rs +++ b/scylla-server/src/schema.rs @@ -32,8 +32,4 @@ diesel::table! { diesel::joinable!(data -> dataType (dataTypeName)); diesel::joinable!(data -> run (runId)); -diesel::allow_tables_to_appear_in_same_query!( - data, - dataType, - run, -); +diesel::allow_tables_to_appear_in_same_query!(data, dataType, run,); diff --git a/scylla-server/src/services/data_service.rs b/scylla-server/src/services/data_service.rs index f1aea2c6..35428e17 100644 --- a/scylla-server/src/services/data_service.rs +++ b/scylla-server/src/services/data_service.rs @@ -46,7 +46,7 @@ pub async fn add_many( .values( client_data .into_iter() - .map(|single_client_data| Into::::into(single_client_data)) + .map(Into::::into) .collect::>(), ) .on_conflict_do_nothing() diff --git a/scylla-server/src/services/data_type_service.rs b/scylla-server/src/services/data_type_service.rs index 684c00a8..b9acd499 100644 --- a/scylla-server/src/services/data_type_service.rs +++ b/scylla-server/src/services/data_type_service.rs @@ -5,7 +5,9 @@ use diesel_async::RunQueryDsl; /// Gets all datatypes /// * `d ` - The connection to the database /// returns: A result containing the data or the QueryError propogated by the db -pub async fn get_all_data_types(db: &mut Database<'_>) -> Result, diesel::result::Error> { +pub async fn get_all_data_types( + db: &mut Database<'_>, +) -> Result, diesel::result::Error> { dataType.load(db).await } @@ -32,5 +34,6 @@ pub async fn upsert_data_type( .do_update() // actually allows for the upsert ability .set(&val) .returning(DataType::as_returning()) - .get_result(db).await + .get_result(db) + .await } diff --git a/scylla-server/src/services/run_service.rs b/scylla-server/src/services/run_service.rs index fa4ebf7d..4c9bdf9d 100644 --- a/scylla-server/src/services/run_service.rs +++ b/scylla-server/src/services/run_service.rs @@ -31,7 +31,8 @@ pub async fn create_run( ) -> Result { diesel::insert_into(run) .values(time.eq(timestamp)) - .get_result(db).await + .get_result(db) + .await } /// Creates a run with a given id @@ -46,7 +47,8 @@ pub async fn create_run_with_id( ) -> Result { diesel::insert_into(run) .values((time.eq(timestamp), id.eq(run_id))) - .get_result(db).await + .get_result(db) + .await } /// Updates a run with GPS points @@ -62,5 +64,6 @@ pub async fn update_run_with_coords( ) -> Result { diesel::update(run.filter(id.eq(run_id))) .set((latitude.eq(lat), longitude.eq(long))) - .get_result(db).await + .get_result(db) + .await } diff --git a/scylla-server/src/transformers/data_transformer.rs b/scylla-server/src/transformers/data_transformer.rs index a7b279ad..36bcf330 100644 --- a/scylla-server/src/transformers/data_transformer.rs +++ b/scylla-server/src/transformers/data_transformer.rs @@ -37,11 +37,7 @@ impl Eq for PublicData {} impl From for PublicData { fn from(value: crate::models::Data) -> Self { PublicData { - values: value - .values - .into_iter() - .flatten() - .collect(), + values: value.values.into_iter().flatten().collect(), time_ms: value.time.timestamp_millis(), } } diff --git a/scylla-server/tests/data_type_service_test.rs b/scylla-server/tests/data_type_service_test.rs index 5a148999..b805b176 100644 --- a/scylla-server/tests/data_type_service_test.rs +++ b/scylla-server/tests/data_type_service_test.rs @@ -44,7 +44,8 @@ async fn test_datatype_create() -> Result<(), diesel::result::Error> { let data = dataType::table .filter(dataType::name.eq(data_type_name.clone())) .select(DataType::as_select()) - .get_result(&mut db).await?; + .get_result(&mut db) + .await?; assert_eq!( PublicDataType::from(data), diff --git a/scylla-server/tests/run_service_test.rs b/scylla-server/tests/run_service_test.rs index 2a5db7de..21a95d11 100644 --- a/scylla-server/tests/run_service_test.rs +++ b/scylla-server/tests/run_service_test.rs @@ -19,7 +19,7 @@ async fn test_get_all_runs() -> Result<(), diesel::result::Error> { async fn test_get_run_by_id() -> Result<(), diesel::result::Error> { let pool = cleanup_and_prepare().await.unwrap(); let mut db = pool.get().await.unwrap(); - + // add a run let run_c = run_service::create_run(&mut db, chrono::DateTime::from_timestamp_millis(1).unwrap()) diff --git a/scylla-server/tests/test_utils.rs b/scylla-server/tests/test_utils.rs index bfb0966b..e1ac0430 100644 --- a/scylla-server/tests/test_utils.rs +++ b/scylla-server/tests/test_utils.rs @@ -1,9 +1,11 @@ use std::time::Duration; +use diesel_async::{ + pooled_connection::{bb8::Pool, AsyncDieselConnectionManager}, + AsyncPgConnection, RunQueryDsl, +}; use dotenvy::dotenv; use scylla_server::schema::{data, dataType, run}; -use diesel_async::{pooled_connection::{bb8::Pool, AsyncDieselConnectionManager}, AsyncPgConnection, RunQueryDsl}; - pub async fn cleanup_and_prepare() -> Result, diesel::result::Error> { dotenv().ok(); @@ -17,7 +19,8 @@ pub async fn cleanup_and_prepare() -> Result, diesel::re .max_lifetime(Some(Duration::from_secs(60 * 60 * 24))) .idle_timeout(Some(Duration::from_secs(60 * 2))) .build(manager) - .await.unwrap(); + .await + .unwrap(); let mut client = pool.get().await.unwrap(); diesel::delete(data::table).execute(&mut client).await?; From fc61e45ceca3456e01b7bd9c437e901b40adcc01 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 21 Dec 2024 15:45:11 -0500 Subject: [PATCH 04/14] switch to jemalloc --- scylla-server/Cargo.lock | 21 +++++++++++++++++++++ scylla-server/Cargo.toml | 3 +++ scylla-server/src/main.rs | 7 +++++++ 3 files changed, 31 insertions(+) diff --git a/scylla-server/Cargo.lock b/scylla-server/Cargo.lock index aaad1555..091bb61f 100755 --- a/scylla-server/Cargo.lock +++ b/scylla-server/Cargo.lock @@ -1897,6 +1897,7 @@ dependencies = [ "serde", "serde_json", "socketioxide", + "tikv-jemallocator", "tokio", "tokio-util", "tower", @@ -2207,6 +2208,26 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.0+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3c60906412afa9c2b5b5a48ca6a5abe5736aec9eb48ad05037a677e52e4e2d" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cec5ff18518d81584f477e9bfdf957f5bb0979b0bac3af4ca30b5b3ae2d2865" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "tinyvec" version = "1.8.1" diff --git a/scylla-server/Cargo.toml b/scylla-server/Cargo.toml index 21a3ef86..7ca01ad4 100644 --- a/scylla-server/Cargo.toml +++ b/scylla-server/Cargo.toml @@ -31,6 +31,8 @@ diesel_migrations = { version = "2.2.0", features = ["postgres"] } rangemap = "1.5.1" axum-macros = "0.4.2" diesel-async = { version = "0.5.2", features = ["postgres", "bb8", "async-connection-wrapper", "sync-connection-wrapper"] } +[target.'cfg(not(target_env = "msvc"))'.dependencies] +tikv-jemallocator = "0.6" [features] top = ["dep:console-subscriber"] @@ -47,3 +49,4 @@ strip = true # Automatically strip symbols from the binary. [[bin]] name = "scylla-server" path = "src/main.rs" + diff --git a/scylla-server/src/main.rs b/scylla-server/src/main.rs index 8ebd142c..c32f6593 100755 --- a/scylla-server/src/main.rs +++ b/scylla-server/src/main.rs @@ -43,6 +43,13 @@ use tower_http::{ use tracing::{debug, info, level_filters::LevelFilter}; use tracing_subscriber::{fmt::format::FmtSpan, EnvFilter}; +#[cfg(not(target_env = "msvc"))] +use tikv_jemallocator::Jemalloc; + +#[cfg(not(target_env = "msvc"))] +#[global_allocator] +static GLOBAL: Jemalloc = Jemalloc; + /// Scylla command line arguments #[derive(Parser, Debug)] #[command(version)] From bcbce397b5a7a4349e7e230a922fe83efd615c2d Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 21 Dec 2024 21:41:16 -0500 Subject: [PATCH 05/14] fail gracefully on file insert, switch to hashset hashset about 0.3% CPU time improvement --- scylla-server/Cargo.toml | 5 +++ .../controllers/file_insertion_controller.rs | 7 ++-- scylla-server/src/db_handler.rs | 34 ++++++++----------- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/scylla-server/Cargo.toml b/scylla-server/Cargo.toml index 7ca01ad4..a8b03163 100644 --- a/scylla-server/Cargo.toml +++ b/scylla-server/Cargo.toml @@ -46,6 +46,11 @@ codegen-units = 1 panic = "abort" strip = true # Automatically strip symbols from the binary. +[profile.profiling] +inherits = "release" +debug = true +strip = false + [[bin]] name = "scylla-server" path = "src/main.rs" diff --git a/scylla-server/src/controllers/file_insertion_controller.rs b/scylla-server/src/controllers/file_insertion_controller.rs index f139523e..3896ebcb 100644 --- a/scylla-server/src/controllers/file_insertion_controller.rs +++ b/scylla-server/src/controllers/file_insertion_controller.rs @@ -45,9 +45,12 @@ pub async fn insert_file( // iterate through all files debug!("Converting file data to insertable data!"); - while let Some(field) = multipart.next_field().await.unwrap() { + while let Ok(Some(field)) = multipart.next_field().await { // round up all of the protobuf segments as a giant list - let data = field.bytes().await.unwrap(); + let Ok(data) = field.bytes().await else { + warn!("Could not decode file insert, perhaps it was interrupted!"); + continue; + }; let mut count_bad_run = 0usize; let mut insertable_data: Vec = Vec::new(); { diff --git a/scylla-server/src/db_handler.rs b/scylla-server/src/db_handler.rs index 587deee7..5d5f8cd0 100644 --- a/scylla-server/src/db_handler.rs +++ b/scylla-server/src/db_handler.rs @@ -1,3 +1,5 @@ +use std::collections::HashSet; + use tokio::sync::mpsc::Receiver; use tokio::{sync::mpsc::Sender, time::Duration}; @@ -12,15 +14,13 @@ use crate::{ClientData, PoolHandle, RUN_ID}; /// upserting of metadata for data, and batch uploading the database pub struct DbHandler { /// The list of data types seen by this instance, used for when to upsert - datatype_list: Vec, + datatype_list: HashSet, /// The broadcast channel which provides serial datapoints for processing receiver: Receiver, /// The database pool handle pool: PoolHandle, /// the queue of data data_queue: Vec, - /// the time since last batch - last_time: tokio::time::Instant, /// upload interval upload_interval: u64, } @@ -63,11 +63,10 @@ impl DbHandler { upload_interval: u64, ) -> DbHandler { DbHandler { - datatype_list: vec![], + datatype_list: HashSet::new(), receiver, pool, data_queue: vec![], - last_time: tokio::time::Instant::now(), upload_interval, } } @@ -172,6 +171,7 @@ impl DbHandler { data_channel: Sender>, cancel_token: CancellationToken, ) { + let mut batch_interval = tokio::time::interval(Duration::from_millis(self.upload_interval)); loop { tokio::select! { _ = cancel_token.cancelled() => { @@ -183,6 +183,15 @@ impl DbHandler { Some(msg) = self.receiver.recv() => { self.handle_msg(msg, &data_channel).await; } + _ = batch_interval.tick() => { + if !self.data_queue.is_empty() { + // mem::take allows us to assign the value of data queue vec::new() while maintaining the memory for data_channel ownership + data_channel + .send(std::mem::take(&mut self.data_queue)) + .await + .expect("Could not comm data to db thread"); + } + } } } } @@ -195,19 +204,6 @@ impl DbHandler { self.receiver.max_capacity() ); - // If the time is greater than upload interval, push to batch upload thread and clear queue - if tokio::time::Instant::now().duration_since(self.last_time) - > Duration::from_millis(self.upload_interval) - && !self.data_queue.is_empty() - { - // mem::take allows us to assign the value of data queue vec::new() while maintaining the memory for data_channel ownership - data_channel - .send(std::mem::take(&mut self.data_queue)) - .await - .expect("Could not comm data to db thread"); - self.last_time = tokio::time::Instant::now(); - } - if !self.datatype_list.contains(&msg.name) { let Ok(mut database) = self.pool.get().await else { warn!("Could not get connection for dataType upsert"); @@ -224,7 +220,7 @@ impl DbHandler { { warn!("DB error datatype upsert: {:?}", msg); } - self.datatype_list.push(msg.name.clone()); + self.datatype_list.insert(msg.name.clone()); } // Check for GPS points, insert them into current run if available From 994244ba40a7ecef6e681b1e612b16a9dbbe1f8b Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 21 Dec 2024 21:58:12 -0500 Subject: [PATCH 06/14] vec with capacity: 0.3% speed increase --- scylla-server/src/db_handler.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scylla-server/src/db_handler.rs b/scylla-server/src/db_handler.rs index 5d5f8cd0..1cdfe4dc 100644 --- a/scylla-server/src/db_handler.rs +++ b/scylla-server/src/db_handler.rs @@ -172,12 +172,13 @@ impl DbHandler { cancel_token: CancellationToken, ) { let mut batch_interval = tokio::time::interval(Duration::from_millis(self.upload_interval)); + // the match batch size to reasonably expect + let mut max_batch_size = 2usize; loop { tokio::select! { _ = cancel_token.cancelled() => { debug!("Pushing final messages to queue"); data_channel.send(self.data_queue).await.expect("Could not comm data to db thread, shutdown"); - self.data_queue = vec![]; break; }, Some(msg) = self.receiver.recv() => { @@ -185,11 +186,14 @@ impl DbHandler { } _ = batch_interval.tick() => { if !self.data_queue.is_empty() { + // set a new max if this batch is larger + max_batch_size = usize::max(max_batch_size, self.data_queue.len()); // mem::take allows us to assign the value of data queue vec::new() while maintaining the memory for data_channel ownership data_channel - .send(std::mem::take(&mut self.data_queue)) + .send(self.data_queue) .await .expect("Could not comm data to db thread"); + self.data_queue = Vec::with_capacity((max_batch_size as f32 * 1.05) as usize); } } } From 4b4539860cc6f03ef934de8f5a747fb389b71216 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 21 Dec 2024 22:17:19 -0500 Subject: [PATCH 07/14] precompute duration --- scylla-server/src/db_handler.rs | 5 +++-- scylla-server/src/mqtt_processor.rs | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scylla-server/src/db_handler.rs b/scylla-server/src/db_handler.rs index 1cdfe4dc..f855ffe0 100644 --- a/scylla-server/src/db_handler.rs +++ b/scylla-server/src/db_handler.rs @@ -115,9 +115,10 @@ impl DbHandler { debug!("A batch of zero messages was sent!"); continue; } - let chunk_size = msgs.len() / ((msgs.len() / 8190) + 1); + let msg_len = msgs.len(); + let chunk_size = msg_len / ((msg_len / 8190) + 1); let chunks = chunk_vec(msgs, chunk_size); - debug!("Batch uploading {} chunks in parrallel", chunks.len()); + info!("Batch uploading {} chunks in parrallel, {} messages.", chunks.len(), msg_len); for chunk in chunks { tokio::spawn(DbHandler::batch_upload(chunk, pool.clone())); } diff --git a/scylla-server/src/mqtt_processor.rs b/scylla-server/src/mqtt_processor.rs index cfaf040d..6bf72504 100644 --- a/scylla-server/src/mqtt_processor.rs +++ b/scylla-server/src/mqtt_processor.rs @@ -38,8 +38,8 @@ pub struct MqttProcessor { upload_ratio: u8, /// static rate limiter rate_limiter: HashMap, - /// time to rate limit in ms - rate_limit_time: u64, + /// time to rate limit + rate_limit_time: Duration, /// rate limit mode rate_limit_mode: RateLimitMode, } @@ -104,7 +104,7 @@ impl MqttProcessor { cancel_token, upload_ratio: opts.upload_ratio, rate_limiter: rate_map, - rate_limit_time: opts.static_rate_limit_time, + rate_limit_time: Duration::from_millis(opts.static_rate_limit_time), rate_limit_mode: opts.rate_limit_mode, }, mqtt_opts, @@ -211,7 +211,7 @@ impl MqttProcessor { // check if we have a previous time for a message based on its topic if let Some(old) = self.rate_limiter.get(topic) { // if the message is less than the rate limit, skip it and do not update the map - if old.elapsed() < Duration::from_millis(self.rate_limit_time) { + if old.elapsed() < self.rate_limit_time { trace!("Static rate limit skipping message with topic {}", topic); return None; } else { From 3f08e87fe7bb01fa41c316b0d539b52b9aea4a31 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 21 Dec 2024 22:58:00 -0500 Subject: [PATCH 08/14] bump deps, fxhashmap fxhasmap -> 0.25% perf --- scylla-server/Cargo.lock | 1245 ++++++++++++++------------- scylla-server/Cargo.toml | 9 +- scylla-server/src/db_handler.rs | 7 +- scylla-server/src/mqtt_processor.rs | 9 +- 4 files changed, 651 insertions(+), 619 deletions(-) diff --git a/scylla-server/Cargo.lock b/scylla-server/Cargo.lock index 091bb61f..36747eae 100755 --- a/scylla-server/Cargo.lock +++ b/scylla-server/Cargo.lock @@ -4,30 +4,18 @@ version = 3 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - -[[package]] -name = "ahash" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "aho-corasick" @@ -38,12 +26,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "allocator-api2" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" - [[package]] name = "android-tzdata" version = "0.1.1" @@ -61,9 +43,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.15" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -76,49 +58,49 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" [[package]] name = "async-stream" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ "async-stream-impl", "futures-core", @@ -127,74 +109,78 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn", ] [[package]] name = "async-trait" -version = "0.1.80" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] -name = "axum" -version = "0.6.20" +name = "aws-lc-rs" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +checksum = "f409eb70b561706bf8abba8ca9c112729c481595893fd06a2dd9af8ed8441148" dependencies = [ - "async-trait", - "axum-core 0.3.4", - "bitflags 1.3.2", - "bytes", - "futures-util", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.29", - "itoa", - "matchit 0.7.3", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper 0.1.2", - "tower", - "tower-layer", - "tower-service", + "aws-lc-sys", + "paste", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8478a5c29ead3f3be14aff8a202ad965cf7da6856860041bfca271becf8ba48b" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", + "libc", + "paste", ] [[package]] name = "axum" -version = "0.7.5" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", - "axum-core 0.4.3", + "axum-core", "bytes", "futures-util", - "http 1.1.0", - "http-body 1.0.0", + "http", + "http-body", "http-body-util", - "hyper 1.4.0", + "hyper", "hyper-util", "itoa", "matchit 0.7.3", @@ -208,9 +194,9 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper", "tokio", - "tower", + "tower 0.5.2", "tower-layer", "tower-service", "tracing", @@ -218,37 +204,20 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http 0.2.12", - "http-body 0.4.6", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.1.0", - "http-body 1.0.0", + "http", + "http-body", "http-body-util", "mime", "pin-project-lite", "rustversion", - "sync_wrapper 0.1.2", + "sync_wrapper", "tower-layer", "tower-service", "tracing", @@ -256,25 +225,26 @@ dependencies = [ [[package]] name = "axum-extra" -version = "0.9.3" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0be6ea09c9b96cb5076af0de2e383bd2bc0c18f827cf1967bdd353e0b910d733" +checksum = "c794b30c904f0a1c2fb7740f7df7f7972dfaa14ef6f57cb6178dc63e5dca2f04" dependencies = [ - "axum 0.7.5", - "axum-core 0.4.3", + "axum", + "axum-core", "bytes", + "fastrand", "futures-util", - "http 1.1.0", - "http-body 1.0.0", + "http", + "http-body", "http-body-util", "mime", + "multer", "pin-project-lite", "serde", "serde_html_form", - "tower", + "tower 0.5.2", "tower-layer", "tower-service", - "tracing", ] [[package]] @@ -285,22 +255,22 @@ checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn", ] [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets", ] [[package]] @@ -328,16 +298,33 @@ dependencies = [ ] [[package]] -name = "bitflags" -version = "1.3.2" +name = "bindgen" +version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn", + "which", +] [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "block-buffer" @@ -362,18 +349,32 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" dependencies = [ "serde", ] [[package]] name = "cc" -version = "1.0.99" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" +checksum = "c31a0499c1dc64f458ad13872de75c0eb7e3fdb0e67964610c914b034fc5956e" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] [[package]] name = "cfg-if" @@ -383,9 +384,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -393,14 +394,25 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.5", + "windows-targets", +] + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", ] [[package]] name = "clap" -version = "4.5.11" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3" +checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" dependencies = [ "clap_builder", "clap_derive", @@ -408,9 +420,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.11" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa" +checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" dependencies = [ "anstream", "anstyle", @@ -420,33 +432,42 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.11" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.67", + "syn", ] [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + +[[package]] +name = "cmake" +version = "0.1.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "c682c223677e0e5b6b7f63a64b9351844c3f1b1678a68b7ee617e30fb082620e" +dependencies = [ + "cc", +] [[package]] name = "colorchoice" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "console-api" -version = "0.7.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a257c22cd7e487dd4a13d413beabc512c5052f0bc048db0da6a84c3d8a6142fd" +checksum = "8030735ecb0d128428b64cd379809817e620a40e5001c54465b99ec5feec2857" dependencies = [ "futures-core", "prost", @@ -457,9 +478,9 @@ dependencies = [ [[package]] name = "console-subscriber" -version = "0.3.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c4cc54bae66f7d9188996404abdf7fdfa23034ef8e43478c8810828abad758" +checksum = "6539aa9c6a4cd31f4b1c040f860a1eac9aa80e7df6b05d506a6e7179936d6a01" dependencies = [ "console-api", "crossbeam-channel", @@ -467,6 +488,7 @@ dependencies = [ "futures-task", "hdrhistogram", "humantime", + "hyper-util", "prost", "prost-types", "serde", @@ -482,9 +504,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.4" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" dependencies = [ "core-foundation-sys", "libc", @@ -492,15 +514,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" dependencies = [ "libc", ] @@ -516,18 +538,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crypto-common" @@ -560,7 +582,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.67", + "syn", ] [[package]] @@ -571,7 +593,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.67", + "syn", ] [[package]] @@ -582,11 +604,11 @@ checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "diesel" -version = "2.2.4" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "158fe8e2e68695bd615d7e4f3227c0727b151330d3e253b525086c348d055d5e" +checksum = "ccf1bedf64cdb9643204a36dd15b19a6ce8e7aa7f7b105868e9f1fad5ffa7d12" dependencies = [ - "bitflags 2.5.0", + "bitflags", "byteorder", "chrono", "diesel_derives", @@ -619,7 +641,7 @@ dependencies = [ "dsl_auto_type", "proc-macro2", "quote", - "syn 2.0.67", + "syn", ] [[package]] @@ -639,7 +661,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25" dependencies = [ - "syn 2.0.67", + "syn", ] [[package]] @@ -670,14 +692,20 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.67", + "syn", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "either" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "encoding_rs" @@ -690,28 +718,29 @@ dependencies = [ [[package]] name = "engineioxide" -version = "0.14.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b9cfc311d0ac3237b8177d2ee8962aa5bb4cfa22faf284356f2ebaf9d698f0" +checksum = "ec84aea019c24ae0cae29a6306b1b9b6bd91a4c950542d804b742435ed797f0c" dependencies = [ "base64 0.22.1", "bytes", "futures-core", "futures-util", - "http 1.1.0", - "http-body 1.0.0", + "http", + "http-body", "http-body-util", - "hyper 1.4.0", + "hyper", "hyper-util", "pin-project-lite", "rand", "serde", "serde_json", "smallvec", - "thiserror", + "thiserror 1.0.69", "tokio", "tokio-tungstenite", - "tower", + "tower-layer", + "tower-service", "tracing", ] @@ -723,12 +752,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -739,9 +768,9 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fastrand" -version = "2.1.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fixedbitset" @@ -751,9 +780,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flate2" -version = "1.0.30" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" dependencies = [ "crc32fast", "miniz_oxide", @@ -761,9 +790,9 @@ dependencies = [ [[package]] name = "flume" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", @@ -785,11 +814,17 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -797,44 +832,44 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn", ] [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-core", "futures-macro", @@ -868,23 +903,29 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.26" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" dependencies = [ + "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap 2.2.6", + "http", + "indexmap 2.7.0", "slab", "tokio", "tokio-util", @@ -902,10 +943,12 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", - "allocator-api2", -] + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "hdrhistogram" @@ -926,12 +969,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" - [[package]] name = "hmac" version = "0.12.1" @@ -943,29 +980,18 @@ dependencies = [ [[package]] name = "home" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] -name = "http" -version = "0.2.12" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "bytes", - "fnv", - "itoa", + "windows-sys 0.59.0", ] [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -974,23 +1000,12 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - -[[package]] -name = "http-body" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.1.0", + "http", ] [[package]] @@ -1001,16 +1016,16 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http 1.1.0", - "http-body 1.0.0", + "http", + "http-body", "pin-project-lite", ] [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" @@ -1026,79 +1041,62 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.29" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" dependencies = [ "bytes", "futures-channel", - "futures-core", "futures-util", "h2", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4fe55fb7a772d59a5ff1dfbff4fe0258d19b89fec4b233e75d35d5d2316badc" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "http 1.1.0", - "http-body 1.0.0", + "http", + "http-body", "httparse", "httpdate", "itoa", "pin-project-lite", "smallvec", "tokio", + "want", ] [[package]] name = "hyper-timeout" -version = "0.4.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 0.14.29", + "hyper", + "hyper-util", "pin-project-lite", "tokio", - "tokio-io-timeout", + "tower-service", ] [[package]] name = "hyper-util" -version = "0.1.6" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", + "futures-channel", "futures-util", - "http 1.1.0", - "http-body 1.0.0", - "hyper 1.4.0", + "http", + "http-body", + "hyper", "pin-project-lite", + "socket2", "tokio", + "tower-service", + "tracing", ] [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1135,12 +1133,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.2", ] [[package]] @@ -1151,25 +1149,35 @@ checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" -version = "0.10.5" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] [[package]] name = "js-sys" -version = "0.3.56" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -1179,11 +1187,27 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" -version = "0.2.155" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "libloading" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets", +] [[package]] name = "linux-raw-sys" @@ -1203,9 +1227,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "matchers" @@ -1224,9 +1248,9 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "matchit" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3c2fcf089c060eb333302d80c5f3ffa8297abecf220f788e4a09ef85f59420" +checksum = "bd0aa4b8ca861b08d68afc8702af3250776898c1508b278e1da9d01e01d4b45c" [[package]] name = "md-5" @@ -1279,22 +1303,22 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.4" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" dependencies = [ - "adler", + "adler2", ] [[package]] name = "mio" -version = "0.8.11" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", "wasi", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1306,7 +1330,7 @@ dependencies = [ "bytes", "encoding_rs", "futures-util", - "http 1.1.0", + "http", "httparse", "memchr", "mime", @@ -1343,30 +1367,20 @@ dependencies = [ "autocfg", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "object" -version = "0.36.0" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "openssl-probe" @@ -1412,9 +1426,15 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets 0.52.5", + "windows-targets", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "percent-encoding" version = "2.3.1" @@ -1441,29 +1461,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn", ] [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -1508,9 +1528,12 @@ dependencies = [ [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "pq-src" @@ -1532,20 +1555,30 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "prettyplease" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.12.6" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +checksum = "2c0fef6c4230e4ccf618a35c59d7ede15dea37de8427500f50aff708806e42ec" dependencies = [ "bytes", "prost-derive", @@ -1553,22 +1586,22 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.6" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3" dependencies = [ "anyhow", "itertools", "proc-macro2", "quote", - "syn 2.0.67", + "syn", ] [[package]] name = "prost-types" -version = "0.12.6" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +checksum = "cc2f1e56baa61e93533aebc21af4d2134b70f66275e0fcdf3cbe43d77ff7e8fc" dependencies = [ "prost", ] @@ -1582,7 +1615,7 @@ dependencies = [ "bytes", "once_cell", "protobuf-support", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -1597,7 +1630,7 @@ dependencies = [ "protobuf-parse", "regex", "tempfile", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -1607,12 +1640,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "322330e133eab455718444b4e033ebfac7c6528972c784fcde28d2cc783c6257" dependencies = [ "anyhow", - "indexmap 2.2.6", + "indexmap 2.7.0", "log", "protobuf", "protobuf-support", "tempfile", - "thiserror", + "thiserror 1.0.69", "which", ] @@ -1622,14 +1655,14 @@ version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b088fd20b938a875ea00843b6faf48579462630015c3788d397ad6a786663252" dependencies = [ - "thiserror", + "thiserror 1.0.69", ] [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -1672,23 +1705,23 @@ checksum = "f60fcc7d6849342eff22c4350c8b9a989ee8ceabc4b481253e8946b9fe83d684" [[package]] name = "redox_syscall" -version = "0.5.2" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.5.0", + "bitflags", ] [[package]] name = "regex" -version = "1.10.5" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", ] [[package]] @@ -1702,13 +1735,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -1719,9 +1752,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "ring" @@ -1747,7 +1780,7 @@ checksum = "3df6368f71f205ff9c33c076d170dd56ebf68e8161c733c0caa07a7a5509ed53" [[package]] name = "rumqttc" version = "0.24.0" -source = "git+https://github.com/bytebeamio/rumqtt?branch=main#db1f261dd5cd6c69bbfd1058ba69ea8ef5f4fc38" +source = "git+https://github.com/bytebeamio/rumqtt?branch=main#4262e6a2d9c150c4c51c3d5efdaaaf085adc1633" dependencies = [ "bytes", "fixedbitset", @@ -1757,7 +1790,7 @@ dependencies = [ "rustls-native-certs", "rustls-pemfile", "rustls-webpki", - "thiserror", + "thiserror 2.0.9", "tokio", "tokio-rustls", "tokio-stream", @@ -1770,27 +1803,40 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hash" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" + [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ - "bitflags 2.5.0", + "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.22.4" +version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ + "aws-lc-rs", "log", - "ring", + "once_cell", "rustls-pki-types", "rustls-webpki", "subtle", @@ -1799,12 +1845,11 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.1" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" dependencies = [ "openssl-probe", - "rustls-pemfile", "rustls-pki-types", "schannel", "security-framework", @@ -1812,26 +1857,26 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" [[package]] name = "rustls-webpki" -version = "0.102.5" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -1839,9 +1884,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "ryu" @@ -1851,11 +1896,11 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1877,7 +1922,7 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" name = "scylla-server" version = "0.0.1" dependencies = [ - "axum 0.7.5", + "axum", "axum-extra", "axum-macros", "chrono", @@ -1894,13 +1939,14 @@ dependencies = [ "rangemap", "ringbuffer", "rumqttc", + "rustc-hash 2.1.0", "serde", "serde_json", "socketioxide", "tikv-jemallocator", "tokio", "tokio-util", - "tower", + "tower 0.5.2", "tower-http", "tracing", "tracing-subscriber", @@ -1908,11 +1954,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.11.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +checksum = "81d3f8c9bfcc3cbb6b0179eb57042d75b1582bdc65c3cb95f3fa999509c03cbc" dependencies = [ - "bitflags 2.5.0", + "bitflags", "core-foundation", "core-foundation-sys", "libc", @@ -1921,9 +1967,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.0" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +checksum = "1863fd3768cd83c56a7f60faa4dc0d403f1b6df0a38c3c25f44b7894e45370d5" dependencies = [ "core-foundation-sys", "libc", @@ -1931,32 +1977,32 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.203" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn", ] [[package]] name = "serde_html_form" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de514ef58196f1fc96dcaef80fe6170a1ce6215df9687a93fe8300e773fefc5" +checksum = "9d2de91cf02bbc07cde38891769ccd5d4f073d22a40683aa4bc7a95781aaa2c4" dependencies = [ "form_urlencoded", - "indexmap 2.2.6", + "indexmap 2.7.0", "itoa", "ryu", "serde", @@ -1964,9 +2010,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.134" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" dependencies = [ "itoa", "memchr", @@ -2036,6 +2082,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -2068,9 +2120,9 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", "windows-sys 0.52.0", @@ -2078,28 +2130,55 @@ dependencies = [ [[package]] name = "socketioxide" -version = "0.14.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f50a295325631d230022f1562fde3d1351edf4d8eac73265f657cc762f655c" +checksum = "4a9ce59f845cfea2fe24cc7adbd512268893ee07bde899ae6477569dbc42dff3" dependencies = [ "bytes", "engineioxide", "futures-core", "futures-util", - "http 1.1.0", - "http-body 1.0.0", - "hyper 1.4.0", - "itoa", - "matchit 0.8.3", + "http", + "http-body", + "hyper", + "matchit 0.8.5", "pin-project-lite", + "rustversion", "serde", - "serde_json", - "thiserror", + "socketioxide-core", + "socketioxide-parser-common", + "thiserror 1.0.69", "tokio", - "tower", + "tower-layer", + "tower-service", "tracing", ] +[[package]] +name = "socketioxide-core" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82826d9a1efb2b201e7d12d231f98870f768d98d02452eb642bca0587a8d51f0" +dependencies = [ + "bytes", + "engineioxide", + "serde", + "thiserror 1.0.69", +] + +[[package]] +name = "socketioxide-parser-common" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9a20b64f78a6d093d73d4d166b5bfc8b5096069865c59cee67b3aafac77eaac" +dependencies = [ + "bytes", + "itoa", + "serde", + "serde_json", + "socketioxide-core", +] + [[package]] name = "spin" version = "0.9.8" @@ -2128,26 +2207,15 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "subtle" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d0208408ba0c3df17ed26eb06992cb1a1268d41b2c0e12e65203fbe3972cee5" - -[[package]] -name = "syn" -version = "1.0.109" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.67" +version = "2.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff8655ed1d86f3af4ee3fd3263786bc14245ad17c4c7e85ba7187fb3ae028c90" +checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" dependencies = [ "proc-macro2", "quote", @@ -2156,46 +2224,61 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - -[[package]] -name = "sync_wrapper" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" [[package]] name = "tempfile" -version = "3.10.1" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", "fastrand", + "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", ] [[package]] name = "thiserror" -version = "1.0.61" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.9", ] [[package]] name = "thiserror-impl" -version = "1.0.61" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -2245,43 +2328,32 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.38.0" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", "tracing", - "windows-sys 0.48.0", -] - -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn", ] [[package]] @@ -2312,20 +2384,19 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.25.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ "rustls", - "rustls-pki-types", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -2334,9 +2405,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.23.1" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6989540ced10490aaf14e6bad2e3d33728a2813310a0c71d1574304c49631cd" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" dependencies = [ "futures-util", "log", @@ -2346,9 +2417,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -2384,11 +2455,11 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.20" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.7.0", "serde", "serde_spanned", "toml_datetime", @@ -2397,26 +2468,29 @@ dependencies = [ [[package]] name = "tonic" -version = "0.11.0" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ "async-stream", "async-trait", - "axum 0.6.20", - "base64 0.21.7", + "axum", + "base64 0.22.1", "bytes", "h2", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.29", + "http", + "http-body", + "http-body-util", + "hyper", "hyper-timeout", + "hyper-util", "percent-encoding", "pin-project", "prost", + "socket2", "tokio", "tokio-stream", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing", @@ -2443,16 +2517,31 @@ dependencies = [ ] [[package]] -name = "tower-http" +name = "tower" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ - "bitflags 2.5.0", + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" +dependencies = [ + "bitflags", "bytes", - "http 1.1.0", - "http-body 1.0.0", - "http-body-util", + "http", + "http-body", "pin-project-lite", "tower-layer", "tower-service", @@ -2461,21 +2550,21 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", "pin-project-lite", @@ -2485,20 +2574,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn", ] [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -2517,9 +2606,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "nu-ansi-term", @@ -2541,19 +2630,19 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e2ce1e47ed2994fd43b04c8f618008d4cabdd5ee34027cf14f9d918edd9c8" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" dependencies = [ "byteorder", "bytes", "data-encoding", - "http 1.1.0", + "http", "httparse", "log", "rand", "sha1", - "thiserror", + "thiserror 1.0.69", "utf-8", ] @@ -2571,9 +2660,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unicode-normalization" @@ -2622,9 +2711,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "want" @@ -2649,34 +2738,34 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.79" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.79" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", - "lazy_static", "log", "proc-macro2", "quote", - "syn 1.0.109", + "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.79" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2684,28 +2773,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.79" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.79" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "web-sys" -version = "0.3.56" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" dependencies = [ "js-sys", "wasm-bindgen", @@ -2762,16 +2851,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.5", -] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", + "windows-targets", ] [[package]] @@ -2780,129 +2860,81 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets", ] [[package]] -name = "windows-targets" -version = "0.48.5" +name = "windows-sys" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", + "windows-targets", ] [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" @@ -2919,6 +2951,7 @@ version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ + "byteorder", "zerocopy-derive", ] @@ -2930,7 +2963,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn", ] [[package]] diff --git a/scylla-server/Cargo.toml b/scylla-server/Cargo.toml index a8b03163..128dc417 100644 --- a/scylla-server/Cargo.toml +++ b/scylla-server/Cargo.toml @@ -13,15 +13,15 @@ protobuf-codegen = "3.7.1" protobuf = { version = "3.7.1", features = ["with-bytes"] } tokio = { version = "1.38.0", features = ["full", "tracing"] } axum = { version = "0.7.5", features = ["multipart"] } -tower = { version = "0.4.13", features = ["timeout"] } -tower-http = { version = "0.5.2", features = ["cors", "trace"] } -socketioxide = { version = "0.14.0", features = ["tracing"] } +tower = { version = "0.5.2", features = ["timeout"] } +tower-http = { version = "0.6.2", features = ["cors", "trace"] } +socketioxide = { version = "0.15.1", features = ["tracing"] } rumqttc = { git = "https://github.com/bytebeamio/rumqtt", branch = "main"} tokio-util = { version= "0.7.11", features = ["full"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["ansi", "env-filter"] } rand = "0.8.5" -console-subscriber = { version = "0.3.0", optional = true } +console-subscriber = { version = "0.4.1", optional = true } ringbuffer = "0.15.0" clap = { version = "4.5.11", features = ["derive", "env"] } axum-extra = { version = "0.9.3", features = ["query"] } @@ -31,6 +31,7 @@ diesel_migrations = { version = "2.2.0", features = ["postgres"] } rangemap = "1.5.1" axum-macros = "0.4.2" diesel-async = { version = "0.5.2", features = ["postgres", "bb8", "async-connection-wrapper", "sync-connection-wrapper"] } +rustc-hash = "2.1.0" [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = "0.6" diff --git a/scylla-server/src/db_handler.rs b/scylla-server/src/db_handler.rs index f855ffe0..69902775 100644 --- a/scylla-server/src/db_handler.rs +++ b/scylla-server/src/db_handler.rs @@ -1,5 +1,4 @@ -use std::collections::HashSet; - +use rustc_hash::FxHashSet; use tokio::sync::mpsc::Receiver; use tokio::{sync::mpsc::Sender, time::Duration}; @@ -14,7 +13,7 @@ use crate::{ClientData, PoolHandle, RUN_ID}; /// upserting of metadata for data, and batch uploading the database pub struct DbHandler { /// The list of data types seen by this instance, used for when to upsert - datatype_list: HashSet, + datatype_list: FxHashSet, /// The broadcast channel which provides serial datapoints for processing receiver: Receiver, /// The database pool handle @@ -63,7 +62,7 @@ impl DbHandler { upload_interval: u64, ) -> DbHandler { DbHandler { - datatype_list: HashSet::new(), + datatype_list: FxHashSet::default(), receiver, pool, data_queue: vec![], diff --git a/scylla-server/src/mqtt_processor.rs b/scylla-server/src/mqtt_processor.rs index 6bf72504..bdc95c05 100644 --- a/scylla-server/src/mqtt_processor.rs +++ b/scylla-server/src/mqtt_processor.rs @@ -1,5 +1,4 @@ use std::{ - collections::HashMap, sync::{atomic::Ordering, Arc}, time::{Duration, SystemTime}, }; @@ -11,6 +10,7 @@ use rumqttc::v5::{ mqttbytes::v5::{Packet, Publish}, AsyncClient, Event, EventLoop, MqttOptions, }; +use rustc_hash::FxHashMap; use socketioxide::SocketIo; use tokio::{sync::mpsc::Sender, time::Instant}; use tokio_util::sync::CancellationToken; @@ -37,7 +37,7 @@ pub struct MqttProcessor { /// Upload ratio, below is not socket sent above is socket sent upload_ratio: u8, /// static rate limiter - rate_limiter: HashMap, + rate_limiter: FxHashMap, /// time to rate limit rate_limit_time: Duration, /// rate limit mode @@ -95,7 +95,6 @@ impl MqttProcessor { .set_session_expiry_interval(Some(u32::MAX)) .set_topic_alias_max(Some(600)); - let rate_map: HashMap = HashMap::new(); ( MqttProcessor { @@ -103,7 +102,7 @@ impl MqttProcessor { io, cancel_token, upload_ratio: opts.upload_ratio, - rate_limiter: rate_map, + rate_limiter: FxHashMap::default(), rate_limit_time: Duration::from_millis(opts.static_rate_limit_time), rate_limit_mode: opts.rate_limit_mode, }, @@ -322,7 +321,7 @@ impl MqttProcessor { if *upload_counter >= self.upload_ratio { match self.io.emit( "message", - serde_json::to_string(&client_data).expect("Could not serialize ClientData"), + &serde_json::to_string(&client_data).expect("Could not serialize ClientData"), ) { Ok(_) => (), Err(err) => match err { From 16d3d40d31ef02539a352469420b3a9647418948 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 21 Dec 2024 23:07:22 -0500 Subject: [PATCH 09/14] fmt --- scylla-server/src/mqtt_processor.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/scylla-server/src/mqtt_processor.rs b/scylla-server/src/mqtt_processor.rs index bdc95c05..d68fdf3e 100644 --- a/scylla-server/src/mqtt_processor.rs +++ b/scylla-server/src/mqtt_processor.rs @@ -95,7 +95,6 @@ impl MqttProcessor { .set_session_expiry_interval(Some(u32::MAX)) .set_topic_alias_max(Some(600)); - ( MqttProcessor { channel, From 7245dfea41fa9f74e54819631319107a8b3306de Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 21 Dec 2024 23:09:27 -0500 Subject: [PATCH 10/14] fix cargo.lock --- scylla-server/Cargo.lock | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/scylla-server/Cargo.lock b/scylla-server/Cargo.lock index 517bb3e9..61879c6a 100755 --- a/scylla-server/Cargo.lock +++ b/scylla-server/Cargo.lock @@ -306,7 +306,7 @@ dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools", + "itertools 0.12.1", "lazy_static", "lazycell", "log", @@ -1156,6 +1156,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.14" @@ -1479,12 +1488,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkg-config" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" - [[package]] name = "postgres-protocol" version = "0.6.7" @@ -1578,7 +1581,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3" dependencies = [ "anyhow", - "itertools", + "itertools 0.13.0", "proc-macro2", "quote", "syn", From 2f0b7c3acff68678b1b90f8719368cc4307defc4 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Thu, 2 Jan 2025 22:52:32 -0500 Subject: [PATCH 11/14] bump axum, fix clippy --- scylla-server/Cargo.lock | 124 ++++++++++++++++++++++---------- scylla-server/Cargo.toml | 6 +- scylla-server/src/db_handler.rs | 4 +- scylla-server/src/main.rs | 6 +- 4 files changed, 93 insertions(+), 47 deletions(-) diff --git a/scylla-server/Cargo.lock b/scylla-server/Cargo.lock index 61879c6a..81f6587d 100755 --- a/scylla-server/Cargo.lock +++ b/scylla-server/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -92,9 +92,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "async-stream" @@ -154,16 +154,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8478a5c29ead3f3be14aff8a202ad965cf7da6856860041bfca271becf8ba48b" +checksum = "923ded50f602b3007e5e63e3f094c479d9c8a9b42d7f4034e4afe456aa48bfd2" dependencies = [ "bindgen", "cc", "cmake", "dunce", "fs_extra", - "libc", "paste", ] @@ -174,16 +173,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.4.5", "bytes", "futures-util", "http", "http-body", "http-body-util", + "itoa", + "matchit 0.7.3", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8" +dependencies = [ + "axum-core 0.5.0", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", "hyper", "hyper-util", "itoa", - "matchit 0.7.3", + "matchit 0.8.4", "memchr", "mime", "multer", @@ -220,28 +246,47 @@ dependencies = [ "sync_wrapper", "tower-layer", "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1362f362fd16024ae199c1970ce98f9661bf5ef94b9808fee734bc3698b733" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", "tracing", ] [[package]] name = "axum-extra" -version = "0.9.6" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c794b30c904f0a1c2fb7740f7df7f7972dfaa14ef6f57cb6178dc63e5dca2f04" +checksum = "460fc6f625a1f7705c6cf62d0d070794e94668988b1c38111baeec177c715f7b" dependencies = [ - "axum", - "axum-core", + "axum 0.8.1", + "axum-core 0.5.0", "bytes", - "fastrand", + "form_urlencoded", "futures-util", "http", "http-body", "http-body-util", "mime", - "multer", "pin-project-lite", "serde", "serde_html_form", + "serde_path_to_error", "tower 0.5.2", "tower-layer", "tower-service", @@ -249,9 +294,9 @@ dependencies = [ [[package]] name = "axum-macros" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" +checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" dependencies = [ "proc-macro2", "quote", @@ -358,9 +403,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.5" +version = "1.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31a0499c1dc64f458ad13872de75c0eb7e3fdb0e67964610c914b034fc5956e" +checksum = "8d6dbb628b8f8555f86d0323c2eb39e3ec81901f4b83e091db8a6a76d316a333" dependencies = [ "jobserver", "libc", @@ -909,9 +954,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "h2" @@ -1257,9 +1302,9 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "matchit" -version = "0.8.5" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd0aa4b8ca861b08d68afc8702af3250776898c1508b278e1da9d01e01d4b45c" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "md-5" @@ -1650,9 +1695,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -1874,9 +1919,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "ryu" @@ -1912,7 +1957,7 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" name = "scylla-server" version = "0.0.1" dependencies = [ - "axum", + "axum 0.8.1", "axum-extra", "axum-macros", "chrono", @@ -1967,18 +2012,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", @@ -2131,7 +2176,7 @@ dependencies = [ "http", "http-body", "hyper", - "matchit 0.8.5", + "matchit 0.8.4", "pin-project-lite", "rustversion", "serde", @@ -2203,9 +2248,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.91" +version = "2.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" +checksum = "987bc0be1cdea8b10216bd06e2ca407d40b9543468fafd3ddfb02f36e77f71f3" dependencies = [ "proc-macro2", "quote", @@ -2220,12 +2265,13 @@ checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" [[package]] name = "tempfile" -version = "3.14.0" +version = "3.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" dependencies = [ "cfg-if", "fastrand", + "getrandom", "once_cell", "rustix", "windows-sys 0.59.0", @@ -2464,7 +2510,7 @@ checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ "async-stream", "async-trait", - "axum", + "axum 0.7.9", "base64 0.22.1", "bytes", "h2", @@ -2928,9 +2974,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.20" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "e6f5bb5257f2407a5425c6e749bfd9692192a73e70a6060516ac04f889087d68" dependencies = [ "memchr", ] diff --git a/scylla-server/Cargo.toml b/scylla-server/Cargo.toml index eecd7da8..8ae5ddad 100644 --- a/scylla-server/Cargo.toml +++ b/scylla-server/Cargo.toml @@ -12,7 +12,7 @@ serde = "1.0.203" protobuf-codegen = "3.7.1" protobuf = { version = "3.7.1", features = ["with-bytes"] } tokio = { version = "1.38.0", features = ["full", "tracing"] } -axum = { version = "0.7.5", features = ["multipart"] } +axum = { version = "0.8.1", features = ["multipart"] } tower = { version = "0.5.2", features = ["timeout"] } tower-http = { version = "0.6.2", features = ["cors", "trace"] } socketioxide = { version = "0.15.1", features = ["tracing"] } @@ -24,12 +24,12 @@ rand = "0.8.5" console-subscriber = { version = "0.4.1", optional = true } ringbuffer = "0.15.0" clap = { version = "4.5.11", features = ["derive", "env"] } -axum-extra = { version = "0.9.3", features = ["query"] } +axum-extra = { version = "0.10.0", features = ["query"] } chrono = { version = "0.4.38", features = ["serde"] } serde_json = "1.0.128" diesel_migrations = { version = "2.2.0", features = ["postgres"] } rangemap = "1.5.1" -axum-macros = "0.4.2" +axum-macros = "0.5.0" diesel-async = { version = "0.5.2", features = ["postgres", "bb8", "async-connection-wrapper", "sync-connection-wrapper"] } rustc-hash = "2.1.0" [target.'cfg(not(target_env = "msvc"))'.dependencies] diff --git a/scylla-server/src/db_handler.rs b/scylla-server/src/db_handler.rs index 69902775..83060e8d 100644 --- a/scylla-server/src/db_handler.rs +++ b/scylla-server/src/db_handler.rs @@ -37,10 +37,10 @@ fn chunk_vec(input: Vec, max_chunk_size: usize) -> Vec> { } // Calculate the number of chunks - let num_chunks = (len + max_chunk_size - 1) / max_chunk_size; + let num_chunks = len.div_ceil(max_chunk_size); // Recompute a balanced chunk size - let chunk_size = usize::max(1, (len + num_chunks - 1) / num_chunks); + let chunk_size = usize::max(1, len.div_ceil(num_chunks)); let mut result = Vec::with_capacity(num_chunks); let mut start = 0; diff --git a/scylla-server/src/main.rs b/scylla-server/src/main.rs index c32f6593..6d8a92d6 100755 --- a/scylla-server/src/main.rs +++ b/scylla-server/src/main.rs @@ -239,17 +239,17 @@ async fn main() -> Result<(), Box> { let app = Router::new() // DATA .route( - "/data/:dataTypeName/:runId", + "/data/{dataTypeName}/{runId}", get(controllers::data_controller::get_data), ) // DATA TYPE .route("/datatypes", get(data_type_controller::get_all_data_types)) .route("/runs", get(run_controller::get_all_runs)) - .route("/runs/:id", get(run_controller::get_run_by_id)) + .route("/runs/{id}", get(run_controller::get_run_by_id)) .route("/runs/new", post(run_controller::new_run)) // CONFIG .route( - "/config/set/:configKey", + "/config/set/{configKey}", post(car_command_controller::send_config_command).layer(Extension(client_sharable)), ) // FILE INSERT From 4c8efb6b94e35deac0dc7d05862242a08abf4e87 Mon Sep 17 00:00:00 2001 From: wyattb Date: Mon, 6 Jan 2025 12:50:03 -0500 Subject: [PATCH 12/14] integration tests fix --- scylla-server/integration_test.sh | 44 +++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/scylla-server/integration_test.sh b/scylla-server/integration_test.sh index c270980c..4e0eb4b2 100755 --- a/scylla-server/integration_test.sh +++ b/scylla-server/integration_test.sh @@ -1,15 +1,37 @@ #!/bin/sh -echo "Starting db" -cd ../compose -docker compose up -d odyssey-timescale -cd ../scylla-server -echo "Migrating DB" -diesel migration run +# Navigate to the compose directory +echo "Navigating to compose directory..." +cd ../compose || { echo "Compose directory not found"; exit 1; } -echo "Running tests" -DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/postgres cargo test -- --test-threads=1 +# Remove any existing odyssey-timescale container +echo "Stopping and removing any existing odyssey-timescale container..." +docker rm -f odyssey-timescale 2>/dev/null || echo "No existing container to remove." -echo "Exiting" -cd ../compose -docker compose down \ No newline at end of file +# Start a new odyssey-timescale container +echo "Starting a new odyssey-timescale container..." +docker compose up -d odyssey-timescale || { echo "Failed to start odyssey-timescale"; exit 1; } + +# Wait for the database to initialize +echo "Waiting for the database to initialize..." +sleep 3 + +# Navigate to the scylla-server directory +cd ../scylla-server || { echo "scylla-server directory not found"; exit 1; } + +# Run database migrations +echo "Running database migrations..." +DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/postgres diesel migration run || { echo "Migration failed"; exit 1; } + +# Run tests +echo "Running tests..." +DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/postgres cargo test -- --test-threads=1 || { echo "Tests failed"; exit 1; } + +# Navigate back to the compose directory +cd ../compose || { echo "Compose directory not found"; exit 1; } + +# Stop and clean up containers +echo "Stopping and cleaning up containers..." +docker compose down || { echo "Failed to clean up containers"; exit 1; } + +echo "Script completed successfully!" From 123732e9f41029a91616fb05d565ecffa1321316 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Mon, 6 Jan 2025 21:55:44 -0500 Subject: [PATCH 13/14] spruce up comments, fix dependency --- scylla-server/Cargo.toml | 2 +- scylla-server/src/db_handler.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scylla-server/Cargo.toml b/scylla-server/Cargo.toml index 8ae5ddad..7881327c 100644 --- a/scylla-server/Cargo.toml +++ b/scylla-server/Cargo.toml @@ -30,7 +30,7 @@ serde_json = "1.0.128" diesel_migrations = { version = "2.2.0", features = ["postgres"] } rangemap = "1.5.1" axum-macros = "0.5.0" -diesel-async = { version = "0.5.2", features = ["postgres", "bb8", "async-connection-wrapper", "sync-connection-wrapper"] } +diesel-async = { version = "0.5.2", features = ["postgres", "bb8", "async-connection-wrapper", "sync-connection-wrapper", "tokio"] } rustc-hash = "2.1.0" [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = "0.6" diff --git a/scylla-server/src/db_handler.rs b/scylla-server/src/db_handler.rs index 83060e8d..206bbf95 100644 --- a/scylla-server/src/db_handler.rs +++ b/scylla-server/src/db_handler.rs @@ -188,11 +188,11 @@ impl DbHandler { if !self.data_queue.is_empty() { // set a new max if this batch is larger max_batch_size = usize::max(max_batch_size, self.data_queue.len()); - // mem::take allows us to assign the value of data queue vec::new() while maintaining the memory for data_channel ownership data_channel .send(self.data_queue) .await .expect("Could not comm data to db thread"); + // give a vector a size that hopefully is big enough to fit the next batch self.data_queue = Vec::with_capacity((max_batch_size as f32 * 1.05) as usize); } } From 17fd0178a5f5d8b7476d221596eb6861c0da960f Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Mon, 6 Jan 2025 21:58:08 -0500 Subject: [PATCH 14/14] the max --- scylla-server/src/db_handler.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scylla-server/src/db_handler.rs b/scylla-server/src/db_handler.rs index 206bbf95..2056acbc 100644 --- a/scylla-server/src/db_handler.rs +++ b/scylla-server/src/db_handler.rs @@ -172,7 +172,7 @@ impl DbHandler { cancel_token: CancellationToken, ) { let mut batch_interval = tokio::time::interval(Duration::from_millis(self.upload_interval)); - // the match batch size to reasonably expect + // the max batch size to reasonably expect let mut max_batch_size = 2usize; loop { tokio::select! {