Implement re_grpc_server (#8622)

rerun-io · Jan 9, 2025 · 0dda8a1 · 0dda8a1
1 parent cbad0cb
commit 0dda8a1
Show file tree

Hide file tree

Showing 26 changed files with 1,601 additions and 16 deletions.
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
@@ -188,6 +188,7 @@ Update instructions:
 | re_data_loader       | Handles loading of Rerun data from file using data loader plugins                                      |
 | re_data_source       | Handles loading of Rerun data from different sources                                                   |
 | re_grpc_client       | Communicate with the Rerun Data Platform over gRPC                                                     |
+| re_grpc_server       | Host an in-memory Storage Node                                                                         |
 | re_sdk_comms         | TCP communication between Rerun SDK and Rerun Server                                                   |
 | re_web_viewer_server | Serves the Rerun web viewer (Wasm and HTML) over HTTP                                                  |
 | re_ws_comms          | WebSocket communication library (encoding, decoding, client, server) between a Rerun server and Viewer |

diff --git a/Cargo.lock b/Cargo.lock
@@ -5616,6 +5616,7 @@ dependencies = [
 name = "re_build_info"
 version = "0.22.0-alpha.1+dev"
 dependencies = [
+ "re_byte_size",
  "serde",
 ]
 
@@ -5639,7 +5640,6 @@ dependencies = [
  "arrow",
  "half",
  "re_arrow2",
- "re_tuid",
  "smallvec",
 ]
 
@@ -6004,6 +6004,27 @@ dependencies = [
  "wasm-bindgen-futures",
 ]
 
+[[package]]
+name = "re_grpc_server"
+version = "0.22.0-alpha.1+dev"
+dependencies = [
+ "re_build_info",
+ "re_byte_size",
+ "re_chunk",
+ "re_format",
+ "re_log",
+ "re_log_encoding",
+ "re_log_types",
+ "re_memory",
+ "re_protos",
+ "re_tracing",
+ "re_types",
+ "tokio",
+ "tokio-stream",
+ "tokio-util",
+ "tonic",
+]
+
 [[package]]
 name = "re_int_histogram"
 version = "0.22.0-alpha.1+dev"
@@ -6147,6 +6168,7 @@ name = "re_protos"
 version = "0.22.0-alpha.1+dev"
 dependencies = [
  "prost",
+ "re_byte_size",
  "thiserror 1.0.65",
  "tonic",
  "tonic-web-wasm-client",
@@ -6444,6 +6466,7 @@ dependencies = [
  "document-features",
  "getrandom",
  "once_cell",
+ "re_byte_size",
  "re_protos",
  "serde",
  "web-time",
@@ -8821,6 +8844,7 @@ dependencies = [
  "futures-core",
  "pin-project-lite",
  "tokio",
+ "tokio-util",
 ]
 
 [[package]]

diff --git a/Cargo.toml b/Cargo.toml
@@ -280,6 +280,7 @@ tinyvec = { version = "1.6", features = ["alloc", "rustc_1_55"] }
 tobj = "4.0"
 tokio = { version = "1.40.0", default-features = false }
 tokio-stream = "0.1.16"
+tokio-util = { version = "0.7.12", default-features = false }
 toml = { version = "0.8.10", default-features = false }
 tonic = { version = "0.12.3", default-features = false }
 tonic-build = { version = "0.12.3", default-features = false }

diff --git a/crates/build/re_build_info/Cargo.toml b/crates/build/re_build_info/Cargo.toml
@@ -27,6 +27,7 @@ serde = ["dep:serde"]
 
 
 [dependencies]
+re_byte_size.workspace = true
 
 # Optional dependencies:
 serde = { workspace = true, optional = true, features = ["derive", "rc"] }
diff --git a/crates/build/re_build_info/src/crate_version.rs b/crates/build/re_build_info/src/crate_version.rs
@@ -470,6 +470,13 @@ impl std::fmt::Display for CrateVersion {
     }
 }
 
+impl re_byte_size::SizeBytes for CrateVersion {
+    #[inline]
+    fn heap_size_bytes(&self) -> u64 {
+        0
+    }
+}
+
 #[test]
 fn test_parse_version() {
     macro_rules! assert_parse_ok {

diff --git a/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs b/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs
@@ -8,7 +8,7 @@
 use camino::Utf8Path;
 
 const PROTOS_DIR: &str = "crates/store/re_protos/proto";
-const INPUT_V0: &[&str] = &["rerun/v0/remote_store.proto", "rerun/v0/log_msg.proto"];
+const INPUT_V0_DIR: &str = "rerun/v0";
 const OUTPUT_V0_RUST_DIR: &str = "crates/store/re_protos/src/v0";
 
 fn main() {
@@ -28,16 +28,27 @@ fn main() {
 
     let definitions_dir_path = workspace_dir.join(PROTOS_DIR);
     let rust_generated_output_dir_path = workspace_dir.join(OUTPUT_V0_RUST_DIR);
+    let proto_paths = std::fs::read_dir(definitions_dir_path.join(INPUT_V0_DIR))
+        .unwrap()
+        .map(|v| {
+            v.unwrap()
+                .path()
+                .strip_prefix(&definitions_dir_path)
+                .unwrap()
+                .to_owned()
+        })
+        .collect::<Vec<_>>();
 
     re_log::info!(
         definitions=?definitions_dir_path,
         output=?rust_generated_output_dir_path,
+        protos=?proto_paths,
         "Running codegen for storage node types",
     );
 
     re_protos_builder::generate_rust_code(
         definitions_dir_path,
-        INPUT_V0,
+        &proto_paths,
         rust_generated_output_dir_path,
     );
 }
diff --git a/crates/store/re_grpc_client/Cargo.toml b/crates/store/re_grpc_client/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "re_grpc_client"
 authors.workspace = true
-description = "gRCP client for the Rerun Data Platform gRPC prototocl"
+description = "gRCP client for the Rerun Data Platform gRPC protocol"
 edition.workspace = true
 homepage.workspace = true
 include.workspace = true

diff --git a/crates/store/re_grpc_client/src/lib.rs b/crates/store/re_grpc_client/src/lib.rs
@@ -91,7 +91,7 @@ const CATALOG_BP_STORE_ID: &str = "catalog_blueprint";
 const CATALOG_REC_STORE_ID: &str = "catalog";
 const CATALOG_APPLICATION_ID: &str = "redap_catalog";
 
-/// Stream an rrd file or metadsasta catalog over gRPC from a Rerun Data Platform server.
+/// Stream an rrd file or metadata catalog over gRPC from a Rerun Data Platform server.
 ///
 /// `on_msg` can be used to wake up the UI thread on Wasm.
 pub fn stream_from_redap(

diff --git a/crates/store/re_grpc_server/Cargo.toml b/crates/store/re_grpc_server/Cargo.toml
@@ -0,0 +1,39 @@
+[package]
+name = "re_grpc_server"
+authors.workspace = true
+description = "gRCP server for the Rerun Data Platform gRPC protocol"
+edition.workspace = true
+homepage.workspace = true
+include.workspace = true
+license.workspace = true
+publish = true
+readme = "README.md"
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+[lints]
+workspace = true
+
+[package.metadata.docs.rs]
+all-features = true
+
+
+[dependencies]
+re_build_info.workspace = true
+re_byte_size.workspace = true
+re_chunk.workspace = true
+re_format.workspace = true
+re_log.workspace = true
+re_log_encoding = { workspace = true, features = ["encoder", "decoder"] }
+re_log_types.workspace = true
+re_memory.workspace = true
+re_protos.workspace = true
+re_tracing.workspace = true
+re_types.workspace = true
+
+# External
+tokio.workspace = true
+tokio-stream = { workspace = true, features = ["sync"] }
+tokio-util.workspace = true
+tonic = { workspace = true, default-features = false, features = ["transport"] }
diff --git a/crates/store/re_grpc_server/README.md b/crates/store/re_grpc_server/README.md
@@ -0,0 +1,10 @@
+# re_grpc_server
+
+Part of the [`rerun`](https://github.com/rerun-io/rerun) family of crates.
+
+[![Latest version](https://img.shields.io/crates/v/re_grpc_server.svg?speculative-link)](https://crates.io/crates/re_grpc_server?speculative-link)
+[![Documentation](https://docs.rs/re_grpc_server/badge.svg?speculative-link)](https://docs.rs/re_grpc_server?speculative-link)
+![MIT](https://img.shields.io/badge/license-MIT-blue.svg)
+![Apache](https://img.shields.io/badge/license-Apache-blue.svg)
+
+Server implementation of an in-memory Storage Node.