From 0f30a0fa28cb3cb755265e4d7bbc6df1df5564d9 Mon Sep 17 00:00:00 2001 From: "evgeny.bovykin" Date: Fri, 29 Nov 2024 13:26:36 +0100 Subject: [PATCH] Split grpc generated files into several --- .../.gitignore | 2 + .../Cargo.toml | 38 ++++++ .../proto/echo.proto | 15 +++ .../src/bin/gen.rs | 7 + .../volo.workspace.yml | 21 +++ volo-build/src/grpc_backend.rs | 127 ++++++++++++++++-- 6 files changed, 202 insertions(+), 8 deletions(-) create mode 100644 tests/code-generation-workspace-split-grpc/.gitignore create mode 100644 tests/code-generation-workspace-split-grpc/Cargo.toml create mode 100644 tests/code-generation-workspace-split-grpc/proto/echo.proto create mode 100644 tests/code-generation-workspace-split-grpc/src/bin/gen.rs create mode 100644 tests/code-generation-workspace-split-grpc/volo.workspace.yml diff --git a/tests/code-generation-workspace-split-grpc/.gitignore b/tests/code-generation-workspace-split-grpc/.gitignore new file mode 100644 index 00000000..f3d1eed0 --- /dev/null +++ b/tests/code-generation-workspace-split-grpc/.gitignore @@ -0,0 +1,2 @@ +rpc_echo +Cargo.lock diff --git a/tests/code-generation-workspace-split-grpc/Cargo.toml b/tests/code-generation-workspace-split-grpc/Cargo.toml new file mode 100644 index 00000000..5afb15c8 --- /dev/null +++ b/tests/code-generation-workspace-split-grpc/Cargo.toml @@ -0,0 +1,38 @@ +[[bin]] +bench = false +name = "gen" +test = false + +[dependencies.pilota-build] +version = "*" + +[dependencies.volo-build] +workspace = true + +[package] +edition = "2021" +name = "code-generation-workspace-split" +publish = false +version = "0.0.0" + +[workspace] +members = [] + +[workspace.dependencies] +anyhow = "1" +async-trait = "0.1" +lazy_static = "1" +serde = "1" +volo-grpc = "*" + +[workspace.dependencies.pilota] +version = "*" + +[workspace.dependencies.volo] +path = "../../volo" + +[workspace.dependencies.volo-build] +path = "../../volo-build" + +[workspace.dependencies.volo-thrift] +path = "../../volo-thrift" diff --git a/tests/code-generation-workspace-split-grpc/proto/echo.proto b/tests/code-generation-workspace-split-grpc/proto/echo.proto new file mode 100644 index 00000000..3793bbec --- /dev/null +++ b/tests/code-generation-workspace-split-grpc/proto/echo.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package echo; + +message EchoRequest { + string message = 1; +} + +message EchoResponse { + string message = 1; +} + +service Echo { + rpc Echo(EchoRequest) returns (EchoResponse) {} +} \ No newline at end of file diff --git a/tests/code-generation-workspace-split-grpc/src/bin/gen.rs b/tests/code-generation-workspace-split-grpc/src/bin/gen.rs new file mode 100644 index 00000000..dc3498f5 --- /dev/null +++ b/tests/code-generation-workspace-split-grpc/src/bin/gen.rs @@ -0,0 +1,7 @@ +use volo_build::plugin::SerdePlugin; + +fn main() { + volo_build::workspace::Builder::protobuf() + .plugin(SerdePlugin) + .gen() +} diff --git a/tests/code-generation-workspace-split-grpc/volo.workspace.yml b/tests/code-generation-workspace-split-grpc/volo.workspace.yml new file mode 100644 index 00000000..8fc06d7a --- /dev/null +++ b/tests/code-generation-workspace-split-grpc/volo.workspace.yml @@ -0,0 +1,21 @@ +common_crate_name: "common" # common_crate_name = "common" by default +touch_all: true +dedups: [] # remove the repeated structure generation in one entry +special_namings: [] +split_generated_files: true +# repos: # exsit if non-local +# { repo }: # repo = extract the name from url by default +# url: { git } # url = repo git url +# ref: { ref } # ref = "HEAD" by default +# lock: { commit hash } # lock is the last commit hash +services: + - idl: + source: local + path: proto/echo.proto + includes: + - proto + codegen_option: + touch: [] + keep_unknown_fields: false # A->B->C, B could use this to transfer the unknown fields which is needed by A and C. + config: + crate_name: rpc_echo \ No newline at end of file diff --git a/volo-build/src/grpc_backend.rs b/volo-build/src/grpc_backend.rs index 55372bd2..6f08caa4 100644 --- a/volo-build/src/grpc_backend.rs +++ b/volo-build/src/grpc_backend.rs @@ -1,3 +1,5 @@ +use std::io::Write; +use std::path::Path; use itertools::Itertools; use pilota_build::{ db::RirDatabase, @@ -6,6 +8,7 @@ use pilota_build::{ tags::protobuf::{ClientStreaming, ServerStreaming}, CodegenBackend, Context, DefId, IdentName, Symbol, }; +use pilota_build::middle::context::Mode; use volo::FastStr; pub struct MkGrpcBackend; @@ -231,6 +234,20 @@ impl VoloGrpcBackend { .into() } } + + fn write_item(stream: &mut String, base_dir: &Path, name: String, impl_str: String) { + let path_buf = base_dir.join(&name); + let path = path_buf.as_path(); + Self::write_file(path, impl_str); + stream.push_str(format!("include!(\"{}\");", &name).as_str()); + } + + fn write_file(path: &Path, stream: String) { + let mut file_writer = std::io::BufWriter::new(std::fs::File::create(path).unwrap()); + file_writer.write_all(stream.as_bytes()).unwrap(); + file_writer.flush().unwrap(); + pilota_build::fmt::fmt_file(path); + } } impl CodegenBackend for VoloGrpcBackend { @@ -254,6 +271,51 @@ impl CodegenBackend for VoloGrpcBackend { let resp_enum_name_send = format!("{}ResponseSend", service_name); let req_enum_name_recv = format!("{}RequestRecv", service_name); let resp_enum_name_recv = format!("{}ResponseRecv", service_name); + + let path = self.cx().item_path(def_id); + let path = path.as_ref(); + + // Locate directory based on the full item path + let base_dir = match self.cx().mode.as_ref() { + // In a workspace mode, the base directory is next to the `.rs` file for the service + Mode::Workspace(info) => { + let mut dir = info.dir.clone(); + if path.is_empty() { + dir + } else { + dir.push(path[0].0.as_str()); + if path.len() > 1 { + dir.push("src"); + for segment in path.iter().skip(1) { + dir.push(Path::new(segment.0.as_str())); + } + } + dir + } + } + // In single file mode, the files directory is the root + // The base directory path is the root + the item path + Mode::SingleFile { file_path } => { + let mut dir = file_path.clone(); + dir.pop(); + for segment in path { + dir.push(Path::new(segment.0.as_str())); + } + dir + } + }; + + let base_dir = if let Some(suffix) = self.cx().names.get(&def_id) { + format!("{}_{suffix}", base_dir.display()) + } else { + base_dir.display().to_string() + }; + let base_dir = Path::new(&base_dir); + + if self.cx().split { + std::fs::create_dir_all(base_dir).expect("Failed to create base directory"); + } + let paths = s .methods .iter() @@ -426,8 +488,9 @@ impl CodegenBackend for VoloGrpcBackend { }}" ); - stream.push_str(&format! { - r#"pub enum {req_enum_name_send} {{ + let req_enum_send_impl = format! { + r#" + pub enum {req_enum_name_send} {{ {req_enum_send_variants} }} @@ -437,8 +500,11 @@ impl CodegenBackend for VoloGrpcBackend { {req_send_into_body} }} }} - }} + }}"# + }; + let req_enum_recv_impl = format! { + r#" pub enum {req_enum_name_recv} {{ {req_enum_recv_variants} }} @@ -450,8 +516,11 @@ impl CodegenBackend for VoloGrpcBackend { _ => ::std::result::Result::Err(::volo_grpc::Status::new(::volo_grpc::Code::Unimplemented, "Method not found.")), }} }} - }} + }}"# + }; + let resp_enum_send_impl = format! { + r#" pub enum {resp_enum_name_send} {{ {resp_enum_send_variants} }} @@ -462,8 +531,11 @@ impl CodegenBackend for VoloGrpcBackend { {resp_send_into_body} }} }} - }} + }}"# + }; + let resp_enum_recv_impl = format! { + r#" pub enum {resp_enum_name_recv} {{ {resp_enum_recv_variants} }} @@ -478,8 +550,11 @@ impl CodegenBackend for VoloGrpcBackend { _ => ::std::result::Result::Err(::volo_grpc::Status::new(::volo_grpc::Code::Unimplemented, "Method not found.")), }} }} - }} + }}"# + }; + let client_impl = format! { + r#" pub struct {client_builder_name} {{}} impl {client_builder_name} {{ pub fn new( @@ -522,8 +597,11 @@ impl CodegenBackend for VoloGrpcBackend { impl, Response=::volo_grpc::Response<{resp_enum_name_recv}>, Error = ::volo_grpc::Status> + Send + Sync + 'static> {oneshot_client_name} {{ {oneshot_client_methods} - }} + }}"# + }; + let server_impl = format! { + r#" pub struct {server_name} {{ inner: ::std::sync::Arc, }} @@ -570,9 +648,42 @@ impl CodegenBackend for VoloGrpcBackend { impl ::volo_grpc::server::NamedService for {server_name} {{ const NAME: &'static str = "{name}"; }}"# - }); + }; + + if self.cx().split { + let mut mod_rs_stream = String::new(); + Self::write_item(&mut mod_rs_stream, base_dir, format!("enum_{}.rs", req_enum_name_send), req_enum_send_impl); + Self::write_item(&mut mod_rs_stream, base_dir, format!("enum_{}.rs", req_enum_name_recv), req_enum_recv_impl); + Self::write_item(&mut mod_rs_stream, base_dir, format!("enum_{}.rs", resp_enum_name_send), resp_enum_send_impl); + Self::write_item(&mut mod_rs_stream, base_dir, format!("enum_{}.rs", resp_enum_name_recv), resp_enum_recv_impl); + + Self::write_item(&mut mod_rs_stream, base_dir, format!("client_{}.rs", client_name), client_impl); + Self::write_item(&mut mod_rs_stream, base_dir, format!("server_{}.rs", server_name), server_impl); + + let mod_rs_file_path = base_dir.join("mod.rs"); + Self::write_file(&mod_rs_file_path, mod_rs_stream); + stream.push_str( + format!( + "include!(\"{}/mod.rs\");", + base_dir.file_name().unwrap().to_str().unwrap() + ) + .as_str(), + ); + } else { + stream.push_str(&format! { + r#" + {req_enum_send_impl} + {req_enum_recv_impl} + {resp_enum_send_impl} + {resp_enum_recv_impl} + + {client_impl} + {server_impl} + "#}); + } } + fn codegen_service_method(&self, _service_def_id: DefId, method: &rir::Method) -> String { let client_streaming = self .cx()