Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add no_std support #158

Merged
merged 33 commits into from
May 17, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
30eb964
Add rust_gen_arrayvec field option for 'repeated' fields
mullr Aug 29, 2019
8ca9750
Introduce WriterBackend trait, eliminating hard dep on std::io
mullr Aug 30, 2019
8ec09c0
Support no_std in runtime crate
mullr Aug 30, 2019
b4db89e
Add no-std example to codegen script
mullr Sep 6, 2019
d5bf6f7
Change rust_gen_arrayvec to (rust_max_length)
mullr Sep 9, 2019
6ae5e9e
remove arrayvec and max_length parameter; use alloc::vec::Vec when in…
xoloki Sep 12, 2019
34a8fb7
let reader use Vec in nostd mode, but not File
xoloki Sep 12, 2019
6139007
add nostd flag to pb-rs
xoloki Sep 12, 2019
3dd078e
use alloc collections in nostd mode
xoloki Sep 12, 2019
93ed580
don't allow from_reader in nostd mode
xoloki Sep 12, 2019
56a5659
restrict serialize_into_vec to std mode; fix perftest config
xoloki Sep 12, 2019
25be3e9
hashmap typedef needs generic args; revert nostd test protobuf
xoloki Sep 13, 2019
35a94a9
regenerate nostd test proto
xoloki Sep 13, 2019
52c8c5f
packed/fixed fields use Cow, so add the appropriate use statement in …
xoloki Sep 13, 2019
3fa017e
remove alloc Vec in nostd mode since the only code that uses it requi…
xoloki Sep 13, 2019
38ec373
generate the nostd test modules using the --nostd flag
xoloki Sep 13, 2019
9dfa1b1
remove arrayvec test code
xoloki Sep 13, 2019
2291026
format fixes
xoloki Sep 13, 2019
ca414b2
copy_from_slice requires both slices to have equal length
xoloki Sep 13, 2019
5ccfe71
fmt fixes
xoloki Sep 13, 2019
a2a06c2
remove generated files; remove ArrayVec type from test proto
xoloki Sep 30, 2019
fcae141
move nostd example into standard examples dir and run it with the others
xoloki Oct 1, 2019
1dcef86
remove old nostd example
xoloki Oct 1, 2019
b56b68e
show that compiler doesn't care about multiple extern crate from sepa…
xoloki Oct 1, 2019
1c5f753
Merge branch 'master' into no_std
jpopesculian Jan 22, 2020
64f2b2c
Adds a hashbrown option
jpopesculian Jan 23, 2020
7f407ef
added global allocator to the pb_rs no_std sample
jpopesculian Jan 23, 2020
0fd2d72
fixes impl fields with core references and alloc Box
jpopesculian Jan 23, 2020
a963dc3
use mod level Box import when necessary for field or impl Owned
jpopesculian Jan 23, 2020
7069090
Reverts write_impl_owned back to no Config
jpopesculian Jan 23, 2020
dd91cde
Merge branch 'master' into no_std
jpopesculian Mar 21, 2020
39ad744
fix perftest to work with no_std
jpopesculian Mar 21, 2020
e78a173
Merge branch 'master' into no_std
tafia May 17, 2020
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions generate_modules.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,17 @@ proto_sets=(
quick-protobuf/tests/rust_protobuf/common
)

nostd_proto_sets=(
quick-protobuf/examples/pb_rs_nostd
)

for ps in "${proto_sets[@]}"; do
cargo run -p pb-rs -- -I "$ps" -d "$ps" "$ps"/*.proto
done

for ps in "${nostd_proto_sets[@]}"; do
cargo run -p pb-rs -- --nostd -I "$ps" -d "$ps" "$ps"/*.proto
done

rm -rf quick-protobuf/examples/pb_rs_v3/owned
mkdir -p quick-protobuf/examples/pb_rs_v3/owned
Expand Down
17 changes: 17 additions & 0 deletions pb-rs/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ pub struct ConfigBuilder {
custom_struct_derive: Vec<String>,
custom_repr: Option<String>,
owned: bool,
nostd: bool,
hashbrown: bool,
}

impl ConfigBuilder {
Expand Down Expand Up @@ -169,6 +171,19 @@ impl ConfigBuilder {
self
}

/// Generate no_std compliant code
pub fn nostd(mut self, val: bool) -> Self {
self.nostd = val;
self
}

/// Use hashbrown as HashMap implementation instead of [std::collections::HashMap] or
/// [alloc::collections::BTreeMap] in a `no_std` environment
pub fn hashbrown(mut self, val: bool) -> Self {
self.hashbrown = val;
self
}

/// Build Config from this ConfigBuilder
pub fn build(self) -> Vec<Config> {
self.in_files
Expand Down Expand Up @@ -198,6 +213,8 @@ impl ConfigBuilder {
custom_rpc_generator: Box::new(|_, _| Ok(())),
custom_includes: Vec::new(),
owned: self.owned,
nostd: self.nostd,
hashbrown: self.hashbrown,
}
})
.collect()
Expand Down
12 changes: 12 additions & 0 deletions pb-rs/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,16 @@ fn run() -> Result<(), ::failure::Error> {
.long("owned")
.required(false)
.help("Generate Owned structs when the proto stuct has a lifetime"),
).arg(
Arg::with_name("NOSTD")
.long("nostd")
.required(false)
.help("Generate no_std compliant code"),
).arg(
Arg::with_name("HASHBROWN")
.long("hashrown")
.required(false)
.help("Use hashrown for HashMap implementation"),
).get_matches();

let in_files = path_vec(values_t!(matches, "INPUT", String));
Expand All @@ -113,6 +123,8 @@ fn run() -> Result<(), ::failure::Error> {
.headers(!matches.is_present("NO_HEADERS"))
.dont_use_cow(matches.is_present("DONT_USE_COW"))
.custom_struct_derive(custom_struct_derive)
.nostd(matches.is_present("NOSTD"))
.hashbrown(matches.is_present("HASHBROWN"))
.custom_repr(custom_repr)
.owned(matches.is_present("OWNED"));

Expand Down
119 changes: 85 additions & 34 deletions pb-rs/src/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@ impl FieldType {
format!("{}{}{}", m.get_modules(desc), m.name, lifetime)
}
FieldType::Map(ref key, ref value) => format!(
"HashMap<{}, {}>",
"KVMap<{}, {}>",
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe most people won't use no_std and they'd rather use regular std::HashMap.
I understand that the distinction is done at import time but I'd like things to be as simple as possible for std users.

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
"KVMap<{}, {}>",
"{}<{}, {}>",
if config.no_std { "KVMap" } else { "HashMap" },

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hey @tafia, sorry for the delay in the reply! Thanks for taking a look 😄 I added the KVMap because the rust_type function doesn't have access to the config object (and I didn't want to change the call definition for a bunch of function), and the alias shouldn't restrict with what the actual person uses (other than reflecting in the docs). Obviously, I'd be happy to change it if you want, but I wanted to check with you first. I could of course also alias BTreeMap to HashMap but that seems a little hacky...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I checked, the alias, actually does not reflect in the docs because its private. It's really pretty invisible to the user of the struct...

key.rust_type(desc)?,
value.rust_type(desc)?
),
Expand Down Expand Up @@ -449,15 +449,15 @@ impl Field {
"i32" => format!("{}i32", *d),
"i64" => format!("{}i64", *d),
"f32" => match &*d.to_lowercase() {
"inf" => "::std::f32::INFINITY".to_string(),
"-inf" => "::std::f32::NEG_INFINITY".to_string(),
"nan" => "::std::f32::NAN".to_string(),
"inf" => "::core::f32::INFINITY".to_string(),
"-inf" => "::core::f32::NEG_INFINITY".to_string(),
"nan" => "::core::f32::NAN".to_string(),
_ => format!("{}f32", *d),
},
"f64" => match &*d.to_lowercase() {
"inf" => "::std::f64::INFINITY".to_string(),
"-inf" => "::std::f64::NEG_INFINITY".to_string(),
"nan" => "::std::f64::NAN".to_string(),
"inf" => "::core::f64::INFINITY".to_string(),
"-inf" => "::core::f64::NEG_INFINITY".to_string(),
"nan" => "::core::f64::NAN".to_string(),
_ => format!("{}f64", *d),
},
"Cow<'a, str>" => format!("Cow::Borrowed({})", d),
Expand Down Expand Up @@ -849,19 +849,43 @@ impl Message {
writeln!(w)?;
writeln!(w, "pub mod mod_{} {{", self.name)?;
writeln!(w)?;
if self
.messages
.iter()
.any(|m| m.all_fields().any(|f| f.typ.has_cow()))
if config.nostd {
writeln!(w, "use alloc::vec::Vec;")?;
}
if self.messages.iter().any(|m| {
m.all_fields()
.any(|f| (f.typ.has_cow() || (f.packed() && f.typ.is_fixed_size())))
}) {
if config.nostd {
writeln!(w, "use alloc::borrow::Cow;")?;
} else {
writeln!(w, "use std::borrow::Cow;")?;
}
}
if config.nostd
&& self.messages.iter().any(|m| {
desc.owned && m.has_lifetime(desc, &mut Vec::new())
|| m.all_fields().any(|f| f.boxed)
})
{
writeln!(w, "use std::borrow::Cow;")?;
writeln!(w)?;
writeln!(w, "use alloc::boxed::Box;")?;
}
if self
.messages
.iter()
.any(|m| m.all_fields().any(|f| f.typ.is_map()))
{
writeln!(w, "use std::collections::HashMap;")?;
if config.hashbrown {
writeln!(w, "use hashbrown::HashMap;")?;
writeln!(w, "type KVMap<K, V> = HashMap<K, V>;")?;
} else if config.nostd {
writeln!(w, "use alloc::collections::BTreeMap;")?;
writeln!(w, "type KVMap<K, V> = BTreeMap<K, V>;")?;
} else {
writeln!(w, "use std::collections::HashMap;")?;
writeln!(w, "type KVMap<K, V> = HashMap<K, V>;")?;
}
}
if !self.messages.is_empty() || !self.oneofs.is_empty() {
writeln!(w, "use super::*;")?;
Expand Down Expand Up @@ -1045,31 +1069,31 @@ impl Message {
struct {name}OwnedInner {{
buf: Vec<u8>,
proto: {name}<'static>,
_pin: std::marker::PhantomPinned,
_pin: core::marker::PhantomPinned,
}}

impl {name}OwnedInner {{
fn new(buf: Vec<u8>) -> Result<std::pin::Pin<Box<Self>>> {{
fn new(buf: Vec<u8>) -> Result<core::pin::Pin<Box<Self>>> {{
let inner = Self {{
buf,
proto: unsafe {{ std::mem::MaybeUninit::zeroed().assume_init() }},
_pin: std::marker::PhantomPinned,
proto: unsafe {{ core::mem::MaybeUninit::zeroed().assume_init() }},
_pin: core::marker::PhantomPinned,
}};
let mut pinned = Box::pin(inner);

let mut reader = BytesReader::from_bytes(&pinned.buf);
let proto = {name}::from_reader(&mut reader, &pinned.buf)?;

unsafe {{
let proto = std::mem::transmute::<_, {name}<'static>>(proto);
let proto = core::mem::transmute::<_, {name}<'static>>(proto);
pinned.as_mut().get_unchecked_mut().proto = proto;
}}
Ok(pinned)
}}
}}

pub struct {name}Owned {{
inner: std::pin::Pin<Box<{name}OwnedInner>>,
inner: core::pin::Pin<Box<{name}OwnedInner>>,
}}

#[allow(dead_code)]
Expand All @@ -1083,8 +1107,8 @@ impl Message {
}}
}}

impl std::fmt::Debug for {name}Owned {{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {{
impl core::fmt::Debug for {name}Owned {{
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {{
self.inner.proto.fmt(f)
}}
}}
Expand Down Expand Up @@ -1123,7 +1147,7 @@ impl Message {
}}
}}
"#,
name = self.name
name = self.name,
)?;
Ok(())
}
Expand All @@ -1144,7 +1168,7 @@ impl Message {
fn write_write_message<W: Write>(&self, w: &mut W, desc: &FileDescriptor) -> Result<()> {
writeln!(
w,
" fn write_message<W: Write>(&self, w: &mut Writer<W>) -> Result<()> {{"
" fn write_message<W: WriterBackend>(&self, w: &mut Writer<W>) -> Result<()> {{"
)?;
for f in self.fields.iter().filter(|f| !f.deprecated) {
f.write_write(w, desc)?;
Expand Down Expand Up @@ -1658,6 +1682,8 @@ pub struct Config {
pub custom_rpc_generator: RpcGeneratorFunction,
pub custom_includes: Vec<String>,
pub owned: bool,
pub nostd: bool,
pub hashbrown: bool,
}

#[derive(Debug, Default, Clone)]
Expand Down Expand Up @@ -2153,33 +2179,58 @@ impl FileDescriptor {
)?;
return Ok(());
}
writeln!(w, "use std::io::Write;")?;
if self
.messages
.iter()
.any(|m| m.all_fields().any(|f| f.typ.has_cow()))

if config.nostd {
writeln!(w, "use alloc::vec::Vec;")?;
}

if self.messages.iter().any(|m| {
m.all_fields()
.any(|f| (f.typ.has_cow() || (f.packed() && f.typ.is_fixed_size())))
}) {
if config.nostd {
writeln!(w, "use alloc::borrow::Cow;")?;
} else {
writeln!(w, "use std::borrow::Cow;")?;
}
}
if config.nostd
&& self.messages.iter().any(|m| {
self.owned && m.has_lifetime(&self, &mut Vec::new())
|| m.all_fields().any(|f| f.boxed)
})
{
writeln!(w, "use std::borrow::Cow;")?;
writeln!(w)?;
writeln!(w, "use alloc::boxed::Box;")?;
}
if self
.messages
.iter()
.any(|m| m.all_fields().any(|f| f.typ.is_map()))
{
writeln!(w, "use std::collections::HashMap;")?;
if config.hashbrown {
writeln!(w, "use hashbrown::HashMap;")?;
writeln!(w, "type KVMap<K, V> = HashMap<K, V>;")?;
} else if config.nostd {
writeln!(w, "use alloc::collections::BTreeMap;")?;
writeln!(w, "type KVMap<K, V> = BTreeMap<K, V>;")?;
} else {
writeln!(w, "use std::collections::HashMap;")?;
writeln!(w, "type KVMap<K, V> = HashMap<K, V>;")?;
}
}
writeln!(
w,
"use quick_protobuf::{{MessageRead, MessageWrite, BytesReader, Writer, Result}};"
"use quick_protobuf::{{MessageRead, MessageWrite, BytesReader, Writer, WriterBackend, Result}};"
)?;

if self.owned {
write!(
w,
"\
use std::convert::TryFrom;\n\
use std::ops::Deref;\n\
use std::ops::DerefMut;\n\
use core::convert::TryFrom;\n\
use core::ops::Deref;\n\
use core::ops::DerefMut;\n\
"
)?;
}
Expand Down
1 change: 1 addition & 0 deletions perftest/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,4 @@ failure = "0.1.5"
protobuf-codegen-pure = "2.0.4"
pb-rs = { path = "../pb-rs" }
prost-build = "0.4.0"
[features]
2 changes: 2 additions & 0 deletions perftest/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ fn main() {
custom_rpc_generator: Box::new(|rpc, writer| generate_rpc_test(rpc, writer)),
custom_includes: Vec::new(),
owned: false,
hashbrown: false,
nostd: false,
};
FileDescriptor::write_proto(&config).unwrap();

Expand Down
Loading