Skip to content

Commit

Permalink
Tagged columnar updates: Rust (#8764)
Browse files Browse the repository at this point in the history
I ended up with something slightly different than the original design,
because it turned out to really not feel that great in practice.
In particular, it required duplicating _a lot_ of generated code for
little (no?) added value.

In short: `.columns` becomes a vanilla method rather than a static
method.

```rust 
// Prepare a point cloud that evolves over 5 timesteps, changing the number of points in the process.
let times = TimeColumn::new_seconds("time", 10..15);

#[rustfmt::skip]
let positions = [
    [1.0, 0.0, 1.0], [0.5, 0.5, 2.0],
    [1.5, -0.5, 1.5], [1.0, 1.0, 2.5], [-0.5, 1.5, 1.0], [-1.5, 0.0, 2.0],
    [2.0, 0.0, 2.0], [1.5, -1.5, 3.0], [0.0, -2.0, 2.5], [1.0, -1.0, 3.5],
    [-2.0, 0.0, 2.0], [-1.5, 1.5, 3.0], [-1.0, 1.0, 3.5],
    [1.0, -1.0, 1.0], [2.0, -2.0, 2.0], [3.0, -1.0, 3.0], [2.0, 0.0, 4.0],
];

// At each timestep, all points in the cloud share the same but changing color and radius.
let colors = [0xFF0000FF, 0x00FF00FF, 0x0000FFFF, 0xFFFF00FF, 0x00FFFFFF];
let radii = [0.05, 0.01, 0.2, 0.1, 0.3];

// Partition our data as expected across the 5 timesteps.
let position = rerun::Points3D::update_fields()
    .with_positions(positions)
    .columns([2, 4, 4, 3, 4])?;
let color_and_radius = rerun::Points3D::update_fields()
    .with_colors(colors)
    .with_radii(radii)
    .columns([1, 1, 1, 1, 1])?;

rec.send_columns_v2("points", [times], position.chain(color_and_radius))?;
```

* Fixes #8753 
* Fixes #7167
  • Loading branch information
teh-cmc authored Jan 21, 2025
1 parent 13f1ea1 commit e6e3b67
Show file tree
Hide file tree
Showing 63 changed files with 2,085 additions and 35 deletions.
41 changes: 39 additions & 2 deletions crates/build/re_types_builder/src/codegen/rust/api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, HashMap, HashSet};

use anyhow::Context as _;
use camino::{Utf8Path, Utf8PathBuf};
use itertools::Itertools as _;
use itertools::Itertools;
use proc_macro2::TokenStream;
use quote::{format_ident, quote};

Expand Down Expand Up @@ -1922,10 +1922,47 @@ fn quote_builder_from_obj(reporter: &Reporter, objects: &Objects, obj: &Object)
}
});

let columnar_methods = obj.is_eager_rust_archetype().then(|| {
let columns_doc = unindent::unindent("\
Partitions the component data into multiple sub-batches.
Specifically, this transforms the existing [`SerializedComponentBatch`]es data into [`SerializedComponentColumn`]s
instead, via [`SerializedComponentBatch::partitioned`].
This makes it possible to use `RecordingStream::send_columns` to send columnar data directly into Rerun.
The specified `lengths` must sum to the total length of the component batch.
[`SerializedComponentColumn`]: [::re_types_core::SerializedComponentColumn]
");
let columns_doc = quote_doc_lines(&columns_doc.lines().map(|l| l.to_owned()).collect_vec());

let fields = required.iter().chain(optional.iter()).map(|field| {
let field_name = format_ident!("{}", field.name);
quote!(self.#field_name.map(|#field_name| #field_name.partitioned(_lengths.clone())).transpose()?)
});

quote! {
#columns_doc
#[inline]
pub fn columns<I>(
self,
_lengths: I, // prefixed so it doesn't conflict with fields of the same name
) -> SerializationResult<impl Iterator<Item = ::re_types_core::SerializedComponentColumn>>
where
I: IntoIterator<Item = usize> + Clone,
{
let columns = [ #(#fields),* ];
let indicator_column = ::re_types_core::indicator_column::<Self>(_lengths.into_iter().count())?;
Ok(columns.into_iter().chain([indicator_column]).flatten())
}
}
});

let with_methods = if obj.is_eager_rust_archetype() {
quote! {
#partial_update_methods

#columnar_methods
#(#eager_with_methods)*
}
} else {
Expand Down
49 changes: 49 additions & 0 deletions crates/store/re_types/src/archetypes/arrows2d.rs

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

46 changes: 46 additions & 0 deletions crates/store/re_types/src/archetypes/arrows3d.rs

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

31 changes: 31 additions & 0 deletions crates/store/re_types/src/archetypes/bar_chart.rs

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

52 changes: 52 additions & 0 deletions crates/store/re_types/src/archetypes/capsules3d.rs

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

46 changes: 46 additions & 0 deletions crates/store/re_types/src/archetypes/depth_image.rs

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

55 changes: 55 additions & 0 deletions crates/store/re_types/src/archetypes/ellipsoids3d.rs

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit e6e3b67

Please sign in to comment.