arrow2-convert migration 3: attr.rust.override_crate & optimizing…

… out dependencies (#3897)

This introduces the `rust.attr.override_crate` attribute, which is
necessary for the serde-codegen story.

This attribute also allows us to generate some fundamental types such as
`InstanceKey` directly into `re_types_core` rather than `re_types`,
making it possible to reduce the number of crates that depend on the
giant `re_types`.

- The `AutoSpaceViews` & `PanelView` components are now back into their
respective home (`re_viewport` & `re_viewer`, respectively).
They were temporarily moved it because we had no support for
`custom_crate`.
They will be joined by their more complicated siblings in the next PR,
which implements the necessary serde-codegen support.

- `InstanceKey`, `Clear` and `ClearIsRecursive` are now generated into
`re_types_core`.
This allows `re_query`, `re_arrow_store` and `re_data_store` to drop
their dependency on `re_types`.

- Generated code now picks up arrow from `re_types_core::external`
instead of importing it directly.
This means crates hosting generated code are not forced into import
`arrow2` directly.

---


`arrow2-convert` migration PR series:
- #3853 
- #3855
- #3897
- #3902

crates/re_arrow_store/Cargo.toml

@@ -40,9 +40,7 @@ re_format.workspace = true
 re_log_types.workspace = true
 re_log.workspace = true
 re_tracing.workspace = true
-# NOTE: We're on our way to make this crate re_types-free, which is why we import both.
 re_types_core.workspace = true
-re_types.workspace = true
 
 # External dependencies:
 ahash.workspace = true

crates/re_arrow_store/src/store_write.rs

@@ -9,8 +9,9 @@ use re_log_types::{
     DataCell, DataCellColumn, DataCellError, DataRow, DataTable, RowId, TimeInt, TimePoint,
     TimeRange,
 };
-use re_types::components::InstanceKey;
-use re_types_core::{ComponentName, ComponentNameSet, Loggable, SizeBytes as _};
+use re_types_core::{
+    components::InstanceKey, ComponentName, ComponentNameSet, Loggable, SizeBytes as _,
+};
 
 use crate::{
     store::MetadataRegistry, DataStore, DataStoreConfig, IndexedBucket, IndexedBucketInner,

crates/re_data_store/Cargo.toml

@@ -32,9 +32,7 @@ re_log_types.workspace = true
 re_log.workspace = true
 re_smart_channel.workspace = true
 re_tracing.workspace = true
-# NOTE: We're on our way to make this crate re_types-free, which is why we import both.
 re_types_core.workspace = true
-re_types.workspace = true
 
 document-features.workspace = true
 itertools.workspace = true

crates/re_data_store/src/entity_tree.rs

@@ -220,8 +220,7 @@ impl EntityTree {
         time_point: &TimePoint,
         path_op: &PathOp,
     ) -> Vec<ComponentPath> {
-        use re_types::{archetypes::Clear, components::ClearIsRecursive};
-        use re_types_core::Archetype as _;
+        use re_types_core::{archetypes::Clear, components::ClearIsRecursive, Archetype as _};
 
         re_tracing::profile_function!();
 

crates/re_data_store/src/instance_path.rs

@@ -1,7 +1,7 @@
 use std::{hash::Hash, str::FromStr};
 
 use re_log_types::{DataPath, EntityPath, EntityPathHash, PathParseError, RowId};
-use re_types::components::InstanceKey;
+use re_types_core::components::InstanceKey;
 
 use crate::{store_db::EntityDb, VersionedInstancePath, VersionedInstancePathHash};
 

crates/re_data_store/src/store_db.rs

@@ -7,8 +7,7 @@ use re_log_types::{
     ApplicationId, ComponentPath, DataCell, DataRow, DataTable, EntityPath, EntityPathHash, LogMsg,
     PathOp, RowId, SetStoreInfo, StoreId, StoreInfo, StoreKind, TimePoint, Timeline,
 };
-use re_types::components::InstanceKey;
-use re_types_core::Loggable;
+use re_types_core::{components::InstanceKey, Loggable};
 
 use crate::{Error, TimesPerTimeline};
 
@@ -117,7 +116,7 @@ impl EntityDb {
 
         // Look for a `ClearIsRecursive` component, and if it's there, go through the clear path
         // instead.
-        use re_types::components::ClearIsRecursive;
+        use re_types_core::components::ClearIsRecursive;
         if let Some(idx) = row.find_cell(&ClearIsRecursive::name()) {
             let cell = &row.cells()[idx];
             let settings = cell.try_to_native_mono::<ClearIsRecursive>().unwrap();

crates/re_log_types/Cargo.toml

@@ -39,7 +39,6 @@ re_log.workspace = true
 re_string_interner.workspace = true
 re_tracing.workspace = true
 re_tuid = { workspace = true, features = ["arrow"] }
-# NOTE: We're on our way to make this crate re_types-free, which is why we import both.
 re_types_core.workspace = true
 re_types = { workspace = true, features = ["image"] }
 

crates/re_log_types/src/example_components.rs

@@ -1,6 +1,27 @@
 //! Example components to be used for tests and docs
 
 use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize};
+use re_types::Loggable as _;
+
+// ----------------------------------------------------------------------------
+
+pub struct MyPoints;
+
+impl re_types::Archetype for MyPoints {
+    type Indicator = re_types::GenericIndicatorComponent<Self>;
+
+    fn name() -> re_types::ArchetypeName {
+        "test.MyPoints".into()
+    }
+
+    fn required_components() -> ::std::borrow::Cow<'static, [re_types::ComponentName]> {
+        vec![MyPoint::name()].into()
+    }
+
+    fn recommended_components() -> std::borrow::Cow<'static, [re_types_core::ComponentName]> {
+        vec![MyColor::name(), MyLabel::name()].into()
+    }
+}
 
 // ----------------------------------------------------------------------------
 
@@ -10,6 +31,12 @@ pub struct MyPoint {
     pub y: f32,
 }
 
+impl MyPoint {
+    pub fn new(x: f32, y: f32) -> Self {
+        Self { x, y }
+    }
+}
+
 use crate as re_log_types;
 
 re_log_types::arrow2convert_component_shim!(MyPoint as "test.Point2D");
@@ -31,6 +58,12 @@ re_log_types::arrow2convert_component_shim!(MyPoint as "test.Point2D");
 #[repr(transparent)]
 pub struct MyColor(pub u32);
 
+impl From<u32> for MyColor {
+    fn from(value: u32) -> Self {
+        Self(value)
+    }
+}
+
 re_log_types::arrow2convert_component_shim!(MyColor as "test.Color");
 
 // ----------------------------------------------------------------------------

crates/re_query/Cargo.toml

@@ -29,9 +29,7 @@ re_arrow_store.workspace = true
 re_data_store.workspace = true
 re_format.workspace = true
 re_log_types.workspace = true
-# NOTE: We're on our way to make this crate re_types-free, which is why we import both.
 re_types_core.workspace = true
-re_types = { workspace = true, features = ["datagen"] }
 re_log.workspace = true
 re_tracing.workspace = true
 
@@ -51,6 +49,8 @@ polars-core = { workspace = true, optional = true, features = [
 
 
 [dev-dependencies]
+re_types = { workspace = true, features = ["datagen"] }
+
 criterion = "0.5"
 itertools = { workspace = true }
 mimalloc.workspace = true

crates/re_query/src/archetype_view.rs

@@ -3,10 +3,9 @@ use std::{collections::BTreeMap, marker::PhantomData};
 use arrow2::array::{Array, PrimitiveArray};
 use re_format::arrow;
 use re_log_types::{DataCell, RowId};
-use re_types::components::InstanceKey;
 use re_types_core::{
-    Archetype, Component, ComponentName, DeserializationError, DeserializationResult, Loggable,
-    SerializationResult,
+    components::InstanceKey, Archetype, Component, ComponentName, DeserializationError,
+    DeserializationResult, Loggable, SerializationResult,
 };
 
 use crate::QueryError;
@@ -41,7 +40,7 @@ impl ComponentWithInstances {
 
     /// Returns the array of [`InstanceKey`]s.
     #[inline]
-    pub fn instance_keys(&self) -> Vec<re_types::components::InstanceKey> {
+    pub fn instance_keys(&self) -> Vec<InstanceKey> {
         re_tracing::profile_function!();
         self.instance_keys.to_native::<InstanceKey>()
     }

crates/re_query/src/dataframe_util.rs

@@ -4,8 +4,7 @@ use arrow2::{
 };
 use polars_core::prelude::*;
 use re_arrow_store::ArrayExt;
-use re_types::components::InstanceKey;
-use re_types_core::{Archetype, Component, Loggable};
+use re_types_core::{components::InstanceKey, Archetype, Component, Loggable};
 
 use crate::{ArchetypeView, ComponentWithInstances, QueryError};
 
@@ -125,7 +124,7 @@ impl ComponentWithInstances {
         let array1 = self.values.as_arrow_ref();
 
         let series0 = Series::try_from((
-            re_types::components::InstanceKey::name().as_ref(),
+            InstanceKey::name().as_ref(),
             array0.as_ref().clean_for_polars(),
         ))?;
         let series1 = Series::try_from((C0::name().as_ref(), array1.as_ref().clean_for_polars()))?;
@@ -145,7 +144,7 @@ impl<A: Archetype> ArchetypeView<A> {
         let array1 = C1::to_arrow_opt(self.iter_optional_component::<C1>()?)?;
 
         let series0 = Series::try_from((
-            re_types::components::InstanceKey::name().as_ref(),
+            InstanceKey::name().as_ref(),
             array0.as_ref().clean_for_polars(),
         ))?;
         let series1 = Series::try_from((C1::name().as_ref(), array1.as_ref().clean_for_polars()))?;
@@ -165,7 +164,7 @@ impl<A: Archetype> ArchetypeView<A> {
         let array2 = C2::to_arrow_opt(self.iter_optional_component::<C2>()?)?;
 
         let series0 = Series::try_from((
-            re_types::components::InstanceKey::name().as_ref(),
+            InstanceKey::name().as_ref(),
             array0.as_ref().clean_for_polars(),
         ))?;
         let series1 = Series::try_from((C1::name().as_ref(), array1.as_ref().clean_for_polars()))?;