fix UB and run tests with miri

.github/workflows/ci.yml

@@ -44,3 +44,22 @@ jobs:
       - name: Pytest - PyVortex
         run: rye run pytest --benchmark-disable test/
         working-directory: pyvortex/
+
+  miri:
+    name: 'miri'
+    runs-on: ubuntu-latest
+    env:
+      RUST_BACKTRACE: 1
+      MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-backtrace=full
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/cleanup
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+
+      - uses: ./.github/actions/setup-rust
+
+      - name: Rust Miri
+        run: cargo +nightly miri test

encodings/fastlanes/src/bitpacking/compress.rs

@@ -320,8 +320,13 @@ mod test {
     }
 
     #[test]
-    fn test_compression_roundtrip() {
+    fn test_compression_roundtrip_fast() {
         compression_roundtrip(125);
+    }
+
+    #[test]
+    #[cfg_attr(miri, ignore)] // This test is too slow on miri
+    fn test_compression_roundtrip() {
         compression_roundtrip(1024);
         compression_roundtrip(10_000);
         compression_roundtrip(10_240);

encodings/fastlanes/src/bitpacking/compute/take.rs

@@ -172,6 +172,7 @@ mod test {
     }
 
     #[test]
+    #[cfg_attr(miri, ignore)] // This test is too slow on miri
     fn take_random_indices() {
         let num_patches: usize = 128;
         let values = (0..u16::MAX as u32 + num_patches as u32).collect::<Vec<_>>();

vortex-array/src/array/varbinview/builder.rs

@@ -6,7 +6,7 @@ use arrow_buffer::NullBufferBuilder;
 use vortex_dtype::DType;
 
 use crate::array::primitive::PrimitiveArray;
-use crate::array::varbinview::{BinaryView, Inlined, Ref, VarBinViewArray, VIEW_SIZE};
+use crate::array::varbinview::{BinaryView, Inlined, Ref, VarBinViewArray, VIEW_SIZE_TO_U64_SIZE};
 use crate::validity::Validity;
 use crate::{ArrayData, IntoArray, IntoArrayData, ToArray};
 
@@ -99,17 +99,17 @@ impl<T: AsRef<[u8]>> VarBinViewBuilder<T> {
         };
 
         // convert Vec<BinaryView> to Vec<u8> which can be stored as an array
-        let views_u8: Vec<u8> = unsafe {
+        let views_u64: Vec<u64> = unsafe {
             let mut views_clone = ManuallyDrop::new(mem::take(&mut self.views));
             Vec::from_raw_parts(
                 views_clone.as_mut_ptr() as _,
-                views_clone.len() * VIEW_SIZE,
-                views_clone.capacity() * VIEW_SIZE,
+                views_clone.len() * VIEW_SIZE_TO_U64_SIZE,
+                views_clone.capacity() * VIEW_SIZE_TO_U64_SIZE,
             )
         };
 
         VarBinViewArray::try_new(
-            PrimitiveArray::from(views_u8).to_array(),
+            PrimitiveArray::from(views_u64).to_array(),
             completed,
             dtype,
             validity,

vortex-array/src/array/varbinview/compute.rs

@@ -2,7 +2,7 @@ use vortex_error::VortexResult;
 use vortex_scalar::Scalar;
 
 use crate::array::varbin::varbin_scalar;
-use crate::array::varbinview::{VarBinViewArray, VIEW_SIZE};
+use crate::array::varbinview::{VarBinViewArray, VIEW_SIZE_TO_U64_SIZE};
 use crate::compute::unary::ScalarAtFn;
 use crate::compute::{slice, ArrayCompute, SliceFn};
 use crate::validity::ArrayValidity;
@@ -32,9 +32,13 @@ impl ScalarAtFn for VarBinViewArray {
 impl SliceFn for VarBinViewArray {
     fn slice(&self, start: usize, stop: usize) -> VortexResult<Array> {
         Ok(Self::try_new(
-            slice(&self.views(), start * VIEW_SIZE, stop * VIEW_SIZE)?
-                .into_array_data()
-                .into_array(),
+            slice(
+                &self.views(),
+                start * VIEW_SIZE_TO_U64_SIZE,
+                stop * VIEW_SIZE_TO_U64_SIZE,
+            )?
+            .into_array_data()
+            .into_array(),
             (0..self.metadata().data_lens.len())
                 .map(|i| self.bytes(i))
                 .collect::<Vec<_>>(),

vortex-array/src/array/varbinview/mod.rs

@@ -38,7 +38,6 @@ struct Inlined {
 }
 
 impl Inlined {
-    #[allow(dead_code)]
     pub fn new(value: &[u8]) -> Self {
         assert!(
             value.len() <= BinaryView::MAX_INLINED_SIZE,
@@ -106,7 +105,9 @@ impl Debug for BinaryView {
     }
 }
 
-pub const VIEW_SIZE: usize = mem::size_of::<BinaryView>();
+// this is a hack; views are 16 bytes with 8-byte alignment
+pub(crate) const VIEW_SIZE_TO_U64_SIZE: usize =
+    mem::size_of::<BinaryView>() / mem::size_of::<u64>();
 
 impl_encoding!("vortex.varbinview", 5u16, VarBinView);
 
@@ -123,8 +124,11 @@ impl VarBinViewArray {
         dtype: DType,
         validity: Validity,
     ) -> VortexResult<Self> {
-        if !matches!(views.dtype(), &DType::BYTES) {
-            vortex_bail!(MismatchedTypes: "u8", views.dtype());
+        if !matches!(
+            views.dtype(),
+            &DType::Primitive(PType::U64, Nullability::NonNullable)
+        ) {
+            vortex_bail!(MismatchedTypes: "u64", views.dtype());
         }
 
         for d in data.iter() {
@@ -141,10 +145,9 @@ impl VarBinViewArray {
             vortex_bail!("incorrect validity {:?}", validity);
         }
 
-        let length = views.len() / VIEW_SIZE;
-
+        let num_views = views.len() / VIEW_SIZE_TO_U64_SIZE;
         let metadata = VarBinViewMetadata {
-            validity: validity.to_metadata(views.len() / VIEW_SIZE)?,
+            validity: validity.to_metadata(num_views)?,
             data_lens: data.iter().map(|a| a.len()).collect_vec(),
         };
 
@@ -155,17 +158,17 @@ impl VarBinViewArray {
             children.push(a)
         }
 
-        Self::try_from_parts(dtype, length, metadata, children.into(), StatsSet::new())
+        Self::try_from_parts(dtype, num_views, metadata, children.into(), StatsSet::new())
     }
 
     fn view_slice(&self) -> &[BinaryView] {
         unsafe {
             slice::from_raw_parts(
                 PrimitiveArray::try_from(self.views())
                     .expect("Views must be a primitive array")
-                    .maybe_null_slice::<u8>()
+                    .maybe_null_slice::<u64>()
                     .as_ptr() as _,
-                self.views().len() / VIEW_SIZE,
+                self.views().len() / VIEW_SIZE_TO_U64_SIZE,
             )
         }
     }
@@ -177,7 +180,11 @@ impl VarBinViewArray {
     #[inline]
     pub fn views(&self) -> Array {
         self.array()
-            .child(0, &DType::BYTES, self.len() * VIEW_SIZE)
+            .child(
+                0,
+                &DType::Primitive(PType::U64, Nullability::NonNullable),
+                self.len() * VIEW_SIZE_TO_U64_SIZE,
+            )
             .expect("missing views")
     }
 
@@ -254,7 +261,7 @@ fn as_arrow(var_bin_view: VarBinViewArray) -> ArrayRef {
         .views()
         .into_primitive()
         .expect("views must be primitive");
-    assert_eq!(views.ptype(), PType::U8);
+    assert_eq!(views.ptype(), PType::U64);
     let nulls = var_bin_view
         .logical_validity()
         .to_null_buffer()
@@ -362,7 +369,7 @@ impl<'a> FromIterator<Option<&'a str>> for VarBinViewArray {
 mod test {
     use vortex_scalar::Scalar;
 
-    use crate::array::varbinview::VarBinViewArray;
+    use crate::array::varbinview::{BinaryView, VarBinViewArray, VIEW_SIZE_TO_U64_SIZE};
     use crate::compute::slice;
     use crate::compute::unary::scalar_at;
     use crate::{Canonical, IntoArray, IntoCanonical};
@@ -408,4 +415,15 @@ mod test {
         assert_eq!(scalar_at(&var_bin, 0).unwrap(), Scalar::from("string1"));
         assert_eq!(scalar_at(&var_bin, 1).unwrap(), Scalar::from("string2"));
     }
+
+    #[test]
+    pub fn views_fit_in_u64() {
+        assert_eq!(VIEW_SIZE_TO_U64_SIZE, 2);
+        assert_eq!(std::mem::size_of::<BinaryView>(), 16);
+        assert_eq!(std::mem::align_of::<BinaryView>(), 8);
+        assert_eq!(
+            std::mem::size_of::<BinaryView>(),
+            std::mem::size_of::<u64>() * VIEW_SIZE_TO_U64_SIZE
+        );
+    }
 }

vortex-datafusion/src/lib.rs

@@ -510,6 +510,7 @@ mod test {
     }
 
     #[tokio::test]
+    #[cfg_attr(miri, ignore)]
     async fn test_datafusion_pushdown() {
         let ctx = SessionContext::new();
 
@@ -538,6 +539,7 @@ mod test {
     }
 
     #[tokio::test]
+    #[cfg_attr(miri, ignore)]
     async fn test_datafusion_no_pushdown() {
         let ctx = SessionContext::new();
 

vortex-sampling-compressor/tests/smoketest.rs

@@ -24,6 +24,7 @@ use vortex_sampling_compressor::compressors::CompressorRef;
 use vortex_sampling_compressor::{CompressConfig, SamplingCompressor};
 
 #[test]
+#[cfg_attr(miri, ignore)] // This test is too slow on miri
 pub fn smoketest_compressor() {
     let compressor = SamplingCompressor::new_with_options(
         HashSet::from([