DataFusion TableProvider for memory arrays

Cargo.toml

@@ -6,6 +6,7 @@ members = [
     "pyvortex",
     "vortex-array",
     "vortex-buffer",
+    "vortex-datafusion",
     "vortex-dtype",
     "vortex-error",
     "vortex-expr",
@@ -41,15 +42,20 @@ arrow-data = "51.0.0"
 arrow-ipc = "51.0.0"
 arrow-schema = "51.0.0"
 arrow-select = "51.0.0"
+async-trait = "0.1"
 bindgen = "0.69.4"
 bytes = "1.6.0"
 bzip2 = "0.4.4"
 cargo_metadata = "0.18.1"
 criterion = { version = "0.5.1", features = ["html_reports"] }
 croaring = "1.0.1"
 csv = "1.3.0"
-datafusion-common = "39.0.0"
-datafusion-expr = "39.0.0"
+datafusion = "37.1.0"
+datafusion-common = "37.1.0"
+datafusion-execution = "37.1.0"
+datafusion-expr = "37.1.0"
+datafusion-physical-expr = "37.1.0"
+datafusion-physical-plan = "37.1.0"
 derive_builder = "0.20.0"
 divan = "0.1.14"
 duckdb = { version = "0.10.1", features = ["bundled"] }

vortex-array/src/array/primitive/compute/filter_indices.rs

@@ -68,7 +68,6 @@ fn apply_predicate<T: NativePType, F: Fn(&T, &T) -> bool>(
 
 #[cfg(test)]
 mod test {
-    use itertools::Itertools;
     use vortex_dtype::field::FieldPath;
     use vortex_expr::{lit, Conjunction, FieldPathOperations};
 
@@ -80,13 +79,11 @@ mod test {
     }
 
     fn to_int_indices(filtered_primitive: BoolArray) -> Vec<u64> {
-        let filtered = filtered_primitive
+        filtered_primitive
             .boolean_buffer()
-            .iter()
-            .enumerate()
-            .flat_map(|(idx, v)| if v { Some(idx as u64) } else { None })
-            .collect_vec();
-        filtered
+            .set_indices()
+            .map(|i| i as u64)
+            .collect()
     }
 
     #[test]

vortex-array/src/view.rs

@@ -34,7 +34,6 @@ impl Debug for ArrayView {
         f.debug_struct("ArrayView")
             .field("encoding", &self.encoding)
             .field("dtype", &self.dtype)
-            // .field("array", &self.array)
             .field("buffers", &self.buffers)
             .field("ctx", &self.ctx)
             .finish()

vortex-buffer/src/string.rs

@@ -3,12 +3,12 @@ use std::str::Utf8Error;
 
 use crate::Buffer;
 
-/// A wrapper around a `Buffer` that guarantees that the buffer contains valid UTF-8.
+/// A wrapper around a [`Buffer`] that guarantees that the buffer contains valid UTF-8.
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd)]
 pub struct BufferString(Buffer);
 
 impl BufferString {
-    /// Creates a new `BufferString` from a `Buffer`.
+    /// Creates a new `BufferString` from a [`Buffer`].
     ///
     /// # Safety
     /// Assumes that the buffer contains valid UTF-8.

vortex-datafusion/Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+name = "vortex-datafusion"
+version.workspace = true
+homepage.workspace = true
+repository.workspace = true
+authors.workspace = true
+license.workspace = true
+keywords.workspace = true
+include.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+vortex-array = { path = "../vortex-array" }
+vortex-dtype = { path = "../vortex-dtype", features = ["arrow"] }
+vortex-error = { path = "../vortex-error" }
+
+arrow-array = { workspace = true }
+arrow-schema = { workspace = true }
+async-trait = { workspace = true }
+
+datafusion = { workspace = true }
+datafusion-common = { workspace = true }
+datafusion-expr = { workspace = true }
+datafusion-execution = { workspace = true }
+datafusion-physical-expr = { workspace = true }
+datafusion-physical-plan = { workspace = true }
+futures = { workspace = true }
+pin-project = { workspace = true }
+
+[dev-dependencies]
+tokio = { workspace = true, features = ["test-util"] }
+
+[lints]
+workspace = true