Replace buffer management with Arrow buffers

Cargo.toml

@@ -1,15 +1,18 @@
 [workspace]
 resolver = "2"
 members = [
+  "stdx/binary-search",
   "tiledb/api",
   "tiledb/common",
   "tiledb/pod",
   "tiledb/proc-macro",
   "tiledb/queries",
+  "tiledb/query-core",
   "tiledb/sys",
   "tiledb/sys-cfg",
   "tiledb/sys-defs",
   "tiledb/utils",
+  "test-utils/arrow-proptest-strategies",
   "test-utils/cells",
   "test-utils/proptest-config",
   "test-utils/signal",
@@ -23,6 +26,7 @@ default-members = [
   "tiledb/pod",
   "tiledb/proc-macro",
   "tiledb/queries",
+  "tiledb/query-core",
   "tiledb/utils",
 ]
 
@@ -35,10 +39,14 @@ version = "0.1.0"
 anyhow = "1.0"
 armerge = "2"
 arrow = { version = "52.0.0", features = ["prettyprint"] }
+arrow-array = { version = "52.0.0" }
+arrow-buffer = { version = "52.0.0" }
+arrow-proptest-strategies = { path = "test-utils/arrow-proptest-strategies" }
 arrow-schema = { version = "52.0.0" }
 bindgen = "0.70"
 cells = { path = "test-utils/cells", version = "0.1.0" }
 cmake = "0.1"
+half = { version = "2.2.1", default-features = false }
 itertools = "0"
 num-traits = "0.2"
 paste = "1.0"
@@ -47,6 +55,7 @@ regex = "1"
 serde = { version = "1", features = ["derive"] }
 serde_json = { version = "1", features = ["float_roundtrip"] }
 signal = { path = "test-utils/signal", version = "0.1.0" }
+stdx-binary-search = { path = "stdx/binary-search", version = "0.1.0" }
 strategy-ext = { path = "test-utils/strategy-ext", version = "0.1.0" }
 tempfile = { version = "3" }
 thiserror = { version = "1" }

stdx/binary-search/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "stdx-binary-search"
+edition.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+[dependencies]
+num-traits = { workspace = true }
+
+[dev-dependencies]
+proptest = { workspace = true }

stdx/binary-search/src/lib.rs

@@ -0,0 +1,153 @@
+use std::ops::Range;
+
+use num_traits::{FromPrimitive, ToPrimitive};
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum Bisect<T> {
+    NeverTrue,
+    UpperBound(T),
+    AlwaysTrue,
+}
+
+/// A type which represents a searchable space of values.
+pub trait Search {
+    type Item;
+
+    /// Performs an efficient search over the items of `self` to find the upper bound
+    /// where `property` is true.
+    ///
+    /// `property` is some function which bisects the search space,
+    /// returning `true` on each value in the first segment and `false` on each value in the second.
+    fn upper_bound<F>(&self, property: F) -> Bisect<Self::Item>
+    where
+        F: Fn(&Self::Item) -> bool;
+}
+
+macro_rules! binary_search_impl {
+    ($($ITYPE:ty),+) => {
+        $(
+        impl Search for Range<$ITYPE> {
+            type Item = $ITYPE;
+
+            fn upper_bound<F>(&self, property: F) -> Bisect<Self::Item>
+            where
+                F: Fn(&Self::Item) -> bool,
+            {
+                if self.is_empty() {
+                    return Bisect::AlwaysTrue
+                } else if self.start + 1 == self.end {
+                    return if property(&self.start) {
+                        Bisect::AlwaysTrue
+                    } else {
+                        Bisect::NeverTrue
+                    }
+                }
+                let mut search = self.clone();
+                while search.start + 1 < search.end {
+                    let midpoint = midpoint(&search);
+                    if property(&midpoint) {
+                        search.start = midpoint;
+                    } else {
+                        search.end = midpoint;
+                    }
+                }
+                if search.end == self.end {
+                    Bisect::AlwaysTrue
+                } else if property(&search.start) {
+                    Bisect::UpperBound(search.start)
+                } else {
+                    Bisect::NeverTrue
+                }
+            }
+        }
+        )+
+    };
+}
+
+fn midpoint<T>(range: &Range<T>) -> T
+where
+    T: Copy + FromPrimitive + ToPrimitive,
+{
+    T::from_i128(
+        (range.start.to_i128().unwrap() + range.end.to_i128().unwrap()) / 2,
+    )
+    .unwrap()
+}
+
+binary_search_impl!(
+    u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize
+);
+
+#[cfg(test)]
+mod tests {
+    use proptest::prelude::*;
+
+    use super::*;
+
+    /// Performs a linear search to return the maximum value in the range
+    /// for which a `property` is true
+    fn linear_search<T, F>(range: Range<T>, property: F) -> Bisect<T>
+    where
+        Range<T>: Iterator<Item = T>,
+        F: Fn(&T) -> bool,
+    {
+        let mut prev = None;
+        for i in range {
+            if property(&i) {
+                prev = Some(i);
+            } else if let Some(prev) = prev {
+                return Bisect::UpperBound(prev);
+            } else {
+                return Bisect::NeverTrue;
+            }
+        }
+        Bisect::AlwaysTrue
+    }
+
+    fn search_results<T, F>(
+        range: Range<T>,
+        property: F,
+    ) -> (Bisect<T>, Bisect<T>)
+    where
+        Range<T>: Iterator<Item = T> + Search<Item = T>,
+        T: Clone + PartialEq,
+        F: Clone + Fn(&T) -> bool,
+    {
+        let linear_search_result =
+            linear_search(range.clone(), property.clone());
+        let binary_search_result = range.upper_bound(property.clone());
+
+        (linear_search_result, binary_search_result)
+    }
+
+    #[test]
+    fn example_simple_less_than() {
+        let cmp = |value: &usize| *value < 5;
+
+        for i in 0..10 {
+            for j in i..10 {
+                let (linear, binary) = search_results(i..j, &cmp);
+                assert_eq!(linear, binary, "i..j = {:?}", i..j)
+            }
+        }
+    }
+
+    proptest! {
+        #[test]
+        fn proptest_simple_less_than(target in any::<usize>(), range in any::<Range<usize>>()) {
+            match range.upper_bound(|value: &usize| *value < target) {
+                Bisect::AlwaysTrue => assert!(range.end <= target),
+                Bisect::NeverTrue => assert!(target < range.start),
+                Bisect::UpperBound(bound) => {
+                    assert_eq!(target - 1, bound);
+                }
+            }
+        }
+
+        #[test]
+        fn proptest_search_compare(target in any::<u8>(), range in any::<Range<u8>>()) {
+            let (linear, binary) = search_results(range, |value: &u8| *value < target);
+            assert_eq!(linear, binary);
+        }
+    }
+}

test-utils/arrow-proptest-strategies/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "arrow-proptest-strategies"
+edition.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+[dependencies]
+arrow-array = { workspace = true }
+arrow-buffer = { workspace = true }
+arrow-schema = { workspace = true }
+half = { workspace = true }
+proptest = { workspace = true }
+strategy-ext = { workspace = true }