Update ion tests and add null & nan comparisions

extension/partiql-extension-ion/Cargo.toml

@@ -34,7 +34,7 @@ unicase = "2.7"
 rust_decimal = { version = "1.36.0", default-features = false, features = ["std"] }
 rust_decimal_macros = "1.36"
 ion-rs_old = { version = "0.18", package = "ion-rs" }
-ion-rs = { version = "1.0.0-rc.11", features = ["experimental"] }
+ion-rs = { version = "1.0.0-rc.11", features = ["experimental", "experimental-ion-hash", "sha2"] }
 
 time = { version = "0.3", features = ["macros"] }
 once_cell = "1"

extension/partiql-extension-ion/src/boxed_ion.rs

@@ -1,8 +1,9 @@
 use crate::util::{PartiqlValueTarget, ToPartiqlValue};
 use ion_rs::{
-    AnyEncoding, Element, ElementReader, IonResult, IonType, OwnedSequenceIterator, Reader,
-    Sequence,
+    AnyEncoding, Element, ElementReader, IonData, IonResult, IonType, OwnedSequenceIterator,
+    Reader, Sequence, Struct, Symbol,
 };
+use itertools::Itertools;
 use partiql_value::boxed_variant::{
     BoxedVariant, BoxedVariantResult, BoxedVariantType, BoxedVariantTypeTag,
     BoxedVariantValueIntoIterator, DynBoxedVariant,
@@ -12,14 +13,14 @@
     DatumSeqRef, DatumTupleOwned, DatumTupleRef, DatumValueOwned, DatumValueRef, OwnedSequenceView,
     OwnedTupleView, RefSequenceView, RefTupleView, SequenceDatum, TupleDatum,
 };
-use partiql_value::{Bag, BindingsName, List, Tuple, Value, Variant};
+use partiql_value::{Bag, BindingsName, List, NullableEq, Tuple, Value, Variant};
 use peekmore::{PeekMore, PeekMoreIterator};
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use std::any::Any;
 use std::borrow::Cow;
 use std::cell::RefCell;
-use std::cmp::Ordering;
+use std::collections::HashMap;
 use std::fmt::{Debug, Display, Formatter};
 use std::hash::{Hash, Hasher};
 use std::ops::DerefMut;
@@ -40,16 +41,33 @@
     }
 
     fn value_eq(&self, l: &DynBoxedVariant, r: &DynBoxedVariant) -> bool {
-        let (l, r) = get_values(l, r);
-
-        l.eq(r)
+        wrap_eq::<true, false>(l, r) == Value::Boolean(true)
+    }
+
+    fn value_eq_param(
+        &self,
+        l: &DynBoxedVariant,
+        r: &DynBoxedVariant,
+        nulls_eq: bool,
+        nans_eq: bool,
+    ) -> bool {
+        let res = match (nulls_eq, nans_eq) {
+            (true, true) => wrap_eq::<true, true>(l, r),
+            (true, false) => wrap_eq::<true, false>(l, r),
+            (false, true) => wrap_eq::<false, true>(l, r),
+            (false, false) => wrap_eq::<false, false>(l, r),
+        };
+        res == Value::Boolean(true)
     }
+}
 
-    fn value_cmp(&self, l: &DynBoxedVariant, r: &DynBoxedVariant) -> Ordering {
-        let (l, r) = get_values(l, r);
-
-        l.cmp(r)
-    }
+fn wrap_eq<const NULLS_EQUAL: bool, const NAN_EQUAL: bool>(
+    l: &DynBoxedVariant,
+    r: &DynBoxedVariant,
+) -> Value {
+    let (l, r) = get_values(l, r);
+    let wrap = IonEqualityValue::<'_, { NULLS_EQUAL }, { NAN_EQUAL }, _>;
+    NullableEq::eq(&wrap(l), &wrap(r))
 }
 
 #[inline]
@@ -109,7 +127,7 @@
     }
 }
 
 pub type IonContextPtr = Rc<RefCell<IonContext>>;
 
 // TODO [EMBDOC] does this serialization work?
 #[derive(Clone)]
@@ -120,7 +138,7 @@
 
 #[cfg(feature = "serde")]
 impl Serialize for BoxedIon {
     fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
     where
         S: Serializer,
     {
@@ -130,7 +148,7 @@
 
 #[cfg(feature = "serde")]
 impl<'de> Deserialize<'de> for BoxedIon {
     fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
     where
         D: Deserializer<'de>,
     {
@@ -139,8 +157,8 @@
 }
 
 impl Hash for BoxedIon {
-    fn hash<H: Hasher>(&self, _: &mut H) {
-        todo!("BoxedIon.hash")
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.doc.hash(state);
     }
 }
 
@@ -206,23 +224,21 @@
     }
 }
 
-impl PartialEq<Self> for BoxedIon {
-    fn eq(&self, other: &Self) -> bool {
-        self.doc.eq(&other.doc)
-    }
-}
-
-impl Eq for BoxedIon {}
+/// A wrapper on [`T`] that specifies if missing and null values should be equal.
+#[derive(Eq, PartialEq)]
+pub struct IonEqualityValue<'a, const NULLS_EQUAL: bool, const NAN_EQUAL: bool, T>(pub &'a T);
 
-impl PartialOrd for BoxedIon {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
+impl<'a, const NULLS_EQUAL: bool, const NAN_EQUAL: bool> NullableEq
+    for IonEqualityValue<'a, NULLS_EQUAL, NAN_EQUAL, BoxedIon>
+{
+    fn eq(&self, rhs: &Self) -> Value {
+        let wrap = IonEqualityValue::<'a, { NULLS_EQUAL }, { NAN_EQUAL }, _>;
+        NullableEq::eq(&wrap(&self.0.doc), &wrap(&rhs.0.doc))
     }
-}
-impl Ord for BoxedIon {
-    fn cmp(&self, other: &Self) -> Ordering {
-        // TODO lowering just to compare is costly... Either find a better way, or lift this out of the extension
-        self.lower().unwrap().cmp(&other.lower().unwrap())
+    #[inline(always)]
+    fn eqg(&self, rhs: &Self) -> Value {
+        let wrap = IonEqualityValue::<'_, true, { NAN_EQUAL }, _>;
+        NullableEq::eq(&wrap(self.0), &wrap(rhs.0))
     }
 }
 
@@ -425,13 +441,13 @@
         Value::from(Variant::from(self))
     }
 
     pub fn new(doc: impl Into<BoxedIonValue>, ctx: IonContextPtr) -> Self {
         Self {
             ctx,
             doc: doc.into(),
         }
     }
     pub fn new_value(doc: impl Into<BoxedIonValue>, ctx: IonContextPtr) -> Value {
         Self::new(doc, ctx).into_value()
     }
 
@@ -446,7 +462,7 @@
         self.child(child).into_value()
     }
 
     pub fn parse(data: Vec<u8>, expected: BoxedIonStreamType) -> IonResult<Self> {
         let mut ctx = IonContext::new_ptr(data)?;
         let doc = Self::init_doc(&mut ctx, expected);
         Ok(Self::new(doc, ctx))
@@ -531,17 +547,153 @@
     Sequence(Sequence),
 }
 
-impl PartialEq<Self> for BoxedIonValue {
-    fn eq(&self, other: &Self) -> bool {
-        match (self, other) {
-            (BoxedIonValue::Value(l), BoxedIonValue::Value(r)) => l == r,
-            (BoxedIonValue::Sequence(l), BoxedIonValue::Sequence(r)) => l == r,
-            _ => false,
+impl Hash for BoxedIonValue {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        match self {
+            BoxedIonValue::Stream() => {
+                todo!("stream not hashable? ")
+            }
+            BoxedIonValue::Value(val) => {
+                let sha = ion_rs::ion_hash::sha256(val).expect("ion hash");
+                state.write(&sha);
+            }
+            BoxedIonValue::Sequence(_) => todo!("ion seq hash"),
         }
     }
 }
 
-impl Eq for BoxedIonValue {}
+impl<'a, const NULLS_EQUAL: bool, const NAN_EQUAL: bool> NullableEq
+    for IonEqualityValue<'a, NULLS_EQUAL, NAN_EQUAL, BoxedIonValue>
+{
+    #[inline(always)]
+    fn eq(&self, other: &Self) -> Value {
+        let wrap = IonEqualityValue::<'a, { NULLS_EQUAL }, { NAN_EQUAL }, Element>;
+        let wrap_seq = IonEqualityValue::<'a, { NULLS_EQUAL }, { NAN_EQUAL }, Sequence>;
+        match (self.0, other.0) {
+            (BoxedIonValue::Value(l), BoxedIonValue::Value(r)) => {
+                NullableEq::eq(&wrap(l), &wrap(r))
+            }
+            (BoxedIonValue::Sequence(l), BoxedIonValue::Sequence(r)) => {
+                NullableEq::eq(&wrap_seq(l), &wrap_seq(r))
+            }
+            _ => Value::Boolean(false),
+        }
+    }
+
+    #[inline(always)]
+    fn eqg(&self, rhs: &Self) -> Value {
+        let wrap = IonEqualityValue::<'_, true, { NAN_EQUAL }, _>;
+        NullableEq::eq(&wrap(self.0), &wrap(rhs.0))
+    }
+}
+
+impl<'a, const NULLS_EQUAL: bool, const NAN_EQUAL: bool> NullableEq
+    for IonEqualityValue<'a, NULLS_EQUAL, NAN_EQUAL, Element>
+{
+    fn eq(&self, other: &Self) -> Value {
+        let wrap_seq = IonEqualityValue::<'a, { NULLS_EQUAL }, { NAN_EQUAL }, Sequence>;
+        let wrap_struct = IonEqualityValue::<'a, { NULLS_EQUAL }, { NAN_EQUAL }, Struct>;
+        let (l, r) = (self.0, other.0);
+        let (lty, rty) = (l.ion_type(), r.ion_type());
+
+        let result = if l.is_null() && r.is_null() {
+            NULLS_EQUAL && l.annotations().eq(r.annotations())
+        } else {
+            match (lty, rty) {
+                (IonType::Float, IonType::Float) => {
+                    let (lf, rf) = (l.as_float().unwrap(), r.as_float().unwrap());
+                    if lf.is_nan() && rf.is_nan() {
+                        NAN_EQUAL && l.annotations().eq(r.annotations())
+                    } else {
+                        lf == rf
+                    }
+                }
+
+                (IonType::List, IonType::List) => {
+                    let (ls, rs) = (l.as_list().unwrap(), r.as_list().unwrap());
+                    l.annotations().eq(r.annotations())
+                        && NullableEq::eq(&wrap_seq(ls), &wrap_seq(rs)) == Value::Boolean(true)
+                }
+                (IonType::SExp, IonType::SExp) => {
+                    let (ls, rs) = (l.as_sexp().unwrap(), r.as_sexp().unwrap());
+                    l.annotations().eq(r.annotations())
+                        && NullableEq::eq(&wrap_seq(ls), &wrap_seq(rs)) == Value::Boolean(true)
+                }
+
+                (IonType::Struct, IonType::Struct) => {
+                    let (ls, rs) = (l.as_struct().unwrap(), r.as_struct().unwrap());
+                    l.annotations().eq(r.annotations())
+                        && NullableEq::eq(&wrap_struct(ls), &wrap_struct(rs))
+                            == Value::Boolean(true)
+                }
+
+                _ => l == r,
+            }
+        };
+
+        Value::Boolean(result)
+    }
+
+    #[inline(always)]
+    fn eqg(&self, rhs: &Self) -> Value {
+        let wrap = IonEqualityValue::<'_, true, { NAN_EQUAL }, _>;
+        NullableEq::eq(&wrap(self.0), &wrap(rhs.0))
+    }
+}
+
+impl<'a, const NULLS_EQUAL: bool, const NAN_EQUAL: bool> NullableEq
+    for IonEqualityValue<'a, NULLS_EQUAL, NAN_EQUAL, Sequence>
+{
+    fn eq(&self, other: &Self) -> Value {
+        let wrap = IonEqualityValue::<'a, { NULLS_EQUAL }, { NAN_EQUAL }, _>;
+        let (l, r) = (self.0, other.0);
+        let l = l.iter().map(wrap);
+        let r = r.iter().map(wrap);
+        let res = l.zip(r).all(|(l, r)| l.eqg(&r) == Value::Boolean(true));
+        Value::Boolean(res)
+    }
+
+    #[inline(always)]
+    fn eqg(&self, rhs: &Self) -> Value {
+        let wrap = IonEqualityValue::<'_, true, { NAN_EQUAL }, _>;
+        NullableEq::eq(&wrap(self.0), &wrap(rhs.0))
+    }
+}
+
+impl<'a, const NULLS_EQUAL: bool, const NAN_EQUAL: bool> NullableEq
+    for IonEqualityValue<'a, NULLS_EQUAL, NAN_EQUAL, Struct>
+{
+    fn eq(&self, other: &Self) -> Value {
+        if self.0.len() != other.0.len() {
+            return Value::Boolean(false);
+        }
+
+        let (l, r) = (self.0, other.0);
+        let l = l.iter();
+        let r = r.iter();
+
+        let sort_fn = |(ls, le): &(&Symbol, &Element), (rs, re): &(&Symbol, &Element)| {
+            ls.cmp(rs).then(IonData::from(le).cmp(&IonData::from(re)))
+        };
+        for ((ls, lv), (rs, rv)) in l.sorted_by(sort_fn).zip(r.sorted_by(sort_fn)) {
+            if ls != rs {
+                return Value::Boolean(false);
+            }
+
+            let wrap = IonEqualityValue::<'a, { NULLS_EQUAL }, { NAN_EQUAL }, _>;
+            if NullableEq::eqg(&wrap(lv), &wrap(rv)) != Value::Boolean(true) {
+                return Value::Boolean(false);
+            }
+        }
+        Value::Boolean(true)
+    }
+
+    #[inline(always)]
+    fn eqg(&self, rhs: &Self) -> Value {
+        let wrap = IonEqualityValue::<'_, true, { NAN_EQUAL }, _>;
+        NullableEq::eq(&wrap(self.0), &wrap(rhs.0))
+    }
+}
 
 impl From<Element> for BoxedIonValue {
     fn from(value: Element) -> Self {

partiql-value/src/boxed_variant.rs

@@ -2,6 +2,7 @@ use dyn_clone::DynClone;
 use dyn_hash::DynHash;
 use partiql_common::pretty::PrettyDoc;
 use std::any::Any;
+use std::borrow::Cow;
 use std::cmp::Ordering;
 use std::error::Error;
 
@@ -28,7 +29,14 @@ pub trait BoxedVariantType: Debug + DynClone {
     fn name(&self) -> &'static str;
 
     fn value_eq(&self, l: &DynBoxedVariant, r: &DynBoxedVariant) -> bool;
-    fn value_cmp(&self, l: &DynBoxedVariant, r: &DynBoxedVariant) -> Ordering;
+
+    fn value_eq_param(
+        &self,
+        l: &DynBoxedVariant,
+        r: &DynBoxedVariant,
+        nulls_eq: bool,
+        nans_eq: bool,
+    ) -> bool;
 }
 
 dyn_clone::clone_trait_object!(BoxedVariantType);
@@ -90,9 +98,12 @@ impl PartialOrd for DynBoxedVariant {
 }
 impl Ord for DynBoxedVariant {
     fn cmp(&self, other: &Self) -> Ordering {
-        self.type_tag()
-            .cmp(&other.type_tag())
-            .then_with(|| self.type_tag().value_cmp(self, other))
+        let missing = |_| Cow::Owned(Value::Missing);
+        self.type_tag().cmp(&other.type_tag()).then_with(|| {
+            self.lower()
+                .unwrap_or_else(missing)
+                .cmp(&other.lower().unwrap_or_else(missing))
+        })
     }
 }
 

partiql-value/src/comparison.rs

@@ -1,5 +1,5 @@
-use crate::Value;
 use crate::{util, Bag, List, Tuple};
+use crate::{Value, Variant};
 
 pub trait Comparable {
     fn is_comparable_to(&self, rhs: &Self) -> bool;
@@ -72,6 +72,7 @@ impl<const GROUP_NULLS: bool, const NAN_EQUAL: bool> NullableEq
         let wrap_list = EqualityValue::<'_, { GROUP_NULLS }, { NAN_EQUAL }, List>;
         let wrap_bag = EqualityValue::<'_, { GROUP_NULLS }, { NAN_EQUAL }, Bag>;
         let wrap_tuple = EqualityValue::<'_, { GROUP_NULLS }, { NAN_EQUAL }, Tuple>;
+        let wrap_var = EqualityValue::<'_, { GROUP_NULLS }, { NAN_EQUAL }, Variant>;
         if GROUP_NULLS {
             if let (Value::Missing | Value::Null, Value::Missing | Value::Null) = (self.0, rhs.0) {
                 return Value::Boolean(true);
@@ -110,6 +111,7 @@ impl<const GROUP_NULLS: bool, const NAN_EQUAL: bool> NullableEq
             (Value::List(l), Value::List(r)) => NullableEq::eq(&wrap_list(l), &wrap_list(r)),
             (Value::Bag(l), Value::Bag(r)) => NullableEq::eq(&wrap_bag(l), &wrap_bag(r)),
             (Value::Tuple(l), Value::Tuple(r)) => NullableEq::eq(&wrap_tuple(l), &wrap_tuple(r)),
+            (Value::Variant(l), Value::Variant(r)) => NullableEq::eq(&wrap_var(l), &wrap_var(r)),
             (_, _) => Value::from(self.0 == rhs.0),
         }
     }

partiql-value/src/variant.rs

@@ -7,7 +7,7 @@
     DatumValue,
 };
 
-use crate::{Comparable, NullSortedValue, Value};
+use crate::{Comparable, EqualityValue, NullSortedValue, NullableEq, Value};
 use delegate::delegate;
 use partiql_common::pretty::{pretty_surrounded_doc, PrettyDoc, ToPretty};
 use pretty::{DocAllocator, DocBuilder};
@@ -120,7 +120,7 @@
     }
 }
 
 pub struct VariantIter<'a>(BoxedVariantValueIter<'a>);
 impl Debug for VariantIter<'_> {
     fn fmt(&self, _f: &mut Formatter<'_>) -> std::fmt::Result {
         todo!()
@@ -202,6 +202,27 @@
 
 impl Eq for Variant {}
 
+impl<const NULLS_EQUAL: bool, const NAN_EQUAL: bool> NullableEq
+    for EqualityValue<'_, NULLS_EQUAL, NAN_EQUAL, Variant>
+{
+    #[inline(always)]
+    fn eq(&self, other: &Self) -> Value {
+        let l = &self.0.variant;
+        let r = &other.0.variant;
+        let lty = l.type_tag();
+        let rty = r.type_tag();
+
+        let res = lty == rty && lty.value_eq_param(l, r, NULLS_EQUAL, NAN_EQUAL);
+        Value::Boolean(res)
+    }
+
+    #[inline(always)]
+    fn eqg(&self, rhs: &Self) -> Value {
+        let wrap = EqualityValue::<'_, true, { NAN_EQUAL }, _>;
+        NullableEq::eq(&wrap(self.0), &wrap(rhs.0))
+    }
+}
+
 #[cfg(feature = "serde")]
 impl Serialize for Variant {
     fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>