Add lowering of Variants

extension/partiql-extension-ion/Cargo.toml

@@ -23,6 +23,7 @@ bench = false
 [dependencies]
 partiql-value = { path = "../../partiql-value", version = "0.11.*" }
 partiql-common = { path = "../../partiql-common", version = "0.11.*" }
+partiql-types = { path = "../../partiql-types", version = "0.11.*" }
 
 serde = { version = "1", features = ["derive"], optional = true }
 typetag = { version = "0.2", optional = true }

extension/partiql-extension-ion/src/boxed_ion.rs

@@ -1,3 +1,4 @@
+use crate::util::{PartiqlValueTarget, ToPartiqlValue};
 use ion_rs::{
     AnyEncoding, Element, ElementReader, IonResult, IonType, OwnedSequenceIterator, Reader,
     Sequence,
@@ -7,11 +8,11 @@ use partiql_value::boxed_variant::{
     BoxedVariant, BoxedVariantResult, BoxedVariantType, BoxedVariantValueIntoIterator,
 };
 use partiql_value::datum::{
-    Datum, DatumCategoryOwned, DatumCategoryRef, DatumSeqOwned, DatumSeqRef, DatumTupleOwned,
-    DatumTupleRef, DatumValueOwned, DatumValueRef, OwnedSequenceView, OwnedTupleView,
-    RefSequenceView, RefTupleView, SequenceDatum, TupleDatum,
+    Datum, DatumCategoryOwned, DatumCategoryRef, DatumLower, DatumLowerResult, DatumSeqOwned,
+    DatumSeqRef, DatumTupleOwned, DatumTupleRef, DatumValueOwned, DatumValueRef, OwnedSequenceView,
+    OwnedTupleView, RefSequenceView, RefTupleView, SequenceDatum, TupleDatum,
 };
-use partiql_value::{BindingsName, Value, Variant};
+use partiql_value::{Bag, BindingsName, List, Tuple, Value, Variant};
 use peekmore::{PeekMore, PeekMoreIterator};
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -132,9 +133,7 @@ impl BoxedVariant for BoxedIon {
                 IonType::SExp => DatumCategoryRef::Sequence(DatumSeqRef::Dynamic(self)),
                 IonType::Null => DatumCategoryRef::Null,
                 IonType::Struct => DatumCategoryRef::Tuple(DatumTupleRef::Dynamic(self)),
-                _ => DatumCategoryRef::Scalar(DatumValueRef::Value(todo!(
-                    "Boxed Ion DatumCategoryRef::Scalar"
-                ))),
+                _ => DatumCategoryRef::Scalar(DatumValueRef::Lower(self)),
             },
         }
     }
@@ -156,6 +155,44 @@ impl BoxedVariant for BoxedIon {
     }
 }
 
+impl DatumLower<Value> for BoxedIon {
+    fn into_lower(self) -> DatumLowerResult<Value> {
+        let Self { ctx, doc } = self;
+        match doc {
+            BoxedIonValue::Stream() => todo!("into_lower stream"),
+            BoxedIonValue::Sequence(seq) => todo!("into_lower seq"),
+            BoxedIonValue::Value(elt) => {
+                let pval = elt.into_partiql_value()?;
+                Ok(match pval {
+                    PartiqlValueTarget::Atom(val) => val,
+                    PartiqlValueTarget::List(l) => {
+                        let vals = l.into_iter().map(|elt| Self::new_value(elt, ctx.clone()));
+                        List::from_iter(vals).into()
+                    }
+                    PartiqlValueTarget::Bag(b) => {
+                        let vals = b.into_iter().map(|elt| Self::new_value(elt, ctx.clone()));
+                        Bag::from_iter(vals).into()
+                    }
+                    PartiqlValueTarget::Struct(s) => {
+                        let vals = s
+                            .into_iter()
+                            .map(|(key, elt)| (key, Self::new_value(elt, ctx.clone())));
+                        Tuple::from_iter(vals).into()
+                    }
+                })
+            }
+        }
+    }
+
+    fn into_lower_boxed(self: Box<Self>) -> DatumLowerResult<Value> {
+        self.into_lower()
+    }
+
+    fn lower(&self) -> DatumLowerResult<Cow<'_, Value>> {
+        self.clone().into_lower().map(Cow::Owned)
+    }
+}
+
 impl SequenceDatum for BoxedIon {
     fn is_ordered(&self) -> bool {
         true
@@ -219,6 +256,10 @@ impl OwnedSequenceView<Value> for BoxedIon {
     fn take_val_boxed(self: Box<Self>, k: i64) -> Option<Value> {
         OwnedSequenceView::take_val(*self, k)
     }
+
+    fn into_iter_boxed(self: Box<Self>) -> Box<dyn Iterator<Item = Value>> {
+        todo!()
+    }
 }
 
 impl TupleDatum for BoxedIon {

extension/partiql-extension-ion/src/lib.rs

@@ -4,10 +4,11 @@
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
 
+pub mod boxed_ion;
 mod common;
 pub mod decode;
-pub mod boxed_ion;
 pub mod encode;
+mod util;
 
 pub use common::Encoding;
 

extension/partiql-extension-ion/src/util.rs

@@ -0,0 +1,129 @@
+use crate::decode::IonDecodeError;
+use ion_rs::{Decimal, Element, IonResult};
+use partiql_value::datum::DatumLowerResult;
+use partiql_value::{DateTime, Value};
+use std::num::NonZeroU8;
+use std::str::FromStr;
+
+pub enum PartiqlValueTarget<T> {
+    Atom(Value),
+    List(Vec<T>),
+    Bag(Vec<T>),
+    Struct(Vec<(String, T)>),
+}
+
+impl<T, V> From<V> for PartiqlValueTarget<T>
+where
+    V: Into<Value>,
+{
+    fn from(value: V) -> Self {
+        PartiqlValueTarget::Atom(value.into())
+    }
+}
+
+pub trait ToPartiqlValue<T> {
+    fn into_partiql_value(self) -> DatumLowerResult<PartiqlValueTarget<T>>;
+}
+
+impl ToPartiqlValue<Element> for Element {
+    fn into_partiql_value(self) -> DatumLowerResult<PartiqlValueTarget<Element>> {
+        let value = self.into_value();
+        match value {
+            ion_rs::Value::Null(_) => Ok(Value::Null.into()),
+            ion_rs::Value::Bool(inner) => Ok(inner.into()),
+            ion_rs::Value::Int(inner) => inner.into_partiql_value(),
+            ion_rs::Value::Float(inner) => Ok(inner.into()),
+            ion_rs::Value::Decimal(inner) => inner.into_partiql_value(),
+            ion_rs::Value::Timestamp(inner) => inner.into_partiql_value(),
+            ion_rs::Value::Symbol(inner) => inner.into_partiql_value(),
+            ion_rs::Value::String(inner) => inner.into_partiql_value(),
+            ion_rs::Value::Clob(inner) => inner.into_partiql_value(),
+            ion_rs::Value::Blob(inner) => inner.into_partiql_value(),
+            ion_rs::Value::List(inner) => inner.into_partiql_value(),
+            ion_rs::Value::SExp(inner) => inner.into_partiql_value(),
+            ion_rs::Value::Struct(inner) => inner.into_partiql_value(),
+        }
+    }
+}
+
+impl ToPartiqlValue<Element> for ion_rs::Int {
+    fn into_partiql_value(self) -> DatumLowerResult<PartiqlValueTarget<Element>> {
+        if let Some(n) = self.as_i64() {
+            Ok(Value::from(n).into())
+        } else {
+            let large = self.as_i128().expect("ion int i128");
+            Ok(Value::from(large).into())
+        }
+    }
+}
+
+impl ToPartiqlValue<Element> for ion_rs::Decimal {
+    fn into_partiql_value(self) -> DatumLowerResult<PartiqlValueTarget<Element>> {
+        let dec = ion_decimal_to_decimal(&self);
+        Ok(dec.expect("ion decimal").into())
+    }
+}
+
+impl ToPartiqlValue<Element> for ion_rs::Timestamp {
+    fn into_partiql_value(self) -> DatumLowerResult<PartiqlValueTarget<Element>> {
+        let ts = self;
+        let offset = ts.offset();
+        let datetime = DateTime::from_ymdhms_nano_offset_minutes(
+            ts.year() as i32,
+            NonZeroU8::new(ts.month() as u8).ok_or(IonDecodeError::ConversionError(
+                "month outside of range".into(),
+            ))?,
+            ts.day() as u8,
+            ts.hour() as u8,
+            ts.minute() as u8,
+            ts.second() as u8,
+            ts.nanoseconds(),
+            offset,
+        );
+        Ok(datetime.into())
+    }
+}
+
+impl ToPartiqlValue<Element> for ion_rs::Symbol {
+    fn into_partiql_value(self) -> DatumLowerResult<PartiqlValueTarget<Element>> {
+        Ok(self.expect_text()?.into())
+    }
+}
+
+impl ToPartiqlValue<Element> for ion_rs::Str {
+    fn into_partiql_value(self) -> DatumLowerResult<PartiqlValueTarget<Element>> {
+        Ok(self.text().into())
+    }
+}
+
+impl ToPartiqlValue<Element> for ion_rs::Bytes {
+    fn into_partiql_value(self) -> DatumLowerResult<PartiqlValueTarget<Element>> {
+        Ok(Value::Blob(Box::new(self.into_bytes())).into())
+    }
+}
+
+impl ToPartiqlValue<Element> for ion_rs::Sequence {
+    fn into_partiql_value(self) -> DatumLowerResult<PartiqlValueTarget<Element>> {
+        Ok(PartiqlValueTarget::List(
+            self.into_iter().collect::<Vec<_>>(),
+        ))
+    }
+}
+
+impl ToPartiqlValue<Element> for ion_rs::Struct {
+    fn into_partiql_value(self) -> DatumLowerResult<PartiqlValueTarget<Element>> {
+        let data: IonResult<Vec<_>> = self
+            .into_iter()
+            .map(|(sym, elt)| sym.expect_text().map(String::from).map(|s| (s, elt)))
+            .collect();
+        Ok(PartiqlValueTarget::Struct(data?))
+    }
+}
+
+fn ion_decimal_to_decimal(ion_dec: &Decimal) -> Result<rust_decimal::Decimal, rust_decimal::Error> {
+    // TODO ion Decimal doesn't give a lot of functionality to get at the data currently
+    // TODO    and it's not clear whether we'll continue with rust decimal or switch to big decimal
+    let ion_dec_str = format!("{ion_dec}").replace('d', "e");
+    rust_decimal::Decimal::from_str(&ion_dec_str)
+        .or_else(|_| rust_decimal::Decimal::from_scientific(&ion_dec_str))
+}

partiql-eval/src/eval/eval_expr_wrapper.rs

@@ -14,7 +14,7 @@ use std::hash::Hash;
 
 use std::marker::PhantomData;
 
-use partiql_value::datum::{DatumCategory, DatumCategoryRef, DatumValueRef};
+use partiql_value::datum::{DatumCategory, DatumCategoryRef, DatumLower, DatumValueRef};
 use std::ops::ControlFlow;
 
 trait TypeSatisfier {
@@ -47,6 +47,12 @@ impl TypeSatisfier for Static {
                             | (Static::DateTime, Value::DateTime(_))
                     )
                 }
+                DatumValueRef::Lower(lower) => {
+                    // TODO this basically clones just to throw the value away after type-check; fix that
+                    let lowered = lower.lower();
+                    let lowered = lowered.expect("lower");
+                    self.satisfies(&lowered)
+                }
             },
             (StaticCategory::Sequence(shape), DatumCategoryRef::Sequence(seq)) => match shape {
                 PartiqlShape::Dynamic | PartiqlShape::Undefined => true,
@@ -340,6 +346,11 @@ where
     for (idx, arg) in args.iter().enumerate() {
         let typ = types(idx);
         let arg = arg.evaluate(bindings, ctx);
+        let arg = match arg {
+            Cow::Borrowed(arg) => arg.lower(),
+            Cow::Owned(arg) => arg.into_lower().map(Cow::Owned),
+        }
+        .expect("lowering failed"); // TODO proper error messaging for lowering
         match ArgC::arg_check(typ, arg) {
             ArgCheckControlFlow::Continue(v) => {
                 if propagate.is_none() {

partiql-eval/src/eval/expr/operators.rs

@@ -19,9 +19,7 @@ use std::fmt::{Debug, Formatter};
 
 use std::marker::PhantomData;
 
-use partiql_value::datum::{
-    DatumCategory, DatumCategoryRef, DatumLowerResult, DatumValue, SequenceDatum, TupleDatum,
-};
+use partiql_value::datum::{DatumCategory, DatumCategoryRef, SequenceDatum, TupleDatum};
 use std::ops::ControlFlow;
 
 /// Represents a literal in (sub)query, e.g. `1` in `a + 1`.

partiql-eval/src/plan.rs

@@ -25,7 +25,7 @@ use crate::eval::expr::{
 use crate::eval::EvalPlan;
 use partiql_catalog::catalog::{Catalog, FunctionEntryFunction};
 use partiql_extension_ion::boxed_ion::BoxedIonType;
-use partiql_value::boxed_variant::{BoxedVariantResult, DynBoxedVariantTypeFactory};
+use partiql_value::boxed_variant::DynBoxedVariantTypeFactory;
 use partiql_value::{Bag, List, Tuple, Value, Variant};
 
 #[macro_export]
@@ -398,7 +398,7 @@ impl<'c> EvaluatorPlanner<'c> {
             ),
             ValueExpr::Lit(lit) => (
                 "literal",
-                match self.plan_lit(lit.as_ref()) {
+                match plan_lit(lit.as_ref()) {
                     Ok(lit) => EvalLitExpr::new(lit).bind::<{ STRICT }>(vec![]),
                     Err(e) => Ok(self.err(e) as Box<dyn EvalExpr>),
                 },
@@ -781,43 +781,43 @@ impl<'c> EvaluatorPlanner<'c> {
 
         self.unwrap_bind(name, bind)
     }
+}
 
-    fn plan_lit(&self, lit: &Lit) -> Result<Value, PlanningError> {
-        let lit_to_val = |lit| self.plan_lit(lit);
-        Ok(match lit {
-            Lit::Null => Value::Null,
-            Lit::Missing => Value::Missing,
-            Lit::Int8(n) => Value::from(*n),
-            Lit::Int16(n) => Value::from(*n),
-            Lit::Int32(n) => Value::from(*n),
-            Lit::Int64(n) => Value::from(*n),
-            Lit::Decimal(d) => Value::from(*d),
-            Lit::Double(f) => Value::from(*f),
-            Lit::Bool(b) => Value::from(*b),
-            Lit::String(s) => Value::from(s.as_ref()),
-            Lit::BoxDocument(contents, _typ) => {
-                let ion_typ = BoxedIonType::default().to_dyn_type_tag();
-                let variant = Variant::new(contents.clone(), ion_typ)
-                    .map_err(|e| PlanningError::IllegalState(e.to_string()));
-                Value::from(variant?)
-            }
-            Lit::Struct(strct) => strct
-                .into_iter()
-                .map(|(k, v)| lit_to_val(v).map(move |v| (k, v)))
-                .collect::<Result<Tuple, _>>()?
-                .into(),
-            Lit::Bag(bag) => bag
-                .into_iter()
-                .map(lit_to_val)
-                .collect::<Result<Bag, _>>()?
-                .into(),
-            Lit::List(list) => list
-                .into_iter()
-                .map(lit_to_val)
-                .collect::<Result<List, _>>()?
-                .into(),
-        })
-    }
+fn plan_lit(lit: &Lit) -> Result<Value, PlanningError> {
+    let lit_to_val = |lit| plan_lit(lit);
+    Ok(match lit {
+        Lit::Null => Value::Null,
+        Lit::Missing => Value::Missing,
+        Lit::Int8(n) => Value::from(*n),
+        Lit::Int16(n) => Value::from(*n),
+        Lit::Int32(n) => Value::from(*n),
+        Lit::Int64(n) => Value::from(*n),
+        Lit::Decimal(d) => Value::from(*d),
+        Lit::Double(f) => Value::from(*f),
+        Lit::Bool(b) => Value::from(*b),
+        Lit::String(s) => Value::from(s.as_ref()),
+        Lit::BoxDocument(contents, _typ) => {
+            let ion_typ = BoxedIonType::default().to_dyn_type_tag();
+            let variant = Variant::new(contents.clone(), ion_typ)
+                .map_err(|e| PlanningError::IllegalState(e.to_string()));
+            Value::from(variant?)
+        }
+        Lit::Struct(strct) => strct
+            .iter()
+            .map(|(k, v)| lit_to_val(v).map(move |v| (k, v)))
+            .collect::<Result<Tuple, _>>()?
+            .into(),
+        Lit::Bag(bag) => bag
+            .iter()
+            .map(lit_to_val)
+            .collect::<Result<Bag, _>>()?
+            .into(),
+        Lit::List(list) => list
+            .iter()
+            .map(lit_to_val)
+            .collect::<Result<List, _>>()?
+            .into(),
+    })
 }
 
 #[cfg(test)]

partiql-value/Cargo.toml

@@ -22,6 +22,7 @@ bench = false
 
 [dependencies]
 partiql-common = { path = "../partiql-common", version = "0.11.*" }
+partiql-types = { path = "../partiql-types", version = "0.11.*" }
 delegate = "0.13"
 ordered-float = "4"
 itertools = "0.13"