diff --git a/datafusion-examples/examples/advanced_udf.rs b/datafusion-examples/examples/advanced_udf.rs index 9a3ee9c8ebcd..22d37043e473 100644 --- a/datafusion-examples/examples/advanced_udf.rs +++ b/datafusion-examples/examples/advanced_udf.rs @@ -96,8 +96,8 @@ impl ScalarUDFImpl for PowUdf { // function, but we check again to make sure assert_eq!(args.len(), 2); let (base, exp) = (&args[0], &args[1]); - assert_eq!(base.data_type(), DataType::Float64); - assert_eq!(exp.data_type(), DataType::Float64); + assert_eq!(base.data_type(), &DataType::Float64); + assert_eq!(exp.data_type(), &DataType::Float64); match (base, exp) { // For demonstration purposes we also implement the scalar / scalar @@ -108,28 +108,31 @@ impl ScalarUDFImpl for PowUdf { // the DataFusion expression simplification logic will often invoke // this path once during planning, and simply use the result during // execution. - ( - ColumnarValue::Scalar(ScalarValue::Float64(base)), - ColumnarValue::Scalar(ScalarValue::Float64(exp)), - ) => { - // compute the output. Note DataFusion treats `None` as NULL. - let res = match (base, exp) { - (Some(base), Some(exp)) => Some(base.powf(*exp)), - // one or both arguments were NULL - _ => None, - }; - Ok(ColumnarValue::Scalar(ScalarValue::from(res))) + (ColumnarValue::Scalar(base), ColumnarValue::Scalar(exp)) => { + match (base.value(), exp.value()) { + (ScalarValue::Float64(base), ScalarValue::Float64(exp)) => { + // compute the output. Note DataFusion treats `None` as NULL. + let res = match (base, exp) { + (Some(base), Some(exp)) => Some(base.powf(*exp)), + // one or both arguments were NULL + _ => None, + }; + Ok(ColumnarValue::from(ScalarValue::from(res))) + } + _ => { + internal_err!("Invalid argument types to pow function") + } + } } // special case if the exponent is a constant - ( - ColumnarValue::Array(base_array), - ColumnarValue::Scalar(ScalarValue::Float64(exp)), - ) => { - let result_array = match exp { + (ColumnarValue::Array(base_array), ColumnarValue::Scalar(exp)) => { + let result_array = match exp.value() { // a ^ null = null - None => new_null_array(base_array.data_type(), base_array.len()), + ScalarValue::Float64(None) => { + new_null_array(base_array.data_type(), base_array.len()) + } // a ^ exp - Some(exp) => { + ScalarValue::Float64(Some(exp)) => { // DataFusion has ensured both arguments are Float64: let base_array = base_array.as_primitive::(); // calculate the result for every row. The `unary` @@ -139,24 +142,25 @@ impl ScalarUDFImpl for PowUdf { compute::unary(base_array, |base| base.powf(*exp)); Arc::new(res) } + _ => return internal_err!("Invalid argument types to pow function"), }; Ok(ColumnarValue::Array(result_array)) } // special case if the base is a constant (note this code is quite // similar to the previous case, so we omit comments) - ( - ColumnarValue::Scalar(ScalarValue::Float64(base)), - ColumnarValue::Array(exp_array), - ) => { - let res = match base { - None => new_null_array(exp_array.data_type(), exp_array.len()), - Some(base) => { + (ColumnarValue::Scalar(base), ColumnarValue::Array(exp_array)) => { + let res = match base.value() { + ScalarValue::Float64(None) => { + new_null_array(exp_array.data_type(), exp_array.len()) + } + ScalarValue::Float64(Some(base)) => { let exp_array = exp_array.as_primitive::(); let res: Float64Array = compute::unary(exp_array, |exp| base.powf(exp)); Arc::new(res) } + _ => return internal_err!("Invalid argument types to pow function"), }; Ok(ColumnarValue::Array(res)) } @@ -169,10 +173,6 @@ impl ScalarUDFImpl for PowUdf { )?; Ok(ColumnarValue::Array(Arc::new(res))) } - // if the types were not float, it is a bug in DataFusion - _ => { - internal_err!("Invalid argument types to pow function") - } } } diff --git a/datafusion-examples/examples/optimizer_rule.rs b/datafusion-examples/examples/optimizer_rule.rs index b4663b345f64..cf24a4b23eb5 100644 --- a/datafusion-examples/examples/optimizer_rule.rs +++ b/datafusion-examples/examples/optimizer_rule.rs @@ -207,7 +207,7 @@ impl ScalarUDFImpl for MyEq { fn invoke(&self, _args: &[ColumnarValue]) -> Result { // this example simply returns "true" which is not what a real // implementation would do. - Ok(ColumnarValue::Scalar(ScalarValue::from(true))) + Ok(ColumnarValue::from(ScalarValue::from(true))) } } diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rs index 3356a85fb6d4..530b2dd9bbf5 100644 --- a/datafusion/common/src/scalar/mod.rs +++ b/datafusion/common/src/scalar/mod.rs @@ -41,7 +41,7 @@ use crate::hash_utils::create_hashes; use crate::utils::{ array_into_fixed_size_list_array, array_into_large_list_array, array_into_list_array, }; -use arrow::compute::kernels::numeric::*; +use arrow::compute::kernels::{self, numeric::*}; use arrow::util::display::{array_value_to_string, ArrayFormatter, FormatOptions}; use arrow::{ array::*, @@ -1704,6 +1704,18 @@ impl ScalarValue { Some(sv) => sv.data_type(), }; + Self::iter_to_array_of_type(scalars, &data_type) + } + + /// Same as [`Self::iter_to_array`] but the target `data_type` can be + /// manually specified instead of being implicitly derived from the type of + /// the first value of the iterator. + pub fn iter_to_array_of_type( + scalars: impl IntoIterator, + data_type: &DataType, + ) -> Result { + let mut scalars = scalars.into_iter().peekable(); + /// Creates an array of $ARRAY_TY by unpacking values of /// SCALAR_TY for primitive types macro_rules! build_array_primitive { @@ -2179,6 +2191,16 @@ impl ScalarValue { Arc::new(array_into_large_list_array(values)) } + pub fn to_array_of_size_and_type( + &self, + size: usize, + target_type: &DataType, + ) -> Result { + let array = self.to_array_of_size(size)?; + let cast_array = kernels::cast::cast(&array, target_type)?; + Ok(cast_array) + } + /// Converts a scalar value into an array of `size` rows. /// /// # Errors diff --git a/datafusion/core/src/physical_optimizer/pruning.rs b/datafusion/core/src/physical_optimizer/pruning.rs index 9bc2bb1d1db9..3c8e5ddd1c74 100644 --- a/datafusion/core/src/physical_optimizer/pruning.rs +++ b/datafusion/core/src/physical_optimizer/pruning.rs @@ -687,14 +687,16 @@ impl BoolVecBuilder { ColumnarValue::Array(array) => { self.combine_array(array.as_boolean()); } - ColumnarValue::Scalar(ScalarValue::Boolean(Some(false))) => { - // False means all containers can not pass the predicate - self.inner = vec![false; self.inner.len()]; - } - _ => { - // Null or true means the rows in container may pass this - // conjunct so we can't prune any containers based on that - } + ColumnarValue::Scalar(scalar) => match scalar.value() { + ScalarValue::Boolean(Some(false)) => { + // False means all containers can not pass the predicate + self.inner = vec![false; self.inner.len()]; + } + _ => { + // Null or true means the rows in container may pass this + // conjunct so we can't prune any containers based on that + } + }, } } diff --git a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs index 013aec48d510..2f47d78db2d5 100644 --- a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs +++ b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs @@ -212,7 +212,7 @@ impl ScalarUDFImpl for Simple0ArgsScalarUDF { } fn invoke_no_args(&self, _number_rows: usize) -> Result { - Ok(ColumnarValue::Scalar(ScalarValue::Int32(Some(100)))) + Ok(ColumnarValue::from(ScalarValue::Int32(Some(100)))) } } @@ -323,7 +323,7 @@ async fn scalar_udf_override_built_in_scalar_function() -> Result<()> { vec![DataType::Int32], DataType::Int32, Volatility::Immutable, - Arc::new(move |_| Ok(ColumnarValue::Scalar(ScalarValue::Int32(Some(1))))), + Arc::new(move |_| Ok(ColumnarValue::from(ScalarValue::Int32(Some(1))))), )); // Make sure that the UDF is used instead of the built-in function @@ -669,7 +669,10 @@ impl ScalarUDFImpl for TakeUDF { // The actual implementation fn invoke(&self, args: &[ColumnarValue]) -> Result { let take_idx = match &args[2] { - ColumnarValue::Scalar(ScalarValue::Int64(Some(v))) if v < &2 => *v as usize, + ColumnarValue::Scalar(scalar) => match scalar.value() { + ScalarValue::Int64(Some(v)) if v < &2 => *v as usize, + _ => unreachable!(), + }, _ => unreachable!(), }; match &args[take_idx] { @@ -1070,11 +1073,12 @@ impl ScalarUDFImpl for MyRegexUdf { fn invoke(&self, args: &[ColumnarValue]) -> Result { match args { - [ColumnarValue::Scalar(ScalarValue::Utf8(value))] => { - Ok(ColumnarValue::Scalar(ScalarValue::Boolean( - self.matches(value.as_deref()), - ))) - } + [ColumnarValue::Scalar(scalar)] => match scalar.value() { + ScalarValue::Utf8(value) => Ok(ColumnarValue::from( + ScalarValue::Boolean(self.matches(value.as_deref())), + )), + _ => exec_err!("regex_udf only accepts a Utf8 arguments"), + }, [ColumnarValue::Array(values)] => { let mut builder = BooleanBuilder::with_capacity(values.len()); for value in values.as_string::() { @@ -1082,7 +1086,7 @@ impl ScalarUDFImpl for MyRegexUdf { } Ok(ColumnarValue::Array(Arc::new(builder.finish()))) } - _ => exec_err!("regex_udf only accepts a Utf8 arguments"), + _ => unreachable!(), } } diff --git a/datafusion/expr-common/src/columnar_value.rs b/datafusion/expr-common/src/columnar_value.rs index bfefb37c98d7..51113557335c 100644 --- a/datafusion/expr-common/src/columnar_value.rs +++ b/datafusion/expr-common/src/columnar_value.rs @@ -25,6 +25,8 @@ use datafusion_common::format::DEFAULT_CAST_OPTIONS; use datafusion_common::{internal_err, Result, ScalarValue}; use std::sync::Arc; +use crate::scalar::Scalar; + /// The result of evaluating an expression. /// /// [`ColumnarValue::Scalar`] represents a single value repeated any number of @@ -89,7 +91,7 @@ pub enum ColumnarValue { /// Array of values Array(ArrayRef), /// A single value - Scalar(ScalarValue), + Scalar(Scalar), } impl From for ColumnarValue { @@ -100,14 +102,14 @@ impl From for ColumnarValue { impl From for ColumnarValue { fn from(value: ScalarValue) -> Self { - ColumnarValue::Scalar(value) + ColumnarValue::Scalar(value.into()) } } impl ColumnarValue { - pub fn data_type(&self) -> DataType { + pub fn data_type(&self) -> &DataType { match self { - ColumnarValue::Array(array_value) => array_value.data_type().clone(), + ColumnarValue::Array(array_value) => array_value.data_type(), ColumnarValue::Scalar(scalar_value) => scalar_value.data_type(), } } @@ -195,9 +197,12 @@ impl ColumnarValue { kernels::cast::cast_with_options(array, cast_type, &cast_options)?, )), ColumnarValue::Scalar(scalar) => { + // TODO(@notfilippo, logical vs physical): if `scalar.data_type` is *logically equivalent* + // to `cast_type` then skip the kernel cast and only change the `data_type` of the scalar. + let scalar_array = if cast_type == &DataType::Timestamp(TimeUnit::Nanosecond, None) { - if let ScalarValue::Float64(Some(float_ts)) = scalar { + if let ScalarValue::Float64(Some(float_ts)) = scalar.value() { ScalarValue::Int64(Some( (float_ts * 1_000_000_000_f64).trunc() as i64, )) @@ -213,7 +218,7 @@ impl ColumnarValue { cast_type, &cast_options, )?; - let cast_scalar = ScalarValue::try_from_array(&cast_array, 0)?; + let cast_scalar = Scalar::try_from_array(&cast_array, 0)?; Ok(ColumnarValue::Scalar(cast_scalar)) } } @@ -250,7 +255,7 @@ mod tests { TestCase { input: vec![ ColumnarValue::Array(make_array(1, 3)), - ColumnarValue::Scalar(ScalarValue::Int32(Some(100))), + ColumnarValue::from(ScalarValue::Int32(Some(100))), ], expected: vec![ make_array(1, 3), @@ -260,7 +265,7 @@ mod tests { // scalar and array TestCase { input: vec![ - ColumnarValue::Scalar(ScalarValue::Int32(Some(100))), + ColumnarValue::from(ScalarValue::Int32(Some(100))), ColumnarValue::Array(make_array(1, 3)), ], expected: vec![ @@ -271,9 +276,9 @@ mod tests { // multiple scalars and array TestCase { input: vec![ - ColumnarValue::Scalar(ScalarValue::Int32(Some(100))), + ColumnarValue::from(ScalarValue::Int32(Some(100))), ColumnarValue::Array(make_array(1, 3)), - ColumnarValue::Scalar(ScalarValue::Int32(Some(200))), + ColumnarValue::from(ScalarValue::Int32(Some(200))), ], expected: vec![ make_array(100, 3), // scalar is expanded @@ -306,7 +311,7 @@ mod tests { fn values_to_arrays_mixed_length_and_scalar() { ColumnarValue::values_to_arrays(&[ ColumnarValue::Array(make_array(1, 3)), - ColumnarValue::Scalar(ScalarValue::Int32(Some(100))), + ColumnarValue::from(ScalarValue::Int32(Some(100))), ColumnarValue::Array(make_array(2, 7)), ]) .unwrap(); diff --git a/datafusion/expr-common/src/lib.rs b/datafusion/expr-common/src/lib.rs index 179dd75ace85..c0dde5c91932 100644 --- a/datafusion/expr-common/src/lib.rs +++ b/datafusion/expr-common/src/lib.rs @@ -31,6 +31,7 @@ pub mod columnar_value; pub mod groups_accumulator; pub mod interval_arithmetic; pub mod operator; +pub mod scalar; pub mod signature; pub mod sort_properties; pub mod type_coercion; diff --git a/datafusion/expr-common/src/scalar.rs b/datafusion/expr-common/src/scalar.rs new file mode 100644 index 000000000000..d4bb04e1f94c --- /dev/null +++ b/datafusion/expr-common/src/scalar.rs @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use arrow::{ + array::{Array, ArrayRef}, + datatypes::DataType, +}; +use datafusion_common::{exec_err, DataFusionError, Result, ScalarValue}; + +/// Represents a physical scalar value obtained after evaluating an expression. +/// +/// Currently [`Scalar`] represents a no-op wrapper around scalar value. This +/// is due to the fact that the current variants of [`ScalarValue`] correspond +/// 1:1 with the variants of [`DataType`]. The goal of this type is aid the +/// process of condensing some of the variants (e.g. [`ScalarValue::LargeUtf8`], +/// [`ScalarValue::Utf8View`], and [`ScalarValue::Utf8`]) into a single variant +/// (e.g. [`ScalarValue::Utf8`]). This type achieves this goal by storing the +/// physical type as [`DataType`] alongside the value represented as +/// [`ScalarValue`]. +/// +/// [`Scalar`] cannot be constructed directly as not all [`DataType`] variants +/// can correctly be represented by a [`ScalarValue`] without casting. For more +/// information see: +/// - [`Scalar::from`] +/// - [`Scalar::try_from_array`] +/// - [`Scalar::new_null_of`] +/// +/// [`Scalar`] is meant to be stored as a variant of [`crate::columnar_value::ColumnarValue`]. +#[derive(Clone, Debug)] +pub struct Scalar { + value: ScalarValue, + data_type: DataType, +} + +impl From for Scalar { + /// Converts a [`ScalarValue`] to a Scalar, resolving the [`DataType`] from + /// the value itself. + fn from(value: ScalarValue) -> Self { + Self { + data_type: value.data_type(), + value, + } + } +} + +impl TryFrom> for Scalar { + type Error = DataFusionError; + + /// Converts a [`arrow::array::Scalar`] to a Scalar, resolving the + /// [`DataType`] from the inner scalar array. + /// + /// See [`Scalar::try_from_array`]. + fn try_from(value: arrow::array::Scalar) -> Result { + let array = value.into_inner(); + Self::try_from_array(&array, 0) + } +} + +impl PartialEq for Scalar { + fn eq(&self, other: &Self) -> bool { + self.value.eq(&other.value) + } +} + +impl Scalar { + /// Converts a value in `array` at `index` into a Scalar, resolving the + /// [`DataType`] from the `array`'s data type. + /// + /// Once [`ScalarValue`] variants will be condensed, this method will prove + /// very useful in order to keep track of the wanted data type (sourced from + /// `array`) as the type information might be lost when simply calling + /// [`ScalarValue::try_from_array`]. + pub fn try_from_array(array: &dyn Array, index: usize) -> Result { + let data_type = array.data_type().clone(); + let value = ScalarValue::try_from_array(array, index)?; + Ok(Self { value, data_type }) + } + + /// Creates a null Scalar of the specified `data_type`. + /// + /// Note that the null `value` stored inside the Scalar might be of a + /// logically equivalent type of `data_type`, but the type information + /// provided will not be lost as it will be stored alongside the value. + pub fn new_null_of(data_type: DataType) -> Result { + Ok(Self { + value: ScalarValue::try_from(&data_type)?, + data_type, + }) + } + + /// Converts an iterator of references [`Scalar`] into an [`ArrayRef`] + /// corresponding to those values. + pub fn iter_to_array(scalars: impl IntoIterator) -> Result { + let mut scalars = scalars.into_iter().peekable(); + + // figure out the type based on the first element + let data_type = match scalars.peek() { + None => return exec_err!("Empty iterator passed to Scalar::iter_to_array"), + Some(sv) => sv.data_type().clone(), + }; + + ScalarValue::iter_to_array_of_type(scalars.map(|scalar| scalar.value), &data_type) + } + + #[inline] + pub fn value(&self) -> &ScalarValue { + &self.value + } + + #[inline] + pub fn into_value(self) -> ScalarValue { + self.value + } + + pub fn data_type(&self) -> &DataType { + &self.data_type + } + + #[inline] + pub fn to_array_of_size(&self, size: usize) -> Result { + self.value.to_array_of_size_and_type(size, &self.data_type) + } + + #[inline] + pub fn to_array(&self) -> Result { + self.to_array_of_size(1) + } + + pub fn to_scalar(&self) -> Result> { + Ok(arrow::array::Scalar::new(self.to_array()?)) + } +} diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs index db0bfd6b1bc2..c033237a54a4 100644 --- a/datafusion/expr/src/expr.rs +++ b/datafusion/expr/src/expr.rs @@ -2530,7 +2530,7 @@ mod test { } fn invoke(&self, _args: &[ColumnarValue]) -> Result { - Ok(ColumnarValue::Scalar(ScalarValue::from("a"))) + Ok(ColumnarValue::from(ScalarValue::from("a"))) } } let udf = Arc::new(ScalarUDF::from(TestScalarUDF { diff --git a/datafusion/expr/src/lib.rs b/datafusion/expr/src/lib.rs index 260065f69af9..95baa6f33bd5 100644 --- a/datafusion/expr/src/lib.rs +++ b/datafusion/expr/src/lib.rs @@ -71,6 +71,7 @@ pub use datafusion_expr_common::accumulator::Accumulator; pub use datafusion_expr_common::columnar_value::ColumnarValue; pub use datafusion_expr_common::groups_accumulator::{EmitTo, GroupsAccumulator}; pub use datafusion_expr_common::operator::Operator; +pub use datafusion_expr_common::scalar::Scalar; pub use datafusion_expr_common::signature::{ ArrayFunctionSignature, Signature, TypeSignature, Volatility, TIMEZONE_WILDCARD, }; diff --git a/datafusion/functions-aggregate/src/approx_percentile_cont.rs b/datafusion/functions-aggregate/src/approx_percentile_cont.rs index 5578aebbf403..51d9ac764c40 100644 --- a/datafusion/functions-aggregate/src/approx_percentile_cont.rs +++ b/datafusion/functions-aggregate/src/approx_percentile_cont.rs @@ -147,7 +147,7 @@ fn get_scalar_value(expr: &Arc) -> Result { let empty_schema = Arc::new(Schema::empty()); let batch = RecordBatch::new_empty(Arc::clone(&empty_schema)); if let ColumnarValue::Scalar(s) = expr.evaluate(&batch)? { - Ok(s) + Ok(s.into_value()) } else { internal_err!("Didn't expect ColumnarValue::Array") } diff --git a/datafusion/functions-nested/benches/map.rs b/datafusion/functions-nested/benches/map.rs index ca23d8b7ff4c..24e892f8b715 100644 --- a/datafusion/functions-nested/benches/map.rs +++ b/datafusion/functions-nested/benches/map.rs @@ -91,8 +91,8 @@ fn criterion_benchmark(c: &mut Criterion) { Arc::new(Int32Array::from(values(&mut rng))), None, ); - let keys = ColumnarValue::Scalar(ScalarValue::List(Arc::new(key_list))); - let values = ColumnarValue::Scalar(ScalarValue::List(Arc::new(value_list))); + let keys = ColumnarValue::from(ScalarValue::List(Arc::new(key_list))); + let values = ColumnarValue::from(ScalarValue::List(Arc::new(value_list))); b.iter(|| { black_box( diff --git a/datafusion/functions-nested/src/array_has.rs b/datafusion/functions-nested/src/array_has.rs index 8f8d123bf5f9..df1a336426d7 100644 --- a/datafusion/functions-nested/src/array_has.rs +++ b/datafusion/functions-nested/src/array_has.rs @@ -106,8 +106,8 @@ impl ScalarUDFImpl for ArrayHas { ColumnarValue::Scalar(scalar_needle) => { // Always return null if the second argument is null // i.e. array_has(array, null) -> null - if scalar_needle.is_null() { - return Ok(ColumnarValue::Scalar(ScalarValue::Boolean(None))); + if scalar_needle.value().is_null() { + return Ok(ColumnarValue::from(ScalarValue::Boolean(None))); } // since the needle is a scalar, convert it to an array of size 1 @@ -118,7 +118,7 @@ impl ScalarUDFImpl for ArrayHas { if let ColumnarValue::Scalar(_) = &args[0] { // If both inputs are scalar, keeps output as scalar let scalar_value = ScalarValue::try_from_array(&array, 0)?; - Ok(ColumnarValue::Scalar(scalar_value)) + Ok(ColumnarValue::from(scalar_value)) } else { Ok(ColumnarValue::Array(array)) } diff --git a/datafusion/functions-nested/src/map.rs b/datafusion/functions-nested/src/map.rs index 29afe4a7f3be..1a5eba2d7064 100644 --- a/datafusion/functions-nested/src/map.rs +++ b/datafusion/functions-nested/src/map.rs @@ -111,7 +111,7 @@ fn check_unique_keys(array: &dyn Array) -> Result<()> { fn get_first_array_ref(columnar_value: &ColumnarValue) -> Result { match columnar_value { - ColumnarValue::Scalar(value) => match value { + ColumnarValue::Scalar(value) => match value.value() { ScalarValue::List(array) => Ok(array.value(0)), ScalarValue::LargeList(array) => Ok(array.value(0)), ScalarValue::FixedSizeList(array) => Ok(array.value(0)), @@ -125,7 +125,7 @@ fn make_map_batch_internal( keys: ArrayRef, values: ArrayRef, can_evaluate_to_const: bool, - data_type: DataType, + data_type: &DataType, ) -> Result { if keys.len() != values.len() { return exec_err!("map requires key and value lists to have the same length"); @@ -172,7 +172,7 @@ fn make_map_batch_internal( let map_array = Arc::new(MapArray::from(map_data)); Ok(if can_evaluate_to_const { - ColumnarValue::Scalar(ScalarValue::try_from_array(map_array.as_ref(), 0)?) + ColumnarValue::from(ScalarValue::try_from_array(map_array.as_ref(), 0)?) } else { ColumnarValue::Array(map_array) }) diff --git a/datafusion/functions-nested/src/utils.rs b/datafusion/functions-nested/src/utils.rs index b9a75724bcde..5382346a0a81 100644 --- a/datafusion/functions-nested/src/utils.rs +++ b/datafusion/functions-nested/src/utils.rs @@ -85,7 +85,7 @@ where if is_scalar { // If all inputs are scalar, keeps output as scalar let result = result.and_then(|arr| ScalarValue::try_from_array(&arr, 0)); - result.map(ColumnarValue::Scalar) + result.map(ColumnarValue::from) } else { result.map(ColumnarValue::Array) } diff --git a/datafusion/functions/benches/concat.rs b/datafusion/functions/benches/concat.rs index 91c46ac775a8..bd3bc31b0c65 100644 --- a/datafusion/functions/benches/concat.rs +++ b/datafusion/functions/benches/concat.rs @@ -28,7 +28,7 @@ fn create_args(size: usize, str_len: usize) -> Vec { let scalar = ScalarValue::Utf8(Some(", ".to_string())); vec![ ColumnarValue::Array(Arc::clone(&array) as ArrayRef), - ColumnarValue::Scalar(scalar), + ColumnarValue::from(scalar), ColumnarValue::Array(array), ] } diff --git a/datafusion/functions/benches/date_bin.rs b/datafusion/functions/benches/date_bin.rs index c881947354fd..7a92037ccc5d 100644 --- a/datafusion/functions/benches/date_bin.rs +++ b/datafusion/functions/benches/date_bin.rs @@ -40,7 +40,7 @@ fn timestamps(rng: &mut ThreadRng) -> TimestampSecondArray { fn criterion_benchmark(c: &mut Criterion) { c.bench_function("date_bin_1000", |b| { let mut rng = rand::thread_rng(); - let interval = ColumnarValue::Scalar(ScalarValue::new_interval_dt(0, 1_000_000)); + let interval = ColumnarValue::from(ScalarValue::new_interval_dt(0, 1_000_000)); let timestamps = ColumnarValue::Array(Arc::new(timestamps(&mut rng)) as ArrayRef); let udf = date_bin(); diff --git a/datafusion/functions/benches/ltrim.rs b/datafusion/functions/benches/ltrim.rs index b3fa5ef4fdff..9d3f98fa9822 100644 --- a/datafusion/functions/benches/ltrim.rs +++ b/datafusion/functions/benches/ltrim.rs @@ -119,7 +119,7 @@ fn create_args( ); vec![ ColumnarValue::Array(string_array), - ColumnarValue::Scalar(pattern), + ColumnarValue::from(pattern), ] } diff --git a/datafusion/functions/benches/make_date.rs b/datafusion/functions/benches/make_date.rs index cb8f1abe6d5d..a865953897eb 100644 --- a/datafusion/functions/benches/make_date.rs +++ b/datafusion/functions/benches/make_date.rs @@ -72,7 +72,7 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("make_date_scalar_col_col_1000", |b| { let mut rng = rand::thread_rng(); - let year = ColumnarValue::Scalar(ScalarValue::Int32(Some(2025))); + let year = ColumnarValue::from(ScalarValue::Int32(Some(2025))); let months = ColumnarValue::Array(Arc::new(months(&mut rng)) as ArrayRef); let days = ColumnarValue::Array(Arc::new(days(&mut rng)) as ArrayRef); @@ -87,8 +87,8 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("make_date_scalar_scalar_col_1000", |b| { let mut rng = rand::thread_rng(); - let year = ColumnarValue::Scalar(ScalarValue::Int32(Some(2025))); - let month = ColumnarValue::Scalar(ScalarValue::Int32(Some(11))); + let year = ColumnarValue::from(ScalarValue::Int32(Some(2025))); + let month = ColumnarValue::from(ScalarValue::Int32(Some(11))); let days = ColumnarValue::Array(Arc::new(days(&mut rng)) as ArrayRef); b.iter(|| { @@ -101,9 +101,9 @@ fn criterion_benchmark(c: &mut Criterion) { }); c.bench_function("make_date_scalar_scalar_scalar", |b| { - let year = ColumnarValue::Scalar(ScalarValue::Int32(Some(2025))); - let month = ColumnarValue::Scalar(ScalarValue::Int32(Some(11))); - let day = ColumnarValue::Scalar(ScalarValue::Int32(Some(26))); + let year = ColumnarValue::from(ScalarValue::Int32(Some(2025))); + let month = ColumnarValue::from(ScalarValue::Int32(Some(11))); + let day = ColumnarValue::from(ScalarValue::Int32(Some(26))); b.iter(|| { black_box( diff --git a/datafusion/functions/benches/nullif.rs b/datafusion/functions/benches/nullif.rs index dfabad335835..31192c1a749f 100644 --- a/datafusion/functions/benches/nullif.rs +++ b/datafusion/functions/benches/nullif.rs @@ -29,7 +29,7 @@ fn criterion_benchmark(c: &mut Criterion) { for size in [1024, 4096, 8192] { let array = Arc::new(create_string_array_with_len::(size, 0.2, 32)); let args = vec![ - ColumnarValue::Scalar(ScalarValue::Utf8(Some("abcd".to_string()))), + ColumnarValue::from(ScalarValue::Utf8(Some("abcd".to_string()))), ColumnarValue::Array(array), ]; c.bench_function(&format!("nullif scalar array: {}", size), |b| { diff --git a/datafusion/functions/benches/to_char.rs b/datafusion/functions/benches/to_char.rs index d9a153e64abc..14819c90a7a5 100644 --- a/datafusion/functions/benches/to_char.rs +++ b/datafusion/functions/benches/to_char.rs @@ -98,7 +98,7 @@ fn criterion_benchmark(c: &mut Criterion) { let mut rng = rand::thread_rng(); let data = ColumnarValue::Array(Arc::new(data(&mut rng)) as ArrayRef); let patterns = - ColumnarValue::Scalar(ScalarValue::Utf8(Some("%Y-%m-%d".to_string()))); + ColumnarValue::from(ScalarValue::Utf8(Some("%Y-%m-%d".to_string()))); b.iter(|| { black_box( @@ -118,10 +118,9 @@ fn criterion_benchmark(c: &mut Criterion) { .and_utc() .timestamp_nanos_opt() .unwrap(); - let data = ColumnarValue::Scalar(TimestampNanosecond(Some(timestamp), None)); - let pattern = ColumnarValue::Scalar(ScalarValue::Utf8(Some( - "%d-%m-%Y %H:%M:%S".to_string(), - ))); + let data = ColumnarValue::from(TimestampNanosecond(Some(timestamp), None)); + let pattern = + ColumnarValue::from(ScalarValue::Utf8(Some("%d-%m-%Y %H:%M:%S".to_string()))); b.iter(|| { black_box( diff --git a/datafusion/functions/src/core/arrowtypeof.rs b/datafusion/functions/src/core/arrowtypeof.rs index cc5e7e619bd8..dd502fb2686d 100644 --- a/datafusion/functions/src/core/arrowtypeof.rs +++ b/datafusion/functions/src/core/arrowtypeof.rs @@ -65,7 +65,7 @@ impl ScalarUDFImpl for ArrowTypeOfFunc { } let input_data_type = args[0].data_type(); - Ok(ColumnarValue::Scalar(ScalarValue::from(format!( + Ok(ColumnarValue::from(ScalarValue::from(format!( "{input_data_type}" )))) } diff --git a/datafusion/functions/src/core/coalesce.rs b/datafusion/functions/src/core/coalesce.rs index 19db58c181e7..245697dc83e3 100644 --- a/datafusion/functions/src/core/coalesce.rs +++ b/datafusion/functions/src/core/coalesce.rs @@ -86,7 +86,7 @@ impl ScalarUDFImpl for CoalesceFunc { if let Some(size) = return_array.next() { // start with nulls as default output - let mut current_value = new_null_array(&return_type, size); + let mut current_value = new_null_array(return_type, size); let mut remainder = BooleanArray::from(vec![true; size]); for arg in args { @@ -97,7 +97,7 @@ impl ScalarUDFImpl for CoalesceFunc { remainder = and(&remainder, &is_null(array)?)?; } ColumnarValue::Scalar(value) => { - if value.is_null() { + if value.value().is_null() { continue; } else { let last_value = value.to_scalar()?; @@ -115,7 +115,7 @@ impl ScalarUDFImpl for CoalesceFunc { let result = args .iter() .filter_map(|x| match x { - ColumnarValue::Scalar(s) if !s.is_null() => Some(x.clone()), + ColumnarValue::Scalar(s) if !s.value().is_null() => Some(x.clone()), _ => None, }) .next() diff --git a/datafusion/functions/src/core/getfield.rs b/datafusion/functions/src/core/getfield.rs index a51f895c5084..6f863809573b 100644 --- a/datafusion/functions/src/core/getfield.rs +++ b/datafusion/functions/src/core/getfield.rs @@ -168,14 +168,14 @@ impl ScalarUDFImpl for GetFieldFunc { } if args[0].data_type().is_null() { - return Ok(ColumnarValue::Scalar(ScalarValue::Null)); + return Ok(ColumnarValue::from(ScalarValue::Null)); } let arrays = ColumnarValue::values_to_arrays(args)?; let array = Arc::clone(&arrays[0]); let name = match &args[1] { - ColumnarValue::Scalar(name) => name, + ColumnarValue::Scalar(name) => name.value(), _ => { return exec_err!( "get_field function requires the argument field_name to be a string" @@ -227,7 +227,7 @@ impl ScalarUDFImpl for GetFieldFunc { "get indexed field is only possible on struct with utf8 indexes. \ Tried with {name:?} index" ), - (DataType::Null, _) => Ok(ColumnarValue::Scalar(ScalarValue::Null)), + (DataType::Null, _) => Ok(ColumnarValue::from(ScalarValue::Null)), (dt, name) => exec_err!( "get indexed field is only possible on lists with int64 indexes or struct \ with utf8 indexes. Tried {dt:?} with {name:?} index" diff --git a/datafusion/functions/src/core/named_struct.rs b/datafusion/functions/src/core/named_struct.rs index 85c332745355..9c3a3664ca32 100644 --- a/datafusion/functions/src/core/named_struct.rs +++ b/datafusion/functions/src/core/named_struct.rs @@ -47,7 +47,10 @@ fn named_struct_expr(args: &[ColumnarValue]) -> Result { let name_column = &chunk[0]; let name = match name_column { - ColumnarValue::Scalar(ScalarValue::Utf8(Some(name_scalar))) => name_scalar, + ColumnarValue::Scalar(scalar) => match scalar.value() { + ScalarValue::Utf8(Some(name_scalar)) => name_scalar, + _ => return exec_err!("named_struct even arguments must be string literals, got {name_column:?} instead at position {}", i * 2) + } _ => return exec_err!("named_struct even arguments must be string literals, got {name_column:?} instead at position {}", i * 2) }; diff --git a/datafusion/functions/src/core/nullif.rs b/datafusion/functions/src/core/nullif.rs index 6fcfbd36416e..9f95392c37bf 100644 --- a/datafusion/functions/src/core/nullif.rs +++ b/datafusion/functions/src/core/nullif.rs @@ -133,10 +133,10 @@ fn nullif_func(args: &[ColumnarValue]) -> Result { (ColumnarValue::Scalar(lhs), ColumnarValue::Scalar(rhs)) => { let val: ScalarValue = match lhs.eq(rhs) { true => lhs.data_type().try_into()?, - false => lhs.clone(), + false => lhs.value().clone(), }; - Ok(ColumnarValue::Scalar(val)) + Ok(ColumnarValue::from(val)) } } } @@ -164,7 +164,7 @@ mod tests { ]); let a = ColumnarValue::Array(Arc::new(a)); - let lit_array = ColumnarValue::Scalar(ScalarValue::Int32(Some(2i32))); + let lit_array = ColumnarValue::from(ScalarValue::Int32(Some(2i32))); let result = nullif_func(&[a, lit_array])?; let result = result.into_array(0).expect("Failed to convert to array"); @@ -190,7 +190,7 @@ mod tests { let a = Int32Array::from(vec![1, 3, 10, 7, 8, 1, 2, 4, 5]); let a = ColumnarValue::Array(Arc::new(a)); - let lit_array = ColumnarValue::Scalar(ScalarValue::Int32(Some(1i32))); + let lit_array = ColumnarValue::from(ScalarValue::Int32(Some(1i32))); let result = nullif_func(&[a, lit_array])?; let result = result.into_array(0).expect("Failed to convert to array"); @@ -215,7 +215,7 @@ mod tests { let a = BooleanArray::from(vec![Some(true), Some(false), None]); let a = ColumnarValue::Array(Arc::new(a)); - let lit_array = ColumnarValue::Scalar(ScalarValue::Boolean(Some(false))); + let lit_array = ColumnarValue::from(ScalarValue::Boolean(Some(false))); let result = nullif_func(&[a, lit_array])?; let result = result.into_array(0).expect("Failed to convert to array"); @@ -232,7 +232,7 @@ mod tests { let a = StringArray::from(vec![Some("foo"), Some("bar"), None, Some("baz")]); let a = ColumnarValue::Array(Arc::new(a)); - let lit_array = ColumnarValue::Scalar(ScalarValue::from("bar")); + let lit_array = ColumnarValue::from(ScalarValue::from("bar")); let result = nullif_func(&[a, lit_array])?; let result = result.into_array(0).expect("Failed to convert to array"); @@ -253,7 +253,7 @@ mod tests { let a = Int32Array::from(vec![Some(1), Some(2), None, None, Some(3), Some(4)]); let a = ColumnarValue::Array(Arc::new(a)); - let lit_array = ColumnarValue::Scalar(ScalarValue::Int32(Some(2i32))); + let lit_array = ColumnarValue::from(ScalarValue::Int32(Some(2i32))); let result = nullif_func(&[lit_array, a])?; let result = result.into_array(0).expect("Failed to convert to array"); @@ -272,8 +272,8 @@ mod tests { #[test] fn nullif_scalar() -> Result<()> { - let a_eq = ColumnarValue::Scalar(ScalarValue::Int32(Some(2i32))); - let b_eq = ColumnarValue::Scalar(ScalarValue::Int32(Some(2i32))); + let a_eq = ColumnarValue::from(ScalarValue::Int32(Some(2i32))); + let b_eq = ColumnarValue::from(ScalarValue::Int32(Some(2i32))); let result_eq = nullif_func(&[a_eq, b_eq])?; let result_eq = result_eq.into_array(1).expect("Failed to convert to array"); @@ -282,8 +282,8 @@ mod tests { assert_eq!(expected_eq.as_ref(), result_eq.as_ref()); - let a_neq = ColumnarValue::Scalar(ScalarValue::Int32(Some(2i32))); - let b_neq = ColumnarValue::Scalar(ScalarValue::Int32(Some(1i32))); + let a_neq = ColumnarValue::from(ScalarValue::Int32(Some(2i32))); + let b_neq = ColumnarValue::from(ScalarValue::Int32(Some(1i32))); let result_neq = nullif_func(&[a_neq, b_neq])?; let result_neq = result_neq diff --git a/datafusion/functions/src/core/nvl.rs b/datafusion/functions/src/core/nvl.rs index a09224acefcd..09d86ddfc0d9 100644 --- a/datafusion/functions/src/core/nvl.rs +++ b/datafusion/functions/src/core/nvl.rs @@ -112,7 +112,7 @@ fn nvl_func(args: &[ColumnarValue]) -> Result { } (ColumnarValue::Scalar(lhs), ColumnarValue::Scalar(rhs)) => { let mut current_value = lhs; - if lhs.is_null() { + if lhs.value().is_null() { current_value = rhs; } return Ok(ColumnarValue::Scalar(current_value.clone())); @@ -147,7 +147,7 @@ mod tests { ]); let a = ColumnarValue::Array(Arc::new(a)); - let lit_array = ColumnarValue::Scalar(ScalarValue::Int32(Some(6i32))); + let lit_array = ColumnarValue::from(ScalarValue::Int32(Some(6i32))); let result = nvl_func(&[a, lit_array])?; let result = result.into_array(0).expect("Failed to convert to array"); @@ -173,7 +173,7 @@ mod tests { let a = Int32Array::from(vec![1, 3, 10, 7, 8, 1, 2, 4, 5]); let a = ColumnarValue::Array(Arc::new(a)); - let lit_array = ColumnarValue::Scalar(ScalarValue::Int32(Some(20i32))); + let lit_array = ColumnarValue::from(ScalarValue::Int32(Some(20i32))); let result = nvl_func(&[a, lit_array])?; let result = result.into_array(0).expect("Failed to convert to array"); @@ -198,7 +198,7 @@ mod tests { let a = BooleanArray::from(vec![Some(true), Some(false), None]); let a = ColumnarValue::Array(Arc::new(a)); - let lit_array = ColumnarValue::Scalar(ScalarValue::Boolean(Some(false))); + let lit_array = ColumnarValue::from(ScalarValue::Boolean(Some(false))); let result = nvl_func(&[a, lit_array])?; let result = result.into_array(0).expect("Failed to convert to array"); @@ -218,7 +218,7 @@ mod tests { let a = StringArray::from(vec![Some("foo"), Some("bar"), None, Some("baz")]); let a = ColumnarValue::Array(Arc::new(a)); - let lit_array = ColumnarValue::Scalar(ScalarValue::from("bax")); + let lit_array = ColumnarValue::from(ScalarValue::from("bax")); let result = nvl_func(&[a, lit_array])?; let result = result.into_array(0).expect("Failed to convert to array"); @@ -239,7 +239,7 @@ mod tests { let a = Int32Array::from(vec![Some(1), Some(2), None, None, Some(3), Some(4)]); let a = ColumnarValue::Array(Arc::new(a)); - let lit_array = ColumnarValue::Scalar(ScalarValue::Int32(Some(2i32))); + let lit_array = ColumnarValue::from(ScalarValue::Int32(Some(2i32))); let result = nvl_func(&[lit_array, a])?; let result = result.into_array(0).expect("Failed to convert to array"); @@ -258,8 +258,8 @@ mod tests { #[test] fn nvl_scalar() -> Result<()> { - let a_null = ColumnarValue::Scalar(ScalarValue::Int32(None)); - let b_null = ColumnarValue::Scalar(ScalarValue::Int32(Some(2i32))); + let a_null = ColumnarValue::from(ScalarValue::Int32(None)); + let b_null = ColumnarValue::from(ScalarValue::Int32(Some(2i32))); let result_null = nvl_func(&[a_null, b_null])?; let result_null = result_null @@ -270,8 +270,8 @@ mod tests { assert_eq!(expected_null.as_ref(), result_null.as_ref()); - let a_nnull = ColumnarValue::Scalar(ScalarValue::Int32(Some(2i32))); - let b_nnull = ColumnarValue::Scalar(ScalarValue::Int32(Some(1i32))); + let a_nnull = ColumnarValue::from(ScalarValue::Int32(Some(2i32))); + let b_nnull = ColumnarValue::from(ScalarValue::Int32(Some(1i32))); let result_nnull = nvl_func(&[a_nnull, b_nnull])?; let result_nnull = result_nnull diff --git a/datafusion/functions/src/core/nvl2.rs b/datafusion/functions/src/core/nvl2.rs index 1144dc0fb7c5..f3027925b26a 100644 --- a/datafusion/functions/src/core/nvl2.rs +++ b/datafusion/functions/src/core/nvl2.rs @@ -126,7 +126,7 @@ fn nvl2_func(args: &[ColumnarValue]) -> Result { internal_err!("except Scalar value, but got Array") } ColumnarValue::Scalar(scalar) => { - if scalar.is_null() { + if scalar.value().is_null() { current_value = &args[2]; } Ok(current_value.clone()) diff --git a/datafusion/functions/src/core/version.rs b/datafusion/functions/src/core/version.rs index 212349e68981..bc3bc1339aa0 100644 --- a/datafusion/functions/src/core/version.rs +++ b/datafusion/functions/src/core/version.rs @@ -76,7 +76,7 @@ impl ScalarUDFImpl for VersionFunc { std::env::consts::ARCH, std::env::consts::OS, ); - Ok(ColumnarValue::Scalar(ScalarValue::Utf8(Some(version)))) + Ok(ColumnarValue::from(ScalarValue::Utf8(Some(version)))) } } @@ -90,7 +90,11 @@ mod test { let version_udf = ScalarUDF::from(VersionFunc::new()); let version = version_udf.invoke_no_args(0).unwrap(); - if let ColumnarValue::Scalar(ScalarValue::Utf8(Some(version))) = version { + let ColumnarValue::Scalar(version) = version else { + panic!("Expected scalar version") + }; + + if let ScalarValue::Utf8(Some(version)) = version.value() { assert!(version.starts_with("Apache DataFusion")); } else { panic!("Expected version string"); diff --git a/datafusion/functions/src/crypto/basic.rs b/datafusion/functions/src/crypto/basic.rs index 716afd84a9c9..5c2390d4b952 100644 --- a/datafusion/functions/src/crypto/basic.rs +++ b/datafusion/functions/src/crypto/basic.rs @@ -120,7 +120,7 @@ pub fn digest(args: &[ColumnarValue]) -> Result { ); } let digest_algorithm = match &args[1] { - ColumnarValue::Scalar(scalar) => match scalar { + ColumnarValue::Scalar(scalar) => match scalar.value() { ScalarValue::Utf8(Some(method)) | ScalarValue::LargeUtf8(Some(method)) => { method.parse::() } @@ -191,10 +191,12 @@ pub fn md5(args: &[ColumnarValue]) -> Result { .collect(); ColumnarValue::Array(Arc::new(string_array)) } - ColumnarValue::Scalar(ScalarValue::Binary(opt)) => { - ColumnarValue::Scalar(ScalarValue::Utf8(opt.map(hex_encode::<_>))) - } - _ => return exec_err!("Impossibly got invalid results from digest"), + ColumnarValue::Scalar(scalar) => match scalar.into_value() { + ScalarValue::Binary(opt) => { + ColumnarValue::from(ScalarValue::Utf8(opt.map(hex_encode::<_>))) + } + _ => return exec_err!("Impossibly got invalid results from digest"), + }, }) } @@ -244,7 +246,7 @@ macro_rules! digest_to_array { impl DigestAlgorithm { /// digest an optional string to its hash value, null values are returned as is pub fn digest_scalar(self, value: Option<&[u8]>) -> ColumnarValue { - ColumnarValue::Scalar(match self { + ColumnarValue::from(match self { Self::Md5 => digest_to_scalar!(Md5, value), Self::Sha224 => digest_to_scalar!(Sha224, value), Self::Sha256 => digest_to_scalar!(Sha256, value), @@ -338,7 +340,7 @@ pub fn digest_process( "Unsupported data type {other:?} for function {digest_algorithm}" ), }, - ColumnarValue::Scalar(scalar) => match scalar { + ColumnarValue::Scalar(scalar) => match scalar.value() { ScalarValue::Utf8(a) | ScalarValue::LargeUtf8(a) => { Ok(digest_algorithm .digest_scalar(a.as_ref().map(|s: &String| s.as_bytes()))) diff --git a/datafusion/functions/src/datetime/common.rs b/datafusion/functions/src/datetime/common.rs index 89b40a3534d3..da1cc2f31c76 100644 --- a/datafusion/functions/src/datetime/common.rs +++ b/datafusion/functions/src/datetime/common.rs @@ -195,10 +195,10 @@ where ))), other => exec_err!("Unsupported data type {other:?} for function {name}"), }, - ColumnarValue::Scalar(scalar) => match scalar { + ColumnarValue::Scalar(scalar) => match scalar.value() { ScalarValue::Utf8(a) | ScalarValue::LargeUtf8(a) => { let result = a.as_ref().map(|x| (op)(x)).transpose()?; - Ok(ColumnarValue::Scalar(S::scalar(result))) + Ok(ColumnarValue::from(S::scalar(result))) } other => exec_err!("Unsupported data type {other:?} for function {name}"), }, @@ -252,7 +252,7 @@ where } }, // if the first argument is a scalar utf8 all arguments are expected to be scalar utf8 - ColumnarValue::Scalar(scalar) => match scalar { + ColumnarValue::Scalar(scalar) => match scalar.value() { ScalarValue::Utf8(a) | ScalarValue::LargeUtf8(a) => { let a = a.as_ref(); // ASK: Why do we trust `a` to be non-null at this point? @@ -261,9 +261,11 @@ where let mut ret = None; for (pos, v) in args.iter().enumerate().skip(1) { - let ColumnarValue::Scalar( - ScalarValue::Utf8(x) | ScalarValue::LargeUtf8(x), - ) = v + let ColumnarValue::Scalar(v) = v else { + return exec_err!("Expected scalar of data type {v:?} for function {name}, arg # {pos}"); + }; + + let (ScalarValue::Utf8(x) | ScalarValue::LargeUtf8(x)) = v.value() else { return exec_err!("Unsupported data type {v:?} for function {name}, arg # {pos}"); }; @@ -271,7 +273,7 @@ where if let Some(s) = x { match op(a.as_str(), s.as_str()) { Ok(r) => { - ret = Some(Ok(ColumnarValue::Scalar(S::scalar(Some( + ret = Some(Ok(ColumnarValue::from(S::scalar(Some( op2(r), ))))); break; @@ -328,7 +330,7 @@ where ColumnarValue::Array(a) => { Ok(Either::Left(as_generic_string_array::(a.as_ref())?)) } - ColumnarValue::Scalar(s) => match s { + ColumnarValue::Scalar(s) => match s.value() { ScalarValue::Utf8(a) | ScalarValue::LargeUtf8(a) => Ok(Either::Right(a)), other => exec_err!( "Unexpected scalar type encountered '{other}' for function '{name}'" diff --git a/datafusion/functions/src/datetime/date_bin.rs b/datafusion/functions/src/datetime/date_bin.rs index 997f1a36ad04..05e45327896a 100644 --- a/datafusion/functions/src/datetime/date_bin.rs +++ b/datafusion/functions/src/datetime/date_bin.rs @@ -135,7 +135,7 @@ impl ScalarUDFImpl for DateBinFunc { fn invoke(&self, args: &[ColumnarValue]) -> Result { if args.len() == 2 { // Default to unix EPOCH - let origin = ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( + let origin = ColumnarValue::from(ScalarValue::TimestampNanosecond( Some(0), Some("+00:00".into()), )); @@ -261,46 +261,48 @@ fn date_bin_impl( origin: &ColumnarValue, ) -> Result { let stride = match stride { - ColumnarValue::Scalar(ScalarValue::IntervalDayTime(Some(v))) => { - let (days, ms) = IntervalDayTimeType::to_parts(*v); - let nanos = (TimeDelta::try_days(days as i64).unwrap() - + TimeDelta::try_milliseconds(ms as i64).unwrap()) - .num_nanoseconds(); - - match nanos { - Some(v) => Interval::Nanoseconds(v), - _ => return exec_err!("DATE_BIN stride argument is too large"), - } - } - ColumnarValue::Scalar(ScalarValue::IntervalMonthDayNano(Some(v))) => { - let (months, days, nanos) = IntervalMonthDayNanoType::to_parts(*v); - - // If interval is months, its origin must be midnight of first date of the month - if months != 0 { - // Return error if days or nanos is not zero - if days != 0 || nanos != 0 { - return not_impl_err!( - "DATE_BIN stride does not support combination of month, day and nanosecond intervals" - ); - } else { - Interval::Months(months as i64) - } - } else { + ColumnarValue::Scalar(v) => match v.value() { + ScalarValue::IntervalDayTime(Some(v)) => { + let (days, ms) = IntervalDayTimeType::to_parts(*v); let nanos = (TimeDelta::try_days(days as i64).unwrap() - + Duration::nanoseconds(nanos)) + + TimeDelta::try_milliseconds(ms as i64).unwrap()) .num_nanoseconds(); + match nanos { Some(v) => Interval::Nanoseconds(v), _ => return exec_err!("DATE_BIN stride argument is too large"), } } - } - ColumnarValue::Scalar(v) => { - return exec_err!( - "DATE_BIN expects stride argument to be an INTERVAL but got {}", - v.data_type() - ); - } + ScalarValue::IntervalMonthDayNano(Some(v)) => { + let (months, days, nanos) = IntervalMonthDayNanoType::to_parts(*v); + + // If interval is months, its origin must be midnight of first date of the month + if months != 0 { + // Return error if days or nanos is not zero + if days != 0 || nanos != 0 { + return not_impl_err!( + "DATE_BIN stride does not support combination of month, day and nanosecond intervals" + ); + } else { + Interval::Months(months as i64) + } + } else { + let nanos = (TimeDelta::try_days(days as i64).unwrap() + + Duration::nanoseconds(nanos)) + .num_nanoseconds(); + match nanos { + Some(v) => Interval::Nanoseconds(v), + _ => return exec_err!("DATE_BIN stride argument is too large"), + } + } + } + _ => { + return exec_err!( + "DATE_BIN expects stride argument to be an INTERVAL but got {}", + v.data_type() + ) + } + }, ColumnarValue::Array(_) => { return not_impl_err!( "DATE_BIN only supports literal values for the stride argument, not arrays" @@ -309,12 +311,12 @@ fn date_bin_impl( }; let origin = match origin { - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(v), _)) => *v, - ColumnarValue::Scalar(v) => { - return exec_err!( + ColumnarValue::Scalar(v) => match v.value() { + ScalarValue::TimestampNanosecond(Some(v), _) => *v, + _ => return exec_err!( "DATE_BIN expects origin argument to be a TIMESTAMP with nanosecond precision but got {}", v.data_type() - ); + ) } ColumnarValue::Array(_) => { return not_impl_err!( @@ -345,38 +347,47 @@ fn date_bin_impl( } Ok(match array { - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(v, tz_opt)) => { - let apply_stride_fn = - stride_map_fn::(origin, stride, stride_fn); - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( - v.map(apply_stride_fn), - tz_opt.clone(), - )) - } - ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(v, tz_opt)) => { - let apply_stride_fn = - stride_map_fn::(origin, stride, stride_fn); - ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond( - v.map(apply_stride_fn), - tz_opt.clone(), - )) - } - ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(v, tz_opt)) => { - let apply_stride_fn = - stride_map_fn::(origin, stride, stride_fn); - ColumnarValue::Scalar(ScalarValue::TimestampMillisecond( - v.map(apply_stride_fn), - tz_opt.clone(), - )) - } - ColumnarValue::Scalar(ScalarValue::TimestampSecond(v, tz_opt)) => { - let apply_stride_fn = - stride_map_fn::(origin, stride, stride_fn); - ColumnarValue::Scalar(ScalarValue::TimestampSecond( - v.map(apply_stride_fn), - tz_opt.clone(), - )) - } + ColumnarValue::Scalar(scalar) => match scalar.value() { + ScalarValue::TimestampNanosecond(v, tz_opt) => { + let apply_stride_fn = + stride_map_fn::(origin, stride, stride_fn); + ColumnarValue::from(ScalarValue::TimestampNanosecond( + v.map(apply_stride_fn), + tz_opt.clone(), + )) + } + ScalarValue::TimestampMicrosecond(v, tz_opt) => { + let apply_stride_fn = + stride_map_fn::(origin, stride, stride_fn); + ColumnarValue::from(ScalarValue::TimestampMicrosecond( + v.map(apply_stride_fn), + tz_opt.clone(), + )) + } + ScalarValue::TimestampMillisecond(v, tz_opt) => { + let apply_stride_fn = + stride_map_fn::(origin, stride, stride_fn); + ColumnarValue::from(ScalarValue::TimestampMillisecond( + v.map(apply_stride_fn), + tz_opt.clone(), + )) + } + ScalarValue::TimestampSecond(v, tz_opt) => { + let apply_stride_fn = + stride_map_fn::(origin, stride, stride_fn); + ColumnarValue::from(ScalarValue::TimestampSecond( + v.map(apply_stride_fn), + tz_opt.clone(), + )) + } + + _ => { + return exec_err!( + "DATE_BIN expects source argument to be a TIMESTAMP but got {}", + array.data_type() + ); + } + }, ColumnarValue::Array(array) => { fn transform_array_with_stride( @@ -427,11 +438,6 @@ fn date_bin_impl( } } } - _ => { - return exec_err!( - "DATE_BIN expects source argument to be a TIMESTAMP scalar or array" - ); - } }) } @@ -454,46 +460,46 @@ mod tests { #[test] fn test_date_bin() { let res = DateBinFunc::new().invoke(&[ - ColumnarValue::Scalar(ScalarValue::IntervalDayTime(Some(IntervalDayTime { + ColumnarValue::from(ScalarValue::IntervalDayTime(Some(IntervalDayTime { days: 0, milliseconds: 1, }))), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), ]); assert!(res.is_ok()); let timestamps = Arc::new((1..6).map(Some).collect::()); let res = DateBinFunc::new().invoke(&[ - ColumnarValue::Scalar(ScalarValue::IntervalDayTime(Some(IntervalDayTime { + ColumnarValue::from(ScalarValue::IntervalDayTime(Some(IntervalDayTime { days: 0, milliseconds: 1, }))), ColumnarValue::Array(timestamps), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), ]); assert!(res.is_ok()); let res = DateBinFunc::new().invoke(&[ - ColumnarValue::Scalar(ScalarValue::IntervalDayTime(Some(IntervalDayTime { + ColumnarValue::from(ScalarValue::IntervalDayTime(Some(IntervalDayTime { days: 0, milliseconds: 1, }))), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), ]); assert!(res.is_ok()); // stride supports month-day-nano let res = DateBinFunc::new().invoke(&[ - ColumnarValue::Scalar(ScalarValue::IntervalMonthDayNano(Some( + ColumnarValue::from(ScalarValue::IntervalMonthDayNano(Some( IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 1, }, ))), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), ]); assert!(res.is_ok()); @@ -502,7 +508,7 @@ mod tests { // // invalid number of arguments - let res = DateBinFunc::new().invoke(&[ColumnarValue::Scalar( + let res = DateBinFunc::new().invoke(&[ColumnarValue::from( ScalarValue::IntervalDayTime(Some(IntervalDayTime { days: 0, milliseconds: 1, @@ -515,9 +521,9 @@ mod tests { // stride: invalid type let res = DateBinFunc::new().invoke(&[ - ColumnarValue::Scalar(ScalarValue::IntervalYearMonth(Some(1))), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::IntervalYearMonth(Some(1))), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), ]); assert_eq!( res.err().unwrap().strip_backtrace(), @@ -526,12 +532,12 @@ mod tests { // stride: invalid value let res = DateBinFunc::new().invoke(&[ - ColumnarValue::Scalar(ScalarValue::IntervalDayTime(Some(IntervalDayTime { + ColumnarValue::from(ScalarValue::IntervalDayTime(Some(IntervalDayTime { days: 0, milliseconds: 0, }))), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), ]); assert_eq!( res.err().unwrap().strip_backtrace(), @@ -540,11 +546,9 @@ mod tests { // stride: overflow of day-time interval let res = DateBinFunc::new().invoke(&[ - ColumnarValue::Scalar(ScalarValue::IntervalDayTime(Some( - IntervalDayTime::MAX, - ))), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::IntervalDayTime(Some(IntervalDayTime::MAX))), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), ]); assert_eq!( res.err().unwrap().strip_backtrace(), @@ -553,9 +557,9 @@ mod tests { // stride: overflow of month-day-nano interval let res = DateBinFunc::new().invoke(&[ - ColumnarValue::Scalar(ScalarValue::new_interval_mdn(0, i32::MAX, 1)), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::new_interval_mdn(0, i32::MAX, 1)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), ]); assert_eq!( res.err().unwrap().strip_backtrace(), @@ -564,9 +568,9 @@ mod tests { // stride: month intervals let res = DateBinFunc::new().invoke(&[ - ColumnarValue::Scalar(ScalarValue::new_interval_mdn(1, 1, 1)), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::new_interval_mdn(1, 1, 1)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), ]); assert_eq!( res.err().unwrap().strip_backtrace(), @@ -575,12 +579,12 @@ mod tests { // origin: invalid type let res = DateBinFunc::new().invoke(&[ - ColumnarValue::Scalar(ScalarValue::IntervalDayTime(Some(IntervalDayTime { + ColumnarValue::from(ScalarValue::IntervalDayTime(Some(IntervalDayTime { days: 0, milliseconds: 1, }))), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), - ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampMicrosecond(Some(1), None)), ]); assert_eq!( res.err().unwrap().strip_backtrace(), @@ -588,12 +592,12 @@ mod tests { ); let res = DateBinFunc::new().invoke(&[ - ColumnarValue::Scalar(ScalarValue::IntervalDayTime(Some(IntervalDayTime { + ColumnarValue::from(ScalarValue::IntervalDayTime(Some(IntervalDayTime { days: 0, milliseconds: 1, }))), - ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(Some(1), None)), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampMicrosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), ]); assert!(res.is_ok()); @@ -610,8 +614,8 @@ mod tests { ); let res = DateBinFunc::new().invoke(&[ ColumnarValue::Array(intervals), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), ]); assert_eq!( res.err().unwrap().strip_backtrace(), @@ -621,11 +625,11 @@ mod tests { // unsupported array type for origin let timestamps = Arc::new((1..6).map(Some).collect::()); let res = DateBinFunc::new().invoke(&[ - ColumnarValue::Scalar(ScalarValue::IntervalDayTime(Some(IntervalDayTime { + ColumnarValue::from(ScalarValue::IntervalDayTime(Some(IntervalDayTime { days: 0, milliseconds: 1, }))), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), ColumnarValue::Array(timestamps), ]); assert_eq!( @@ -744,9 +748,9 @@ mod tests { .with_timezone_opt(tz_opt.clone()); let result = DateBinFunc::new() .invoke(&[ - ColumnarValue::Scalar(ScalarValue::new_interval_dt(1, 0)), + ColumnarValue::from(ScalarValue::new_interval_dt(1, 0)), ColumnarValue::Array(Arc::new(input)), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( + ColumnarValue::from(ScalarValue::TimestampNanosecond( Some(string_to_timestamp_nanos(origin).unwrap()), tz_opt.clone(), )), diff --git a/datafusion/functions/src/datetime/date_part.rs b/datafusion/functions/src/datetime/date_part.rs index 8ee82d872651..7f8c488b572c 100644 --- a/datafusion/functions/src/datetime/date_part.rs +++ b/datafusion/functions/src/datetime/date_part.rs @@ -140,16 +140,17 @@ impl ScalarUDFImpl for DatePartFunc { } let (part, array) = (&args[0], &args[1]); - let part = if let ColumnarValue::Scalar(ScalarValue::Utf8(Some(v))) = part { - v - } else if let ColumnarValue::Scalar(ScalarValue::Utf8View(Some(v))) = part { - v - } else { + let ColumnarValue::Scalar(part) = part else { return exec_err!( "First argument of `DATE_PART` must be non-null scalar Utf8" ); }; + let part = match part.value() { + ScalarValue::Utf8(Some(v)) | ScalarValue::Utf8View(Some(v)) => v, + _ => unreachable!(), + }; + let is_scalar = matches!(array, ColumnarValue::Scalar(_)); let array = match array { @@ -192,7 +193,7 @@ impl ScalarUDFImpl for DatePartFunc { }; Ok(if is_scalar { - ColumnarValue::Scalar(ScalarValue::try_from_array(arr.as_ref(), 0)?) + ColumnarValue::from(ScalarValue::try_from_array(arr.as_ref(), 0)?) } else { ColumnarValue::Array(arr) }) diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs index 0ef839c49f0b..4cde51f0429a 100644 --- a/datafusion/functions/src/datetime/date_trunc.rs +++ b/datafusion/functions/src/datetime/date_trunc.rs @@ -137,17 +137,17 @@ impl ScalarUDFImpl for DateTruncFunc { fn invoke(&self, args: &[ColumnarValue]) -> Result { let (granularity, array) = (&args[0], &args[1]); - let granularity = if let ColumnarValue::Scalar(ScalarValue::Utf8(Some(v))) = - granularity - { - v.to_lowercase() - } else if let ColumnarValue::Scalar(ScalarValue::Utf8View(Some(v))) = granularity - { - v.to_lowercase() - } else { + let ColumnarValue::Scalar(granularity) = granularity else { return exec_err!("Granularity of `date_trunc` must be non-null scalar Utf8"); }; + let granularity = match granularity.value() { + ScalarValue::Utf8(Some(v)) | ScalarValue::Utf8View(Some(v)) => { + v.to_lowercase() + } + _ => unreachable!(), + }; + fn process_array( array: &dyn Array, granularity: String, @@ -171,22 +171,30 @@ impl ScalarUDFImpl for DateTruncFunc { let parsed_tz = parse_tz(tz_opt)?; let value = general_date_trunc(T::UNIT, v, parsed_tz, granularity.as_str())?; let value = ScalarValue::new_timestamp::(value, tz_opt.clone()); - Ok(ColumnarValue::Scalar(value)) + Ok(ColumnarValue::from(value)) } Ok(match array { - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(v, tz_opt)) => { - process_scalar::(v, granularity, tz_opt)? - } - ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(v, tz_opt)) => { - process_scalar::(v, granularity, tz_opt)? - } - ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(v, tz_opt)) => { - process_scalar::(v, granularity, tz_opt)? - } - ColumnarValue::Scalar(ScalarValue::TimestampSecond(v, tz_opt)) => { - process_scalar::(v, granularity, tz_opt)? - } + ColumnarValue::Scalar(scalar) => match scalar.value() { + ScalarValue::TimestampNanosecond(v, tz_opt) => { + process_scalar::(v, granularity, tz_opt)? + } + ScalarValue::TimestampMicrosecond(v, tz_opt) => { + process_scalar::(v, granularity, tz_opt)? + } + ScalarValue::TimestampMillisecond(v, tz_opt) => { + process_scalar::(v, granularity, tz_opt)? + } + ScalarValue::TimestampSecond(v, tz_opt) => { + process_scalar::(v, granularity, tz_opt)? + } + value => { + return exec_err!( + "second argument of `date_trunc` is an unsupported scalar type: {}", + value.data_type() + ) + } + }, ColumnarValue::Array(array) => { let array_type = array.data_type(); if let Timestamp(unit, tz_opt) = array_type { @@ -216,11 +224,6 @@ impl ScalarUDFImpl for DateTruncFunc { return exec_err!("second argument of `date_trunc` is an unsupported array type: {array_type}"); } } - _ => { - return exec_err!( - "second argument of `date_trunc` must be timestamp scalar or array" - ); - } }) } @@ -689,7 +692,7 @@ mod tests { .with_timezone_opt(tz_opt.clone()); let result = DateTruncFunc::new() .invoke(&[ - ColumnarValue::Scalar(ScalarValue::from("day")), + ColumnarValue::from(ScalarValue::from("day")), ColumnarValue::Array(Arc::new(input)), ]) .unwrap(); @@ -847,7 +850,7 @@ mod tests { .with_timezone_opt(tz_opt.clone()); let result = DateTruncFunc::new() .invoke(&[ - ColumnarValue::Scalar(ScalarValue::from("hour")), + ColumnarValue::from(ScalarValue::from("hour")), ColumnarValue::Array(Arc::new(input)), ]) .unwrap(); diff --git a/datafusion/functions/src/datetime/make_date.rs b/datafusion/functions/src/datetime/make_date.rs index ded7b454f9eb..ed30f5f5a26c 100644 --- a/datafusion/functions/src/datetime/make_date.rs +++ b/datafusion/functions/src/datetime/make_date.rs @@ -94,7 +94,7 @@ impl ScalarUDFImpl for MakeDateFunc { let ColumnarValue::Scalar(s) = col else { return exec_err!("Expected scalar value"); }; - let ScalarValue::Int32(Some(i)) = s else { + let ScalarValue::Int32(Some(i)) = s.value() else { return exec_err!("Unable to parse date from null/empty value"); }; Ok(*i) @@ -143,7 +143,7 @@ impl ScalarUDFImpl for MakeDateFunc { |days: i32| value = days, )?; - ColumnarValue::Scalar(ScalarValue::Date32(Some(value))) + ColumnarValue::from(ScalarValue::Date32(Some(value))) }; Ok(value) @@ -192,44 +192,65 @@ mod tests { fn test_make_date() { let res = MakeDateFunc::new() .invoke(&[ - ColumnarValue::Scalar(ScalarValue::Int32(Some(2024))), - ColumnarValue::Scalar(ScalarValue::Int64(Some(1))), - ColumnarValue::Scalar(ScalarValue::UInt32(Some(14))), + ColumnarValue::from(ScalarValue::Int32(Some(2024))), + ColumnarValue::from(ScalarValue::Int64(Some(1))), + ColumnarValue::from(ScalarValue::UInt32(Some(14))), ]) .expect("that make_date parsed values without error"); - if let ColumnarValue::Scalar(ScalarValue::Date32(date)) = res { + let ColumnarValue::Scalar(scalar) = res else { + panic!("Expected a scalar value") + }; + + if let ScalarValue::Date32(date) = scalar.value() { assert_eq!(19736, date.unwrap()); } else { - panic!("Expected a scalar value") + panic!( + "Expected a Date32 scalar, got {}", + scalar.value().data_type() + ) } let res = MakeDateFunc::new() .invoke(&[ - ColumnarValue::Scalar(ScalarValue::Int64(Some(2024))), - ColumnarValue::Scalar(ScalarValue::UInt64(Some(1))), - ColumnarValue::Scalar(ScalarValue::UInt32(Some(14))), + ColumnarValue::from(ScalarValue::Int64(Some(2024))), + ColumnarValue::from(ScalarValue::UInt64(Some(1))), + ColumnarValue::from(ScalarValue::UInt32(Some(14))), ]) .expect("that make_date parsed values without error"); - if let ColumnarValue::Scalar(ScalarValue::Date32(date)) = res { + let ColumnarValue::Scalar(scalar) = res else { + panic!("Expected a scalar value") + }; + + if let ScalarValue::Date32(date) = scalar.value() { assert_eq!(19736, date.unwrap()); } else { - panic!("Expected a scalar value") + panic!( + "Expected a Date32 scalar, got {}", + scalar.value().data_type() + ) } let res = MakeDateFunc::new() .invoke(&[ - ColumnarValue::Scalar(ScalarValue::Utf8(Some("2024".to_string()))), - ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some("1".to_string()))), - ColumnarValue::Scalar(ScalarValue::Utf8(Some("14".to_string()))), + ColumnarValue::from(ScalarValue::Utf8(Some("2024".to_string()))), + ColumnarValue::from(ScalarValue::LargeUtf8(Some("1".to_string()))), + ColumnarValue::from(ScalarValue::Utf8(Some("14".to_string()))), ]) .expect("that make_date parsed values without error"); - if let ColumnarValue::Scalar(ScalarValue::Date32(date)) = res { + let ColumnarValue::Scalar(scalar) = res else { + panic!("Expected a scalar value") + }; + + if let ScalarValue::Date32(date) = scalar.value() { assert_eq!(19736, date.unwrap()); } else { - panic!("Expected a scalar value") + panic!( + "Expected a Date32 scalar, got {}", + scalar.value().data_type() + ) } let years = Arc::new((2021..2025).map(Some).collect::()); @@ -261,7 +282,7 @@ mod tests { // invalid number of arguments let res = MakeDateFunc::new() - .invoke(&[ColumnarValue::Scalar(ScalarValue::Int32(Some(1)))]); + .invoke(&[ColumnarValue::from(ScalarValue::Int32(Some(1)))]); assert_eq!( res.err().unwrap().strip_backtrace(), "Execution error: make_date function requires 3 arguments, got 1" @@ -269,9 +290,9 @@ mod tests { // invalid type let res = MakeDateFunc::new().invoke(&[ - ColumnarValue::Scalar(ScalarValue::IntervalYearMonth(Some(1))), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::IntervalYearMonth(Some(1))), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), ]); assert_eq!( res.err().unwrap().strip_backtrace(), @@ -280,9 +301,9 @@ mod tests { // overflow of month let res = MakeDateFunc::new().invoke(&[ - ColumnarValue::Scalar(ScalarValue::Int32(Some(2023))), - ColumnarValue::Scalar(ScalarValue::UInt64(Some(u64::MAX))), - ColumnarValue::Scalar(ScalarValue::Int32(Some(22))), + ColumnarValue::from(ScalarValue::Int32(Some(2023))), + ColumnarValue::from(ScalarValue::UInt64(Some(u64::MAX))), + ColumnarValue::from(ScalarValue::Int32(Some(22))), ]); assert_eq!( res.err().unwrap().strip_backtrace(), @@ -291,9 +312,9 @@ mod tests { // overflow of day let res = MakeDateFunc::new().invoke(&[ - ColumnarValue::Scalar(ScalarValue::Int32(Some(2023))), - ColumnarValue::Scalar(ScalarValue::Int32(Some(22))), - ColumnarValue::Scalar(ScalarValue::UInt32(Some(u32::MAX))), + ColumnarValue::from(ScalarValue::Int32(Some(2023))), + ColumnarValue::from(ScalarValue::Int32(Some(22))), + ColumnarValue::from(ScalarValue::UInt32(Some(u32::MAX))), ]); assert_eq!( res.err().unwrap().strip_backtrace(), diff --git a/datafusion/functions/src/datetime/to_char.rs b/datafusion/functions/src/datetime/to_char.rs index f2e5af978ca0..706540b6d1f8 100644 --- a/datafusion/functions/src/datetime/to_char.rs +++ b/datafusion/functions/src/datetime/to_char.rs @@ -115,22 +115,22 @@ impl ScalarUDFImpl for ToCharFunc { } match &args[1] { - ColumnarValue::Scalar(ScalarValue::Utf8(None)) - | ColumnarValue::Scalar(ScalarValue::Null) => { - _to_char_scalar(args[0].clone(), None) - } - // constant format - ColumnarValue::Scalar(ScalarValue::Utf8(Some(format))) => { - // invoke to_char_scalar with the known string, without converting to array - _to_char_scalar(args[0].clone(), Some(format)) - } + ColumnarValue::Scalar(scalar) => match scalar.value() { + ScalarValue::Utf8(None) | ScalarValue::Null => { + _to_char_scalar(args[0].clone(), None) + } + ScalarValue::Utf8(Some(format)) => { + // invoke to_char_scalar with the known string, without converting to array + _to_char_scalar(args[0].clone(), Some(format)) + } + _ => { + exec_err!( + "Format for `to_char` must be non-null Utf8, received {:?}", + args[1].data_type() + ) + } + }, ColumnarValue::Array(_) => _to_char_array(args), - _ => { - exec_err!( - "Format for `to_char` must be non-null Utf8, received {:?}", - args[1].data_type() - ) - } } } @@ -177,13 +177,13 @@ fn _to_char_scalar( ) -> Result { // it's possible that the expression is a scalar however because // of the implementation in arrow-rs we need to convert it to an array - let data_type = &expression.data_type(); + let data_type = expression.data_type().clone(); let is_scalar_expression = matches!(&expression, ColumnarValue::Scalar(_)); let array = expression.into_array(1)?; if format.is_none() { if is_scalar_expression { - return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None))); + return Ok(ColumnarValue::from(ScalarValue::Utf8(None))); } else { return Ok(ColumnarValue::Array(new_null_array( &DataType::Utf8, @@ -192,7 +192,7 @@ fn _to_char_scalar( } } - let format_options = match _build_format_options(data_type, format) { + let format_options = match _build_format_options(&data_type, format) { Ok(value) => value, Err(value) => return value, }; @@ -204,7 +204,7 @@ fn _to_char_scalar( if let Ok(formatted) = formatted { if is_scalar_expression { - Ok(ColumnarValue::Scalar(ScalarValue::Utf8(Some( + Ok(ColumnarValue::from(ScalarValue::Utf8(Some( formatted.first().unwrap().to_string(), )))) } else { @@ -252,10 +252,10 @@ fn _to_char_array(args: &[ColumnarValue]) -> Result { results, )) as ArrayRef)), ColumnarValue::Scalar(_) => match results.first().unwrap() { - Some(value) => Ok(ColumnarValue::Scalar(ScalarValue::Utf8(Some( + Some(value) => Ok(ColumnarValue::from(ScalarValue::Utf8(Some( value.to_string(), )))), - None => Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None))), + None => Ok(ColumnarValue::from(ScalarValue::Utf8(None))), }, } } @@ -351,13 +351,15 @@ mod tests { for (value, format, expected) in scalar_data { let result = ToCharFunc::new() - .invoke(&[ColumnarValue::Scalar(value), ColumnarValue::Scalar(format)]) + .invoke(&[ColumnarValue::from(value), ColumnarValue::from(format)]) .expect("that to_char parsed values without error"); - if let ColumnarValue::Scalar(ScalarValue::Utf8(date)) = result { - assert_eq!(expected, date.unwrap()); - } else { - panic!("Expected a scalar value") + match result { + ColumnarValue::Scalar(scalar) => match scalar.into_value() { + ScalarValue::Utf8(date) => assert_eq!(expected, date.unwrap()), + _ => unreachable!(), + }, + _ => panic!("Expected a scalar value"), } } @@ -426,15 +428,17 @@ mod tests { for (value, format, expected) in scalar_array_data { let result = ToCharFunc::new() .invoke(&[ - ColumnarValue::Scalar(value), + ColumnarValue::from(value), ColumnarValue::Array(Arc::new(format) as ArrayRef), ]) .expect("that to_char parsed values without error"); - if let ColumnarValue::Scalar(ScalarValue::Utf8(date)) = result { - assert_eq!(expected, date.unwrap()); - } else { - panic!("Expected a scalar value") + match result { + ColumnarValue::Scalar(scalar) => match scalar.into_value() { + ScalarValue::Utf8(date) => assert_eq!(expected, date.unwrap()), + _ => unreachable!(), + }, + _ => panic!("Expected a scalar value"), } } @@ -552,7 +556,7 @@ mod tests { let result = ToCharFunc::new() .invoke(&[ ColumnarValue::Array(value as ArrayRef), - ColumnarValue::Scalar(format), + ColumnarValue::from(format), ]) .expect("that to_char parsed values without error"); @@ -585,8 +589,8 @@ mod tests { // // invalid number of arguments - let result = ToCharFunc::new() - .invoke(&[ColumnarValue::Scalar(ScalarValue::Int32(Some(1)))]); + let result = + ToCharFunc::new().invoke(&[ColumnarValue::from(ScalarValue::Int32(Some(1)))]); assert_eq!( result.err().unwrap().strip_backtrace(), "Execution error: to_char function requires 2 arguments, got 1" @@ -594,8 +598,8 @@ mod tests { // invalid type let result = ToCharFunc::new().invoke(&[ - ColumnarValue::Scalar(ScalarValue::Int32(Some(1))), - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)), + ColumnarValue::from(ScalarValue::Int32(Some(1))), + ColumnarValue::from(ScalarValue::TimestampNanosecond(Some(1), None)), ]); assert_eq!( result.err().unwrap().strip_backtrace(), diff --git a/datafusion/functions/src/datetime/to_date.rs b/datafusion/functions/src/datetime/to_date.rs index 288641b84dd7..ae2ef58899f2 100644 --- a/datafusion/functions/src/datetime/to_date.rs +++ b/datafusion/functions/src/datetime/to_date.rs @@ -156,18 +156,22 @@ mod tests { for tc in &test_cases { let date_scalar = ScalarValue::Utf8(Some(tc.date_str.to_string())); let to_date_result = - ToDateFunc::new().invoke(&[ColumnarValue::Scalar(date_scalar)]); + ToDateFunc::new().invoke(&[ColumnarValue::from(date_scalar)]); match to_date_result { - Ok(ColumnarValue::Scalar(ScalarValue::Date32(date_val))) => { - let expected = Date32Type::parse_formatted(tc.date_str, "%Y-%m-%d"); - assert_eq!( - date_val, expected, - "{}: to_date created wrong value", - tc.name - ); - } - _ => panic!("Could not convert '{}' to Date", tc.date_str), + Ok(ColumnarValue::Scalar(scalar)) => match scalar.into_value() { + ScalarValue::Date32(date_val) => { + let expected = + Date32Type::parse_formatted(tc.date_str, "%Y-%m-%d"); + assert_eq!( + date_val, expected, + "{}: to_date created wrong value", + tc.name + ); + } + _ => panic!("Could not convert '{}' to Date", tc.date_str), + }, + _ => unreachable!(), } } } @@ -226,19 +230,23 @@ mod tests { let format_scalar = ScalarValue::Utf8(Some(tc.format_str.to_string())); let to_date_result = ToDateFunc::new().invoke(&[ - ColumnarValue::Scalar(formatted_date_scalar), - ColumnarValue::Scalar(format_scalar), + ColumnarValue::from(formatted_date_scalar), + ColumnarValue::from(format_scalar), ]); match to_date_result { - Ok(ColumnarValue::Scalar(ScalarValue::Date32(date_val))) => { - let expected = Date32Type::parse_formatted(tc.date_str, "%Y-%m-%d"); - assert_eq!(date_val, expected, "{}: to_date created wrong value for date '{}' with format string '{}'", tc.name, tc.formatted_date, tc.format_str); - } - _ => panic!( - "Could not convert '{}' with format string '{}'to Date", - tc.date_str, tc.format_str - ), + Ok(ColumnarValue::Scalar(scalar)) => match scalar.into_value() { + ScalarValue::Date32(date_val) => { + let expected = + Date32Type::parse_formatted(tc.date_str, "%Y-%m-%d"); + assert_eq!(date_val, expected, "{}: to_date created wrong value for date '{}' with format string '{}'", tc.name, tc.formatted_date, tc.format_str); + } + _ => panic!( + "Could not convert '{}' with format string '{}'to Date", + tc.date_str, tc.format_str + ), + }, + _ => unreachable!(), } } } @@ -250,20 +258,23 @@ mod tests { let format2_scalar = ScalarValue::Utf8(Some("%Y/%m/%d".into())); let to_date_result = ToDateFunc::new().invoke(&[ - ColumnarValue::Scalar(formatted_date_scalar), - ColumnarValue::Scalar(format1_scalar), - ColumnarValue::Scalar(format2_scalar), + ColumnarValue::from(formatted_date_scalar), + ColumnarValue::from(format1_scalar), + ColumnarValue::from(format2_scalar), ]); match to_date_result { - Ok(ColumnarValue::Scalar(ScalarValue::Date32(date_val))) => { - let expected = Date32Type::parse_formatted("2023-01-31", "%Y-%m-%d"); - assert_eq!( - date_val, expected, - "to_date created wrong value for date with 2 format strings" - ); - } - _ => panic!("Conversion failed",), + Ok(ColumnarValue::Scalar(scalar)) => match scalar.into_value() { + ScalarValue::Date32(date_val) => { + let expected = Date32Type::parse_formatted("2023-01-31", "%Y-%m-%d"); + assert_eq!( + date_val, expected, + "to_date created wrong value for date with 2 format strings" + ); + } + _ => panic!("Conversion failed",), + }, + _ => unreachable!(), } } @@ -278,14 +289,18 @@ mod tests { let formatted_date_scalar = ScalarValue::Utf8(Some(date_str.into())); let to_date_result = - ToDateFunc::new().invoke(&[ColumnarValue::Scalar(formatted_date_scalar)]); + ToDateFunc::new().invoke(&[ColumnarValue::from(formatted_date_scalar)]); match to_date_result { - Ok(ColumnarValue::Scalar(ScalarValue::Date32(date_val))) => { - let expected = Date32Type::parse_formatted("2020-09-08", "%Y-%m-%d"); - assert_eq!(date_val, expected, "to_date created wrong value"); - } - _ => panic!("Conversion of {} failed", date_str), + Ok(ColumnarValue::Scalar(scalar)) => match scalar.into_value() { + ScalarValue::Date32(date_val) => { + let expected = + Date32Type::parse_formatted("2020-09-08", "%Y-%m-%d"); + assert_eq!(date_val, expected, "to_date created wrong value"); + } + _ => panic!("Conversion of {} failed", date_str), + }, + _ => unreachable!(), } } } @@ -296,18 +311,21 @@ mod tests { let date_scalar = ScalarValue::Utf8(Some(date_str.into())); let to_date_result = - ToDateFunc::new().invoke(&[ColumnarValue::Scalar(date_scalar)]); + ToDateFunc::new().invoke(&[ColumnarValue::from(date_scalar)]); match to_date_result { - Ok(ColumnarValue::Scalar(ScalarValue::Date32(date_val))) => { - let expected = Date32Type::parse_formatted("2024-12-31", "%Y-%m-%d"); - assert_eq!( - date_val, expected, - "to_date created wrong value for {}", - date_str - ); - } - _ => panic!("Conversion of {} failed", date_str), + Ok(ColumnarValue::Scalar(scalar)) => match scalar.into_value() { + ScalarValue::Date32(date_val) => { + let expected = Date32Type::parse_formatted("2024-12-31", "%Y-%m-%d"); + assert_eq!( + date_val, expected, + "to_date created wrong value for {}", + date_str + ); + } + _ => panic!("Conversion of {} failed", date_str), + }, + _ => unreachable!(), } } @@ -317,13 +335,15 @@ mod tests { let date_scalar = ScalarValue::Utf8(Some(date_str.into())); let to_date_result = - ToDateFunc::new().invoke(&[ColumnarValue::Scalar(date_scalar)]); + ToDateFunc::new().invoke(&[ColumnarValue::from(date_scalar)]); - if let Ok(ColumnarValue::Scalar(ScalarValue::Date32(_))) = to_date_result { - panic!( - "Conversion of {} succeded, but should have failed, ", - date_str - ); + if let Ok(ColumnarValue::Scalar(scalar)) = to_date_result { + if let ScalarValue::Date32(_) = scalar.value() { + panic!( + "Conversion of {} succeded, but should have failed, ", + date_str + ) + } } } } diff --git a/datafusion/functions/src/datetime/to_local_time.rs b/datafusion/functions/src/datetime/to_local_time.rs index 634e28e6f393..d5ce1281cd2b 100644 --- a/datafusion/functions/src/datetime/to_local_time.rs +++ b/datafusion/functions/src/datetime/to_local_time.rs @@ -97,50 +97,48 @@ impl ToLocalTimeFunc { let tz: Tz = timezone.parse()?; match time_value { - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( - Some(ts), - Some(_), - )) => { - let adjusted_ts = - adjust_to_local_time::(*ts, tz)?; - Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( - Some(adjusted_ts), - None, - ))) - } - ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond( - Some(ts), - Some(_), - )) => { - let adjusted_ts = - adjust_to_local_time::(*ts, tz)?; - Ok(ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond( - Some(adjusted_ts), - None, - ))) - } - ColumnarValue::Scalar(ScalarValue::TimestampMillisecond( - Some(ts), - Some(_), - )) => { - let adjusted_ts = - adjust_to_local_time::(*ts, tz)?; - Ok(ColumnarValue::Scalar(ScalarValue::TimestampMillisecond( - Some(adjusted_ts), - None, - ))) - } - ColumnarValue::Scalar(ScalarValue::TimestampSecond( - Some(ts), - Some(_), - )) => { - let adjusted_ts = - adjust_to_local_time::(*ts, tz)?; - Ok(ColumnarValue::Scalar(ScalarValue::TimestampSecond( - Some(adjusted_ts), - None, - ))) - } + ColumnarValue::Scalar(scalar) => match scalar.value() { + ScalarValue::TimestampNanosecond(Some(ts), Some(_)) => { + let adjusted_ts = + adjust_to_local_time::(*ts, tz)?; + Ok(ColumnarValue::from(ScalarValue::TimestampNanosecond( + Some(adjusted_ts), + None, + ))) + } + ScalarValue::TimestampMicrosecond(Some(ts), Some(_)) => { + let adjusted_ts = adjust_to_local_time::< + TimestampMicrosecondType, + >(*ts, tz)?; + Ok(ColumnarValue::from(ScalarValue::TimestampMicrosecond( + Some(adjusted_ts), + None, + ))) + } + ScalarValue::TimestampMillisecond(Some(ts), Some(_)) => { + let adjusted_ts = adjust_to_local_time::< + TimestampMillisecondType, + >(*ts, tz)?; + Ok(ColumnarValue::from(ScalarValue::TimestampMillisecond( + Some(adjusted_ts), + None, + ))) + } + ScalarValue::TimestampSecond(Some(ts), Some(_)) => { + let adjusted_ts = + adjust_to_local_time::(*ts, tz)?; + Ok(ColumnarValue::from(ScalarValue::TimestampSecond( + Some(adjusted_ts), + None, + ))) + } + _ => { + exec_err!( + "to_local_time function requires timestamp argument, got {:?}", + time_value.data_type() + ) + } + }, ColumnarValue::Array(array) => { fn transform_array( array: &ArrayRef, @@ -185,12 +183,6 @@ impl ToLocalTimeFunc { } } } - _ => { - exec_err!( - "to_local_time function requires timestamp argument, got {:?}", - time_value.data_type() - ) - } } } _ => { @@ -359,7 +351,7 @@ mod tests { use arrow::datatypes::{DataType, TimeUnit}; use chrono::NaiveDateTime; use datafusion_common::ScalarValue; - use datafusion_expr::{ColumnarValue, ScalarUDFImpl}; + use datafusion_expr::{ColumnarValue, Scalar, ScalarUDFImpl}; use super::{adjust_to_local_time, ToLocalTimeFunc}; @@ -486,11 +478,11 @@ mod tests { fn test_to_local_time_helper(input: ScalarValue, expected: ScalarValue) { let res = ToLocalTimeFunc::new() - .invoke(&[ColumnarValue::Scalar(input)]) + .invoke(&[ColumnarValue::from(input)]) .unwrap(); match res { ColumnarValue::Scalar(res) => { - assert_eq!(res, expected); + assert_eq!(res, Scalar::from(expected)); } _ => panic!("unexpected return type"), } diff --git a/datafusion/functions/src/datetime/to_timestamp.rs b/datafusion/functions/src/datetime/to_timestamp.rs index cbb6f37603d2..b8f148236c58 100644 --- a/datafusion/functions/src/datetime/to_timestamp.rs +++ b/datafusion/functions/src/datetime/to_timestamp.rs @@ -169,7 +169,7 @@ impl ScalarUDFImpl for ToTimestampFunc { args[0].cast_to(&Timestamp(Nanosecond, None), None) } DataType::Timestamp(_, Some(tz)) => { - args[0].cast_to(&Timestamp(Nanosecond, Some(tz)), None) + args[0].cast_to(&Timestamp(Nanosecond, Some(Arc::clone(tz))), None) } DataType::Utf8 => { to_timestamp_impl::(args, "to_timestamp") @@ -219,7 +219,7 @@ impl ScalarUDFImpl for ToTimestampSecondsFunc { args[0].cast_to(&Timestamp(Second, None), None) } DataType::Timestamp(_, Some(tz)) => { - args[0].cast_to(&Timestamp(Second, Some(tz)), None) + args[0].cast_to(&Timestamp(Second, Some(Arc::clone(tz))), None) } DataType::Utf8 => { to_timestamp_impl::(args, "to_timestamp_seconds") @@ -269,7 +269,7 @@ impl ScalarUDFImpl for ToTimestampMillisFunc { args[0].cast_to(&Timestamp(Millisecond, None), None) } DataType::Timestamp(_, Some(tz)) => { - args[0].cast_to(&Timestamp(Millisecond, Some(tz)), None) + args[0].cast_to(&Timestamp(Millisecond, Some(Arc::clone(tz))), None) } DataType::Utf8 => { to_timestamp_impl::(args, "to_timestamp_millis") @@ -319,7 +319,7 @@ impl ScalarUDFImpl for ToTimestampMicrosFunc { args[0].cast_to(&Timestamp(Microsecond, None), None) } DataType::Timestamp(_, Some(tz)) => { - args[0].cast_to(&Timestamp(Microsecond, Some(tz)), None) + args[0].cast_to(&Timestamp(Microsecond, Some(Arc::clone(tz))), None) } DataType::Utf8 => { to_timestamp_impl::(args, "to_timestamp_micros") @@ -369,7 +369,7 @@ impl ScalarUDFImpl for ToTimestampNanosFunc { args[0].cast_to(&Timestamp(Nanosecond, None), None) } DataType::Timestamp(_, Some(tz)) => { - args[0].cast_to(&Timestamp(Nanosecond, Some(tz)), None) + args[0].cast_to(&Timestamp(Nanosecond, Some(Arc::clone(tz))), None) } DataType::Utf8 => { to_timestamp_impl::(args, "to_timestamp_nanos") @@ -803,7 +803,7 @@ mod tests { for udf in &udfs { for array in arrays { - let rt = udf.return_type(&[array.data_type()]).unwrap(); + let rt = udf.return_type(&[array.data_type().clone()]).unwrap(); assert!(matches!(rt, DataType::Timestamp(_, Some(_)))); let res = udf @@ -846,7 +846,7 @@ mod tests { for udf in &udfs { for array in arrays { - let rt = udf.return_type(&[array.data_type()]).unwrap(); + let rt = udf.return_type(&[array.data_type().clone()]).unwrap(); assert!(matches!(rt, DataType::Timestamp(_, None))); let res = udf @@ -896,9 +896,9 @@ mod tests { // test UTF8 let string_array = [ ColumnarValue::Array(Arc::new(data.clone()) as ArrayRef), - ColumnarValue::Scalar(ScalarValue::Utf8(Some("%s".to_string()))), - ColumnarValue::Scalar(ScalarValue::Utf8(Some("%c".to_string()))), - ColumnarValue::Scalar(ScalarValue::Utf8(Some("%+".to_string()))), + ColumnarValue::from(ScalarValue::Utf8(Some("%s".to_string()))), + ColumnarValue::from(ScalarValue::Utf8(Some("%c".to_string()))), + ColumnarValue::from(ScalarValue::Utf8(Some("%+".to_string()))), ]; let parsed_timestamps = func(&string_array) .expect("that to_timestamp with format args parsed values without error"); @@ -925,9 +925,9 @@ mod tests { // test LargeUTF8 let string_array = [ ColumnarValue::Array(Arc::new(data.clone()) as ArrayRef), - ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some("%s".to_string()))), - ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some("%c".to_string()))), - ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some("%+".to_string()))), + ColumnarValue::from(ScalarValue::LargeUtf8(Some("%s".to_string()))), + ColumnarValue::from(ScalarValue::LargeUtf8(Some("%c".to_string()))), + ColumnarValue::from(ScalarValue::LargeUtf8(Some("%+".to_string()))), ]; let parsed_timestamps = func(&string_array) .expect("that to_timestamp with format args parsed values without error"); @@ -959,9 +959,9 @@ mod tests { // test other types let string_array = [ ColumnarValue::Array(Arc::new(data.clone()) as ArrayRef), - ColumnarValue::Scalar(ScalarValue::Int32(Some(1))), - ColumnarValue::Scalar(ScalarValue::Int32(Some(2))), - ColumnarValue::Scalar(ScalarValue::Int32(Some(3))), + ColumnarValue::from(ScalarValue::Int32(Some(1))), + ColumnarValue::from(ScalarValue::Int32(Some(2))), + ColumnarValue::from(ScalarValue::Int32(Some(3))), ]; let expected = "Unsupported data type Int32 for function".to_string(); diff --git a/datafusion/functions/src/encoding/inner.rs b/datafusion/functions/src/encoding/inner.rs index d9ce299a2602..adf0b66df31d 100644 --- a/datafusion/functions/src/encoding/inner.rs +++ b/datafusion/functions/src/encoding/inner.rs @@ -174,7 +174,7 @@ fn encode_process(value: &ColumnarValue, encoding: Encoding) -> Result { - match scalar { + match scalar.value() { ScalarValue::Utf8(a) => { Ok(encoding.encode_scalar(a.as_ref().map(|s: &String| s.as_bytes()))) } @@ -205,7 +205,7 @@ fn decode_process(value: &ColumnarValue, encoding: Encoding) -> Result { - match scalar { + match scalar.value() { ScalarValue::Utf8(a) => { encoding.decode_scalar(a.as_ref().map(|s: &String| s.as_bytes())) } @@ -266,7 +266,7 @@ macro_rules! decode_to_array { impl Encoding { fn encode_scalar(self, value: Option<&[u8]>) -> ColumnarValue { - ColumnarValue::Scalar(match self { + ColumnarValue::from(match self { Self::Base64 => ScalarValue::Utf8( value.map(|v| general_purpose::STANDARD_NO_PAD.encode(v)), ), @@ -275,7 +275,7 @@ impl Encoding { } fn encode_large_scalar(self, value: Option<&[u8]>) -> ColumnarValue { - ColumnarValue::Scalar(match self { + ColumnarValue::from(match self { Self::Base64 => ScalarValue::LargeUtf8( value.map(|v| general_purpose::STANDARD_NO_PAD.encode(v)), ), @@ -310,7 +310,7 @@ impl Encoding { fn decode_scalar(self, value: Option<&[u8]>) -> Result { let value = match value { Some(value) => value, - None => return Ok(ColumnarValue::Scalar(ScalarValue::Binary(None))), + None => return Ok(ColumnarValue::from(ScalarValue::Binary(None))), }; let out = match self { @@ -332,13 +332,13 @@ impl Encoding { })?, }; - Ok(ColumnarValue::Scalar(ScalarValue::Binary(Some(out)))) + Ok(ColumnarValue::from(ScalarValue::Binary(Some(out)))) } fn decode_large_scalar(self, value: Option<&[u8]>) -> Result { let value = match value { Some(value) => value, - None => return Ok(ColumnarValue::Scalar(ScalarValue::LargeBinary(None))), + None => return Ok(ColumnarValue::from(ScalarValue::LargeBinary(None))), }; let out = match self { @@ -360,7 +360,7 @@ impl Encoding { })?, }; - Ok(ColumnarValue::Scalar(ScalarValue::LargeBinary(Some(out)))) + Ok(ColumnarValue::from(ScalarValue::LargeBinary(Some(out)))) } fn decode_binary_array(self, value: &dyn Array) -> Result @@ -425,7 +425,7 @@ fn encode(args: &[ColumnarValue]) -> Result { ); } let encoding = match &args[1] { - ColumnarValue::Scalar(scalar) => match scalar { + ColumnarValue::Scalar(scalar) => match scalar.value() { ScalarValue::Utf8(Some(method)) | ScalarValue::LargeUtf8(Some(method)) => { method.parse::() } @@ -451,7 +451,7 @@ fn decode(args: &[ColumnarValue]) -> Result { ); } let encoding = match &args[1] { - ColumnarValue::Scalar(scalar) => match scalar { + ColumnarValue::Scalar(scalar) => match scalar.value() { ScalarValue::Utf8(Some(method)) | ScalarValue::LargeUtf8(Some(method)) => { method.parse::() } diff --git a/datafusion/functions/src/math/log.rs b/datafusion/functions/src/math/log.rs index ad7cff1f7149..b6f01f5e4659 100644 --- a/datafusion/functions/src/math/log.rs +++ b/datafusion/functions/src/math/log.rs @@ -110,7 +110,7 @@ impl ScalarUDFImpl for LogFunc { fn invoke(&self, args: &[ColumnarValue]) -> Result { let args = ColumnarValue::values_to_arrays(args)?; - let mut base = ColumnarValue::Scalar(ScalarValue::Float32(Some(10.0))); + let mut base = ColumnarValue::from(ScalarValue::Float32(Some(10.0))); let mut x = &args[0]; if args.len() == 2 { @@ -120,11 +120,18 @@ impl ScalarUDFImpl for LogFunc { // note in f64::log params order is different than in sql. e.g in sql log(base, x) == f64::log(x, base) let arr: ArrayRef = match args[0].data_type() { DataType::Float64 => match base { - ColumnarValue::Scalar(ScalarValue::Float32(Some(base))) => { - Arc::new(make_function_scalar_inputs!(x, "x", Float64Array, { - |value: f64| f64::log(value, base as f64) - })) - } + ColumnarValue::Scalar(scalar) => match scalar.into_value() { + ScalarValue::Float32(Some(base)) => { + Arc::new(make_function_scalar_inputs!(x, "x", Float64Array, { + |value: f64| f64::log(value, base as f64) + })) + } + _ => { + return exec_err!( + "log function requires a Float32 scalar for base" + ) + } + }, ColumnarValue::Array(base) => Arc::new(make_function_inputs2!( x, base, @@ -133,17 +140,17 @@ impl ScalarUDFImpl for LogFunc { Float64Array, { f64::log } )), - _ => { - return exec_err!("log function requires a scalar or array for base") - } }, DataType::Float32 => match base { - ColumnarValue::Scalar(ScalarValue::Float32(Some(base))) => { - Arc::new(make_function_scalar_inputs!(x, "x", Float32Array, { - |value: f32| f32::log(value, base) - })) - } + ColumnarValue::Scalar(scalar) => match scalar.into_value() { + ScalarValue::Float32(Some(base)) => { + Arc::new(make_function_scalar_inputs!(x, "x", Float32Array, { + |value: f32| f32::log(value, base) + })) + } + _ => return exec_err!("log function requires a Float32 scalar"), + }, ColumnarValue::Array(base) => Arc::new(make_function_inputs2!( x, base, @@ -152,9 +159,6 @@ impl ScalarUDFImpl for LogFunc { Float32Array, { f32::log } )), - _ => { - return exec_err!("log function requires a scalar or array for base") - } }, other => { return exec_err!("Unsupported data type {other:?} for function log") diff --git a/datafusion/functions/src/math/pi.rs b/datafusion/functions/src/math/pi.rs index c2fe4efb1139..b4adfc190e69 100644 --- a/datafusion/functions/src/math/pi.rs +++ b/datafusion/functions/src/math/pi.rs @@ -64,7 +64,7 @@ impl ScalarUDFImpl for PiFunc { } fn invoke_no_args(&self, _number_rows: usize) -> Result { - Ok(ColumnarValue::Scalar(ScalarValue::Float64(Some( + Ok(ColumnarValue::from(ScalarValue::Float64(Some( std::f64::consts::PI, )))) } diff --git a/datafusion/functions/src/math/round.rs b/datafusion/functions/src/math/round.rs index 89554a76febb..9067a1748e19 100644 --- a/datafusion/functions/src/math/round.rs +++ b/datafusion/functions/src/math/round.rs @@ -108,7 +108,7 @@ pub fn round(args: &[ArrayRef]) -> Result { ); } - let mut decimal_places = ColumnarValue::Scalar(ScalarValue::Int64(Some(0))); + let mut decimal_places = ColumnarValue::from(ScalarValue::Int64(Some(0))); if args.len() == 2 { decimal_places = ColumnarValue::Array(Arc::clone(&args[1])); @@ -116,25 +116,30 @@ pub fn round(args: &[ArrayRef]) -> Result { match args[0].data_type() { DataType::Float64 => match decimal_places { - ColumnarValue::Scalar(ScalarValue::Int64(Some(decimal_places))) => { - let decimal_places: i32 = decimal_places.try_into().map_err(|e| { - exec_datafusion_err!( - "Invalid value for decimal places: {decimal_places}: {e}" - ) - })?; - - Ok(Arc::new(make_function_scalar_inputs!( - &args[0], - "value", - Float64Array, - { - |value: f64| { - (value * 10.0_f64.powi(decimal_places)).round() - / 10.0_f64.powi(decimal_places) + ColumnarValue::Scalar(scalar) => match scalar.into_value() { + ScalarValue::Int64(Some(decimal_places)) => { + let decimal_places: i32 = decimal_places.try_into().map_err(|e| { + exec_datafusion_err!( + "Invalid value for decimal places: {decimal_places}: {e}" + ) + })?; + + Ok(Arc::new(make_function_scalar_inputs!( + &args[0], + "value", + Float64Array, + { + |value: f64| { + (value * 10.0_f64.powi(decimal_places)).round() + / 10.0_f64.powi(decimal_places) + } } - } - )) as ArrayRef) - } + )) as ArrayRef) + } + _ => { + exec_err!("round function requires a Int64 scalar for decimal_places") + } + }, ColumnarValue::Array(decimal_places) => { let options = CastOptions { safe: false, // raise error if the cast is not possible @@ -159,31 +164,33 @@ pub fn round(args: &[ArrayRef]) -> Result { } )) as ArrayRef) } - _ => { - exec_err!("round function requires a scalar or array for decimal_places") - } }, DataType::Float32 => match decimal_places { - ColumnarValue::Scalar(ScalarValue::Int64(Some(decimal_places))) => { - let decimal_places: i32 = decimal_places.try_into().map_err(|e| { - exec_datafusion_err!( - "Invalid value for decimal places: {decimal_places}: {e}" - ) - })?; - - Ok(Arc::new(make_function_scalar_inputs!( - &args[0], - "value", - Float32Array, - { - |value: f32| { - (value * 10.0_f32.powi(decimal_places)).round() - / 10.0_f32.powi(decimal_places) + ColumnarValue::Scalar(scalar) => match scalar.into_value() { + ScalarValue::Int64(Some(decimal_places)) => { + let decimal_places: i32 = decimal_places.try_into().map_err(|e| { + exec_datafusion_err!( + "Invalid value for decimal places: {decimal_places}: {e}" + ) + })?; + + Ok(Arc::new(make_function_scalar_inputs!( + &args[0], + "value", + Float32Array, + { + |value: f32| { + (value * 10.0_f32.powi(decimal_places)).round() + / 10.0_f32.powi(decimal_places) + } } - } - )) as ArrayRef) - } + )) as ArrayRef) + } + _ => { + exec_err!("round function requires a Int64 scalar for decimal_places") + } + }, ColumnarValue::Array(_) => { let ColumnarValue::Array(decimal_places) = decimal_places.cast_to(&Int32, None).map_err(|e| { @@ -208,9 +215,6 @@ pub fn round(args: &[ArrayRef]) -> Result { } )) as ArrayRef) } - _ => { - exec_err!("round function requires a scalar or array for decimal_places") - } }, other => exec_err!("Unsupported data type {other:?} for function round"), diff --git a/datafusion/functions/src/math/trunc.rs b/datafusion/functions/src/math/trunc.rs index 3344438454c4..2a2b33c1e279 100644 --- a/datafusion/functions/src/math/trunc.rs +++ b/datafusion/functions/src/math/trunc.rs @@ -115,16 +115,22 @@ fn trunc(args: &[ArrayRef]) -> Result { //or then invoke the compute_truncate method to process precision let num = &args[0]; let precision = if args.len() == 1 { - ColumnarValue::Scalar(Int64(Some(0))) + ColumnarValue::from(Int64(Some(0))) } else { - ColumnarValue::Array(Arc::clone(&args[1])) + ColumnarValue::from(Arc::clone(&args[1])) }; match args[0].data_type() { Float64 => match precision { - ColumnarValue::Scalar(Int64(Some(0))) => Ok(Arc::new( - make_function_scalar_inputs!(num, "num", Float64Array, { f64::trunc }), - ) as ArrayRef), + ColumnarValue::Scalar(scalar) => match scalar.value() { + Int64(Some(0)) => Ok(Arc::new(make_function_scalar_inputs!( + num, + "num", + Float64Array, + { f64::trunc } + )) as ArrayRef), + _ => exec_err!("trunc function requires a Int64 scalar for precision"), + }, ColumnarValue::Array(precision) => Ok(Arc::new(make_function_inputs2!( num, precision, @@ -134,12 +140,17 @@ fn trunc(args: &[ArrayRef]) -> Result { Int64Array, { compute_truncate64 } )) as ArrayRef), - _ => exec_err!("trunc function requires a scalar or array for precision"), }, Float32 => match precision { - ColumnarValue::Scalar(Int64(Some(0))) => Ok(Arc::new( - make_function_scalar_inputs!(num, "num", Float32Array, { f32::trunc }), - ) as ArrayRef), + ColumnarValue::Scalar(scalar) => match scalar.value() { + Int64(Some(0)) => Ok(Arc::new(make_function_scalar_inputs!( + num, + "num", + Float32Array, + { f32::trunc } + )) as ArrayRef), + _ => exec_err!("trunc function requires a Int64 scalar for precision"), + }, ColumnarValue::Array(precision) => Ok(Arc::new(make_function_inputs2!( num, precision, @@ -149,7 +160,6 @@ fn trunc(args: &[ArrayRef]) -> Result { Int64Array, { compute_truncate32 } )) as ArrayRef), - _ => exec_err!("trunc function requires a scalar or array for precision"), }, other => exec_err!("Unsupported data type {other:?} for function trunc"), } diff --git a/datafusion/functions/src/regex/regexplike.rs b/datafusion/functions/src/regex/regexplike.rs index 20029ba005c4..4cff633a1772 100644 --- a/datafusion/functions/src/regex/regexplike.rs +++ b/datafusion/functions/src/regex/regexplike.rs @@ -100,7 +100,7 @@ impl ScalarUDFImpl for RegexpLikeFunc { if is_scalar { // If all inputs are scalar, keeps output as scalar let result = result.and_then(|arr| ScalarValue::try_from_array(&arr, 0)); - result.map(ColumnarValue::Scalar) + result.map(ColumnarValue::from) } else { result.map(ColumnarValue::Array) } diff --git a/datafusion/functions/src/regex/regexpmatch.rs b/datafusion/functions/src/regex/regexpmatch.rs index bf40eff11d30..a2c314e3761b 100644 --- a/datafusion/functions/src/regex/regexpmatch.rs +++ b/datafusion/functions/src/regex/regexpmatch.rs @@ -102,7 +102,7 @@ impl ScalarUDFImpl for RegexpMatchFunc { if is_scalar { // If all inputs are scalar, keeps output as scalar let result = result.and_then(|arr| ScalarValue::try_from_array(&arr, 0)); - result.map(ColumnarValue::Scalar) + result.map(ColumnarValue::from) } else { result.map(ColumnarValue::Array) } diff --git a/datafusion/functions/src/regex/regexpreplace.rs b/datafusion/functions/src/regex/regexpreplace.rs index 3eb72a1fb5f5..5c885de202d0 100644 --- a/datafusion/functions/src/regex/regexpreplace.rs +++ b/datafusion/functions/src/regex/regexpreplace.rs @@ -118,7 +118,7 @@ impl ScalarUDFImpl for RegexpReplaceFunc { if is_scalar { // If all inputs are scalar, keeps output as scalar let result = result.and_then(|arr| ScalarValue::try_from_array(&arr, 0)); - result.map(ColumnarValue::Scalar) + result.map(ColumnarValue::from) } else { result.map(ColumnarValue::Array) } diff --git a/datafusion/functions/src/string/ascii.rs b/datafusion/functions/src/string/ascii.rs index 68ba3f5ff15f..526803eb36e4 100644 --- a/datafusion/functions/src/string/ascii.rs +++ b/datafusion/functions/src/string/ascii.rs @@ -122,7 +122,7 @@ mod tests { ($INPUT:expr, $EXPECTED:expr) => { test_function!( AsciiFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8($INPUT))], + &[ColumnarValue::from(ScalarValue::Utf8($INPUT))], $EXPECTED, i32, Int32, @@ -131,7 +131,7 @@ mod tests { test_function!( AsciiFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::LargeUtf8($INPUT))], + &[ColumnarValue::from(ScalarValue::LargeUtf8($INPUT))], $EXPECTED, i32, Int32, @@ -140,7 +140,7 @@ mod tests { test_function!( AsciiFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8View($INPUT))], + &[ColumnarValue::from(ScalarValue::Utf8View($INPUT))], $EXPECTED, i32, Int32, diff --git a/datafusion/functions/src/string/bit_length.rs b/datafusion/functions/src/string/bit_length.rs index 65ec1a4a7734..c37851443c37 100644 --- a/datafusion/functions/src/string/bit_length.rs +++ b/datafusion/functions/src/string/bit_length.rs @@ -77,13 +77,13 @@ impl ScalarUDFImpl for BitLengthFunc { match &args[0] { ColumnarValue::Array(v) => Ok(ColumnarValue::Array(bit_length(v.as_ref())?)), - ColumnarValue::Scalar(v) => match v { - ScalarValue::Utf8(v) => Ok(ColumnarValue::Scalar(ScalarValue::Int32( + ColumnarValue::Scalar(v) => match v.value() { + ScalarValue::Utf8(v) => Ok(ColumnarValue::from(ScalarValue::Int32( v.as_ref().map(|x| (x.len() * 8) as i32), ))), - ScalarValue::LargeUtf8(v) => Ok(ColumnarValue::Scalar( - ScalarValue::Int64(v.as_ref().map(|x| (x.len() * 8) as i64)), - )), + ScalarValue::LargeUtf8(v) => Ok(ColumnarValue::from(ScalarValue::Int64( + v.as_ref().map(|x| (x.len() * 8) as i64), + ))), _ => unreachable!(), }, } diff --git a/datafusion/functions/src/string/common.rs b/datafusion/functions/src/string/common.rs index 9365a6d83331..693106c798ad 100644 --- a/datafusion/functions/src/string/common.rs +++ b/datafusion/functions/src/string/common.rs @@ -230,18 +230,18 @@ where } other => exec_err!("Unsupported data type {other:?} for function {name}"), }, - ColumnarValue::Scalar(scalar) => match scalar { + ColumnarValue::Scalar(scalar) => match scalar.value() { ScalarValue::Utf8(a) => { let result = a.as_ref().map(|x| op(x)); - Ok(ColumnarValue::Scalar(ScalarValue::Utf8(result))) + Ok(ColumnarValue::from(ScalarValue::Utf8(result))) } ScalarValue::LargeUtf8(a) => { let result = a.as_ref().map(|x| op(x)); - Ok(ColumnarValue::Scalar(ScalarValue::LargeUtf8(result))) + Ok(ColumnarValue::from(ScalarValue::LargeUtf8(result))) } ScalarValue::Utf8View(a) => { let result = a.as_ref().map(|x| op(x)); - Ok(ColumnarValue::Scalar(ScalarValue::Utf8(result))) + Ok(ColumnarValue::from(ScalarValue::Utf8(result))) } other => exec_err!("Unsupported data type {other:?} for function {name}"), }, diff --git a/datafusion/functions/src/string/concat.rs b/datafusion/functions/src/string/concat.rs index 98f57efef90d..8a9fae7e9f79 100644 --- a/datafusion/functions/src/string/concat.rs +++ b/datafusion/functions/src/string/concat.rs @@ -86,13 +86,13 @@ impl ScalarUDFImpl for ConcatFunc { fn invoke(&self, args: &[ColumnarValue]) -> Result { let mut return_datatype = DataType::Utf8; args.iter().for_each(|col| { - if col.data_type() == DataType::Utf8View { - return_datatype = col.data_type(); + if col.data_type() == &DataType::Utf8View { + return_datatype = col.data_type().clone(); } - if col.data_type() == DataType::LargeUtf8 + if col.data_type() == &DataType::LargeUtf8 && return_datatype != DataType::Utf8View { - return_datatype = col.data_type(); + return_datatype = col.data_type().clone(); } }); @@ -108,20 +108,22 @@ impl ScalarUDFImpl for ConcatFunc { if array_len.is_none() { let mut result = String::new(); for arg in args { - if let ColumnarValue::Scalar(ScalarValue::Utf8(Some(v))) = arg { - result.push_str(v); + if let ColumnarValue::Scalar(scalar) = arg { + if let ScalarValue::Utf8(Some(v)) = scalar.value() { + result.push_str(v); + } } } return match return_datatype { DataType::Utf8View => { - Ok(ColumnarValue::Scalar(ScalarValue::Utf8View(Some(result)))) + Ok(ColumnarValue::from(ScalarValue::Utf8View(Some(result)))) } DataType::Utf8 => { - Ok(ColumnarValue::Scalar(ScalarValue::Utf8(Some(result)))) + Ok(ColumnarValue::from(ScalarValue::Utf8(Some(result)))) } DataType::LargeUtf8 => { - Ok(ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some(result)))) + Ok(ColumnarValue::from(ScalarValue::LargeUtf8(Some(result)))) } other => { plan_err!("Concat function does not support datatype of {other}") @@ -136,14 +138,17 @@ impl ScalarUDFImpl for ConcatFunc { for arg in args { match arg { - ColumnarValue::Scalar(ScalarValue::Utf8(maybe_value)) - | ColumnarValue::Scalar(ScalarValue::LargeUtf8(maybe_value)) - | ColumnarValue::Scalar(ScalarValue::Utf8View(maybe_value)) => { - if let Some(s) = maybe_value { - data_size += s.len() * len; - columns.push(ColumnarValueRef::Scalar(s.as_bytes())); + ColumnarValue::Scalar(scalar) => match scalar.value() { + ScalarValue::Utf8(maybe_value) + | ScalarValue::LargeUtf8(maybe_value) + | ScalarValue::Utf8View(maybe_value) => { + if let Some(s) = maybe_value { + data_size += s.len() * len; + columns.push(ColumnarValueRef::Scalar(s.as_bytes())); + } } - } + _ => unreachable!(), + }, ColumnarValue::Array(array) => { match array.data_type() { DataType::Utf8 => { @@ -184,7 +189,6 @@ impl ScalarUDFImpl for ConcatFunc { } }; } - _ => unreachable!(), } } @@ -306,9 +310,9 @@ mod tests { test_function!( ConcatFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("aa")), - ColumnarValue::Scalar(ScalarValue::from("bb")), - ColumnarValue::Scalar(ScalarValue::from("cc")), + ColumnarValue::from(ScalarValue::from("aa")), + ColumnarValue::from(ScalarValue::from("bb")), + ColumnarValue::from(ScalarValue::from("cc")), ], Ok(Some("aabbcc")), &str, @@ -318,9 +322,9 @@ mod tests { test_function!( ConcatFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("aa")), - ColumnarValue::Scalar(ScalarValue::Utf8(None)), - ColumnarValue::Scalar(ScalarValue::from("cc")), + ColumnarValue::from(ScalarValue::from("aa")), + ColumnarValue::from(ScalarValue::Utf8(None)), + ColumnarValue::from(ScalarValue::from("cc")), ], Ok(Some("aacc")), &str, @@ -329,7 +333,7 @@ mod tests { ); test_function!( ConcatFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8(None))], + &[ColumnarValue::from(ScalarValue::Utf8(None))], Ok(Some("")), &str, Utf8, @@ -338,10 +342,10 @@ mod tests { test_function!( ConcatFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("aa")), - ColumnarValue::Scalar(ScalarValue::Utf8View(None)), - ColumnarValue::Scalar(ScalarValue::LargeUtf8(None)), - ColumnarValue::Scalar(ScalarValue::from("cc")), + ColumnarValue::from(ScalarValue::from("aa")), + ColumnarValue::from(ScalarValue::Utf8View(None)), + ColumnarValue::from(ScalarValue::LargeUtf8(None)), + ColumnarValue::from(ScalarValue::from("cc")), ], Ok(Some("aacc")), &str, @@ -351,9 +355,9 @@ mod tests { test_function!( ConcatFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("aa")), - ColumnarValue::Scalar(ScalarValue::LargeUtf8(None)), - ColumnarValue::Scalar(ScalarValue::from("cc")), + ColumnarValue::from(ScalarValue::from("aa")), + ColumnarValue::from(ScalarValue::LargeUtf8(None)), + ColumnarValue::from(ScalarValue::from("cc")), ], Ok(Some("aacc")), &str, @@ -368,7 +372,7 @@ mod tests { fn concat() -> Result<()> { let c0 = ColumnarValue::Array(Arc::new(StringArray::from(vec!["foo", "bar", "baz"]))); - let c1 = ColumnarValue::Scalar(ScalarValue::Utf8(Some(",".to_string()))); + let c1 = ColumnarValue::from(ScalarValue::Utf8(Some(",".to_string()))); let c2 = ColumnarValue::Array(Arc::new(StringArray::from(vec![ Some("x"), None, diff --git a/datafusion/functions/src/string/concat_ws.rs b/datafusion/functions/src/string/concat_ws.rs index 1134c525cfca..6503a54e65c1 100644 --- a/datafusion/functions/src/string/concat_ws.rs +++ b/datafusion/functions/src/string/concat_ws.rs @@ -94,14 +94,17 @@ impl ScalarUDFImpl for ConcatWsFunc { // Scalar if array_len.is_none() { let sep = match &args[0] { - ColumnarValue::Scalar(ScalarValue::Utf8(Some(s))) - | ColumnarValue::Scalar(ScalarValue::Utf8View(Some(s))) - | ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some(s))) => s, - ColumnarValue::Scalar(ScalarValue::Utf8(None)) - | ColumnarValue::Scalar(ScalarValue::Utf8View(None)) - | ColumnarValue::Scalar(ScalarValue::LargeUtf8(None)) => { - return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None))); - } + ColumnarValue::Scalar(scalar) => match scalar.value() { + ScalarValue::Utf8(Some(s)) + | ScalarValue::Utf8View(Some(s)) + | ScalarValue::LargeUtf8(Some(s)) => s, + ScalarValue::Utf8(None) + | ScalarValue::Utf8View(None) + | ScalarValue::LargeUtf8(None) => { + return Ok(ColumnarValue::from(ScalarValue::Utf8(None))); + } + _ => unreachable!(), + }, _ => unreachable!(), }; @@ -110,35 +113,41 @@ impl ScalarUDFImpl for ConcatWsFunc { for arg in iter.by_ref() { match arg { - ColumnarValue::Scalar(ScalarValue::Utf8(Some(s))) - | ColumnarValue::Scalar(ScalarValue::Utf8View(Some(s))) - | ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some(s))) => { - result.push_str(s); - break; - } - ColumnarValue::Scalar(ScalarValue::Utf8(None)) - | ColumnarValue::Scalar(ScalarValue::Utf8View(None)) - | ColumnarValue::Scalar(ScalarValue::LargeUtf8(None)) => {} + ColumnarValue::Scalar(scalar) => match scalar.value() { + ScalarValue::Utf8(Some(s)) + | ScalarValue::Utf8View(Some(s)) + | ScalarValue::LargeUtf8(Some(s)) => { + result.push_str(s); + break; + } + ScalarValue::Utf8(None) + | ScalarValue::Utf8View(None) + | ScalarValue::LargeUtf8(None) => {} + _ => unreachable!(), + }, _ => unreachable!(), } } for arg in iter.by_ref() { match arg { - ColumnarValue::Scalar(ScalarValue::Utf8(Some(s))) - | ColumnarValue::Scalar(ScalarValue::Utf8View(Some(s))) - | ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some(s))) => { - result.push_str(sep); - result.push_str(s); - } - ColumnarValue::Scalar(ScalarValue::Utf8(None)) - | ColumnarValue::Scalar(ScalarValue::Utf8View(None)) - | ColumnarValue::Scalar(ScalarValue::LargeUtf8(None)) => {} + ColumnarValue::Scalar(scalar) => match scalar.value() { + ScalarValue::Utf8(Some(s)) + | ScalarValue::Utf8View(Some(s)) + | ScalarValue::LargeUtf8(Some(s)) => { + result.push_str(sep); + result.push_str(s); + } + ScalarValue::Utf8(None) + | ScalarValue::Utf8View(None) + | ScalarValue::LargeUtf8(None) => {} + _ => unreachable!(), + }, _ => unreachable!(), } } - return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(Some(result)))); + return Ok(ColumnarValue::from(ScalarValue::Utf8(Some(result)))); } // Array @@ -147,13 +156,18 @@ impl ScalarUDFImpl for ConcatWsFunc { // parse sep let sep = match &args[0] { - ColumnarValue::Scalar(ScalarValue::Utf8(Some(s))) => { - data_size += s.len() * len * (args.len() - 2); // estimate - ColumnarValueRef::Scalar(s.as_bytes()) - } - ColumnarValue::Scalar(ScalarValue::Utf8(None)) => { - return Ok(ColumnarValue::Array(Arc::new(StringArray::new_null(len)))); - } + ColumnarValue::Scalar(scalar) => match scalar.value() { + ScalarValue::Utf8(Some(s)) => { + data_size += s.len() * len * (args.len() - 2); // estimate + ColumnarValueRef::Scalar(s.as_bytes()) + } + ScalarValue::Utf8(None) => { + return Ok(ColumnarValue::Array(Arc::new(StringArray::new_null( + len, + )))); + } + _ => unreachable!(), + }, ColumnarValue::Array(array) => { let string_array = as_string_array(array)?; data_size += string_array.values().len() * (args.len() - 2); // estimate @@ -163,20 +177,22 @@ impl ScalarUDFImpl for ConcatWsFunc { ColumnarValueRef::NonNullableArray(string_array) } } - _ => unreachable!(), }; let mut columns = Vec::with_capacity(args.len() - 1); for arg in &args[1..] { match arg { - ColumnarValue::Scalar(ScalarValue::Utf8(maybe_value)) - | ColumnarValue::Scalar(ScalarValue::LargeUtf8(maybe_value)) - | ColumnarValue::Scalar(ScalarValue::Utf8View(maybe_value)) => { - if let Some(s) = maybe_value { - data_size += s.len() * len; - columns.push(ColumnarValueRef::Scalar(s.as_bytes())); + ColumnarValue::Scalar(scalar) => match scalar.value() { + ScalarValue::Utf8(maybe_value) + | ScalarValue::Utf8View(maybe_value) + | ScalarValue::LargeUtf8(maybe_value) => { + if let Some(s) = maybe_value { + data_size += s.len() * len; + columns.push(ColumnarValueRef::Scalar(s.as_bytes())); + } } - } + _ => unreachable!(), + }, ColumnarValue::Array(array) => { match array.data_type() { DataType::Utf8 => { @@ -217,7 +233,6 @@ impl ScalarUDFImpl for ConcatWsFunc { } }; } - _ => unreachable!(), } } @@ -366,10 +381,10 @@ mod tests { test_function!( ConcatWsFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("|")), - ColumnarValue::Scalar(ScalarValue::from("aa")), - ColumnarValue::Scalar(ScalarValue::from("bb")), - ColumnarValue::Scalar(ScalarValue::from("cc")), + ColumnarValue::from(ScalarValue::from("|")), + ColumnarValue::from(ScalarValue::from("aa")), + ColumnarValue::from(ScalarValue::from("bb")), + ColumnarValue::from(ScalarValue::from("cc")), ], Ok(Some("aa|bb|cc")), &str, @@ -379,8 +394,8 @@ mod tests { test_function!( ConcatWsFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("|")), - ColumnarValue::Scalar(ScalarValue::Utf8(None)), + ColumnarValue::from(ScalarValue::from("|")), + ColumnarValue::from(ScalarValue::Utf8(None)), ], Ok(Some("")), &str, @@ -390,10 +405,10 @@ mod tests { test_function!( ConcatWsFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8(None)), - ColumnarValue::Scalar(ScalarValue::from("aa")), - ColumnarValue::Scalar(ScalarValue::from("bb")), - ColumnarValue::Scalar(ScalarValue::from("cc")), + ColumnarValue::from(ScalarValue::Utf8(None)), + ColumnarValue::from(ScalarValue::from("aa")), + ColumnarValue::from(ScalarValue::from("bb")), + ColumnarValue::from(ScalarValue::from("cc")), ], Ok(None), &str, @@ -403,10 +418,10 @@ mod tests { test_function!( ConcatWsFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("|")), - ColumnarValue::Scalar(ScalarValue::from("aa")), - ColumnarValue::Scalar(ScalarValue::Utf8(None)), - ColumnarValue::Scalar(ScalarValue::from("cc")), + ColumnarValue::from(ScalarValue::from("|")), + ColumnarValue::from(ScalarValue::from("aa")), + ColumnarValue::from(ScalarValue::Utf8(None)), + ColumnarValue::from(ScalarValue::from("cc")), ], Ok(Some("aa|cc")), &str, @@ -420,7 +435,7 @@ mod tests { #[test] fn concat_ws() -> Result<()> { // sep is scalar - let c0 = ColumnarValue::Scalar(ScalarValue::Utf8(Some(",".to_string()))); + let c0 = ColumnarValue::from(ScalarValue::Utf8(Some(",".to_string()))); let c1 = ColumnarValue::Array(Arc::new(StringArray::from(vec!["foo", "bar", "baz"]))); let c2 = ColumnarValue::Array(Arc::new(StringArray::from(vec![ diff --git a/datafusion/functions/src/string/contains.rs b/datafusion/functions/src/string/contains.rs index c319f80661c3..4133552160d4 100644 --- a/datafusion/functions/src/string/contains.rs +++ b/datafusion/functions/src/string/contains.rs @@ -209,8 +209,8 @@ mod tests { test_function!( ContainsFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from("alph")), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from("alph")), ], Ok(Some(true)), bool, @@ -220,8 +220,8 @@ mod tests { test_function!( ContainsFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from("dddddd")), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from("dddddd")), ], Ok(Some(false)), bool, @@ -231,8 +231,8 @@ mod tests { test_function!( ContainsFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from("pha")), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from("pha")), ], Ok(Some(true)), bool, @@ -243,10 +243,8 @@ mod tests { test_function!( ContainsFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View(Some(String::from( - "Apache" - )))), - ColumnarValue::Scalar(ScalarValue::Utf8View(Some(String::from("pac")))), + ColumnarValue::from(ScalarValue::Utf8View(Some(String::from("Apache")))), + ColumnarValue::from(ScalarValue::Utf8View(Some(String::from("pac")))), ], Ok(Some(true)), bool, @@ -256,10 +254,8 @@ mod tests { test_function!( ContainsFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View(Some(String::from( - "Apache" - )))), - ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::from("ap")))), + ColumnarValue::from(ScalarValue::Utf8View(Some(String::from("Apache")))), + ColumnarValue::from(ScalarValue::Utf8(Some(String::from("ap")))), ], Ok(Some(false)), bool, @@ -269,10 +265,8 @@ mod tests { test_function!( ContainsFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View(Some(String::from( - "Apache" - )))), - ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some(String::from( + ColumnarValue::from(ScalarValue::Utf8View(Some(String::from("Apache")))), + ColumnarValue::from(ScalarValue::LargeUtf8(Some(String::from( "DataFusion" )))), ], diff --git a/datafusion/functions/src/string/ends_with.rs b/datafusion/functions/src/string/ends_with.rs index 03a1795954d0..82fde772c282 100644 --- a/datafusion/functions/src/string/ends_with.rs +++ b/datafusion/functions/src/string/ends_with.rs @@ -111,8 +111,8 @@ mod tests { test_function!( EndsWithFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from("alph")), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from("alph")), ], Ok(Some(false)), bool, @@ -122,8 +122,8 @@ mod tests { test_function!( EndsWithFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from("bet")), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from("bet")), ], Ok(Some(true)), bool, @@ -133,8 +133,8 @@ mod tests { test_function!( EndsWithFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8(None)), - ColumnarValue::Scalar(ScalarValue::from("alph")), + ColumnarValue::from(ScalarValue::Utf8(None)), + ColumnarValue::from(ScalarValue::from("alph")), ], Ok(None), bool, @@ -144,8 +144,8 @@ mod tests { test_function!( EndsWithFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::Utf8(None)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::Utf8(None)), ], Ok(None), bool, diff --git a/datafusion/functions/src/string/initcap.rs b/datafusion/functions/src/string/initcap.rs index 4e1eb213ef57..081fa94af8d9 100644 --- a/datafusion/functions/src/string/initcap.rs +++ b/datafusion/functions/src/string/initcap.rs @@ -137,7 +137,7 @@ mod tests { fn test_functions() -> Result<()> { test_function!( InitcapFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::from("hi THOMAS"))], + &[ColumnarValue::from(ScalarValue::from("hi THOMAS"))], Ok(Some("Hi Thomas")), &str, Utf8, @@ -145,7 +145,7 @@ mod tests { ); test_function!( InitcapFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::from(""))], + &[ColumnarValue::from(ScalarValue::from(""))], Ok(Some("")), &str, Utf8, @@ -153,7 +153,7 @@ mod tests { ); test_function!( InitcapFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::from(""))], + &[ColumnarValue::from(ScalarValue::from(""))], Ok(Some("")), &str, Utf8, @@ -161,7 +161,7 @@ mod tests { ); test_function!( InitcapFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8(None))], + &[ColumnarValue::from(ScalarValue::Utf8(None))], Ok(None), &str, Utf8, @@ -169,7 +169,7 @@ mod tests { ); test_function!( InitcapFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8View(Some( + &[ColumnarValue::from(ScalarValue::Utf8View(Some( "hi THOMAS".to_string() )))], Ok(Some("Hi Thomas")), @@ -179,7 +179,7 @@ mod tests { ); test_function!( InitcapFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8View(Some( + &[ColumnarValue::from(ScalarValue::Utf8View(Some( "hi THOMAS wIth M0re ThAN 12 ChaRs".to_string() )))], Ok(Some("Hi Thomas With M0re Than 12 Chars")), @@ -189,7 +189,7 @@ mod tests { ); test_function!( InitcapFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8View(Some( + &[ColumnarValue::from(ScalarValue::Utf8View(Some( "".to_string() )))], Ok(Some("")), @@ -199,7 +199,7 @@ mod tests { ); test_function!( InitcapFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8View(None))], + &[ColumnarValue::from(ScalarValue::Utf8View(None))], Ok(None), &str, Utf8, diff --git a/datafusion/functions/src/string/octet_length.rs b/datafusion/functions/src/string/octet_length.rs index f792914d862e..93e62961bca8 100644 --- a/datafusion/functions/src/string/octet_length.rs +++ b/datafusion/functions/src/string/octet_length.rs @@ -77,16 +77,16 @@ impl ScalarUDFImpl for OctetLengthFunc { match &args[0] { ColumnarValue::Array(v) => Ok(ColumnarValue::Array(length(v.as_ref())?)), - ColumnarValue::Scalar(v) => match v { - ScalarValue::Utf8(v) => Ok(ColumnarValue::Scalar(ScalarValue::Int32( + ColumnarValue::Scalar(v) => match v.value() { + ScalarValue::Utf8(v) => Ok(ColumnarValue::from(ScalarValue::Int32( + v.as_ref().map(|x| x.len() as i32), + ))), + ScalarValue::LargeUtf8(v) => Ok(ColumnarValue::from(ScalarValue::Int64( + v.as_ref().map(|x| x.len() as i64), + ))), + ScalarValue::Utf8View(v) => Ok(ColumnarValue::from(ScalarValue::Int32( v.as_ref().map(|x| x.len() as i32), ))), - ScalarValue::LargeUtf8(v) => Ok(ColumnarValue::Scalar( - ScalarValue::Int64(v.as_ref().map(|x| x.len() as i64)), - )), - ScalarValue::Utf8View(v) => Ok(ColumnarValue::Scalar( - ScalarValue::Int32(v.as_ref().map(|x| x.len() as i32)), - )), _ => unreachable!(), }, } @@ -111,7 +111,7 @@ mod tests { fn test_functions() -> Result<()> { test_function!( OctetLengthFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Int32(Some(12)))], + &[ColumnarValue::from(ScalarValue::Int32(Some(12)))], exec_err!( "The OCTET_LENGTH function can only accept strings, but got Int32." ), @@ -133,8 +133,8 @@ mod tests { test_function!( OctetLengthFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::from("chars")))), - ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::from("chars")))) + ColumnarValue::from(ScalarValue::Utf8(Some(String::from("chars")))), + ColumnarValue::from(ScalarValue::Utf8(Some(String::from("chars")))) ], exec_err!("octet_length function requires 1 argument, got 2"), i32, @@ -143,9 +143,9 @@ mod tests { ); test_function!( OctetLengthFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8(Some( - String::from("chars") - )))], + &[ColumnarValue::from(ScalarValue::Utf8(Some(String::from( + "chars" + ))))], Ok(Some(5)), i32, Int32, @@ -153,9 +153,9 @@ mod tests { ); test_function!( OctetLengthFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8(Some( - String::from("josé") - )))], + &[ColumnarValue::from(ScalarValue::Utf8(Some(String::from( + "josé" + ))))], Ok(Some(5)), i32, Int32, @@ -163,9 +163,9 @@ mod tests { ); test_function!( OctetLengthFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8(Some( - String::from("") - )))], + &[ColumnarValue::from(ScalarValue::Utf8(Some(String::from( + "" + ))))], Ok(Some(0)), i32, Int32, @@ -173,7 +173,7 @@ mod tests { ); test_function!( OctetLengthFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8(None))], + &[ColumnarValue::from(ScalarValue::Utf8(None))], Ok(None), i32, Int32, @@ -181,7 +181,7 @@ mod tests { ); test_function!( OctetLengthFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8View(Some( + &[ColumnarValue::from(ScalarValue::Utf8View(Some( String::from("joséjoséjoséjosé") )))], Ok(Some(20)), @@ -191,7 +191,7 @@ mod tests { ); test_function!( OctetLengthFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8View(Some( + &[ColumnarValue::from(ScalarValue::Utf8View(Some( String::from("josé") )))], Ok(Some(5)), @@ -201,7 +201,7 @@ mod tests { ); test_function!( OctetLengthFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8View(Some( + &[ColumnarValue::from(ScalarValue::Utf8View(Some( String::from("") )))], Ok(Some(0)), diff --git a/datafusion/functions/src/string/repeat.rs b/datafusion/functions/src/string/repeat.rs index 20e4462784b8..0788e0428d7d 100644 --- a/datafusion/functions/src/string/repeat.rs +++ b/datafusion/functions/src/string/repeat.rs @@ -147,8 +147,8 @@ mod tests { test_function!( RepeatFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::from("Pg")))), - ColumnarValue::Scalar(ScalarValue::Int64(Some(4))), + ColumnarValue::from(ScalarValue::Utf8(Some(String::from("Pg")))), + ColumnarValue::from(ScalarValue::Int64(Some(4))), ], Ok(Some("PgPgPgPg")), &str, @@ -158,8 +158,8 @@ mod tests { test_function!( RepeatFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8(None)), - ColumnarValue::Scalar(ScalarValue::Int64(Some(4))), + ColumnarValue::from(ScalarValue::Utf8(None)), + ColumnarValue::from(ScalarValue::Int64(Some(4))), ], Ok(None), &str, @@ -169,8 +169,8 @@ mod tests { test_function!( RepeatFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::from("Pg")))), - ColumnarValue::Scalar(ScalarValue::Int64(None)), + ColumnarValue::from(ScalarValue::Utf8(Some(String::from("Pg")))), + ColumnarValue::from(ScalarValue::Int64(None)), ], Ok(None), &str, @@ -181,8 +181,8 @@ mod tests { test_function!( RepeatFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View(Some(String::from("Pg")))), - ColumnarValue::Scalar(ScalarValue::Int64(Some(4))), + ColumnarValue::from(ScalarValue::Utf8View(Some(String::from("Pg")))), + ColumnarValue::from(ScalarValue::Int64(Some(4))), ], Ok(Some("PgPgPgPg")), &str, @@ -192,8 +192,8 @@ mod tests { test_function!( RepeatFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View(None)), - ColumnarValue::Scalar(ScalarValue::Int64(Some(4))), + ColumnarValue::from(ScalarValue::Utf8View(None)), + ColumnarValue::from(ScalarValue::Int64(Some(4))), ], Ok(None), &str, @@ -203,8 +203,8 @@ mod tests { test_function!( RepeatFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View(Some(String::from("Pg")))), - ColumnarValue::Scalar(ScalarValue::Int64(None)), + ColumnarValue::from(ScalarValue::Utf8View(Some(String::from("Pg")))), + ColumnarValue::from(ScalarValue::Int64(None)), ], Ok(None), &str, diff --git a/datafusion/functions/src/string/replace.rs b/datafusion/functions/src/string/replace.rs index 13fa3d55672d..7d3109ab63b9 100644 --- a/datafusion/functions/src/string/replace.rs +++ b/datafusion/functions/src/string/replace.rs @@ -136,9 +136,9 @@ mod tests { test_function!( ReplaceFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::from("aabbdqcbb")))), - ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::from("bb")))), - ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::from("ccc")))), + ColumnarValue::from(ScalarValue::Utf8(Some(String::from("aabbdqcbb")))), + ColumnarValue::from(ScalarValue::Utf8(Some(String::from("bb")))), + ColumnarValue::from(ScalarValue::Utf8(Some(String::from("ccc")))), ], Ok(Some("aacccdqcccc")), &str, @@ -149,11 +149,9 @@ mod tests { test_function!( ReplaceFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some(String::from( - "aabbb" - )))), - ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some(String::from("bbb")))), - ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some(String::from("cc")))), + ColumnarValue::from(ScalarValue::LargeUtf8(Some(String::from("aabbb")))), + ColumnarValue::from(ScalarValue::LargeUtf8(Some(String::from("bbb")))), + ColumnarValue::from(ScalarValue::LargeUtf8(Some(String::from("cc")))), ], Ok(Some("aacc")), &str, @@ -164,11 +162,9 @@ mod tests { test_function!( ReplaceFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View(Some(String::from( - "aabbbcw" - )))), - ColumnarValue::Scalar(ScalarValue::Utf8View(Some(String::from("bb")))), - ColumnarValue::Scalar(ScalarValue::Utf8View(Some(String::from("cc")))), + ColumnarValue::from(ScalarValue::Utf8View(Some(String::from("aabbbcw")))), + ColumnarValue::from(ScalarValue::Utf8View(Some(String::from("bb")))), + ColumnarValue::from(ScalarValue::Utf8View(Some(String::from("cc")))), ], Ok(Some("aaccbcw")), &str, diff --git a/datafusion/functions/src/string/split_part.rs b/datafusion/functions/src/string/split_part.rs index 8d292315a35a..438e2e611359 100644 --- a/datafusion/functions/src/string/split_part.rs +++ b/datafusion/functions/src/string/split_part.rs @@ -173,7 +173,7 @@ impl ScalarUDFImpl for SplitPartFunc { if is_scalar { // If all inputs are scalar, keep the output as scalar let result = result.and_then(|arr| ScalarValue::try_from_array(&arr, 0)); - result.map(ColumnarValue::Scalar) + result.map(ColumnarValue::from) } else { result.map(ColumnarValue::Array) } @@ -242,11 +242,11 @@ mod tests { test_function!( SplitPartFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::from( + ColumnarValue::from(ScalarValue::Utf8(Some(String::from( "abc~@~def~@~ghi" )))), - ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::from("~@~")))), - ColumnarValue::Scalar(ScalarValue::Int64(Some(2))), + ColumnarValue::from(ScalarValue::Utf8(Some(String::from("~@~")))), + ColumnarValue::from(ScalarValue::Int64(Some(2))), ], Ok(Some("def")), &str, @@ -256,11 +256,11 @@ mod tests { test_function!( SplitPartFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::from( + ColumnarValue::from(ScalarValue::Utf8(Some(String::from( "abc~@~def~@~ghi" )))), - ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::from("~@~")))), - ColumnarValue::Scalar(ScalarValue::Int64(Some(20))), + ColumnarValue::from(ScalarValue::Utf8(Some(String::from("~@~")))), + ColumnarValue::from(ScalarValue::Int64(Some(20))), ], Ok(Some("")), &str, @@ -270,11 +270,11 @@ mod tests { test_function!( SplitPartFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::from( + ColumnarValue::from(ScalarValue::Utf8(Some(String::from( "abc~@~def~@~ghi" )))), - ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::from("~@~")))), - ColumnarValue::Scalar(ScalarValue::Int64(Some(-1))), + ColumnarValue::from(ScalarValue::Utf8(Some(String::from("~@~")))), + ColumnarValue::from(ScalarValue::Int64(Some(-1))), ], Ok(Some("ghi")), &str, @@ -284,11 +284,11 @@ mod tests { test_function!( SplitPartFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::from( + ColumnarValue::from(ScalarValue::Utf8(Some(String::from( "abc~@~def~@~ghi" )))), - ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::from("~@~")))), - ColumnarValue::Scalar(ScalarValue::Int64(Some(0))), + ColumnarValue::from(ScalarValue::Utf8(Some(String::from("~@~")))), + ColumnarValue::from(ScalarValue::Int64(Some(0))), ], exec_err!("field position must not be zero"), &str, diff --git a/datafusion/functions/src/string/starts_with.rs b/datafusion/functions/src/string/starts_with.rs index 8450697cbf30..e0c0fbdb3df7 100644 --- a/datafusion/functions/src/string/starts_with.rs +++ b/datafusion/functions/src/string/starts_with.rs @@ -117,18 +117,18 @@ mod tests { .into_iter() .flat_map(|(a, b, c)| { let utf_8_args = vec![ - ColumnarValue::Scalar(ScalarValue::Utf8(a.map(|s| s.to_string()))), - ColumnarValue::Scalar(ScalarValue::Utf8(b.map(|s| s.to_string()))), + ColumnarValue::from(ScalarValue::Utf8(a.map(|s| s.to_string()))), + ColumnarValue::from(ScalarValue::Utf8(b.map(|s| s.to_string()))), ]; let large_utf_8_args = vec![ - ColumnarValue::Scalar(ScalarValue::LargeUtf8(a.map(|s| s.to_string()))), - ColumnarValue::Scalar(ScalarValue::LargeUtf8(b.map(|s| s.to_string()))), + ColumnarValue::from(ScalarValue::LargeUtf8(a.map(|s| s.to_string()))), + ColumnarValue::from(ScalarValue::LargeUtf8(b.map(|s| s.to_string()))), ]; let utf_8_view_args = vec![ - ColumnarValue::Scalar(ScalarValue::Utf8View(a.map(|s| s.to_string()))), - ColumnarValue::Scalar(ScalarValue::Utf8View(b.map(|s| s.to_string()))), + ColumnarValue::from(ScalarValue::Utf8View(a.map(|s| s.to_string()))), + ColumnarValue::from(ScalarValue::Utf8View(b.map(|s| s.to_string()))), ]; vec![(utf_8_args, c), (large_utf_8_args, c), (utf_8_view_args, c)] diff --git a/datafusion/functions/src/unicode/character_length.rs b/datafusion/functions/src/unicode/character_length.rs index c9dc96b2a935..768e88c019a6 100644 --- a/datafusion/functions/src/unicode/character_length.rs +++ b/datafusion/functions/src/unicode/character_length.rs @@ -139,7 +139,7 @@ mod tests { ($INPUT:expr, $EXPECTED:expr) => { test_function!( CharacterLengthFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8($INPUT))], + &[ColumnarValue::from(ScalarValue::Utf8($INPUT))], $EXPECTED, i32, Int32, @@ -148,7 +148,7 @@ mod tests { test_function!( CharacterLengthFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::LargeUtf8($INPUT))], + &[ColumnarValue::from(ScalarValue::LargeUtf8($INPUT))], $EXPECTED, i64, Int64, @@ -157,7 +157,7 @@ mod tests { test_function!( CharacterLengthFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8View($INPUT))], + &[ColumnarValue::from(ScalarValue::Utf8View($INPUT))], $EXPECTED, i32, Int32, @@ -181,7 +181,7 @@ mod tests { #[cfg(not(feature = "unicode_expressions"))] test_function!( CharacterLengthFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::from("josé"))))], + &[ColumnarValue::from(ScalarValue::Utf8(Some(String::from("josé"))))], internal_err!( "function character_length requires compilation with feature flag: unicode_expressions." ), diff --git a/datafusion/functions/src/unicode/left.rs b/datafusion/functions/src/unicode/left.rs index c49784948dd0..f1f84c98ef5e 100644 --- a/datafusion/functions/src/unicode/left.rs +++ b/datafusion/functions/src/unicode/left.rs @@ -153,8 +153,8 @@ mod tests { test_function!( LeftFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("abcde")), - ColumnarValue::Scalar(ScalarValue::from(2i64)), + ColumnarValue::from(ScalarValue::from("abcde")), + ColumnarValue::from(ScalarValue::from(2i64)), ], Ok(Some("ab")), &str, @@ -164,8 +164,8 @@ mod tests { test_function!( LeftFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("abcde")), - ColumnarValue::Scalar(ScalarValue::from(200i64)), + ColumnarValue::from(ScalarValue::from("abcde")), + ColumnarValue::from(ScalarValue::from(200i64)), ], Ok(Some("abcde")), &str, @@ -175,8 +175,8 @@ mod tests { test_function!( LeftFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("abcde")), - ColumnarValue::Scalar(ScalarValue::from(-2i64)), + ColumnarValue::from(ScalarValue::from("abcde")), + ColumnarValue::from(ScalarValue::from(-2i64)), ], Ok(Some("abc")), &str, @@ -186,8 +186,8 @@ mod tests { test_function!( LeftFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("abcde")), - ColumnarValue::Scalar(ScalarValue::from(-200i64)), + ColumnarValue::from(ScalarValue::from("abcde")), + ColumnarValue::from(ScalarValue::from(-200i64)), ], Ok(Some("")), &str, @@ -197,8 +197,8 @@ mod tests { test_function!( LeftFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("abcde")), - ColumnarValue::Scalar(ScalarValue::from(0i64)), + ColumnarValue::from(ScalarValue::from("abcde")), + ColumnarValue::from(ScalarValue::from(0i64)), ], Ok(Some("")), &str, @@ -208,8 +208,8 @@ mod tests { test_function!( LeftFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8(None)), - ColumnarValue::Scalar(ScalarValue::from(2i64)), + ColumnarValue::from(ScalarValue::Utf8(None)), + ColumnarValue::from(ScalarValue::from(2i64)), ], Ok(None), &str, @@ -219,8 +219,8 @@ mod tests { test_function!( LeftFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("abcde")), - ColumnarValue::Scalar(ScalarValue::Int64(None)), + ColumnarValue::from(ScalarValue::from("abcde")), + ColumnarValue::from(ScalarValue::Int64(None)), ], Ok(None), &str, @@ -230,8 +230,8 @@ mod tests { test_function!( LeftFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("joséésoj")), - ColumnarValue::Scalar(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::from("joséésoj")), + ColumnarValue::from(ScalarValue::from(5i64)), ], Ok(Some("joséé")), &str, @@ -241,8 +241,8 @@ mod tests { test_function!( LeftFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("joséésoj")), - ColumnarValue::Scalar(ScalarValue::from(-3i64)), + ColumnarValue::from(ScalarValue::from("joséésoj")), + ColumnarValue::from(ScalarValue::from(-3i64)), ], Ok(Some("joséé")), &str, @@ -253,8 +253,8 @@ mod tests { test_function!( LeftFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("abcde")), - ColumnarValue::Scalar(ScalarValue::from(2i64)), + ColumnarValue::from(ScalarValue::from("abcde")), + ColumnarValue::from(ScalarValue::from(2i64)), ], internal_err!( "function left requires compilation with feature flag: unicode_expressions." diff --git a/datafusion/functions/src/unicode/lpad.rs b/datafusion/functions/src/unicode/lpad.rs index e102673c4253..5dfe4e0d82aa 100644 --- a/datafusion/functions/src/unicode/lpad.rs +++ b/datafusion/functions/src/unicode/lpad.rs @@ -265,8 +265,8 @@ mod tests { test_function!( LPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8($INPUT)), - ColumnarValue::Scalar($LENGTH) + ColumnarValue::from(ScalarValue::Utf8($INPUT)), + ColumnarValue::from($LENGTH) ], $EXPECTED, &str, @@ -277,8 +277,8 @@ mod tests { test_function!( LPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::LargeUtf8($INPUT)), - ColumnarValue::Scalar($LENGTH) + ColumnarValue::from(ScalarValue::LargeUtf8($INPUT)), + ColumnarValue::from($LENGTH) ], $EXPECTED, &str, @@ -289,8 +289,8 @@ mod tests { test_function!( LPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View($INPUT)), - ColumnarValue::Scalar($LENGTH) + ColumnarValue::from(ScalarValue::Utf8View($INPUT)), + ColumnarValue::from($LENGTH) ], $EXPECTED, &str, @@ -304,9 +304,9 @@ mod tests { test_function!( LPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8($INPUT)), - ColumnarValue::Scalar($LENGTH), - ColumnarValue::Scalar(ScalarValue::Utf8($REPLACE)) + ColumnarValue::from(ScalarValue::Utf8($INPUT)), + ColumnarValue::from($LENGTH), + ColumnarValue::from(ScalarValue::Utf8($REPLACE)) ], $EXPECTED, &str, @@ -317,9 +317,9 @@ mod tests { test_function!( LPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8($INPUT)), - ColumnarValue::Scalar($LENGTH), - ColumnarValue::Scalar(ScalarValue::LargeUtf8($REPLACE)) + ColumnarValue::from(ScalarValue::Utf8($INPUT)), + ColumnarValue::from($LENGTH), + ColumnarValue::from(ScalarValue::LargeUtf8($REPLACE)) ], $EXPECTED, &str, @@ -330,9 +330,9 @@ mod tests { test_function!( LPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8($INPUT)), - ColumnarValue::Scalar($LENGTH), - ColumnarValue::Scalar(ScalarValue::Utf8View($REPLACE)) + ColumnarValue::from(ScalarValue::Utf8($INPUT)), + ColumnarValue::from($LENGTH), + ColumnarValue::from(ScalarValue::Utf8View($REPLACE)) ], $EXPECTED, &str, @@ -344,9 +344,9 @@ mod tests { test_function!( LPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::LargeUtf8($INPUT)), - ColumnarValue::Scalar($LENGTH), - ColumnarValue::Scalar(ScalarValue::Utf8($REPLACE)) + ColumnarValue::from(ScalarValue::LargeUtf8($INPUT)), + ColumnarValue::from($LENGTH), + ColumnarValue::from(ScalarValue::Utf8($REPLACE)) ], $EXPECTED, &str, @@ -357,9 +357,9 @@ mod tests { test_function!( LPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::LargeUtf8($INPUT)), - ColumnarValue::Scalar($LENGTH), - ColumnarValue::Scalar(ScalarValue::LargeUtf8($REPLACE)) + ColumnarValue::from(ScalarValue::LargeUtf8($INPUT)), + ColumnarValue::from($LENGTH), + ColumnarValue::from(ScalarValue::LargeUtf8($REPLACE)) ], $EXPECTED, &str, @@ -370,9 +370,9 @@ mod tests { test_function!( LPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::LargeUtf8($INPUT)), - ColumnarValue::Scalar($LENGTH), - ColumnarValue::Scalar(ScalarValue::Utf8View($REPLACE)) + ColumnarValue::from(ScalarValue::LargeUtf8($INPUT)), + ColumnarValue::from($LENGTH), + ColumnarValue::from(ScalarValue::Utf8View($REPLACE)) ], $EXPECTED, &str, @@ -384,9 +384,9 @@ mod tests { test_function!( LPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View($INPUT)), - ColumnarValue::Scalar($LENGTH), - ColumnarValue::Scalar(ScalarValue::Utf8($REPLACE)) + ColumnarValue::from(ScalarValue::Utf8View($INPUT)), + ColumnarValue::from($LENGTH), + ColumnarValue::from(ScalarValue::Utf8($REPLACE)) ], $EXPECTED, &str, @@ -397,9 +397,9 @@ mod tests { test_function!( LPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View($INPUT)), - ColumnarValue::Scalar($LENGTH), - ColumnarValue::Scalar(ScalarValue::LargeUtf8($REPLACE)) + ColumnarValue::from(ScalarValue::Utf8View($INPUT)), + ColumnarValue::from($LENGTH), + ColumnarValue::from(ScalarValue::LargeUtf8($REPLACE)) ], $EXPECTED, &str, @@ -410,9 +410,9 @@ mod tests { test_function!( LPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View($INPUT)), - ColumnarValue::Scalar($LENGTH), - ColumnarValue::Scalar(ScalarValue::Utf8View($REPLACE)) + ColumnarValue::from(ScalarValue::Utf8View($INPUT)), + ColumnarValue::from($LENGTH), + ColumnarValue::from(ScalarValue::Utf8View($REPLACE)) ], $EXPECTED, &str, diff --git a/datafusion/functions/src/unicode/reverse.rs b/datafusion/functions/src/unicode/reverse.rs index da16d3ee3752..b7af2d31e711 100644 --- a/datafusion/functions/src/unicode/reverse.rs +++ b/datafusion/functions/src/unicode/reverse.rs @@ -117,7 +117,7 @@ mod tests { ($INPUT:expr, $EXPECTED:expr) => { test_function!( ReverseFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8($INPUT))], + &[ColumnarValue::from(ScalarValue::Utf8($INPUT))], $EXPECTED, &str, Utf8, @@ -126,7 +126,7 @@ mod tests { test_function!( ReverseFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::LargeUtf8($INPUT))], + &[ColumnarValue::from(ScalarValue::LargeUtf8($INPUT))], $EXPECTED, &str, LargeUtf8, @@ -135,7 +135,7 @@ mod tests { test_function!( ReverseFunc::new(), - &[ColumnarValue::Scalar(ScalarValue::Utf8View($INPUT))], + &[ColumnarValue::from(ScalarValue::Utf8View($INPUT))], $EXPECTED, &str, Utf8, diff --git a/datafusion/functions/src/unicode/right.rs b/datafusion/functions/src/unicode/right.rs index 9d542bb2c006..7fadb058c19b 100644 --- a/datafusion/functions/src/unicode/right.rs +++ b/datafusion/functions/src/unicode/right.rs @@ -156,8 +156,8 @@ mod tests { test_function!( RightFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("abcde")), - ColumnarValue::Scalar(ScalarValue::from(2i64)), + ColumnarValue::from(ScalarValue::from("abcde")), + ColumnarValue::from(ScalarValue::from(2i64)), ], Ok(Some("de")), &str, @@ -167,8 +167,8 @@ mod tests { test_function!( RightFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("abcde")), - ColumnarValue::Scalar(ScalarValue::from(200i64)), + ColumnarValue::from(ScalarValue::from("abcde")), + ColumnarValue::from(ScalarValue::from(200i64)), ], Ok(Some("abcde")), &str, @@ -178,8 +178,8 @@ mod tests { test_function!( RightFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("abcde")), - ColumnarValue::Scalar(ScalarValue::from(-2i64)), + ColumnarValue::from(ScalarValue::from("abcde")), + ColumnarValue::from(ScalarValue::from(-2i64)), ], Ok(Some("cde")), &str, @@ -189,8 +189,8 @@ mod tests { test_function!( RightFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("abcde")), - ColumnarValue::Scalar(ScalarValue::from(-200i64)), + ColumnarValue::from(ScalarValue::from("abcde")), + ColumnarValue::from(ScalarValue::from(-200i64)), ], Ok(Some("")), &str, @@ -200,8 +200,8 @@ mod tests { test_function!( RightFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("abcde")), - ColumnarValue::Scalar(ScalarValue::from(0i64)), + ColumnarValue::from(ScalarValue::from("abcde")), + ColumnarValue::from(ScalarValue::from(0i64)), ], Ok(Some("")), &str, @@ -211,8 +211,8 @@ mod tests { test_function!( RightFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8(None)), - ColumnarValue::Scalar(ScalarValue::from(2i64)), + ColumnarValue::from(ScalarValue::Utf8(None)), + ColumnarValue::from(ScalarValue::from(2i64)), ], Ok(None), &str, @@ -222,8 +222,8 @@ mod tests { test_function!( RightFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("abcde")), - ColumnarValue::Scalar(ScalarValue::Int64(None)), + ColumnarValue::from(ScalarValue::from("abcde")), + ColumnarValue::from(ScalarValue::Int64(None)), ], Ok(None), &str, @@ -233,8 +233,8 @@ mod tests { test_function!( RightFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("joséésoj")), - ColumnarValue::Scalar(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::from("joséésoj")), + ColumnarValue::from(ScalarValue::from(5i64)), ], Ok(Some("éésoj")), &str, @@ -244,8 +244,8 @@ mod tests { test_function!( RightFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("joséésoj")), - ColumnarValue::Scalar(ScalarValue::from(-3i64)), + ColumnarValue::from(ScalarValue::from("joséésoj")), + ColumnarValue::from(ScalarValue::from(-3i64)), ], Ok(Some("éésoj")), &str, @@ -256,8 +256,8 @@ mod tests { test_function!( RightFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("abcde")), - ColumnarValue::Scalar(ScalarValue::from(2i64)), + ColumnarValue::from(ScalarValue::from("abcde")), + ColumnarValue::from(ScalarValue::from(2i64)), ], internal_err!( "function right requires compilation with feature flag: unicode_expressions." diff --git a/datafusion/functions/src/unicode/rpad.rs b/datafusion/functions/src/unicode/rpad.rs index c1d6f327928f..4c8c2c2ca5f5 100644 --- a/datafusion/functions/src/unicode/rpad.rs +++ b/datafusion/functions/src/unicode/rpad.rs @@ -281,8 +281,8 @@ mod tests { test_function!( RPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("josé")), - ColumnarValue::Scalar(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::from("josé")), + ColumnarValue::from(ScalarValue::from(5i64)), ], Ok(Some("josé ")), &str, @@ -292,8 +292,8 @@ mod tests { test_function!( RPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("hi")), - ColumnarValue::Scalar(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::from("hi")), + ColumnarValue::from(ScalarValue::from(5i64)), ], Ok(Some("hi ")), &str, @@ -303,8 +303,8 @@ mod tests { test_function!( RPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("hi")), - ColumnarValue::Scalar(ScalarValue::from(0i64)), + ColumnarValue::from(ScalarValue::from("hi")), + ColumnarValue::from(ScalarValue::from(0i64)), ], Ok(Some("")), &str, @@ -314,8 +314,8 @@ mod tests { test_function!( RPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("hi")), - ColumnarValue::Scalar(ScalarValue::Int64(None)), + ColumnarValue::from(ScalarValue::from("hi")), + ColumnarValue::from(ScalarValue::Int64(None)), ], Ok(None), &str, @@ -325,8 +325,8 @@ mod tests { test_function!( RPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8(None)), - ColumnarValue::Scalar(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::Utf8(None)), + ColumnarValue::from(ScalarValue::from(5i64)), ], Ok(None), &str, @@ -336,9 +336,9 @@ mod tests { test_function!( RPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("hi")), - ColumnarValue::Scalar(ScalarValue::from(5i64)), - ColumnarValue::Scalar(ScalarValue::from("xy")), + ColumnarValue::from(ScalarValue::from("hi")), + ColumnarValue::from(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::from("xy")), ], Ok(Some("hixyx")), &str, @@ -348,9 +348,9 @@ mod tests { test_function!( RPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("hi")), - ColumnarValue::Scalar(ScalarValue::from(21i64)), - ColumnarValue::Scalar(ScalarValue::from("abcdef")), + ColumnarValue::from(ScalarValue::from("hi")), + ColumnarValue::from(ScalarValue::from(21i64)), + ColumnarValue::from(ScalarValue::from("abcdef")), ], Ok(Some("hiabcdefabcdefabcdefa")), &str, @@ -360,9 +360,9 @@ mod tests { test_function!( RPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("hi")), - ColumnarValue::Scalar(ScalarValue::from(5i64)), - ColumnarValue::Scalar(ScalarValue::from(" ")), + ColumnarValue::from(ScalarValue::from("hi")), + ColumnarValue::from(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::from(" ")), ], Ok(Some("hi ")), &str, @@ -372,9 +372,9 @@ mod tests { test_function!( RPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("hi")), - ColumnarValue::Scalar(ScalarValue::from(5i64)), - ColumnarValue::Scalar(ScalarValue::from("")), + ColumnarValue::from(ScalarValue::from("hi")), + ColumnarValue::from(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::from("")), ], Ok(Some("hi")), &str, @@ -384,9 +384,9 @@ mod tests { test_function!( RPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8(None)), - ColumnarValue::Scalar(ScalarValue::from(5i64)), - ColumnarValue::Scalar(ScalarValue::from("xy")), + ColumnarValue::from(ScalarValue::Utf8(None)), + ColumnarValue::from(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::from("xy")), ], Ok(None), &str, @@ -396,9 +396,9 @@ mod tests { test_function!( RPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("hi")), - ColumnarValue::Scalar(ScalarValue::Int64(None)), - ColumnarValue::Scalar(ScalarValue::from("xy")), + ColumnarValue::from(ScalarValue::from("hi")), + ColumnarValue::from(ScalarValue::Int64(None)), + ColumnarValue::from(ScalarValue::from("xy")), ], Ok(None), &str, @@ -408,9 +408,9 @@ mod tests { test_function!( RPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("hi")), - ColumnarValue::Scalar(ScalarValue::from(5i64)), - ColumnarValue::Scalar(ScalarValue::Utf8(None)), + ColumnarValue::from(ScalarValue::from("hi")), + ColumnarValue::from(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::Utf8(None)), ], Ok(None), &str, @@ -420,9 +420,9 @@ mod tests { test_function!( RPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("josé")), - ColumnarValue::Scalar(ScalarValue::from(10i64)), - ColumnarValue::Scalar(ScalarValue::from("xy")), + ColumnarValue::from(ScalarValue::from("josé")), + ColumnarValue::from(ScalarValue::from(10i64)), + ColumnarValue::from(ScalarValue::from("xy")), ], Ok(Some("joséxyxyxy")), &str, @@ -432,9 +432,9 @@ mod tests { test_function!( RPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("josé")), - ColumnarValue::Scalar(ScalarValue::from(10i64)), - ColumnarValue::Scalar(ScalarValue::from("éñ")), + ColumnarValue::from(ScalarValue::from("josé")), + ColumnarValue::from(ScalarValue::from(10i64)), + ColumnarValue::from(ScalarValue::from("éñ")), ], Ok(Some("josééñéñéñ")), &str, @@ -445,8 +445,8 @@ mod tests { test_function!( RPadFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("josé")), - ColumnarValue::Scalar(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::from("josé")), + ColumnarValue::from(ScalarValue::from(5i64)), ], internal_err!( "function rpad requires compilation with feature flag: unicode_expressions." diff --git a/datafusion/functions/src/unicode/strpos.rs b/datafusion/functions/src/unicode/strpos.rs index 6da67c8a2798..e2f3fee14a36 100644 --- a/datafusion/functions/src/unicode/strpos.rs +++ b/datafusion/functions/src/unicode/strpos.rs @@ -199,8 +199,8 @@ mod tests { test_function!( StrposFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::$t1(Some($lhs.to_owned()))), - ColumnarValue::Scalar(ScalarValue::$t2(Some($rhs.to_owned()))), + ColumnarValue::from(ScalarValue::$t1(Some($lhs.to_owned()))), + ColumnarValue::from(ScalarValue::$t2(Some($rhs.to_owned()))), ], Ok(Some($result)), $t3, diff --git a/datafusion/functions/src/unicode/substr.rs b/datafusion/functions/src/unicode/substr.rs index 5e311f1e1891..39a962e8e8a1 100644 --- a/datafusion/functions/src/unicode/substr.rs +++ b/datafusion/functions/src/unicode/substr.rs @@ -456,8 +456,8 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View(None)), - ColumnarValue::Scalar(ScalarValue::from(1i64)), + ColumnarValue::from(ScalarValue::Utf8View(None)), + ColumnarValue::from(ScalarValue::from(1i64)), ], Ok(None), &str, @@ -467,10 +467,10 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View(Some(String::from( + ColumnarValue::from(ScalarValue::Utf8View(Some(String::from( "alphabet" )))), - ColumnarValue::Scalar(ScalarValue::from(0i64)), + ColumnarValue::from(ScalarValue::from(0i64)), ], Ok(Some("alphabet")), &str, @@ -480,11 +480,11 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View(Some(String::from( + ColumnarValue::from(ScalarValue::Utf8View(Some(String::from( "this és longer than 12B" )))), - ColumnarValue::Scalar(ScalarValue::from(5i64)), - ColumnarValue::Scalar(ScalarValue::from(2i64)), + ColumnarValue::from(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::from(2i64)), ], Ok(Some(" é")), &str, @@ -494,10 +494,10 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View(Some(String::from( + ColumnarValue::from(ScalarValue::Utf8View(Some(String::from( "this is longer than 12B" )))), - ColumnarValue::Scalar(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::from(5i64)), ], Ok(Some(" is longer than 12B")), &str, @@ -507,10 +507,10 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View(Some(String::from( + ColumnarValue::from(ScalarValue::Utf8View(Some(String::from( "joséésoj" )))), - ColumnarValue::Scalar(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::from(5i64)), ], Ok(Some("ésoj")), &str, @@ -520,11 +520,11 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View(Some(String::from( + ColumnarValue::from(ScalarValue::Utf8View(Some(String::from( "alphabet" )))), - ColumnarValue::Scalar(ScalarValue::from(3i64)), - ColumnarValue::Scalar(ScalarValue::from(2i64)), + ColumnarValue::from(ScalarValue::from(3i64)), + ColumnarValue::from(ScalarValue::from(2i64)), ], Ok(Some("ph")), &str, @@ -534,11 +534,11 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8View(Some(String::from( + ColumnarValue::from(ScalarValue::Utf8View(Some(String::from( "alphabet" )))), - ColumnarValue::Scalar(ScalarValue::from(3i64)), - ColumnarValue::Scalar(ScalarValue::from(20i64)), + ColumnarValue::from(ScalarValue::from(3i64)), + ColumnarValue::from(ScalarValue::from(20i64)), ], Ok(Some("phabet")), &str, @@ -548,8 +548,8 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from(0i64)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from(0i64)), ], Ok(Some("alphabet")), &str, @@ -559,8 +559,8 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("joséésoj")), - ColumnarValue::Scalar(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::from("joséésoj")), + ColumnarValue::from(ScalarValue::from(5i64)), ], Ok(Some("ésoj")), &str, @@ -570,8 +570,8 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("joséésoj")), - ColumnarValue::Scalar(ScalarValue::from(-5i64)), + ColumnarValue::from(ScalarValue::from("joséésoj")), + ColumnarValue::from(ScalarValue::from(-5i64)), ], Ok(Some("joséésoj")), &str, @@ -581,8 +581,8 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from(1i64)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from(1i64)), ], Ok(Some("alphabet")), &str, @@ -592,8 +592,8 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from(2i64)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from(2i64)), ], Ok(Some("lphabet")), &str, @@ -603,8 +603,8 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from(3i64)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from(3i64)), ], Ok(Some("phabet")), &str, @@ -614,8 +614,8 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from(-3i64)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from(-3i64)), ], Ok(Some("alphabet")), &str, @@ -625,8 +625,8 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from(30i64)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from(30i64)), ], Ok(Some("")), &str, @@ -636,8 +636,8 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::Int64(None)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::Int64(None)), ], Ok(None), &str, @@ -647,9 +647,9 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from(3i64)), - ColumnarValue::Scalar(ScalarValue::from(2i64)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from(3i64)), + ColumnarValue::from(ScalarValue::from(2i64)), ], Ok(Some("ph")), &str, @@ -659,9 +659,9 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from(3i64)), - ColumnarValue::Scalar(ScalarValue::from(20i64)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from(3i64)), + ColumnarValue::from(ScalarValue::from(20i64)), ], Ok(Some("phabet")), &str, @@ -671,9 +671,9 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from(0i64)), - ColumnarValue::Scalar(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from(0i64)), + ColumnarValue::from(ScalarValue::from(5i64)), ], Ok(Some("alph")), &str, @@ -684,9 +684,9 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from(-5i64)), - ColumnarValue::Scalar(ScalarValue::from(10i64)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from(-5i64)), + ColumnarValue::from(ScalarValue::from(10i64)), ], Ok(Some("alph")), &str, @@ -697,9 +697,9 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from(-5i64)), - ColumnarValue::Scalar(ScalarValue::from(4i64)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from(-5i64)), + ColumnarValue::from(ScalarValue::from(4i64)), ], Ok(Some("")), &str, @@ -710,9 +710,9 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from(-5i64)), - ColumnarValue::Scalar(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from(-5i64)), + ColumnarValue::from(ScalarValue::from(5i64)), ], Ok(Some("")), &str, @@ -722,9 +722,9 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::Int64(None)), - ColumnarValue::Scalar(ScalarValue::from(20i64)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::Int64(None)), + ColumnarValue::from(ScalarValue::from(20i64)), ], Ok(None), &str, @@ -734,9 +734,9 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from(3i64)), - ColumnarValue::Scalar(ScalarValue::Int64(None)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from(3i64)), + ColumnarValue::from(ScalarValue::Int64(None)), ], Ok(None), &str, @@ -746,9 +746,9 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from(1i64)), - ColumnarValue::Scalar(ScalarValue::from(-1i64)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from(1i64)), + ColumnarValue::from(ScalarValue::from(-1i64)), ], exec_err!("negative substring length not allowed: substr(, 1, -1)"), &str, @@ -758,9 +758,9 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("joséésoj")), - ColumnarValue::Scalar(ScalarValue::from(5i64)), - ColumnarValue::Scalar(ScalarValue::from(2i64)), + ColumnarValue::from(ScalarValue::from("joséésoj")), + ColumnarValue::from(ScalarValue::from(5i64)), + ColumnarValue::from(ScalarValue::from(2i64)), ], Ok(Some("és")), &str, @@ -771,8 +771,8 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("alphabet")), - ColumnarValue::Scalar(ScalarValue::from(0i64)), + ColumnarValue::from(ScalarValue::from("alphabet")), + ColumnarValue::from(ScalarValue::from(0i64)), ], internal_err!( "function substr requires compilation with feature flag: unicode_expressions." @@ -784,8 +784,8 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("abc")), - ColumnarValue::Scalar(ScalarValue::from(-9223372036854775808i64)), + ColumnarValue::from(ScalarValue::from("abc")), + ColumnarValue::from(ScalarValue::from(-9223372036854775808i64)), ], Ok(Some("abc")), &str, @@ -795,9 +795,9 @@ mod tests { test_function!( SubstrFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("overflow")), - ColumnarValue::Scalar(ScalarValue::from(-9223372036854775808i64)), - ColumnarValue::Scalar(ScalarValue::from(1i64)), + ColumnarValue::from(ScalarValue::from("overflow")), + ColumnarValue::from(ScalarValue::from(-9223372036854775808i64)), + ColumnarValue::from(ScalarValue::from(1i64)), ], exec_err!("negative overflow when calculating skip value"), &str, diff --git a/datafusion/functions/src/unicode/substrindex.rs b/datafusion/functions/src/unicode/substrindex.rs index 6591ee26403a..9ca3d018d884 100644 --- a/datafusion/functions/src/unicode/substrindex.rs +++ b/datafusion/functions/src/unicode/substrindex.rs @@ -213,9 +213,9 @@ mod tests { test_function!( SubstrIndexFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("www.apache.org")), - ColumnarValue::Scalar(ScalarValue::from(".")), - ColumnarValue::Scalar(ScalarValue::from(1i64)), + ColumnarValue::from(ScalarValue::from("www.apache.org")), + ColumnarValue::from(ScalarValue::from(".")), + ColumnarValue::from(ScalarValue::from(1i64)), ], Ok(Some("www")), &str, @@ -225,9 +225,9 @@ mod tests { test_function!( SubstrIndexFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("www.apache.org")), - ColumnarValue::Scalar(ScalarValue::from(".")), - ColumnarValue::Scalar(ScalarValue::from(2i64)), + ColumnarValue::from(ScalarValue::from("www.apache.org")), + ColumnarValue::from(ScalarValue::from(".")), + ColumnarValue::from(ScalarValue::from(2i64)), ], Ok(Some("www.apache")), &str, @@ -237,9 +237,9 @@ mod tests { test_function!( SubstrIndexFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("www.apache.org")), - ColumnarValue::Scalar(ScalarValue::from(".")), - ColumnarValue::Scalar(ScalarValue::from(-2i64)), + ColumnarValue::from(ScalarValue::from("www.apache.org")), + ColumnarValue::from(ScalarValue::from(".")), + ColumnarValue::from(ScalarValue::from(-2i64)), ], Ok(Some("apache.org")), &str, @@ -249,9 +249,9 @@ mod tests { test_function!( SubstrIndexFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("www.apache.org")), - ColumnarValue::Scalar(ScalarValue::from(".")), - ColumnarValue::Scalar(ScalarValue::from(-1i64)), + ColumnarValue::from(ScalarValue::from("www.apache.org")), + ColumnarValue::from(ScalarValue::from(".")), + ColumnarValue::from(ScalarValue::from(-1i64)), ], Ok(Some("org")), &str, @@ -261,9 +261,9 @@ mod tests { test_function!( SubstrIndexFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("www.apache.org")), - ColumnarValue::Scalar(ScalarValue::from(".")), - ColumnarValue::Scalar(ScalarValue::from(0i64)), + ColumnarValue::from(ScalarValue::from("www.apache.org")), + ColumnarValue::from(ScalarValue::from(".")), + ColumnarValue::from(ScalarValue::from(0i64)), ], Ok(Some("")), &str, @@ -273,9 +273,9 @@ mod tests { test_function!( SubstrIndexFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("")), - ColumnarValue::Scalar(ScalarValue::from(".")), - ColumnarValue::Scalar(ScalarValue::from(1i64)), + ColumnarValue::from(ScalarValue::from("")), + ColumnarValue::from(ScalarValue::from(".")), + ColumnarValue::from(ScalarValue::from(1i64)), ], Ok(Some("")), &str, @@ -285,9 +285,9 @@ mod tests { test_function!( SubstrIndexFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("www.apache.org")), - ColumnarValue::Scalar(ScalarValue::from("")), - ColumnarValue::Scalar(ScalarValue::from(1i64)), + ColumnarValue::from(ScalarValue::from("www.apache.org")), + ColumnarValue::from(ScalarValue::from("")), + ColumnarValue::from(ScalarValue::from(1i64)), ], Ok(Some("")), &str, diff --git a/datafusion/functions/src/unicode/translate.rs b/datafusion/functions/src/unicode/translate.rs index a42b9c6cb857..d49559d452c8 100644 --- a/datafusion/functions/src/unicode/translate.rs +++ b/datafusion/functions/src/unicode/translate.rs @@ -171,9 +171,9 @@ mod tests { test_function!( TranslateFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("12345")), - ColumnarValue::Scalar(ScalarValue::from("143")), - ColumnarValue::Scalar(ScalarValue::from("ax")) + ColumnarValue::from(ScalarValue::from("12345")), + ColumnarValue::from(ScalarValue::from("143")), + ColumnarValue::from(ScalarValue::from("ax")) ], Ok(Some("a2x5")), &str, @@ -183,9 +183,9 @@ mod tests { test_function!( TranslateFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::Utf8(None)), - ColumnarValue::Scalar(ScalarValue::from("143")), - ColumnarValue::Scalar(ScalarValue::from("ax")) + ColumnarValue::from(ScalarValue::Utf8(None)), + ColumnarValue::from(ScalarValue::from("143")), + ColumnarValue::from(ScalarValue::from("ax")) ], Ok(None), &str, @@ -195,9 +195,9 @@ mod tests { test_function!( TranslateFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("12345")), - ColumnarValue::Scalar(ScalarValue::Utf8(None)), - ColumnarValue::Scalar(ScalarValue::from("ax")) + ColumnarValue::from(ScalarValue::from("12345")), + ColumnarValue::from(ScalarValue::Utf8(None)), + ColumnarValue::from(ScalarValue::from("ax")) ], Ok(None), &str, @@ -207,9 +207,9 @@ mod tests { test_function!( TranslateFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("12345")), - ColumnarValue::Scalar(ScalarValue::from("143")), - ColumnarValue::Scalar(ScalarValue::Utf8(None)) + ColumnarValue::from(ScalarValue::from("12345")), + ColumnarValue::from(ScalarValue::from("143")), + ColumnarValue::from(ScalarValue::Utf8(None)) ], Ok(None), &str, @@ -219,9 +219,9 @@ mod tests { test_function!( TranslateFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("é2íñ5")), - ColumnarValue::Scalar(ScalarValue::from("éñí")), - ColumnarValue::Scalar(ScalarValue::from("óü")), + ColumnarValue::from(ScalarValue::from("é2íñ5")), + ColumnarValue::from(ScalarValue::from("éñí")), + ColumnarValue::from(ScalarValue::from("óü")), ], Ok(Some("ó2ü5")), &str, @@ -232,9 +232,9 @@ mod tests { test_function!( TranslateFunc::new(), &[ - ColumnarValue::Scalar(ScalarValue::from("12345")), - ColumnarValue::Scalar(ScalarValue::from("143")), - ColumnarValue::Scalar(ScalarValue::from("ax")), + ColumnarValue::from(ScalarValue::from("12345")), + ColumnarValue::from(ScalarValue::from("143")), + ColumnarValue::from(ScalarValue::from("ax")), ], internal_err!( "function translate requires compilation with feature flag: unicode_expressions." diff --git a/datafusion/functions/src/utils.rs b/datafusion/functions/src/utils.rs index 818b4c64bd20..b1f5580ebe56 100644 --- a/datafusion/functions/src/utils.rs +++ b/datafusion/functions/src/utils.rs @@ -113,7 +113,7 @@ where if is_scalar { // If all inputs are scalar, keeps output as scalar let result = result.and_then(|arr| ScalarValue::try_from_array(&arr, 0)); - result.map(ColumnarValue::Scalar) + result.map(ColumnarValue::from) } else { result.map(ColumnarValue::Array) } @@ -133,7 +133,7 @@ pub mod test { let expected: Result> = $EXPECTED; let func = $FUNC; - let type_array = $ARGS.iter().map(|arg| arg.data_type()).collect::>(); + let type_array = $ARGS.iter().map(|arg| arg.data_type().clone()).collect::>(); let return_type = func.return_type(&type_array); match expected { diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs index 7a8746572cfd..b20a303eb5c0 100644 --- a/datafusion/optimizer/src/analyzer/type_coercion.rs +++ b/datafusion/optimizer/src/analyzer/type_coercion.rs @@ -1238,7 +1238,7 @@ mod test { } fn invoke(&self, _args: &[ColumnarValue]) -> Result { - Ok(ColumnarValue::Scalar(ScalarValue::from("a"))) + Ok(ColumnarValue::from(ScalarValue::from("a"))) } } diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index 6f0a64b85cb6..b1a19fbf1086 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -3047,7 +3047,7 @@ Projection: a, b } fn invoke(&self, _args: &[ColumnarValue]) -> Result { - Ok(ColumnarValue::Scalar(ScalarValue::from(1))) + Ok(ColumnarValue::from(ScalarValue::from(1))) } } diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index fc3921d29615..fea53554fced 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -670,7 +670,7 @@ impl<'a> ConstEvaluator<'a> { } ColumnarValue::Scalar(s) => { // TODO: support the optimization for `Map` type after support impl hash for it - if matches!(&s, ScalarValue::Map(_)) { + if matches!(&s.value(), ScalarValue::Map(_)) { ConstSimplifyResult::SimplifyRuntimeError( DataFusionError::NotImplemented( "Const evaluate for Map type is still not supported" @@ -679,7 +679,7 @@ impl<'a> ConstEvaluator<'a> { expr, ) } else { - ConstSimplifyResult::Simplified(s) + ConstSimplifyResult::Simplified(s.into_value()) } } } diff --git a/datafusion/physical-expr-common/src/datum.rs b/datafusion/physical-expr-common/src/datum.rs index c47ec9d75d50..6ecd47839546 100644 --- a/datafusion/physical-expr-common/src/datum.rs +++ b/datafusion/physical-expr-common/src/datum.rs @@ -21,10 +21,11 @@ use arrow::buffer::NullBuffer; use arrow::compute::SortOptions; use arrow::error::ArrowError; use datafusion_common::DataFusionError; +use datafusion_common::Result; use datafusion_common::{arrow_datafusion_err, internal_err}; -use datafusion_common::{Result, ScalarValue}; use datafusion_expr_common::columnar_value::ColumnarValue; use datafusion_expr_common::operator::Operator; +use datafusion_expr_common::scalar::Scalar; use std::sync::Arc; /// Applies a binary [`Datum`] kernel `f` to `lhs` and `rhs` @@ -47,7 +48,7 @@ pub fn apply( ), (ColumnarValue::Scalar(left), ColumnarValue::Scalar(right)) => { let array = f(&left.to_scalar()?, &right.to_scalar()?)?; - let scalar = ScalarValue::try_from_array(array.as_ref(), 0)?; + let scalar = Scalar::try_from_array(array.as_ref(), 0)?; Ok(ColumnarValue::Scalar(scalar)) } } diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs index 236b24dd4094..8c4a0fd3c989 100644 --- a/datafusion/physical-expr/src/expressions/binary.rs +++ b/datafusion/physical-expr/src/expressions/binary.rs @@ -253,8 +253,8 @@ impl PhysicalExpr for BinaryExpr { let lhs = self.left.evaluate(batch)?; let rhs = self.right.evaluate(batch)?; - let left_data_type = lhs.data_type(); - let right_data_type = rhs.data_type(); + let left_data_type = lhs.data_type().clone(); + let right_data_type = rhs.data_type().clone(); let schema = batch.schema(); let input_schema = schema.as_ref(); @@ -296,12 +296,15 @@ impl PhysicalExpr for BinaryExpr { let scalar_result = match (&lhs, &rhs) { (ColumnarValue::Array(array), ColumnarValue::Scalar(scalar)) => { // if left is array and right is literal(not NULL) - use scalar operations - if scalar.is_null() { + if scalar.value().is_null() { None } else { - self.evaluate_array_scalar(array, scalar.clone())?.map(|r| { - r.and_then(|a| to_result_type_array(&self.op, a, &result_type)) - }) + self.evaluate_array_scalar(array, scalar.value().clone())? + .map(|r| { + r.and_then(|a| { + to_result_type_array(&self.op, a, &result_type) + }) + }) } } (_, _) => None, // default to array implementation diff --git a/datafusion/physical-expr/src/expressions/case.rs b/datafusion/physical-expr/src/expressions/case.rs index 712175c9afbe..618b61856e68 100644 --- a/datafusion/physical-expr/src/expressions/case.rs +++ b/datafusion/physical-expr/src/expressions/case.rs @@ -223,12 +223,10 @@ impl CaseExpr { .evaluate_selection(batch, &when_match)?; current_value = match then_value { - ColumnarValue::Scalar(ScalarValue::Null) => { - nullif(current_value.as_ref(), &when_match)? - } - ColumnarValue::Scalar(then_value) => { - zip(&when_match, &then_value.to_scalar()?, ¤t_value)? - } + ColumnarValue::Scalar(scalar) => match scalar.value() { + ScalarValue::Null => nullif(current_value.as_ref(), &when_match)?, + _ => zip(&when_match, &scalar.to_scalar()?, ¤t_value)?, + }, ColumnarValue::Array(then_value) => { zip(&when_match, &then_value, ¤t_value)? } @@ -294,12 +292,10 @@ impl CaseExpr { .evaluate_selection(batch, &when_value)?; current_value = match then_value { - ColumnarValue::Scalar(ScalarValue::Null) => { - nullif(current_value.as_ref(), &when_value)? - } - ColumnarValue::Scalar(then_value) => { - zip(&when_value, &then_value.to_scalar()?, ¤t_value)? - } + ColumnarValue::Scalar(scalar) => match scalar.value() { + ScalarValue::Null => nullif(current_value.as_ref(), &when_value)?, + _ => zip(&when_value, &scalar.to_scalar()?, ¤t_value)?, + }, ColumnarValue::Array(then_value) => { zip(&when_value, &then_value, ¤t_value)? } diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rs index 0a3e5fcefcf6..b44afe40b3c5 100644 --- a/datafusion/physical-expr/src/expressions/in_list.rs +++ b/datafusion/physical-expr/src/expressions/in_list.rs @@ -40,7 +40,7 @@ use datafusion_common::hash_utils::HashValue; use datafusion_common::{ exec_err, internal_err, not_impl_err, DFSchema, Result, ScalarValue, }; -use datafusion_expr::ColumnarValue; +use datafusion_expr::{ColumnarValue, Scalar}; use datafusion_physical_expr_common::datum::compare_with_eq; use ahash::RandomState; @@ -222,13 +222,15 @@ fn evaluate_list( exec_err!("InList expression must evaluate to a scalar") } // Flatten dictionary values - ColumnarValue::Scalar(ScalarValue::Dictionary(_, v)) => Ok(*v), - ColumnarValue::Scalar(s) => Ok(s), + ColumnarValue::Scalar(s) => match s.value() { + ScalarValue::Dictionary(_, v) => Ok(Scalar::from(v.as_ref().clone())), + _ => Ok(s), + }, }) }) .collect::>>()?; - ScalarValue::iter_to_array(scalars) + Scalar::iter_to_array(scalars) } fn try_cast_static_filter_to_set( diff --git a/datafusion/physical-expr/src/expressions/is_not_null.rs b/datafusion/physical-expr/src/expressions/is_not_null.rs index 58559352d44c..50c3cbab9baf 100644 --- a/datafusion/physical-expr/src/expressions/is_not_null.rs +++ b/datafusion/physical-expr/src/expressions/is_not_null.rs @@ -76,8 +76,8 @@ impl PhysicalExpr for IsNotNullExpr { let is_not_null = super::is_null::compute_is_not_null(array)?; Ok(ColumnarValue::Array(Arc::new(is_not_null))) } - ColumnarValue::Scalar(scalar) => Ok(ColumnarValue::Scalar( - ScalarValue::Boolean(Some(!scalar.is_null())), + ColumnarValue::Scalar(scalar) => Ok(ColumnarValue::from( + ScalarValue::Boolean(Some(!scalar.value().is_null())), )), } } diff --git a/datafusion/physical-expr/src/expressions/is_null.rs b/datafusion/physical-expr/src/expressions/is_null.rs index 3cdb49bcab42..cdc5f101002e 100644 --- a/datafusion/physical-expr/src/expressions/is_null.rs +++ b/datafusion/physical-expr/src/expressions/is_null.rs @@ -80,8 +80,8 @@ impl PhysicalExpr for IsNullExpr { ColumnarValue::Array(array) => { Ok(ColumnarValue::Array(Arc::new(compute_is_null(array)?))) } - ColumnarValue::Scalar(scalar) => Ok(ColumnarValue::Scalar( - ScalarValue::Boolean(Some(scalar.is_null())), + ColumnarValue::Scalar(scalar) => Ok(ColumnarValue::from( + ScalarValue::Boolean(Some(scalar.value().is_null())), )), } } diff --git a/datafusion/physical-expr/src/expressions/literal.rs b/datafusion/physical-expr/src/expressions/literal.rs index ed24e9028153..e064abbca35c 100644 --- a/datafusion/physical-expr/src/expressions/literal.rs +++ b/datafusion/physical-expr/src/expressions/literal.rs @@ -72,7 +72,7 @@ impl PhysicalExpr for Literal { } fn evaluate(&self, _batch: &RecordBatch) -> Result { - Ok(ColumnarValue::Scalar(self.value.clone())) + Ok(ColumnarValue::from(self.value.clone())) } fn children(&self) -> Vec<&Arc> { diff --git a/datafusion/physical-expr/src/expressions/negative.rs b/datafusion/physical-expr/src/expressions/negative.rs index b5ebc250cb89..fd6584cccbe0 100644 --- a/datafusion/physical-expr/src/expressions/negative.rs +++ b/datafusion/physical-expr/src/expressions/negative.rs @@ -84,7 +84,7 @@ impl PhysicalExpr for NegativeExpr { Ok(ColumnarValue::Array(result)) } ColumnarValue::Scalar(scalar) => { - Ok(ColumnarValue::Scalar((scalar.arithmetic_negate())?)) + Ok(ColumnarValue::from((scalar.value().arithmetic_negate())?)) } } } diff --git a/datafusion/physical-expr/src/expressions/not.rs b/datafusion/physical-expr/src/expressions/not.rs index b69954e00bba..7a0afaa1a637 100644 --- a/datafusion/physical-expr/src/expressions/not.rs +++ b/datafusion/physical-expr/src/expressions/not.rs @@ -78,13 +78,11 @@ impl PhysicalExpr for NotExpr { ))) } ColumnarValue::Scalar(scalar) => { - if scalar.is_null() { - return Ok(ColumnarValue::Scalar(ScalarValue::Boolean(None))); + if scalar.value().is_null() { + return Ok(ColumnarValue::from(ScalarValue::Boolean(None))); } - let bool_value: bool = scalar.try_into()?; - Ok(ColumnarValue::Scalar(ScalarValue::Boolean(Some( - !bool_value, - )))) + let bool_value: bool = scalar.into_value().try_into()?; + Ok(ColumnarValue::from(ScalarValue::Boolean(Some(!bool_value)))) } } } diff --git a/datafusion/physical-expr/src/expressions/try_cast.rs b/datafusion/physical-expr/src/expressions/try_cast.rs index 43b6c993d2b2..8490f78b1fe5 100644 --- a/datafusion/physical-expr/src/expressions/try_cast.rs +++ b/datafusion/physical-expr/src/expressions/try_cast.rs @@ -92,7 +92,7 @@ impl PhysicalExpr for TryCastExpr { let array = scalar.to_array()?; let cast_array = cast_with_options(&array, &self.cast_type, &options)?; let cast_scalar = ScalarValue::try_from_array(&cast_array, 0)?; - Ok(ColumnarValue::Scalar(cast_scalar)) + Ok(ColumnarValue::from(cast_scalar)) } } } diff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rs index 7cbfd49afb86..6f505b8d4349 100644 --- a/datafusion/physical-plan/src/lib.rs +++ b/datafusion/physical-plan/src/lib.rs @@ -27,7 +27,7 @@ pub use datafusion_common::hash_utils; pub use datafusion_common::utils::project_schema; pub use datafusion_common::{internal_err, ColumnStatistics, Statistics}; pub use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream}; -pub use datafusion_expr::{Accumulator, ColumnarValue}; +pub use datafusion_expr::{Accumulator, ColumnarValue, Scalar}; pub use datafusion_physical_expr::window::WindowExpr; use datafusion_physical_expr::PhysicalSortExpr; pub use datafusion_physical_expr::{ diff --git a/datafusion/physical-plan/src/values.rs b/datafusion/physical-plan/src/values.rs index e01aea1fdd6b..bae67f70f057 100644 --- a/datafusion/physical-plan/src/values.rs +++ b/datafusion/physical-plan/src/values.rs @@ -26,12 +26,12 @@ use super::{ }; use crate::{ memory::MemoryStream, ColumnarValue, DisplayFormatType, ExecutionPlan, Partitioning, - PhysicalExpr, + PhysicalExpr, Scalar, }; use arrow::datatypes::{Schema, SchemaRef}; use arrow::record_batch::{RecordBatch, RecordBatchOptions}; -use datafusion_common::{internal_err, plan_err, Result, ScalarValue}; +use datafusion_common::{internal_err, plan_err, Result}; use datafusion_execution::TaskContext; use datafusion_physical_expr::EquivalenceProperties; @@ -74,7 +74,7 @@ impl ValuesExec { match r { Ok(ColumnarValue::Scalar(scalar)) => Ok(scalar), Ok(ColumnarValue::Array(a)) if a.len() == 1 => { - ScalarValue::try_from_array(&a, 0) + Scalar::try_from_array(&a, 0) } Ok(ColumnarValue::Array(a)) => { plan_err!( @@ -85,7 +85,7 @@ impl ValuesExec { } }) .collect::>>() - .and_then(ScalarValue::iter_to_array) + .and_then(Scalar::iter_to_array) }) .collect::>>()?; let batch = RecordBatch::try_new_with_options( @@ -219,6 +219,7 @@ mod tests { use crate::test::{self, make_partition}; use arrow_schema::{DataType, Field}; + use datafusion_common::ScalarValue; #[tokio::test] async fn values_empty_case() -> Result<()> {