From 104a7d10a8c4f8a3a16d7899b2c411a7bfff259f Mon Sep 17 00:00:00 2001 From: Josh Pschorr Date: Wed, 2 Oct 2024 14:44:08 -0700 Subject: [PATCH] Add UDFs to catalog; Add extension UDFs for TUPLEUNION/TUPLECONCAT (#496) Extensions and scalar UDFs --- Adds ability to create and register scalar User Defined Functions (UDFs) TUPLEUNION & TUPLECONCAT --- Adds `TUPLEUNION` and `TUPLECONCAT` functions via an extension crate under the `extension` crate namespace as scalar UDFs. `TUPLEUNION` is as per 6.3.2 of the spec (https://partiql.org/assets/PartiQL-Specification.pdf#subsubsection.6.3.2) `TUPLECONCAT` is inspired by description of concatenating binding environments as per section 3.3 of the spec (https://partiql.org/assets/PartiQL-Specification.pdf#subsection.3.3) and as requested in #495. --- `tupleunion({ 'bob': 1, 'sally': 'error' }, { 'sally': 1 }, { 'sally': 2 }, { 'sally': 3 }, { 'sally': 4 })` -> `{ 'bob': 1, 'sally': 'error', 'sally': 1, 'sally': 2, 'sally': 3, 'sally': 4 }` --- `tupleconcat({ 'bob': 1, 'sally': 'error' }, { 'sally': 1 }, { 'sally': 2 }, { 'sally': 3 }, { 'sally': 4 })` -> `{ 'sally': 4, 'bob': 1 }` --- CHANGELOG.md | 4 + Cargo.toml | 1 + deny.toml | 5 +- .../src/lib.rs | 20 +- .../Cargo.toml | 33 ++ .../partiql-extension-value-functions/LICENSE | 175 +++++++++ .../src/lib.rs | 96 +++++ partiql-ast-passes/src/name_resolver.rs | 2 +- partiql-catalog/Cargo.toml | 5 +- partiql-catalog/src/call_defs.rs | 39 +- partiql-catalog/src/catalog.rs | 294 +++++++++++++++ partiql-catalog/src/extension.rs | 11 + partiql-catalog/src/lib.rs | 345 +----------------- partiql-catalog/src/scalar_fn.rs | 82 +++++ partiql-catalog/src/table_fn.rs | 43 +++ partiql-common/src/catalog.rs | 51 +++ partiql-common/src/lib.rs | 4 + partiql-conformance-tests/tests/mod.rs | 2 +- partiql-eval/benches/bench_eval.rs | 2 +- partiql-eval/src/error.rs | 6 +- partiql-eval/src/eval/eval_expr_wrapper.rs | 117 +++--- partiql-eval/src/eval/expr/base_table.rs | 2 +- partiql-eval/src/eval/expr/functions.rs | 55 +++ partiql-eval/src/eval/expr/mod.rs | 2 + partiql-eval/src/eval/expr/strings.rs | 27 +- partiql-eval/src/lib.rs | 2 +- partiql-eval/src/plan.rs | 87 +++-- partiql-logical-planner/src/functions.rs | 82 +++++ partiql-logical-planner/src/lib.rs | 5 +- partiql-logical-planner/src/lower.rs | 11 +- partiql-logical-planner/src/typer.rs | 4 +- partiql-logical/Cargo.toml | 1 + partiql-logical/src/lib.rs | 3 + partiql-value/src/tuple.rs | 14 +- partiql/Cargo.toml | 11 +- partiql/benches/bench_agg.rs | 3 +- partiql/benches/bench_eval_multi_like.rs | 5 +- partiql/src/lib.rs | 2 +- partiql/src/subquery_tests.rs | 2 +- partiql/tests/extension_error.rs | 15 +- .../snapshots/tuple_ops__tupleconcat.snap | 5 + .../snapshots/tuple_ops__tuplemerge.snap | 5 + .../snapshots/tuple_ops__tupleunion.snap | 5 + partiql/tests/tuple_ops.rs | 141 +++++++ partiql/tests/user_context.rs | 15 +- 45 files changed, 1339 insertions(+), 502 deletions(-) create mode 100644 extension/partiql-extension-value-functions/Cargo.toml create mode 100644 extension/partiql-extension-value-functions/LICENSE create mode 100644 extension/partiql-extension-value-functions/src/lib.rs create mode 100644 partiql-catalog/src/catalog.rs create mode 100644 partiql-catalog/src/extension.rs create mode 100644 partiql-catalog/src/scalar_fn.rs create mode 100644 partiql-catalog/src/table_fn.rs create mode 100644 partiql-common/src/catalog.rs create mode 100644 partiql-eval/src/eval/expr/functions.rs create mode 100644 partiql-logical-planner/src/functions.rs create mode 100644 partiql/tests/snapshots/tuple_ops__tupleconcat.snap create mode 100644 partiql/tests/snapshots/tuple_ops__tuplemerge.snap create mode 100644 partiql/tests/snapshots/tuple_ops__tupleunion.snap create mode 100644 partiql/tests/tuple_ops.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index f68a59b6..35ca4452 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - partiql-ast: fixed pretty-printing of `PIVOT` - partiql-ast: improved pretty-printing of `CASE` and various clauses +- *BREAKING* partiql-catalog: refactored structure of crate; module paths have changes ### Added - Added `partiql-common`. @@ -16,6 +17,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - *BREAKING* Added thread-safe `PartiqlShapeBuilder` and automatic `NodeId` generation for the `StaticType`. - Added a static thread safe `shape_builder` function that provides a convenient way for using `PartiqlShapeBuilder` for creating new shapes. - Added `partiql_common::meta::PartiqlMetadata` +- Added ability for crate importers to add scalar *U*ser *D*efined *F*unctions (UDFs) to the catalog +- Added `extension/partiql-extension-value-functions` crate demonstrating use of scalar UDFs +- Added `TUPLEUNION` and `TUPLECONCAT` functions in the `extension/partiql-extension-value-functions` crate ### Removed - *BREAKING* Removed `partiql-source-map`. diff --git a/Cargo.toml b/Cargo.toml index acd82026..4b6989ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ members = [ "extension/partiql-extension-ddl", "extension/partiql-extension-ion", "extension/partiql-extension-ion-functions", + "extension/partiql-extension-value-functions", "extension/partiql-extension-visualize", ] diff --git a/deny.toml b/deny.toml index 26081a2d..37526387 100644 --- a/deny.toml +++ b/deny.toml @@ -15,6 +15,9 @@ ignore = [ # Advisory: https://rustsec.org/advisories/RUSTSEC-2020-0071 # `chrono` uses an old version of `time`, but `chrono` >= 4.2 doesn't use the code path with the issue "RUSTSEC-2020-0071", + # Advisory: https://rustsec.org/advisories/RUSTSEC-2024-0375 + # `atty` is unmaintained, but we still use a version of criterion that brings it in + "RUSTSEC-2024-0375" ] @@ -103,7 +106,7 @@ deny = [ { name = "time", version = "<0.2.23", wrappers = ["chrono"] }, # Advisory: https://rustsec.org/advisories/RUSTSEC-2020-0071 # `chrono` uses an old version of `time`, but `chrono` >= 4.2 doesn't use the code path with the issue - { name = "chrono", version = "<0.4.20" } + { name = "chrono", version = "<0.4.20" }, ] diff --git a/extension/partiql-extension-ion-functions/src/lib.rs b/extension/partiql-extension-ion-functions/src/lib.rs index 5b1bfb85..2187e166 100644 --- a/extension/partiql-extension-ion-functions/src/lib.rs +++ b/extension/partiql-extension-ion-functions/src/lib.rs @@ -3,10 +3,10 @@ use ion_rs_old::data_source::ToIonDataSource; use partiql_catalog::call_defs::{CallDef, CallSpec, CallSpecArg}; -use partiql_catalog::TableFunction; -use partiql_catalog::{ - BaseTableExpr, BaseTableExprResult, BaseTableExprResultError, BaseTableExprResultValueIter, - BaseTableFunctionInfo, Catalog, +use partiql_catalog::catalog::Catalog; +use partiql_catalog::table_fn::{ + BaseTableExpr, BaseTableExprResult, BaseTableExprResultValueIter, BaseTableFunctionInfo, + TableFunction, }; use partiql_extension_ion::decode::{IonDecoderBuilder, IonDecoderConfig}; use partiql_extension_ion::Encoding; @@ -15,6 +15,7 @@ use partiql_value::Value; use std::borrow::Cow; use partiql_catalog::context::SessionContext; +use partiql_catalog::extension::ExtensionResultError; use std::error::Error; use std::fmt::Debug; use std::fs::File; @@ -51,7 +52,7 @@ impl From for IonExtensionError { #[derive(Debug)] pub struct IonExtension {} -impl partiql_catalog::Extension for IonExtension { +impl partiql_catalog::extension::Extension for IonExtension { fn name(&self) -> String { "ion".into() } @@ -114,12 +115,12 @@ impl BaseTableExpr for EvalFnReadIon { let error = IonExtensionError::FunctionError( "expected string path argument".to_string(), ); - Err(Box::new(error) as BaseTableExprResultError) + Err(Box::new(error) as ExtensionResultError) } } } else { let error = IonExtensionError::FunctionError("expected path argument".to_string()); - Err(Box::new(error) as BaseTableExprResultError) + Err(Box::new(error) as ExtensionResultError) } } } @@ -151,7 +152,7 @@ fn parse_ion_read<'a>(mut reader: impl 'a + Read + Seek) -> BaseTableExprResult< } fn parse_ion_buff<'a, I: 'a + ToIonDataSource>(input: I) -> BaseTableExprResult<'a> { - let err_map = |e| Box::new(e) as BaseTableExprResultError; + let err_map = |e| Box::new(e) as ExtensionResultError; let reader = ion_rs_old::ReaderBuilder::new().build(input).unwrap(); let decoder = IonDecoderBuilder::new(IonDecoderConfig::default().with_mode(Encoding::Ion)).build(reader); @@ -163,8 +164,9 @@ fn parse_ion_buff<'a, I: 'a + ToIonDataSource>(input: I) -> BaseTableExprResult< mod tests { use super::*; + use partiql_catalog::catalog::{Catalog, PartiqlCatalog}; use partiql_catalog::context::SystemContext; - use partiql_catalog::{Catalog, Extension, PartiqlCatalog}; + use partiql_catalog::extension::Extension; use partiql_eval::env::basic::MapBindings; use partiql_eval::eval::BasicContext; use partiql_eval::plan::EvaluationMode; diff --git a/extension/partiql-extension-value-functions/Cargo.toml b/extension/partiql-extension-value-functions/Cargo.toml new file mode 100644 index 00000000..2798a2fb --- /dev/null +++ b/extension/partiql-extension-value-functions/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "partiql-extension-value-functions" +description = "PartiQL Value function extensions" +authors.workspace = true +homepage.workspace = true +repository.workspace = true +license = "Apache-2.0" +readme = "../../README.md" +keywords = ["sql", "query", "compilers", "interpreters"] +categories = ["database", "compilers"] +exclude = [ + "**/.git/**", + "**/.github/**", + "**/.travis.yml", + "**/.appveyor.yml", +] +version.workspace = true +edition.workspace = true + +[lib] +bench = false + +[dependencies] +partiql-value = { path = "../../partiql-value", version = "0.10.*" } +partiql-catalog = { path = "../../partiql-catalog", version = "0.10.*" } +partiql-logical = { path = "../../partiql-logical", version = "0.10.*" } + +ordered-float = "3.*" +unicase = "2.6" +time = { version = "0.3", features = ["macros"] } + +[features] +default = [] diff --git a/extension/partiql-extension-value-functions/LICENSE b/extension/partiql-extension-value-functions/LICENSE new file mode 100644 index 00000000..67db8588 --- /dev/null +++ b/extension/partiql-extension-value-functions/LICENSE @@ -0,0 +1,175 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/extension/partiql-extension-value-functions/src/lib.rs b/extension/partiql-extension-value-functions/src/lib.rs new file mode 100644 index 00000000..bea7d7ad --- /dev/null +++ b/extension/partiql-extension-value-functions/src/lib.rs @@ -0,0 +1,96 @@ +#![deny(rust_2018_idioms)] +#![deny(clippy::all)] + +use partiql_catalog::call_defs::ScalarFnCallDef; +use partiql_catalog::catalog::Catalog; +use partiql_catalog::context::SessionContext; +use partiql_catalog::scalar_fn::{ + vararg_scalar_fn_overloads, ScalarFnExpr, ScalarFnExprResult, ScalarFunction, + SimpleScalarFunctionInfo, +}; +use partiql_value::{Tuple, Value}; +use std::borrow::Cow; +use std::error::Error; + +#[derive(Debug, Default)] +pub struct PartiqlValueFnExtension {} + +impl partiql_catalog::extension::Extension for PartiqlValueFnExtension { + fn name(&self) -> String { + "value-functions".into() + } + + fn load(&self, catalog: &mut dyn Catalog) -> Result<(), Box> { + for scfn in [function_catalog_tupleunion, function_catalog_tupleconcat] { + match catalog.add_scalar_function(scfn()) { + Ok(_) => continue, + Err(e) => return Err(Box::new(e) as Box), + } + } + Ok(()) + } +} + +fn function_catalog_tupleunion() -> ScalarFunction { + let scalar_fn = Box::new(TupleUnionFnExpr::default()); + let call_def = ScalarFnCallDef { + names: vec!["tupleunion"], + overloads: vararg_scalar_fn_overloads(scalar_fn), + }; + + let info = SimpleScalarFunctionInfo::new(call_def); + ScalarFunction::new(Box::new(info)) +} + +/// Represents a built-in tupleunion function, +/// e.g. `tupleunion({ 'bob': 1 }, { 'sally': 2 }, { 'sally': 2 })` -> `{'bob: 1, 'sally':1, 'sally':2}`. +#[derive(Debug, Clone, Default)] +struct TupleUnionFnExpr {} +impl ScalarFnExpr for TupleUnionFnExpr { + fn evaluate<'c>( + &self, + args: &[Cow<'_, Value>], + _ctx: &'c dyn SessionContext<'c>, + ) -> ScalarFnExprResult<'c> { + let mut t = Tuple::default(); + for arg in args { + t.extend( + arg.as_tuple_ref() + .pairs() + .map(|(k, v)| (k.as_str(), v.clone())), + ) + } + Ok(Cow::Owned(Value::from(t))) + } +} + +fn function_catalog_tupleconcat() -> ScalarFunction { + let scalar_fn = Box::new(TupleConcatFnExpr::default()); + let call_def = ScalarFnCallDef { + names: vec!["tupleconcat"], + overloads: vararg_scalar_fn_overloads(scalar_fn), + }; + + let info = SimpleScalarFunctionInfo::new(call_def); + ScalarFunction::new(Box::new(info)) +} + +/// Represents a built-in tupleconcat function, +/// e.g. `tupleconcat({ 'bob': 1 }, { 'sally': 2 }, { 'sally': 2 })` -> `{'bob: 1, 'sally':2}`. +#[derive(Debug, Clone, Default)] +struct TupleConcatFnExpr {} +impl ScalarFnExpr for TupleConcatFnExpr { + fn evaluate<'c>( + &self, + args: &[Cow<'_, Value>], + _ctx: &'c dyn SessionContext<'c>, + ) -> ScalarFnExprResult<'c> { + let result = args + .iter() + .map(|val| val.as_tuple_ref()) + .reduce(|l, r| Cow::Owned(l.tuple_concat(&r))) + .map(|v| v.into_owned()) + .unwrap_or_default(); + Ok(Cow::Owned(Value::from(result))) + } +} diff --git a/partiql-ast-passes/src/name_resolver.rs b/partiql-ast-passes/src/name_resolver.rs index 52147ba9..e08899ca 100644 --- a/partiql-ast-passes/src/name_resolver.rs +++ b/partiql-ast-passes/src/name_resolver.rs @@ -4,7 +4,7 @@ use indexmap::{IndexMap, IndexSet}; use partiql_ast::ast; use partiql_ast::ast::{GroupByExpr, GroupKey}; use partiql_ast::visit::{Traverse, Visit, Visitor}; -use partiql_catalog::Catalog; +use partiql_catalog::catalog::Catalog; use partiql_common::node::NodeId; use std::sync::atomic::{AtomicU32, Ordering}; diff --git a/partiql-catalog/Cargo.toml b/partiql-catalog/Cargo.toml index 40f2b85c..41a09fd9 100644 --- a/partiql-catalog/Cargo.toml +++ b/partiql-catalog/Cargo.toml @@ -21,8 +21,8 @@ edition.workspace = true bench = false [dependencies] +partiql-common = { path = "../partiql-common", version = "0.10.*" } partiql-value = { path = "../partiql-value", version = "0.10.*" } -partiql-parser = { path = "../partiql-parser", version = "0.10.*" } partiql-logical = { path = "../partiql-logical", version = "0.10.*" } partiql-types = { path = "../partiql-types", version = "0.10.*" } @@ -31,5 +31,4 @@ ordered-float = "4" itertools = "0.13" unicase = "2.7" -[dev-dependencies] -criterion = "0.5" +dyn-clone = "1" diff --git a/partiql-catalog/src/call_defs.rs b/partiql-catalog/src/call_defs.rs index a8a7284c..0d2b6957 100644 --- a/partiql-catalog/src/call_defs.rs +++ b/partiql-catalog/src/call_defs.rs @@ -4,6 +4,7 @@ use partiql_logical::ValueExpr; use std::fmt::{Debug, Formatter}; use thiserror::Error; +use crate::scalar_fn::ScalarFnExpr; use unicase::UniCase; /// An error that can happen during call lookup @@ -15,13 +16,6 @@ pub enum CallLookupError { InvalidNumberOfArguments(String), } -#[derive(Debug, Eq, PartialEq)] -pub enum CallArgument { - Positional(ValueExpr), - Named(String, ValueExpr), - Star, -} - #[derive(Debug)] pub struct CallDef { pub names: Vec<&'static str>, @@ -29,6 +23,7 @@ pub struct CallDef { } impl CallDef { + // Used when lowering AST -> plan pub fn lookup(&self, args: &[CallArgument], name: &str) -> Result { 'overload: for overload in &self.overloads { let formals = &overload.input; @@ -53,6 +48,13 @@ impl CallDef { } } +#[derive(Debug, Eq, PartialEq)] +pub enum CallArgument { + Positional(ValueExpr), + Named(String, ValueExpr), + Star, +} + #[derive(Debug, Copy, Clone)] pub enum CallSpecArg { Positional, @@ -75,8 +77,6 @@ impl CallSpecArg { } } -impl CallSpecArg {} - pub struct CallSpec { pub input: Vec, pub output: Box) -> logical::ValueExpr + Send + Sync>, @@ -87,3 +87,24 @@ impl Debug for CallSpec { write!(f, "CallSpec [{:?}]", &self.input) } } + +#[derive(Debug, Clone)] +pub struct ScalarFnCallDef { + pub names: Vec<&'static str>, + pub overloads: ScalarFnCallSpecs, +} + +pub type ScalarFnCallSpecs = Vec; + +#[derive(Clone)] +pub struct ScalarFnCallSpec { + // TODO: Include Scalar Function attributes (e.g., isNullCall and isMissingCall, etc.): https://github.com/partiql/partiql-lang-rust/issues/499 + pub input: Vec, + pub output: Box, +} + +impl Debug for ScalarFnCallSpec { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "ScalarFnCallSpec [{:?}]", &self.input) + } +} diff --git a/partiql-catalog/src/catalog.rs b/partiql-catalog/src/catalog.rs new file mode 100644 index 00000000..d52eb4a9 --- /dev/null +++ b/partiql-catalog/src/catalog.rs @@ -0,0 +1,294 @@ +use crate::call_defs::ScalarFnCallSpecs; +use crate::scalar_fn::ScalarFunction; +use crate::table_fn::TableFunction; +use partiql_common::catalog::{CatalogId, EntryId, ObjectId}; +use partiql_types::PartiqlShape; +use std::collections::HashMap; +use std::fmt::Debug; +use std::sync::atomic::{AtomicU64, Ordering}; +use thiserror::Error; +use unicase::UniCase; + +/// Contains the errors that occur during Catalog related operations +#[derive(Error, Debug, Clone, PartialEq)] +#[error("Catalog error: encountered errors")] +pub struct CatalogError { + pub errors: Vec, +} + +impl CatalogError { + #[must_use] + pub fn new(errors: Vec) -> Self { + CatalogError { errors } + } +} + +/// Catalog Error kind +/// +/// ### Notes +/// This is marked `#[non_exhaustive]`, to reserve the right to add more variants in the future. +#[derive(Error, Debug, Clone, PartialEq)] +#[non_exhaustive] +pub enum CatalogErrorKind { + /// Entry exists error. + #[error("Catalog error: entry already exists for `{0}`")] + EntryExists(String), + + /// Entry error. + #[error("Catalog error: `{0}`")] + EntryError(String), + + /// Any other catalog error. + #[error("Catalog error: unknown error")] + Unknown, +} + +pub trait Catalog: Debug { + fn add_table_function(&mut self, info: TableFunction) -> Result; + fn add_scalar_function(&mut self, info: ScalarFunction) -> Result; + + fn add_type_entry(&mut self, entry: TypeEnvEntry<'_>) -> Result; + + fn get_function(&self, name: &str) -> Option>; + + fn get_function_by_id(&self, id: ObjectId) -> Option>; + + fn resolve_type(&self, name: &str) -> Option; +} + +#[derive(Debug)] +pub struct TypeEnvEntry<'a> { + name: UniCase, + aliases: Vec<&'a str>, + ty: PartiqlShape, +} + +impl<'a> TypeEnvEntry<'a> { + #[must_use] + pub fn new(name: &str, aliases: &[&'a str], ty: PartiqlShape) -> Self { + TypeEnvEntry { + name: UniCase::from(name.to_string()), + aliases: aliases.to_vec(), + ty, + } + } +} + +#[derive(Debug, Clone)] +pub struct TypeEntry { + id: ObjectId, + ty: PartiqlShape, +} + +impl TypeEntry { + #[must_use] + pub fn id(&self) -> &ObjectId { + &self.id + } + + #[must_use] + pub fn ty(&self) -> &PartiqlShape { + &self.ty + } +} + +#[derive(Debug)] +#[allow(dead_code)] +pub struct FunctionEntry<'a> { + id: ObjectId, + function: &'a FunctionEntryFunction, +} + +#[derive(Debug)] +pub enum FunctionEntryFunction { + Table(TableFunction), + Scalar(ScalarFnCallSpecs), + Aggregate(), +} + +impl<'a> FunctionEntry<'a> { + pub fn id(&self) -> &ObjectId { + &self.id + } + + #[must_use] + pub fn entry(&'a self) -> &'a FunctionEntryFunction { + self.function + } +} + +#[derive(Debug)] +pub struct PartiqlCatalog { + functions: CatalogEntrySet, + types: CatalogEntrySet, + id: CatalogId, +} + +impl Default for PartiqlCatalog { + fn default() -> Self { + PartiqlCatalog { + functions: Default::default(), + types: Default::default(), + id: 1.into(), + } + } +} + +impl PartiqlCatalog {} + +impl Catalog for PartiqlCatalog { + fn add_table_function(&mut self, info: TableFunction) -> Result { + let call_def = info.call_def(); + let names = call_def.names.clone(); + if let Some((name, aliases)) = names.split_first() { + let eid = self + .functions + .add(name, aliases, FunctionEntryFunction::Table(info))?; + Ok((self.id, eid).into()) + } else { + Err(CatalogError::new(vec![CatalogErrorKind::EntryError( + "Function definition has no name".into(), + )])) + } + } + + fn add_scalar_function(&mut self, info: ScalarFunction) -> Result { + let id = self.id; + let call_def = info.into_call_def(); + let names = call_def.names; + if let Some((name, aliases)) = names.split_first() { + self.functions + .add( + name, + aliases, + FunctionEntryFunction::Scalar(call_def.overloads), + ) + .map(|eid| ObjectId::new(id, eid)) + } else { + Err(CatalogError::new(vec![CatalogErrorKind::EntryError( + "Function definition has no name".into(), + )])) + } + } + + fn add_type_entry(&mut self, entry: TypeEnvEntry<'_>) -> Result { + let eid = self + .types + .add(entry.name.as_ref(), entry.aliases.as_slice(), entry.ty); + + match eid { + Ok(eid) => Ok((self.id, eid).into()), + Err(e) => Err(e), + } + } + + fn get_function(&self, name: &str) -> Option> { + self.functions + .find_by_name(name) + .map(|(e, f)| self.to_function_entry(e, f)) + } + + fn get_function_by_id(&self, id: ObjectId) -> Option> { + assert_eq!(self.id, id.catalog_id()); + self.functions + .find_by_id(&id.entry_id()) + .map(|(e, f)| self.to_function_entry(e, f)) + } + + fn resolve_type(&self, name: &str) -> Option { + self.types.find_by_name(name).map(|(eid, entry)| TypeEntry { + id: (self.id, eid).into(), + ty: entry.clone(), + }) + } +} + +impl PartiqlCatalog { + fn to_function_entry<'a>( + &'a self, + eid: EntryId, + entry: &'a FunctionEntryFunction, + ) -> FunctionEntry<'a> { + FunctionEntry { + id: (self.id, eid).into(), + function: entry, + } + } +} + +#[derive(Debug)] +struct CatalogEntrySet { + entries: HashMap, + by_name: HashMap, EntryId>, + by_alias: HashMap, EntryId>, + + next_id: AtomicU64, +} + +impl Default for CatalogEntrySet { + fn default() -> Self { + CatalogEntrySet { + entries: Default::default(), + by_name: Default::default(), + by_alias: Default::default(), + next_id: 1.into(), + } + } +} + +impl CatalogEntrySet { + fn add(&mut self, name: &str, aliases: &[&str], info: T) -> Result { + let mut errors = vec![]; + let name = UniCase::from(name); + let aliases: Vec> = aliases + .iter() + .map(|a| UniCase::from((*a).to_string())) + .collect(); + + for a in &aliases { + if self.by_alias.contains_key(a) { + errors.push(CatalogErrorKind::EntryExists(a.as_ref().to_string())); + } + } + + if self.by_name.contains_key(&name) { + errors.push(CatalogErrorKind::EntryExists(name.to_string())); + } + + let id = self.next_id.fetch_add(1, Ordering::SeqCst).into(); + + if let Some(_old_val) = self.entries.insert(id, info) { + errors.push(CatalogErrorKind::Unknown); + } + + match errors.is_empty() { + true => { + self.by_name.insert(name, id); + + for a in aliases { + self.by_alias.insert(a, id); + } + + Ok(id) + } + _ => Err(CatalogError::new(errors)), + } + } + + fn find_by_id(&self, eid: &EntryId) -> Option<(EntryId, &T)> { + self.entries.get(eid).map(|e| (*eid, e)) + } + + fn find_by_name(&self, name: &str) -> Option<(EntryId, &T)> { + let name = UniCase::from(name); + let eid = self.by_name.get(&name).or(self.by_alias.get(&name)); + + eid.and_then(|eid| self.find_by_id(eid)) + } +} + +#[cfg(test)] +mod tests { + #[test] + fn todo() {} +} diff --git a/partiql-catalog/src/extension.rs b/partiql-catalog/src/extension.rs new file mode 100644 index 00000000..a6095378 --- /dev/null +++ b/partiql-catalog/src/extension.rs @@ -0,0 +1,11 @@ +use crate::catalog::Catalog; +use std::error::Error; +use std::fmt::Debug; + +pub trait Extension: Debug { + fn name(&self) -> String; + + fn load(&self, catalog: &mut dyn Catalog) -> Result<(), Box>; +} + +pub type ExtensionResultError = Box; diff --git a/partiql-catalog/src/lib.rs b/partiql-catalog/src/lib.rs index 25ddba7d..ac65a659 100644 --- a/partiql-catalog/src/lib.rs +++ b/partiql-catalog/src/lib.rs @@ -1,348 +1,11 @@ #![deny(rust_2018_idioms)] #![deny(clippy::all)] -use crate::call_defs::CallDef; - -use partiql_types::PartiqlShape; -use partiql_value::Value; -use std::borrow::Cow; - -use crate::context::SessionContext; -use std::collections::HashMap; -use std::error::Error; -use std::fmt::Debug; -use std::sync::atomic::{AtomicU64, Ordering}; -use thiserror::Error; -use unicase::UniCase; - pub mod call_defs; pub mod context; -pub trait Extension: Debug { - fn name(&self) -> String; - fn load(&self, catalog: &mut dyn Catalog) -> Result<(), Box>; -} - -#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Copy, Clone, Hash)] -struct CatalogId(u64); - -impl From for CatalogId { - fn from(value: u64) -> Self { - CatalogId(value) - } -} - -#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Copy, Clone, Hash)] -struct EntryId(u64); - -impl From for EntryId { - fn from(value: u64) -> Self { - EntryId(value) - } -} - -#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Copy, Clone, Hash)] -pub struct ObjectId { - catalog_id: CatalogId, - entry_id: EntryId, -} - -pub type BaseTableExprResultError = Box; - -pub type BaseTableExprResultValueIter<'a> = - Box>>; -pub type BaseTableExprResult<'a> = - Result, BaseTableExprResultError>; - -pub trait BaseTableExpr: Debug { - fn evaluate<'c>( - &self, - args: &[Cow<'_, Value>], - ctx: &'c dyn SessionContext<'c>, - ) -> BaseTableExprResult<'c>; -} - -pub trait BaseTableFunctionInfo: Debug { - fn call_def(&self) -> &CallDef; - fn plan_eval(&self) -> Box; -} - -#[derive(Debug)] -pub struct TableFunction { - info: Box, -} - -impl TableFunction { - #[must_use] - pub fn new(info: Box) -> Self { - TableFunction { info } - } -} - -/// Contains the errors that occur during Catalog related operations -#[derive(Error, Debug, Clone, PartialEq)] -#[error("Catalog error: encountered errors")] -pub struct CatalogError { - pub errors: Vec, -} - -impl CatalogError { - #[must_use] - pub fn new(errors: Vec) -> Self { - CatalogError { errors } - } -} - -/// Catalog Error kind -/// -/// ### Notes -/// This is marked `#[non_exhaustive]`, to reserve the right to add more variants in the future. -#[derive(Error, Debug, Clone, PartialEq)] -#[non_exhaustive] -pub enum CatalogErrorKind { - /// Entry exists error. - #[error("Catalog error: entry already exists for `{0}`")] - EntryExists(String), - - /// Entry error. - #[error("Catalog error: `{0}`")] - EntryError(String), - - /// Any other catalog error. - #[error("Catalog error: unknown error")] - Unknown, -} - -pub trait Catalog: Debug { - fn add_table_function(&mut self, info: TableFunction) -> Result; - - fn add_type_entry(&mut self, entry: TypeEnvEntry<'_>) -> Result; - - fn get_function(&self, name: &str) -> Option>; - - fn resolve_type(&self, name: &str) -> Option; -} - -#[derive(Debug)] -pub struct TypeEnvEntry<'a> { - name: UniCase, - aliases: Vec<&'a str>, - ty: PartiqlShape, -} - -impl<'a> TypeEnvEntry<'a> { - #[must_use] - pub fn new(name: &str, aliases: &[&'a str], ty: PartiqlShape) -> Self { - TypeEnvEntry { - name: UniCase::from(name.to_string()), - aliases: aliases.to_vec(), - ty, - } - } -} - -#[derive(Debug, Clone)] -pub struct TypeEntry { - id: ObjectId, - ty: PartiqlShape, -} - -impl TypeEntry { - #[must_use] - pub fn id(&self) -> &ObjectId { - &self.id - } - - #[must_use] - pub fn ty(&self) -> &PartiqlShape { - &self.ty - } -} - -#[derive(Debug)] -#[allow(dead_code)] -pub struct FunctionEntry<'a> { - id: ObjectId, - function: &'a FunctionEntryFunction, -} - -#[derive(Debug)] -pub enum FunctionEntryFunction { - Table(TableFunction), - Scalar(), - Aggregate(), -} - -impl<'a> FunctionEntry<'a> { - #[must_use] - pub fn call_def(&'a self) -> &'a CallDef { - match &self.function { - FunctionEntryFunction::Table(tf) => tf.info.call_def(), - FunctionEntryFunction::Scalar() => todo!(), - FunctionEntryFunction::Aggregate() => todo!(), - } - } - - #[must_use] - pub fn plan_eval(&'a self) -> Box { - match &self.function { - FunctionEntryFunction::Table(tf) => tf.info.plan_eval(), - FunctionEntryFunction::Scalar() => todo!(), - FunctionEntryFunction::Aggregate() => todo!(), - } - } -} - -#[derive(Debug)] -pub struct PartiqlCatalog { - functions: CatalogEntrySet, - types: CatalogEntrySet, - id: CatalogId, -} - -impl Default for PartiqlCatalog { - fn default() -> Self { - PartiqlCatalog { - functions: Default::default(), - types: Default::default(), - id: CatalogId(1), - } - } -} - -impl PartiqlCatalog {} - -impl Catalog for PartiqlCatalog { - fn add_table_function(&mut self, info: TableFunction) -> Result { - let call_def = info.info.call_def(); - let names = call_def.names.clone(); - if let Some((name, aliases)) = names.split_first() { - let id = self - .functions - .add(name, aliases, FunctionEntryFunction::Table(info))?; - Ok(ObjectId { - catalog_id: self.id, - entry_id: id, - }) - } else { - Err(CatalogError::new(vec![CatalogErrorKind::EntryError( - "Function definition has no name".into(), - )])) - } - } - - fn add_type_entry(&mut self, entry: TypeEnvEntry<'_>) -> Result { - let id = self - .types - .add(entry.name.as_ref(), entry.aliases.as_slice(), entry.ty); - - match id { - Ok(id) => Ok(ObjectId { - catalog_id: self.id, - entry_id: id, - }), - Err(e) => Err(e), - } - } - - fn get_function(&self, name: &str) -> Option> { - self.functions - .find_by_name(name) - .map(|(eid, entry)| FunctionEntry { - id: ObjectId { - catalog_id: self.id, - entry_id: eid, - }, - function: entry, - }) - } - - fn resolve_type(&self, name: &str) -> Option { - self.types.find_by_name(name).map(|(eid, entry)| TypeEntry { - id: ObjectId { - catalog_id: self.id, - entry_id: eid, - }, - ty: entry.clone(), - }) - } -} - -#[derive(Debug)] -struct CatalogEntrySet { - entries: HashMap, - by_name: HashMap, EntryId>, - by_alias: HashMap, EntryId>, - - next_id: AtomicU64, -} - -impl Default for CatalogEntrySet { - fn default() -> Self { - CatalogEntrySet { - entries: Default::default(), - by_name: Default::default(), - by_alias: Default::default(), - next_id: 1.into(), - } - } -} - -impl CatalogEntrySet { - fn add(&mut self, name: &str, aliases: &[&str], info: T) -> Result { - let mut errors = vec![]; - let name = UniCase::from(name); - let aliases: Vec> = aliases - .iter() - .map(|a| UniCase::from((*a).to_string())) - .collect(); - - for a in &aliases { - if self.by_alias.contains_key(a) { - errors.push(CatalogErrorKind::EntryExists(a.as_ref().to_string())); - } - } - - if self.by_name.contains_key(&name) { - errors.push(CatalogErrorKind::EntryExists(name.to_string())); - } - - let id = self.next_id.fetch_add(1, Ordering::SeqCst).into(); - - if let Some(_old_val) = self.entries.insert(id, info) { - errors.push(CatalogErrorKind::Unknown); - } - - match errors.is_empty() { - true => { - self.by_name.insert(name, id); - - for a in aliases { - self.by_alias.insert(a, id); - } - - Ok(id) - } - _ => Err(CatalogError::new(errors)), - } - } - - fn find_by_name(&self, name: &str) -> Option<(EntryId, &T)> { - let name = UniCase::from(name); - - let eid = self.by_name.get(&name).or(self.by_alias.get(&name)); - - if let Some(eid) = eid { - self.entries.get(eid).map(|e| (*eid, e)) - } else { - None - } - } -} - -#[cfg(test)] -mod tests { - #[test] - fn todo() {} -} +pub mod catalog; +pub mod extension; +pub mod scalar_fn; +pub mod table_fn; diff --git a/partiql-catalog/src/scalar_fn.rs b/partiql-catalog/src/scalar_fn.rs new file mode 100644 index 00000000..d7904b43 --- /dev/null +++ b/partiql-catalog/src/scalar_fn.rs @@ -0,0 +1,82 @@ +use crate::call_defs::{CallSpecArg, ScalarFnCallDef, ScalarFnCallSpec}; +use crate::context::SessionContext; + +use crate::extension::ExtensionResultError; +use dyn_clone::DynClone; +use partiql_common::FN_VAR_ARG_MAX; +use partiql_value::Value; +use std::borrow::Cow; +use std::fmt::{Debug, Formatter}; + +pub type ScalarFnExprResultValue<'a> = Cow<'a, Value>; +pub type ScalarFnExprResult<'a> = Result, ExtensionResultError>; + +pub trait ScalarFnExpr: DynClone + Debug { + fn evaluate<'c>( + &self, + args: &[Cow<'_, Value>], + ctx: &'c dyn SessionContext<'c>, + ) -> ScalarFnExprResult<'c>; +} + +dyn_clone::clone_trait_object!(ScalarFnExpr); + +pub trait ScalarFunctionInfo: Debug { + fn call_def(&self) -> &ScalarFnCallDef; + + fn into_call_def(self: Box) -> ScalarFnCallDef; +} + +pub struct SimpleScalarFunctionInfo { + call_def: ScalarFnCallDef, +} + +impl SimpleScalarFunctionInfo { + pub fn new(call_def: ScalarFnCallDef) -> Self { + Self { call_def } + } +} + +impl ScalarFunctionInfo for SimpleScalarFunctionInfo { + fn call_def(&self) -> &ScalarFnCallDef { + &self.call_def + } + + fn into_call_def(self: Box) -> ScalarFnCallDef { + self.call_def + } +} + +impl Debug for SimpleScalarFunctionInfo { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.call_def.fmt(f) + } +} + +#[derive(Debug)] +pub struct ScalarFunction { + info: Box, +} + +impl ScalarFunction { + pub fn new(info: Box) -> Self { + ScalarFunction { info } + } + + pub fn call_def(&self) -> &ScalarFnCallDef { + self.info.call_def() + } + + pub fn into_call_def(self) -> ScalarFnCallDef { + self.info.into_call_def() + } +} + +pub fn vararg_scalar_fn_overloads(scalar_fn_expr: Box) -> Vec { + (1..=FN_VAR_ARG_MAX) + .map(|n| ScalarFnCallSpec { + input: std::iter::repeat(CallSpecArg::Positional).take(n).collect(), + output: scalar_fn_expr.clone(), + }) + .collect() +} diff --git a/partiql-catalog/src/table_fn.rs b/partiql-catalog/src/table_fn.rs new file mode 100644 index 00000000..b66b71af --- /dev/null +++ b/partiql-catalog/src/table_fn.rs @@ -0,0 +1,43 @@ +use crate::call_defs::CallDef; +use crate::context::SessionContext; +use crate::extension::ExtensionResultError; +use partiql_value::Value; +use std::borrow::Cow; +use std::fmt::Debug; + +pub type BaseTableExprResultValueIter<'a> = + Box>>; +pub type BaseTableExprResult<'a> = Result, ExtensionResultError>; + +pub trait BaseTableExpr: Debug { + fn evaluate<'c>( + &self, + args: &[Cow<'_, Value>], + ctx: &'c dyn SessionContext<'c>, + ) -> BaseTableExprResult<'c>; +} + +pub trait BaseTableFunctionInfo: Debug { + fn call_def(&self) -> &CallDef; + fn plan_eval(&self) -> Box; +} + +#[derive(Debug)] +pub struct TableFunction { + info: Box, +} + +impl TableFunction { + #[must_use] + pub fn new(info: Box) -> Self { + TableFunction { info } + } + + pub fn call_def(&self) -> &CallDef { + self.info.call_def() + } + + pub fn plan_eval(&self) -> Box { + self.info.plan_eval() + } +} diff --git a/partiql-common/src/catalog.rs b/partiql-common/src/catalog.rs new file mode 100644 index 00000000..be534f62 --- /dev/null +++ b/partiql-common/src/catalog.rs @@ -0,0 +1,51 @@ +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Copy, Clone, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct CatalogId(u64); + +impl From for CatalogId { + fn from(value: u64) -> Self { + CatalogId(value) + } +} + +#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Copy, Clone, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct EntryId(u64); + +impl From for EntryId { + fn from(value: u64) -> Self { + EntryId(value) + } +} + +#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Copy, Clone, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct ObjectId { + catalog_id: CatalogId, + entry_id: EntryId, +} + +impl ObjectId { + pub fn new(catalog_id: CatalogId, entry_id: EntryId) -> Self { + Self { + catalog_id, + entry_id, + } + } + + pub fn catalog_id(&self) -> CatalogId { + self.catalog_id + } + pub fn entry_id(&self) -> EntryId { + self.entry_id + } +} + +impl From<(CatalogId, EntryId)> for ObjectId { + fn from((catalog_id, entry_id): (CatalogId, EntryId)) -> Self { + ObjectId::new(catalog_id, entry_id) + } +} diff --git a/partiql-common/src/lib.rs b/partiql-common/src/lib.rs index 7efacd9a..8238c394 100644 --- a/partiql-common/src/lib.rs +++ b/partiql-common/src/lib.rs @@ -1,5 +1,9 @@ #![deny(rust_2018_idioms)] #![deny(clippy::all)] + +pub static FN_VAR_ARG_MAX: usize = 10; pub mod metadata; pub mod node; pub mod syntax; + +pub mod catalog; diff --git a/partiql-conformance-tests/tests/mod.rs b/partiql-conformance-tests/tests/mod.rs index 74e1d507..651afe15 100644 --- a/partiql-conformance-tests/tests/mod.rs +++ b/partiql-conformance-tests/tests/mod.rs @@ -1,5 +1,4 @@ use partiql_ast_passes::error::AstTransformationError; -use partiql_catalog::{Catalog, PartiqlCatalog}; use partiql_eval as eval; use partiql_eval::error::{EvalErr, PlanErr}; @@ -8,6 +7,7 @@ use partiql_logical as logical; use partiql_parser::{Parsed, ParserError, ParserResult}; use partiql_value::DateTime; +use partiql_catalog::catalog::{Catalog, PartiqlCatalog}; use partiql_catalog::context::SystemContext; use thiserror::Error; diff --git a/partiql-eval/benches/bench_eval.rs b/partiql-eval/benches/bench_eval.rs index f1d7ecf5..cf5eaabb 100644 --- a/partiql-eval/benches/bench_eval.rs +++ b/partiql-eval/benches/bench_eval.rs @@ -2,8 +2,8 @@ use std::borrow::Cow; use std::time::Duration; use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use partiql_catalog::catalog::PartiqlCatalog; use partiql_catalog::context::SystemContext; -use partiql_catalog::PartiqlCatalog; use partiql_eval::env::basic::MapBindings; use partiql_eval::eval::{BasicContext, EvalPlan}; diff --git a/partiql-eval/src/error.rs b/partiql-eval/src/error.rs index 50f22112..bda6dd89 100644 --- a/partiql-eval/src/error.rs +++ b/partiql-eval/src/error.rs @@ -1,7 +1,7 @@ use crate::eval::evaluable::Evaluable; use crate::eval::expr::EvalExpr; use crate::eval::EvalContext; -use partiql_catalog::BaseTableExprResultError; +use partiql_catalog::extension::ExtensionResultError; use partiql_value::{Tuple, Value}; use std::borrow::Cow; use thiserror::Error; @@ -45,8 +45,8 @@ pub enum EvaluationError { NotYetImplemented(String), /// Error originating in an extension - #[error("Base Table Expression Error")] - ExtensionResultError(#[from] BaseTableExprResultError), + #[error("Extension Result Expression Error")] + ExtensionResultError(#[from] ExtensionResultError), } /// Used when an error occurs during the logical to eval plan conversion. Allows the conversion diff --git a/partiql-eval/src/eval/eval_expr_wrapper.rs b/partiql-eval/src/eval/eval_expr_wrapper.rs index b9a21d4f..b88d3cc6 100644 --- a/partiql-eval/src/eval/eval_expr_wrapper.rs +++ b/partiql-eval/src/eval/eval_expr_wrapper.rs @@ -258,7 +258,7 @@ impl, ArgC: ArgChecker &'a self, bindings: &'a Tuple, ctx: &'c dyn EvalContext<'c>, - ) -> ControlFlow; N]> + ) -> ControlFlow; N]> where 'c: 'a, { @@ -273,57 +273,86 @@ impl, ArgC: ArgChecker ControlFlow::Break(Missing) }; - let mut result = Vec::with_capacity(N); + match evaluate_args::<{ STRICT }, ArgC, _>(&self.args, |n| &self.types[n], bindings, ctx) { + ControlFlow::Continue(result) => match result.try_into() { + Ok(a) => ControlFlow::Continue(a), + Err(args) => err_arg_count_mismatch(args), + }, + ControlFlow::Break(v) => ControlFlow::Break(v), + } + } +} - let mut propagate = None; - for i in 0..N { - let typ = &self.types[i]; - let arg = self.args[i].evaluate(bindings, ctx); +pub(crate) fn evaluate_args< + 'a, + 'c, + 't, + const STRICT: bool, + ArgC: ArgChecker, + F: Fn(usize) -> &'t PartiqlShape, +>( + args: &'a [Box], + types: F, + bindings: &'a Tuple, + ctx: &'c dyn EvalContext<'c>, +) -> ControlFlow>> +where + 'c: 'a, +{ + let mut result = Vec::with_capacity(args.len()); - match ArgC::arg_check(typ, arg) { - ArgCheckControlFlow::Continue(v) => { - if propagate.is_none() { - result.push(v); - } - } - ArgCheckControlFlow::Propagate(v) => { - propagate = match propagate { - None => Some(v), - Some(prev) => match (prev, v) { - (Null, Missing) => Missing, - (Missing, _) => Missing, - (Null, _) => Null, - (_, new) => new, - } - .into(), - }; + let mut propagate = None; + + for (idx, arg) in args.iter().enumerate() { + let typ = types(idx); + let arg = arg.evaluate(bindings, ctx); + + match ArgC::arg_check(typ, arg) { + ArgCheckControlFlow::Continue(v) => { + if propagate.is_none() { + result.push(v); } - ArgCheckControlFlow::ShortCircuit(v) => return ControlFlow::Break(v), - ArgCheckControlFlow::ErrorOrShortCircuit(v) => { - if STRICT { - let signature = self.types.iter().map(|typ| format!("{}", typ)).join(","); - let before = (0..i).map(|_| "_"); - let arg = "MISSING"; // TODO display actual argument? - let after = (i + 1..N).map(|_| "_"); - let arg_pattern = before.chain(std::iter::once(arg)).chain(after).join(","); - let msg = format!("expected `({signature})`, found `({arg_pattern})`"); - ctx.add_error(EvaluationError::IllegalState(msg)); + } + ArgCheckControlFlow::Propagate(v) => { + propagate = match propagate { + None => Some(v), + Some(prev) => match (prev, v) { + (Null, Missing) => Missing, + (Missing, _) => Missing, + (Null, _) => Null, + (_, new) => new, } - return ControlFlow::Break(v); + .into(), + }; + } + ArgCheckControlFlow::ShortCircuit(v) => return ControlFlow::Break(v), + ArgCheckControlFlow::ErrorOrShortCircuit(v) => { + if STRICT { + let arg_end = args.len(); + let arg_count = 0..arg_end; + + let signature = (arg_count) + .map(types) + .map(|typ| format!("{}", typ)) + .join(","); + let before = (0..idx).map(|_| "_"); + let arg = "MISSING"; // TODO display actual argument? + let after = (idx + 1..arg_end).map(|_| "_"); + let arg_pattern = before.chain(std::iter::once(arg)).chain(after).join(","); + let msg = format!("expected `({signature})`, found `({arg_pattern})`"); + ctx.add_error(EvaluationError::IllegalState(msg)); } + return ControlFlow::Break(v); } } + } - if let Some(v) = propagate { - // If `propagate` is a `Some`, then argument type checking failed, propagate the value - ControlFlow::Break(v) - } else { - // If `propagate` is `None`, then try to convert the `result` vec into an array of `N` - match result.try_into() { - Ok(a) => ControlFlow::Continue(a), - Err(args) => err_arg_count_mismatch(args), - } - } + if let Some(v) = propagate { + // If `propagate` is a `Some`, then argument type checking failed, propagate the value + ControlFlow::Break(v) + } else { + // If `propagate` is `None`, then return result + ControlFlow::Continue(result) } } diff --git a/partiql-eval/src/eval/expr/base_table.rs b/partiql-eval/src/eval/expr/base_table.rs index 1d6effc9..daad61a3 100644 --- a/partiql-eval/src/eval/expr/base_table.rs +++ b/partiql-eval/src/eval/expr/base_table.rs @@ -1,7 +1,7 @@ use crate::eval::expr::EvalExpr; use crate::eval::EvalContext; use itertools::Itertools; -use partiql_catalog::BaseTableExpr; +use partiql_catalog::table_fn::BaseTableExpr; use partiql_value::Value::Missing; use partiql_value::{Bag, Tuple, Value}; diff --git a/partiql-eval/src/eval/expr/functions.rs b/partiql-eval/src/eval/expr/functions.rs new file mode 100644 index 00000000..1d6e77e4 --- /dev/null +++ b/partiql-eval/src/eval/expr/functions.rs @@ -0,0 +1,55 @@ +use crate::eval::eval_expr_wrapper::{evaluate_args, DefaultArgChecker, PropagateMissing}; + +use crate::eval::expr::{BindError, BindEvalExpr, EvalExpr}; +use crate::eval::EvalContext; + +use partiql_types::{PartiqlShapeBuilder, StructType}; +use partiql_value::{Tuple, Value}; + +use std::borrow::Cow; +use std::fmt::Debug; + +use crate::error::EvaluationError; +use partiql_catalog::call_defs::ScalarFnCallSpec; +use partiql_catalog::scalar_fn::ScalarFnExpr; +use std::ops::ControlFlow; + +impl BindEvalExpr for ScalarFnCallSpec { + fn bind( + &self, + args: Vec>, + ) -> Result, BindError> { + let plan = self.output.clone(); + Ok(Box::new(EvalExprFnScalar::<{ STRICT }> { plan, args })) + } +} + +#[derive(Debug)] +pub(crate) struct EvalExprFnScalar { + plan: Box, + args: Vec>, +} + +impl EvalExpr for EvalExprFnScalar { + fn evaluate<'a, 'c>( + &'a self, + bindings: &'a Tuple, + ctx: &'c dyn EvalContext<'c>, + ) -> Cow<'a, Value> + where + 'c: 'a, + { + type Check = DefaultArgChecker>; + let typ = PartiqlShapeBuilder::init_or_get().new_struct(StructType::new_any()); + match evaluate_args::<{ STRICT }, Check, _>(&self.args, |_| &typ, bindings, ctx) { + ControlFlow::Break(v) => Cow::Owned(v), + ControlFlow::Continue(args) => match self.plan.evaluate(&args, ctx.as_session()) { + Ok(v) => v, + Err(e) => { + ctx.add_error(EvaluationError::ExtensionResultError(e)); + Cow::Owned(Value::Missing) + } + }, + } + } +} diff --git a/partiql-eval/src/eval/expr/mod.rs b/partiql-eval/src/eval/expr/mod.rs index 8e107e19..4f19c0bf 100644 --- a/partiql-eval/src/eval/expr/mod.rs +++ b/partiql-eval/src/eval/expr/mod.rs @@ -14,7 +14,9 @@ mod path; pub(crate) use path::*; mod pattern_match; pub(crate) use pattern_match::*; +mod functions; mod operators; + pub(crate) use operators::*; use crate::eval::EvalContext; diff --git a/partiql-eval/src/eval/expr/strings.rs b/partiql-eval/src/eval/expr/strings.rs index 8e0d50a9..d5b2ff9e 100644 --- a/partiql-eval/src/eval/expr/strings.rs +++ b/partiql-eval/src/eval/expr/strings.rs @@ -1,17 +1,14 @@ use crate::eval::eval_expr_wrapper::{ - BinaryValueExpr, EvalExprWrapper, ExecuteEvalExpr, QuaternaryValueExpr, TernaryValueExpr, - UnaryValueExpr, + BinaryValueExpr, QuaternaryValueExpr, TernaryValueExpr, UnaryValueExpr, }; use crate::eval::expr::{BindError, BindEvalExpr, EvalExpr}; -use crate::eval::EvalContext; use itertools::Itertools; use partiql_types::{type_int, type_string}; use partiql_value::Value; use partiql_value::Value::Missing; -use std::borrow::{Borrow, Cow}; use std::fmt::Debug; #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -60,28 +57,6 @@ impl BindEvalExpr for EvalStringFn { } } -impl ExecuteEvalExpr<1> for EvalExprWrapper -where - F: Fn(&Box) -> R, - R: Into, -{ - #[inline] - fn evaluate<'a, 'c>( - &'a self, - args: [Cow<'a, Value>; 1], - _ctx: &'c dyn EvalContext<'c>, - ) -> Cow<'a, Value> - where - 'c: 'a, - { - let [value] = args; - Cow::Owned(match value.borrow() { - Value::String(s) => ((self.f)(s)).into(), - _ => Missing, - }) - } -} - #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub(crate) enum EvalTrimFn { /// Represents a built-in trim string function, e.g. `trim(both from ' foobar ')`. diff --git a/partiql-eval/src/lib.rs b/partiql-eval/src/lib.rs index a7f3a53c..846a16db 100644 --- a/partiql-eval/src/lib.rs +++ b/partiql-eval/src/lib.rs @@ -12,8 +12,8 @@ mod tests { use crate::env::basic::MapBindings; use crate::plan; + use partiql_catalog::catalog::PartiqlCatalog; use partiql_catalog::context::SystemContext; - use partiql_catalog::PartiqlCatalog; use rust_decimal_macros::dec; use partiql_logical as logical; diff --git a/partiql-eval/src/plan.rs b/partiql-eval/src/plan.rs index d3f3b83e..961d6187 100644 --- a/partiql-eval/src/plan.rs +++ b/partiql-eval/src/plan.rs @@ -1,9 +1,8 @@ use itertools::{Either, Itertools}; +use partiql_logical as logical; use petgraph::prelude::StableGraph; use std::collections::HashMap; -use partiql_logical as logical; - use partiql_logical::{ AggFunc, BagOperator, BinaryOp, BindingsOp, CallName, GroupingStrategy, IsTypeExpr, JoinKind, LogicalPlan, OpId, PathComponent, Pattern, PatternMatchExpr, SearchedCase, SetQuantifier, @@ -25,7 +24,7 @@ use crate::eval::expr::{ EvalPath, EvalSearchedCaseExpr, EvalStringFn, EvalTrimFn, EvalTupleExpr, EvalVarRef, }; use crate::eval::EvalPlan; -use partiql_catalog::Catalog; +use partiql_catalog::catalog::{Catalog, FunctionEntryFunction}; use partiql_value::Value::Null; #[macro_export] @@ -711,27 +710,65 @@ impl<'c> EvaluatorPlanner<'c> { "coll_every", EvalCollFn::Every(setq.into()).bind::<{ STRICT }>(args), ), - CallName::ByName(name) => match self.catalog.get_function(name) { - None => { - self.errors.push(PlanningError::IllegalState(format!( - "Function to exist in catalog {name}", - ))); - - ( - name.as_str(), - Ok(Box::new(ErrorNode::new()) as Box), - ) - } - Some(function) => { - let eval = function.plan_eval(); - - ( - name.as_str(), - Ok(Box::new(EvalFnBaseTableExpr { args, expr: eval }) - as Box), - ) - } - }, + CallName::ByName(name) => { + let plan = match self.catalog.get_function(name) { + None => { + self.errors.push(PlanningError::IllegalState(format!( + "Function call spec {name} does not exist in catalog", + ))); + + Ok(Box::new(ErrorNode::new()) as Box) + } + Some(function) => match function.entry() { + FunctionEntryFunction::Scalar(_) => { + todo!("Scalar functions in catalog by name") + } + FunctionEntryFunction::Table(tbl_fn) => { + Ok(Box::new(EvalFnBaseTableExpr { + args, + expr: tbl_fn.plan_eval(), + }) as Box) + } + FunctionEntryFunction::Aggregate() => { + todo!("Aggregate functions in catalog by name") + } + }, + }; + (name.as_str(), plan) + } + CallName::ById(name, oid, overload_idx) => { + let func = self.catalog.get_function_by_id(*oid); + let plan = match func { + Some(func) => match func.entry() { + FunctionEntryFunction::Table(_) => { + todo!("table functions in catalog by id") + } + FunctionEntryFunction::Scalar(scfn) => { + match scfn.get(*overload_idx) { + None => { + self.errors.push(PlanningError::IllegalState(format!( + "Function call spec {name} overload #{overload_idx} does not exist in catalog", + ))); + + Ok(Box::new(ErrorNode::new()) as Box) + } + Some(overload) => overload.bind::<{ STRICT }>(args), + } + } + FunctionEntryFunction::Aggregate() => { + todo!("Aggregate functions in catalog by id") + } + }, + None => { + self.errors.push(PlanningError::IllegalState(format!( + "Function call spec {name} does not exist in catalog", + ))); + + Ok(Box::new(ErrorNode::new()) as Box) + } + }; + (name.as_str(), plan) + } } } }; @@ -743,7 +780,7 @@ impl<'c> EvaluatorPlanner<'c> { #[cfg(test)] mod tests { use super::*; - use partiql_catalog::PartiqlCatalog; + use partiql_catalog::catalog::PartiqlCatalog; use partiql_logical::CallExpr; use partiql_logical::ExprQuery; use partiql_value::Value; diff --git a/partiql-logical-planner/src/functions.rs b/partiql-logical-planner/src/functions.rs new file mode 100644 index 00000000..1ab6705a --- /dev/null +++ b/partiql-logical-planner/src/functions.rs @@ -0,0 +1,82 @@ +use partiql_catalog::call_defs::{CallArgument, CallLookupError, CallSpecArg, ScalarFnCallSpecs}; +use partiql_catalog::catalog::{FunctionEntry, FunctionEntryFunction}; +use partiql_common::catalog::ObjectId; +use partiql_logical::{CallExpr, CallName, ValueExpr}; +use unicase::UniCase; + +pub(crate) trait Function { + fn resolve(&self, name: &str, args: &[CallArgument]) -> Result; +} + +impl<'a> Function for FunctionEntry<'a> { + fn resolve(&self, name: &str, args: &[CallArgument]) -> Result { + let oid = self.id(); + match self.entry() { + FunctionEntryFunction::Table(tbl) => { + tbl.call_def().lookup(args, name).map_err(Into::into) + } + FunctionEntryFunction::Scalar(scfn) => { + ScalarFnResolver { oid, scfn }.resolve(name, args) + } + FunctionEntryFunction::Aggregate() => { + todo!("Aggregate function resolution") + } + } + } +} + +struct ScalarFnResolver<'a> { + pub oid: &'a ObjectId, + pub scfn: &'a ScalarFnCallSpecs, +} + +impl<'a> Function for ScalarFnResolver<'a> { + fn resolve(&self, name: &str, args: &[CallArgument]) -> Result { + let oid = self.oid; + let overloads = self.scfn; + 'overload: for (idx, overload) in overloads.iter().enumerate() { + let formals = &overload.input; + if formals.len() != args.len() { + continue 'overload; + } + + let mut actuals = vec![]; + for i in 0..formals.len() { + let formal = &formals[i]; + let actual = &args[i]; + if let Some(vexpr) = formal.resolve_argument(actual) { + actuals.push(vexpr); + } else { + continue 'overload; + } + } + + // the current overload matches the argument arity and types + return Ok(ValueExpr::Call(CallExpr { + name: CallName::ById(name.to_string(), *oid, idx), + arguments: actuals.into_iter().cloned().collect(), + })); + } + Err(CallLookupError::InvalidNumberOfArguments(name.into())) + } +} + +pub(crate) trait FormalArg { + fn resolve_argument<'a>(&self, arg: &'a CallArgument) -> Option<&'a ValueExpr>; +} + +impl FormalArg for CallSpecArg { + fn resolve_argument<'a>(&self, arg: &'a CallArgument) -> Option<&'a ValueExpr> { + match (self, arg) { + (CallSpecArg::Positional, CallArgument::Positional(ve)) => Some(ve), + (CallSpecArg::Named(formal_name), CallArgument::Named(arg_name, ve)) => { + if formal_name == &UniCase::new(arg_name.as_str()) { + Some(ve) + } else { + None + } + } + _ => None, + } + } +} diff --git a/partiql-logical-planner/src/lib.rs b/partiql-logical-planner/src/lib.rs index 18dc8849..a3313dcd 100644 --- a/partiql-logical-planner/src/lib.rs +++ b/partiql-logical-planner/src/lib.rs @@ -8,9 +8,10 @@ use partiql_ast_passes::name_resolver::NameResolver; use partiql_logical as logical; use partiql_parser::Parsed; -use partiql_catalog::{Catalog, PartiqlCatalog}; +use partiql_catalog::catalog::{Catalog, PartiqlCatalog}; mod builtins; +mod functions; mod lower; mod typer; @@ -41,8 +42,8 @@ impl<'c> LogicalPlanner<'c> { mod tests { use assert_matches::assert_matches; use partiql_ast_passes::error::AstTransformationError; + use partiql_catalog::catalog::PartiqlCatalog; use partiql_catalog::context::SystemContext; - use partiql_catalog::PartiqlCatalog; use partiql_eval::env::basic::MapBindings; use partiql_eval::eval::BasicContext; diff --git a/partiql-logical-planner/src/lower.rs b/partiql-logical-planner/src/lower.rs index b691aad5..912f1199 100644 --- a/partiql-logical-planner/src/lower.rs +++ b/partiql-logical-planner/src/lower.rs @@ -32,8 +32,9 @@ use partiql_catalog::call_defs::{CallArgument, CallDef}; use partiql_ast_passes::error::{AstTransformError, AstTransformationError}; +use crate::functions::Function; use partiql_ast_passes::name_resolver::NameRef; -use partiql_catalog::Catalog; +use partiql_catalog::catalog::Catalog; use partiql_common::node::NodeId; use partiql_extension_ion::decode::{IonDecoderBuilder, IonDecoderConfig}; use partiql_extension_ion::Encoding; @@ -1117,8 +1118,7 @@ impl<'a, 'ast> Visitor<'ast> for AstToLogical<'a> { let args = self.exit_call(); let name = call.func_name.value.to_lowercase(); - let call_def_to_vexpr = - |call_def: &CallDef| call_def.lookup(&args, &name).map_err(Into::into); + let call_def_to_vexpr = |call_def: &CallDef| call_def.lookup(&args, &name); let call_expr = self .fnsym_tab @@ -1127,8 +1127,9 @@ impl<'a, 'ast> Visitor<'ast> for AstToLogical<'a> { .or_else(|| { self.catalog .get_function(&name) - .map(|e| call_def_to_vexpr(e.call_def())) + .map(|e| e.resolve(&name, &args)) }) + .map(|res| res.map_err(Into::into)) .unwrap_or_else(|| Err(AstTransformError::UnsupportedFunction(name.clone()))); let expr = match call_expr { @@ -2004,7 +2005,7 @@ fn parse_embedded_ion_str(contents: &str) -> Result { mod tests { use super::*; use crate::LogicalPlanner; - use partiql_catalog::{PartiqlCatalog, TypeEnvEntry}; + use partiql_catalog::catalog::{PartiqlCatalog, TypeEnvEntry}; use partiql_logical::BindingsOp::Project; use partiql_logical::ValueExpr; use partiql_types::type_dynamic; diff --git a/partiql-logical-planner/src/typer.rs b/partiql-logical-planner/src/typer.rs index f42b8851..84fc38ff 100644 --- a/partiql-logical-planner/src/typer.rs +++ b/partiql-logical-planner/src/typer.rs @@ -1,7 +1,7 @@ use crate::typer::LookupOrder::{GlobalLocal, LocalGlobal}; use indexmap::{IndexMap, IndexSet}; use partiql_ast::ast::{CaseSensitivity, SymbolPrimitive}; -use partiql_catalog::Catalog; +use partiql_catalog::catalog::Catalog; use partiql_logical::{BindingsOp, LogicalPlan, OpId, PathComponent, ValueExpr, VarRefType}; use partiql_types::{ type_array, type_bag, type_bool, type_decimal, type_dynamic, type_int, type_string, @@ -603,7 +603,7 @@ mod tests { use super::*; use crate::{logical, LogicalPlanner}; use partiql_ast_passes::error::AstTransformationError; - use partiql_catalog::{PartiqlCatalog, TypeEnvEntry}; + use partiql_catalog::catalog::{PartiqlCatalog, TypeEnvEntry}; use partiql_parser::{Parsed, Parser}; use partiql_types::{ struct_fields, type_bag, type_int_with_const_id, type_string_with_const_id, diff --git a/partiql-logical/Cargo.toml b/partiql-logical/Cargo.toml index 52ef3eb6..b08db8b6 100644 --- a/partiql-logical/Cargo.toml +++ b/partiql-logical/Cargo.toml @@ -22,6 +22,7 @@ bench = false [dependencies] partiql-value = { path = "../partiql-value", version = "0.10.*" } +partiql-common = { path = "../partiql-common", version = "0.10.*" } ordered-float = "4" itertools = "0.13" unicase = "2.7" diff --git a/partiql-logical/src/lib.rs b/partiql-logical/src/lib.rs index 32d2b91c..675f7757 100644 --- a/partiql-logical/src/lib.rs +++ b/partiql-logical/src/lib.rs @@ -55,6 +55,8 @@ use partiql_value::{BindingsName, Value}; use std::collections::HashMap; use std::fmt::{Debug, Display, Formatter}; +use partiql_common::catalog::ObjectId; + #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -703,6 +705,7 @@ pub enum CallName { CollAny(SetQuantifier), CollEvery(SetQuantifier), ByName(String), + ById(String, ObjectId, usize), } /// Indicates if a set should be reduced to its distinct elements or not. diff --git a/partiql-value/src/tuple.rs b/partiql-value/src/tuple.rs index f592e3c7..ef038cda 100644 --- a/partiql-value/src/tuple.rs +++ b/partiql-value/src/tuple.rs @@ -58,8 +58,8 @@ impl Tuple { other .pairs() .chain(self.pairs()) - .map(|(a, v)| (a, v.clone())) .unique_by(|(a, _)| *a) + .map(|(a, v)| (a, v.clone())) .collect() } @@ -190,6 +190,18 @@ where } } +impl Extend<(S, T)> for Tuple +where + S: AsRef, + T: Into, +{ + fn extend>(&mut self, iter: I) { + for (k, v) in iter { + self.insert(k.as_ref(), v.into()); + } + } +} + impl Iterator for Tuple { type Item = (String, Value); diff --git a/partiql/Cargo.toml b/partiql/Cargo.toml index 86077d04..fa411b3a 100644 --- a/partiql/Cargo.toml +++ b/partiql/Cargo.toml @@ -9,10 +9,10 @@ readme = "../README.md" keywords = ["sql", "parser", "query", "compilers", "interpreters"] categories = ["database", "compilers", "parser-implementations"] exclude = [ - "**/.git/**", - "**/.github/**", - "**/.travis.yml", - "**/.appveyor.yml", + "**/.git/**", + "**/.github/**", + "**/.travis.yml", + "**/.appveyor.yml", ] version.workspace = true edition.workspace = true @@ -26,11 +26,12 @@ bench = false partiql-parser = { path = "../partiql-parser" } partiql-ast = { path = "../partiql-ast" } partiql-ast-passes = { path = "../partiql-ast-passes" } -partiql-catalog = { path = "../partiql-catalog"} +partiql-catalog = { path = "../partiql-catalog" } partiql-value = { path = "../partiql-value" } partiql-logical = { path = "../partiql-logical" } partiql-logical-planner = { path = "../partiql-logical-planner" } partiql-eval = { path = "../partiql-eval" } +partiql-extension-value-functions = { path = "../extension/partiql-extension-value-functions" } insta = "1.40.0" diff --git a/partiql/benches/bench_agg.rs b/partiql/benches/bench_agg.rs index d28b4c62..abe88f95 100644 --- a/partiql/benches/bench_agg.rs +++ b/partiql/benches/bench_agg.rs @@ -2,9 +2,8 @@ use std::time::Duration; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use itertools::Itertools; +use partiql_catalog::catalog::{Catalog, PartiqlCatalog}; use partiql_catalog::context::SystemContext; -use partiql_catalog::{Catalog, PartiqlCatalog}; - use partiql_eval::env::basic::MapBindings; use partiql_eval::eval::{BasicContext, EvalPlan}; use partiql_eval::plan::{EvaluationMode, EvaluatorPlanner}; diff --git a/partiql/benches/bench_eval_multi_like.rs b/partiql/benches/bench_eval_multi_like.rs index 8de96226..cac40f0f 100644 --- a/partiql/benches/bench_eval_multi_like.rs +++ b/partiql/benches/bench_eval_multi_like.rs @@ -2,15 +2,14 @@ use std::time::Duration; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use itertools::Itertools; +use partiql_catalog::catalog::{Catalog, PartiqlCatalog}; use partiql_catalog::context::SystemContext; -use partiql_catalog::{Catalog, PartiqlCatalog}; -use rand::{Rng, SeedableRng}; - use partiql_eval::env::basic::MapBindings; use partiql_eval::eval::{BasicContext, EvalPlan}; use partiql_eval::plan::{EvaluationMode, EvaluatorPlanner}; use partiql_logical::{BindingsOp, LogicalPlan}; use partiql_logical_planner::LogicalPlanner; +use rand::{Rng, SeedableRng}; use partiql_parser::{Parser, ParserResult}; use partiql_value::{tuple, Bag, DateTime, Value}; diff --git a/partiql/src/lib.rs b/partiql/src/lib.rs index a842a25f..609cd354 100644 --- a/partiql/src/lib.rs +++ b/partiql/src/lib.rs @@ -6,8 +6,8 @@ mod subquery_tests; #[cfg(test)] mod tests { use partiql_ast_passes::error::AstTransformationError; + use partiql_catalog::catalog::{Catalog, PartiqlCatalog}; use partiql_catalog::context::SystemContext; - use partiql_catalog::{Catalog, PartiqlCatalog}; use partiql_eval as eval; use partiql_eval::env::basic::MapBindings; use partiql_eval::error::{EvalErr, PlanErr}; diff --git a/partiql/src/subquery_tests.rs b/partiql/src/subquery_tests.rs index db456489..e8d167f3 100644 --- a/partiql/src/subquery_tests.rs +++ b/partiql/src/subquery_tests.rs @@ -3,8 +3,8 @@ #[cfg(test)] mod tests { + use partiql_catalog::catalog::{Catalog, PartiqlCatalog}; use partiql_catalog::context::SystemContext; - use partiql_catalog::{Catalog, PartiqlCatalog}; use partiql_eval::env::basic::MapBindings; use partiql_eval::error::EvalErr; use partiql_eval::eval::BasicContext; diff --git a/partiql/tests/extension_error.rs b/partiql/tests/extension_error.rs index 764707c1..3b71f856 100644 --- a/partiql/tests/extension_error.rs +++ b/partiql/tests/extension_error.rs @@ -6,10 +6,11 @@ use std::error::Error; use thiserror::Error; use partiql_catalog::call_defs::{CallDef, CallSpec, CallSpecArg}; +use partiql_catalog::catalog::{Catalog, PartiqlCatalog}; use partiql_catalog::context::{SessionContext, SystemContext}; -use partiql_catalog::{ - BaseTableExpr, BaseTableExprResult, BaseTableExprResultError, BaseTableFunctionInfo, Catalog, - Extension, PartiqlCatalog, TableFunction, +use partiql_catalog::extension::{Extension, ExtensionResultError}; +use partiql_catalog::table_fn::{ + BaseTableExpr, BaseTableExprResult, BaseTableFunctionInfo, TableFunction, }; use partiql_eval::env::basic::MapBindings; use partiql_eval::error::{EvalErr, EvaluationError}; @@ -23,7 +24,7 @@ use partiql_logical as logical; #[derive(Debug)] pub struct UserCtxTestExtension {} -impl partiql_catalog::Extension for UserCtxTestExtension { +impl partiql_catalog::extension::Extension for UserCtxTestExtension { fn name(&self) -> String { "test_extension".into() } @@ -95,12 +96,12 @@ impl BaseTableExpr for EvalTestCtxTable { Value::String(_name) => Ok(Box::new(TestDataGen {})), _ => { let error = UserCtxError::BadArgs; - Err(Box::new(error) as BaseTableExprResultError) + Err(Box::new(error) as ExtensionResultError) } } } else { let error = UserCtxError::BadArgs; - Err(Box::new(error) as BaseTableExprResultError) + Err(Box::new(error) as ExtensionResultError) } } } @@ -108,7 +109,7 @@ impl BaseTableExpr for EvalTestCtxTable { struct TestDataGen {} impl Iterator for TestDataGen { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option { Some(Err(Box::new(UserCtxError::Runtime))) diff --git a/partiql/tests/snapshots/tuple_ops__tupleconcat.snap b/partiql/tests/snapshots/tuple_ops__tupleconcat.snap new file mode 100644 index 00000000..dec3414e --- /dev/null +++ b/partiql/tests/snapshots/tuple_ops__tupleconcat.snap @@ -0,0 +1,5 @@ +--- +source: partiql/tests/tuple_ops.rs +expression: tuple +--- +{ 'sally': 4, 'bob': 1 } diff --git a/partiql/tests/snapshots/tuple_ops__tuplemerge.snap b/partiql/tests/snapshots/tuple_ops__tuplemerge.snap new file mode 100644 index 00000000..dec3414e --- /dev/null +++ b/partiql/tests/snapshots/tuple_ops__tuplemerge.snap @@ -0,0 +1,5 @@ +--- +source: partiql/tests/tuple_ops.rs +expression: tuple +--- +{ 'sally': 4, 'bob': 1 } diff --git a/partiql/tests/snapshots/tuple_ops__tupleunion.snap b/partiql/tests/snapshots/tuple_ops__tupleunion.snap new file mode 100644 index 00000000..82540561 --- /dev/null +++ b/partiql/tests/snapshots/tuple_ops__tupleunion.snap @@ -0,0 +1,5 @@ +--- +source: partiql/tests/tuple_ops.rs +expression: tuple +--- +{ 'bob': 1, 'sally': 'error', 'sally': 1, 'sally': 2, 'sally': 3, 'sally': 4 } diff --git a/partiql/tests/tuple_ops.rs b/partiql/tests/tuple_ops.rs new file mode 100644 index 00000000..37112401 --- /dev/null +++ b/partiql/tests/tuple_ops.rs @@ -0,0 +1,141 @@ +use assert_matches::assert_matches; +use partiql_ast_passes::error::AstTransformationError; +use partiql_catalog::catalog::{Catalog, PartiqlCatalog}; +use partiql_catalog::context::SystemContext; +use partiql_catalog::extension::Extension; +use partiql_eval as eval; +use partiql_eval::env::basic::MapBindings; +use partiql_eval::error::{EvalErr, PlanErr}; +use partiql_eval::eval::{BasicContext, EvalPlan, EvalResult, Evaluated}; +use partiql_eval::plan::EvaluationMode; +use partiql_extension_value_functions::PartiqlValueFnExtension; +use partiql_logical as logical; +use partiql_parser::{Parsed, ParserError, ParserResult}; +use partiql_value::{DateTime, Value}; +use std::error::Error; +use thiserror::Error; + +#[derive(Error, Debug)] +enum TestError<'a> { + #[error("Parse error: {0:?}")] + Parse(ParserError<'a>), + #[error("Lower error: {0:?}")] + Lower(AstTransformationError), + #[error("Plan error: {0:?}")] + Plan(PlanErr), + #[error("Evaluation error: {0:?}")] + Eval(EvalErr), + #[error("Other: {0:?}")] + Other(Box), +} + +impl<'a> From> for TestError<'a> { + fn from(err: ParserError<'a>) -> Self { + TestError::Parse(err) + } +} + +impl From for TestError<'_> { + fn from(err: AstTransformationError) -> Self { + TestError::Lower(err) + } +} + +impl From for TestError<'_> { + fn from(err: PlanErr) -> Self { + TestError::Plan(err) + } +} + +impl From for TestError<'_> { + fn from(err: EvalErr) -> Self { + TestError::Eval(err) + } +} + +impl From> for TestError<'_> { + fn from(err: Box) -> Self { + TestError::Other(err) + } +} + +#[track_caller] +#[inline] +fn parse(statement: &str) -> ParserResult<'_> { + partiql_parser::Parser::default().parse(statement) +} + +#[track_caller] +#[inline] +fn lower( + catalog: &dyn Catalog, + parsed: &Parsed<'_>, +) -> Result, AstTransformationError> { + let planner = partiql_logical_planner::LogicalPlanner::new(catalog); + planner.lower(parsed) +} + +#[track_caller] +#[inline] +fn compile( + mode: EvaluationMode, + catalog: &dyn Catalog, + logical: logical::LogicalPlan, +) -> Result { + let mut planner = eval::plan::EvaluatorPlanner::new(mode, catalog); + planner.compile(&logical) +} + +#[track_caller] +#[inline] +fn evaluate(mut plan: EvalPlan, bindings: MapBindings) -> EvalResult { + let sys = SystemContext { + now: DateTime::from_system_now_utc(), + }; + let ctx = BasicContext::new(bindings, sys); + plan.execute_mut(&ctx) +} + +#[track_caller] +#[inline] +fn eval(statement: &str, mode: EvaluationMode) -> Result> { + let mut catalog = PartiqlCatalog::default(); + let ext = PartiqlValueFnExtension::default(); + ext.load(&mut catalog)?; + + let parsed = parse(statement)?; + let lowered = lower(&catalog, &parsed)?; + let bindings = Default::default(); + let plan = compile(mode, &catalog, lowered)?; + Ok(evaluate(plan, bindings)?) +} + +#[test] +fn tupleunion() { + let query = "tupleunion({ 'bob': 1, 'sally': 'error' }, { 'sally': 1 }, { 'sally': 2 }, { 'sally': 3 }, { 'sally': 4 })"; + + let res = eval(query, EvaluationMode::Permissive); + assert_matches!(res, Ok(_)); + + let res = res.unwrap().result; + assert_matches!(res, Value::Tuple(_)); + let tuple = res.as_tuple_ref(); + assert_eq!(tuple.len(), 6); + + insta::assert_debug_snapshot!(tuple); +} + +#[test] +fn tupleconcat() { + let query = "tupleconcat({ 'bob': 1, 'sally': 'error' }, { 'sally': 1 }, { 'sally': 2 }, { 'sally': 3 }, { 'sally': 4 })"; + + let res = eval(query, EvaluationMode::Permissive); + assert_matches!(res, Ok(_)); + + let res = res.unwrap().result; + assert_matches!(res, Value::Tuple(_)); + let tuple = res.as_tuple_ref(); + assert_eq!(tuple.len(), 2); + + insta::assert_debug_snapshot!(tuple); +} diff --git a/partiql/tests/user_context.rs b/partiql/tests/user_context.rs index 5ecead28..079acfb4 100644 --- a/partiql/tests/user_context.rs +++ b/partiql/tests/user_context.rs @@ -7,10 +7,11 @@ use std::error::Error; use thiserror::Error; use partiql_catalog::call_defs::{CallDef, CallSpec, CallSpecArg}; +use partiql_catalog::catalog::{Catalog, PartiqlCatalog}; use partiql_catalog::context::{SessionContext, SystemContext}; -use partiql_catalog::{ - BaseTableExpr, BaseTableExprResult, BaseTableExprResultError, BaseTableFunctionInfo, Catalog, - Extension, PartiqlCatalog, TableFunction, +use partiql_catalog::extension::{Extension, ExtensionResultError}; +use partiql_catalog::table_fn::{ + BaseTableExpr, BaseTableExprResult, BaseTableFunctionInfo, TableFunction, }; use partiql_eval::env::basic::MapBindings; use partiql_eval::eval::BasicContext; @@ -23,7 +24,7 @@ use partiql_logical as logical; #[derive(Debug)] pub struct UserCtxTestExtension {} -impl partiql_catalog::Extension for UserCtxTestExtension { +impl partiql_catalog::extension::Extension for UserCtxTestExtension { fn name(&self) -> String { "test_extension".into() } @@ -93,12 +94,12 @@ impl BaseTableExpr for EvalTestCtxTable { Value::String(name) => generated_data(name.to_string(), ctx), _ => { let error = UserCtxError::Unknown; - Err(Box::new(error) as BaseTableExprResultError) + Err(Box::new(error) as ExtensionResultError) } } } else { let error = UserCtxError::Unknown; - Err(Box::new(error) as BaseTableExprResultError) + Err(Box::new(error) as ExtensionResultError) } } } @@ -109,7 +110,7 @@ struct TestDataGen<'a> { } impl<'a> Iterator for TestDataGen<'a> { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option { if let Some(cv) = self.ctx.user_context(&self.name) {