diff --git a/lib/partiql-transpiler/src/main/kotlin/org/partiql/transpiler/sql/RexToSql.kt b/lib/partiql-transpiler/src/main/kotlin/org/partiql/transpiler/sql/RexToSql.kt index ab8c8859d3..59261b6b3d 100644 --- a/lib/partiql-transpiler/src/main/kotlin/org/partiql/transpiler/sql/RexToSql.kt +++ b/lib/partiql-transpiler/src/main/kotlin/org/partiql/transpiler/sql/RexToSql.kt @@ -3,6 +3,8 @@ package org.partiql.transpiler.sql import org.partiql.ast.Ast import org.partiql.ast.Expr import org.partiql.ast.Identifier +import org.partiql.ast.Select +import org.partiql.ast.SetQuantifier import org.partiql.plan.Fn import org.partiql.plan.PlanNode import org.partiql.plan.Rel @@ -11,6 +13,7 @@ import org.partiql.plan.visitor.PlanBaseVisitor import org.partiql.transpiler.TranspilerProblem import org.partiql.transpiler.sql.SqlTransform.Companion.translate import org.partiql.types.StaticType +import org.partiql.types.StructType import org.partiql.value.PartiQLValueExperimental import org.partiql.value.StringValue @@ -77,9 +80,7 @@ public open class RexToSql( override fun visitRexOpGlobal(node: Rex.Op.Global, ctx: StaticType): Expr { val global = transform.getGlobal(node.ref) - if (global == null) { - error("Malformed plan, resolved global (\$global ${node.ref}) does not exist") - } + ?: error("Malformed plan, resolved global (\$global ${node.ref}) does not exist") val identifier = global val scope = Expr.Var.Scope.DEFAULT return Ast.exprVar(identifier, scope) @@ -91,6 +92,23 @@ public open class RexToSql( return Ast.exprPath(root, steps) } + // TODO: Fix this. + @OptIn(PartiQLValueExperimental::class) + override fun visitRexOpTupleUnion(node: Rex.Op.TupleUnion, ctx: StaticType): Expr { + return Ast.create { + val args = node.args.map { arg -> + when (arg) { + is Rex.Op.TupleUnion.Arg.Struct -> visitRex(arg.v, ctx) + is Rex.Op.TupleUnion.Arg.Spread -> visitRex(arg.v, ctx) + } + } + exprCall( + identifierSymbol("TUPLEUNION", Identifier.CaseSensitivity.INSENSITIVE), + args = args + ) + } + } + private fun visitRexOpPathStep(node: Rex.Op.Path.Step): Expr.Path.Step = when (node) { is Rex.Op.Path.Step.Index -> visitRexOpPathStepIndex(node) is Rex.Op.Path.Step.Unpivot -> Ast.exprPathStepUnpivot() @@ -151,18 +169,86 @@ public open class RexToSql( } override fun visitRexOpSelect(node: Rex.Op.Select, ctx: StaticType): Expr { - // val typeEnv = node.rel.type.schema + val typeEnv = node.rel.type.schema val relToSql = RelToSql(transform) - // val rexToSql = RexToSql(transform, typeEnv) + val rexToSql = RexToSql(transform, typeEnv) val sfw = relToSql.apply(node.rel) assert(sfw.select != null) { "SELECT from RelToSql should never be null" } - if (node.constructor.isDefault(node.rel.type.schema)) { + return if (node.constructor.isDefault(node.rel.type.schema)) { // SELECT - return sfw.build() + sfw.build() } else { - // SELECT VALUE - // TODO rewrite the constructor replacing variable references with the projected expressions - throw UnsupportedOperationException("SELECT VALUE is not supported") + // Attempt to convert TUPLEUNION into SELECT. If unable, go ahead with SELECT VALUE. + val setq = getSetQuantifier(sfw.select!!) + val select = convertTupleUnionToSqlSelect(node.constructor.op, node.rel, setq) + ?: Ast.create { selectValue(rexToSql.apply(node.constructor), setq) } + sfw.select = select + sfw.build() + } + } + + /** + * Grabs the [SetQuantifier] of a [Select]. + */ + private fun getSetQuantifier(select: Select): SetQuantifier? = when (select) { + is Select.Project -> select.setq + is Select.Value -> select.setq + is Select.Star -> select.setq + is Select.Pivot -> null + } + + /** + * Attempts to convert the [op] (TUPLEUNION) into the projections that it holds. For this to occur, all tuples in TUPLEUNION + * **must** be closed content [StructType]s. We assume that the [input] is always a [Rel.Op.Project]. + * + * Example: + * ``` + * SELECT VALUE TUPLEUNION({ 'a': , 'b': }, { 'c': }) FROM t + * -- Gets converted into: + * SELECT a, b, c FROM t + * ``` + * + * If unable to convert into SQL-style projections (due to open content structs, non-struct arguments, etc), we + * return null. + */ + private fun convertTupleUnionToSqlSelect(op: Rex.Op, input: Rel, setq: SetQuantifier?): Select.Project? { + val relProject = input.op as? Rel.Op.Project ?: error("Malformed plan, the top rel should be a project.") + if (op !is Rex.Op.TupleUnion) { return null } + if (op.args.size != input.type.schema.size) { return null } + if (op.args.size != relProject.projections.size) { return null } + val projections = relProject.projections.flatMapIndexed { index, project -> + val newRexToSql = RexToSql(transform, relProject.input.type.schema) + val newExpr = newRexToSql.apply(project) + when (val arg = op.args[index]) { + is Rex.Op.TupleUnion.Arg.Spread -> { + val structType = project.type as? StructType ?: return null + if (structType.contentClosed.not()) { return null } + structType.fields.map { field -> + Ast.create { + selectProjectItemExpression( + expr = exprPath(newExpr, listOf(exprPathStepSymbol(identifierSymbol(field.key, caseSensitivity = Identifier.CaseSensitivity.INSENSITIVE)))), // TODO: Sensitivity + asAlias = identifierSymbol(field.key, Identifier.CaseSensitivity.INSENSITIVE) // TODO: Sensitivity + ) + } + } + } + is Rex.Op.TupleUnion.Arg.Struct -> { + listOf( + Ast.create { + selectProjectItemExpression( + expr = newExpr, + asAlias = identifierSymbol(arg.k, Identifier.CaseSensitivity.INSENSITIVE) // TODO: Sensitivity + ) + } + ) + } + } + } + return Ast.create { + selectProject( + items = projections, + setq = setq + ) } } @@ -179,7 +265,7 @@ public open class RexToSql( /** * Returns true iff this [Rex] is the default constructor for [schema] derived from an SQL SELECT. * - * See [RelConverter.defaultConstructor] to see how the default constructor is created. + * See [org.partiql.planner.transforms.RelConverter.defaultConstructor] to see how the default constructor is created. */ @OptIn(PartiQLValueExperimental::class) private fun Rex.isDefault(schema: List): Boolean { diff --git a/lib/partiql-transpiler/src/test/resources/cases/tests_00.ion b/lib/partiql-transpiler/src/test/resources/cases/tests_00.ion index f9dccd9aab..7f0e621f86 100644 --- a/lib/partiql-transpiler/src/test/resources/cases/tests_00.ion +++ b/lib/partiql-transpiler/src/test/resources/cases/tests_00.ion @@ -159,20 +159,347 @@ suite::{ }, '0004': { statement: ''' - SELECT s_store_sk FROM + SELECT * FROM tpc_ds.store AS store LEFT JOIN tpc_ds.store_returns AS returns ON s_store_sk = sr_store_sk ''', schema: { - type: "bag", - items: { - type: "struct", - fields: [ + type:"bag", + items:{ + type:"struct", + fields:[ + { + name:"s_store_sk", + type:"string" + }, + { + name:"s_store_id", + type:"string" + }, + { + name:"s_rec_start_date", + type:[ + "date", + "null" + ] + }, + { + name:"s_rec_end_date", + type:[ + "date", + "null" + ] + }, + { + name:"s_closed_date_sk", + type:[ + "null", + "string" + ] + }, + { + name:"s_store_name", + type:[ + "null", + "string" + ] + }, + { + name:"s_number_employees", + type:[ + "int32", + "null" + ] + }, + { + name:"s_floor_space", + type:[ + "int32", + "null" + ] + }, + { + name:"s_hours", + type:[ + "null", + "string" + ] + }, + { + name:"s_manager", + type:[ + "null", + "string" + ] + }, + { + name:"s_market_id", + type:[ + "int32", + "null" + ] + }, + { + name:"s_geography_class", + type:[ + "null", + "string" + ] + }, + { + name:"s_market_desc", + type:[ + "null", + "string" + ] + }, + { + name:"s_market_manager", + type:[ + "null", + "string" + ] + }, + { + name:"s_division_id", + type:[ + "int32", + "null" + ] + }, + { + name:"s_division_name", + type:[ + "null", + "string" + ] + }, + { + name:"s_company_id", + type:[ + "int32", + "null" + ] + }, + { + name:"s_company_name", + type:[ + "null", + "string" + ] + }, + { + name:"s_street_number", + type:[ + "null", + "string" + ] + }, + { + name:"s_street_name", + type:[ + "null", + "string" + ] + }, + { + name:"s_street_type", + type:[ + "null", + "string" + ] + }, + { + name:"s_suite_number", + type:[ + "null", + "string" + ] + }, + { + name:"s_city", + type:[ + "null", + "string" + ] + }, + { + name:"s_county", + type:[ + "null", + "string" + ] + }, + { + name:"s_state", + type:[ + "null", + "string" + ] + }, + { + name:"s_zip", + type:[ + "null", + "string" + ] + }, { - name: "s_store_sk", - type: "string" + name:"s_country", + type:[ + "null", + "string" + ] + }, + { + name:"s_gmt_offset", + type:[ + "float64", + "null" + ] + }, + { + name:"s_tax_precentage", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_returned_date_sk", + type:[ + "null", + "string" + ] + }, + { + name:"sr_return_time_sk", + type:[ + "null", + "string" + ] + }, + { + name:"sr_item_sk", + type:"string" + }, + { + name:"sr_customer_sk", + type:[ + "null", + "string" + ] + }, + { + name:"sr_cdemo_sk", + type:[ + "null", + "string" + ] + }, + { + name:"sr_hdemo_sk", + type:[ + "null", + "string" + ] + }, + { + name:"sr_addr_sk", + type:[ + "null", + "string" + ] + }, + { + name:"sr_store_sk", + type:[ + "null", + "string" + ] + }, + { + name:"sr_reason_sk", + type:[ + "null", + "string" + ] + }, + { + name:"sr_ticket_number", + type:"string" + }, + { + name:"sr_return_quantity", + type:[ + "int32", + "null" + ] + }, + { + name:"sr_return_amt", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_return_tax", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_return_amt_inc_tax", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_fee", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_return_ship_cost", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_refunded_cash", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_reversed_charge", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_store_credit", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_net_loss", + type:[ + "float64", + "null" + ] } ] } @@ -199,5 +526,245 @@ suite::{ }, }, }, + '0006': { + statement: ''' + SELECT item.i_rec.*, item.*, item.i_rec.i_rec_start_date FROM pql.item AS item + ''', + schema: { + type:"bag", + items:{ + type:"struct", + fields:[ + { + name:"i_rec_start_date", + type:[ + "int64", + "null" + ] + }, + { + name:"i_rec_end_date", + type:[ + "int64", + "null" + ] + }, + { + name:"i_item_sk", + type:[ + "null", + "string" + ] + }, + { + name:"i_item_id", + type:[ + "null", + "string" + ] + }, + { + name:"i_rec", + type:{ + type:"struct", + fields:[ + { + name:"i_rec_start_date", + type:[ + "int64", + "null" + ] + }, + { + name:"i_rec_end_date", + type:[ + "int64", + "null" + ] + } + ] + } + }, + { + name:"i_item_desc", + type:[ + "null", + "string" + ] + }, + { + name:"pricing", + type:{ + type:"struct", + fields:[ + { + name:"i_current_price", + type:[ + "float64", + "null" + ] + }, + { + name:"i_wholesale_cost", + type:[ + "float64", + "null" + ] + } + ] + } + }, + { + name:"i_brand_id", + type:[ + "int32", + "null" + ] + }, + { + name:"i_brand", + type:[ + "null", + "string" + ] + }, + { + name:"i_class_id", + type:[ + "int32", + "null" + ] + }, + { + name:"i_class", + type:[ + "null", + "string" + ] + }, + { + name:"i_category_id", + type:[ + "int32", + "null" + ] + }, + { + name:"i_category", + type:[ + "null", + "string" + ] + }, + { + name:"i_manufact_id", + type:[ + "int32", + "null" + ] + }, + { + name:"i_manufact", + type:[ + "null", + "string" + ] + }, + { + name:"i_size", + type:[ + "null", + "string" + ] + }, + { + name:"i_formulation", + type:[ + "null", + "string" + ] + }, + { + name:"i_color", + type:[ + "null", + "string" + ] + }, + { + name:"i_units", + type:[ + "null", + "string" + ] + }, + { + name:"i_container", + type:[ + "null", + "string" + ] + }, + { + name:"manager_info", + type:{ + type:"struct", + fields:[ + { + name:"manager_id", + type:[ + "int32" + ] + }, + { + name:"manager_name", + type:[ + "null", + "string" + ] + }, + { + name:"manager_address", + type:[ + "null", + { + type:"struct", + fields:[ + { + name:"zip_code", + type:"int32" + }, + { + name:"house_number", + type:[ + "int32", + "null" + ] + } + ] + } + ] + } + ] + } + }, + { + name:"i_product_name", + type:[ + "null", + "string" + ] + }, + { + name:"i_rec_start_date", + type:[ + "int64", + "null" + ] + } + ] + } + } + }, } } diff --git a/lib/partiql-transpiler/src/test/resources/targets/partiql/partiql_results_00.ion b/lib/partiql-transpiler/src/test/resources/targets/partiql/partiql_results_00.ion index 731e84fedc..4019eaf5ae 100644 --- a/lib/partiql-transpiler/src/test/resources/targets/partiql/partiql_results_00.ion +++ b/lib/partiql-transpiler/src/test/resources/targets/partiql/partiql_results_00.ion @@ -55,13 +55,28 @@ target::{ }, '0004': { statement: ''' - SELECT - store.s_store_sk AS s_store_sk - FROM - tpc_ds."store" AS store - LEFT OUTER JOIN - tpc_ds."store_returns" AS returns - ON store.s_store_sk = returns.sr_store_sk + SELECT store.s_store_sk AS s_store_sk, store.s_store_id AS s_store_id, store.s_rec_start_date AS s_rec_start_date, + store.s_rec_end_date AS s_rec_end_date, store.s_closed_date_sk AS s_closed_date_sk, store.s_store_name AS s_store_name, + store.s_number_employees AS s_number_employees, store.s_floor_space AS s_floor_space, store.s_hours AS s_hours, + store.s_manager AS s_manager, store.s_market_id AS s_market_id, store.s_geography_class AS s_geography_class, + store.s_market_desc AS s_market_desc, store.s_market_manager AS s_market_manager, store.s_division_id AS s_division_id, + store.s_division_name AS s_division_name, store.s_company_id AS s_company_id, store.s_company_name AS s_company_name, + store.s_street_number AS s_street_number, store.s_street_name AS s_street_name, store.s_street_type AS s_street_type, + store.s_suite_number AS s_suite_number, store.s_city AS s_city, store.s_county AS s_county, store.s_state AS s_state, + store.s_zip AS s_zip, store.s_country AS s_country, store.s_gmt_offset AS s_gmt_offset, + store.s_tax_precentage AS s_tax_precentage, returns.sr_returned_date_sk AS sr_returned_date_sk, + returns.sr_return_time_sk AS sr_return_time_sk, returns.sr_item_sk AS sr_item_sk, + returns.sr_customer_sk AS sr_customer_sk, returns.sr_cdemo_sk AS sr_cdemo_sk, returns.sr_hdemo_sk AS sr_hdemo_sk, + returns.sr_addr_sk AS sr_addr_sk, returns.sr_store_sk AS sr_store_sk, returns.sr_reason_sk AS sr_reason_sk, + returns.sr_ticket_number AS sr_ticket_number, returns.sr_return_quantity AS sr_return_quantity, + returns.sr_return_amt AS sr_return_amt, returns.sr_return_tax AS sr_return_tax, + returns.sr_return_amt_inc_tax AS sr_return_amt_inc_tax, returns.sr_fee AS sr_fee, + returns.sr_return_ship_cost AS sr_return_ship_cost, returns.sr_refunded_cash AS sr_refunded_cash, + returns.sr_reversed_charge AS sr_reversed_charge, returns.sr_store_credit AS sr_store_credit, + returns.sr_net_loss AS sr_net_loss + FROM tpc_ds."store" AS store + LEFT OUTER JOIN tpc_ds."store_returns" AS returns + ON store.s_store_sk = returns.sr_store_sk ''' }, '0005': { @@ -69,5 +84,17 @@ target::{ SELECT CURRENT_USER AS CURRENT_USER, CURRENT_DATE AS CURRENT_DATE FROM store_sales AS store_sales ''', }, + '0006': { + statement: ''' + SELECT item.i_rec.i_rec_start_date AS i_rec_start_date, item.i_rec.i_rec_end_date AS i_rec_end_date, + item.i_item_sk AS i_item_sk, item.i_item_id AS i_item_id, item.i_rec AS i_rec, item.i_item_desc AS i_item_desc, + item.pricing AS pricing, item.i_brand_id AS i_brand_id, item.i_brand AS i_brand, item.i_class_id AS i_class_id, + item.i_class AS i_class, item.i_category_id AS i_category_id, item.i_category AS i_category, + item.i_manufact_id AS i_manufact_id, item.i_manufact AS i_manufact, item.i_size AS i_size, + item.i_formulation AS i_formulation, item.i_color AS i_color, item.i_units AS i_units, + item.i_container AS i_container, item.manager_info AS manager_info, item.i_product_name AS i_product_name, + item.i_rec.i_rec_start_date AS i_rec_start_date FROM pql."item" AS item + ''', + }, }, } diff --git a/lib/partiql-transpiler/src/test/resources/targets/redshift/redshift_results_00.ion b/lib/partiql-transpiler/src/test/resources/targets/redshift/redshift_results_00.ion index 525af5135c..bac3c8d4ce 100644 --- a/lib/partiql-transpiler/src/test/resources/targets/redshift/redshift_results_00.ion +++ b/lib/partiql-transpiler/src/test/resources/targets/redshift/redshift_results_00.ion @@ -56,13 +56,28 @@ target::{ }, '0004': { statement: ''' - SELECT - store.s_store_sk AS s_store_sk - FROM - tpc_ds."store" AS store - LEFT OUTER JOIN - tpc_ds."store_returns" AS returns - ON store.s_store_sk = returns.sr_store_sk + SELECT store.s_store_sk AS s_store_sk, store.s_store_id AS s_store_id, store.s_rec_start_date AS s_rec_start_date, + store.s_rec_end_date AS s_rec_end_date, store.s_closed_date_sk AS s_closed_date_sk, store.s_store_name AS s_store_name, + store.s_number_employees AS s_number_employees, store.s_floor_space AS s_floor_space, store.s_hours AS s_hours, + store.s_manager AS s_manager, store.s_market_id AS s_market_id, store.s_geography_class AS s_geography_class, + store.s_market_desc AS s_market_desc, store.s_market_manager AS s_market_manager, store.s_division_id AS s_division_id, + store.s_division_name AS s_division_name, store.s_company_id AS s_company_id, store.s_company_name AS s_company_name, + store.s_street_number AS s_street_number, store.s_street_name AS s_street_name, store.s_street_type AS s_street_type, + store.s_suite_number AS s_suite_number, store.s_city AS s_city, store.s_county AS s_county, store.s_state AS s_state, + store.s_zip AS s_zip, store.s_country AS s_country, store.s_gmt_offset AS s_gmt_offset, + store.s_tax_precentage AS s_tax_precentage, returns.sr_returned_date_sk AS sr_returned_date_sk, + returns.sr_return_time_sk AS sr_return_time_sk, returns.sr_item_sk AS sr_item_sk, + returns.sr_customer_sk AS sr_customer_sk, returns.sr_cdemo_sk AS sr_cdemo_sk, returns.sr_hdemo_sk AS sr_hdemo_sk, + returns.sr_addr_sk AS sr_addr_sk, returns.sr_store_sk AS sr_store_sk, returns.sr_reason_sk AS sr_reason_sk, + returns.sr_ticket_number AS sr_ticket_number, returns.sr_return_quantity AS sr_return_quantity, + returns.sr_return_amt AS sr_return_amt, returns.sr_return_tax AS sr_return_tax, + returns.sr_return_amt_inc_tax AS sr_return_amt_inc_tax, returns.sr_fee AS sr_fee, + returns.sr_return_ship_cost AS sr_return_ship_cost, returns.sr_refunded_cash AS sr_refunded_cash, + returns.sr_reversed_charge AS sr_reversed_charge, returns.sr_store_credit AS sr_store_credit, + returns.sr_net_loss AS sr_net_loss + FROM tpc_ds."store" AS store + LEFT OUTER JOIN tpc_ds."store_returns" AS returns + ON store.s_store_sk = returns.sr_store_sk ''' }, '0005': { @@ -70,5 +85,17 @@ target::{ SELECT CURRENT_USER AS CURRENT_USER, CURRENT_DATE AS CURRENT_DATE FROM store_sales AS store_sales ''', }, + '0006': { + statement: ''' + SELECT item.i_rec.i_rec_start_date AS i_rec_start_date, item.i_rec.i_rec_end_date AS i_rec_end_date, + item.i_item_sk AS i_item_sk, item.i_item_id AS i_item_id, item.i_rec AS i_rec, item.i_item_desc AS i_item_desc, + item.pricing AS pricing, item.i_brand_id AS i_brand_id, item.i_brand AS i_brand, item.i_class_id AS i_class_id, + item.i_class AS i_class, item.i_category_id AS i_category_id, item.i_category AS i_category, + item.i_manufact_id AS i_manufact_id, item.i_manufact AS i_manufact, item.i_size AS i_size, + item.i_formulation AS i_formulation, item.i_color AS i_color, item.i_units AS i_units, + item.i_container AS i_container, item.manager_info AS manager_info, item.i_product_name AS i_product_name, + item.i_rec.i_rec_start_date AS i_rec_start_date FROM pql."item" AS item + ''', + }, }, } diff --git a/lib/partiql-transpiler/src/test/resources/targets/trino/trino_results_00.ion b/lib/partiql-transpiler/src/test/resources/targets/trino/trino_results_00.ion index c55c4265a2..f26e33c254 100644 --- a/lib/partiql-transpiler/src/test/resources/targets/trino/trino_results_00.ion +++ b/lib/partiql-transpiler/src/test/resources/targets/trino/trino_results_00.ion @@ -55,13 +55,28 @@ target::{ }, '0004': { statement: ''' - SELECT - store.s_store_sk AS s_store_sk - FROM - tpc_ds."store" AS store - LEFT OUTER JOIN - tpc_ds."store_returns" AS returns - ON store.s_store_sk = returns.sr_store_sk + SELECT store.s_store_sk AS s_store_sk, store.s_store_id AS s_store_id, store.s_rec_start_date AS s_rec_start_date, + store.s_rec_end_date AS s_rec_end_date, store.s_closed_date_sk AS s_closed_date_sk, store.s_store_name AS s_store_name, + store.s_number_employees AS s_number_employees, store.s_floor_space AS s_floor_space, store.s_hours AS s_hours, + store.s_manager AS s_manager, store.s_market_id AS s_market_id, store.s_geography_class AS s_geography_class, + store.s_market_desc AS s_market_desc, store.s_market_manager AS s_market_manager, store.s_division_id AS s_division_id, + store.s_division_name AS s_division_name, store.s_company_id AS s_company_id, store.s_company_name AS s_company_name, + store.s_street_number AS s_street_number, store.s_street_name AS s_street_name, store.s_street_type AS s_street_type, + store.s_suite_number AS s_suite_number, store.s_city AS s_city, store.s_county AS s_county, store.s_state AS s_state, + store.s_zip AS s_zip, store.s_country AS s_country, store.s_gmt_offset AS s_gmt_offset, + store.s_tax_precentage AS s_tax_precentage, returns.sr_returned_date_sk AS sr_returned_date_sk, + returns.sr_return_time_sk AS sr_return_time_sk, returns.sr_item_sk AS sr_item_sk, + returns.sr_customer_sk AS sr_customer_sk, returns.sr_cdemo_sk AS sr_cdemo_sk, returns.sr_hdemo_sk AS sr_hdemo_sk, + returns.sr_addr_sk AS sr_addr_sk, returns.sr_store_sk AS sr_store_sk, returns.sr_reason_sk AS sr_reason_sk, + returns.sr_ticket_number AS sr_ticket_number, returns.sr_return_quantity AS sr_return_quantity, + returns.sr_return_amt AS sr_return_amt, returns.sr_return_tax AS sr_return_tax, + returns.sr_return_amt_inc_tax AS sr_return_amt_inc_tax, returns.sr_fee AS sr_fee, + returns.sr_return_ship_cost AS sr_return_ship_cost, returns.sr_refunded_cash AS sr_refunded_cash, + returns.sr_reversed_charge AS sr_reversed_charge, returns.sr_store_credit AS sr_store_credit, + returns.sr_net_loss AS sr_net_loss + FROM tpc_ds."store" AS store + LEFT OUTER JOIN tpc_ds."store_returns" AS returns + ON store.s_store_sk = returns.sr_store_sk ''' }, '0005': { @@ -69,5 +84,17 @@ target::{ SELECT CURRENT_USER AS CURRENT_USER, CURRENT_DATE AS CURRENT_DATE FROM store_sales AS store_sales ''', }, + '0006': { + statement: ''' + SELECT item.i_rec.i_rec_start_date AS i_rec_start_date, item.i_rec.i_rec_end_date AS i_rec_end_date, + item.i_item_sk AS i_item_sk, item.i_item_id AS i_item_id, item.i_rec AS i_rec, item.i_item_desc AS i_item_desc, + item.pricing AS pricing, item.i_brand_id AS i_brand_id, item.i_brand AS i_brand, item.i_class_id AS i_class_id, + item.i_class AS i_class, item.i_category_id AS i_category_id, item.i_category AS i_category, + item.i_manufact_id AS i_manufact_id, item.i_manufact AS i_manufact, item.i_size AS i_size, + item.i_formulation AS i_formulation, item.i_color AS i_color, item.i_units AS i_units, + item.i_container AS i_container, item.manager_info AS manager_info, item.i_product_name AS i_product_name, + item.i_rec.i_rec_start_date AS i_rec_start_date FROM pql."item" AS item + ''', + }, }, } diff --git a/partiql-ast/src/main/kotlin/org/partiql/ast/normalize/NormalizeSelectStar.kt b/partiql-ast/src/main/kotlin/org/partiql/ast/normalize/NormalizeSelectStar.kt index 7105e2e382..2a8eb815bb 100644 --- a/partiql-ast/src/main/kotlin/org/partiql/ast/normalize/NormalizeSelectStar.kt +++ b/partiql-ast/src/main/kotlin/org/partiql/ast/normalize/NormalizeSelectStar.kt @@ -1,5 +1,6 @@ package org.partiql.ast.normalize +import org.partiql.ast.AstNode import org.partiql.ast.AstPass import org.partiql.ast.Expr import org.partiql.ast.From @@ -14,7 +15,9 @@ import org.partiql.ast.util.AstRewriter /** * Rewrites * - `SELECT * FROM A AS x, B AS y AT i` -> `SELECT x.* AS _1, y.* as _2, i AS i FROM A AS x, B AS y AT i` - * - TODO GROUP BY + * - `SELECT x.* FROM A AS x` (where x.* is a Select.Project.Item.Star) -> `SELECT x.* FROM A AS x` (where x.* is an unpivot step) + * - TODO: GROUP BY + * - TODO: LET * * Requires [NormalizeFromSource] */ @@ -43,6 +46,16 @@ internal object NormalizeSelectStar : AstPass { sfw.copy(select = sel) } + override fun visitSelectProject(node: Select.Project, ctx: Unit): AstNode = ast { + val items = node.items.mapIndexed { index, item -> + when (item) { + is Select.Project.Item.All -> item.expr.star(index) + is Select.Project.Item.Expression -> item + } + } + node.copy(items = items) + } + // Helpers private fun From.aliases(): List> = when (this) { @@ -65,6 +78,25 @@ internal object NormalizeSelectStar : AstPass { selectProjectItemExpression(expr, alias) } + // a.* -> a.* AS _i + private fun Expr.star(i: Int): Select.Project.Item.Expression { + val thiz = this + return ast { + val expr = when (thiz) { + is Expr.Path -> exprPath { + root = thiz.root + steps = (thiz.steps + mutableListOf(exprPathStepUnpivot())).toMutableList() + } + else -> exprPath { + root = this@star + steps += exprPathStepUnpivot() + } + } + val alias = expr.toBinder(i) + selectProjectItemExpression(expr, alias) + } + } + // t -> t AS t private fun String.simple() = ast { val expr = exprVar(id(this@simple), Expr.Var.Scope.DEFAULT) diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/transforms/RelConverter.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/transforms/RelConverter.kt index 819f2c88b9..a2e20b7130 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/transforms/RelConverter.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/transforms/RelConverter.kt @@ -16,6 +16,8 @@ import org.partiql.plan.Rel import org.partiql.plan.Rex import org.partiql.planner.Env import org.partiql.types.StaticType +import org.partiql.types.StructType +import org.partiql.types.TupleConstraint import org.partiql.value.PartiQLValueExperimental import org.partiql.value.boolValue import org.partiql.value.stringValue @@ -106,13 +108,22 @@ internal object RelConverter { rexOpStructField(k, v) } val op = rexOpStruct(fields) - rex(StaticType.STRUCT, op) + val type = StructType( + fields = fields.mapIndexed { i, it -> StructType.Field(schema[i].name, schema[i].type) }, + contentClosed = true, + constraints = setOf( + TupleConstraint.Open(false) + ) + ) + rex(type, op) } /** * Produces the `TUPLEUNION` constructor defined in Section 6.3.2 `SQL's SELECT *`. * * See https://partiql.org/assets/PartiQL-Specification.pdf#page=28 + * + * We assume that at least one of the [op].projections is a PROJECT ALL. */ private fun tupleUnionConstructor(op: Rel.Op.Project, type: Rel.Type): Pair = with(Plan) { val projections = mutableListOf() @@ -125,7 +136,7 @@ internal object RelConverter { projections.add(item.removeUnpivot()) rexOpTupleUnionArgSpread(k, v) } - else -> { + false -> { projections.add(item) rexOpTupleUnionArgStruct(k, v) } diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/typer/PlanTyper.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/typer/PlanTyper.kt index 2d01f35ed3..ddec64eb62 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/typer/PlanTyper.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/typer/PlanTyper.kt @@ -272,7 +272,8 @@ internal class PlanTyper( */ override fun visitRexOpPath(node: Rex.Op.Path, ctx: StaticType?): Rex = rewrite { // 1. Resolve path prefix - val (root, steps) = when (val rootOp = node.root.op) { + val visitedRoot = visitRex(node.root, ctx) + val (root, steps) = when (val rootOp = visitedRoot.op) { is Rex.Op.Var.Unresolved -> { // Rewrite the root val path = rexPathToBindingPath(rootOp, node.steps) @@ -300,7 +301,7 @@ internal class PlanTyper( // rewrite root rex(type, op) to steps } - else -> node.root to node.steps + else -> visitedRoot to node.steps } // 2. Evaluate remaining path steps val type = steps.fold(root.type) { type, step -> @@ -310,7 +311,11 @@ internal class PlanTyper( is Rex.Op.Path.Step.Wildcard -> error("Wildcard path type inference implemented") } } - rex(type, rexOpPath(root, steps)) + val rex = when (steps.isEmpty()) { + true -> root.op + false -> rexOpPath(root, steps) + } + rex(type, rex) } override fun visitRexOpCall(node: Rex.Op.Call, ctx: StaticType?): Rex = rewrite { diff --git a/partiql-planner/src/testFixtures/kotlin/org/partiql/planner/test/Parsing.kt b/partiql-planner/src/testFixtures/kotlin/org/partiql/planner/test/Parsing.kt index 08d7894ac8..2db05ca7aa 100644 --- a/partiql-planner/src/testFixtures/kotlin/org/partiql/planner/test/Parsing.kt +++ b/partiql-planner/src/testFixtures/kotlin/org/partiql/planner/test/Parsing.kt @@ -28,6 +28,7 @@ import org.partiql.types.StructType import org.partiql.types.SymbolType import org.partiql.types.TimeType import org.partiql.types.TimestampType +import org.partiql.types.TupleConstraint // Use some generated serde eventually @@ -127,7 +128,7 @@ public fun StructElement.toStructType(): StaticType { val type = it.getAngry("type").toStaticType() StructType.Field(name, type) } - return StructType(fields) + return StructType(fields, contentClosed = true, constraints = setOf(TupleConstraint.Open(false))) } public fun StaticType.toIon(): IonElement = when (this) { diff --git a/partiql-planner/src/testFixtures/resources/tests/suite_00.ion b/partiql-planner/src/testFixtures/resources/tests/suite_00.ion index e074635e02..acee8748e2 100644 --- a/partiql-planner/src/testFixtures/resources/tests/suite_00.ion +++ b/partiql-planner/src/testFixtures/resources/tests/suite_00.ion @@ -124,7 +124,7 @@ suite::{ // TODO: Add support for SELECT * so we can assert on the schema '0004': { statement: ''' - SELECT s_store_sk + SELECT * FROM tpc_ds.store AS store LEFT JOIN @@ -135,10 +135,364 @@ suite::{ type: "bag", items: { type: "struct", - fields: [ + fields:[ { name:"s_store_sk", type:"string" + }, + { + name:"s_store_id", + type:"string" + }, + { + name:"s_rec_start_date", + type:[ + "date", + "null" + ] + }, + { + name:"s_rec_end_date", + type:[ + "date", + "null" + ] + }, + { + name:"s_closed_date_sk", + type:[ + "null", + "string" + ] + }, + { + name:"s_store_name", + type:[ + "null", + "string" + ] + }, + { + name:"s_number_employees", + type:[ + "int32", + "null" + ] + }, + { + name:"s_floor_space", + type:[ + "int32", + "null" + ] + }, + { + name:"s_hours", + type:[ + "null", + "string" + ] + }, + { + name:"s_manager", + type:[ + "null", + "string" + ] + }, + { + name:"s_market_id", + type:[ + "int32", + "null" + ] + }, + { + name:"s_geography_class", + type:[ + "null", + "string" + ] + }, + { + name:"s_market_desc", + type:[ + "null", + "string" + ] + }, + { + name:"s_market_manager", + type:[ + "null", + "string" + ] + }, + { + name:"s_division_id", + type:[ + "int32", + "null" + ] + }, + { + name:"s_division_name", + type:[ + "null", + "string" + ] + }, + { + name:"s_company_id", + type:[ + "int32", + "null" + ] + }, + { + name:"s_company_name", + type:[ + "null", + "string" + ] + }, + { + name:"s_street_number", + type:[ + "null", + "string" + ] + }, + { + name:"s_street_name", + type:[ + "null", + "string" + ] + }, + { + name:"s_street_type", + type:[ + "null", + "string" + ] + }, + { + name:"s_suite_number", + type:[ + "null", + "string" + ] + }, + { + name:"s_city", + type:[ + "null", + "string" + ] + }, + { + name:"s_county", + type:[ + "null", + "string" + ] + }, + { + name:"s_state", + type:[ + "null", + "string" + ] + }, + { + name:"s_zip", + type:[ + "null", + "string" + ] + }, + { + name:"s_country", + type:[ + "null", + "string" + ] + }, + { + name:"s_gmt_offset", + type:[ + "float64", + "null" + ] + }, + { + name:"s_tax_precentage", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_returned_date_sk", + type:[ + "null", + "string" + ] + }, + { + name:"sr_return_time_sk", + type:[ + "null", + "string" + ] + }, + { + name:"sr_item_sk", + type:"string" + }, + { + name:"sr_customer_sk", + type:[ + "null", + "string" + ] + }, + { + name:"sr_cdemo_sk", + type:[ + "null", + "string" + ] + }, + { + name:"sr_hdemo_sk", + type:[ + "null", + "string" + ] + }, + { + name:"sr_addr_sk", + type:[ + "null", + "string" + ] + }, + { + name:"sr_store_sk", + type:[ + "null", + "string" + ] + }, + { + name:"sr_reason_sk", + type:[ + "null", + "string" + ] + }, + { + name:"sr_ticket_number", + type:"string" + }, + { + name:"sr_return_quantity", + type:[ + "int32", + "null" + ] + }, + { + name:"sr_return_amt", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_return_tax", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_return_amt_inc_tax", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_fee", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_return_ship_cost", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_refunded_cash", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_reversed_charge", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_store_credit", + type:[ + "float64", + "null" + ] + }, + { + name:"sr_net_loss", + type:[ + "float64", + "null" + ] + } + ] + } + } + }, + '0005': { + statement: ''' + SELECT item.i_rec.*, item.i_rec.i_rec_start_date FROM pql.item AS item + ''', + schema: { + type: "bag", + items: { + type: "struct", + fields:[ + { + name:"i_rec_start_date", + type:[ + "int64", + "null" + ] + }, + { + name:"i_rec_end_date", + type:[ + "int64", + "null" + ] } ] } diff --git a/test/sprout-tests/src/test/kotlin/org/partiql/sprout/tests/example/ToStringTests.kt b/test/sprout-tests/src/test/kotlin/org/partiql/sprout/tests/example/ToStringTests.kt new file mode 100644 index 0000000000..7d9f93bcf6 --- /dev/null +++ b/test/sprout-tests/src/test/kotlin/org/partiql/sprout/tests/example/ToStringTests.kt @@ -0,0 +1,79 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at: + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ + +package org.partiql.sprout.tests.example + +import com.amazon.ionelement.api.createIonElementLoader +import org.junit.jupiter.api.Test +import org.partiql.sprout.tests.example.builder.ExampleFactoryImpl +import kotlin.test.assertEquals + +/** + * While toString isn't a contract, here are some tests for making sure at least some things work. + * + * Notably, the following definitely won't get properly converted to Ion: + * - Maps + * - Imported Types + * - Escape Characters + */ +class ToStringTests { + private val factory = ExampleFactoryImpl() + private val loader = createIonElementLoader() + + @Test + fun simpleProductAndEnum() { + val product = factory.identifierSymbol( + symbol = "helloworld!", + caseSensitivity = Identifier.CaseSensitivity.SENSITIVE + ) + val expected = loader.loadSingleElement("IdentifierSymbol::{ symbol: \"helloworld!\", caseSensitivity: IdentifierCaseSensitivity::SENSITIVE }") + val actual = loader.loadSingleElement(product.toString()) + assertEquals(expected, actual) + } + + @Test + fun emptyProduct() { + val product = factory.exprEmpty() + val expected = loader.loadSingleElement("ExprEmpty::{ }") + val actual = loader.loadSingleElement(product.toString()) + assertEquals(expected, actual) + } + + @Test + fun list() { + val product = factory.identifierQualified( + root = factory.identifierSymbol( + symbol = "hello", + caseSensitivity = Identifier.CaseSensitivity.INSENSITIVE + ), + steps = listOf( + factory.identifierSymbol( + symbol = "world", + caseSensitivity = Identifier.CaseSensitivity.SENSITIVE + ), + ) + ) + val expectedString = """ + IdentifierQualified::{ + root: IdentifierSymbol::{ symbol: "hello", caseSensitivity: IdentifierCaseSensitivity::INSENSITIVE }, + steps: [ + IdentifierSymbol::{ symbol: "world", caseSensitivity: IdentifierCaseSensitivity::SENSITIVE }, + ] + } + """.trimIndent() + val expected = loader.loadSingleElement(expectedString) + val actual = loader.loadSingleElement(product.toString()) + assertEquals(expected, actual) + } +}