Merge pull request #1273 from partiql/planner-main

Merge Planner to Main

.github/workflows/build.yml

@@ -23,6 +23,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
+          ref: ${{ github.event.pull_request.head.sha }}
           submodules: recursive
 
       - name: Use Java ${{ matrix.java }}

CHANGELOG.md

@@ -31,18 +31,28 @@ Thank you to all who have contributed!
 - Adds top-level IR node creation functions.
 - Adds `componentN` functions (destructuring) to IR nodes via Kotlin data classes
 - Adds public `tag` field to IR nodes for associating metadata
+- Adds AST Normalization Pass. 
+- Adds PartiQLPlanner Interface, which is responsible for translate an AST to a Plan. 
 
 ### Changed
 - StaticTypeInferencer and PlanTyper will not raise an error when an expression is inferred to `NULL` or `unionOf(NULL, MISSING)`. In these cases the StaticTypeInferencer and PlanTyper will still raise the Problem Code `ExpressionAlwaysReturnsNullOrMissing` but the severity of the problem has been changed to warning. In the case an expression always returns `MISSING`, problem code `ExpressionAlwaysReturnsMissing` will be raised, which will have problem severity of error.
+- **Breaking** The default integer literal type is now 32-bit; if the literal can not fit in a 32-bit integer, it overflows to 64-bit. 
+- **BREAKING** `PartiQLValueType` now distinguishes between Arbitrary Precision Decimal and Fixed Precision Decimal. 
+- **BREAKING** Function Signature Changes. Now Function signature has two subclasses, `Scalar` and `Aggregation`. 
+- **BREAKING** In the produced plan: 
+  - The new plan is fully resolved and typed.
+  - Operators will be converted to function call. 
+
 
 ### Deprecated
 
 ### Fixed
 - Fixes the CLI hanging on invalid queries. See issue #1230.
 
 ### Removed
-- [Breaking] Removed IR factory in favor of static top-level functions. Change `Ast.foo()`
+- **Breaking** Removed IR factory in favor of static top-level functions. Change `Ast.foo()`
   to `foo()`
+- **Breaking** Removed `org.partiql.lang.planner.transforms.AstToPlan`. Use `org.partiql.planner.PartiQLPlanner`. 
 
 ### Security
 
@@ -153,6 +163,8 @@ classes in `:partiql-ast` and `:partiql-plan`.
   - `org.partiql.lang.errors.ProblemSeverity` -> `org.partiql.errors.ProblemSeverity`
   - `org.partiql.lang.errors.ProblemHandler` -> `org.partiql.errors.ProblemHandler`
 - **Breaking** the `sourceLocation` field of `org.partiql.errors.Problem` was changed from `org.partiql.lang.ast.SoureceLocationMeta` to `org.partiql.errors.ProblemLocation`.
+- Removed `Nullable<Value` implementations of PartiQLValue and made the standard implementations nullable.
+- Using PartiQLValueType requires optin; this was a miss from an earlier commit.
 - **Breaking** removed redundant ValueParameter from FunctionParameter as all parameters are values.
 - Introduces `isNullCall` and `isNullable` properties to FunctionSignature.
 - Removed `Nullable...Value` implementations of PartiQLValue and made the standard implementations nullable.

buildSrc/src/main/kotlin/partiql.versions.kt

@@ -44,6 +44,7 @@ object Versions {
     const val kasechange = "1.3.0"
     const val ktlint = "11.6.0"
     const val pig = "0.6.2"
+    const val shadow = "7.1.2"
 
     // Testing
     const val assertj = "3.11.0"
@@ -113,4 +114,6 @@ object Plugins {
     const val jmh = "me.champeau.gradle.jmh"
     const val ktlint = "org.jlleitschuh.gradle.ktlint"
     const val library = "org.gradle.java-library"
+    const val shadow = "com.github.johnrengelman.shadow"
+    const val testFixtures = "org.gradle.java-test-fixtures"
 }

gradle.properties

@@ -1,5 +1,5 @@
 group=org.partiql
-version=0.13.3-SNAPSHOT
+version=0.14.0-SNAPSHOT
 
 ossrhUsername=EMPTY
 ossrhPassword=EMPTY

partiql-ast/src/main/kotlin/org/partiql/ast/helpers/ToBinder.kt

@@ -0,0 +1,65 @@
+package org.partiql.ast.helpers
+
+import org.partiql.ast.Expr
+import org.partiql.ast.Identifier
+import org.partiql.ast.builder.ast
+import org.partiql.value.PartiQLValueExperimental
+import org.partiql.value.StringValue
+
+private val col = { index: () -> Int -> "_${index()}" }
+
+/**
+ * Produces a "binder" (AS alias) for an expression following the given rules:
+ *
+ *  1. If item is an id, use the last symbol
+ *  2. If item is a path with a final symbol step, use the symbol — else 4
+ *  3. If item is a cast, use the value name
+ *  4. Else, use item index with prefix _
+ *
+ *  See https://github.com/partiql/partiql-lang-kotlin/issues/1122
+ */
+public fun Expr.toBinder(index: () -> Int): Identifier.Symbol = when (this) {
+    is Expr.Var -> this.identifier.toBinder()
+    is Expr.Path -> this.toBinder(index)
+    is Expr.Cast -> this.value.toBinder(index)
+    is Expr.SessionAttribute -> this.attribute.name.uppercase().toBinder()
+    else -> col(index).toBinder()
+}
+
+/**
+ * Simple toBinder that uses an int literal rather than a closure.
+ *
+ * @param index
+ * @return
+ */
+public fun Expr.toBinder(index: Int): Identifier.Symbol = toBinder { index }
+
+private fun String.toBinder(): Identifier.Symbol = ast {
+    // Every binder preserves case
+    identifierSymbol(this@toBinder, Identifier.CaseSensitivity.SENSITIVE)
+}
+
+private fun Identifier.toBinder(): Identifier.Symbol = when (this@toBinder) {
+    is Identifier.Qualified -> when (steps.isEmpty()) {
+        true -> root.symbol.toBinder()
+        else -> steps.last().symbol.toBinder()
+    }
+    is Identifier.Symbol -> symbol.toBinder()
+}
+
+@OptIn(PartiQLValueExperimental::class)
+private fun Expr.Path.toBinder(index: () -> Int): Identifier.Symbol {
+    if (steps.isEmpty()) return root.toBinder(index)
+    return when (val last = steps.last()) {
+        is Expr.Path.Step.Symbol -> last.symbol.toBinder()
+        is Expr.Path.Step.Index -> {
+            val k = last.key
+            if (k is Expr.Lit && k.value is StringValue) {
+                k.value.value!!.toBinder()
+            } else {
+                col(index).toBinder()
+            }
+        }
+        else -> col(index).toBinder()
+    }
+}

partiql-ast/src/main/kotlin/org/partiql/ast/normalize/AstPass.kt

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at:
+ *
+ *      http://aws.amazon.com/apache2.0/
+ *
+ *  or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
+ *  language governing permissions and limitations under the License.
+ */
+
+package org.partiql.ast.normalize
+
+import org.partiql.ast.Statement
+
+/**
+ * Wraps a rewriter with a default entry point.
+ */
+public interface AstPass {
+
+    public fun apply(statement: Statement): Statement
+}

partiql-ast/src/main/kotlin/org/partiql/ast/normalize/Normalize.kt

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at:
+ *
+ *      http://aws.amazon.com/apache2.0/
+ *
+ *  or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
+ *  language governing permissions and limitations under the License.
+ */
+
+package org.partiql.ast.normalize
+
+import org.partiql.ast.Statement
+
+/**
+ * AST normalization
+ */
+public fun Statement.normalize(): Statement {
+    // could be a fold, but this is nice for setting breakpoints
+    var ast = this
+    ast = NormalizeFromSource.apply(ast)
+    ast = NormalizeSelect.apply(ast)
+    ast = NormalizeGroupBy.apply(ast)
+    return ast
+}

partiql-ast/src/main/kotlin/org/partiql/ast/normalize/NormalizeFromSource.kt

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at:
+ *
+ *      http://aws.amazon.com/apache2.0/
+ *
+ *  or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
+ *  language governing permissions and limitations under the License.
+ */
+
+package org.partiql.ast.normalize
+
+import org.partiql.ast.AstNode
+import org.partiql.ast.Expr
+import org.partiql.ast.From
+import org.partiql.ast.Statement
+import org.partiql.ast.fromJoin
+import org.partiql.ast.helpers.toBinder
+import org.partiql.ast.util.AstRewriter
+
+/**
+ * Assign aliases to any FROM source which does not have one.
+ */
+internal object NormalizeFromSource : AstPass {
+
+    override fun apply(statement: Statement): Statement = Visitor.visitStatement(statement, 0) as Statement
+
+    private object Visitor : AstRewriter<Int>() {
+
+        // Each SFW starts the ctx count again.
+        override fun visitExprSFW(node: Expr.SFW, ctx: Int): AstNode = super.visitExprSFW(node, 0)
+
+        override fun visitStatementDMLBatchLegacy(node: Statement.DML.BatchLegacy, ctx: Int): AstNode =
+            super.visitStatementDMLBatchLegacy(node, 0)
+
+        override fun visitFrom(node: From, ctx: Int) = super.visitFrom(node, ctx) as From
+
+        override fun visitFromJoin(node: From.Join, ctx: Int): From {
+            val lhs = visitFrom(node.lhs, ctx)
+            val rhs = visitFrom(node.rhs, ctx + 1)
+            val condition = node.condition?.let { visitExpr(it, ctx) as Expr }
+            return if (lhs !== node.lhs || rhs !== node.rhs || condition !== node.condition) {
+                fromJoin(lhs, rhs, node.type, condition)
+            } else {
+                node
+            }
+        }
+
+        override fun visitFromValue(node: From.Value, ctx: Int): From {
+            val expr = visitExpr(node.expr, ctx) as Expr
+            val asAlias = node.asAlias ?: expr.toBinder(ctx)
+            return if (expr !== node.expr || asAlias !== node.asAlias) {
+                node.copy(expr = expr, asAlias = asAlias)
+            } else {
+                node
+            }
+        }
+    }
+}

partiql-ast/src/main/kotlin/org/partiql/ast/normalize/NormalizeGroupBy.kt

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at:
+ *
+ *      http://aws.amazon.com/apache2.0/
+ *
+ *  or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
+ *  language governing permissions and limitations under the License.
+ */
+
+package org.partiql.ast.normalize
+
+import org.partiql.ast.Expr
+import org.partiql.ast.GroupBy
+import org.partiql.ast.Statement
+import org.partiql.ast.groupByKey
+import org.partiql.ast.helpers.toBinder
+import org.partiql.ast.util.AstRewriter
+
+/**
+ * Adds a unique binder to each group key.
+ */
+object NormalizeGroupBy : AstPass {
+
+    override fun apply(statement: Statement) = Visitor.visitStatement(statement, 0) as Statement
+
+    private object Visitor : AstRewriter<Int>() {
+
+        override fun visitGroupByKey(node: GroupBy.Key, ctx: Int): GroupBy.Key {
+            val expr = visitExpr(node.expr, 0) as Expr
+            val alias = when (node.asAlias) {
+                null -> expr.toBinder(ctx)
+                else -> node.asAlias
+            }
+            return if (expr !== node.expr || alias !== node.asAlias) {
+                groupByKey(expr, alias)
+            } else {
+                node
+            }
+        }
+    }
+}