Merge physical plan evaluator and basic planner to main (#612)

Merges the following PRs to `main`:

- #583 
- #584 
- #587 
- #588 
- #589 
- #609 
- #590 
- #592 

Also, this adds a TODO comment (ea40e4a, requested by @am357 in offline chat) and excludes `PlannerPipeline` from the new aggregate tests which were pulled in when merging from `main` and were not part of the other PRs (34025b3).

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

docs/user/BuiltInFunctions.md

@@ -226,8 +226,6 @@ CAST(<<'a', 'b'>> AS bag) -- <<'a', 'b'>> (REPL does not display << >> and comma
 
 ### CHAR_LENGTH, CHARACTER_LENGTH
 
-
-
 Counts the number of characters in the specified string, where 'character' is defined as a single unicode code point.
 
 *Note:* `CHAR_LENGTH` and `CHARACTER_LENGTH` are synonyms. 
@@ -455,9 +453,31 @@ EXTRACT(TIMEZONE_MINUTE FROM TIME WITH TIME ZONE '23:12:59-08:30')  -- -30
 ```
 *Note* that `timezone_hour` and `timezone_minute` are **not supported** for `DATE` and `TIME` (without time zone) type. 
 
+### `FILTER_DISTINCT`
+
+Signature
+: `FILTER_DISTINCT: Container -> Bag`
+
+Header
+: `FILTER_DISTINCT(c)`
+
+Purpose
+: Returns a bag of distinct values contained within a bag, list, sexp, or struct.  If the container is a struct,
+the field names are not considered.
+
+Examples
+:
+
+```sql
+FILTER_DISTINCT([0, 0, 1])                  -- <<0, 1>>
+FILTER_DISTINCT(<<0, 0, 1>>)                -- <<0, 1>>
+FILTER_DISTINCT(SEXP(0, 0, 1))              -- <<0, 1>>
+FILTER_DISTINCT({'a': 0, 'b': 0, 'c': 1})   -- <<0, 1>>
+```
+
 ### LOWER 
 
-Given a string convert all upper case characters to lower case characters. 
+Given a string convert all upper case characters to lower case characters.
 
 Signature
 : `LOWER: String -> String`

examples/src/kotlin/org/partiql/examples/ParserErrorExample.kt

@@ -28,7 +28,7 @@ class ParserErrorExample(out: PrintStream) : Example(out) {
 
         throw Exception("ParserException was not thrown")
     } catch (e: ParserException) {
-        val errorContext = e.errorContext!!
+        val errorContext = e.errorContext
 
         val errorInformation = "errorCode: ${e.errorCode}" +
             "\nLINE_NUMBER: ${errorContext[Property.LINE_NUMBER]}" +

lang/src/org/partiql/lang/CompilerPipeline.kt

@@ -39,7 +39,7 @@ import org.partiql.lang.types.StaticType
 import org.partiql.lang.util.interruptibleFold
 
 /**
- * Contains all of the information needed for processing steps.
+ * Contains all information needed for processing steps.
  */
 data class StepContext(
     /** The instance of [ExprValueFactory] that is used by the pipeline. */
@@ -102,6 +102,11 @@ interface CompilerPipeline {
      */
     val procedures: @JvmSuppressWildcards Map<String, StoredProcedure>
 
+    /**
+     * The configured global type bindings.
+     */
+    val globalTypeBindings: Bindings<StaticType>?
+
     /** Compiles the specified PartiQL query using the configured parser. */
     fun compile(query: String): Expression
 
@@ -205,7 +210,7 @@ interface CompilerPipeline {
         fun build(): CompilerPipeline {
             val compileOptionsToUse = compileOptions ?: CompileOptions.standard()
 
-            when (compileOptionsToUse.thunkReturnTypeAssertions) {
+            when (compileOptionsToUse.thunkOptions.thunkReturnTypeAssertions) {
                 ThunkReturnTypeAssertions.DISABLED -> { /* intentionally blank */ }
                 ThunkReturnTypeAssertions.ENABLED -> {
                     check(this.globalTypeBindings != null) {
@@ -244,7 +249,7 @@ internal class CompilerPipelineImpl(
     override val customDataTypes: List<CustomType>,
     override val procedures: Map<String, StoredProcedure>,
     private val preProcessingSteps: List<ProcessingStep>,
-    private val globalTypeBindings: Bindings<StaticType>?
+    override val globalTypeBindings: Bindings<StaticType>?
 ) : CompilerPipeline {
 
     private val compiler = EvaluatingCompiler(

lang/src/org/partiql/lang/SqlException.kt

@@ -18,6 +18,7 @@ import org.partiql.lang.errors.ErrorCode
 import org.partiql.lang.errors.Property
 import org.partiql.lang.errors.PropertyValueMap
 import org.partiql.lang.errors.UNKNOWN
+import org.partiql.lang.util.propertyValueMapOf
 
 /**
  * General exception class for the interpreter.
@@ -33,20 +34,18 @@ import org.partiql.lang.errors.UNKNOWN
  *
  * @param message the message for this exception
  * @param errorCode the error code for this exception
- * @param propertyValueMap context for this error
+ * @param errorContextArg context for this error, includes details like line & character offsets, among others.
+ * TODO: https://github.com/partiql/partiql-lang-kotlin/issues/616
  * @param cause for this exception
- *
- * @constructor a custom error [message], the [errorCode], error context as a [propertyValueMap] and optional [cause] creates an
- * [SqlException]. This is the constructor for the second configuration explained above.
- *
  */
 open class SqlException(
     override var message: String,
     val errorCode: ErrorCode,
-    val errorContext: PropertyValueMap? = null,
+    errorContextArg: PropertyValueMap? = null,
     cause: Throwable? = null
-) :
-    RuntimeException(message, cause) {
+) : RuntimeException(message, cause) {
+
+    val errorContext: PropertyValueMap = errorContextArg ?: propertyValueMapOf()
 
     /**
      * Indicates if this exception is due to an internal error or not.
@@ -81,7 +80,7 @@ open class SqlException(
      *
      *  * ErrorCategory is one of `Lexer Error`, `Parser Error`, `Runtime Error`
      *  * ErrorLocation is the line and column where the error occurred
-     *  * Errormessatge is the **generated** error message
+     *  * ErrorMessage is the **generated** error message
      *
      *
      * TODO: Prepend to the auto-generated message the file name.
@@ -90,6 +89,10 @@ open class SqlException(
     fun generateMessage(): String =
         "${errorCategory(errorCode)}: ${errorLocation(errorContext)}: ${errorMessage(errorCode, errorContext)}"
 
+    /** Same as [generateMessage] but without the location. */
+    fun generateMessageNoLocation(): String =
+        "${errorCategory(errorCode)}: ${errorMessage(errorCode, errorContext)}"
+
     private fun errorMessage(errorCode: ErrorCode?, propertyValueMap: PropertyValueMap?): String =
         errorCode?.getErrorMessage(propertyValueMap) ?: UNKNOWN
 

lang/src/org/partiql/lang/ast/passes/SemanticException.kt

@@ -37,7 +37,7 @@ class SemanticException(
     constructor(err: Problem, cause: Throwable? = null) :
         this(
             message = "",
-            errorCode = ErrorCode.SEMANTIC_INFERENCER_ERROR,
+            errorCode = ErrorCode.SEMANTIC_PROBLEM,
             errorContext = propertyValueMapOf(
                 Property.LINE_NUMBER to err.sourceLocation.lineNum,
                 Property.COLUMN_NUMBER to err.sourceLocation.charOffset,

lang/src/org/partiql/lang/domains/util.kt

@@ -1,5 +1,6 @@
 package org.partiql.lang.domains
 
+import com.amazon.ionelement.api.IonElement
 import com.amazon.ionelement.api.MetaContainer
 import com.amazon.ionelement.api.emptyMetaContainer
 import com.amazon.ionelement.api.metaContainerOf
@@ -14,6 +15,19 @@ import org.partiql.lang.eval.BindingCase
 fun PartiqlAst.Builder.id(name: String) =
     id(name, caseInsensitive(), unqualified())
 
+// TODO:  once https://github.com/partiql/partiql-ir-generator/issues/6 has been completed, we can delete this.
+fun PartiqlLogical.Builder.id(name: String) =
+    id(name, caseInsensitive(), unqualified())
+
+// TODO:  once https://github.com/partiql/partiql-ir-generator/issues/6 has been completed, we can delete this.
+fun PartiqlLogical.Builder.pathExpr(exp: PartiqlLogical.Expr) =
+    pathExpr(exp, caseInsensitive())
+
+// Workaround for a bug in PIG that is fixed in its next release:
+// https://github.com/partiql/partiql-ir-generator/issues/41
+fun List<IonElement>.asAnyElement() =
+    this.map { it.asAnyElement() }
+
 val MetaContainer.staticType: StaticTypeMeta? get() = this[StaticTypeMeta.TAG] as StaticTypeMeta?
 
 /** Constructs a container with the specified metas. */
@@ -60,17 +74,17 @@ fun PartiqlAst.CaseSensitivity.toBindingCase(): BindingCase = when (this) {
 }
 
 /**
- * Returns the [SourceLocationMeta] as an error context if the [SourceLocationMeta.TAG] exists in the passed
- * [metaContainer]. Otherwise, returns an empty map.
+ * Converts a [PartiqlLogical.CaseSensitivity] to a [BindingCase].
  */
-fun errorContextFrom(metaContainer: MetaContainer?): PropertyValueMap {
-    if (metaContainer == null) {
-        return PropertyValueMap()
-    }
-    val location = metaContainer[SourceLocationMeta.TAG] as? SourceLocationMeta
-    return if (location != null) {
-        org.partiql.lang.eval.errorContextFrom(location)
-    } else {
-        PropertyValueMap()
-    }
+fun PartiqlLogical.CaseSensitivity.toBindingCase(): BindingCase = when (this) {
+    is PartiqlLogical.CaseSensitivity.CaseInsensitive -> BindingCase.INSENSITIVE
+    is PartiqlLogical.CaseSensitivity.CaseSensitive -> BindingCase.SENSITIVE
+}
+
+/**
+ * Converts a [PartiqlLogical.CaseSensitivity] to a [BindingCase].
+ */
+fun PartiqlPhysical.CaseSensitivity.toBindingCase(): BindingCase = when (this) {
+    is PartiqlPhysical.CaseSensitivity.CaseInsensitive -> BindingCase.INSENSITIVE
+    is PartiqlPhysical.CaseSensitivity.CaseSensitive -> BindingCase.SENSITIVE
 }

lang/src/org/partiql/lang/errors/ErrorCode.kt

@@ -647,7 +647,7 @@ enum class ErrorCode(
                 "got: ${errorContext?.get(Property.ACTUAL_ARGUMENT_TYPES) ?: UNKNOWN}"
     },
 
-    SEMANTIC_INFERENCER_ERROR(
+    SEMANTIC_PROBLEM(
         ErrorCategory.SEMANTIC,
         LOCATION + setOf(Property.MESSAGE),
         ""
@@ -980,12 +980,6 @@ enum class ErrorCode(
         ErrorBehaviorInPermissiveMode.RETURN_MISSING
     ),
 
-    EVALUATOR_SQL_EXCEPTION(
-        ErrorCategory.EVALUATOR,
-        LOCATION,
-        "SQL exception"
-    ),
-
     EVALUATOR_COUNT_START_NOT_ALLOWED(
         ErrorCategory.EVALUATOR,
         LOCATION,

lang/src/org/partiql/lang/eval/CompileOptions.kt

@@ -141,14 +141,14 @@ enum class ThunkReturnTypeAssertions {
  * @param defaultTimezoneOffset Default timezone offset to be used when TIME WITH TIME ZONE does not explicitly
  * specify the time zone. Defaults to [ZoneOffset.UTC]
  */
+@Suppress("DataClassPrivateConstructor")
 data class CompileOptions private constructor (
     val undefinedVariable: UndefinedVariableBehavior,
     val projectionIteration: ProjectionIterationBehavior = ProjectionIterationBehavior.FILTER_MISSING,
     val visitorTransformMode: VisitorTransformMode = VisitorTransformMode.DEFAULT,
     val thunkOptions: ThunkOptions = ThunkOptions.standard(),
     val typingMode: TypingMode = TypingMode.LEGACY,
     val typedOpBehavior: TypedOpBehavior = TypedOpBehavior.LEGACY,
-    val thunkReturnTypeAssertions: ThunkReturnTypeAssertions = ThunkReturnTypeAssertions.DISABLED,
     val defaultTimezoneOffset: ZoneOffset = ZoneOffset.UTC
 ) {
 
@@ -177,7 +177,7 @@ data class CompileOptions private constructor (
         fun build(options: CompileOptions, block: Builder.() -> Unit) = Builder(options).apply(block).build()
 
         /**
-         * Creates a [CompileOptions] instance with the standard values.
+         * Creates a [CompileOptions] instance with the standard values for use by the legacy AST compiler.
          */
         @JvmStatic
         fun standard() = Builder().build()
@@ -194,7 +194,7 @@ data class CompileOptions private constructor (
         fun typingMode(value: TypingMode) = set { copy(typingMode = value) }
         fun typedOpBehavior(value: TypedOpBehavior) = set { copy(typedOpBehavior = value) }
         fun thunkOptions(value: ThunkOptions) = set { copy(thunkOptions = value) }
-        fun evaluationTimeTypeChecks(value: ThunkReturnTypeAssertions) = set { copy(thunkReturnTypeAssertions = value) }
+        fun thunkOptions(build: ThunkOptions.Builder.() -> Unit) = set { copy(thunkOptions = ThunkOptions.build(build)) }
         fun defaultTimezoneOffset(value: ZoneOffset) = set { copy(defaultTimezoneOffset = value) }
 
         private inline fun set(block: CompileOptions.() -> CompileOptions): Builder {

lang/src/org/partiql/lang/eval/EvaluatingCompiler.kt

@@ -86,6 +86,25 @@ import java.util.TreeSet
 import java.util.regex.Pattern
 import kotlin.Comparator
 
+/**
+ * A thunk with no parameters other than the current environment.
+ *
+ * See https://en.wikipedia.org/wiki/Thunk
+ *
+ * This name was chosen because it is a thunk that accepts an instance of `Environment`.
+ */
+private typealias ThunkEnv = Thunk<Environment>
+
+/**
+ * A thunk taking a single [T] argument and the current environment.
+ *
+ * See https://en.wikipedia.org/wiki/Thunk
+ *
+ * This name was chosen because it is a thunk that accepts an instance of `Environment` and an [ExprValue] as
+ * its arguments.
+ */
+private typealias ThunkEnvValue<T> = ThunkValue<Environment, T>
+
 /**
  * A basic compiler that converts an instance of [PartiqlAst] to an [Expression].
  *
@@ -115,7 +134,7 @@ internal class EvaluatingCompiler(
     private val compileOptions: CompileOptions = CompileOptions.standard()
 ) {
     private val errorSignaler = compileOptions.typingMode.createErrorSignaler(valueFactory)
-    private val thunkFactory = compileOptions.typingMode.createThunkFactory(compileOptions, valueFactory)
+    private val thunkFactory = compileOptions.typingMode.createThunkFactory<Environment>(compileOptions.thunkOptions, valueFactory)
 
     private val compilationContextStack = Stack<CompilationContext>()
 
@@ -3048,7 +3067,7 @@ private class SingleProjectionElement(val name: ExprValue, val thunk: ThunkEnv)
  */
 private class MultipleProjectionElement(val thunks: List<ThunkEnv>) : ProjectionElement()
 
-private val MetaContainer.sourceLocationMeta get() = this[SourceLocationMeta.TAG] as? SourceLocationMeta
+internal val MetaContainer.sourceLocationMeta get() = this[SourceLocationMeta.TAG] as? SourceLocationMeta
 
 private fun StaticType.getTypes() = when (val flattened = this.flatten()) {
     is AnyOfType -> flattened.types

lang/src/org/partiql/lang/eval/Exceptions.kt

@@ -125,6 +125,10 @@ fun fillErrorContext(errorContext: PropertyValueMap, metaContainer: MetaContaine
     }
 }
 
+/**
+ * Returns the [SourceLocationMeta] as an error context if the [SourceLocationMeta.TAG] exists in the passed
+ * [metaContainer]. Otherwise, returns an empty map.
+ */
 fun errorContextFrom(metaContainer: MetaContainer?): PropertyValueMap {
     if (metaContainer == null) {
         return PropertyValueMap()