identifier normalization

partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt

@@ -1084,7 +1084,7 @@ class PartiQLEngineDefaultTest {
     ) {
 
         private val engine = PartiQLEngine.builder().build()
-        private val planner = PartiQLPlannerBuilder().build()
+        private val planner = PartiQLPlannerBuilder().casePreserve().lookUpBehavior("INSENSITIVE").build()
         private val parser = PartiQLParser.default()
         private val loader = createIonElementLoader()
 
@@ -1156,7 +1156,7 @@ class PartiQLEngineDefaultTest {
     ) {
 
         private val engine = PartiQLEngine.builder().build()
-        private val planner = PartiQLPlannerBuilder().build()
+        private val planner = PartiQLPlannerBuilder().casePreserve().lookUpBehavior("INSENSITIVE").build()
         private val parser = PartiQLParser.default()
 
         internal fun assert() {

partiql-planner/api/partiql-planner.api

@@ -37,7 +37,9 @@ public final class org/partiql/planner/PartiQLPlannerBuilder {
 	public final fun addPass (Lorg/partiql/planner/PartiQLPlannerPass;)Lorg/partiql/planner/PartiQLPlannerBuilder;
 	public final fun addPasses ([Lorg/partiql/planner/PartiQLPlannerPass;)Lorg/partiql/planner/PartiQLPlannerBuilder;
 	public final fun build ()Lorg/partiql/planner/PartiQLPlanner;
+	public final fun casePreserve ()Lorg/partiql/planner/PartiQLPlannerBuilder;
 	public final fun catalogs ([Lkotlin/Pair;)Lorg/partiql/planner/PartiQLPlannerBuilder;
+	public final fun lookUpBehavior (Ljava/lang/String;)Lorg/partiql/planner/PartiQLPlannerBuilder;
 	public final fun passes (Ljava/util/List;)Lorg/partiql/planner/PartiQLPlannerBuilder;
 	public final fun signalMode ()Lorg/partiql/planner/PartiQLPlannerBuilder;
 }

partiql-planner/src/main/kotlin/org/partiql/planner/PartiQLPlanner.kt

@@ -56,6 +56,10 @@ public interface PartiQLPlanner {
         public fun builder(): PartiQLPlannerBuilder = PartiQLPlannerBuilder()
 
         @JvmStatic
-        public fun default(): PartiQLPlanner = PartiQLPlannerBuilder().build()
+        public fun default(): PartiQLPlanner =
+            PartiQLPlannerBuilder()
+                .casePreserve()
+                .lookUpBehavior("INSENSITIVE")
+                .build()
     }
 }

partiql-planner/src/main/kotlin/org/partiql/planner/PartiQLPlannerBuilder.kt

@@ -1,7 +1,9 @@
 package org.partiql.planner
 
+import org.partiql.planner.internal.BooleanFlag
 import org.partiql.planner.internal.PartiQLPlannerDefault
 import org.partiql.planner.internal.PlannerFlag
+import org.partiql.planner.internal.RValue
 import org.partiql.spi.connector.ConnectorMetadata
 
 /**
@@ -50,7 +52,24 @@ public class PartiQLPlannerBuilder {
      * Java style method for setting the planner to signal mode
      */
     public fun signalMode(): PartiQLPlannerBuilder = this.apply {
-        this.flags.add(PlannerFlag.SIGNAL_MODE)
+        this.flags.add(BooleanFlag.SIGNAL_MODE)
+    }
+
+    public fun casePreserve(): PartiQLPlannerBuilder = this.apply {
+        this.flags.add(BooleanFlag.CASE_PRESERVATION)
+    }
+
+    public fun lookUpBehavior(behavior: String): PartiQLPlannerBuilder = this.apply {
+        this.flags.removeAll(
+            this.flags.filterIsInstance<RValue>().toSet()
+        )
+        when (behavior.uppercase()) {
+            "FOLDING_UP" -> this.flags.add(RValue.FOLDING_UP)
+            "FOLDING_DOWN" -> this.flags.add(RValue.FOLDING_DOWN)
+            "SENSITIVE" -> this.flags.add(RValue.SENSITIVE)
+            "INSENSITIVE" -> this.flags.add(RValue.INSENSITIVE)
+            else -> error("Illegal flag, expect one of ${RValue.values()}")
+        }
     }
 
     /**

partiql-planner/src/main/kotlin/org/partiql/planner/internal/IdentifierManager.kt

@@ -0,0 +1,39 @@
+package org.partiql.planner.internal
+
+import org.partiql.ast.Binder
+import org.partiql.ast.Identifier
+import org.partiql.ast.binder
+import org.partiql.ast.identifierSymbol
+
+internal class IdentifierManager(
+    val casePreservation: Boolean,
+    val rValue: RValue
+) {
+    fun normalizeBinder(binder: Binder): Binder =
+        when (binder.caseSensitivity) {
+            Binder.CaseSensitivity.SENSITIVE -> binder
+            Binder.CaseSensitivity.INSENSITIVE -> {
+                if (casePreservation) {
+                    binder(binder.symbol, Binder.CaseSensitivity.SENSITIVE)
+                }
+                // This is a conscious decision
+                // Even though the case preservation is off, we still need a way to store the binder
+                // To make this compatible with the current implementation,
+                // the decision here is: we normalize by preserving case.
+                else {
+                    binder(binder.symbol, Binder.CaseSensitivity.SENSITIVE)
+                }
+            }
+        }
+
+    fun normalizeRvalue(id: Identifier.Symbol): Identifier.Symbol =
+        when (id.caseSensitivity) {
+            Identifier.CaseSensitivity.SENSITIVE -> id
+            Identifier.CaseSensitivity.INSENSITIVE -> when (rValue) {
+                RValue.FOLDING_UP -> identifierSymbol(id.symbol.uppercase(), Identifier.CaseSensitivity.SENSITIVE)
+                RValue.FOLDING_DOWN -> identifierSymbol(id.symbol.lowercase(), Identifier.CaseSensitivity.SENSITIVE)
+                RValue.SENSITIVE -> identifierSymbol(id.symbol, Identifier.CaseSensitivity.SENSITIVE)
+                RValue.INSENSITIVE -> identifierSymbol(id.symbol, Identifier.CaseSensitivity.INSENSITIVE)
+            }
+        }
+}

partiql-planner/src/main/kotlin/org/partiql/planner/internal/PartiQLPlannerDefault.kt

@@ -27,7 +27,7 @@ internal class PartiQLPlannerDefault(
         val env = Env(session)
 
         // 1. Normalize
-        val ast = statement.normalize()
+        val ast = statement.normalize(flags)
 
         // 2. AST to Rel/Rex
         val root = AstToPlan.apply(ast, env)

partiql-planner/src/main/kotlin/org/partiql/planner/internal/PlannerFlag.kt

@@ -1,6 +1,9 @@
 package org.partiql.planner.internal
 
-internal enum class PlannerFlag {
+// Marker interface
+internal interface PlannerFlag
+
+internal enum class BooleanFlag : PlannerFlag {
     /**
      * Determine the planner behavior upon encounter an operation that always returns MISSING.
      *
@@ -16,5 +19,38 @@ internal enum class PlannerFlag {
      *
      *    The result plan will turn the problematic operation into a missing node.
      */
-    SIGNAL_MODE
+    SIGNAL_MODE,
+
+    /**
+     * Determines lvalue behavior
+     *
+     * If this flag is included:
+     *    Lvalue (AS binding, DDL, etc) are case-sensitive
+     *
+     * If not included:
+     *   Lvalue are normalized using an implementation defined normalization rule
+     */
+    CASE_PRESERVATION
+}
+
+internal enum class RValue : PlannerFlag {
+    /**
+     * Using upper case text to look up
+     */
+    FOLDING_UP,
+
+    /**
+     * Using lower case text to look up
+     */
+    FOLDING_DOWN,
+
+    /**
+     * Using original text to look up
+     */
+    SENSITIVE,
+
+    /**
+     * Match behavior: text comparison with case ignored.
+     */
+    INSENSITIVE
 }

partiql-planner/src/main/kotlin/org/partiql/planner/internal/astPasses/Normalize.kt

@@ -15,14 +15,20 @@
 package org.partiql.planner.internal.astPasses
 
 import org.partiql.ast.Statement
+import org.partiql.planner.internal.BooleanFlag
+import org.partiql.planner.internal.PlannerFlag
+import org.partiql.planner.internal.RValue
 
 /**
  * AST normalization
  */
-internal fun Statement.normalize(): Statement {
+internal fun Statement.normalize(flags: Set<PlannerFlag>): Statement {
     // could be a fold, but this is nice for setting breakpoints
     var ast = this
     ast = NormalizeFromSource.apply(ast)
     ast = NormalizeGroupBy.apply(ast)
+    val casePreservation = flags.any { it == BooleanFlag.CASE_PRESERVATION }
+    val rValue = flags.first { it is RValue } as RValue
+    ast = NormalizeIdentifier(casePreservation, rValue).apply(ast)
     return ast
 }

partiql-planner/src/main/kotlin/org/partiql/planner/internal/astPasses/NormalizeIdentifier.kt

@@ -0,0 +1,41 @@
+package org.partiql.planner.internal.astPasses
+
+import org.partiql.ast.Binder
+import org.partiql.ast.Identifier
+import org.partiql.ast.Statement
+import org.partiql.ast.normalize.AstPass
+import org.partiql.ast.util.AstRewriter
+import org.partiql.planner.internal.IdentifierManager
+import org.partiql.planner.internal.RValue
+
+/**
+ * Case Preservation:
+ *   - If on, turn case-insensitive binder to case-sensitive binder, with symbol preserve the original case
+ *   - If off, normalize the binder.
+ *      - The normalization function in the spec perhaps will end up being implementation defined.
+ *      - Therefore, for now we normalize by preserving case.
+ *      - This is to stay consistent with the current behavior, and reduce one moving element during testing....
+ *
+ * Identifier Normalization:
+ *   - Determines the normalization for identifier (variable look up)
+ *   - Folding down: Using lower case text to look up
+ *   - Folding up: Using upper case text to look up.
+ *   - Case Sensitive: Using original text to look up
+ *   - Case Insensitive: Matching behavior, string comparison with case ignored.
+ */
+internal class NormalizeIdentifier(
+    casePreservation: Boolean,
+    rValue: RValue
+) : AstPass {
+
+    val identifierManager = IdentifierManager(casePreservation, rValue)
+    override fun apply(statement: Statement): Statement = Visitor(identifierManager).visitStatement(statement, Unit) as Statement
+
+    private class Visitor(val identifierManager: IdentifierManager) : AstRewriter<Unit>() {
+
+        override fun visitIdentifierSymbol(node: Identifier.Symbol, ctx: Unit) =
+            identifierManager.normalizeRvalue(node)
+
+        override fun visitBinder(node: Binder, ctx: Unit) = identifierManager.normalizeBinder(node)
+    }
+}

partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt

@@ -5,6 +5,7 @@ import org.partiql.plan.PlanNode
 import org.partiql.plan.partiQLPlan
 import org.partiql.plan.rexOpCast
 import org.partiql.plan.rexOpErr
+import org.partiql.planner.internal.BooleanFlag
 import org.partiql.planner.internal.PlannerFlag
 import org.partiql.planner.internal.ProblemGenerator
 import org.partiql.planner.internal.ir.Identifier
@@ -27,7 +28,7 @@ import org.partiql.value.PartiQLValueExperimental
 internal class PlanTransform(
     flags: Set<PlannerFlag>
 ) {
-    private val signalMode = flags.contains(PlannerFlag.SIGNAL_MODE)
+    private val signalMode = flags.contains(BooleanFlag.SIGNAL_MODE)
 
     fun transform(node: PartiQLPlan, onProblem: ProblemCallback): org.partiql.plan.PartiQLPlan {
         val symbols = Symbols.empty()

partiql-planner/src/test/kotlin/org/partiql/planner/PlanTest.kt

@@ -88,8 +88,8 @@ class PlanTest {
         val problemCollector = ProblemCollector()
         val ast = PartiQLParser.default().parse(test.statement).root
         val planner = when (isSignalMode) {
-            true -> PartiQLPlanner.builder().signalMode().build()
-            else -> PartiQLPlanner.builder().build()
+            true -> PartiQLPlanner.builder().signalMode().casePreserve().lookUpBehavior("INSENSITIVE").build()
+            else -> PartiQLPlanner.builder().casePreserve().lookUpBehavior("INSENSITIVE").build()
         }
         planner.plan(ast, session, problemCollector)
     }

partiql-planner/src/test/kotlin/org/partiql/planner/PlannerErrorReportingTests.kt

@@ -398,8 +398,8 @@ internal class PlannerErrorReportingTests {
 
     private fun runTestCase(tc: TestCase) {
         val planner = when (tc.isSignal) {
-            true -> PartiQLPlanner.builder().signalMode().build()
-            else -> PartiQLPlanner.builder().build()
+            true -> PartiQLPlanner.builder().signalMode().casePreserve().lookUpBehavior("INSENSITIVE").build()
+            else -> PartiQLPlanner.builder().casePreserve().lookUpBehavior("INSENSITIVE").build()
         }
         val pc = ProblemCollector()
         val res = planner.plan(statement(tc.query), session, pc)

partiql-planner/src/test/kotlin/org/partiql/planner/internal/typer/PlanTyperTestsPorted.kt

@@ -130,7 +130,12 @@ internal class PlanTyperTestsPorted {
     companion object {
 
         private val parser = PartiQLParser.default()
-        private val planner = PartiQLPlanner.builder().signalMode().build()
+        private val planner = PartiQLPlanner
+            .builder()
+            .signalMode()
+            .casePreserve()
+            .lookUpBehavior("INSENSITIVE")
+            .build()
 
         private fun assertProblemExists(problem: Problem) = ProblemHandler { problems, ignoreSourceLocation ->
             val message = buildString {