From e8622ee8c775ccb60a7df8921a20d62e9df71ad1 Mon Sep 17 00:00:00 2001 From: John Ed Quinn Date: Thu, 14 Dec 2023 14:35:40 -0800 Subject: [PATCH 1/6] Structures globals within plan to lie within catalogs --- .../src/main/resources/partiql_plan.ion | 22 +++++-- .../partiql/planner/PartiQLPlannerDefault.kt | 2 +- .../org/partiql/planner/internal/Env.kt | 64 ++++++++++++++----- .../org/partiql/planner/internal/ir/Nodes.kt | 54 +++++++++++----- .../org/partiql/planner/internal/ir/Plan.kt | 16 +++-- .../internal/ir/builder/PlanBuilder.kt | 34 +++++++--- .../internal/ir/builder/PlanBuilders.kt | 54 +++++++++++----- .../planner/internal/ir/util/PlanRewriter.kt | 27 +++++--- .../internal/ir/visitor/PlanBaseVisitor.kt | 6 +- .../internal/ir/visitor/PlanVisitor.kt | 6 +- .../internal/transforms/PlanTransform.kt | 22 ++++--- .../planner/internal/typer/PlanTyper.kt | 11 ++-- .../org/partiql/planner/internal/EnvTest.kt | 31 ++++----- .../planner/internal/typer/PlanTyperTest.kt | 16 ++--- 14 files changed, 239 insertions(+), 126 deletions(-) diff --git a/partiql-plan/src/main/resources/partiql_plan.ion b/partiql-plan/src/main/resources/partiql_plan.ion index 170fa29017..15ff3cbbe9 100644 --- a/partiql-plan/src/main/resources/partiql_plan.ion +++ b/partiql-plan/src/main/resources/partiql_plan.ion @@ -8,15 +8,22 @@ imports::{ } parti_q_l_plan::{ - globals: list::[global], // (globals ...) + catalogs: list::[catalog], // (catalogs ...) statement: statement, // (statement ...) } -// Globals - -global::{ - path: '.identifier.qualified', - type: static_type, +// Represent an instance of a database. +// - Currently, `values` represents all values from this catalog to be used in this plan. +// - Eventually, TODO functions may be resolved to a specific namespace within a catalog. +catalog::{ + name: string, + values: list::[value], + _: [ + value::{ + path: list::[string], // This should NOT be empty. + type: static_type + } + ] } // Functions @@ -70,7 +77,8 @@ rex::{ }, global::{ - ref: int, + catalog_ref: int, // The index of the relevant catalog in the plan's catalogs list + value_ref: Int // The index of the relevant value in a catalog's values list }, path::{ diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/PartiQLPlannerDefault.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/PartiQLPlannerDefault.kt index e0dcda8028..618b3928dd 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/PartiQLPlannerDefault.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/PartiQLPlannerDefault.kt @@ -37,7 +37,7 @@ internal class PartiQLPlannerDefault( val typer = PlanTyper(env, onProblem) val internal = org.partiql.planner.internal.ir.PartiQLPlan( version = PartiQLVersion.VERSION_0_1, - globals = env.globals, + catalogs = env.catalogs, statement = typer.resolve(root), ) diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/Env.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/Env.kt index 1517f020d6..988758c5b0 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/Env.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/Env.kt @@ -3,13 +3,11 @@ package org.partiql.planner.internal import org.partiql.planner.Header import org.partiql.planner.PartiQLPlanner import org.partiql.planner.internal.ir.Agg +import org.partiql.planner.internal.ir.Catalog import org.partiql.planner.internal.ir.Fn -import org.partiql.planner.internal.ir.Global import org.partiql.planner.internal.ir.Identifier import org.partiql.planner.internal.ir.Rel import org.partiql.planner.internal.ir.Rex -import org.partiql.planner.internal.ir.global -import org.partiql.planner.internal.ir.identifierQualified import org.partiql.planner.internal.ir.identifierSymbol import org.partiql.planner.internal.typer.FnResolver import org.partiql.spi.BindingCase @@ -95,13 +93,15 @@ internal sealed interface ResolvedVar { * Metadata for a resolved global variable * * @property type Resolved StaticType - * @property ordinal Index offset in the environment `globals` list + * @property ordinal The relevant catalog's index offset in the [Env.catalogs] list * @property depth The depth/level of the path match. + * @property position The relevant value's index offset in the [Catalog.values] list */ class Global( override val type: StaticType, override val ordinal: Int, val depth: Int, + val position: Int ) : ResolvedVar } @@ -134,7 +134,7 @@ internal class Env( /** * Collect the list of all referenced globals during planning. */ - public val globals = mutableListOf() + public val catalogs = mutableListOf() /** * Encapsulate all function resolving logic within [FnResolver]. @@ -149,7 +149,7 @@ internal class Env( /** * Map of catalog names to its underlying connector */ - private val catalogs: Map + private val connectors: Map // Initialize connectors init { @@ -163,7 +163,7 @@ internal class Env( // initialize connector with given config catalogs[catalog] = connector.create(catalog, config) } - this.catalogs = catalogs.toMap() + this.connectors = catalogs.toMap() } /** @@ -209,8 +209,8 @@ internal class Env( * @return */ private fun getMetadata(catalogName: BindingName): Handle? { - val catalogKey = catalogs.keys.firstOrNull { catalogName.isEquivalentTo(it) } ?: return null - val connector = catalogs[catalogKey] ?: return null + val catalogKey = connectors.keys.firstOrNull { catalogName.isEquivalentTo(it) } ?: return null + val connector = connectors[catalogKey] ?: return null val metadata = connector.getMetadata(connectorSession) return catalogKey to metadata } @@ -232,15 +232,47 @@ internal class Env( getObjectHandle(cat, catalogPath)?.let { handle -> getObjectDescriptor(handle).let { type -> val depth = calculateMatched(originalPath, catalogPath, handle.second.absolutePath) - val qualifiedPath = identifierQualified( - root = handle.first.toIdentifier(), - steps = handle.second.absolutePath.steps.map { it.toIdentifier() } - ) - val global = global(qualifiedPath, type) - globals.add(global) + val (catalogIndex, valueIndex) = getOrAddCatalogValue(handle.first, handle.second.absolutePath.steps, type) // Return resolution metadata - ResolvedVar.Global(type, globals.size - 1, depth) + ResolvedVar.Global(type, catalogIndex, depth, valueIndex) + } + } + } + } + + /** + * @return a [Pair] where [Pair.first] is the catalog index and [Pair.second] is the value index within that catalog + */ + private fun getOrAddCatalogValue(catalogName: String, valuePath: List, valueType: StaticType): Pair { + val catalogIndex = getOrAddCatalog(catalogName) + val values = catalogs[catalogIndex].values + return values.indexOfFirst { value -> + value.path == valuePath + }.let { index -> + when (index) { + -1 -> { + catalogs[catalogIndex] = catalogs[catalogIndex].copy( + values = values + listOf(Catalog.Value(valuePath, valueType)) + ) + catalogIndex to 0 + } + else -> { + catalogIndex to index + } + } + } + } + + private fun getOrAddCatalog(catalogName: String): Int { + return catalogs.indexOfFirst { catalog -> + catalog.name == catalogName + }.let { + when (it) { + -1 -> { + catalogs.add(Catalog(catalogName, emptyList())) + catalogs.lastIndex } + else -> it } } } diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Nodes.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Nodes.kt index d357cfbb28..7f11c91416 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Nodes.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Nodes.kt @@ -4,9 +4,10 @@ package org.partiql.planner.internal.ir import org.partiql.planner.internal.ir.builder.AggResolvedBuilder import org.partiql.planner.internal.ir.builder.AggUnresolvedBuilder +import org.partiql.planner.internal.ir.builder.CatalogBuilder +import org.partiql.planner.internal.ir.builder.CatalogValueBuilder import org.partiql.planner.internal.ir.builder.FnResolvedBuilder import org.partiql.planner.internal.ir.builder.FnUnresolvedBuilder -import org.partiql.planner.internal.ir.builder.GlobalBuilder import org.partiql.planner.internal.ir.builder.IdentifierQualifiedBuilder import org.partiql.planner.internal.ir.builder.IdentifierSymbolBuilder import org.partiql.planner.internal.ir.builder.PartiQlPlanBuilder @@ -80,13 +81,13 @@ internal data class PartiQLPlan( @JvmField internal val version: PartiQLVersion, @JvmField - internal val globals: List, + internal val catalogs: List, @JvmField internal val statement: Statement, ) : PlanNode() { internal override val children: List by lazy { val kids = mutableListOf() - kids.addAll(globals) + kids.addAll(catalogs) kids.add(statement) kids.filterNotNull() } @@ -100,24 +101,41 @@ internal data class PartiQLPlan( } } -internal data class Global( +internal data class Catalog( @JvmField - internal val path: Identifier.Qualified, + public val name: String, @JvmField - internal val type: StaticType, + public val values: List, ) : PlanNode() { - internal override val children: List by lazy { + public override val children: List by lazy { val kids = mutableListOf() - kids.add(path) + kids.addAll(values) kids.filterNotNull() } - internal override fun accept(visitor: PlanVisitor, ctx: C): R = - visitor.visitGlobal(this, ctx) + public override fun accept(visitor: PlanVisitor, ctx: C): R = + visitor.visitCatalog(this, ctx) - internal companion object { + public data class Value( + @JvmField + public val path: List, + @JvmField + public val type: StaticType, + ) : PlanNode() { + public override val children: List = emptyList() + + public override fun accept(visitor: PlanVisitor, ctx: C): R = + visitor.visitCatalogValue(this, ctx) + + public companion object { + @JvmStatic + public fun builder(): CatalogValueBuilder = CatalogValueBuilder() + } + } + + public companion object { @JvmStatic - internal fun builder(): GlobalBuilder = GlobalBuilder() + public fun builder(): CatalogBuilder = CatalogBuilder() } } @@ -380,16 +398,18 @@ internal data class Rex( internal data class Global( @JvmField - internal val ref: Int, + public val catalogRef: Int, + @JvmField + public val valueRef: Int, ) : Op() { - internal override val children: List = emptyList() + public override val children: List = emptyList() - internal override fun accept(visitor: PlanVisitor, ctx: C): R = + public override fun accept(visitor: PlanVisitor, ctx: C): R = visitor.visitRexOpGlobal(this, ctx) - internal companion object { + public companion object { @JvmStatic - internal fun builder(): RexOpGlobalBuilder = RexOpGlobalBuilder() + public fun builder(): RexOpGlobalBuilder = RexOpGlobalBuilder() } } diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Plan.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Plan.kt index ddd3b9547d..538595c644 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Plan.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Plan.kt @@ -10,11 +10,16 @@ import org.partiql.value.PartiQLValueExperimental internal fun partiQLPlan( version: PartiQLVersion, - globals: List, + catalogs: List, statement: Statement, -): PartiQLPlan = PartiQLPlan(version, globals, statement) +): PartiQLPlan = PartiQLPlan(version, catalogs, statement) -internal fun global(path: Identifier.Qualified, type: StaticType): Global = Global(path, type) +internal fun catalog(name: String, values: List): Catalog = Catalog(name, values) + +internal fun catalogValue(path: List, type: StaticType): Catalog.Value = Catalog.Value( + path, + type +) internal fun fnResolved(signature: FunctionSignature.Scalar): Fn.Resolved = Fn.Resolved(signature) @@ -44,7 +49,10 @@ internal fun rexOpVarResolved(ref: Int): Rex.Op.Var.Resolved = Rex.Op.Var.Resolv internal fun rexOpVarUnresolved(identifier: Identifier, scope: Rex.Op.Var.Scope): Rex.Op.Var.Unresolved = Rex.Op.Var.Unresolved(identifier, scope) -internal fun rexOpGlobal(ref: Int): Rex.Op.Global = Rex.Op.Global(ref) +internal fun rexOpGlobal(catalogRef: Int, valueRef: Int): Rex.Op.Global = Rex.Op.Global( + catalogRef, + valueRef +) internal fun rexOpPath(root: Rex, steps: List): Rex.Op.Path = Rex.Op.Path( root, diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/builder/PlanBuilder.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/builder/PlanBuilder.kt index 8ea53d5c5f..053222690e 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/builder/PlanBuilder.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/builder/PlanBuilder.kt @@ -3,8 +3,8 @@ package org.partiql.planner.internal.ir.builder import org.partiql.planner.internal.ir.Agg +import org.partiql.planner.internal.ir.Catalog import org.partiql.planner.internal.ir.Fn -import org.partiql.planner.internal.ir.Global import org.partiql.planner.internal.ir.Identifier import org.partiql.planner.internal.ir.PartiQLPlan import org.partiql.planner.internal.ir.PartiQLVersion @@ -22,21 +22,31 @@ internal fun plan(block: PlanBuilder.() -> T) = PlanBuilder().blo internal class PlanBuilder { internal fun partiQLPlan( version: PartiQLVersion? = null, - globals: MutableList = mutableListOf(), + catalogs: MutableList = mutableListOf(), statement: Statement? = null, block: PartiQlPlanBuilder.() -> Unit = {}, ): PartiQLPlan { - val builder = PartiQlPlanBuilder(version, globals, statement) + val builder = PartiQlPlanBuilder(version, catalogs, statement) builder.block() return builder.build() } - internal fun global( - path: Identifier.Qualified? = null, + public fun catalog( + name: String? = null, + values: MutableList = mutableListOf(), + block: CatalogBuilder.() -> Unit = {}, + ): Catalog { + val builder = CatalogBuilder(name, values) + builder.block() + return builder.build() + } + + public fun catalogValue( + path: MutableList = mutableListOf(), type: StaticType? = null, - block: GlobalBuilder.() -> Unit = {}, - ): Global { - val builder = GlobalBuilder(path, type) + block: CatalogValueBuilder.() -> Unit = {}, + ): Catalog.Value { + val builder = CatalogValueBuilder(path, type) builder.block() return builder.build() } @@ -140,8 +150,12 @@ internal class PlanBuilder { return builder.build() } - internal fun rexOpGlobal(ref: Int? = null, block: RexOpGlobalBuilder.() -> Unit = {}): Rex.Op.Global { - val builder = RexOpGlobalBuilder(ref) + public fun rexOpGlobal( + catalogRef: Int? = null, + valueRef: Int? = null, + block: RexOpGlobalBuilder.() -> Unit = {}, + ): Rex.Op.Global { + val builder = RexOpGlobalBuilder(catalogRef, valueRef) builder.block() return builder.build() } diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/builder/PlanBuilders.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/builder/PlanBuilders.kt index 8f4cf3197d..f0bf8a9726 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/builder/PlanBuilders.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/builder/PlanBuilders.kt @@ -3,8 +3,8 @@ package org.partiql.planner.internal.ir.builder import org.partiql.planner.internal.ir.Agg +import org.partiql.planner.internal.ir.Catalog import org.partiql.planner.internal.ir.Fn -import org.partiql.planner.internal.ir.Global import org.partiql.planner.internal.ir.Identifier import org.partiql.planner.internal.ir.PartiQLPlan import org.partiql.planner.internal.ir.PartiQLVersion @@ -18,15 +18,15 @@ import org.partiql.value.PartiQLValueExperimental internal class PartiQlPlanBuilder( internal var version: PartiQLVersion? = null, - internal var globals: MutableList = mutableListOf(), + internal var catalogs: MutableList = mutableListOf(), internal var statement: Statement? = null, ) { internal fun version(version: PartiQLVersion?): PartiQlPlanBuilder = this.apply { this.version = version } - internal fun globals(globals: MutableList): PartiQlPlanBuilder = this.apply { - this.globals = globals + internal fun catalogs(catalogs: MutableList): PartiQlPlanBuilder = this.apply { + this.catalogs = catalogs } internal fun statement(statement: Statement?): PartiQlPlanBuilder = this.apply { @@ -34,25 +34,40 @@ internal class PartiQlPlanBuilder( } internal fun build(): PartiQLPlan = PartiQLPlan( - version = version!!, globals = globals, + version = version!!, catalogs = catalogs, statement = statement!! ) } -internal class GlobalBuilder( - internal var path: Identifier.Qualified? = null, - internal var type: StaticType? = null, +internal class CatalogBuilder( + public var name: String? = null, + public var values: MutableList = mutableListOf(), ) { - internal fun path(path: Identifier.Qualified?): GlobalBuilder = this.apply { + public fun name(name: String?): CatalogBuilder = this.apply { + this.name = name + } + + public fun values(values: MutableList): CatalogBuilder = this.apply { + this.values = values + } + + public fun build(): Catalog = Catalog(name = name!!, values = values) +} + +internal class CatalogValueBuilder( + public var path: MutableList = mutableListOf(), + public var type: StaticType? = null, +) { + public fun path(path: MutableList): CatalogValueBuilder = this.apply { this.path = path } - internal fun type(type: StaticType?): GlobalBuilder = this.apply { + public fun type(type: StaticType?): CatalogValueBuilder = this.apply { this.type = type } - internal fun build(): Global = Global(path = path!!, type = type!!) + public fun build(): Catalog.Value = Catalog.Value(path = path, type = type!!) } internal class FnResolvedBuilder( @@ -205,13 +220,22 @@ internal class RexOpVarUnresolvedBuilder( } internal class RexOpGlobalBuilder( - internal var ref: Int? = null, + public var catalogRef: Int? = null, + public var valueRef: Int? = null, ) { - internal fun ref(ref: Int?): RexOpGlobalBuilder = this.apply { - this.ref = ref + public fun catalogRef(catalogRef: Int?): RexOpGlobalBuilder = this.apply { + this.catalogRef = catalogRef } - internal fun build(): Rex.Op.Global = Rex.Op.Global(ref = ref!!) + public fun valueRef(valueRef: Int?): RexOpGlobalBuilder = this.apply { + this.valueRef = valueRef + } + + public fun build(): Rex.Op.Global = Rex.Op.Global( + catalogRef = catalogRef!!, + valueRef = + valueRef!! + ) } internal class RexOpPathBuilder( diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/util/PlanRewriter.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/util/PlanRewriter.kt index 0aae2cd0da..493ddedadf 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/util/PlanRewriter.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/util/PlanRewriter.kt @@ -6,8 +6,8 @@ package org.partiql.planner.internal.ir.util import org.partiql.planner.internal.ir.Agg +import org.partiql.planner.internal.ir.Catalog import org.partiql.planner.internal.ir.Fn -import org.partiql.planner.internal.ir.Global import org.partiql.planner.internal.ir.Identifier import org.partiql.planner.internal.ir.PartiQLPlan import org.partiql.planner.internal.ir.PlanNode @@ -87,25 +87,31 @@ internal abstract class PlanRewriter : PlanBaseVisitor() { override fun visitPartiQLPlan(node: PartiQLPlan, ctx: C): PlanNode { val version = node.version - val globals = _visitList(node.globals, ctx, ::visitGlobal) + val globals = _visitList(node.catalogs, ctx, ::visitCatalog) val statement = visitStatement(node.statement, ctx) as Statement - return if (version !== node.version || globals !== node.globals || statement !== node.statement) { + return if (version !== node.version || globals !== node.catalogs || statement !== node.statement) { PartiQLPlan(version, globals, statement) } else { node } } - override fun visitGlobal(node: Global, ctx: C): PlanNode { - val path = visitIdentifierQualified(node.path, ctx) as Identifier.Qualified - val type = node.type - return if (path !== node.path || type !== node.type) { - Global(path, type) + public override fun visitCatalog(node: Catalog, ctx: C): PlanNode { + val name = node.name + val values = _visitList(node.values, ctx, ::visitCatalogValue) + return if (name !== node.name || values !== node.values) { + Catalog(name, values) } else { node } } + public override fun visitCatalogValue(node: Catalog.Value, ctx: C): PlanNode { + val path = node.path + val type = node.type + return node + } + override fun visitFnResolved(node: Fn.Resolved, ctx: C): PlanNode { val signature = node.signature return node @@ -191,8 +197,9 @@ internal abstract class PlanRewriter : PlanBaseVisitor() { } } - override fun visitRexOpGlobal(node: Rex.Op.Global, ctx: C): PlanNode { - val ref = node.ref + public override fun visitRexOpGlobal(node: Rex.Op.Global, ctx: C): PlanNode { + val catalogRef = node.catalogRef + val valueRef = node.valueRef return node } diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/visitor/PlanBaseVisitor.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/visitor/PlanBaseVisitor.kt index afe8dac281..fe97f33efd 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/visitor/PlanBaseVisitor.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/visitor/PlanBaseVisitor.kt @@ -3,8 +3,8 @@ package org.partiql.planner.internal.ir.visitor import org.partiql.planner.internal.ir.Agg +import org.partiql.planner.internal.ir.Catalog import org.partiql.planner.internal.ir.Fn -import org.partiql.planner.internal.ir.Global import org.partiql.planner.internal.ir.Identifier import org.partiql.planner.internal.ir.PartiQLPlan import org.partiql.planner.internal.ir.PlanNode @@ -18,7 +18,9 @@ internal abstract class PlanBaseVisitor : PlanVisitor { override fun visitPartiQLPlan(node: PartiQLPlan, ctx: C): R = defaultVisit(node, ctx) - override fun visitGlobal(node: Global, ctx: C): R = defaultVisit(node, ctx) + public override fun visitCatalog(node: Catalog, ctx: C): R = defaultVisit(node, ctx) + + public override fun visitCatalogValue(node: Catalog.Value, ctx: C): R = defaultVisit(node, ctx) override fun visitFn(node: Fn, ctx: C): R = when (node) { is Fn.Resolved -> visitFnResolved(node, ctx) diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/visitor/PlanVisitor.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/visitor/PlanVisitor.kt index f3114e780c..c665579e52 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/visitor/PlanVisitor.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/visitor/PlanVisitor.kt @@ -3,8 +3,8 @@ package org.partiql.planner.internal.ir.visitor import org.partiql.planner.internal.ir.Agg +import org.partiql.planner.internal.ir.Catalog import org.partiql.planner.internal.ir.Fn -import org.partiql.planner.internal.ir.Global import org.partiql.planner.internal.ir.Identifier import org.partiql.planner.internal.ir.PartiQLPlan import org.partiql.planner.internal.ir.PlanNode @@ -18,7 +18,9 @@ internal interface PlanVisitor { fun visitPartiQLPlan(node: PartiQLPlan, ctx: C): R - fun visitGlobal(node: Global, ctx: C): R + public fun visitCatalog(node: Catalog, ctx: C): R + + public fun visitCatalogValue(node: Catalog.Value, ctx: C): R fun visitFn(node: Fn, ctx: C): R diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt index b52698db6b..95f3e426d0 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt @@ -8,8 +8,8 @@ import org.partiql.plan.rexOpLit import org.partiql.plan.rexOpPathStepKey import org.partiql.plan.rexOpPathStepSymbol import org.partiql.planner.internal.ir.Agg +import org.partiql.planner.internal.ir.Catalog import org.partiql.planner.internal.ir.Fn -import org.partiql.planner.internal.ir.Global import org.partiql.planner.internal.ir.Identifier import org.partiql.planner.internal.ir.PartiQLPlan import org.partiql.planner.internal.ir.Rel @@ -35,15 +35,18 @@ internal object PlanTransform : PlanBaseVisitor() { } override fun visitPartiQLPlan(node: PartiQLPlan, ctx: ProblemCallback): org.partiql.plan.PartiQLPlan { - val globals = node.globals.map { visitGlobal(it, ctx) } + val catalogs = node.catalogs.map { visitCatalog(it, ctx) } val statement = visitStatement(node.statement, ctx) - return partiQLPlan(globals, statement) + return partiQLPlan(catalogs, statement) } - override fun visitGlobal(node: Global, ctx: ProblemCallback): org.partiql.plan.Global { - val path = visitIdentifierQualified(node.path, ctx) - val type = node.type - return org.partiql.plan.global(path, type) + override fun visitCatalog(node: Catalog, ctx: ProblemCallback): org.partiql.plan.Catalog { + val values = node.values.map { visitCatalogValue(it, ctx) } + return org.partiql.plan.Catalog(node.name, values) + } + + override fun visitCatalogValue(node: Catalog.Value, ctx: ProblemCallback): org.partiql.plan.Catalog.Value { + return org.partiql.plan.Catalog.Value(node.path, node.type) } override fun visitFnResolved(node: Fn.Resolved, ctx: ProblemCallback) = org.partiql.plan.fn(node.signature) @@ -108,7 +111,10 @@ internal object PlanTransform : PlanBaseVisitor() { override fun visitRexOpVarUnresolved(node: Rex.Op.Var.Unresolved, ctx: ProblemCallback) = org.partiql.plan.Rex.Op.Err("Unresolved variable $node") - override fun visitRexOpGlobal(node: Rex.Op.Global, ctx: ProblemCallback) = org.partiql.plan.Rex.Op.Global(node.ref) + override fun visitRexOpGlobal(node: Rex.Op.Global, ctx: ProblemCallback) = org.partiql.plan.Rex.Op.Global( + node.catalogRef, + node.valueRef + ) override fun visitRexOpPath(node: Rex.Op.Path, ctx: ProblemCallback): org.partiql.plan.Rex.Op.Path { val root = visitRex(node.root, ctx) diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt index c6280900f0..9c971c805b 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt @@ -84,10 +84,7 @@ import org.partiql.types.NullType import org.partiql.types.SexpType import org.partiql.types.StaticType import org.partiql.types.StaticType.Companion.ANY -import org.partiql.types.StaticType.Companion.BOOL import org.partiql.types.StaticType.Companion.MISSING -import org.partiql.types.StaticType.Companion.NULL -import org.partiql.types.StaticType.Companion.STRING import org.partiql.types.StringType import org.partiql.types.StructType import org.partiql.types.TupleConstraint @@ -423,15 +420,15 @@ internal class PlanTyper( } val type = resolvedVar.type val op = when (resolvedVar) { - is ResolvedVar.Global -> rexOpGlobal(resolvedVar.ordinal) + is ResolvedVar.Global -> rexOpGlobal(resolvedVar.ordinal, resolvedVar.position) is ResolvedVar.Local -> resolvedLocalPath(resolvedVar) } return rex(type, op) } override fun visitRexOpGlobal(node: Rex.Op.Global, ctx: StaticType?): Rex { - val global = env.globals[node.ref] - val type = global.type + val catalog = env.catalogs[node.catalogRef] + val type = catalog.values[node.valueRef].type return rex(type, node) } @@ -466,7 +463,7 @@ internal class PlanTyper( true -> emptyList() false -> visitedSteps.subList(remainingFirstIndex, visitedSteps.size) } - rexOpGlobal(resolvedVar.ordinal) to remaining + rexOpGlobal(resolvedVar.ordinal, resolvedVar.position) to remaining } } // rewrite root diff --git a/partiql-planner/src/test/kotlin/org/partiql/planner/internal/EnvTest.kt b/partiql-planner/src/test/kotlin/org/partiql/planner/internal/EnvTest.kt index 725f0f481f..81e9ae35f8 100644 --- a/partiql-planner/src/test/kotlin/org/partiql/planner/internal/EnvTest.kt +++ b/partiql-planner/src/test/kotlin/org/partiql/planner/internal/EnvTest.kt @@ -8,11 +8,8 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.partiql.planner.PartiQLHeader import org.partiql.planner.PartiQLPlanner -import org.partiql.planner.internal.ir.Global -import org.partiql.planner.internal.ir.Identifier +import org.partiql.planner.internal.ir.Catalog import org.partiql.planner.internal.ir.Rex -import org.partiql.planner.internal.ir.identifierQualified -import org.partiql.planner.internal.ir.identifierSymbol import org.partiql.plugins.local.LocalPlugin import org.partiql.spi.BindingCase import org.partiql.spi.BindingName @@ -38,15 +35,11 @@ class EnvTest { private val EMPTY_TYPE_ENV = TypeEnv(schema = emptyList(), ResolutionStrategy.GLOBAL) - private val GLOBAL_OS = Global( - path = identifierQualified( - root = identifierSymbol("pql", Identifier.CaseSensitivity.SENSITIVE), - steps = listOf( - identifierSymbol("main", Identifier.CaseSensitivity.SENSITIVE), - identifierSymbol("os", Identifier.CaseSensitivity.SENSITIVE) - ) - ), - type = StaticType.STRING + private val GLOBAL_OS = Catalog( + name = "pql", + values = listOf( + Catalog.Value(path = listOf("main", "os"), type = StaticType.STRING) + ) ) } @@ -71,29 +64,29 @@ class EnvTest { fun testGlobalMatchingSensitiveName() { val path = BindingPath(listOf(BindingName("os", BindingCase.SENSITIVE))) assertNotNull(env.resolve(path, EMPTY_TYPE_ENV, Rex.Op.Var.Scope.DEFAULT)) - assertEquals(1, env.globals.size) - assert(env.globals.contains(GLOBAL_OS)) + assertEquals(1, env.catalogs.size) + assert(env.catalogs.contains(GLOBAL_OS)) } @Test fun testGlobalMatchingInsensitiveName() { val path = BindingPath(listOf(BindingName("oS", BindingCase.INSENSITIVE))) assertNotNull(env.resolve(path, EMPTY_TYPE_ENV, Rex.Op.Var.Scope.DEFAULT)) - assertEquals(1, env.globals.size) - assert(env.globals.contains(GLOBAL_OS)) + assertEquals(1, env.catalogs.size) + assert(env.catalogs.contains(GLOBAL_OS)) } @Test fun testGlobalNotMatchingSensitiveName() { val path = BindingPath(listOf(BindingName("oS", BindingCase.SENSITIVE))) assertNull(env.resolve(path, EMPTY_TYPE_ENV, Rex.Op.Var.Scope.DEFAULT)) - assert(env.globals.isEmpty()) + assert(env.catalogs.isEmpty()) } @Test fun testGlobalNotMatchingInsensitiveName() { val path = BindingPath(listOf(BindingName("nonexistent", BindingCase.INSENSITIVE))) assertNull(env.resolve(path, EMPTY_TYPE_ENV, Rex.Op.Var.Scope.DEFAULT)) - assert(env.globals.isEmpty()) + assert(env.catalogs.isEmpty()) } } diff --git a/partiql-planner/src/test/kotlin/org/partiql/planner/internal/typer/PlanTyperTest.kt b/partiql-planner/src/test/kotlin/org/partiql/planner/internal/typer/PlanTyperTest.kt index 3e7394afb1..b284cfb962 100644 --- a/partiql-planner/src/test/kotlin/org/partiql/planner/internal/typer/PlanTyperTest.kt +++ b/partiql-planner/src/test/kotlin/org/partiql/planner/internal/typer/PlanTyperTest.kt @@ -258,7 +258,7 @@ class PlanTyperTest { op = rexOpPath( root = rex( ORDERED_DUPLICATES_STRUCT, - rexOpGlobal(0) + rexOpGlobal(0, 0) ), steps = listOf( rexOpPathStepSymbol(identifierSymbol("definition", Identifier.CaseSensitivity.SENSITIVE)), @@ -297,7 +297,7 @@ class PlanTyperTest { op = rexOpPath( root = rex( ORDERED_DUPLICATES_STRUCT, - rexOpGlobal(0) + rexOpGlobal(0, 0) ), steps = listOf( rexOpPathStepSymbol(identifierSymbol("DEFINITION", Identifier.CaseSensitivity.SENSITIVE)), @@ -336,7 +336,7 @@ class PlanTyperTest { op = rexOpPath( root = rex( DUPLICATES_STRUCT, - rexOpGlobal(0) + rexOpGlobal(0, 0) ), steps = listOf( rexOpPathStepSymbol(identifierSymbol("DEFINITION", Identifier.CaseSensitivity.INSENSITIVE)), @@ -375,7 +375,7 @@ class PlanTyperTest { op = rexOpPath( root = rex( DUPLICATES_STRUCT, - rexOpGlobal(0) + rexOpGlobal(0, 0) ), steps = listOf( rexOpPathStepSymbol(identifierSymbol("DEFINITION", Identifier.CaseSensitivity.SENSITIVE)), @@ -414,7 +414,7 @@ class PlanTyperTest { op = rexOpPath( root = rex( DUPLICATES_STRUCT, - rexOpGlobal(0) + rexOpGlobal(0, 0) ), steps = listOf( rexOpPathStepSymbol(identifierSymbol("definition", Identifier.CaseSensitivity.SENSITIVE)), @@ -453,7 +453,7 @@ class PlanTyperTest { op = rexOpPath( root = rex( OPEN_DUPLICATES_STRUCT, - rexOpGlobal(0) + rexOpGlobal(0, 0) ), steps = listOf( rexOpPathStepSymbol(identifierSymbol("definition", Identifier.CaseSensitivity.SENSITIVE)), @@ -492,7 +492,7 @@ class PlanTyperTest { op = rexOpPath( root = rex( CLOSED_UNION_DUPLICATES_STRUCT, - rexOpGlobal(0) + rexOpGlobal(0, 0) ), steps = listOf( rexOpPathStepSymbol(identifierSymbol("definition", Identifier.CaseSensitivity.INSENSITIVE)), @@ -531,7 +531,7 @@ class PlanTyperTest { op = rexOpPath( root = rex( CLOSED_UNION_DUPLICATES_STRUCT, - rexOpGlobal(0) + rexOpGlobal(0, 0) ), steps = listOf( rexOpPathStepSymbol(identifierSymbol("definition", Identifier.CaseSensitivity.SENSITIVE)), From 0d72dfb85bbb5751d8471418ab35f69fe5ae5631 Mon Sep 17 00:00:00 2001 From: John Ed Quinn Date: Thu, 14 Dec 2023 14:40:10 -0800 Subject: [PATCH 2/6] Updates CHANGELOG --- CHANGELOG.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 100d6337f4..b47ed8cbf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,13 +40,14 @@ Thank you to all who have contributed! ### Changed - StaticTypeInferencer and PlanTyper will not raise an error when an expression is inferred to `NULL` or `unionOf(NULL, MISSING)`. In these cases the StaticTypeInferencer and PlanTyper will still raise the Problem Code `ExpressionAlwaysReturnsNullOrMissing` but the severity of the problem has been changed to warning. In the case an expression always returns `MISSING`, problem code `ExpressionAlwaysReturnsMissing` will be raised, which will have problem severity of error. -- **Breaking** The default integer literal type is now 32-bit; if the literal can not fit in a 32-bit integer, it overflows to 64-bit. -- **BREAKING** `PartiQLValueType` now distinguishes between Arbitrary Precision Decimal and Fixed Precision Decimal. -- **BREAKING** Function Signature Changes. Now Function signature has two subclasses, `Scalar` and `Aggregation`. -- **BREAKING** In the produced plan: +- **BREAKING**: The default integer literal type is now 32-bit; if the literal can not fit in a 32-bit integer, it overflows to 64-bit. +- **BREAKING**: `PartiQLValueType` now distinguishes between Arbitrary Precision Decimal and Fixed Precision Decimal. +- **BREAKING**: Function Signature Changes. Now Function signature has two subclasses, `Scalar` and `Aggregation`. +- **BREAKING**: In the produced plan: - The new plan is fully resolved and typed. - Operators will be converted to function call. - Changes the return type of `filter_distinct` to a list if input collection is list +- **BREAKING**: Globals within the logical plan are now represented in a structured manner to reflect the existence of catalogs. Each value is now mandated to be case-sensitive (we have moved from Identifiers to string literals). ### Deprecated From b6137a3b69bac512cbf7203d8631d38ca4c34b2a Mon Sep 17 00:00:00 2001 From: John Ed Quinn Date: Fri, 15 Dec 2023 11:10:10 -0800 Subject: [PATCH 3/6] Updates naming of value to symbol --- .../src/main/resources/partiql_plan.ion | 23 +++++++++++++------ .../internal/transforms/PlanTransform.kt | 7 +++--- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/partiql-plan/src/main/resources/partiql_plan.ion b/partiql-plan/src/main/resources/partiql_plan.ion index 15ff3cbbe9..198d548f72 100644 --- a/partiql-plan/src/main/resources/partiql_plan.ion +++ b/partiql-plan/src/main/resources/partiql_plan.ion @@ -13,15 +13,25 @@ parti_q_l_plan::{ } // Represent an instance of a database. -// - Currently, `values` represents all values from this catalog to be used in this plan. +// - Currently, `symbols` represents all values from this catalog to be used in this plan. // - Eventually, TODO functions may be resolved to a specific namespace within a catalog. catalog::{ name: string, - values: list::[value], + symbols: list::[symbol], _: [ - value::{ - path: list::[string], // This should NOT be empty. - type: static_type + // A reference to a value contained within a catalog. + symbol::{ + // The path to a value WITHIN a catalog. Note: This should not start with the catalog's name. Also, this + // should not be empty + path: list::[string], + type: static_type, + _: [ + // A reference to a symbol + ref::{ + catalog: int, + symbol: int + } + ] } ] } @@ -77,8 +87,7 @@ rex::{ }, global::{ - catalog_ref: int, // The index of the relevant catalog in the plan's catalogs list - value_ref: Int // The index of the relevant value in a catalog's values list + ref: '.catalog.symbol.ref' }, path::{ diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt index 95f3e426d0..ac98212df7 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt @@ -45,8 +45,8 @@ internal object PlanTransform : PlanBaseVisitor() { return org.partiql.plan.Catalog(node.name, values) } - override fun visitCatalogValue(node: Catalog.Value, ctx: ProblemCallback): org.partiql.plan.Catalog.Value { - return org.partiql.plan.Catalog.Value(node.path, node.type) + override fun visitCatalogValue(node: Catalog.Value, ctx: ProblemCallback): org.partiql.plan.Catalog.Symbol { + return org.partiql.plan.Catalog.Symbol(node.path, node.type) } override fun visitFnResolved(node: Fn.Resolved, ctx: ProblemCallback) = org.partiql.plan.fn(node.signature) @@ -112,8 +112,7 @@ internal object PlanTransform : PlanBaseVisitor() { org.partiql.plan.Rex.Op.Err("Unresolved variable $node") override fun visitRexOpGlobal(node: Rex.Op.Global, ctx: ProblemCallback) = org.partiql.plan.Rex.Op.Global( - node.catalogRef, - node.valueRef + ref = org.partiql.plan.Catalog.Symbol.Ref(catalog = node.catalogRef, symbol = node.valueRef) ) override fun visitRexOpPath(node: Rex.Op.Path, ctx: ProblemCallback): org.partiql.plan.Rex.Op.Path { From f31fffef687c7cc3222f26cba2138ebb693079e1 Mon Sep 17 00:00:00 2001 From: John Ed Quinn Date: Fri, 15 Dec 2023 11:56:57 -0800 Subject: [PATCH 4/6] Updates internal IRs to match public API --- .../org/partiql/planner/internal/Env.kt | 6 +- .../org/partiql/planner/internal/ir/Nodes.kt | 66 ++++++++++++------- .../org/partiql/planner/internal/ir/Plan.kt | 16 ++--- .../internal/ir/builder/PlanBuilder.kt | 33 ++++++---- .../internal/ir/builder/PlanBuilders.kt | 60 ++++++++++------- .../planner/internal/ir/util/PlanRewriter.kt | 27 +++++--- .../internal/ir/visitor/PlanBaseVisitor.kt | 5 +- .../internal/ir/visitor/PlanVisitor.kt | 4 +- .../internal/transforms/PlanTransform.kt | 12 ++-- .../planner/internal/typer/PlanTyper.kt | 9 +-- .../org/partiql/planner/internal/EnvTest.kt | 4 +- .../planner/internal/typer/PlanTyperTest.kt | 5 ++ 12 files changed, 154 insertions(+), 93 deletions(-) diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/Env.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/Env.kt index 988758c5b0..f8d09a01c0 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/Env.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/Env.kt @@ -245,14 +245,14 @@ internal class Env( */ private fun getOrAddCatalogValue(catalogName: String, valuePath: List, valueType: StaticType): Pair { val catalogIndex = getOrAddCatalog(catalogName) - val values = catalogs[catalogIndex].values - return values.indexOfFirst { value -> + val symbols = catalogs[catalogIndex].symbols + return symbols.indexOfFirst { value -> value.path == valuePath }.let { index -> when (index) { -1 -> { catalogs[catalogIndex] = catalogs[catalogIndex].copy( - values = values + listOf(Catalog.Value(valuePath, valueType)) + symbols = symbols + listOf(Catalog.Symbol(valuePath, valueType)) ) catalogIndex to 0 } diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Nodes.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Nodes.kt index 7f11c91416..75e4fe4532 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Nodes.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Nodes.kt @@ -5,7 +5,8 @@ package org.partiql.planner.internal.ir import org.partiql.planner.internal.ir.builder.AggResolvedBuilder import org.partiql.planner.internal.ir.builder.AggUnresolvedBuilder import org.partiql.planner.internal.ir.builder.CatalogBuilder -import org.partiql.planner.internal.ir.builder.CatalogValueBuilder +import org.partiql.planner.internal.ir.builder.CatalogSymbolBuilder +import org.partiql.planner.internal.ir.builder.CatalogSymbolRefBuilder import org.partiql.planner.internal.ir.builder.FnResolvedBuilder import org.partiql.planner.internal.ir.builder.FnUnresolvedBuilder import org.partiql.planner.internal.ir.builder.IdentifierQualifiedBuilder @@ -103,39 +104,56 @@ internal data class PartiQLPlan( internal data class Catalog( @JvmField - public val name: String, + internal val name: String, @JvmField - public val values: List, + internal val symbols: List, ) : PlanNode() { - public override val children: List by lazy { + internal override val children: List by lazy { val kids = mutableListOf() - kids.addAll(values) + kids.addAll(symbols) kids.filterNotNull() } - public override fun accept(visitor: PlanVisitor, ctx: C): R = + internal override fun accept(visitor: PlanVisitor, ctx: C): R = visitor.visitCatalog(this, ctx) - public data class Value( + internal data class Symbol( @JvmField - public val path: List, + internal val path: List, @JvmField - public val type: StaticType, + internal val type: StaticType, ) : PlanNode() { - public override val children: List = emptyList() + internal override val children: List = emptyList() + + internal override fun accept(visitor: PlanVisitor, ctx: C): R = + visitor.visitCatalogSymbol(this, ctx) + + internal data class Ref( + @JvmField + internal val catalog: Int, + @JvmField + internal val symbol: Int, + ) : PlanNode() { + internal override val children: List = emptyList() + + internal override fun accept(visitor: PlanVisitor, ctx: C): R = + visitor.visitCatalogSymbolRef(this, ctx) - public override fun accept(visitor: PlanVisitor, ctx: C): R = - visitor.visitCatalogValue(this, ctx) + internal companion object { + @JvmStatic + internal fun builder(): CatalogSymbolRefBuilder = CatalogSymbolRefBuilder() + } + } - public companion object { + internal companion object { @JvmStatic - public fun builder(): CatalogValueBuilder = CatalogValueBuilder() + internal fun builder(): CatalogSymbolBuilder = CatalogSymbolBuilder() } } - public companion object { + internal companion object { @JvmStatic - public fun builder(): CatalogBuilder = CatalogBuilder() + internal fun builder(): CatalogBuilder = CatalogBuilder() } } @@ -398,18 +416,20 @@ internal data class Rex( internal data class Global( @JvmField - public val catalogRef: Int, - @JvmField - public val valueRef: Int, + internal val ref: Catalog.Symbol.Ref, ) : Op() { - public override val children: List = emptyList() + internal override val children: List by lazy { + val kids = mutableListOf() + kids.add(ref) + kids.filterNotNull() + } - public override fun accept(visitor: PlanVisitor, ctx: C): R = + internal override fun accept(visitor: PlanVisitor, ctx: C): R = visitor.visitRexOpGlobal(this, ctx) - public companion object { + internal companion object { @JvmStatic - public fun builder(): RexOpGlobalBuilder = RexOpGlobalBuilder() + internal fun builder(): RexOpGlobalBuilder = RexOpGlobalBuilder() } } diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Plan.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Plan.kt index 538595c644..50f374b782 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Plan.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Plan.kt @@ -14,12 +14,13 @@ internal fun partiQLPlan( statement: Statement, ): PartiQLPlan = PartiQLPlan(version, catalogs, statement) -internal fun catalog(name: String, values: List): Catalog = Catalog(name, values) +internal fun catalog(name: String, symbols: List): Catalog = Catalog(name, symbols) -internal fun catalogValue(path: List, type: StaticType): Catalog.Value = Catalog.Value( - path, - type -) +internal fun catalogSymbol(path: List, type: StaticType): Catalog.Symbol = + Catalog.Symbol(path, type) + +internal fun catalogSymbolRef(catalog: Int, symbol: Int): Catalog.Symbol.Ref = + Catalog.Symbol.Ref(catalog, symbol) internal fun fnResolved(signature: FunctionSignature.Scalar): Fn.Resolved = Fn.Resolved(signature) @@ -49,10 +50,7 @@ internal fun rexOpVarResolved(ref: Int): Rex.Op.Var.Resolved = Rex.Op.Var.Resolv internal fun rexOpVarUnresolved(identifier: Identifier, scope: Rex.Op.Var.Scope): Rex.Op.Var.Unresolved = Rex.Op.Var.Unresolved(identifier, scope) -internal fun rexOpGlobal(catalogRef: Int, valueRef: Int): Rex.Op.Global = Rex.Op.Global( - catalogRef, - valueRef -) +internal fun rexOpGlobal(ref: Catalog.Symbol.Ref): Rex.Op.Global = Rex.Op.Global(ref) internal fun rexOpPath(root: Rex, steps: List): Rex.Op.Path = Rex.Op.Path( root, diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/builder/PlanBuilder.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/builder/PlanBuilder.kt index 053222690e..79f8871aba 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/builder/PlanBuilder.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/builder/PlanBuilder.kt @@ -31,22 +31,32 @@ internal class PlanBuilder { return builder.build() } - public fun catalog( + internal fun catalog( name: String? = null, - values: MutableList = mutableListOf(), + symbols: MutableList = mutableListOf(), block: CatalogBuilder.() -> Unit = {}, ): Catalog { - val builder = CatalogBuilder(name, values) + val builder = CatalogBuilder(name, symbols) builder.block() return builder.build() } - public fun catalogValue( + internal fun catalogSymbol( path: MutableList = mutableListOf(), type: StaticType? = null, - block: CatalogValueBuilder.() -> Unit = {}, - ): Catalog.Value { - val builder = CatalogValueBuilder(path, type) + block: CatalogSymbolBuilder.() -> Unit = {}, + ): Catalog.Symbol { + val builder = CatalogSymbolBuilder(path, type) + builder.block() + return builder.build() + } + + internal fun catalogSymbolRef( + catalog: Int? = null, + symbol: Int? = null, + block: CatalogSymbolRefBuilder.() -> Unit = {}, + ): Catalog.Symbol.Ref { + val builder = CatalogSymbolRefBuilder(catalog, symbol) builder.block() return builder.build() } @@ -150,12 +160,11 @@ internal class PlanBuilder { return builder.build() } - public fun rexOpGlobal( - catalogRef: Int? = null, - valueRef: Int? = null, - block: RexOpGlobalBuilder.() -> Unit = {}, + internal fun rexOpGlobal( + ref: Catalog.Symbol.Ref? = null, + block: RexOpGlobalBuilder.() -> Unit = {} ): Rex.Op.Global { - val builder = RexOpGlobalBuilder(catalogRef, valueRef) + val builder = RexOpGlobalBuilder(ref) builder.block() return builder.build() } diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/builder/PlanBuilders.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/builder/PlanBuilders.kt index f0bf8a9726..b13aff6ce0 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/builder/PlanBuilders.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/builder/PlanBuilders.kt @@ -41,33 +41,52 @@ internal class PartiQlPlanBuilder( } internal class CatalogBuilder( - public var name: String? = null, - public var values: MutableList = mutableListOf(), + internal var name: String? = null, + internal var symbols: MutableList = mutableListOf(), ) { - public fun name(name: String?): CatalogBuilder = this.apply { + internal fun name(name: String?): CatalogBuilder = this.apply { this.name = name } - public fun values(values: MutableList): CatalogBuilder = this.apply { - this.values = values + internal fun symbols(symbols: MutableList): CatalogBuilder = this.apply { + this.symbols = symbols } - public fun build(): Catalog = Catalog(name = name!!, values = values) + internal fun build(): Catalog = Catalog(name = name!!, symbols = symbols) } -internal class CatalogValueBuilder( - public var path: MutableList = mutableListOf(), - public var type: StaticType? = null, +internal class CatalogSymbolBuilder( + internal var path: MutableList = mutableListOf(), + internal var type: StaticType? = null, ) { - public fun path(path: MutableList): CatalogValueBuilder = this.apply { + internal fun path(path: MutableList): CatalogSymbolBuilder = this.apply { this.path = path } - public fun type(type: StaticType?): CatalogValueBuilder = this.apply { + internal fun type(type: StaticType?): CatalogSymbolBuilder = this.apply { this.type = type } - public fun build(): Catalog.Value = Catalog.Value(path = path, type = type!!) + internal fun build(): Catalog.Symbol = Catalog.Symbol(path = path, type = type!!) +} + +internal class CatalogSymbolRefBuilder( + internal var catalog: Int? = null, + internal var symbol: Int? = null, +) { + internal fun catalog(catalog: Int?): CatalogSymbolRefBuilder = this.apply { + this.catalog = catalog + } + + internal fun symbol(symbol: Int?): CatalogSymbolRefBuilder = this.apply { + this.symbol = symbol + } + + internal fun build(): Catalog.Symbol.Ref = Catalog.Symbol.Ref( + catalog = catalog!!, + symbol = + symbol!! + ) } internal class FnResolvedBuilder( @@ -220,22 +239,13 @@ internal class RexOpVarUnresolvedBuilder( } internal class RexOpGlobalBuilder( - public var catalogRef: Int? = null, - public var valueRef: Int? = null, + internal var ref: Catalog.Symbol.Ref? = null, ) { - public fun catalogRef(catalogRef: Int?): RexOpGlobalBuilder = this.apply { - this.catalogRef = catalogRef - } - - public fun valueRef(valueRef: Int?): RexOpGlobalBuilder = this.apply { - this.valueRef = valueRef + internal fun ref(ref: Catalog.Symbol.Ref?): RexOpGlobalBuilder = this.apply { + this.ref = ref } - public fun build(): Rex.Op.Global = Rex.Op.Global( - catalogRef = catalogRef!!, - valueRef = - valueRef!! - ) + internal fun build(): Rex.Op.Global = Rex.Op.Global(ref = ref!!) } internal class RexOpPathBuilder( diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/util/PlanRewriter.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/util/PlanRewriter.kt index 493ddedadf..cad7648d06 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/util/PlanRewriter.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/util/PlanRewriter.kt @@ -96,22 +96,28 @@ internal abstract class PlanRewriter : PlanBaseVisitor() { } } - public override fun visitCatalog(node: Catalog, ctx: C): PlanNode { + override fun visitCatalog(node: Catalog, ctx: C): PlanNode { val name = node.name - val values = _visitList(node.values, ctx, ::visitCatalogValue) - return if (name !== node.name || values !== node.values) { - Catalog(name, values) + val symbols = _visitList(node.symbols, ctx, ::visitCatalogSymbol) + return if (name !== node.name || symbols !== node.symbols) { + Catalog(name, symbols) } else { node } } - public override fun visitCatalogValue(node: Catalog.Value, ctx: C): PlanNode { + override fun visitCatalogSymbol(node: Catalog.Symbol, ctx: C): PlanNode { val path = node.path val type = node.type return node } + override fun visitCatalogSymbolRef(node: Catalog.Symbol.Ref, ctx: C): PlanNode { + val catalog = node.catalog + val symbol = node.symbol + return node + } + override fun visitFnResolved(node: Fn.Resolved, ctx: C): PlanNode { val signature = node.signature return node @@ -197,10 +203,13 @@ internal abstract class PlanRewriter : PlanBaseVisitor() { } } - public override fun visitRexOpGlobal(node: Rex.Op.Global, ctx: C): PlanNode { - val catalogRef = node.catalogRef - val valueRef = node.valueRef - return node + override fun visitRexOpGlobal(node: Rex.Op.Global, ctx: C): PlanNode { + val ref = visitCatalogSymbolRef(node.ref, ctx) as Catalog.Symbol.Ref + return if (ref !== node.ref) { + Rex.Op.Global(ref) + } else { + node + } } override fun visitRexOpPath(node: Rex.Op.Path, ctx: C): PlanNode { diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/visitor/PlanBaseVisitor.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/visitor/PlanBaseVisitor.kt index fe97f33efd..a5aab5250c 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/visitor/PlanBaseVisitor.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/visitor/PlanBaseVisitor.kt @@ -20,7 +20,10 @@ internal abstract class PlanBaseVisitor : PlanVisitor { public override fun visitCatalog(node: Catalog, ctx: C): R = defaultVisit(node, ctx) - public override fun visitCatalogValue(node: Catalog.Value, ctx: C): R = defaultVisit(node, ctx) + public override fun visitCatalogSymbol(node: Catalog.Symbol, ctx: C): R = defaultVisit(node, ctx) + + public override fun visitCatalogSymbolRef(node: Catalog.Symbol.Ref, ctx: C): R = + defaultVisit(node, ctx) override fun visitFn(node: Fn, ctx: C): R = when (node) { is Fn.Resolved -> visitFnResolved(node, ctx) diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/visitor/PlanVisitor.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/visitor/PlanVisitor.kt index c665579e52..e2707c2c17 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/visitor/PlanVisitor.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/visitor/PlanVisitor.kt @@ -20,7 +20,9 @@ internal interface PlanVisitor { public fun visitCatalog(node: Catalog, ctx: C): R - public fun visitCatalogValue(node: Catalog.Value, ctx: C): R + public fun visitCatalogSymbol(node: Catalog.Symbol, ctx: C): R + + public fun visitCatalogSymbolRef(node: Catalog.Symbol.Ref, ctx: C): R fun visitFn(node: Fn, ctx: C): R diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt index ac98212df7..4afe25c8e5 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt @@ -41,14 +41,18 @@ internal object PlanTransform : PlanBaseVisitor() { } override fun visitCatalog(node: Catalog, ctx: ProblemCallback): org.partiql.plan.Catalog { - val values = node.values.map { visitCatalogValue(it, ctx) } - return org.partiql.plan.Catalog(node.name, values) + val symbols = node.symbols.map { visitCatalogSymbol(it, ctx) } + return org.partiql.plan.Catalog(node.name, symbols) } - override fun visitCatalogValue(node: Catalog.Value, ctx: ProblemCallback): org.partiql.plan.Catalog.Symbol { + override fun visitCatalogSymbol(node: Catalog.Symbol, ctx: ProblemCallback): org.partiql.plan.Catalog.Symbol { return org.partiql.plan.Catalog.Symbol(node.path, node.type) } + override fun visitCatalogSymbolRef(node: Catalog.Symbol.Ref, ctx: ProblemCallback): org.partiql.plan.Catalog.Symbol.Ref { + return org.partiql.plan.Catalog.Symbol.Ref(node.catalog, node.symbol) + } + override fun visitFnResolved(node: Fn.Resolved, ctx: ProblemCallback) = org.partiql.plan.fn(node.signature) override fun visitFnUnresolved(node: Fn.Unresolved, ctx: ProblemCallback): org.partiql.plan.Rex.Op { @@ -112,7 +116,7 @@ internal object PlanTransform : PlanBaseVisitor() { org.partiql.plan.Rex.Op.Err("Unresolved variable $node") override fun visitRexOpGlobal(node: Rex.Op.Global, ctx: ProblemCallback) = org.partiql.plan.Rex.Op.Global( - ref = org.partiql.plan.Catalog.Symbol.Ref(catalog = node.catalogRef, symbol = node.valueRef) + ref = visitCatalogSymbolRef(node.ref, ctx) ) override fun visitRexOpPath(node: Rex.Op.Path, ctx: ProblemCallback): org.partiql.plan.Rex.Op.Path { diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt index 9c971c805b..3beeee7826 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt @@ -32,6 +32,7 @@ import org.partiql.planner.internal.ir.Rel import org.partiql.planner.internal.ir.Rex import org.partiql.planner.internal.ir.Statement import org.partiql.planner.internal.ir.aggResolved +import org.partiql.planner.internal.ir.catalogSymbolRef import org.partiql.planner.internal.ir.fnResolved import org.partiql.planner.internal.ir.identifierSymbol import org.partiql.planner.internal.ir.rel @@ -420,15 +421,15 @@ internal class PlanTyper( } val type = resolvedVar.type val op = when (resolvedVar) { - is ResolvedVar.Global -> rexOpGlobal(resolvedVar.ordinal, resolvedVar.position) + is ResolvedVar.Global -> rexOpGlobal(catalogSymbolRef(resolvedVar.ordinal, resolvedVar.position)) is ResolvedVar.Local -> resolvedLocalPath(resolvedVar) } return rex(type, op) } override fun visitRexOpGlobal(node: Rex.Op.Global, ctx: StaticType?): Rex { - val catalog = env.catalogs[node.catalogRef] - val type = catalog.values[node.valueRef].type + val catalog = env.catalogs[node.ref.catalog] + val type = catalog.symbols[node.ref.symbol].type return rex(type, node) } @@ -463,7 +464,7 @@ internal class PlanTyper( true -> emptyList() false -> visitedSteps.subList(remainingFirstIndex, visitedSteps.size) } - rexOpGlobal(resolvedVar.ordinal, resolvedVar.position) to remaining + rexOpGlobal(catalogSymbolRef(resolvedVar.ordinal, resolvedVar.position)) to remaining } } // rewrite root diff --git a/partiql-planner/src/test/kotlin/org/partiql/planner/internal/EnvTest.kt b/partiql-planner/src/test/kotlin/org/partiql/planner/internal/EnvTest.kt index 81e9ae35f8..32590a4e21 100644 --- a/partiql-planner/src/test/kotlin/org/partiql/planner/internal/EnvTest.kt +++ b/partiql-planner/src/test/kotlin/org/partiql/planner/internal/EnvTest.kt @@ -37,8 +37,8 @@ class EnvTest { private val GLOBAL_OS = Catalog( name = "pql", - values = listOf( - Catalog.Value(path = listOf("main", "os"), type = StaticType.STRING) + symbols = listOf( + Catalog.Symbol(path = listOf("main", "os"), type = StaticType.STRING) ) ) } diff --git a/partiql-planner/src/test/kotlin/org/partiql/planner/internal/typer/PlanTyperTest.kt b/partiql-planner/src/test/kotlin/org/partiql/planner/internal/typer/PlanTyperTest.kt index b284cfb962..de4154e91d 100644 --- a/partiql-planner/src/test/kotlin/org/partiql/planner/internal/typer/PlanTyperTest.kt +++ b/partiql-planner/src/test/kotlin/org/partiql/planner/internal/typer/PlanTyperTest.kt @@ -9,6 +9,7 @@ import org.partiql.planner.PartiQLPlanner import org.partiql.planner.internal.Env import org.partiql.planner.internal.ir.Identifier import org.partiql.planner.internal.ir.Rex +import org.partiql.planner.internal.ir.catalogSymbolRef import org.partiql.planner.internal.ir.identifierSymbol import org.partiql.planner.internal.ir.rex import org.partiql.planner.internal.ir.rexOpGlobal @@ -542,4 +543,8 @@ class PlanTyperTest { val actual = typer.resolve(input) assertEquals(expected, actual) } + + private fun rexOpGlobal(catalog: Int, ref: Int) = rexOpGlobal( + catalogSymbolRef(catalog, ref) + ) } From 3d877f84ce92924742f125558c0a47892f873c54 Mon Sep 17 00:00:00 2001 From: John Ed Quinn Date: Fri, 15 Dec 2023 12:24:50 -0800 Subject: [PATCH 5/6] Re-adds the imports removed from merge --- .../kotlin/org/partiql/planner/internal/typer/PlanTyper.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt index ab74fb8797..91ca52df53 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt @@ -85,7 +85,10 @@ import org.partiql.types.NullType import org.partiql.types.SexpType import org.partiql.types.StaticType import org.partiql.types.StaticType.Companion.ANY +import org.partiql.types.StaticType.Companion.BOOL import org.partiql.types.StaticType.Companion.MISSING +import org.partiql.types.StaticType.Companion.NULL +import org.partiql.types.StaticType.Companion.STRING import org.partiql.types.StringType import org.partiql.types.StructType import org.partiql.types.TupleConstraint From 22d3c33f300bc2ac3e8db7ac30f79a4f787298ee Mon Sep 17 00:00:00 2001 From: John Ed Quinn Date: Fri, 15 Dec 2023 13:37:46 -0800 Subject: [PATCH 6/6] Updates CHANGELOG --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2914c8af4..d7a6959e4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,6 @@ Thank you to all who have contributed! - The new plan is fully resolved and typed. - Operators will be converted to function call. - Changes the return type of `filter_distinct` to a list if input collection is list -- **BREAKING**: Globals within the logical plan are now represented in a structured manner to reflect the existence of catalogs. Each value is now mandated to be case-sensitive (we have moved from Identifiers to string literals). - Changes the `PartiQLValue` collections to implement Iterable rather than Sequence, allowing for multiple consumption. - **BREAKING** Moves PartiQLParserBuilder.standard().build() to be PartiQLParser.default().