Skip to content

Commit

Permalink
Address RelExclude comments; slight refactor of subsumption + testing…
Browse files Browse the repository at this point in the history 
…; add comments
  • Loading branch information
@alancai98
alancai98 committed Jan 11, 2024
1 parent ea5baf5 commit 3be4a5e
Show file tree
Hide file tree
Showing 6 changed files with 82 additions and 68 deletions.
83 changes: 39 additions & 44 deletions partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelExclude.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,28 +31,23 @@ internal class RelExclude(
}

override fun next(): Record? {
while (true) {
val row = input.next() ?: return null
val newRecord = exclusions.fold(row) { curRecord, expr ->
excludeOnRecord(curRecord, expr)
}
return newRecord
}
val record = input.next() ?: return null
return exclusions.fold(record) { rec, path -> exclude(rec, path) }
}

override fun close() {
input.close()
}

@OptIn(PartiQLValueExperimental::class)
private fun excludeOnRecord(
private fun exclude(
record: Record,
path: Rel.Op.Exclude.Path
): Record {
val values = record.values
val value = values.getOrNull(path.root.ref)
val newValues = if (value != null) {
values[path.root.ref] = excludeOnPartiQLValue(value, path.steps)
values[path.root.ref] = exclude(value, path.steps)
values
} else {
values
Expand All @@ -61,57 +56,57 @@ internal class RelExclude(
}

@OptIn(PartiQLValueExperimental::class)
private fun excludeOnStructValue(
private fun exclude(
structValue: StructValue<*>,
exclusions: List<Rel.Op.Exclude.Step>
): PartiQLValue {
val attrSymbolsToRemove = mutableSetOf<String>()
val attrKeysToRemove = mutableSetOf<String>()
val structSymbolsToRemove = mutableSetOf<String>()
val structKeysToRemove = mutableSetOf<String>() // keys stored as lowercase strings
val branches = mutableMapOf<Rel.Op.Exclude.Type, List<Rel.Op.Exclude.Step>>()
exclusions.forEach { exclusion ->
when (exclusion.substeps.isEmpty()) {
true -> {
when (val leafType = exclusion.type) {
is Rel.Op.Exclude.Type.StructWildcard -> {
// tuple wildcard at current level. return empty struct
// struct wildcard at current level. return empty struct
return structValue<PartiQLValue>()
}
is Rel.Op.Exclude.Type.StructSymbol -> attrSymbolsToRemove.add(leafType.symbol)
is Rel.Op.Exclude.Type.StructKey -> attrKeysToRemove.add(leafType.key.lowercase())
else -> { /* non-coll step; do nothing */ }
is Rel.Op.Exclude.Type.StructSymbol -> structSymbolsToRemove.add(leafType.symbol)
is Rel.Op.Exclude.Type.StructKey -> structKeysToRemove.add(leafType.key.lowercase())
else -> { /* coll step; do nothing */ }
}
}
false -> {
when (exclusion.type) {
is Rel.Op.Exclude.Type.StructWildcard, is Rel.Op.Exclude.Type.StructSymbol, is Rel.Op.Exclude.Type.StructKey -> branches[exclusion.type] =
exclusion.substeps
else -> { /* non-coll step; do nothing */ }
else -> { /* coll step; do nothing */ }
}
}
}
}
val finalStruct = structValue.entries.mapNotNull { structField ->
if (attrSymbolsToRemove.contains(structField.first) || attrKeysToRemove.contains(structField.first.lowercase())) {
if (structSymbolsToRemove.contains(structField.first) || structKeysToRemove.contains(structField.first.lowercase())) {
// struct attr is to be removed at current level
null
} else {
// deeper level exclusions
val name = structField.first
var value = structField.second
// apply case-sensitive tuple attr exclusions at deeper levels
val structFieldCaseSensitiveKey = relOpExcludeTypeStructKey(name)
branches[structFieldCaseSensitiveKey]?.let {
value = excludeOnPartiQLValue(value, it)
// apply struct key exclusions at deeper levels
val structKey = relOpExcludeTypeStructKey(name)
branches[structKey]?.let {
value = exclude(value, it)
}
// apply case-insensitive tuple attr exclusions at deeper levels
val structFieldCaseInsensitiveKey = relOpExcludeTypeStructSymbol(name)
branches[structFieldCaseInsensitiveKey]?.let {
value = excludeOnPartiQLValue(value, it)
// apply struct symbol exclusions at deeper levels
val structSymbol = relOpExcludeTypeStructSymbol(name)
branches[structSymbol]?.let {
value = exclude(value, it)
}
// apply tuple wildcard exclusions at deeper levels
val tupleWildcardKey = relOpExcludeTypeStructWildcard()
branches[tupleWildcardKey]?.let {
value = excludeOnPartiQLValue(value, it)
// apply struct wildcard exclusions at deeper levels
val structWildcard = relOpExcludeTypeStructWildcard()
branches[structWildcard]?.let {
value = exclude(value, it)
}
Pair(name, value)
}
Expand All @@ -134,7 +129,7 @@ internal class RelExclude(
}

@OptIn(PartiQLValueExperimental::class)
private fun excludeOnCollValue(
private fun exclude(
coll: CollectionValue<*>,
type: PartiQLValueType,
exclusions: List<Rel.Op.Exclude.Step>
Expand All @@ -152,14 +147,14 @@ internal class RelExclude(
is Rel.Op.Exclude.Type.CollIndex -> {
indexesToRemove.add(leafType.index)
}
else -> { /* non-coll step; do nothing */ }
else -> { /* struct step; do nothing */ }
}
}
false -> {
when (exclusion.type) {
is Rel.Op.Exclude.Type.CollWildcard, is Rel.Op.Exclude.Type.CollIndex -> branches[exclusion.type] =
exclusion.substeps
else -> { /* non-coll step; do nothing */ }
else -> { /* struct step; do nothing */ }
}
}
}
Expand All @@ -173,15 +168,15 @@ internal class RelExclude(
var value = element
if (coll is ListValue || coll is SexpValue) {
// apply collection index exclusions at deeper levels for lists and sexps
val elementKey = relOpExcludeTypeCollIndex(index)
branches[elementKey]?.let {
value = excludeOnPartiQLValue(element, it)
val collIndex = relOpExcludeTypeCollIndex(index)
branches[collIndex]?.let {
value = exclude(element, it)
}
}
// apply collection wildcard exclusions at deeper levels for lists, bags, and sexps
val collectionWildcardKey = relOpExcludeTypeCollWildcard()
branches[collectionWildcardKey]?.let {
value = excludeOnPartiQLValue(value, it)
val collWildcard = relOpExcludeTypeCollWildcard()
branches[collWildcard]?.let {
value = exclude(value, it)
}
value
}
Expand All @@ -190,12 +185,12 @@ internal class RelExclude(
}

@OptIn(PartiQLValueExperimental::class)
private fun excludeOnPartiQLValue(initialPartiQLValue: PartiQLValue, exclusions: List<Rel.Op.Exclude.Step>): PartiQLValue {
private fun exclude(initialPartiQLValue: PartiQLValue, exclusions: List<Rel.Op.Exclude.Step>): PartiQLValue {
return when (initialPartiQLValue) {
is StructValue<*> -> excludeOnStructValue(initialPartiQLValue, exclusions)
is BagValue<*> -> excludeOnCollValue(initialPartiQLValue, PartiQLValueType.BAG, exclusions)
is ListValue<*> -> excludeOnCollValue(initialPartiQLValue, PartiQLValueType.LIST, exclusions)
is SexpValue<*> -> excludeOnCollValue(initialPartiQLValue, PartiQLValueType.SEXP, exclusions)
is StructValue<*> -> exclude(initialPartiQLValue, exclusions)
is BagValue<*> -> exclude(initialPartiQLValue, PartiQLValueType.BAG, exclusions)
is ListValue<*> -> exclude(initialPartiQLValue, PartiQLValueType.LIST, exclusions)
is SexpValue<*> -> exclude(initialPartiQLValue, PartiQLValueType.SEXP, exclusions)
else -> {
initialPartiQLValue
}
Expand Down
1 change: 0 additions & 1 deletion partiql-planner/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ dependencies {
testImplementation(project(":partiql-parser"))
testImplementation(project(":plugins:partiql-local"))
testImplementation(project(":plugins:partiql-memory"))
testImplementation(testFixtures(project(":partiql-lang"))) // TODO replace usage of `partiql-lang` test fixture
// Test Fixtures
testFixturesImplementation(project(":partiql-spi"))
}
Expand Down
27 changes: 16 additions & 11 deletions partiql-planner/src/main/kotlin/org/partiql/planner/internal/exclude/Subsumption.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import org.partiql.planner.internal.ir.relOpExcludeTypeStructSymbol
import org.partiql.planner.internal.ir.relOpExcludeTypeStructWildcard

/**
* An internal data structure of exclude steps that makes it a bit easier to remove any redundant exclude steps.
*
* Store as a map of exclude type to its substeps; previous representation as a set of `Rel.Op.Exclude.Step` didn't
* work since retrieval of element in set required looping through the whole list.
*/
Expand All @@ -27,12 +29,14 @@ internal class ExcludeRepr(val steps: Map<Rel.Op.Exclude.Type, ExcludeRepr>) {
}
}

internal fun isLeaf(): Boolean = steps.isEmpty()

internal fun removeRedundantSteps(): ExcludeRepr {
val newSubsteps = mutableMapOf<Rel.Op.Exclude.Type, ExcludeRepr>()
val collIndexSteps = mutableMapOf<Rel.Op.Exclude.Type, ExcludeRepr>()
val structSymbolSteps = mutableMapOf<Rel.Op.Exclude.Type, ExcludeRepr>()
val structKeySteps = mutableMapOf<Rel.Op.Exclude.Type, ExcludeRepr>()
// Group the steps by their type and add wildcards to `substeps`
// Group the steps by their type and first add wildcards to `substeps`
steps.forEach { (type, substeps) ->
when (type) {
is Rel.Op.Exclude.Type.StructWildcard -> {
Expand All @@ -48,7 +52,7 @@ internal class ExcludeRepr(val steps: Map<Rel.Op.Exclude.Type, ExcludeRepr>) {
is Rel.Op.Exclude.Type.StructKey -> structKeySteps[type] = substeps.removeRedundantSteps()
}
}
// Next add non-wildcard steps
// Next add non-wildcard steps and check if they are subsumed by previously added steps
stepsSubsume(newSubsteps, collIndexSteps)?.let {
newSubsteps.putAll(it)
}
Expand All @@ -62,6 +66,7 @@ internal class ExcludeRepr(val steps: Map<Rel.Op.Exclude.Type, ExcludeRepr>) {
}
}

// null indicates all the steps on rhs are subsumed by the lhs
private fun stepsSubsume(lhs: Map<Rel.Op.Exclude.Type, ExcludeRepr>, rhs: Map<Rel.Op.Exclude.Type, ExcludeRepr>?): Map<Rel.Op.Exclude.Type, ExcludeRepr>? {
if (rhs == null) {
return null
Expand All @@ -72,7 +77,7 @@ private fun stepsSubsume(lhs: Map<Rel.Op.Exclude.Type, ExcludeRepr>, rhs: Map<Re
is Rel.Op.Exclude.Type.CollWildcard -> {
val lhsCollWildcard = lhs[relOpExcludeTypeCollWildcard()]
if (lhsCollWildcard != null) {
newSubsteps = if (lhsCollWildcard.steps.isEmpty()) {
newSubsteps = if (lhsCollWildcard.isLeaf()) {
// coll wildcard leaf in lhs
null
} else {
Expand All @@ -83,7 +88,7 @@ private fun stepsSubsume(lhs: Map<Rel.Op.Exclude.Type, ExcludeRepr>, rhs: Map<Re
is Rel.Op.Exclude.Type.CollIndex -> {
val lhsCollWildcard = lhs[relOpExcludeTypeCollWildcard()]
if (lhsCollWildcard != null) {
newSubsteps = if (lhsCollWildcard.steps.isEmpty()) {
newSubsteps = if (lhsCollWildcard.isLeaf()) {
// coll wildcard leaf in lhs
null
} else {
Expand All @@ -92,7 +97,7 @@ private fun stepsSubsume(lhs: Map<Rel.Op.Exclude.Type, ExcludeRepr>, rhs: Map<Re
}
val lhsCollIndex = lhs[type]
if (lhsCollIndex != null) {
newSubsteps = if (lhsCollIndex.steps.isEmpty()) {
newSubsteps = if (lhsCollIndex.isLeaf()) {
// coll index leaf in lhs
null
} else {
Expand All @@ -103,7 +108,7 @@ private fun stepsSubsume(lhs: Map<Rel.Op.Exclude.Type, ExcludeRepr>, rhs: Map<Re
is Rel.Op.Exclude.Type.StructWildcard -> {
val lhsStructWildcard = lhs[relOpExcludeTypeStructWildcard()]
if (lhsStructWildcard != null) {
newSubsteps = if (lhsStructWildcard.steps.isEmpty()) {
newSubsteps = if (lhsStructWildcard.isLeaf()) {
// struct wildcard leaf in lhs
null
} else {
Expand All @@ -114,7 +119,7 @@ private fun stepsSubsume(lhs: Map<Rel.Op.Exclude.Type, ExcludeRepr>, rhs: Map<Re
is Rel.Op.Exclude.Type.StructSymbol -> {
val lhsStructWildcard = lhs[relOpExcludeTypeStructWildcard()]
if (lhsStructWildcard != null) {
newSubsteps = if (lhsStructWildcard.steps.isEmpty()) {
newSubsteps = if (lhsStructWildcard.isLeaf()) {
// struct wildcard leaf in lhs
null
} else {
Expand All @@ -123,7 +128,7 @@ private fun stepsSubsume(lhs: Map<Rel.Op.Exclude.Type, ExcludeRepr>, rhs: Map<Re
}
val lhsStructSymbol = lhs[type]
if (lhsStructSymbol != null) {
newSubsteps = if (lhsStructSymbol.steps.isEmpty()) {
newSubsteps = if (lhsStructSymbol.isLeaf()) {
// struct symbol leaf in lhs
null
} else {
Expand All @@ -134,7 +139,7 @@ private fun stepsSubsume(lhs: Map<Rel.Op.Exclude.Type, ExcludeRepr>, rhs: Map<Re
is Rel.Op.Exclude.Type.StructKey -> {
val lhsStructWildcard = lhs[relOpExcludeTypeStructWildcard()]
if (lhsStructWildcard != null) {
newSubsteps = if (lhsStructWildcard.steps.isEmpty()) {
newSubsteps = if (lhsStructWildcard.isLeaf()) {
// struct wildcard leaf in lhs
null
} else {
Expand All @@ -144,7 +149,7 @@ private fun stepsSubsume(lhs: Map<Rel.Op.Exclude.Type, ExcludeRepr>, rhs: Map<Re
val keyAsSymbol = relOpExcludeTypeStructSymbol(type.key)
val lhsStructSymbol = lhs[keyAsSymbol]
if (lhsStructSymbol != null) {
newSubsteps = if (lhsStructSymbol.steps.isEmpty()) {
newSubsteps = if (lhsStructSymbol.isLeaf()) {
// struct symbol leaf in lhs
null
} else {
Expand All @@ -153,7 +158,7 @@ private fun stepsSubsume(lhs: Map<Rel.Op.Exclude.Type, ExcludeRepr>, rhs: Map<Re
}
val lhsStructKey = lhs[type]
if (lhsStructKey != null) {
newSubsteps = if (lhsStructKey.steps.isEmpty()) {
newSubsteps = if (lhsStructKey.isLeaf()) {
// struct key leaf in lhs
null
} else {
Expand Down
2 changes: 1 addition & 1 deletion partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1425,7 +1425,7 @@ internal class PlanTyper(
if (id.isEquivalentTo(it.name)) {
matchedRoot = true
// recompute the StaticType of this binding after applying the exclusions
val type = it.type.exclude(item.steps, false)
val type = it.type.exclude(item.steps, lastStepOptional = false)
it.copy(type = type)
} else {
it
Expand Down
15 changes: 10 additions & 5 deletions partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/TypeUtils.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,13 +139,18 @@ private fun StaticType.asRuntimeType(): PartiQLValueType = when (this) {
}

/**
* Applies the given exclusion path to produce the reduced StaticType
* Applies the given exclusion path to produce the reduced [StaticType]. [lastStepOptional] indicates if a previous
* step in the exclude path includes a collection index exclude step. Currently, for paths with the last step as
* a struct symbol/key, the type inference will define that struct value as optional if [lastStepOptional] is true.
* Note, this specific behavior could change depending on `EXCLUDE`'s static typing behavior in a future RFC.
*
* e.g. EXCLUDE t.a[1].field_x will define the struct value `field_x` as optional
*
* @param steps
* @param lastStepOptional
* @return
*/
internal fun StaticType.exclude(steps: List<Rel.Op.Exclude.Step>, lastStepOptional: Boolean = true): StaticType {
internal fun StaticType.exclude(steps: List<Rel.Op.Exclude.Step>, lastStepOptional: Boolean = false): StaticType {
val type = this
return steps.fold(type) { acc, step ->
when (acc) {
Expand All @@ -166,7 +171,7 @@ internal fun StaticType.exclude(steps: List<Rel.Op.Exclude.Step>, lastStepOption
* @param lastStepOptional
* @return
*/
internal fun StructType.exclude(step: Rel.Op.Exclude.Step, lastStepOptional: Boolean = true): StaticType {
internal fun StructType.exclude(step: Rel.Op.Exclude.Step, lastStepOptional: Boolean = false): StaticType {
val type = step.type
val substeps = step.substeps
val output = fields.mapNotNull { field ->
Expand Down Expand Up @@ -211,13 +216,13 @@ internal fun StructType.exclude(step: Rel.Op.Exclude.Step, lastStepOptional: Boo
* @param lastStepOptional
* @return
*/
internal fun CollectionType.exclude(step: Rel.Op.Exclude.Step, lastStepOptional: Boolean = true): StaticType {
internal fun CollectionType.exclude(step: Rel.Op.Exclude.Step, lastStepOptional: Boolean = false): StaticType {
var e = this.elementType
val substeps = step.substeps
when (step.type) {
is Rel.Op.Exclude.Type.CollIndex -> {
if (substeps.isNotEmpty()) {
e = e.exclude(substeps, true)
e = e.exclude(substeps, lastStepOptional = true)
}
}
is Rel.Op.Exclude.Type.CollWildcard -> {
Expand Down
Loading

0 comments on commit 3be4a5e

Please sign in to comment.