diff --git a/partiql-lang/src/main/kotlin/org/partiql/lang/eval/EvaluatingCompiler.kt b/partiql-lang/src/main/kotlin/org/partiql/lang/eval/EvaluatingCompiler.kt index c2a9a5c791..38a96c8a1c 100644 --- a/partiql-lang/src/main/kotlin/org/partiql/lang/eval/EvaluatingCompiler.kt +++ b/partiql-lang/src/main/kotlin/org/partiql/lang/eval/EvaluatingCompiler.kt @@ -20,6 +20,7 @@ import com.amazon.ion.IonValue import com.amazon.ion.Timestamp import com.amazon.ion.system.IonSystemBuilder import com.amazon.ionelement.api.MetaContainer +import com.amazon.ionelement.api.emptyMetaContainer import com.amazon.ionelement.api.ionBool import com.amazon.ionelement.api.toIonValue import org.partiql.errors.ErrorCode @@ -73,6 +74,7 @@ import org.partiql.lang.util.take import org.partiql.lang.util.times import org.partiql.lang.util.totalMinutes import org.partiql.lang.util.unaryMinus +import org.partiql.pig.runtime.LongPrimitive import org.partiql.pig.runtime.SymbolPrimitive import org.partiql.types.AnyOfType import org.partiql.types.AnyType @@ -1829,6 +1831,8 @@ internal open class EvaluatingCompiler( val limitThunk = selectExpr.limit?.let { compileAstExpr(it) } val limitLocationMeta = selectExpr.limit?.metas?.sourceLocation + val excludeExprs = selectExpr.excludeClause?.let { compileExcludeClause(it) } + fun rowsWithOffsetAndLimit(rows: Sequence, env: Environment): Sequence { val rowsWithOffset = when (offsetThunk) { null -> rows @@ -1863,7 +1867,16 @@ internal open class EvaluatingCompiler( else -> evalOrderBy(sourcedRows, orderByThunk, orderByLocationMeta) } - val projectedRows = orderedRows.map { (joinedValues, projectEnv) -> + val excludedBindings = when (excludeExprs) { + null -> orderedRows + else -> { + orderedRows.map { row -> + FromProduction(values = row.values, env = evalExclude(row.env, excludeExprs)) + } + } + } + + val projectedRows = excludedBindings.map { (joinedValues, projectEnv) -> selectProjectionThunk(projectEnv, joinedValues) } @@ -1929,8 +1942,17 @@ internal open class EvaluatingCompiler( else -> evalOrderBy(sourceThunks(env), orderByThunk, orderByLocationMeta) } + val excludedBindings = when (excludeExprs) { + null -> orderedRows + else -> { + orderedRows.map { row -> + FromProduction(values = row.values, env = evalExclude(row.env, excludeExprs)) + } + } + } + val fromProductions: Sequence = - rowsWithOffsetAndLimit(orderedRows, env) + rowsWithOffsetAndLimit(excludedBindings, env) val registers = createRegisterBank() // note: the group key can be anything here because we only ever have a single @@ -1977,9 +1999,6 @@ internal open class EvaluatingCompiler( val havingThunk = selectExpr.having?.let { compileHaving(it) } - val filterHavingAndProject: (Environment, Group) -> ExprValue? = - createFilterHavingAndProjectClosure(havingThunk, selectProjectionThunk) - val getGroupEnv: (Environment, Group) -> Environment = createGetGroupEnvClosure(groupAsName) @@ -2026,10 +2045,30 @@ internal open class EvaluatingCompiler( else -> evalOrderBy(groupByEnvValuePairs, orderByThunk, orderByLocationMeta) } + // apply HAVING row filter + val havingGroupEnvPairs = when (havingThunk) { + null -> orderedGroupEnvPairs + else -> { + orderedGroupEnvPairs.filter { (groupByEnv, _) -> + val havingClauseResult = havingThunk(groupByEnv) + havingClauseResult.isNotUnknown() && havingClauseResult.booleanValue() + } + } + } + + // apply EXCLUDE expressions + val excludedBindings = when (excludeExprs) { + null -> havingGroupEnvPairs + else -> { + havingGroupEnvPairs.map { (groupByEnv, currentGroup) -> + Pair(evalExclude(groupByEnv, excludeExprs), currentGroup) + } + } + } // generate the final group by projection - val projectedRows = orderedGroupEnvPairs.mapNotNull { (groupByEnv, groupValue) -> - filterHavingAndProject(groupByEnv, groupValue) - }.asSequence().let { rowsWithOffsetAndLimit(it, env) } + val projectedRows = excludedBindings.map { (groupByEnv, currentGroup) -> + selectProjectionThunk(groupByEnv, listOf(currentGroup.key)) + }.let { rowsWithOffsetAndLimit(it, env) } // if order by is specified, return list otherwise bag when (orderByThunk) { @@ -2067,7 +2106,15 @@ internal open class EvaluatingCompiler( val asThunk = compileAstExpr(asExpr) val atThunk = compileAstExpr(atExpr) thunkFactory.thunkEnv(metas) { env -> - val sourceValue = rowsWithOffsetAndLimit(sourceThunks(env).asSequence(), env) + val excludedBindings = when (excludeExprs) { + null -> sourceThunks(env) + else -> { + sourceThunks(env).map { row -> + FromProduction(values = row.values, env = evalExclude(row.env, excludeExprs)) + } + } + } + val sourceValue = rowsWithOffsetAndLimit(excludedBindings, env) val seq = sourceValue .map { (_, env) -> Pair(asThunk(env), atThunk(env)) } .filter { (name, _) -> name.type.isText } @@ -2159,6 +2206,126 @@ internal open class EvaluatingCompiler( CompiledGroupByItem(alias.text.exprValue(), uniqueName, compileAstExpr(it.expr)) } + /** + * Represents an instance of a compiled `EXCLUDE` expression. Notably, this expr will have redundant steps removed. + */ + private data class CompiledExcludeExpr(val root: PartiqlAst.Identifier, val exclusions: RemoveAndOtherSteps) + /** + * Represents all the exclusions at the current level and other nested levels. + */ + private data class RemoveAndOtherSteps(val remove: Set, val steps: Map) { + companion object { + fun empty(): RemoveAndOtherSteps { + return RemoveAndOtherSteps(emptySet(), emptyMap()) + } + } + } + + /** + * Creates a list of compiled exclude expressions. + */ + private fun compileExcludeClause(excludeClause: PartiqlAst.ExcludeOp): List { + val excludeExprs = excludeClause.exprs + fun addToCompiledExcludeExprs(curCompiledExpr: RemoveAndOtherSteps, steps: List): RemoveAndOtherSteps { + // subsumption cases + // when steps.size == 1: look at remove set + // when steps.size > 1: look at other steps + val first = steps.first() + var entryRemove = curCompiledExpr.remove.toMutableSet() + val entrySteps = curCompiledExpr.steps.toMutableMap() + if (steps.size == 1) { + when (first) { + is PartiqlAst.ExcludeStep.ExcludeTupleAttr -> { + if (entryRemove.contains(PartiqlAst.build { excludeTupleWildcard() })) { + // contains wildcard; do not add; a.b and a.* -> a[*] + } else { + entryRemove.add(first) + } + } + is PartiqlAst.ExcludeStep.ExcludeTupleWildcard -> { + if (entryRemove.any { it is PartiqlAst.ExcludeStep.ExcludeCollectionWildcard || it is PartiqlAst.ExcludeStep.ExcludeCollectionIndex }) { + // todo mistyping; and other mistyping + } else { + entryRemove = mutableSetOf(first) + } + } + is PartiqlAst.ExcludeStep.ExcludeCollectionIndex -> { + if (entryRemove.contains(PartiqlAst.build { excludeCollectionWildcard() })) { + // contains wildcard; do not add; a[*] and a[*] -> a[*] + } else { + entryRemove.add(first) + } + } + is PartiqlAst.ExcludeStep.ExcludeCollectionWildcard -> { + if (entryRemove.any { it is PartiqlAst.ExcludeStep.ExcludeTupleWildcard || it is PartiqlAst.ExcludeStep.ExcludeTupleAttr }) { + // todo mistyping; and other mistyping + } else { + entryRemove = mutableSetOf(first) + } + } + } + } else { + // remove at deeper level; need to check if first step is already removed in previous step + when (first) { + is PartiqlAst.ExcludeStep.ExcludeTupleAttr -> { + if (entryRemove.contains(PartiqlAst.build { excludeTupleWildcard() }) || entryRemove.contains(first)) { + // contains wildcard or first; do not add; a.b.c and a.* -> a.* + } else { + val existingEntry = entrySteps.getOrDefault(first, RemoveAndOtherSteps.empty()) + val newEntry = addToCompiledExcludeExprs(existingEntry, steps.drop(1)) + entrySteps[first] = newEntry + } + } + is PartiqlAst.ExcludeStep.ExcludeTupleWildcard -> { + if (entryRemove.any { it is PartiqlAst.ExcludeStep.ExcludeCollectionWildcard || it is PartiqlAst.ExcludeStep.ExcludeCollectionIndex }) { + // todo mistyping; and other mistyping + } else { + if (entryRemove.any { it is PartiqlAst.ExcludeStep.ExcludeTupleWildcard }) { + // do nothing + } else { + val existingEntry = entrySteps.getOrDefault(first, RemoveAndOtherSteps.empty()) + val newEntry = addToCompiledExcludeExprs(existingEntry, steps.drop(1)) + entrySteps[first] = newEntry + } + } + } + is PartiqlAst.ExcludeStep.ExcludeCollectionIndex -> { + if (entryRemove.contains(PartiqlAst.build { excludeCollectionWildcard() }) || entryRemove.contains(first)) { + // contains wildcard; do not add; a[*] and a[*][1] -> a[*] + } else { + val existingEntry = entrySteps.getOrDefault(first, RemoveAndOtherSteps.empty()) + val newEntry = addToCompiledExcludeExprs(existingEntry, steps.drop(1)) + entrySteps[first] = newEntry + } + } + is PartiqlAst.ExcludeStep.ExcludeCollectionWildcard -> { + if (entryRemove.any { it is PartiqlAst.ExcludeStep.ExcludeTupleWildcard || it is PartiqlAst.ExcludeStep.ExcludeTupleAttr }) { + // todo mistyping; and other mistyping + } else { + if (entryRemove.any { it is PartiqlAst.ExcludeStep.ExcludeCollectionWildcard }) { + // do nothing + } else { + val existingEntry = entrySteps.getOrDefault(first, RemoveAndOtherSteps.empty()) + val newEntry = addToCompiledExcludeExprs(existingEntry, steps.drop(1)) + entrySteps[first] = newEntry + } + } + } + } + } + return RemoveAndOtherSteps(entryRemove, entrySteps) + } + val compiledExcludeExprs = excludeExprs + .groupBy { it.root } + .map { (root, exclusions) -> + val compiledExclusions = exclusions.fold(RemoveAndOtherSteps.empty()) { acc, exclusion -> + addToCompiledExcludeExprs(acc, exclusion.steps) + } + CompiledExcludeExpr(root, compiledExclusions) + } + return compiledExcludeExprs + } + /** * Create a thunk that uses the compiled GROUP BY expressions to create the group key. */ @@ -2196,6 +2363,144 @@ internal open class EvaluatingCompiler( CompiledOrderByItem(comparator, compileAstExpr(it.expr)) } + /** + * Returns an [ExprValue] created from a sequence of [seq]. Requires [type] to be a sequence type + * (i.e. [ExprValueType.isSequence] == true). + */ + private fun newSequence(type: ExprValueType, seq: Sequence): ExprValue { + return when (type) { + ExprValueType.LIST -> ExprValue.newList(seq) + ExprValueType.BAG -> ExprValue.newBag(seq) + ExprValueType.SEXP -> ExprValue.newSexp(seq) + else -> error("Sequence type required") + } + } + + private fun evalExclude( + env: Environment, + excludeExprs: List + ): Environment { + fun excludeExprValue(initialExprValue: ExprValue, exclusions: RemoveAndOtherSteps): ExprValue { + if (initialExprValue is Named) { + return excludeExprValue(initialExprValue.value, exclusions) + } + val toRemove = exclusions.remove + val otherSteps = exclusions.steps + when (initialExprValue.type) { + ExprValueType.STRUCT -> { + initialExprValue as StructExprValue + if (toRemove.any { it is PartiqlAst.ExcludeStep.ExcludeTupleWildcard }) { + return StructExprValue(sequence = emptySequence(), ordering = initialExprValue.ordering) + } + val attrsToRemove = toRemove.filterIsInstance() + .map { it.attr.name.text } + .toSet() + val sequenceWithRemoved = initialExprValue.mapNotNull { structField -> + if (attrsToRemove.contains(structField.name?.stringValue())) { + null + } else { + structField + } + } + val finalSequence = sequenceWithRemoved.map { structField -> + var expr = structField + val name = structField.name!! + val structFieldKey = PartiqlAst.build { + PartiqlAst.ExcludeStep.ExcludeTupleAttr( + PartiqlAst.Identifier( + SymbolPrimitive( + structField.name?.stringValue()!!, + emptyMetaContainer() + ), + caseInsensitive() + ) + ) + } + if (otherSteps.contains(structFieldKey)) { + expr = excludeExprValue(structField, otherSteps[structFieldKey]!!) + } + val tupleWildcardEntry = + otherSteps[PartiqlAst.build { excludeTupleWildcard(emptyMetaContainer()) }] + if (tupleWildcardEntry != null) { + expr = excludeExprValue(expr, tupleWildcardEntry) + } + expr.namedValue(name) + } + return ExprValue.newStruct(values = finalSequence, ordering = initialExprValue.ordering) + } + + ExprValueType.LIST, ExprValueType.BAG, ExprValueType.SEXP -> { + if (toRemove.any { it is PartiqlAst.ExcludeStep.ExcludeCollectionWildcard }) { + return newSequence(initialExprValue.type, emptySequence()) + } else { + // remove some elements + val indexesToRemove = toRemove.filterIsInstance() + .map { it.index.value } + .toSet() + val sequenceWithRemoved = initialExprValue.mapNotNull { element -> + if (indexesToRemove.contains(element.name?.longValue())) { + null + } else { + element + } + }.asSequence() + val finalSequence = sequenceWithRemoved.map { element -> + var expr = element + if (initialExprValue.type == ExprValueType.LIST || initialExprValue.type == ExprValueType.SEXP) { + val elementKey = PartiqlAst.build { + PartiqlAst.ExcludeStep.ExcludeCollectionIndex( + LongPrimitive( + element.name?.longValue()!!, + emptyMetaContainer() + ) + ) + } + if (otherSteps.contains(elementKey)) { + expr = excludeExprValue(element, otherSteps[elementKey]!!) + } + } + val collectionWildcardEntry = + otherSteps[PartiqlAst.build { excludeCollectionWildcard(emptyMetaContainer()) }] + if (collectionWildcardEntry != null) { + expr = excludeExprValue(expr, collectionWildcardEntry) + } + expr + } + return newSequence(initialExprValue.type, finalSequence) + } + } + else -> { + return initialExprValue + } + } + } + + fun excludeBindings( + initialBindings: Bindings, + root: PartiqlAst.Identifier, + exclusions: RemoveAndOtherSteps + ): Bindings { + val bindingNameString = root.name.text + val bindingName = BindingName(bindingNameString, BindingCase.INSENSITIVE) + val bindingAtAttr = initialBindings[bindingName] + return if (bindingAtAttr != null) { + val newBindings = Bindings.buildLazyBindings { + val newExprValue = excludeExprValue(bindingAtAttr, exclusions) + addBinding(bindingNameString) { + newExprValue + } + } + newBindings.delegate(initialBindings) + } else { + initialBindings + } + } + val newBindings = excludeExprs.fold(env.current) { accBindings, expr -> + excludeBindings(accBindings, expr.root, expr.exclusions) + } + return env.nest(newLocals = newBindings) + } + private fun evalOrderBy( rows: Sequence, orderByItems: List, @@ -2271,33 +2576,6 @@ internal open class EvaluatingCompiler( } } - /** - * Returns a closure which performs the final projection and returns the - * result. If a HAVING clause was included, a different closure is returned - * that evaluates the HAVING clause and performs filtering. - */ - private fun createFilterHavingAndProjectClosure( - havingThunk: ThunkEnv?, - selectProjectionThunk: ThunkEnvValue> - ): (Environment, Group) -> ExprValue? = - when { - havingThunk != null -> { groupByEnv, currentGroup -> - // Create a closure that executes the HAVING clause and returns null if the - // HAVING criteria is not met - val havingClauseResult = havingThunk(groupByEnv) - if (havingClauseResult.isNotUnknown() && havingClauseResult.booleanValue()) { - selectProjectionThunk(groupByEnv, listOf(currentGroup.key)) - } else { - null - } - } - else -> { groupByEnv, currentGroup -> - // Create a closure that simply performs the final projection and - // returns the result. - selectProjectionThunk(groupByEnv, listOf(currentGroup.key)) - } - } - private fun compileCallAgg(expr: PartiqlAst.Expr.CallAgg, metas: MetaContainer): ThunkEnv { if (metas.containsKey(IsCountStarMeta.TAG) && currentCompilationContext.expressionContext != ExpressionContext.SELECT_LIST) { err( diff --git a/partiql-lang/src/main/kotlin/org/partiql/lang/eval/ExprValueExtensions.kt b/partiql-lang/src/main/kotlin/org/partiql/lang/eval/ExprValueExtensions.kt index 9b5ce3239b..1c598488f0 100644 --- a/partiql-lang/src/main/kotlin/org/partiql/lang/eval/ExprValueExtensions.kt +++ b/partiql-lang/src/main/kotlin/org/partiql/lang/eval/ExprValueExtensions.kt @@ -93,10 +93,12 @@ val ExprValue.orderedNames: List? fun ExprValue.asNamed(): Named = object : Named { override val name: ExprValue get() = this@asNamed + override val value = this@asNamed } /** Binds the given name value as a [Named] facet delegate over this [ExprValue]. */ fun ExprValue.namedValue(nameValue: ExprValue): ExprValue = object : ExprValue by this, Named { + override val value = this@namedValue override val name = nameValue override fun asFacet(type: Class?): T? = downcast(type) ?: this@namedValue.asFacet(type) diff --git a/partiql-lang/src/main/kotlin/org/partiql/lang/eval/Named.kt b/partiql-lang/src/main/kotlin/org/partiql/lang/eval/Named.kt index 2122f6873a..42c0709e1e 100644 --- a/partiql-lang/src/main/kotlin/org/partiql/lang/eval/Named.kt +++ b/partiql-lang/src/main/kotlin/org/partiql/lang/eval/Named.kt @@ -26,4 +26,9 @@ interface Named { * a `struct` or an `int` for values that have some ordinal in a collection. */ val name: ExprValue + + /** + * TODO ALAN: docs + */ + val value: ExprValue } diff --git a/partiql-lang/src/main/kotlin/org/partiql/lang/eval/StructExprValue.kt b/partiql-lang/src/main/kotlin/org/partiql/lang/eval/StructExprValue.kt index 75eccc52f3..bc0b78369c 100644 --- a/partiql-lang/src/main/kotlin/org/partiql/lang/eval/StructExprValue.kt +++ b/partiql-lang/src/main/kotlin/org/partiql/lang/eval/StructExprValue.kt @@ -26,7 +26,7 @@ enum class StructOrdering { * Provides a [ExprValueType.STRUCT] implementation lazily backed by a sequence. */ internal open class StructExprValue( - private val ordering: StructOrdering, + internal val ordering: StructOrdering, private val sequence: Sequence ) : BaseExprValue() { diff --git a/partiql-lang/src/test/kotlin/org/partiql/lang/eval/EvaluatingCompilerExcludeTests.kt b/partiql-lang/src/test/kotlin/org/partiql/lang/eval/EvaluatingCompilerExcludeTests.kt new file mode 100644 index 0000000000..40ea677a64 --- /dev/null +++ b/partiql-lang/src/test/kotlin/org/partiql/lang/eval/EvaluatingCompilerExcludeTests.kt @@ -0,0 +1,795 @@ +package org.partiql.lang.eval + +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ArgumentsSource +import org.partiql.lang.eval.evaluatortestframework.CompilerPipelineFactory +import org.partiql.lang.eval.evaluatortestframework.EvaluatorTestAdapter +import org.partiql.lang.eval.evaluatortestframework.EvaluatorTestCase +import org.partiql.lang.eval.evaluatortestframework.PipelineEvaluatorTestAdapter +import org.partiql.lang.util.ArgumentsProviderBase + +class EvaluatingCompilerExcludeTests : EvaluatorTestBase() { + + private val testHarness: EvaluatorTestAdapter = PipelineEvaluatorTestAdapter(CompilerPipelineFactory()) + + class ExcludeTests : ArgumentsProviderBase() { + override fun getParameters(): List = listOf( + EvaluatorTestCase( + "SELECT t.* EXCLUDE t.a FROM <<{'a': {'b': 2}, 'foo': 'bar', 'foo2': 'bar2'}>> AS t", + """<<{'foo': 'bar', 'foo2': 'bar2'}>>""" + ), + EvaluatorTestCase( // EXCLUDE tuple attr using bracket syntax; same output as above + "SELECT t.* EXCLUDE t['a'] FROM <<{'a': {'b': 2}, 'foo': 'bar', 'foo2': 'bar2'}>> AS t", + """<<{'foo': 'bar', 'foo2': 'bar2'}>>""" + ), + EvaluatorTestCase( // multiple binding tuples; select star + """ + SELECT * + EXCLUDE t.a FROM + << + {'a': {'b': 1}, 'foo': 'bar', 'foo2': 'bar1'}, + {'a': {'b': 2}, 'foo': 'bar', 'foo2': 'bar2'}, + {'a': {'b': 3}, 'foo': 'bar', 'foo2': 'bar3'} + >> AS t + """.trimIndent(), + """<< + {'foo': 'bar', 'foo2': 'bar1'}, + {'foo': 'bar', 'foo2': 'bar2'}, + {'foo': 'bar', 'foo2': 'bar3'} + >>""".trimMargin() + ), + EvaluatorTestCase( // multiple binding tuples; select list + """ + SELECT t.* + EXCLUDE t.a FROM + << + {'a': {'b': 1}, 'foo': 'bar', 'foo2': 'bar1'}, + {'a': {'b': 2}, 'foo': 'bar', 'foo2': 'bar2'}, + {'a': {'b': 3}, 'foo': 'bar', 'foo2': 'bar3'} + >> AS t + """.trimIndent(), + """<< + {'foo': 'bar', 'foo2': 'bar1'}, + {'foo': 'bar', 'foo2': 'bar2'}, + {'foo': 'bar', 'foo2': 'bar3'} + >>""".trimMargin() + ), + EvaluatorTestCase( // multiple binding tuples; + """ + SELECT VALUE t + EXCLUDE t.a FROM + << + {'a': {'b': 1}, 'foo': 'bar', 'foo2': 'bar1'}, + {'a': {'b': 2}, 'foo': 'bar', 'foo2': 'bar2'}, + {'a': {'b': 3}, 'foo': 'bar', 'foo2': 'bar3'} + >> AS t + """.trimIndent(), + """<< + {'foo': 'bar', 'foo2': 'bar1'}, + {'foo': 'bar', 'foo2': 'bar2'}, + {'foo': 'bar', 'foo2': 'bar3'} + >>""".trimMargin() + ), + EvaluatorTestCase( // EXCLUDE deeper nested field; no fields remaining + "SELECT t.* EXCLUDE t.a.b FROM <<{'a': {'b': 2}, 'foo': 'bar', 'foo2': 'bar2'}>> AS t", + """<<{'a': {}, 'foo': 'bar', 'foo2': 'bar2'}>>""" + ), + EvaluatorTestCase( // EXCLUDE deeper nested field; other field remaining + "SELECT t.* EXCLUDE t.a.b FROM <<{'a': {'b': 2, 'c': 3}, 'foo': 'bar', 'foo2': 'bar2'}>> AS t", + """<<{'a': {'c': 3}, 'foo': 'bar', 'foo2': 'bar2'}>>""" + ), + EvaluatorTestCase( // EXCLUDE multiple nested paths + "SELECT t.* EXCLUDE t.a.c, t.a.d, t.foo FROM <<{'a': {'b': 2, 'c': 3, 'd': 4}, 'foo': 'bar', 'foo2': 'bar2'}>> AS t", + """<<{'a': {'b': 2}, 'foo2': 'bar2'}>>""" + ), + EvaluatorTestCase( // EXCLUDE overlapping paths + """ + SELECT t.* + EXCLUDE t.a.c, t.a -- `t.a` and `t.a.c` overlap; still exclude `t.a` + FROM <<{'a': {'b': 2, 'c': 3, 'd': 4}, 'foo': 'bar', 'foo2': 'bar2'}>> AS t + """.trimIndent(), + """<<{'foo': 'bar', 'foo2': 'bar2'}>>""" + ), + EvaluatorTestCase( // EXCLUDE select star + """SELECT * EXCLUDE c.ssn FROM [ + { + 'name': 'Alan', + 'custId': 1, + 'address': { + 'city': 'Seattle', + 'zipcode': 98109, + 'street': '123 Seaplane Dr.' + }, + 'ssn': 123456789 + } + ] AS c + """.trimIndent(), + """ + << + { + 'name': 'Alan', + 'custId': 1, + 'address': { + 'city': 'Seattle', + 'zipcode': 98109, + 'street': '123 Seaplane Dr.' + } + } + >>""" + ), + EvaluatorTestCase( // EXCLUDE select star with FROM source list + """SELECT * EXCLUDE c.ssn FROM [ + { + 'name': 'Alan', + 'custId': 1, + 'address': { + 'city': 'Seattle', + 'zipcode': 98109, + 'street': '123 Seaplane Dr.' + }, + 'ssn': 123456789 + } + ] AS c + """.trimIndent(), + """ + << + { + 'name': 'Alan', + 'custId': 1, + 'address': { + 'city': 'Seattle', + 'zipcode': 98109, + 'street': '123 Seaplane Dr.' + } + } + >>""" + ), + EvaluatorTestCase( // EXCLUDE select star with multiple paths and FROM source list + """ + SELECT * EXCLUDE c.ssn, c.address.street FROM [ + { + 'name': 'Alan', + 'custId': 1, + 'address': { + 'city': 'Seattle', + 'zipcode': 98109, + 'street': '123 Seaplane Dr.' + }, + 'ssn': 123456789 + } + ] AS c + """.trimIndent(), + """ + << + { + 'name': 'Alan', + 'custId': 1, + 'address': { + 'city': 'Seattle', + 'zipcode': 98109 + } + } + >>""" + ), + EvaluatorTestCase( // EXCLUDE select star list index and list index field + """ + SELECT * + EXCLUDE + t.a.b.c[0], + t.a.b.c[1].field + FROM [{ + 'a': { + 'b': { + 'c': [ + { + 'field': 0, -- c[0]; entire struct to be excluded + 'index': 0 + }, + { + 'field': 1, -- c[1]; `field` to be excluded + 'index': 1 + }, + { + 'field': 2, -- c[2]; field unchanged + 'index': 2 + } + ] + } + }, + 'foo': 'bar' + }] AS t + """, + """<<{'a': {'b': {'c': [{'index': 1}, {'field': 2, 'index': 2}]}}, 'foo': 'bar'}>>""" + ), + EvaluatorTestCase( // EXCLUDE select star collection index as last step + """ + SELECT * + EXCLUDE + t.a.b.c[0] + FROM [{ + 'a': { + 'b': { + 'c': [0, 1, 2] + } + }, + 'foo': 'bar' + }] AS t + """, + """<<{'a': {'b': {'c': [1, 2]}}, 'foo': 'bar'}>>""" + ), + EvaluatorTestCase( // EXCLUDE select star collection wildcard as last step on list + """ + SELECT * + EXCLUDE + t.a[*] + FROM [{ + 'a': [0, 1, 2] + }] AS t + """, + """<<{'a': []}>>""" + ), + EvaluatorTestCase( // EXCLUDE select star collection wildcard and tuple path on list + """ + SELECT * + EXCLUDE + t.a.b.c[*].field_x + FROM [{ + 'a': { + 'b': { + 'c': [ + { -- c[0]; field_x to be removed + 'field_x': 0, + 'field_y': 0 + }, + { -- c[1]; field_x to be removed + 'field_x': 1, + 'field_y': 1 + }, + { -- c[2]; field_x to be removed + 'field_x': 2, + 'field_y': 2 + } + ] + } + }, + 'foo': 'bar' + }] AS t + """, + """<<{'a': {'b': {'c': [{'field_y': 0}, {'field_y': 1}, {'field_y': 2}]}}, 'foo': 'bar'}>>""" + ), + EvaluatorTestCase( // EXCLUDE select star tuple wildcard as final step + """ + SELECT * + EXCLUDE + t.a.b.c[*].* + FROM [{ + 'a': { + 'b': { + 'c': [ + { -- c[0] + 'field_x': 0, + 'field_y': 0 + }, + { -- c[1] + 'field_x': 1, + 'field_y': 1 + }, + { -- c[2] + 'field_x': 2, + 'field_y': 2 + } + ] + } + }, + 'foo': 'bar' + }] AS t + """, + """<<{'a': {'b': {'c': [{}, {}, {}]}}, 'foo': 'bar'}>>""" + ), + EvaluatorTestCase( // EXCLUDE select star order by + """ + SELECT * + EXCLUDE + t.a + FROM [ + { + 'a': 2, + 'foo': 'bar2' + }, + { + 'a': 1, + 'foo': 'bar1' + }, + { + 'a': 3, + 'foo': 'bar3' + } + ] AS t + ORDER BY t.a + """, + """[{'foo': 'bar1'}, {'foo': 'bar2'}, {'foo': 'bar3'}]""" + ), + EvaluatorTestCase( // EXCLUDE select star with JOIN + """ + SELECT * + EXCLUDE bar.d + FROM + << + {'a': 1, 'b': 11}, + {'a': 2, 'b': 22} + >> AS foo, + << + {'c': 3, 'd': 33}, + {'c': 4, 'd': 44} + >> AS bar + """, + """<<{'a': 1, 'b': 11, 'c': 3}, {'a': 1, 'b': 11, 'c': 4}, {'a': 2, 'b': 22, 'c': 3}, {'a': 2, 'b': 22, 'c': 4}>>""" + ), + EvaluatorTestCase( // EXCLUDE select list with multiple fields in FROM source struct + """ + SELECT t.b EXCLUDE t.b[*].b_1 + FROM << + { + 'a': {'a_1':1,'a_2':2}, + 'b': [ {'b_1':3,'b_2':4}, {'b_1':5,'b_2':6} ], -- every `b_1` to be excluded + 'c': 7, + 'd': 8 + } >> AS t + """, + """<<{'b': [{'b_2': 4}, {'b_2': 6}]}>>""" + ), + EvaluatorTestCase( // EXCLUDE select star with multiple fields in FROM source struct + """ + SELECT * EXCLUDE t.b[*].b_1 + FROM << + { + 'a': {'a_1':1,'a_2':2}, + 'b': [ {'b_1':3,'b_2':4}, {'b_1':5,'b_2':6} ], -- every `b_1` to be excluded + 'c': 7, + 'd': 8 + } >> AS t + """, + """<<{'a': {'a_1': 1, 'a_2': 2}, 'b': [{'b_2': 4}, {'b_2': 6}], 'c': 7, 'd': 8}>>""" + ), + EvaluatorTestCase( // EXCLUDE select value with multiple fields in FROM source struct + """ + SELECT VALUE t.b EXCLUDE t.b[*].b_1 + FROM << + { + 'a': {'a_1':1,'a_2':2}, + 'b': [ {'b_1':3,'b_2':4}, {'b_1':5,'b_2':6} ], + 'c': 7, + 'd': 8 + } >> AS t + """, + """<<[{'b_2': 4}, {'b_2': 6}]>>""" + ), + EvaluatorTestCase( // EXCLUDE select star collection wildcard and nested tuple attr + """ + SELECT * EXCLUDE t.a[*].b.c + FROM << + { + 'a': [ -- `c` attr to be excluded from each element of `a` + { 'b': { 'c': 0, 'd': 'zero' } }, + { 'b': { 'c': 1, 'd': 'one' } }, + { 'b': { 'c': 2, 'd': 'two' } } + ] + } + >> AS t + """, + """<<{'a': [{'b': {'d': 'zero'}}, {'b': {'d': 'one'}}, {'b': {'d': 'two'}}]}>>""" + ), + EvaluatorTestCase( // EXCLUDE select star collection index and nested tuple attr + """ + SELECT * EXCLUDE t.a[1].b.c + FROM << + { + 'a': [ + { 'b': { 'c': 0, 'd': 'zero' } }, + { 'b': { 'c': 1, 'd': 'one' } }, -- exclude `c` from just this index + { 'b': { 'c': 2, 'd': 'two' } } + ] + } + >> AS t + """, + """<<{'a': [{'b': {'c': 0, 'd': 'zero'}}, {'b': {'d': 'one'}}, {'b': {'c': 2, 'd': 'two'}}]}>>""" + ), + EvaluatorTestCase( // EXCLUDE select star collection wildcard and nested tuple wildcard + """ + SELECT * EXCLUDE t.a[*].b.* + FROM << + { + 'a': [ -- exclude all of `b`'s attrs from each element of `a` + { 'b': { 'c': 0, 'd': 'zero' } }, + { 'b': { 'c': 1, 'd': 'one' } }, + { 'b': { 'c': 2, 'd': 'two' } } + ] + } + >> AS t + """, + """<<{'a': [{'b': {}}, {'b': {}}, {'b': {}}]}>>""" + ), + EvaluatorTestCase( // EXCLUDE select star collection index and nested tuple wildcard + """ + SELECT * EXCLUDE t.a[1].b.* + FROM << + { + 'a': [ + { 'b': { 'c': 0, 'd': 'zero' } }, + { 'b': { 'c': 1, 'd': 'one' } }, -- exclude all of `b`'s attrs from just this index + { 'b': { 'c': 2, 'd': 'two' } } + ] + } + >> AS t + """, + """<<{'a': [{'b': {'c': 0, 'd': 'zero'}}, {'b': {}}, {'b': {'c': 2, 'd': 'two'}}]}>>""" + ), + EvaluatorTestCase( // EXCLUDE select star collection wildcard and nested collection wildcard + """ + SELECT * EXCLUDE t.a[*].b.d[*].e + FROM << + { + 'a': [ + { 'b': { 'c': 0, 'd': [{'e': 'zero0', 'f': true}, {'e': 'zero1', 'f': false}] } }, -- all `e` to be excluded + { 'b': { 'c': 1, 'd': [{'e': 'one0', 'f': true}, {'e': 'one1', 'f': false}] } }, -- all `e` to be excluded + { 'b': { 'c': 2, 'd': [{'e': 'two0', 'f': true}, {'e': 'two1', 'f': false}] } } -- all `e` to be excluded + ] + } + >> AS t + """, + """ + << + { + 'a': [ + { 'b': { 'c': 0, 'd': [ { 'f': true }, { 'f': false } ] } }, + { 'b': { 'c': 1, 'd': [ { 'f': true }, { 'f': false } ] } }, + { 'b': { 'c': 2, 'd': [ { 'f': true }, { 'f': false } ] } } + ] + } + >> + """ + ), + EvaluatorTestCase( // EXCLUDE select star collection index and nested collection wildcard + """ + SELECT * EXCLUDE t.a[1].b.d[*].e + FROM << + { + 'a': [ + { 'b': { 'c': 0, 'd': [{'e': 'zero0', 'f': true}, {'e': 'zero1', 'f': false}] } }, + { 'b': { 'c': 1, 'd': [{'e': 'one0', 'f': true}, {'e': 'one1', 'f': false}] } }, -- only `e` from this index to be excluded + { 'b': { 'c': 2, 'd': [{'e': 'two0', 'f': true}, {'e': 'two1', 'f': false}] } } + ] + } + >> AS t + """, + """ + << + { + 'a': [ + { 'b': { 'c': 0, 'd': [ { 'e': 'zero0', 'f': true }, { 'e': 'zero1', 'f': false } ] } }, + { 'b': { 'c': 1, 'd': [ { 'f': true }, { 'f': false } ] } }, + { 'b': { 'c': 2, 'd': [ { 'e': 'two0', 'f': true }, { 'e': 'two1', 'f': false } ] } } + ] + } + >> + """ + ), + EvaluatorTestCase( // EXCLUDE select star collection index and nested collection index + """ + SELECT * EXCLUDE t.a[1].b.d[0].e + FROM << + { + 'a': [ + { 'b': { 'c': 0, 'd': [{'e': 'zero0', 'f': true}, {'e': 'zero1', 'f': false}] } }, + { 'b': { 'c': 1, 'd': [{'e': 'one0', 'f': true}, {'e': 'one1', 'f': false}] } }, -- `e` from 0-th index of `d` to be excluded + { 'b': { 'c': 2, 'd': [{'e': 'two0', 'f': true}, {'e': 'two1', 'f': false}] } } + ] + } + >> AS t + """, + """ + << + { + 'a': [ + { 'b': { 'c': 0, 'd': [ { 'e': 'zero0', 'f': true }, { 'e': 'zero1', 'f': false } ] } }, + { 'b': { 'c': 1, 'd': [ { 'f': true }, { 'e': 'one1', 'f': false } ] } }, + { 'b': { 'c': 2, 'd': [ { 'e': 'two0', 'f': true }, { 'e': 'two1', 'f': false } ] } } + ] + } + >> + """ + ), + EvaluatorTestCase( // EXCLUDE select star tuple wildcard and subsequent tuple attr + """ + SELECT * + EXCLUDE + t.a.*.bar + FROM [ + { + 'a': { + 'b': { 'foo': 1, 'bar': 2 }, + 'c': { 'foo': 11, 'bar': 22 }, + 'd': { 'foo': 111, 'bar': 222 } + }, + 'foo': 'bar' + } + ] AS t + """, + """ + << + { + 'a': { + 'b': { 'foo': 1 }, + 'c': { 'foo': 11 }, + 'd': { 'foo': 111 } + }, + 'foo': 'bar' + } + >> + """ + ), + EvaluatorTestCase( // EXCLUDE select star with ORDER BY + """ + SELECT * + EXCLUDE + t.a + FROM [ + { + 'a': 2, + 'foo': 'bar2' + }, + { + 'a': 1, + 'foo': 'bar1' + }, + { + 'a': 3, + 'foo': 'bar3' + } + ] AS t + ORDER BY t.a + """, + """ + [ + { + 'foo': 'bar1' + }, + { + 'foo': 'bar2' + }, + { + 'foo': 'bar3' + } + ] + """ + ), + EvaluatorTestCase( // exclude select star with GROUP BY + """ + SELECT * + EXCLUDE g[*].t.c + FROM + << + { 'a': 1, 'b': 11, 'c': 111 }, + { 'a': 1, 'b': 22, 'c': 222 }, + { 'a': 2, 'b': 33, 'c': 333 } + >> AS t + GROUP BY t.a AS a GROUP AS g + """, + """ + << + { + 'a': 1, + 'g': << + { 't': { 'a': 1, 'b': 11 } }, + { 't': { 'a': 1, 'b': 22 } } + >> + }, + { + 'a': 2, + 'g': << + { 't': { 'a': 2, 'b': 33 } } + >> + } + >> + """ + ), + EvaluatorTestCase( // EXCLUDE select star with DISTINCT + """ + SELECT DISTINCT * + EXCLUDE t.a + FROM + << + { 'a': 1, 'b': 11, 'c': 111 }, + { 'a': 2, 'b': 11, 'c': 111 }, -- `b` and `c` same as above; `a` is different but will be excluded + { 'a': 1, 'b': 22, 'c': 222 } -- `b` and `c` different from above two rows; will be kept + >> AS t + """, + """ + << + {'b': 11, 'c': 111}, + {'b': 22, 'c': 222} + >> + """ + ), + EvaluatorTestCase( // EXCLUDE select star with ORDER BY, LIMIT, OFFSET + """ + SELECT DISTINCT * + EXCLUDE t.a + FROM + << + { 'a': 1, 'b': 11, 'c': 111 }, + { 'a': 2, 'b': 22, 'c': 222 }, + { 'a': 3, 'b': 33, 'c': 333 }, -- kept + { 'a': 4, 'b': 44, 'c': 444 }, -- kept + { 'a': 5, 'b': 55, 'c': 555 } + >> AS t + ORDER BY a + LIMIT 2 + OFFSET 2 + """, + """ + [ + {'b': 33, 'c': 333}, + {'b': 44, 'c': 444} + ] + """ + ), + EvaluatorTestCase( // EXCLUDE PIVOT + """ + PIVOT t.v AT t.attr + EXCLUDE t.v[*].excludeValue + FROM + << + { 'attr': 'a', 'v': [{'keepValue': 1, 'excludeValue': 11}, {'keepValue': 4, 'excludeValue': 44}] }, + { 'attr': 'b', 'v': [{'keepValue': 2, 'excludeValue': 22}, {'keepValue': 5, 'excludeValue': 55}] }, + { 'attr': 'c', 'v': [{'keepValue': 3, 'excludeValue': 33}, {'keepValue': 6, 'excludeValue': 66}] } + >> AS t + """, + """ + { + 'a': [{'keepValue': 1}, {'keepValue': 4}], + 'b': [{'keepValue': 2}, {'keepValue': 5}], + 'c': [{'keepValue': 3}, {'keepValue': 6}] + } + """ + ), + EvaluatorTestCase( // EXCLUDE UNPIVOT + """ + SELECT v, attr + EXCLUDE v.foo + FROM UNPIVOT + { + 'a': {'foo': 1, 'bar': 11}, + 'a': {'foo': 2, 'bar': 22}, + 'b': {'foo': 3, 'bar': 33} + } AS v AT attr + """, + """ + << + {'v': {'bar': 11}, 'attr': 'a'}, + {'v': {'bar': 22}, 'attr': 'a'}, + {'v': {'bar': 33}, 'attr': 'b'} + >> + """ + ), + EvaluatorTestCase( // EXCLUDE collection index on list + """ + SELECT * + EXCLUDE t.a[1] + FROM + << + {'a': [0, 1, 2]} -- index `1` to be excluded + >> AS t + """, + """<<{'a': [0, 2]}>>""" + ), + EvaluatorTestCase( // EXCLUDE collection index on bag -- nothing gets excluded; no error + """ + SELECT * + EXCLUDE t.a[1] + FROM + << + {'a': <<0, 1, 2>>} + >> AS t + """, + """<<{'a': <<0, 1, 2>>}>>""" + ), + EvaluatorTestCase( // EXCLUDE collection index on sexp + """ + SELECT * + EXCLUDE t.a[1] + FROM + << + {'a': `(0 1 2)`} -- index `1` to be excluded + >> AS t + """, + """<<{'a': `(0 2)`}>>""" + ), + EvaluatorTestCase( // EXCLUDE collection wildcard on list + """ + SELECT * + EXCLUDE t.a[*] + FROM + << + {'a': [0, 1, 2]} -- all indexes to be excluded; empty list as result + >> AS t + """, + """<<{'a': []}>>""" + ), + EvaluatorTestCase( // EXCLUDE collection wildcard on bag + """ + SELECT * + EXCLUDE t.a[*] + FROM + << + {'a': <<0, 1, 2>>} -- all indexes to be excluded; empty bag as result + >> AS t + """, + """<<{'a': <<>>}>>""" + ), + EvaluatorTestCase( // EXCLUDE collection wildcard on sexp + """ + SELECT * + EXCLUDE t.a[*] + FROM + << + {'a': `(0 1 2)`} -- all indexes to be excluded; empty sexp as result + >> AS t + """, + """<<{'a': `()`}>>""" + ), + EvaluatorTestCase( // EXCLUDE with duplicates + """ + SELECT * + EXCLUDE t.a + FROM + << + { + 'a': 1, -- to be excluded + 'a': 2, -- to be excluded + 'a': 3, -- to be excluded + 'b': 4, + 'c': 5 + } + >> AS t + """, + """<<{'b': 4, 'c': 5}>>""" + ), + EvaluatorTestCase( // EXCLUDE with non-existent paths; no error + """ + SELECT * + EXCLUDE t.path_does_not_exist, t.path_does_not_exist.*.foo, t.a['does not exist'], t.a + FROM + <<{'a': 1, 'b': 2}>> AS t -- only exclude `a` + """, + """<<{'b': 2}>>""" + ), + EvaluatorTestCase( // EXCLUDE with different FROM source bindings + """ + SELECT * + EXCLUDE t.a[*].bar, t.a.bar, t.a.*.bar -- EXCLUDE all `bar` + FROM + << + {'a': [{'foo': 0, 'bar': 1, 'baz': 2}, {'foo': 3, 'bar': 4, 'baz': 5}]}, + {'a': {'foo': 6, 'bar': 7, 'baz': 8}}, + {'a': {'a1': {'foo': 9, 'bar': 10, 'baz': 11}, 'a2': {'foo': 12, 'bar': 13, 'baz': 14}}} + >> AS t + """, + """ + << + {'a': [{'foo': 0, 'baz': 2}, {'foo': 3, 'baz': 5}]}, + {'a': {'foo': 6, 'baz': 8}}, + {'a': {'a1': {'foo': 9, 'baz': 11}, 'a2': {'foo': 12, 'baz': 14}}} + >> + """ + ), + ) + } + + @ParameterizedTest + @ArgumentsSource(ExcludeTests::class) + fun validExcludeTests(tc: EvaluatorTestCase) = testHarness.runEvaluatorTestCase( + tc, + EvaluationSession.standard() + ) +} diff --git a/partiql-lang/src/test/kotlin/org/partiql/lang/eval/EvaluatingCompilerSelectStarTests.kt b/partiql-lang/src/test/kotlin/org/partiql/lang/eval/EvaluatingCompilerSelectStarTests.kt index 7a95ed7fa5..9c4222a5bb 100644 --- a/partiql-lang/src/test/kotlin/org/partiql/lang/eval/EvaluatingCompilerSelectStarTests.kt +++ b/partiql-lang/src/test/kotlin/org/partiql/lang/eval/EvaluatingCompilerSelectStarTests.kt @@ -26,7 +26,8 @@ class EvaluatingCompilerSelectStarTests : EvaluatorTestBase() { class AddressedExprValue( private val innerExprValue: ExprValue, override val name: ExprValue, - override val address: ExprValue + override val address: ExprValue, + override val value: ExprValue, ) : ExprValue by innerExprValue, Named, Addressed { // Need to override the asFacet provided by [innerExprValue] since it won't implement either facet. @@ -38,7 +39,8 @@ class EvaluatingCompilerSelectStarTests : EvaluatorTestBase() { AddressedExprValue( ExprValue.of(ion.singleValue(ionText)), ExprValue.newInt(index), - ExprValue.newString(address) + ExprValue.newString(address), + ExprValue.missingValue ) @Test diff --git a/partiql-lang/src/test/kotlin/org/partiql/lang/planner/transforms/PartiQLSchemaInferencerTests.kt b/partiql-lang/src/test/kotlin/org/partiql/lang/planner/transforms/PartiQLSchemaInferencerTests.kt index edd60a52f8..4ba3e365ff 100644 --- a/partiql-lang/src/test/kotlin/org/partiql/lang/planner/transforms/PartiQLSchemaInferencerTests.kt +++ b/partiql-lang/src/test/kotlin/org/partiql/lang/planner/transforms/PartiQLSchemaInferencerTests.kt @@ -1589,7 +1589,7 @@ class PartiQLSchemaInferencerTests { ) ), SuccessTestCase( - name = "EXCLUDE SELECT star list tuple wildcard", + name = "EXCLUDE SELECT star tuple wildcard as last step", query = """SELECT * EXCLUDE t.a.b.c[*].* @@ -1676,7 +1676,7 @@ class PartiQLSchemaInferencerTests { ) ), SuccessTestCase( - name = "EXCLUDE SELECT star with JOINs", + name = "EXCLUDE SELECT star with JOIN", query = """SELECT * EXCLUDE bar.d FROM @@ -1938,9 +1938,9 @@ class PartiQLSchemaInferencerTests { FROM << { 'a': [ - { 'b': { 'c': 0, 'd': [{'e': 'zero', 'f': true}] } }, - { 'b': { 'c': 1, 'd': [{'e': 'one', 'f': true}] } }, - { 'b': { 'c': 2, 'd': [{'e': 'two', 'f': true}] } } + { 'b': { 'c': 0, 'd': [ { 'e': 'zero', 'f': true } ] } }, + { 'b': { 'c': 1, 'd': [ { 'e': 'one', 'f': true } ] } }, + { 'b': { 'c': 2, 'd': [ { 'e': 'two', 'f': true } ] } } ] } >> AS t""",