diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index decbef88db..b802c24a78 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,6 +19,8 @@ jobs: java: [11, 17] steps: - uses: actions/checkout@v2 + with: + submodules: recursive - name: Use Java ${{ matrix.java }} uses: actions/setup-java@v1 with: @@ -30,6 +32,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + submodules: recursive - name: Use Java 17 uses: actions/setup-java@v1 with: @@ -50,17 +54,17 @@ jobs: file: lang/build/reports/jacoco/test/jacocoTestReport.xml flags: LANG - - name: Upload PTS coverage + - name: Upload PARTIQL_PTS coverage uses: codecov/codecov-action@v3 with: - file: pts/build/reports/jacoco/test/jacocoTestReport.xml - flags: PTS + file: partiql-pts/build/reports/jacoco/test/jacocoTestReport.xml + flags: PARTIQL_PTS - - name: Upload TEST_SCRIPT coverage + - name: Upload PARTIQL_TEST_SCRIPT coverage uses: codecov/codecov-action@v3 with: - file: testscript/build/reports/jacoco/test/jacocoTestReport.xml - flags: TEST_SCRIPT + file: partiql-testscript/build/reports/jacoco/test/jacocoTestReport.xml + flags: PARTIQL_TEST_SCRIPT - name: Upload EXAMPLES coverage uses: codecov/codecov-action@v3 diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..d5539cefcb --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "test/partiql-tests"] + path = test/partiql-tests + url = https://github.com/partiql/partiql-tests.git diff --git a/build.gradle b/build.gradle index 9a713d5f86..864f4c9cfd 100644 --- a/build.gradle +++ b/build.gradle @@ -156,7 +156,7 @@ subprojects { proj -> processResources.dependsOn generateVersionAndHashProperties }) - if (project.name == "partiql-isl") { + if (project.name == "partiql-isl" || project.name == "partiql-tests-runner") { // skip since partiql-isl uses the conventional Gradle layout return } diff --git a/settings.gradle b/settings.gradle index eb78a5a5cf..1819c96333 100644 --- a/settings.gradle +++ b/settings.gradle @@ -18,9 +18,10 @@ rootProject.name = 'PartiQL' include('lang', 'cli', 'examples', - 'testscript', - 'pts', 'extensions', 'partiql-grammar', - 'lib:partiql-isl' + 'lib:partiql-isl', + 'test:partiql-pts', + 'test:partiql-tests-runner', + 'test:partiql-testscript' ) diff --git a/pts/build.gradle b/test/partiql-pts/build.gradle similarity index 96% rename from pts/build.gradle rename to test/partiql-pts/build.gradle index 69c0b98880..49e9a8d031 100644 --- a/pts/build.gradle +++ b/test/partiql-pts/build.gradle @@ -33,6 +33,6 @@ dependencies { // test-time dependencies testImplementation project(':lang') - testImplementation project(':testscript') + testImplementation project(':test:partiql-testscript') testImplementation 'org.junit.vintage:junit-vintage-engine:5.7.0' } diff --git a/pts/src/.gitkeep b/test/partiql-pts/src/.gitkeep similarity index 100% rename from pts/src/.gitkeep rename to test/partiql-pts/src/.gitkeep diff --git a/pts/test-resources/junit-platform.properties b/test/partiql-pts/test-resources/junit-platform.properties similarity index 100% rename from pts/test-resources/junit-platform.properties rename to test/partiql-pts/test-resources/junit-platform.properties diff --git a/pts/test/org/partiql/lang/pts/PartiQlPtsEvaluator.kt b/test/partiql-pts/test/org/partiql/lang/pts/PartiQlPtsEvaluator.kt similarity index 100% rename from pts/test/org/partiql/lang/pts/PartiQlPtsEvaluator.kt rename to test/partiql-pts/test/org/partiql/lang/pts/PartiQlPtsEvaluator.kt diff --git a/pts/test/org/partiql/lang/pts/PtsTest.kt b/test/partiql-pts/test/org/partiql/lang/pts/PtsTest.kt similarity index 83% rename from pts/test/org/partiql/lang/pts/PtsTest.kt rename to test/partiql-pts/test/org/partiql/lang/pts/PtsTest.kt index fa12cdb9eb..80e5819a31 100644 --- a/pts/test/org/partiql/lang/pts/PtsTest.kt +++ b/test/partiql-pts/test/org/partiql/lang/pts/PtsTest.kt @@ -13,5 +13,5 @@ class PtsTest : Junit4PtsTest() { override fun getEvaluator(): Evaluator = PartiQlPtsEvaluator(PtsEquality.getDefault()) - override fun getPtsFilePaths() = listOf("../testscript/pts/test-scripts") + override fun getPtsFilePaths() = listOf("../partiql-testscript/pts/test-scripts") } diff --git a/test/partiql-tests b/test/partiql-tests new file mode 160000 index 0000000000..a19b3f4027 --- /dev/null +++ b/test/partiql-tests @@ -0,0 +1 @@ +Subproject commit a19b3f402752cdb103211b0c1f46410b3d43237e diff --git a/test/partiql-tests-runner/README.md b/test/partiql-tests-runner/README.md new file mode 100644 index 0000000000..df87dc15b5 --- /dev/null +++ b/test/partiql-tests-runner/README.md @@ -0,0 +1,12 @@ +# PartiQL Kotlin Test Runner + +This package checks whether the conformance tests (in [partiql-tests](https://github.com/partiql/partiql-tests)) run +using the `partiql-lang-kotlin` implementation return the correct result. + +This package enables: +1. Verifying conformance tests are defined correctly (e.g. verify evaluation environment is not missing a table) +2. Identifying areas in the Kotlin implementation that diverge from the [PartiQL Specification](https://partiql.org/assets/PartiQL-Specification.pdf) + +Eventually, the `partiql-test-runner` module will replace the `partiql-pts` and `partiql-testscript` modules along with some other +tests in `lang` that were ported to `partiql-tests` +(see [partiql-lang-kotlin#789](https://github.com/partiql/partiql-lang-kotlin/issues/789)). diff --git a/test/partiql-tests-runner/build.gradle b/test/partiql-tests-runner/build.gradle new file mode 100644 index 0000000000..a3b411a76d --- /dev/null +++ b/test/partiql-tests-runner/build.gradle @@ -0,0 +1,33 @@ +/* + * Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at: + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ + +plugins { + id 'java-library' +} + +dependencies { + implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") + + testImplementation("org.jetbrains.kotlin:kotlin-test") + testImplementation("org.junit.jupiter:junit-jupiter-params:5.8.2") + + testImplementation(project(':lang')) + +} + +test { + useJUnitPlatform() + environment("PARTIQL_EVAL_TESTS_DATA", file("../partiql-tests/partiql-tests-data/eval/").absolutePath) + environment("PARTIQL_EVAL_EQUIV_TESTS_DATA", file("../partiql-tests/partiql-tests-data/eval-equiv/").absolutePath) +} diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ArgumentsProviderBase.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ArgumentsProviderBase.kt new file mode 100644 index 0000000000..771c7bc25a --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ArgumentsProviderBase.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at: + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ + +package org.partiql.runner + +import org.junit.jupiter.api.extension.ExtensionContext +import org.junit.jupiter.params.provider.Arguments +import org.junit.jupiter.params.provider.ArgumentsProvider +import java.util.stream.Stream + +/** + * Reduces some of the boilerplate associated with the style of parameterized testing frequently + * utilized in this package. + * + * Since JUnit5 requires `@JvmStatic` on its `@MethodSource` argument factory methods, this requires all + * of the argument lists to reside in the companion object of a test class. This can be annoying since it + * forces the test to be separated from its tests cases. + * + * Classes that derive from this class can be defined near the `@ParameterizedTest` functions instead. + */ +abstract class ArgumentsProviderBase : ArgumentsProvider { + + abstract fun getParameters(): List + + @Throws(Exception::class) + override fun provideArguments(extensionContext: ExtensionContext): Stream? { + return getParameters().map { Arguments.of(it) }.stream() + } +} diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Parse.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Parse.kt new file mode 100644 index 0000000000..43a98091b6 --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Parse.kt @@ -0,0 +1,128 @@ +package org.partiql.runner + +import com.amazon.ion.IonList +import com.amazon.ion.IonStruct +import com.amazon.ion.IonSymbol +import com.amazon.ion.IonType +import com.amazon.ion.IonValue +import org.partiql.lang.eval.CompileOptions +import org.partiql.lang.eval.TypingMode +import org.partiql.lang.util.asIonStruct +import org.partiql.lang.util.stringValue + +/** + * Parses the [testStruct] to a list of [TestCase]s with respect to the environments and equivalence classes provided + * in the [curNamespace]. + */ +private fun parseTestCase(testStruct: IonStruct, curNamespace: Namespace): List { + val testCases = mutableListOf() + val name = testStruct.get("name").stringValue() ?: error("Expected test case to have field `name`") + val statement = testStruct.get("statement") ?: error("Expected test case to have field `statement`") + val env = testStruct.get("env") ?: curNamespace.env + val assertList = when (val assert = testStruct.get("assert") ?: error("Expected test case to have field `assert`")) { + is IonStruct -> listOf(assert) + is IonList -> assert.toList() + else -> error("Expect `assert` field to be an IonStruct or IonList") + } + + testCases += assertList.map { assertion -> + val assertionStruct = assertion as IonStruct + val evalModeList = when (val evalModeIonValue = assertionStruct.get("evalMode")) { + is IonSymbol -> listOf(evalModeIonValue.stringValue()) + is IonList -> evalModeIonValue.toList().map { it.stringValue() } + else -> error("evalMode expects IonSymbol or IonList") + } + + evalModeList.map { evalMode -> + val compileOption = when (evalMode) { + "EvalModeError" -> CompileOptions.build { typingMode(TypingMode.LEGACY) } + "EvalModeCoerce" -> CompileOptions.build { typingMode(TypingMode.PERMISSIVE) } + else -> error("unsupported eval modes") + } + val evalResult: Assertion = when (assertionStruct.get("result").stringValue()) { + "EvaluationSuccess" -> Assertion.EvaluationSuccess(assertionStruct.get("output")) + "EvaluationFail" -> Assertion.EvaluationFailure + else -> error("expected one of EvaluationSuccess or EvaluationFail") + } + + when (statement.type) { + // statement being an IonString indicates that this is an Eval test case + IonType.STRING -> EvalTestCase( + name = name, + statement = statement.stringValue() ?: error("Expected `statement` to be a string"), + env = env.asIonStruct(), + compileOptions = compileOption, + assertion = evalResult + ) + // statement being an IonSymbol indicates that this is an eval equivalence test case + IonType.SYMBOL -> { + val equivClassId = statement.stringValue() ?: error("Expected `statement` to be a symbol") + EvalEquivTestCase( + name = name, + statements = curNamespace.equivClasses[equivClassId] ?: error("Equiv class $equivClassId not defined in current namespace"), + env = env.asIonStruct(), + compileOptions = compileOption, + assertion = evalResult + ) + } + else -> TODO("Support other test case categories: https://github.com/partiql/partiql-tests/issues/35") + } + } + }.flatten() + return testCases +} + +private fun parseEquivalenceClass(equivClassStruct: IonStruct): EquivalenceClass { + val id = equivClassStruct.get("id") ?: error("Expected field `id` for equivalence class struct: $equivClassStruct ") + val statements = equivClassStruct.get("statements") ?: error("Expected field `statements` for equivalence class struct: $equivClassStruct") + + val idAsString = id.stringValue() ?: error("Expected `id` to be an IonSymbol") + val statementsAsStrings = (statements as IonList).map { statement -> + statement.stringValue() ?: error("Expected each statement within equivalence class to be a string $statement") + } + return EquivalenceClass( + idAsString, + statementsAsStrings + ) +} + +/** + * Parses [data] with the [curNamespace] into a new [Namespace]. + */ +internal fun parseNamespace(curNamespace: Namespace, data: IonValue): Namespace { + return when (data) { + is IonList -> { + val newNamespace = Namespace( + env = curNamespace.env, + namespaces = mutableListOf(), + testCases = mutableListOf(), + equivClasses = mutableMapOf() + ) + data.forEach { d -> + parseNamespace(newNamespace, d) + } + curNamespace.namespaces.add(newNamespace) + curNamespace + } + is IonStruct -> { + // environment, equivalence class, or test case. add to current namespace + val annotations = data.typeAnnotations + when { + annotations.contains("envs") -> { + curNamespace.env = data + } + annotations.contains("equiv_class") -> { + // equivalence class + val equivClass = parseEquivalenceClass(data) + curNamespace.equivClasses[equivClass.id] = equivClass.statements + } + annotations.isEmpty() -> { + // test case + curNamespace.testCases.addAll(parseTestCase(data, curNamespace)) + } + } + curNamespace + } + else -> error("Document parsing requires an IonList or IonStruct") + } +} diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/PartiQLEqualityChecker.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/PartiQLEqualityChecker.kt new file mode 100644 index 0000000000..dd4bb6617e --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/PartiQLEqualityChecker.kt @@ -0,0 +1,106 @@ +package org.partiql.runner + +import com.amazon.ion.IonDecimal +import com.amazon.ion.IonList +import com.amazon.ion.IonSequence +import com.amazon.ion.IonSexp +import com.amazon.ion.IonStruct +import com.amazon.ion.IonTimestamp +import com.amazon.ion.IonType +import com.amazon.ion.IonValue +import java.lang.IllegalArgumentException + +/** + * Checks the equality of two PartiQL values defined using its [IonValue] representation. This definition first requires + * the types to be the same, whereas PartiQL's equal operator can assert equivalence with implicit type coercion. This + * differs from Ion's definition of equality in the following ways: + * 1. Bag comparison checks ignore ordering of IonLists + * 2. Null checks check for `missing` annotation + */ +class PartiQLEqualityChecker { + fun areEqual(left: IonValue, right: IonValue): Boolean { + if (left.type != right.type) { + return false + } + + // typed nulls + if (!left.isMissing() && !right.isMissing() && (left.isNullValue || right.isNullValue)) { + return left.isNullValue && right.isNullValue + } + + return when (left.type!!) { + IonType.NULL -> { + if (left.isMissing() || right.isMissing()) { + left.isMissing() && right.isMissing() + } else { + right.isNullValue + } + } + IonType.BOOL, + IonType.INT, + IonType.FLOAT, + IonType.SYMBOL, + IonType.STRING, + IonType.CLOB, + IonType.BLOB -> left == right + IonType.DECIMAL -> { + val leftDecimal = left as IonDecimal + val rightDecimal = right as IonDecimal + + // we use compareTo to ignore differences in scale since + // for PartiQL 1.0 == 1.00 while that's not true for Ion + leftDecimal.bigDecimalValue().compareTo(rightDecimal.bigDecimalValue()) == 0 + } + IonType.TIMESTAMP -> { + val leftTimestamp = left as IonTimestamp + val rightTimestamp = right as IonTimestamp + + leftTimestamp.timestampValue().compareTo(rightTimestamp.timestampValue()) == 0 + } + IonType.LIST -> { + val leftList = left as IonList + val rightList = right as IonList + + if (leftList.isBag() || rightList.isBag()) { + ptsBagEquals(leftList, rightList) + } else { + ptsSequenceEquals(leftList, rightList) + } + } + IonType.SEXP -> ptsSequenceEquals(left as IonSexp, right as IonSexp) + IonType.STRUCT -> left as IonStruct == right as IonStruct + IonType.DATAGRAM -> throw IllegalArgumentException("DATAGRAM are not a valid type in CTS") + } + } + + private fun IonList.isBag(): Boolean = + this.typeAnnotations.size == 1 && + this.typeAnnotations[0] == BAG_ANNOTATION + + private fun ptsSequenceEquals(left: IonSequence, right: IonSequence): Boolean = + left.size == right.size && + left.asSequence() + .mapIndexed { index, leftElement -> index to leftElement } + .all { (index, leftElement) -> areEqual(leftElement, right[index]) } + + // bags can contain repeated elements, so they are equal if and only if: + // * Same size + // * All elements in one are contained in the other at the same quantities + private fun ptsBagEquals(left: IonList, right: IonList): Boolean = + when { + left.size != right.size -> false + left.isBag() && right.isBag() -> { + left.all { leftEl -> + val leftQtd = left.count { areEqual(leftEl, it) } + val rightQtd = right.count { areEqual(leftEl, it) } + + leftQtd == rightQtd + } + } + else -> false + } + + private fun IonValue.isMissing(): Boolean = this.isNullValue && + this.hasTypeAnnotation(MISSING_ANNOTATION) && + this.typeAnnotations.size == 1 +} diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Schema.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Schema.kt new file mode 100644 index 0000000000..f1883a5391 --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Schema.kt @@ -0,0 +1,43 @@ +package org.partiql.runner + +import com.amazon.ion.IonStruct +import com.amazon.ion.IonValue +import org.partiql.lang.eval.CompileOptions + +data class Namespace( + var env: IonStruct, + val namespaces: MutableList, + val testCases: MutableList, + val equivClasses: MutableMap> +) + +data class EquivalenceClass(val id: String, val statements: List) + +sealed class Assertion { + data class EvaluationSuccess(val expectedResult: IonValue) : Assertion() + object EvaluationFailure : Assertion() + // TODO: other assertion and test categories: https://github.com/partiql/partiql-tests/issues/35 +} + +sealed class TestCase { + abstract val name: String + abstract val env: IonStruct + abstract val compileOptions: CompileOptions + abstract val assertion: Assertion +} + +data class EvalTestCase( + override val name: String, + val statement: String, + override val env: IonStruct, + override val compileOptions: CompileOptions, + override val assertion: Assertion +) : TestCase() + +data class EvalEquivTestCase( + override val name: String, + val statements: List, + override val env: IonStruct, + override val compileOptions: CompileOptions, + override val assertion: Assertion +) : TestCase() diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/TestRunner.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/TestRunner.kt new file mode 100644 index 0000000000..0fbdb177ce --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/TestRunner.kt @@ -0,0 +1,272 @@ +/* + * Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at: + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ + +package org.partiql.runner + +import com.amazon.ion.system.IonSystemBuilder +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ArgumentsSource +import org.partiql.lang.CompilerPipeline +import org.partiql.lang.SqlException +import org.partiql.lang.eval.CompileOptions +import org.partiql.lang.eval.EvaluationSession +import org.partiql.lang.eval.TypingMode +import java.io.File + +private val PARTIQL_EVAL_TEST_DATA_DIR = System.getenv("PARTIQL_EVAL_TESTS_DATA") +private val PARTIQL_EVAL_EQUIV_TEST_DATA_DIR = System.getenv("PARTIQL_EVAL_EQUIV_TESTS_DATA") + +private val ION = IonSystemBuilder.standard().build() + +private val COERCE_EVAL_MODE_COMPILE_OPTIONS = CompileOptions.build { typingMode(TypingMode.PERMISSIVE) } +private val ERROR_EVAL_MODE_COMPILE_OPTIONS = CompileOptions.build { typingMode(TypingMode.LEGACY) } + +/* +The skip lists defined in this file show how the current Kotlin implementation diverges from the PartiQL spec. Most of +the divergent behavior is due to `partiql-lang-kotlin` not having a STRICT/ERROR typing mode. The [LEGACY typing mode](https://github.com/partiql/partiql-lang-kotlin/blob/main/lang/src/org/partiql/lang/eval/CompileOptions.kt#L53-L62) +(which is closer to STRICT/ERROR but not a complete match) was used for testing the STRICT/ERROR typing mode behavior. + +A lot of the other behavior differences is due to not supporting some syntax mentioned in the spec (like `COLL_*` +aggregation functions) and due to not supporting coercions. + +The remaining divergent behavior causing certain conformance tests to fail are likely bugs. Tracking issue: +https://github.com/partiql/partiql-lang-kotlin/issues/804. + */ +private val LANG_KOTLIN_EVAL_SKIP_LIST = listOf( + // from the spec: no explicit CAST to string means the query is "treated as an array navigation with wrongly typed + // data" and will return `MISSING` + Pair("tuple navigation with array notation without explicit CAST to string", COERCE_EVAL_MODE_COMPILE_OPTIONS), + // same as above, but since in error mode, should give an error + Pair("tuple navigation with array notation without explicit CAST to string", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // for the following, partiql-lang-kotlin doesn't have a STRICT/ERROR typing mode. tested using + // partiql-lang-kotlin's LEGACY typing mode, which has some semantic differences from STRICT/ERROR typing mode. + Pair("path on string", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("tuple navigation missing attribute dot notation", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("tuple navigation missing attribute array notation", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("single source FROM with bag and AT clause", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("single source FROM with scalar and AT clause", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("single source FROM with tuple and AT clause", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("single source FROM with absent value null and AT clause", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("single source FROM with absent value missing and AT clause", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("cast and operations with missing argument", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("missing value in arithmetic expression", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("equality of scalar missing", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("arithmetic with null/missing", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // TODO: clarify behavior. spec (section 8) says it should return NULL based on 3-value logic + Pair("missing and true", COERCE_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't currently implement subquery coercion. The inner SFW query returns a bag of two elements that when + // coerced to a scalar should return MISSING in COERCE mode. As a result, `customerName` should be missing from the + // first tuple. + Pair("inner select evaluating to collection with more than one element", COERCE_EVAL_MODE_COMPILE_OPTIONS), + + // coll_* aggregate functions not supported in plk -- results in parser error. coll_* functions will be supported in + // https://github.com/partiql/partiql-lang-kotlin/issues/222 + Pair("coll_count without group by", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("coll_count without group by", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("coll_count with result of subquery", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("coll_count with result of subquery", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // WITH keyword not supported resulting in parse error. windowing will be supported in plk in https://github.com/partiql/partiql-lang-kotlin/issues/603 + Pair("windowing simplified with grouping", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("windowing simplified with grouping", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't have STRICT/ERROR mode. LEGACY mode used which doesn't error when RHS of `IN` expression is not a + // bag, list, or sexp + Pair("notInPredicateSingleExpr", ERROR_EVAL_MODE_COMPILE_OPTIONS), +) + +private val LANG_KOTLIN_EVAL_EQUIV_SKIP_LIST = listOf( + // plk gives a parser error for tuple path navigation in which the path expression is a string literal + // e.g. { 'a': 1, 'b': 2}.'a' -> 1 (see section 4 of spec) + Pair("equiv tuple path navigation with array notation", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv tuple path navigation with array notation", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't support a STRICT/ERROR mode. + Pair("equiv attribute value pair unpivot non-missing", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv attribute value pair unpivot missing", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't support `LATERAL` keyword which results in a parser error + Pair("equiv of comma, cross join, and join", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv of comma, cross join, and join", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't support `TUPLEUNION` function which results in an evaluation error + Pair("equiv tupleunion with select list", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv tupleunion with select list", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't support coercion of subqueries which results in different outputs + Pair("equiv coercion of a SELECT subquery into a scalar", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv coercion of a SELECT subquery into a scalar", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv coercion of a SELECT subquery into an array", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv coercion of a SELECT subquery into an array", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv coercions with explicit literals", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv coercions with explicit literals", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't support `GROUP ALL` and `COLL_*` aggregate functions. Currently, results in a parser error + Pair("equiv group_all", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv group_all", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't support `COLL_*` aggregate functions. Currently, results in an evaluation error + Pair("equiv group by with aggregates", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv group by with aggregates", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't support using aliases created in select list in `GROUP BY` (and `ORDER BY`). GH issue to track: + // https://github.com/partiql/partiql-lang-kotlin/issues/571 + Pair("equiv aliases from select clause", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv aliases from select clause", ERROR_EVAL_MODE_COMPILE_OPTIONS), +) + +/** + * Checks all the PartiQL conformance test data in [PARTIQL_EVAL_TEST_DATA_DIR] conforms to the test data schema. + */ +class TestRunner { + private fun parseTestFile(file: File): Namespace { + val loadedData = file.readText() + val dataInIon = ION.loader.load(loadedData) + val emptyNamespace = Namespace( + env = ION.newEmptyStruct(), + namespaces = mutableListOf(), + testCases = mutableListOf(), + equivClasses = mutableMapOf() + ) + dataInIon.forEach { d -> + parseNamespace(emptyNamespace, d) + } + return emptyNamespace + } + + private fun allTestsFromNamespace(ns: Namespace): List { + return ns.testCases + ns.namespaces.fold(listOf()) { acc, subns -> + acc + allTestsFromNamespace(subns) + } + } + + private fun loadTests(path: String, skipList: List> = emptyList()): List { + val allFiles = File(path).walk() + .filter { it.isFile } + .filter { it.path.endsWith(".ion") } + .toList() + val filesAsNamespaces = allFiles.map { file -> + parseTestFile(file) + } + + val allTestCases = filesAsNamespaces.flatMap { ns -> + allTestsFromNamespace(ns) + }.filter { + // Currently, just filtering the expected failing tests defined by the `skipList`. As an enhancement to the + // test runner, we could instead run the failing tests to assert they still fail. + !skipList.contains(Pair(it.name, it.compileOptions)) + } + return allTestCases + } + + private fun runEvalTestCase(evalTC: EvalTestCase) { + val compilerPipeline = CompilerPipeline.builder(ION).compileOptions(evalTC.compileOptions).build() + val globals = evalTC.env.toExprValue(compilerPipeline.valueFactory).bindings + val session = EvaluationSession.build { globals(globals) } + val expression = compilerPipeline.compile(evalTC.statement) + + try { + val actualResult = expression.eval(session) + when (evalTC.assertion) { + is Assertion.EvaluationSuccess -> { + val actualResultAsIon = actualResult.toIonValue(ION) + if (!PartiQLEqualityChecker().areEqual(evalTC.assertion.expectedResult, actualResultAsIon)) { + error("Expected and actual results differ:\nExpected: ${evalTC.assertion.expectedResult}\nActual: $actualResultAsIon\nMode: ${evalTC.compileOptions.typingMode}") + } + } + is Assertion.EvaluationFailure -> { + error("Expected error to be thrown but none was thrown.\n${evalTC.name}\nActual result: ${actualResult.toIonValue(ION)}") + } + } + } catch (e: SqlException) { + when (evalTC.assertion) { + is Assertion.EvaluationSuccess -> { + error("Expected success but exception thrown") + } + is Assertion.EvaluationFailure -> { + // Expected failure and test threw when evaluated + } + } + } + } + + private fun runEvalEquivTestCase(evalEquivTestCase: EvalEquivTestCase) { + val compilerPipeline = CompilerPipeline.builder(ION).compileOptions(evalEquivTestCase.compileOptions).build() + val globals = evalEquivTestCase.env.toExprValue(compilerPipeline.valueFactory).bindings + val session = EvaluationSession.build { globals(globals) } + val statements = evalEquivTestCase.statements + + statements.forEach { statement -> + val expression = compilerPipeline.compile(statement) + try { + val actualResult = expression.eval(session) + when (evalEquivTestCase.assertion) { + is Assertion.EvaluationSuccess -> { + val actualResultAsIon = actualResult.toIonValue(ION) + if (!PartiQLEqualityChecker().areEqual(evalEquivTestCase.assertion.expectedResult, actualResultAsIon)) { + error("Expected and actual results differ:\nExpected: ${evalEquivTestCase.assertion.expectedResult}\nActual: $actualResultAsIon\nMode: ${evalEquivTestCase.compileOptions.typingMode}") + } + } + is Assertion.EvaluationFailure -> { + error("Expected error to be thrown but none was thrown.\n${evalEquivTestCase.name}\nActual result: ${actualResult.toIonValue(ION)}") + } + } + } catch (e: SqlException) { + when (evalEquivTestCase.assertion) { + is Assertion.EvaluationSuccess -> { + error("Expected success but exception thrown: $e") + } + is Assertion.EvaluationFailure -> { + // Expected failure and test threw when evaluated + } + } + } + } + } + + // Tests the eval tests with the Kotlin implementation + @ParameterizedTest + @ArgumentsSource(EvalTestCases::class) + fun validatePartiQLEvalTestData(tc: TestCase) { + when (tc) { + is EvalTestCase -> TestRunner().runEvalTestCase(tc) + else -> error("Unsupported test case category") + } + } + + class EvalTestCases : ArgumentsProviderBase() { + override fun getParameters(): List { + return TestRunner().loadTests(PARTIQL_EVAL_TEST_DATA_DIR, LANG_KOTLIN_EVAL_SKIP_LIST) + } + } + + // Tests the eval equivalence tests with the Kotlin implementation + @ParameterizedTest + @ArgumentsSource(EvalEquivTestCases::class) + fun validatePartiQLEvalEquivTestData(tc: TestCase) { + when (tc) { + is EvalEquivTestCase -> TestRunner().runEvalEquivTestCase(tc) + else -> error("Unsupported test case category") + } + } + + class EvalEquivTestCases : ArgumentsProviderBase() { + override fun getParameters(): List { + return TestRunner().loadTests(PARTIQL_EVAL_EQUIV_TEST_DATA_DIR, LANG_KOTLIN_EVAL_EQUIV_SKIP_LIST) + } + } +} diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Util.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Util.kt new file mode 100644 index 0000000000..e77f2b16dc --- /dev/null +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Util.kt @@ -0,0 +1,81 @@ +package org.partiql.runner + +import com.amazon.ion.IonList +import com.amazon.ion.IonNull +import com.amazon.ion.IonSequence +import com.amazon.ion.IonSexp +import com.amazon.ion.IonStruct +import com.amazon.ion.IonSystem +import com.amazon.ion.IonValue +import org.partiql.lang.eval.ExprValue +import org.partiql.lang.eval.ExprValueFactory +import org.partiql.lang.eval.ExprValueType +import org.partiql.lang.eval.StructOrdering +import org.partiql.lang.eval.name +import org.partiql.lang.eval.namedValue +import org.partiql.lang.eval.stringValue + +const val BAG_ANNOTATION = "\$bag" +const val MISSING_ANNOTATION = "\$missing" + +/** + * Converts the conformance test's encoding of PartiQL values in Ion to an [ExprValue]. The conformance tests have a + * slightly different encoding than the default conversion function provided by [ExprValueFactory]. E.g. Ion value + * annotation for bag. + */ +internal fun IonValue.toExprValue(exprValueFactory: ExprValueFactory): ExprValue { + // Need to create a different IonValue to ExprValue conversion function because the default provided by + // `ExprValueFactory`'s [newFromIonValue] relies on a different encoding of PartiQL-specific types than the + // conformance tests (e.g. `ExprValueFactory` uses $partiql_bag rather than $bag) + val elem = this + val annotations = elem.typeAnnotations + return when { + (elem is IonList) && annotations.contains(BAG_ANNOTATION) -> { + val elemsAsExprValues = elem.map { + it.toExprValue(exprValueFactory) + } + exprValueFactory.newBag(elemsAsExprValues) + } + elem is IonNull && elem.hasTypeAnnotation(MISSING_ANNOTATION) -> exprValueFactory.missingValue + // TODO: other PartiQL types not in Ion + elem is IonList -> exprValueFactory.newList(elem.map { it.toExprValue(exprValueFactory) }) + elem is IonSexp -> exprValueFactory.newSexp(elem.map { it.toExprValue(exprValueFactory) }) + elem is IonStruct -> { + exprValueFactory.newStruct(elem.map { it.toExprValue(exprValueFactory).namedValue(exprValueFactory.newString(it.fieldName)) }, StructOrdering.UNORDERED) + } + else -> exprValueFactory.newFromIonValue(elem) + } +} + +/** + * Converts an [ExprValue] to the conformance test suite's modeling of PartiQL values in Ion. + */ +internal fun ExprValue.toIonValue(ion: IonSystem): IonValue { + fun ExprValue.foldToIonSequence(initial: S): S = + this.fold(initial) { seq, el -> seq.apply { add(el.toIonValue(ion)) } } + + return when (this.type) { + ExprValueType.MISSING -> ion.singleValue("$MISSING_ANNOTATION::null").clone() + ExprValueType.NULL, + ExprValueType.BOOL, + ExprValueType.INT, + ExprValueType.FLOAT, + ExprValueType.DECIMAL, + ExprValueType.DATE, + ExprValueType.TIME, + ExprValueType.TIMESTAMP, + ExprValueType.SYMBOL, + ExprValueType.STRING, + ExprValueType.CLOB, + ExprValueType.BLOB -> this.ionValue.clone() + ExprValueType.LIST -> this.foldToIonSequence(ion.newEmptyList()) + ExprValueType.SEXP -> this.foldToIonSequence(ion.newEmptySexp()) + ExprValueType.STRUCT -> this.fold(ion.newEmptyStruct()) { struct, el -> + struct.apply { add(el.name!!.stringValue(), el.toIonValue(ion)) } + } + ExprValueType.BAG -> { + val bag = ion.newEmptyList().apply { addTypeAnnotation(BAG_ANNOTATION) } + this.foldToIonSequence(bag) + } + } +} diff --git a/testscript/Readme.md b/test/partiql-testscript/Readme.md similarity index 100% rename from testscript/Readme.md rename to test/partiql-testscript/Readme.md diff --git a/testscript/build.gradle b/test/partiql-testscript/build.gradle similarity index 100% rename from testscript/build.gradle rename to test/partiql-testscript/build.gradle diff --git a/testscript/pts/test-scripts-ignored/README.md b/test/partiql-testscript/pts/test-scripts-ignored/README.md similarity index 100% rename from testscript/pts/test-scripts-ignored/README.md rename to test/partiql-testscript/pts/test-scripts-ignored/README.md diff --git a/testscript/pts/test-scripts-ignored/ported/mysql/select_slow.ion b/test/partiql-testscript/pts/test-scripts-ignored/ported/mysql/select_slow.ion similarity index 100% rename from testscript/pts/test-scripts-ignored/ported/mysql/select_slow.ion rename to test/partiql-testscript/pts/test-scripts-ignored/ported/mysql/select_slow.ion diff --git a/testscript/pts/test-scripts-ignored/ported/mysql/select_slow.sqlts b/test/partiql-testscript/pts/test-scripts-ignored/ported/mysql/select_slow.sqlts similarity index 100% rename from testscript/pts/test-scripts-ignored/ported/mysql/select_slow.sqlts rename to test/partiql-testscript/pts/test-scripts-ignored/ported/mysql/select_slow.sqlts diff --git a/testscript/pts/test-scripts/binding/empty-symbol.sqlts b/test/partiql-testscript/pts/test-scripts/binding/empty-symbol.sqlts similarity index 100% rename from testscript/pts/test-scripts/binding/empty-symbol.sqlts rename to test/partiql-testscript/pts/test-scripts/binding/empty-symbol.sqlts diff --git a/testscript/pts/test-scripts/builtin-aggregations-functions/avg.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-aggregations-functions/avg.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-aggregations-functions/avg.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-aggregations-functions/avg.sqlts diff --git a/testscript/pts/test-scripts/builtin-aggregations-functions/count.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-aggregations-functions/count.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-aggregations-functions/count.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-aggregations-functions/count.sqlts diff --git a/testscript/pts/test-scripts/builtin-aggregations-functions/max.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-aggregations-functions/max.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-aggregations-functions/max.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-aggregations-functions/max.sqlts diff --git a/testscript/pts/test-scripts/builtin-aggregations-functions/min.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-aggregations-functions/min.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-aggregations-functions/min.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-aggregations-functions/min.sqlts diff --git a/testscript/pts/test-scripts/builtin-aggregations-functions/sum.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-aggregations-functions/sum.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-aggregations-functions/sum.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-aggregations-functions/sum.sqlts diff --git a/testscript/pts/test-scripts/builtin-functions/Readme.md b/test/partiql-testscript/pts/test-scripts/builtin-functions/Readme.md similarity index 100% rename from testscript/pts/test-scripts/builtin-functions/Readme.md rename to test/partiql-testscript/pts/test-scripts/builtin-functions/Readme.md diff --git a/testscript/pts/test-scripts/builtin-functions/char_length.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-functions/char_length.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-functions/char_length.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-functions/char_length.sqlts diff --git a/testscript/pts/test-scripts/builtin-functions/coalesce.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-functions/coalesce.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-functions/coalesce.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-functions/coalesce.sqlts diff --git a/testscript/pts/test-scripts/builtin-functions/date_add.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-functions/date_add.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-functions/date_add.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-functions/date_add.sqlts diff --git a/testscript/pts/test-scripts/builtin-functions/date_diff.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-functions/date_diff.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-functions/date_diff.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-functions/date_diff.sqlts diff --git a/testscript/pts/test-scripts/builtin-functions/exists.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-functions/exists.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-functions/exists.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-functions/exists.sqlts diff --git a/testscript/pts/test-scripts/builtin-functions/extract.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-functions/extract.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-functions/extract.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-functions/extract.sqlts diff --git a/testscript/pts/test-scripts/builtin-functions/lower.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-functions/lower.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-functions/lower.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-functions/lower.sqlts diff --git a/testscript/pts/test-scripts/builtin-functions/nullif.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-functions/nullif.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-functions/nullif.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-functions/nullif.sqlts diff --git a/testscript/pts/test-scripts/builtin-functions/size.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-functions/size.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-functions/size.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-functions/size.sqlts diff --git a/testscript/pts/test-scripts/builtin-functions/substring.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-functions/substring.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-functions/substring.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-functions/substring.sqlts diff --git a/testscript/pts/test-scripts/builtin-functions/to_string.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-functions/to_string.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-functions/to_string.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-functions/to_string.sqlts diff --git a/testscript/pts/test-scripts/builtin-functions/to_timestamp.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-functions/to_timestamp.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-functions/to_timestamp.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-functions/to_timestamp.sqlts diff --git a/testscript/pts/test-scripts/builtin-functions/trim.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-functions/trim.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-functions/trim.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-functions/trim.sqlts diff --git a/testscript/pts/test-scripts/builtin-functions/upper.sqlts b/test/partiql-testscript/pts/test-scripts/builtin-functions/upper.sqlts similarity index 100% rename from testscript/pts/test-scripts/builtin-functions/upper.sqlts rename to test/partiql-testscript/pts/test-scripts/builtin-functions/upper.sqlts diff --git a/testscript/pts/test-scripts/case-sensitivity/path-expressions.sqlts b/test/partiql-testscript/pts/test-scripts/case-sensitivity/path-expressions.sqlts similarity index 100% rename from testscript/pts/test-scripts/case-sensitivity/path-expressions.sqlts rename to test/partiql-testscript/pts/test-scripts/case-sensitivity/path-expressions.sqlts diff --git a/testscript/pts/test-scripts/cast/Readme.md b/test/partiql-testscript/pts/test-scripts/cast/Readme.md similarity index 100% rename from testscript/pts/test-scripts/cast/Readme.md rename to test/partiql-testscript/pts/test-scripts/cast/Readme.md diff --git a/testscript/pts/test-scripts/cast/cast-to-int.sqlts b/test/partiql-testscript/pts/test-scripts/cast/cast-to-int.sqlts similarity index 100% rename from testscript/pts/test-scripts/cast/cast-to-int.sqlts rename to test/partiql-testscript/pts/test-scripts/cast/cast-to-int.sqlts diff --git a/testscript/pts/test-scripts/cast/cast-to-missing.sqlts b/test/partiql-testscript/pts/test-scripts/cast/cast-to-missing.sqlts similarity index 100% rename from testscript/pts/test-scripts/cast/cast-to-missing.sqlts rename to test/partiql-testscript/pts/test-scripts/cast/cast-to-missing.sqlts diff --git a/testscript/pts/test-scripts/cast/cast-to-null.sqlts b/test/partiql-testscript/pts/test-scripts/cast/cast-to-null.sqlts similarity index 100% rename from testscript/pts/test-scripts/cast/cast-to-null.sqlts rename to test/partiql-testscript/pts/test-scripts/cast/cast-to-null.sqlts diff --git a/testscript/pts/test-scripts/group-by/simple-group-by.sqlts b/test/partiql-testscript/pts/test-scripts/group-by/simple-group-by.sqlts similarity index 100% rename from testscript/pts/test-scripts/group-by/simple-group-by.sqlts rename to test/partiql-testscript/pts/test-scripts/group-by/simple-group-by.sqlts diff --git a/testscript/pts/test-scripts/join/join-with-condition.sqlts b/test/partiql-testscript/pts/test-scripts/join/join-with-condition.sqlts similarity index 100% rename from testscript/pts/test-scripts/join/join-with-condition.sqlts rename to test/partiql-testscript/pts/test-scripts/join/join-with-condition.sqlts diff --git a/testscript/pts/test-scripts/operators/concat.sqlts b/test/partiql-testscript/pts/test-scripts/operators/concat.sqlts similarity index 100% rename from testscript/pts/test-scripts/operators/concat.sqlts rename to test/partiql-testscript/pts/test-scripts/operators/concat.sqlts diff --git a/testscript/pts/test-scripts/ported/mysql/select.ion b/test/partiql-testscript/pts/test-scripts/ported/mysql/select.ion similarity index 100% rename from testscript/pts/test-scripts/ported/mysql/select.ion rename to test/partiql-testscript/pts/test-scripts/ported/mysql/select.ion diff --git a/testscript/pts/test-scripts/ported/mysql/select.sqlts b/test/partiql-testscript/pts/test-scripts/ported/mysql/select.sqlts similarity index 100% rename from testscript/pts/test-scripts/ported/mysql/select.sqlts rename to test/partiql-testscript/pts/test-scripts/ported/mysql/select.sqlts diff --git a/testscript/pts/test-scripts/ported/mysql/select_bugs.sqlts b/test/partiql-testscript/pts/test-scripts/ported/mysql/select_bugs.sqlts similarity index 100% rename from testscript/pts/test-scripts/ported/mysql/select_bugs.sqlts rename to test/partiql-testscript/pts/test-scripts/ported/mysql/select_bugs.sqlts diff --git a/testscript/pts/test-scripts/ported/pg/int8.ion b/test/partiql-testscript/pts/test-scripts/ported/pg/int8.ion similarity index 100% rename from testscript/pts/test-scripts/ported/pg/int8.ion rename to test/partiql-testscript/pts/test-scripts/ported/pg/int8.ion diff --git a/testscript/pts/test-scripts/ported/pg/int8.sqlts b/test/partiql-testscript/pts/test-scripts/ported/pg/int8.sqlts similarity index 100% rename from testscript/pts/test-scripts/ported/pg/int8.sqlts rename to test/partiql-testscript/pts/test-scripts/ported/pg/int8.sqlts diff --git a/testscript/pts/test-scripts/ported/pg/join.ion b/test/partiql-testscript/pts/test-scripts/ported/pg/join.ion similarity index 100% rename from testscript/pts/test-scripts/ported/pg/join.ion rename to test/partiql-testscript/pts/test-scripts/ported/pg/join.ion diff --git a/testscript/pts/test-scripts/ported/pg/join.sqlts b/test/partiql-testscript/pts/test-scripts/ported/pg/join.sqlts similarity index 100% rename from testscript/pts/test-scripts/ported/pg/join.sqlts rename to test/partiql-testscript/pts/test-scripts/ported/pg/join.sqlts diff --git a/testscript/pts/test-scripts/ported/pg/select.ion b/test/partiql-testscript/pts/test-scripts/ported/pg/select.ion similarity index 100% rename from testscript/pts/test-scripts/ported/pg/select.ion rename to test/partiql-testscript/pts/test-scripts/ported/pg/select.ion diff --git a/testscript/pts/test-scripts/ported/pg/select.sqlts b/test/partiql-testscript/pts/test-scripts/ported/pg/select.sqlts similarity index 100% rename from testscript/pts/test-scripts/ported/pg/select.sqlts rename to test/partiql-testscript/pts/test-scripts/ported/pg/select.sqlts diff --git a/testscript/pts/test-scripts/primitive-types/README.md b/test/partiql-testscript/pts/test-scripts/primitive-types/README.md similarity index 100% rename from testscript/pts/test-scripts/primitive-types/README.md rename to test/partiql-testscript/pts/test-scripts/primitive-types/README.md diff --git a/testscript/pts/test-scripts/primitive-types/bool.sqlts b/test/partiql-testscript/pts/test-scripts/primitive-types/bool.sqlts similarity index 100% rename from testscript/pts/test-scripts/primitive-types/bool.sqlts rename to test/partiql-testscript/pts/test-scripts/primitive-types/bool.sqlts diff --git a/testscript/pts/test-scripts/primitive-types/int.sqlts b/test/partiql-testscript/pts/test-scripts/primitive-types/int.sqlts similarity index 100% rename from testscript/pts/test-scripts/primitive-types/int.sqlts rename to test/partiql-testscript/pts/test-scripts/primitive-types/int.sqlts diff --git a/testscript/pts/test-scripts/primitive-types/naughty-strings.sqlts b/test/partiql-testscript/pts/test-scripts/primitive-types/naughty-strings.sqlts similarity index 100% rename from testscript/pts/test-scripts/primitive-types/naughty-strings.sqlts rename to test/partiql-testscript/pts/test-scripts/primitive-types/naughty-strings.sqlts diff --git a/testscript/pts/test-scripts/primitive-types/null.sqlts b/test/partiql-testscript/pts/test-scripts/primitive-types/null.sqlts similarity index 100% rename from testscript/pts/test-scripts/primitive-types/null.sqlts rename to test/partiql-testscript/pts/test-scripts/primitive-types/null.sqlts diff --git a/testscript/pts/test-scripts/primitive-types/string.sqlts b/test/partiql-testscript/pts/test-scripts/primitive-types/string.sqlts similarity index 100% rename from testscript/pts/test-scripts/primitive-types/string.sqlts rename to test/partiql-testscript/pts/test-scripts/primitive-types/string.sqlts diff --git a/testscript/pts/test-scripts/regression/README.md b/test/partiql-testscript/pts/test-scripts/regression/README.md similarity index 100% rename from testscript/pts/test-scripts/regression/README.md rename to test/partiql-testscript/pts/test-scripts/regression/README.md diff --git a/testscript/pts/test-scripts/regression/empty-string-for-like-pattern.sqlts b/test/partiql-testscript/pts/test-scripts/regression/empty-string-for-like-pattern.sqlts similarity index 100% rename from testscript/pts/test-scripts/regression/empty-string-for-like-pattern.sqlts rename to test/partiql-testscript/pts/test-scripts/regression/empty-string-for-like-pattern.sqlts diff --git a/testscript/pts/test-scripts/regression/repeated-field-on-struct.sqlts b/test/partiql-testscript/pts/test-scripts/regression/repeated-field-on-struct.sqlts similarity index 100% rename from testscript/pts/test-scripts/regression/repeated-field-on-struct.sqlts rename to test/partiql-testscript/pts/test-scripts/regression/repeated-field-on-struct.sqlts diff --git a/testscript/pts/test-scripts/select/select-from-where/like.sqlts b/test/partiql-testscript/pts/test-scripts/select/select-from-where/like.sqlts similarity index 100% rename from testscript/pts/test-scripts/select/select-from-where/like.sqlts rename to test/partiql-testscript/pts/test-scripts/select/select-from-where/like.sqlts diff --git a/testscript/pts/test-scripts/select/select-from/select-distinct.sqlts b/test/partiql-testscript/pts/test-scripts/select/select-from/select-distinct.sqlts similarity index 100% rename from testscript/pts/test-scripts/select/select-from/select-distinct.sqlts rename to test/partiql-testscript/pts/test-scripts/select/select-from/select-distinct.sqlts diff --git a/testscript/pts/test-scripts/select/select-from/select-star.sqlts b/test/partiql-testscript/pts/test-scripts/select/select-from/select-star.sqlts similarity index 100% rename from testscript/pts/test-scripts/select/select-from/select-star.sqlts rename to test/partiql-testscript/pts/test-scripts/select/select-from/select-star.sqlts diff --git a/testscript/pts/test-scripts/select/select-value/select-distinct-value.sqlts b/test/partiql-testscript/pts/test-scripts/select/select-value/select-distinct-value.sqlts similarity index 100% rename from testscript/pts/test-scripts/select/select-value/select-distinct-value.sqlts rename to test/partiql-testscript/pts/test-scripts/select/select-value/select-distinct-value.sqlts diff --git a/testscript/pts/test-scripts/single_working.sqlts b/test/partiql-testscript/pts/test-scripts/single_working.sqlts similarity index 100% rename from testscript/pts/test-scripts/single_working.sqlts rename to test/partiql-testscript/pts/test-scripts/single_working.sqlts diff --git a/testscript/src/org/partiql/testscript/PtsException.kt b/test/partiql-testscript/src/org/partiql/testscript/PtsException.kt similarity index 100% rename from testscript/src/org/partiql/testscript/PtsException.kt rename to test/partiql-testscript/src/org/partiql/testscript/PtsException.kt diff --git a/testscript/src/org/partiql/testscript/Result.kt b/test/partiql-testscript/src/org/partiql/testscript/Result.kt similarity index 100% rename from testscript/src/org/partiql/testscript/Result.kt rename to test/partiql-testscript/src/org/partiql/testscript/Result.kt diff --git a/testscript/src/org/partiql/testscript/compiler/Compiler.kt b/test/partiql-testscript/src/org/partiql/testscript/compiler/Compiler.kt similarity index 100% rename from testscript/src/org/partiql/testscript/compiler/Compiler.kt rename to test/partiql-testscript/src/org/partiql/testscript/compiler/Compiler.kt diff --git a/testscript/src/org/partiql/testscript/compiler/CompilerErrors.kt b/test/partiql-testscript/src/org/partiql/testscript/compiler/CompilerErrors.kt similarity index 100% rename from testscript/src/org/partiql/testscript/compiler/CompilerErrors.kt rename to test/partiql-testscript/src/org/partiql/testscript/compiler/CompilerErrors.kt diff --git a/testscript/src/org/partiql/testscript/compiler/CompilerException.kt b/test/partiql-testscript/src/org/partiql/testscript/compiler/CompilerException.kt similarity index 100% rename from testscript/src/org/partiql/testscript/compiler/CompilerException.kt rename to test/partiql-testscript/src/org/partiql/testscript/compiler/CompilerException.kt diff --git a/testscript/src/org/partiql/testscript/compiler/TestScriptExpression.kt b/test/partiql-testscript/src/org/partiql/testscript/compiler/TestScriptExpression.kt similarity index 100% rename from testscript/src/org/partiql/testscript/compiler/TestScriptExpression.kt rename to test/partiql-testscript/src/org/partiql/testscript/compiler/TestScriptExpression.kt diff --git a/testscript/src/org/partiql/testscript/evaluator/DefaultPtsEquality.kt b/test/partiql-testscript/src/org/partiql/testscript/evaluator/DefaultPtsEquality.kt similarity index 100% rename from testscript/src/org/partiql/testscript/evaluator/DefaultPtsEquality.kt rename to test/partiql-testscript/src/org/partiql/testscript/evaluator/DefaultPtsEquality.kt diff --git a/testscript/src/org/partiql/testscript/evaluator/Evaluator.java b/test/partiql-testscript/src/org/partiql/testscript/evaluator/Evaluator.java similarity index 100% rename from testscript/src/org/partiql/testscript/evaluator/Evaluator.java rename to test/partiql-testscript/src/org/partiql/testscript/evaluator/Evaluator.java diff --git a/testscript/src/org/partiql/testscript/evaluator/PtsEquality.java b/test/partiql-testscript/src/org/partiql/testscript/evaluator/PtsEquality.java similarity index 100% rename from testscript/src/org/partiql/testscript/evaluator/PtsEquality.java rename to test/partiql-testscript/src/org/partiql/testscript/evaluator/PtsEquality.java diff --git a/testscript/src/org/partiql/testscript/evaluator/TestResult.kt b/test/partiql-testscript/src/org/partiql/testscript/evaluator/TestResult.kt similarity index 100% rename from testscript/src/org/partiql/testscript/evaluator/TestResult.kt rename to test/partiql-testscript/src/org/partiql/testscript/evaluator/TestResult.kt diff --git a/testscript/src/org/partiql/testscript/extensions/FileExtensions.kt b/test/partiql-testscript/src/org/partiql/testscript/extensions/FileExtensions.kt similarity index 100% rename from testscript/src/org/partiql/testscript/extensions/FileExtensions.kt rename to test/partiql-testscript/src/org/partiql/testscript/extensions/FileExtensions.kt diff --git a/testscript/src/org/partiql/testscript/extensions/Interpolation.kt b/test/partiql-testscript/src/org/partiql/testscript/extensions/Interpolation.kt similarity index 100% rename from testscript/src/org/partiql/testscript/extensions/Interpolation.kt rename to test/partiql-testscript/src/org/partiql/testscript/extensions/Interpolation.kt diff --git a/testscript/src/org/partiql/testscript/extensions/IonExtensions.kt b/test/partiql-testscript/src/org/partiql/testscript/extensions/IonExtensions.kt similarity index 100% rename from testscript/src/org/partiql/testscript/extensions/IonExtensions.kt rename to test/partiql-testscript/src/org/partiql/testscript/extensions/IonExtensions.kt diff --git a/testscript/src/org/partiql/testscript/extensions/ListExtensions.kt b/test/partiql-testscript/src/org/partiql/testscript/extensions/ListExtensions.kt similarity index 100% rename from testscript/src/org/partiql/testscript/extensions/ListExtensions.kt rename to test/partiql-testscript/src/org/partiql/testscript/extensions/ListExtensions.kt diff --git a/testscript/src/org/partiql/testscript/junitRunner/Junit4PtsTest.java b/test/partiql-testscript/src/org/partiql/testscript/junitRunner/Junit4PtsTest.java similarity index 100% rename from testscript/src/org/partiql/testscript/junitRunner/Junit4PtsTest.java rename to test/partiql-testscript/src/org/partiql/testscript/junitRunner/Junit4PtsTest.java diff --git a/testscript/src/org/partiql/testscript/junitRunner/PtsRunner.kt b/test/partiql-testscript/src/org/partiql/testscript/junitRunner/PtsRunner.kt similarity index 100% rename from testscript/src/org/partiql/testscript/junitRunner/PtsRunner.kt rename to test/partiql-testscript/src/org/partiql/testscript/junitRunner/PtsRunner.kt diff --git a/testscript/src/org/partiql/testscript/parser/Ion.kt b/test/partiql-testscript/src/org/partiql/testscript/parser/Ion.kt similarity index 100% rename from testscript/src/org/partiql/testscript/parser/Ion.kt rename to test/partiql-testscript/src/org/partiql/testscript/parser/Ion.kt diff --git a/testscript/src/org/partiql/testscript/parser/NamedInputStream.kt b/test/partiql-testscript/src/org/partiql/testscript/parser/NamedInputStream.kt similarity index 100% rename from testscript/src/org/partiql/testscript/parser/NamedInputStream.kt rename to test/partiql-testscript/src/org/partiql/testscript/parser/NamedInputStream.kt diff --git a/testscript/src/org/partiql/testscript/parser/Parser.kt b/test/partiql-testscript/src/org/partiql/testscript/parser/Parser.kt similarity index 100% rename from testscript/src/org/partiql/testscript/parser/Parser.kt rename to test/partiql-testscript/src/org/partiql/testscript/parser/Parser.kt diff --git a/testscript/src/org/partiql/testscript/parser/ParserError.kt b/test/partiql-testscript/src/org/partiql/testscript/parser/ParserError.kt similarity index 100% rename from testscript/src/org/partiql/testscript/parser/ParserError.kt rename to test/partiql-testscript/src/org/partiql/testscript/parser/ParserError.kt diff --git a/testscript/src/org/partiql/testscript/parser/ParserException.kt b/test/partiql-testscript/src/org/partiql/testscript/parser/ParserException.kt similarity index 100% rename from testscript/src/org/partiql/testscript/parser/ParserException.kt rename to test/partiql-testscript/src/org/partiql/testscript/parser/ParserException.kt diff --git a/testscript/src/org/partiql/testscript/parser/ScriptLocation.kt b/test/partiql-testscript/src/org/partiql/testscript/parser/ScriptLocation.kt similarity index 100% rename from testscript/src/org/partiql/testscript/parser/ScriptLocation.kt rename to test/partiql-testscript/src/org/partiql/testscript/parser/ScriptLocation.kt diff --git a/testscript/src/org/partiql/testscript/parser/ast/AstNode.kt b/test/partiql-testscript/src/org/partiql/testscript/parser/ast/AstNode.kt similarity index 100% rename from testscript/src/org/partiql/testscript/parser/ast/AstNode.kt rename to test/partiql-testscript/src/org/partiql/testscript/parser/ast/AstNode.kt diff --git a/testscript/src/org/partiql/testscript/parser/ast/MacroNodes.kt b/test/partiql-testscript/src/org/partiql/testscript/parser/ast/MacroNodes.kt similarity index 100% rename from testscript/src/org/partiql/testscript/parser/ast/MacroNodes.kt rename to test/partiql-testscript/src/org/partiql/testscript/parser/ast/MacroNodes.kt diff --git a/testscript/src/org/partiql/testscript/parser/ast/builders/AppendTestBuilder.kt b/test/partiql-testscript/src/org/partiql/testscript/parser/ast/builders/AppendTestBuilder.kt similarity index 100% rename from testscript/src/org/partiql/testscript/parser/ast/builders/AppendTestBuilder.kt rename to test/partiql-testscript/src/org/partiql/testscript/parser/ast/builders/AppendTestBuilder.kt diff --git a/testscript/src/org/partiql/testscript/parser/ast/builders/BaseBuilder.kt b/test/partiql-testscript/src/org/partiql/testscript/parser/ast/builders/BaseBuilder.kt similarity index 100% rename from testscript/src/org/partiql/testscript/parser/ast/builders/BaseBuilder.kt rename to test/partiql-testscript/src/org/partiql/testscript/parser/ast/builders/BaseBuilder.kt diff --git a/testscript/src/org/partiql/testscript/parser/ast/builders/ForBuilder.kt b/test/partiql-testscript/src/org/partiql/testscript/parser/ast/builders/ForBuilder.kt similarity index 100% rename from testscript/src/org/partiql/testscript/parser/ast/builders/ForBuilder.kt rename to test/partiql-testscript/src/org/partiql/testscript/parser/ast/builders/ForBuilder.kt diff --git a/testscript/src/org/partiql/testscript/parser/ast/builders/TestBuilder.kt b/test/partiql-testscript/src/org/partiql/testscript/parser/ast/builders/TestBuilder.kt similarity index 100% rename from testscript/src/org/partiql/testscript/parser/ast/builders/TestBuilder.kt rename to test/partiql-testscript/src/org/partiql/testscript/parser/ast/builders/TestBuilder.kt diff --git a/testscript/test-resources/junit-platform.properties b/test/partiql-testscript/test-resources/junit-platform.properties similarity index 100% rename from testscript/test-resources/junit-platform.properties rename to test/partiql-testscript/test-resources/junit-platform.properties diff --git a/testscript/test/org/partiql/testscript/SpecConverter.kt b/test/partiql-testscript/test/org/partiql/testscript/SpecConverter.kt similarity index 100% rename from testscript/test/org/partiql/testscript/SpecConverter.kt rename to test/partiql-testscript/test/org/partiql/testscript/SpecConverter.kt diff --git a/testscript/test/org/partiql/testscript/compiler/CompilerTest.kt b/test/partiql-testscript/test/org/partiql/testscript/compiler/CompilerTest.kt similarity index 100% rename from testscript/test/org/partiql/testscript/compiler/CompilerTest.kt rename to test/partiql-testscript/test/org/partiql/testscript/compiler/CompilerTest.kt diff --git a/testscript/test/org/partiql/testscript/evaluator/DefaultPtsEqualityTest.kt b/test/partiql-testscript/test/org/partiql/testscript/evaluator/DefaultPtsEqualityTest.kt similarity index 100% rename from testscript/test/org/partiql/testscript/evaluator/DefaultPtsEqualityTest.kt rename to test/partiql-testscript/test/org/partiql/testscript/evaluator/DefaultPtsEqualityTest.kt diff --git a/testscript/test/org/partiql/testscript/parser/BaseParseTests.kt b/test/partiql-testscript/test/org/partiql/testscript/parser/BaseParseTests.kt similarity index 100% rename from testscript/test/org/partiql/testscript/parser/BaseParseTests.kt rename to test/partiql-testscript/test/org/partiql/testscript/parser/BaseParseTests.kt diff --git a/testscript/test/org/partiql/testscript/parser/ParserForTests.kt b/test/partiql-testscript/test/org/partiql/testscript/parser/ParserForTests.kt similarity index 100% rename from testscript/test/org/partiql/testscript/parser/ParserForTests.kt rename to test/partiql-testscript/test/org/partiql/testscript/parser/ParserForTests.kt diff --git a/testscript/test/org/partiql/testscript/parser/ParserMacroTests.kt b/test/partiql-testscript/test/org/partiql/testscript/parser/ParserMacroTests.kt similarity index 100% rename from testscript/test/org/partiql/testscript/parser/ParserMacroTests.kt rename to test/partiql-testscript/test/org/partiql/testscript/parser/ParserMacroTests.kt diff --git a/testscript/test/org/partiql/testscript/parser/ParserTests.kt b/test/partiql-testscript/test/org/partiql/testscript/parser/ParserTests.kt similarity index 100% rename from testscript/test/org/partiql/testscript/parser/ParserTests.kt rename to test/partiql-testscript/test/org/partiql/testscript/parser/ParserTests.kt diff --git a/testscript/test/org/partiql/testscript/parser/util.kt b/test/partiql-testscript/test/org/partiql/testscript/parser/util.kt similarity index 100% rename from testscript/test/org/partiql/testscript/parser/util.kt rename to test/partiql-testscript/test/org/partiql/testscript/parser/util.kt diff --git a/testscript/test/resources/environment.ion b/test/partiql-testscript/test/resources/environment.ion similarity index 100% rename from testscript/test/resources/environment.ion rename to test/partiql-testscript/test/resources/environment.ion diff --git a/testscript/test/resources/multiple_values.ion b/test/partiql-testscript/test/resources/multiple_values.ion similarity index 100% rename from testscript/test/resources/multiple_values.ion rename to test/partiql-testscript/test/resources/multiple_values.ion diff --git a/testscript/test/resources/not_struct.ion b/test/partiql-testscript/test/resources/not_struct.ion similarity index 100% rename from testscript/test/resources/not_struct.ion rename to test/partiql-testscript/test/resources/not_struct.ion