Add Kotlin conformance test runner

.github/workflows/build.yml

@@ -19,6 +19,8 @@ jobs:
         java: [11, 17]
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Use Java ${{ matrix.java }}
         uses: actions/setup-java@v1
         with:
@@ -30,6 +32,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Use Java 17
         uses: actions/setup-java@v1
         with:
@@ -50,17 +54,17 @@ jobs:
           file: lang/build/reports/jacoco/test/jacocoTestReport.xml
           flags: LANG
 
-      - name: Upload PTS coverage
+      - name: Upload PARTIQL_PTS coverage
         uses: codecov/codecov-action@v3
         with:
-          file: pts/build/reports/jacoco/test/jacocoTestReport.xml
-          flags: PTS
+          file: partiql-pts/build/reports/jacoco/test/jacocoTestReport.xml
+          flags: PARTIQL_PTS
 
-      - name: Upload TEST_SCRIPT coverage
+      - name: Upload PARTIQL_TEST_SCRIPT coverage
         uses: codecov/codecov-action@v3
         with:
-          file: testscript/build/reports/jacoco/test/jacocoTestReport.xml
-          flags: TEST_SCRIPT
+          file: partiql-testscript/build/reports/jacoco/test/jacocoTestReport.xml
+          flags: PARTIQL_TEST_SCRIPT
 
       - name: Upload EXAMPLES coverage
         uses: codecov/codecov-action@v3

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test/partiql-tests"]
+	path = test/partiql-tests
+	url = https://github.com/partiql/partiql-tests.git

build.gradle

@@ -156,7 +156,7 @@ subprojects { proj ->
         processResources.dependsOn generateVersionAndHashProperties
     })
 
-    if (project.name == "partiql-isl") {
+    if (project.name == "partiql-isl" || project.name == "partiql-tests-runner") {
         // skip since partiql-isl uses the conventional Gradle layout
         return
     }

settings.gradle

@@ -18,9 +18,10 @@ rootProject.name = 'PartiQL'
 include('lang',
         'cli', 
         'examples', 
-        'testscript',
-        'pts',
         'extensions',
         'partiql-grammar',
-        'lib:partiql-isl'
+        'lib:partiql-isl',
+        'test:partiql-pts',
+        'test:partiql-tests-runner',
+        'test:partiql-testscript'
 )

pts/build.gradle → test/partiql-pts/build.gradle

@@ -33,6 +33,6 @@ dependencies {
 
     // test-time dependencies
     testImplementation project(':lang')
-    testImplementation project(':testscript')
+    testImplementation project(':test:partiql-testscript')
     testImplementation 'org.junit.vintage:junit-vintage-engine:5.7.0'
 }

pts/test/org/partiql/lang/pts/PtsTest.kt → test/partiql-pts/test/org/partiql/lang/pts/PtsTest.kt

@@ -13,5 +13,5 @@ class PtsTest : Junit4PtsTest() {
 
     override fun getEvaluator(): Evaluator = PartiQlPtsEvaluator(PtsEquality.getDefault())
 
-    override fun getPtsFilePaths() = listOf("../testscript/pts/test-scripts")
+    override fun getPtsFilePaths() = listOf("../partiql-testscript/pts/test-scripts")
 }

test/partiql-tests-runner/README.md

@@ -0,0 +1,12 @@
+# PartiQL Kotlin Test Runner
+
+This package checks whether the conformance tests (in [partiql-tests](https://github.com/partiql/partiql-tests)) run 
+using the `partiql-lang-kotlin` implementation return the correct result.
+
+This package will allow us to:
+1. verify conformance tests are defined correctly (e.g. verify evaluation environment is not missing a table)
+2. identify areas in the Kotlin implementation that diverge from the PartiQL specification
+
+Eventually, the Kotlin test runner should replace the `pts` and `testscript` Gradle subprojects along with some other 
+tests in `lang` that were ported to `partiql-tests` 
+(see [partiql-lang-kotlin#789](https://github.com/partiql/partiql-lang-kotlin/issues/789)).

test/partiql-tests-runner/build.gradle

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at:
+ *
+ *       http://aws.amazon.com/apache2.0/
+ *
+ *  or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
+ *  language governing permissions and limitations under the License.
+ */
+
+plugins {
+    id 'java-library'
+}
+
+dependencies {
+    implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8")
+
+    testImplementation("org.jetbrains.kotlin:kotlin-test")
+    testImplementation("org.junit.jupiter:junit-jupiter-params:5.8.2")
+
+    implementation(project(':lang'))
+
+}
+
+test {
+    useJUnitPlatform()
+    environment("PARTIQL_EVAL_TESTS_DATA", file("../partiql-tests/partiql-tests-data/eval/").absolutePath)
+    environment("PARTIQL_EVAL_EQUIV_TESTS_DATA", file("../partiql-tests/partiql-tests-data/eval-equiv/").absolutePath)
+}

test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ArgumentsProviderBase.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at:
+ *
+ *       http://aws.amazon.com/apache2.0/
+ *
+ *  or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
+ *  language governing permissions and limitations under the License.
+ */
+
+package org.partiql.runner
+
+import org.junit.jupiter.api.extension.ExtensionContext
+import org.junit.jupiter.params.provider.Arguments
+import org.junit.jupiter.params.provider.ArgumentsProvider
+import java.util.stream.Stream
+
+/**
+ * Reduces some of the boilerplate associated with the style of parameterized testing frequently
+ * utilized in this package.
+ *
+ * Since JUnit5 requires `@JvmStatic` on its `@MethodSource` argument factory methods, this requires all
+ * of the argument lists to reside in the companion object of a test class.  This can be annoying since it
+ * forces the test to be separated from its tests cases.
+ *
+ * Classes that derive from this class can be defined near the `@ParameterizedTest` functions instead.
+ */
+abstract class ArgumentsProviderBase : ArgumentsProvider {
+
+    abstract fun getParameters(): List<Any>
+
+    @Throws(Exception::class)
+    override fun provideArguments(extensionContext: ExtensionContext): Stream<out Arguments>? {
+        return getParameters().map { Arguments.of(it) }.stream()
+    }
+}

test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Parse.kt

@@ -0,0 +1,128 @@
+package org.partiql.runner
+
+import com.amazon.ion.IonList
+import com.amazon.ion.IonStruct
+import com.amazon.ion.IonSymbol
+import com.amazon.ion.IonType
+import com.amazon.ion.IonValue
+import org.partiql.lang.eval.CompileOptions
+import org.partiql.lang.eval.TypingMode
+import org.partiql.lang.util.asIonStruct
+import org.partiql.lang.util.stringValue
+
+/**
+ * Parses the [testStruct] to a list of [TestCase]s with respect to the environments and equivalence classes provided
+ * in the [curNamespace].
+ */
+private fun parseTestCase(testStruct: IonStruct, curNamespace: Namespace): List<TestCase> {
+    val testCases = mutableListOf<TestCase>()
+    val name = testStruct.get("name").stringValue() ?: error("Expected test case to have field `name`")
+    val statement = testStruct.get("statement") ?: error("Expected test case to have field `statement`")
+    val env = testStruct.get("env") ?: curNamespace.env
+    val assertList = when (val assert = testStruct.get("assert") ?: error("Expected test case to have field `assert`")) {
+        is IonStruct -> listOf(assert)
+        is IonList -> assert.toList()
+        else -> error("Expect `assert` field to be an IonStruct or IonList")
+    }
+
+    testCases += assertList.map { assertion ->
+        val assertionStruct = assertion as IonStruct
+        val evalModeList = when (val evalModeIonValue = assertionStruct.get("evalMode")) {
+            is IonSymbol -> listOf(evalModeIonValue.stringValue())
+            is IonList -> evalModeIonValue.toList().map { it.stringValue() }
+            else -> error("evalMode expects IonSymbol or IonList")
+        }
+
+        evalModeList.map { evalMode ->
+            val compileOption = when (evalMode) {
+                "EvalModeError" -> CompileOptions.build { typingMode(TypingMode.LEGACY) }
+                "EvalModeCoerce" -> CompileOptions.build { typingMode(TypingMode.PERMISSIVE) }
+                else -> error("unsupported eval modes")
+            }
+            val evalResult: Assertion = when (assertionStruct.get("result").stringValue()) {
+                "EvaluationSuccess" -> Assertion.EvaluationSuccess(assertionStruct.get("output"))
+                "EvaluationFail" -> Assertion.EvaluationFailure
+                else -> error("expected one of EvaluationSuccess or EvaluationFail")
+            }
+
+            when (statement.type) {
+                // statement being an IonString indicates that this is an Eval test case
+                IonType.STRING -> EvalTestCase(
+                    name = name,
+                    statement = statement.stringValue() ?: error("Expected `statement` to be a string"),
+                    env = env.asIonStruct(),
+                    compileOptions = compileOption,
+                    assertion = evalResult
+                )
+                // statement being an IonSymbol indicates that this is an eval equivalence test case
+                IonType.SYMBOL -> {
+                    val equivClassId = statement.stringValue() ?: error("Expected `statement` to be a symbol")
+                    EvalEquivTestCase(
+                        name = name,
+                        statements = curNamespace.equivClasses[equivClassId] ?: error("Equiv class $equivClassId not defined in current namespace"),
+                        env = env.asIonStruct(),
+                        compileOptions = compileOption,
+                        assertion = evalResult
+                    )
+                }
+                else -> TODO("Support other test case categories: https://github.com/partiql/partiql-tests/issues/35")
+            }
+        }
+    }.flatten()
+    return testCases
+}
+
+private fun parseEquivalenceClass(equivClassStruct: IonStruct): EquivalenceClass {
+    val id = equivClassStruct.get("id") ?: error("Expected field `id` for equivalence class struct: $equivClassStruct ")
+    val statements = equivClassStruct.get("statements") ?: error("Expected field `statements` for equivalence class struct: $equivClassStruct")
+
+    val idAsString = id.stringValue() ?: error("Expected `id` to be an IonSymbol")
+    val statementsAsStrings = (statements as IonList).map { statement ->
+        statement.stringValue() ?: error("Expected each statement within equivalence class to be a string $statement")
+    }
+    return EquivalenceClass(
+        idAsString,
+        statementsAsStrings
+    )
+}
+
+/**
+ * Parses [data] with the [curNamespace] into a new [Namespace].
+ */
+internal fun parseNamespace(curNamespace: Namespace, data: IonValue): Namespace {
+    return when (data) {
+        is IonList -> {
+            val newNamespace = Namespace(
+                env = curNamespace.env,
+                namespaces = mutableListOf(),
+                testCases = mutableListOf(),
+                equivClasses = mutableMapOf()
+            )
+            data.forEach { d ->
+                parseNamespace(newNamespace, d)
+            }
+            curNamespace.namespaces.add(newNamespace)
+            curNamespace
+        }
+        is IonStruct -> {
+            // environment, equivalence class, or test case. add to current namespace
+            val annotations = data.typeAnnotations
+            when {
+                annotations.contains("envs") -> {
+                    curNamespace.env = data
+                }
+                annotations.contains("equiv_class") -> {
+                    // equivalence class
+                    val equivClass = parseEquivalenceClass(data)
+                    curNamespace.equivClasses[equivClass.id] = equivClass.statements
+                }
+                annotations.isEmpty() -> {
+                    // test case
+                    curNamespace.testCases.addAll(parseTestCase(data, curNamespace))
+                }
+            }
+            curNamespace
+        }
+        else -> error("Document parsing requires an IonList or IonStruct")
+    }
+}