Refactors partiql-tests-runner for multiple engines (#1289)

.github/workflows/conformance-report.yml

@@ -27,7 +27,7 @@ jobs:
       # Run the conformance tests and save to an Ion file.
       - name: gradle test of the conformance tests (can fail) and save to Ion file
         continue-on-error: true
-        run: gradle :test:partiql-tests-runner:test --tests "*ConformanceTestsReportRunner" -PconformanceReport
+        run: gradle :test:partiql-tests-runner:test --tests "*ConformanceTestReport" -PconformanceReport
       # Upload conformance report for future viewing and comparison with future runs.
       - name: Upload `conformance_test_results.ion`
         uses: actions/upload-artifact@v3
@@ -86,7 +86,7 @@ jobs:
         continue-on-error: true
         run: |
           cd ${{ github.event.pull_request.base.sha }}
-          gradle :test:partiql-tests-runner:test --tests "*ConformanceTestsReportRunner" -PconformanceReport
+          gradle :test:partiql-tests-runner:test --tests "*ConformanceTestReport" -PconformanceReport
       - name: (If download of target branch conformance report fails) Move conformance test report of target branch to ./artifact directory
         if: ${{ steps.download-report.outcome == 'failure' }}
         continue-on-error: true

test/partiql-tests-runner/README.md

@@ -12,19 +12,19 @@ This package enables:
 
 ```shell
 # default, test data from partiql-tests submodule will be used
-./gradlew :test:partiql-tests-runner:test --tests "*ConformanceTestsReportRunner" -PconformanceReport
+./gradlew :test:partiql-tests-runner:test --tests "*ConformanceTestReport" -PconformanceReport
 
 # override test data location
 PARTIQL_TESTS_DATA=/path/to/partiql-tests/data \
-./gradlew :test:partiql-tests-runner:test --tests "*ConformanceTestsReportRunner" -PconformanceReport
+./gradlew :test:partiql-tests-runner:test --tests "*ConformanceTestReport" -PconformanceReport
 ```
 The report is written into file `test/partiql-tests-runner/conformance_test_results.ion`.
 
 ## Run Conformance Tests in UI
 
 The above project property `-PconformanceReport` is checked in `test/partiql-tests-runner/build.gradle.kts`,
 to exclude the conformance test suite from executing during a normal project-build test run. 
-Unfortunately, this also disables running `ConformanceTestsReportRunner` in a UI runner. 
+Unfortunately, this also disables running `ConformanceTestReport` in a UI runner. 
 To make that possible locally, temporarily comment out the check in `test/partiql-tests-runner/build.gradle.kts`.
 
 ## Compare Conformance Reports locally

test/partiql-tests-runner/build.gradle.kts

@@ -40,11 +40,11 @@ tasks.test {
     environment(Env.PARTIQL_EVAL, file("$tests/eval/").absolutePath)
     environment(Env.PARTIQL_EQUIV, file("$tests/eval-equiv/").absolutePath)
 
-    // To make it possible to run ConformanceTestsReportRunner in unit test UI runner, comment out this check:
+    // To make it possible to run ConformanceTestReport in unit test UI runner, comment out this check:
     if (!project.hasProperty("conformanceReport")) {
-        exclude("org/partiql/runner/TestRunner\$ConformanceTestsReportRunner.class")
+        exclude("org/partiql/runner/ConformanceTestReport.class")
     }
 
     // May 2023: Disabled conformance testing during regular project build, because fail lists are out of date.
-    exclude("org/partiql/runner/TestRunner\$DefaultConformanceTestRunner.class")
+    exclude("org/partiql/runner/ConformanceTest.class")
 }

test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTest.kt

@@ -0,0 +1,44 @@
+package org.partiql.runner
+
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.ArgumentsSource
+import org.partiql.runner.schema.TestCase
+import org.partiql.runner.skip.LANG_KOTLIN_EVAL_EQUIV_FAIL_LIST
+import org.partiql.runner.skip.LANG_KOTLIN_EVAL_FAIL_LIST
+import org.partiql.runner.test.TestProvider
+import org.partiql.runner.test.TestRunner
+import org.partiql.runner.test.executor.LegacyExecutor
+
+/**
+ * Runs the conformance tests with an expected list of failing tests. Ensures that tests not in the failing list
+ * succeed with the expected result. Ensures that tests included in the failing list fail.
+ *
+ * These tests are included in the normal test/building.
+ * Update May 2023: Now excluded from the normal build, because the fail lists are out of date.
+ * TODO: Come up with a low-burden method of maintaining fail / exclusion lists.
+ */
+class ConformanceTest {
+
+    private val factory = LegacyExecutor.Factory
+    private val runner = TestRunner(factory)
+
+    // Tests the eval tests with the Kotlin implementation
+    @ParameterizedTest(name = "{arguments}")
+    @ArgumentsSource(TestProvider.Eval::class)
+    fun validatePartiQLEvalTestData(tc: TestCase) {
+        when (tc) {
+            is TestCase.Eval -> runner.test(tc, LANG_KOTLIN_EVAL_FAIL_LIST)
+            else -> error("Unsupported test case category")
+        }
+    }
+
+    // Tests the eval equivalence tests with the Kotlin implementation
+    @ParameterizedTest(name = "{arguments}")
+    @ArgumentsSource(TestProvider.Equiv::class)
+    fun validatePartiQLEvalEquivTestData(tc: TestCase) {
+        when (tc) {
+            is TestCase.Equiv -> runner.test(tc, LANG_KOTLIN_EVAL_EQUIV_FAIL_LIST)
+            else -> error("Unsupported test case category")
+        }
+    }
+}

test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTestReport.kt

@@ -0,0 +1,44 @@
+package org.partiql.runner
+
+import org.junit.jupiter.api.extension.ExtendWith
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.ArgumentsSource
+import org.partiql.runner.report.ReportGenerator
+import org.partiql.runner.schema.TestCase
+import org.partiql.runner.test.TestProvider
+import org.partiql.runner.test.TestRunner
+import org.partiql.runner.test.executor.LegacyExecutor
+
+/**
+ * Runs the conformance tests without a fail list, so we can document the passing/failing tests in the conformance
+ * report.
+ *
+ * These tests are excluded from normal testing/building unless the `conformanceReport` gradle property is
+ * specified (i.e. `gradle test ... -PconformanceReport`)
+ */
+@ExtendWith(ReportGenerator::class)
+class ConformanceTestReport {
+
+    private val factory = LegacyExecutor.Factory
+    private val runner = TestRunner(factory)
+
+    // Tests the eval tests with the Kotlin implementation without a fail list
+    @ParameterizedTest(name = "{arguments}")
+    @ArgumentsSource(TestProvider.Eval::class)
+    fun validatePartiQLEvalTestData(tc: TestCase) {
+        when (tc) {
+            is TestCase.Eval -> runner.test(tc, emptyList())
+            else -> error("Unsupported test case category")
+        }
+    }
+
+    // Tests the eval equivalence tests with the Kotlin implementation without a fail list
+    @ParameterizedTest(name = "{arguments}")
+    @ArgumentsSource(TestProvider.Equiv::class)
+    fun validatePartiQLEvalEquivTestData(tc: TestCase) {
+        when (tc) {
+            is TestCase.Equiv -> runner.test(tc, emptyList())
+            else -> error("Unsupported test case category")
+        }
+    }
+}

test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Ion.kt

@@ -0,0 +1,8 @@
+package org.partiql.runner
+
+import com.amazon.ion.system.IonSystemBuilder
+
+/**
+ * IonSystem for legacy pipelines and value comparison.
+ */
+public val ION = IonSystemBuilder.standard().build()

test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/report/ReportGenerator.kt

@@ -0,0 +1,55 @@
+package org.partiql.runner.report
+
+import com.amazon.ion.IonType
+import com.amazon.ion.system.IonTextWriterBuilder
+import org.junit.jupiter.api.extension.AfterAllCallback
+import org.junit.jupiter.api.extension.ExtensionContext
+import org.junit.jupiter.api.extension.TestWatcher
+import java.io.File
+
+class ReportGenerator : TestWatcher, AfterAllCallback {
+    var failingTests = emptySet<String>()
+    var passingTests = emptySet<String>()
+    var ignoredTests = emptySet<String>()
+    override fun testFailed(context: ExtensionContext?, cause: Throwable?) {
+        failingTests += context?.displayName ?: ""
+        super.testFailed(context, cause)
+    }
+
+    override fun testSuccessful(context: ExtensionContext?) {
+        passingTests += context?.displayName ?: ""
+        super.testSuccessful(context)
+    }
+
+    override fun afterAll(p0: ExtensionContext?) {
+        val file = File("./conformance_test_results.ion")
+        val outputStream = file.outputStream()
+        val writer = IonTextWriterBuilder.pretty().build(outputStream)
+        writer.stepIn(IonType.STRUCT) // in: outer struct
+
+        // set struct field for passing
+        writer.setFieldName("passing")
+        writer.stepIn(IonType.LIST)
+        passingTests.forEach { passingTest ->
+            writer.writeString(passingTest)
+        }
+        writer.stepOut()
+        // set struct field for failing
+        writer.setFieldName("failing")
+        writer.stepIn(IonType.LIST)
+        failingTests.forEach { failingTest ->
+            writer.writeString(failingTest)
+        }
+        writer.stepOut()
+
+        // set struct field for ignored
+        writer.setFieldName("ignored")
+        writer.stepIn(IonType.LIST)
+        ignoredTests.forEach { ignoredTest ->
+            writer.writeString(ignoredTest)
+        }
+        writer.stepOut()
+
+        writer.stepOut() // out: outer struct
+    }
+}

test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/Assertion.kt

@@ -0,0 +1,9 @@
+package org.partiql.runner.schema
+
+import com.amazon.ion.IonValue
+
+sealed class Assertion {
+    data class EvaluationSuccess(val expectedResult: IonValue) : Assertion()
+    object EvaluationFailure : Assertion()
+    // TODO: other assertion and test categories: https://github.com/partiql/partiql-tests/issues/35
+}

test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/EquivalenceClass.kt

@@ -0,0 +1,6 @@
+package org.partiql.runner.schema
+
+data class EquivalenceClass(
+    val id: String,
+    val statements: List<String>,
+)

test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/Namespace.kt

@@ -0,0 +1,10 @@
+package org.partiql.runner.schema
+
+import com.amazon.ion.IonStruct
+
+data class Namespace(
+    var env: IonStruct,
+    val namespaces: MutableList<Namespace>,
+    val testCases: MutableList<TestCase>,
+    val equivClasses: MutableMap<String, List<String>>
+)

test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/Parse.kt → test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/Parse.kt

@@ -1,4 +1,4 @@
-package org.partiql.runner
+package org.partiql.runner.schema
 
 import com.amazon.ion.IonList
 import com.amazon.ion.IonStruct
@@ -47,7 +47,7 @@ private fun parseTestCase(testStruct: IonStruct, curNamespace: Namespace): List<
 
             when (statement.type) {
                 // statement being an IonString indicates that this is an Eval test case
-                IonType.STRING -> EvalTestCase(
+                IonType.STRING -> TestCase.Eval(
                     name = name,
                     statement = statement.stringValue() ?: error("Expected `statement` to be a string"),
                     env = env.asIonStruct(),
@@ -57,7 +57,7 @@ private fun parseTestCase(testStruct: IonStruct, curNamespace: Namespace): List<
                 // statement being an IonSymbol indicates that this is an eval equivalence test case
                 IonType.SYMBOL -> {
                     val equivClassId = statement.stringValue() ?: error("Expected `statement` to be a symbol")
-                    EvalEquivTestCase(
+                    TestCase.Equiv(
                         name = name,
                         statements = curNamespace.equivClasses[equivClassId] ?: error("Equiv class $equivClassId not defined in current namespace"),
                         env = env.asIonStruct(),

test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/schema/TestCase.kt

@@ -0,0 +1,35 @@
+package org.partiql.runner.schema
+
+import com.amazon.ion.IonStruct
+import org.partiql.lang.eval.CompileOptions
+
+sealed class TestCase {
+    abstract val name: String
+    abstract val env: IonStruct
+    abstract val compileOptions: CompileOptions
+    abstract val assertion: Assertion
+
+    data class Equiv(
+        override val name: String,
+        val statements: List<String>,
+        override val env: IonStruct,
+        override val compileOptions: CompileOptions,
+        override val assertion: Assertion
+    ) : TestCase() {
+        override fun toString(): String {
+            return name + ", compileOption: " + compileOptions.typingMode
+        }
+    }
+
+    data class Eval(
+        override val name: String,
+        val statement: String,
+        override val env: IonStruct,
+        override val compileOptions: CompileOptions,
+        override val assertion: Assertion
+    ) : TestCase() {
+        override fun toString(): String {
+            return name + ", compileOption: " + compileOptions.typingMode
+        }
+    }
+}