From 066da8fd11c5292245121e2e59a7df5735e58603 Mon Sep 17 00:00:00 2001 From: Alan Cai Date: Mon, 26 Sep 2022 15:00:52 -0700 Subject: [PATCH] Adds Kotlin test runner to verify ported tests are ported correctly (#34) --- .github/workflows/build.yml | 4 + partiql-tests-data/eval-equiv/spec-tests.ion | 6 +- partiql-tests-data/eval/primitives/basic.ion | 2 +- .../eval/query/select/projection.ion | 8 +- .../eval/query/select/select.ion | 5 +- .../query/undefined-variable-behavior.ion | 26 +- partiql-tests-data/eval/spec-tests.ion | 34 +- partiql-tests-kotlin-test-runner/.gitignore | 335 ++++++++++++++++++ partiql-tests-kotlin-test-runner/README.md | 13 + .../build.gradle.kts | 47 +++ .../gradle/wrapper/gradle-wrapper.properties | 5 + partiql-tests-kotlin-test-runner/gradlew | 234 ++++++++++++ partiql-tests-kotlin-test-runner/gradlew.bat | 89 +++++ .../settings.gradle.kts | 15 + .../kotlintestrunner/ArgumentsProviderBase.kt | 40 +++ .../partiql/tests/kotlintestrunner/Parse.kt | 128 +++++++ .../PartiQLEqualityChecker.kt | 106 ++++++ .../partiql/tests/kotlintestrunner/Schema.kt | 43 +++ .../tests/kotlintestrunner/TestRunner.kt | 295 +++++++++++++++ .../partiql/tests/kotlintestrunner/Util.kt | 81 +++++ 20 files changed, 1465 insertions(+), 51 deletions(-) create mode 100644 partiql-tests-kotlin-test-runner/.gitignore create mode 100644 partiql-tests-kotlin-test-runner/README.md create mode 100644 partiql-tests-kotlin-test-runner/build.gradle.kts create mode 100644 partiql-tests-kotlin-test-runner/gradle/wrapper/gradle-wrapper.properties create mode 100755 partiql-tests-kotlin-test-runner/gradlew create mode 100644 partiql-tests-kotlin-test-runner/gradlew.bat create mode 100644 partiql-tests-kotlin-test-runner/settings.gradle.kts create mode 100644 partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/ArgumentsProviderBase.kt create mode 100644 partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/Parse.kt create mode 100644 partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/PartiQLEqualityChecker.kt create mode 100644 partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/Schema.kt create mode 100644 partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/TestRunner.kt create mode 100644 partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/Util.kt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d4e3bca..9bc9eee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,3 +26,7 @@ jobs: working-directory: ./partiql-tests-validator run: gradle build + - name: Validate Ported Tests with Kotlin Test Runner + working-directory: ./partiql-tests-kotlin-test-runner + run: gradle build + diff --git a/partiql-tests-data/eval-equiv/spec-tests.ion b/partiql-tests-data/eval-equiv/spec-tests.ion index 3d79ba3..173e0cc 100644 --- a/partiql-tests-data/eval-equiv/spec-tests.ion +++ b/partiql-tests-data/eval-equiv/spec-tests.ion @@ -24,7 +24,7 @@ ] }, { - name: "equiv tuple path navigation", + name: "equiv tuple path navigation with array notation", statement: tuple_navigation_with_array_notation, assert: { evalMode: [EvalModeCoerce, EvalModeError], @@ -172,7 +172,7 @@ }, { name: "equiv of comma, cross join, and join", - statement: comma_cross_join_and_join, + statement: comma_cross_join_and_join_and_lateral, env: { customers: [ {id: 5, name: "Joe"}, @@ -540,7 +540,7 @@ ] }, { - name: "equiv group by with aggregates", + name: "equiv aliases from select clause", statement: aliases_from_select_clause, env: { people: $bag::[{name: "zoe", age: 10, tag: "child"}, diff --git a/partiql-tests-data/eval/primitives/basic.ion b/partiql-tests-data/eval/primitives/basic.ion index 94e4da8..3ee5bc9 100644 --- a/partiql-tests-data/eval/primitives/basic.ion +++ b/partiql-tests-data/eval/primitives/basic.ion @@ -324,7 +324,7 @@ basic::[ result:EvaluationSuccess, output:$bag::[ null, - null, + $missing::null, 2 ] } diff --git a/partiql-tests-data/eval/query/select/projection.ion b/partiql-tests-data/eval/query/select/projection.ion index ab8ffce..cc26bd9 100644 --- a/partiql-tests-data/eval/query/select/projection.ion +++ b/partiql-tests-data/eval/query/select/projection.ion @@ -21,11 +21,7 @@ assert:{ evalMode:[EvalModeCoerce, EvalModeError], result:EvaluationSuccess, - output:$bag::[ - { - someColumn:$missing::null - } - ] + output:$bag::[{}] } }, { @@ -36,7 +32,7 @@ result:EvaluationSuccess, output:$bag::[ { - someColumn:$missing::null + _1:{} } ] } diff --git a/partiql-tests-data/eval/query/select/select.ion b/partiql-tests-data/eval/query/select/select.ion index fe246ec..a3a464a 100644 --- a/partiql-tests-data/eval/query/select/select.ion +++ b/partiql-tests-data/eval/query/select/select.ion @@ -116,6 +116,7 @@ envs::{ c:100 }, d:3., + someAlias:"hello", } { @@ -408,8 +409,8 @@ select_join::[ } }, { - name:"selectNonCorrelatedJoin", // Note that the joined s is coming from the global scope without @-operator - statement:"SELECT s.id AS id, v AS title FROM stores AS s, s AS v", + name:"selectNonCorrelatedJoin", // Note that the joined someAlias is coming from the global scope without @-operator + statement:"SELECT someAlias.id AS id, v AS title FROM stores AS someAlias, someAlias AS v", assert:{ evalMode:[EvalModeCoerce, EvalModeError], result:EvaluationSuccess, diff --git a/partiql-tests-data/eval/query/undefined-variable-behavior.ion b/partiql-tests-data/eval/query/undefined-variable-behavior.ion index 005d697..440ab76 100644 --- a/partiql-tests-data/eval/query/undefined-variable-behavior.ion +++ b/partiql-tests-data/eval/query/undefined-variable-behavior.ion @@ -51,21 +51,15 @@ undefined_variable_behavior::[ { name:"undefinedUnqualifiedVariableInSelectWithUndefinedVariableBehaviorMissing", statement:"SELECT s.a, s.undefined_variable, s.b FROM `[{a:100, b:200}]` s", - assert:[ - { - evalMode:EvalModeCoerce, - result:EvaluationSuccess, - output:$bag::[ - { - a:100, - b:200 - } - ] - }, - { - evalMode:EvalModeError, - result:EvaluationFail, - } - ] + assert:{ + evalMode:[EvalModeCoerce, EvalModeError], + result:EvaluationSuccess, + output:$bag::[ + { + a:100, + b:200 + } + ] + } } ] diff --git a/partiql-tests-data/eval/spec-tests.ion b/partiql-tests-data/eval/spec-tests.ion index 47408f2..fc1bcc7 100644 --- a/partiql-tests-data/eval/spec-tests.ion +++ b/partiql-tests-data/eval/spec-tests.ion @@ -245,7 +245,7 @@ assert: { evalMode: [EvalModeCoerce, EvalModeError], result: EvaluationSuccess, - output: $bag::[{price: 840.05, sym: "amzn"}, {price: 31.06, sym: "tdc"}] + output: $bag::[{price: 840.05, symbol: "amzn"}, {price: 31.06, symbol: "tdc"}] } } ] @@ -388,32 +388,20 @@ { name: "cast and operations with missing argument", statement: "SELECT VALUE {'a':3*v.a, 'b':3*(CAST(v.b AS INTEGER))} FROM [{'a':1, 'b':'1'}, {'a':2}] v", - assert: [ - { - evalMode: EvalModeCoerce, - result: EvaluationSuccess, - output: $bag::[{a:3, b:3}, {a:6}] - }, - { - evalMode: EvalModeError, - result: EvaluationFail, - }, - ] + assert: { + evalMode: [EvalModeCoerce, EvalModeError], + result: EvaluationSuccess, + output: $bag::[{a:3, b:3}, {a:6}] + } }, { name: "missing value in arithmetic expression", statement: "5 + missing", - assert: [ - { - evalMode: EvalModeCoerce, - result: EvaluationSuccess, - output: $missing::null - }, - { - evalMode: EvalModeError, - result: EvaluationFail, - }, - ] + assert: { + evalMode: [EvalModeCoerce, EvalModeError], + result: EvaluationSuccess, + output: $missing::null + } }, { name: "data type mismatch in comparison expression", diff --git a/partiql-tests-kotlin-test-runner/.gitignore b/partiql-tests-kotlin-test-runner/.gitignore new file mode 100644 index 0000000..e68379e --- /dev/null +++ b/partiql-tests-kotlin-test-runner/.gitignore @@ -0,0 +1,335 @@ +build +/gradle-build +*.iml +*.swp +*.~ +~$* +.gradle +out/ + + +# Created by https://www.toptal.com/developers/gitignore/api/vim,git,java,emacs,kotlin,eclipse,intellij+all,macos +# Edit at https://www.toptal.com/developers/gitignore?templates=vim,git,java,emacs,kotlin,eclipse,intellij+all,macos + +### Eclipse ### +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +### Eclipse Patch ### +# Eclipse Core +.project + +# JDT-specific (Eclipse Java Development Tools) +.classpath + +# Annotation Processing +.apt_generated + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# CDT- autotools +.autotools + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ + +# Annotation Processing +.apt_generated/ +.apt_generated_test/ + +# Scala IDE specific (Scala & Java development for Eclipse) +.cache-main +.scala_dependencies +.worksheet + +# Uncomment this line if you wish to ignore the project description file. +# Typically, this file would be tracked if it contains build/dependency configurations: +#.project + +### Eclipse Patch ### +# Spring Boot Tooling +.sts4-cache/ + +### Emacs ### +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data + + +### Git ### +# Created by git for backups. To disable backups in Git: +# $ git config --global mergetool.keepBackup false +*.orig + +# Created by git when using merge tools for conflicts +*.BACKUP.* +*.BASE.* +*.LOCAL.* +*.REMOTE.* +*_BACKUP_*.txt +*_BASE_*.txt +*_LOCAL_*.txt +*_REMOTE_*.txt + +### Intellij+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### Intellij+all Patch ### +# Ignores the whole .idea folder and all .iml files +# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 + +.idea/ + +# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 + +*.iml +modules.xml +.idea/misc.xml +*.ipr + +# Sonarlint plugin +.idea/sonarlint + +### Java ### +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +### Kotlin ### +# Compiled class file + +# Log file + +# BlueJ files + +# Mobile Tools for Java (J2ME) + +# Package Files # + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +# End of https://www.toptal.com/developers/gitignore/api/vim,git,java,emacs,kotlin,eclipse,intellij+all,macos diff --git a/partiql-tests-kotlin-test-runner/README.md b/partiql-tests-kotlin-test-runner/README.md new file mode 100644 index 0000000..037c6c2 --- /dev/null +++ b/partiql-tests-kotlin-test-runner/README.md @@ -0,0 +1,13 @@ +# PartiQL Kotlin Test Runner + +This package checks whether the conformance tests run using the `partiql-lang-kotlin` implementation return the correct +result. The `partiql-tests-validator` package does schema level validation to assert the test data conforms to the +test model schema, whereas this package will do validation on the test data's results. + +This package will allow us to: +1. verify conformance tests are defined correctly (e.g. verify evaluation environment is not missing a table) +2. identify areas in the Kotlin implementation that diverge from the PartiQL specification + +Eventually, the Kotlin test runner should be moved to `partiql-lang-kotlin` ([partiql-tests#34](https://github.com/partiql/partiql-tests/issues/36) +for removal from `partiql-tests` and [partiql-lang-kotlin#789](https://github.com/partiql/partiql-lang-kotlin/issues/789) +for adding to `partiql-lang-kotlin`) and can replace the tests that were ported to `partiql-tests`. diff --git a/partiql-tests-kotlin-test-runner/build.gradle.kts b/partiql-tests-kotlin-test-runner/build.gradle.kts new file mode 100644 index 0000000..4b0effd --- /dev/null +++ b/partiql-tests-kotlin-test-runner/build.gradle.kts @@ -0,0 +1,47 @@ +/* + * Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at: + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ + +plugins { + id("org.jetbrains.kotlin.jvm") version "1.5.0" + id("org.jlleitschuh.gradle.ktlint") version "10.3.0" + `java-library` +} + +repositories { + mavenCentral() +} + +object Versions { + const val jupiter = "5.8.2" +} + +dependencies { + implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") + + testImplementation("org.jetbrains.kotlin:kotlin-test") + testImplementation("org.junit.jupiter:junit-jupiter-params:${Versions.jupiter}") + + testImplementation("org.partiql:partiql-lang-kotlin:0.7.0") +} + +tasks.test { + useJUnitPlatform() + environment("PARTIQL_EVAL_TESTS_DATA", file("../partiql-tests-data/eval/").absolutePath) + environment("PARTIQL_EVAL_EQUIV_TESTS_DATA", file("../partiql-tests-data/eval-equiv/").absolutePath) +} + +java { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} diff --git a/partiql-tests-kotlin-test-runner/gradle/wrapper/gradle-wrapper.properties b/partiql-tests-kotlin-test-runner/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..ffed3a2 --- /dev/null +++ b/partiql-tests-kotlin-test-runner/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/partiql-tests-kotlin-test-runner/gradlew b/partiql-tests-kotlin-test-runner/gradlew new file mode 100755 index 0000000..1b6c787 --- /dev/null +++ b/partiql-tests-kotlin-test-runner/gradlew @@ -0,0 +1,234 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit + +APP_NAME="Gradle" +APP_BASE_NAME=${0##*/} + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/partiql-tests-kotlin-test-runner/gradlew.bat b/partiql-tests-kotlin-test-runner/gradlew.bat new file mode 100644 index 0000000..107acd3 --- /dev/null +++ b/partiql-tests-kotlin-test-runner/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/partiql-tests-kotlin-test-runner/settings.gradle.kts b/partiql-tests-kotlin-test-runner/settings.gradle.kts new file mode 100644 index 0000000..cc43139 --- /dev/null +++ b/partiql-tests-kotlin-test-runner/settings.gradle.kts @@ -0,0 +1,15 @@ +/* + * Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at: + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ + +rootProject.name = "kotlin-test-runner" diff --git a/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/ArgumentsProviderBase.kt b/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/ArgumentsProviderBase.kt new file mode 100644 index 0000000..b13e2cc --- /dev/null +++ b/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/ArgumentsProviderBase.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at: + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ + +package org.partiql.tests.kotlintestrunner + +import org.junit.jupiter.api.extension.ExtensionContext +import org.junit.jupiter.params.provider.Arguments +import org.junit.jupiter.params.provider.ArgumentsProvider +import java.util.stream.Stream + +/** + * Reduces some of the boilerplate associated with the style of parameterized testing frequently + * utilized in this package. + * + * Since JUnit5 requires `@JvmStatic` on its `@MethodSource` argument factory methods, this requires all + * of the argument lists to reside in the companion object of a test class. This can be annoying since it + * forces the test to be separated from its tests cases. + * + * Classes that derive from this class can be defined near the `@ParameterizedTest` functions instead. + */ +abstract class ArgumentsProviderBase : ArgumentsProvider { + + abstract fun getParameters(): List + + @Throws(Exception::class) + override fun provideArguments(extensionContext: ExtensionContext): Stream? { + return getParameters().map { Arguments.of(it) }.stream() + } +} diff --git a/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/Parse.kt b/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/Parse.kt new file mode 100644 index 0000000..6068db1 --- /dev/null +++ b/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/Parse.kt @@ -0,0 +1,128 @@ +package org.partiql.tests.kotlintestrunner + +import com.amazon.ion.IonList +import com.amazon.ion.IonStruct +import com.amazon.ion.IonSymbol +import com.amazon.ion.IonType +import com.amazon.ion.IonValue +import org.partiql.lang.eval.CompileOptions +import org.partiql.lang.eval.TypingMode +import org.partiql.lang.util.asIonStruct +import org.partiql.lang.util.stringValue + +/** + * Parses the [testStruct] to a list of [TestCase]s with respect to the environments and equivalence classes provided + * in the [curNamespace]. + */ +private fun parseTestCase(testStruct: IonStruct, curNamespace: Namespace): List { + val testCases = mutableListOf() + val name = testStruct.get("name").stringValue() ?: error("Expected test case to have field `name`") + val statement = testStruct.get("statement") ?: error("Expected test case to have field `statement`") + val env = testStruct.get("env") ?: curNamespace.env + val assertList = when (val assert = testStruct.get("assert") ?: error("Expected test case to have field `assert`")) { + is IonStruct -> listOf(assert) + is IonList -> assert.toList() + else -> error("Expect `assert` field to be an IonStruct or IonList") + } + + testCases += assertList.map { assertion -> + val assertionStruct = assertion as IonStruct + val evalModeList = when (val evalModeIonValue = assertionStruct.get("evalMode")) { + is IonSymbol -> listOf(evalModeIonValue.stringValue()) + is IonList -> evalModeIonValue.toList().map { it.stringValue() } + else -> error("evalMode expects IonSymbol or IonList") + } + + evalModeList.map { evalMode -> + val compileOption = when (evalMode) { + "EvalModeError" -> CompileOptions.build { typingMode(TypingMode.LEGACY) } + "EvalModeCoerce" -> CompileOptions.build { typingMode(TypingMode.PERMISSIVE) } + else -> error("unsupported eval modes") + } + val evalResult: Assertion = when (assertionStruct.get("result").stringValue()) { + "EvaluationSuccess" -> Assertion.EvaluationSuccess(assertionStruct.get("output")) + "EvaluationFail" -> Assertion.EvaluationFailure + else -> error("expected one of EvaluationSuccess or EvaluationFail") + } + + when (statement.type) { + // statement being an IonString indicates that this is an Eval test case + IonType.STRING -> EvalTestCase( + name = name, + statement = statement.stringValue() ?: error("Expected `statement` to be a string"), + env = env.asIonStruct(), + compileOptions = compileOption, + assertion = evalResult + ) + // statement being an IonSymbol indicates that this is an eval equivalence test case + IonType.SYMBOL -> { + val equivClassId = statement.stringValue() ?: error("Expected `statement` to be a symbol") + EvalEquivTestCase( + name = name, + statements = curNamespace.equivClasses[equivClassId] ?: error("Equiv class $equivClassId not defined in current namespace"), + env = env.asIonStruct(), + compileOptions = compileOption, + assertion = evalResult + ) + } + else -> TODO("Support other test case categories: https://github.com/partiql/partiql-tests/issues/35") + } + } + }.flatten() + return testCases +} + +private fun parseEquivalenceClass(equivClassStruct: IonStruct): EquivalenceClass { + val id = equivClassStruct.get("id") ?: error("Expected field `id` for equivalence class struct: $equivClassStruct ") + val statements = equivClassStruct.get("statements") ?: error("Expected field `statements` for equivalence class struct: $equivClassStruct") + + val idAsString = id.stringValue() ?: error("Expected `id` to be an IonSymbol") + val statementsAsStrings = (statements as IonList).map { statement -> + statement.stringValue() ?: error("Expected each statement within equivalence class to be a string $statement") + } + return EquivalenceClass( + idAsString, + statementsAsStrings + ) +} + +/** + * Parses [data] with the [curNamespace] into a new [Namespace]. + */ +internal fun parseNamespace(curNamespace: Namespace, data: IonValue): Namespace { + return when (data) { + is IonList -> { + val newNamespace = Namespace( + env = curNamespace.env, + namespaces = mutableListOf(), + testCases = mutableListOf(), + equivClasses = mutableMapOf() + ) + data.forEach { d -> + parseNamespace(newNamespace, d) + } + curNamespace.namespaces.add(newNamespace) + curNamespace + } + is IonStruct -> { + // environment, equivalence class, or test case. add to current namespace + val annotations = data.typeAnnotations + when { + annotations.contains("envs") -> { + curNamespace.env = data + } + annotations.contains("equiv_class") -> { + // equivalence class + val equivClass = parseEquivalenceClass(data) + curNamespace.equivClasses[equivClass.id] = equivClass.statements + } + annotations.isEmpty() -> { + // test case + curNamespace.testCases.addAll(parseTestCase(data, curNamespace)) + } + } + curNamespace + } + else -> error("Document parsing requires an IonList or IonStruct") + } +} diff --git a/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/PartiQLEqualityChecker.kt b/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/PartiQLEqualityChecker.kt new file mode 100644 index 0000000..0e35ee0 --- /dev/null +++ b/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/PartiQLEqualityChecker.kt @@ -0,0 +1,106 @@ +package org.partiql.tests.kotlintestrunner + +import com.amazon.ion.IonDecimal +import com.amazon.ion.IonList +import com.amazon.ion.IonSequence +import com.amazon.ion.IonSexp +import com.amazon.ion.IonStruct +import com.amazon.ion.IonTimestamp +import com.amazon.ion.IonType +import com.amazon.ion.IonValue +import java.lang.IllegalArgumentException + +/** + * Checks the equality of two PartiQL values defined using its [IonValue] representation. This definition first requires + * the types to be the same, whereas PartiQL's equal operator can assert equivalence with implicit type coercion. This + * differs from Ion's definition of equality in the following ways: + * 1. Bag comparison checks ignore ordering of IonLists + * 2. Null checks check for `missing` annotation + */ +class PartiQLEqualityChecker { + fun areEqual(left: IonValue, right: IonValue): Boolean { + if (left.type != right.type) { + return false + } + + // typed nulls + if (!left.isMissing() && !right.isMissing() && (left.isNullValue || right.isNullValue)) { + return left.isNullValue && right.isNullValue + } + + return when (left.type!!) { + IonType.NULL -> { + if (left.isMissing() || right.isMissing()) { + left.isMissing() && right.isMissing() + } else { + right.isNullValue + } + } + IonType.BOOL, + IonType.INT, + IonType.FLOAT, + IonType.SYMBOL, + IonType.STRING, + IonType.CLOB, + IonType.BLOB -> left == right + IonType.DECIMAL -> { + val leftDecimal = left as IonDecimal + val rightDecimal = right as IonDecimal + + // we use compareTo to ignore differences in scale since + // for PartiQL 1.0 == 1.00 while that's not true for Ion + leftDecimal.bigDecimalValue().compareTo(rightDecimal.bigDecimalValue()) == 0 + } + IonType.TIMESTAMP -> { + val leftTimestamp = left as IonTimestamp + val rightTimestamp = right as IonTimestamp + + leftTimestamp.timestampValue().compareTo(rightTimestamp.timestampValue()) == 0 + } + IonType.LIST -> { + val leftList = left as IonList + val rightList = right as IonList + + if (leftList.isBag() || rightList.isBag()) { + ptsBagEquals(leftList, rightList) + } else { + ptsSequenceEquals(leftList, rightList) + } + } + IonType.SEXP -> ptsSequenceEquals(left as IonSexp, right as IonSexp) + IonType.STRUCT -> left as IonStruct == right as IonStruct + IonType.DATAGRAM -> throw IllegalArgumentException("DATAGRAM are not a valid type in CTS") + } + } + + private fun IonList.isBag(): Boolean = + this.typeAnnotations.size == 1 && + this.typeAnnotations[0] == BAG_ANNOTATION + + private fun ptsSequenceEquals(left: IonSequence, right: IonSequence): Boolean = + left.size == right.size && + left.asSequence() + .mapIndexed { index, leftElement -> index to leftElement } + .all { (index, leftElement) -> areEqual(leftElement, right[index]) } + + // bags can contain repeated elements, so they are equal if and only if: + // * Same size + // * All elements in one are contained in the other at the same quantities + private fun ptsBagEquals(left: IonList, right: IonList): Boolean = + when { + left.size != right.size -> false + left.isBag() && right.isBag() -> { + left.all { leftEl -> + val leftQtd = left.count { areEqual(leftEl, it) } + val rightQtd = right.count { areEqual(leftEl, it) } + + leftQtd == rightQtd + } + } + else -> false + } + + private fun IonValue.isMissing(): Boolean = this.isNullValue && + this.hasTypeAnnotation(MISSING_ANNOTATION) && + this.typeAnnotations.size == 1 +} diff --git a/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/Schema.kt b/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/Schema.kt new file mode 100644 index 0000000..8bb564c --- /dev/null +++ b/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/Schema.kt @@ -0,0 +1,43 @@ +package org.partiql.tests.kotlintestrunner + +import com.amazon.ion.IonStruct +import com.amazon.ion.IonValue +import org.partiql.lang.eval.CompileOptions + +data class Namespace( + var env: IonStruct, + val namespaces: MutableList, + val testCases: MutableList, + val equivClasses: MutableMap> +) + +data class EquivalenceClass(val id: String, val statements: List) + +sealed class Assertion { + data class EvaluationSuccess(val expectedResult: IonValue) : Assertion() + object EvaluationFailure : Assertion() + // TODO: other assertion and test categories: https://github.com/partiql/partiql-tests/issues/35 +} + +sealed class TestCase { + abstract val name: String + abstract val env: IonStruct + abstract val compileOptions: CompileOptions + abstract val assertion: Assertion +} + +data class EvalTestCase( + override val name: String, + val statement: String, + override val env: IonStruct, + override val compileOptions: CompileOptions, + override val assertion: Assertion +) : TestCase() + +data class EvalEquivTestCase( + override val name: String, + val statements: List, + override val env: IonStruct, + override val compileOptions: CompileOptions, + override val assertion: Assertion +) : TestCase() diff --git a/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/TestRunner.kt b/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/TestRunner.kt new file mode 100644 index 0000000..acbfcee --- /dev/null +++ b/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/TestRunner.kt @@ -0,0 +1,295 @@ +/* + * Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at: + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ + +package org.partiql.tests.kotlintestrunner + +import com.amazon.ion.system.IonSystemBuilder +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ArgumentsSource +import org.partiql.lang.CompilerPipeline +import org.partiql.lang.SqlException +import org.partiql.lang.eval.CompileOptions +import org.partiql.lang.eval.EvaluationSession +import org.partiql.lang.eval.TypingMode +import java.io.File + +private val PARTIQL_EVAL_TEST_DATA_DIR = System.getenv("PARTIQL_EVAL_TESTS_DATA") +private val PARTIQL_EVAL_EQUIV_TEST_DATA_DIR = System.getenv("PARTIQL_EVAL_EQUIV_TESTS_DATA") + +private val ION = IonSystemBuilder.standard().build() + +private val COERCE_EVAL_MODE_COMPILE_OPTIONS = CompileOptions.build { typingMode(TypingMode.PERMISSIVE) } +private val ERROR_EVAL_MODE_COMPILE_OPTIONS = CompileOptions.build { typingMode(TypingMode.LEGACY) } + +/* +The skip lists defined in this file show how the current Kotlin implementation diverges from the PartiQL spec. Most of +the divergent behavior is due to `partiql-lang-kotlin` not having a STRICT/ERROR typing mode. The [LEGACY typing mode](https://github.com/partiql/partiql-lang-kotlin/blob/main/lang/src/org/partiql/lang/eval/CompileOptions.kt#L53-L62) +(which is closer to STRICT/ERROR but not a complete match) was used for testing the STRICT/ERROR typing mode behavior. + +A lot of the other behavior differences is due to not supporting some syntax mentioned in the spec (like `COLL_*` +aggregation functions) and due to not supporting coercions. + +The remaining divergent behavior causing certain conformance tests to fail are likely bugs. Tracking issue: +https://github.com/partiql/partiql-lang-kotlin/issues/804. + */ +private val LANG_KOTLIN_EVAL_SKIP_LIST = listOf( + // from the spec: no explicit CAST to string means the query is "treated as an array navigation with wrongly typed + // data" and will return `MISSING` + Pair("tuple navigation with array notation without explicit CAST to string", COERCE_EVAL_MODE_COMPILE_OPTIONS), + // same as above, but since in error mode, should give an error + Pair("tuple navigation with array notation without explicit CAST to string", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // for the following, partiql-lang-kotlin doesn't have a STRICT/ERROR typing mode. tested using + // partiql-lang-kotlin's LEGACY typing mode, which has some semantic differences from STRICT/ERROR typing mode. + Pair("path on string", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("tuple navigation missing attribute dot notation", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("tuple navigation missing attribute array notation", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("single source FROM with bag and AT clause", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("single source FROM with scalar and AT clause", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("single source FROM with tuple and AT clause", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("single source FROM with absent value null and AT clause", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("single source FROM with absent value missing and AT clause", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("cast and operations with missing argument", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("missing value in arithmetic expression", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("equality of scalar missing", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("arithmetic with null/missing", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // TODO: clarify behavior. spec (section 8) says it should return NULL based on 3-value logic + Pair("missing and true", COERCE_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't currently implement subquery coercion. The inner SFW query returns a bag of two elements that when + // coerced to a scalar should return MISSING in COERCE mode. As a result, `customerName` should be missing from the + // first tuple. + Pair("inner select evaluating to collection with more than one element", COERCE_EVAL_MODE_COMPILE_OPTIONS), + + // coll_* aggregate functions not supported in plk -- results in parser error. coll_* functions will be supported in + // https://github.com/partiql/partiql-lang-kotlin/issues/222 + Pair("coll_count without group by", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("coll_count without group by", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("coll_count with result of subquery", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("coll_count with result of subquery", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // WITH keyword not supported resulting in parse error. windowing will be supported in plk in https://github.com/partiql/partiql-lang-kotlin/issues/603 + Pair("windowing simplified with grouping", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("windowing simplified with grouping", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // outer set ops not yet supported released to Maven (will be part of next release). implemented as part of + // https://github.com/partiql/partiql-lang-kotlin/pull/690 + Pair("outerUnionDistinct", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerUnionDistinct", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerUnionAll", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerUnionAll", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerIntersectDistinct", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerIntersectDistinct", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerIntersectAll", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerIntersectAll", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerExceptDistinct", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerExceptDistinct", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerExceptAll", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerExceptAll", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerUnionCoerceScalar", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerUnionCoerceScalar", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerUnionCoerceStruct", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerUnionCoerceStruct", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerUnionCoerceNullMissing", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerUnionCoerceNullMissing", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerUnionCoerceList", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("outerUnionCoerceList", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't have STRICT/ERROR mode. LEGACY mode used which doesn't error when RHS of `IN` expression is not a + // bag, list, or sexp + Pair("notInPredicateSingleExpr", ERROR_EVAL_MODE_COMPILE_OPTIONS), +) + +private val LANG_KOTLIN_EVAL_EQUIV_SKIP_LIST = listOf( + // plk gives a parser error for tuple path navigation in which the path expression is a string literal + // e.g. { 'a': 1, 'b': 2}.'a' -> 1 (see section 4 of spec) + Pair("equiv tuple path navigation with array notation", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv tuple path navigation with array notation", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't support a STRICT/ERROR mode. + Pair("equiv attribute value pair unpivot non-missing", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv attribute value pair unpivot missing", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't support `LATERAL` keyword which results in a parser error + Pair("equiv of comma, cross join, and join", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv of comma, cross join, and join", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't support `TUPLEUNION` function which results in an evaluation error + Pair("equiv tupleunion with select list", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv tupleunion with select list", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't support coercion of subqueries which results in different outputs + Pair("equiv coercion of a SELECT subquery into a scalar", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv coercion of a SELECT subquery into a scalar", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv coercion of a SELECT subquery into an array", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv coercion of a SELECT subquery into an array", ERROR_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv coercions with explicit literals", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv coercions with explicit literals", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't support `GROUP ALL` and `COLL_*` aggregate functions. Currently, results in a parser error + Pair("equiv group_all", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv group_all", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't support `COLL_*` aggregate functions. Currently, results in an evaluation error + Pair("equiv group by with aggregates", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv group by with aggregates", ERROR_EVAL_MODE_COMPILE_OPTIONS), + + // plk doesn't support using aliases created in select list in `GROUP BY` (and `ORDER BY`). GH issue to track: + // https://github.com/partiql/partiql-lang-kotlin/issues/571 + Pair("equiv aliases from select clause", COERCE_EVAL_MODE_COMPILE_OPTIONS), + Pair("equiv aliases from select clause", ERROR_EVAL_MODE_COMPILE_OPTIONS), +) + +/** + * Checks all the PartiQL conformance test data in [PARTIQL_EVAL_TEST_DATA_DIR] conforms to the test data schema. + */ +class TestRunner { + private fun parseTestFile(file: File): Namespace { + val loadedData = file.readText() + val dataInIon = ION.loader.load(loadedData) + val emptyNamespace = Namespace( + env = ION.newEmptyStruct(), + namespaces = mutableListOf(), + testCases = mutableListOf(), + equivClasses = mutableMapOf() + ) + dataInIon.forEach { d -> + parseNamespace(emptyNamespace, d) + } + return emptyNamespace + } + + private fun allTestsFromNamespace(ns: Namespace): List { + return ns.testCases + ns.namespaces.fold(listOf()) { acc, subns -> + acc + allTestsFromNamespace(subns) + } + } + + private fun loadTests(path: String, skipList: List> = emptyList()): List { + val allFiles = File(path).walk() + .filter { it.isFile } + .filter { it.path.endsWith(".ion") } + .toList() + val filesAsNamespaces = allFiles.map { file -> + parseTestFile(file) + } + + val allTestCases = filesAsNamespaces.flatMap { ns -> + allTestsFromNamespace(ns) + }.filter { + // Currently, just filtering the expected failing tests defined by the `skipList`. As an enhancement to the + // test runner, we could instead run the failing tests to assert they still fail. + !skipList.contains(Pair(it.name, it.compileOptions)) + } + return allTestCases + } + + private fun runEvalTestCase(evalTC: EvalTestCase) { + val compilerPipeline = CompilerPipeline.builder(ION).compileOptions(evalTC.compileOptions).build() + val globals = evalTC.env.toExprValue(compilerPipeline.valueFactory).bindings + val session = EvaluationSession.build { globals(globals) } + val expression = compilerPipeline.compile(evalTC.statement) + + try { + val actualResult = expression.eval(session) + when (evalTC.assertion) { + is Assertion.EvaluationSuccess -> { + val actualResultAsIon = actualResult.toIonValue(ION) + if (!PartiQLEqualityChecker().areEqual(evalTC.assertion.expectedResult, actualResultAsIon)) { + error("Expected and actual results differ:\nExpected: ${evalTC.assertion.expectedResult}\nActual: $actualResultAsIon\nMode: ${evalTC.compileOptions.typingMode}") + } + } + is Assertion.EvaluationFailure -> { + error("Expected error to be thrown but none was thrown.\n${evalTC.name}\nActual result: ${actualResult.toIonValue(ION)}") + } + } + } catch (e: SqlException) { + when (evalTC.assertion) { + is Assertion.EvaluationSuccess -> { + error("Expected success but exception thrown") + } + is Assertion.EvaluationFailure -> { + // Expected failure and test threw when evaluated + } + } + } + } + + private fun runEvalEquivTestCase(evalEquivTestCase: EvalEquivTestCase) { + val compilerPipeline = CompilerPipeline.builder(ION).compileOptions(evalEquivTestCase.compileOptions).build() + val globals = evalEquivTestCase.env.toExprValue(compilerPipeline.valueFactory).bindings + val session = EvaluationSession.build { globals(globals) } + val statements = evalEquivTestCase.statements + + statements.forEach { statement -> + val expression = compilerPipeline.compile(statement) + try { + val actualResult = expression.eval(session) + when (evalEquivTestCase.assertion) { + is Assertion.EvaluationSuccess -> { + val actualResultAsIon = actualResult.toIonValue(ION) + if (!PartiQLEqualityChecker().areEqual(evalEquivTestCase.assertion.expectedResult, actualResultAsIon)) { + error("Expected and actual results differ:\nExpected: ${evalEquivTestCase.assertion.expectedResult}\nActual: $actualResultAsIon\nMode: ${evalEquivTestCase.compileOptions.typingMode}") + } + } + is Assertion.EvaluationFailure -> { + error("Expected error to be thrown but none was thrown.\n${evalEquivTestCase.name}\nActual result: ${actualResult.toIonValue(ION)}") + } + } + } catch (e: SqlException) { + when (evalEquivTestCase.assertion) { + is Assertion.EvaluationSuccess -> { + error("Expected success but exception thrown: $e") + } + is Assertion.EvaluationFailure -> { + // Expected failure and test threw when evaluated + } + } + } + } + } + + // Tests the eval tests with the Kotlin implementation + @ParameterizedTest + @ArgumentsSource(EvalTestCases::class) + fun validatePartiQLEvalTestData(tc: TestCase) { + when (tc) { + is EvalTestCase -> TestRunner().runEvalTestCase(tc) + else -> error("Unsupported test case category") + } + } + + class EvalTestCases : ArgumentsProviderBase() { + override fun getParameters(): List { + return TestRunner().loadTests(PARTIQL_EVAL_TEST_DATA_DIR, LANG_KOTLIN_EVAL_SKIP_LIST) + } + } + + // Tests the eval equivalence tests with the Kotlin implementation + @ParameterizedTest + @ArgumentsSource(EvalEquivTestCases::class) + fun validatePartiQLEvalEquivTestData(tc: TestCase) { + when (tc) { + is EvalEquivTestCase -> TestRunner().runEvalEquivTestCase(tc) + else -> error("Unsupported test case category") + } + } + + class EvalEquivTestCases : ArgumentsProviderBase() { + override fun getParameters(): List { + return TestRunner().loadTests(PARTIQL_EVAL_EQUIV_TEST_DATA_DIR, LANG_KOTLIN_EVAL_EQUIV_SKIP_LIST) + } + } +} diff --git a/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/Util.kt b/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/Util.kt new file mode 100644 index 0000000..cf822fc --- /dev/null +++ b/partiql-tests-kotlin-test-runner/src/test/kotlin/org/partiql/tests/kotlintestrunner/Util.kt @@ -0,0 +1,81 @@ +package org.partiql.tests.kotlintestrunner + +import com.amazon.ion.IonList +import com.amazon.ion.IonNull +import com.amazon.ion.IonSequence +import com.amazon.ion.IonSexp +import com.amazon.ion.IonStruct +import com.amazon.ion.IonSystem +import com.amazon.ion.IonValue +import org.partiql.lang.eval.ExprValue +import org.partiql.lang.eval.ExprValueFactory +import org.partiql.lang.eval.ExprValueType +import org.partiql.lang.eval.StructOrdering +import org.partiql.lang.eval.name +import org.partiql.lang.eval.namedValue +import org.partiql.lang.eval.stringValue + +const val BAG_ANNOTATION = "\$bag" +const val MISSING_ANNOTATION = "\$missing" + +/** + * Converts the conformance test's encoding of PartiQL values in Ion to an [ExprValue]. The conformance tests have a + * slightly different encoding than the default conversion function provided by [ExprValueFactory]. E.g. Ion value + * annotation for bag. + */ +internal fun IonValue.toExprValue(exprValueFactory: ExprValueFactory): ExprValue { + // Need to create a different IonValue to ExprValue conversion function because the default provided by + // `ExprValueFactory`'s [newFromIonValue] relies on a different encoding of PartiQL-specific types than the + // conformance tests (e.g. `ExprValueFactory` uses $partiql_bag rather than $bag) + val elem = this + val annotations = elem.typeAnnotations + return when { + (elem is IonList) && annotations.contains(BAG_ANNOTATION) -> { + val elemsAsExprValues = elem.map { + it.toExprValue(exprValueFactory) + } + exprValueFactory.newBag(elemsAsExprValues) + } + elem is IonNull && elem.hasTypeAnnotation(MISSING_ANNOTATION) -> exprValueFactory.missingValue + // TODO: other PartiQL types not in Ion + elem is IonList -> exprValueFactory.newList(elem.map { it.toExprValue(exprValueFactory) }) + elem is IonSexp -> exprValueFactory.newSexp(elem.map { it.toExprValue(exprValueFactory) }) + elem is IonStruct -> { + exprValueFactory.newStruct(elem.map { it.toExprValue(exprValueFactory).namedValue(exprValueFactory.newString(it.fieldName)) }, StructOrdering.UNORDERED) + } + else -> exprValueFactory.newFromIonValue(elem) + } +} + +/** + * Converts an [ExprValue] to the conformance test suite's modeling of PartiQL values in Ion. + */ +internal fun ExprValue.toIonValue(ion: IonSystem): IonValue { + fun ExprValue.foldToIonSequence(initial: S): S = + this.fold(initial) { seq, el -> seq.apply { add(el.toIonValue(ion)) } } + + return when (this.type) { + ExprValueType.MISSING -> ion.singleValue("$MISSING_ANNOTATION::null").clone() + ExprValueType.NULL, + ExprValueType.BOOL, + ExprValueType.INT, + ExprValueType.FLOAT, + ExprValueType.DECIMAL, + ExprValueType.DATE, + ExprValueType.TIME, + ExprValueType.TIMESTAMP, + ExprValueType.SYMBOL, + ExprValueType.STRING, + ExprValueType.CLOB, + ExprValueType.BLOB -> this.ionValue.clone() + ExprValueType.LIST -> this.foldToIonSequence(ion.newEmptyList()) + ExprValueType.SEXP -> this.foldToIonSequence(ion.newEmptySexp()) + ExprValueType.STRUCT -> this.fold(ion.newEmptyStruct()) { struct, el -> + struct.apply { add(el.name!!.stringValue(), el.toIonValue(ion)) } + } + ExprValueType.BAG -> { + val bag = ion.newEmptyList().apply { addTypeAnnotation(BAG_ANNOTATION) } + this.foldToIonSequence(bag) + } + } +}