Add FILTER_DISTINCT function.

docs/user/BuiltInFunctions.md

@@ -226,8 +226,6 @@ CAST(<<'a', 'b'>> AS bag) -- <<'a', 'b'>> (REPL does not display << >> and comma
 
 ### CHAR_LENGTH, CHARACTER_LENGTH
 
-
-
 Counts the number of characters in the specified string, where 'character' is defined as a single unicode code point.
 
 *Note:* `CHAR_LENGTH` and `CHARACTER_LENGTH` are synonyms. 
@@ -455,9 +453,31 @@ EXTRACT(TIMEZONE_MINUTE FROM TIME WITH TIME ZONE '23:12:59-08:30')  -- -30
 ```
 *Note* that `timezone_hour` and `timezone_minute` are **not supported** for `DATE` and `TIME` (without time zone) type. 
 
+### `FILTER_DISTINCT`
+
+Signature
+: `FILTER_DISTINCT: Container -> Bag`
+
+Header
+: `FILTER_DISTINCT(c)`
+
+Purpose
+: Returns a bag of distinct values contained within a bag, list, sexp, or struct.  If the container is a struct,
+the field names are not considered.
+
+Examples
+:
+
+```sql
+FILTER_DISTINCT([0, 0, 1])                  -- <<0, 1>>
+FILTER_DISTINCT(<<0, 0, 1>>)                -- <<0, 1>>
+FILTER_DISTINCT(SEXP(0, 0, 1))              -- <<0, 1>>
+FILTER_DISTINCT({'a': 0, 'b': 0, 'c': 1})   -- <<0, 1>>
+```
+
 ### LOWER 
 
-Given a string convert all upper case characters to lower case characters. 
+Given a string convert all upper case characters to lower case characters.
 
 Signature
 : `LOWER: String -> String`

lang/src/org/partiql/lang/eval/builtins/BuiltinFunctions.kt

@@ -15,15 +15,18 @@
 package org.partiql.lang.eval.builtins
 
 import com.amazon.ion.system.IonSystemBuilder
+import org.partiql.lang.eval.DEFAULT_COMPARATOR
 import org.partiql.lang.eval.EvaluationSession
 import org.partiql.lang.eval.ExprFunction
 import org.partiql.lang.eval.ExprValue
 import org.partiql.lang.eval.ExprValueFactory
 import org.partiql.lang.eval.stringValue
+import org.partiql.lang.eval.unnamedValue
 import org.partiql.lang.types.AnyOfType
 import org.partiql.lang.types.FunctionSignature
 import org.partiql.lang.types.StaticType
 import org.partiql.lang.types.UnknownArguments
+import java.util.TreeSet
 
 internal const val DYNAMIC_LOOKUP_FUNCTION_NAME = "\$__dynamic_lookup__"
 
@@ -42,6 +45,7 @@ internal fun createBuiltinFunctions(valueFactory: ExprValueFactory) =
         createCharacterLength("character_length", valueFactory),
         createCharacterLength("char_length", valueFactory),
         createUtcNow(valueFactory),
+        createFilterDistinct(valueFactory),
         DateAddExprFunction(valueFactory),
         DateDiffExprFunction(valueFactory),
         ExtractExprFunction(valueFactory),
@@ -79,6 +83,29 @@ internal fun createUtcNow(valueFactory: ExprValueFactory): ExprFunction = object
         valueFactory.newTimestamp(session.now)
 }
 
+internal fun createFilterDistinct(valueFactory: ExprValueFactory): ExprFunction = object : ExprFunction {
+    override val signature = FunctionSignature(
+        "filter_distinct",
+        listOf(StaticType.unionOf(StaticType.BAG, StaticType.LIST, StaticType.SEXP, StaticType.STRUCT)),
+        returnType = StaticType.BAG
+    )
+
+    override fun callWithRequired(session: EvaluationSession, required: List<ExprValue>): ExprValue {
+        val argument = required.first()
+        // We cannot use a [HashSet] here because [ExprValue] does not implement .equals() and .hashCode()
+        val encountered = TreeSet(DEFAULT_COMPARATOR)
+        return valueFactory.newBag(
+            sequence {
+                argument.asSequence().forEach {
+                    if (!encountered.contains(it)) {
+                        encountered.add(it.unnamedValue())
+                        yield(it)
+                    }
+                }
+            }
+        )
+    }
+}
 internal fun createCharacterLength(name: String, valueFactory: ExprValueFactory): ExprFunction =
     object : ExprFunction {
         override val signature: FunctionSignature

lang/test/org/partiql/lang/eval/builtins/functions/FilterDistinctEvaluationTest.kt

@@ -0,0 +1,73 @@
+package org.partiql.lang.eval.builtins.functions
+
+import org.junit.Test
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.ArgumentsSource
+import org.partiql.lang.errors.ErrorCode
+import org.partiql.lang.errors.Property
+import org.partiql.lang.eval.EvaluatorTestBase
+import org.partiql.lang.eval.builtins.ExprFunctionTestCase
+import org.partiql.lang.eval.builtins.checkInvalidArity
+import org.partiql.lang.util.ArgumentsProviderBase
+import org.partiql.lang.util.propertyValueMapOf
+import org.partiql.lang.util.to
+
+class FilterDistinctEvaluationTest : EvaluatorTestBase() {
+    // Pass test cases
+    @ParameterizedTest
+    @ArgumentsSource(ToStringPassCases::class)
+    fun runPassTests(testCase: ExprFunctionTestCase) =
+        runEvaluatorTestCase(query = testCase.source, expectedResult = testCase.expectedLegacyModeResult)
+
+    // We rely on the built-in [DEFAULT_COMPARATOR] for the actual definition of equality, which is not being tested
+    // here.
+    class ToStringPassCases : ArgumentsProviderBase() {
+        override fun getParameters(): List<Any> = listOf(
+
+            // These three tests ensure we can accept lists, bags, s-expressions and structs
+            ExprFunctionTestCase("filter_distinct([0, 0, 1])", "[0, 1]"), // list
+            ExprFunctionTestCase("filter_distinct(<<0, 0, 1>>)", "[0, 1]"), // bag
+            ExprFunctionTestCase("filter_distinct(SEXP(0, 0, 1))", "[0, 1]"), // s-exp
+            ExprFunctionTestCase("filter_distinct({'a': 0, 'b': 0, 'c': 1})", "[0, 1]"), // struct
+
+            // Some "smoke tests" to ensure the basic plumbing is working right.
+            ExprFunctionTestCase("filter_distinct(['foo', 'foo', 1, 1, `symbol`, `symbol`])", "[\"foo\", 1, symbol]"),
+            ExprFunctionTestCase("filter_distinct([{ 'a': 1 }, { 'a': 1 }, { 'a': 1 }])", "[{ 'a': 1 }]"),
+            ExprFunctionTestCase("filter_distinct([[1, 1], [1, 1], [2, 2]])", "[[1,1], [2, 2]]"),
+        )
+    }
+
+    // Error test cases: Invalid arguments
+    data class InvalidArgTestCase(
+        val source: String,
+        val actualArgumentType: String
+    )
+
+    @ParameterizedTest
+    @ArgumentsSource(InvalidArgCases::class)
+    fun toStringInvalidArgumentTests(testCase: InvalidArgTestCase) = runEvaluatorErrorTestCase(
+        query = testCase.source,
+        expectedErrorCode = ErrorCode.EVALUATOR_INCORRECT_TYPE_OF_ARGUMENTS_TO_FUNC_CALL,
+        expectedErrorContext = propertyValueMapOf(
+            1, 1,
+            Property.FUNCTION_NAME to "filter_distinct",
+            Property.EXPECTED_ARGUMENT_TYPES to "BAG, LIST, SEXP, or STRUCT",
+            Property.ACTUAL_ARGUMENT_TYPES to testCase.actualArgumentType,
+            Property.ARGUMENT_POSITION to 1
+        ),
+        expectedPermissiveModeResult = "MISSING",
+    )
+
+    class InvalidArgCases : ArgumentsProviderBase() {
+        override fun getParameters(): List<Any> = listOf(
+            InvalidArgTestCase("filter_distinct(1)", "INT"),
+            InvalidArgTestCase("filter_distinct(1.0)", "DECIMAL"),
+            InvalidArgTestCase("filter_distinct('foo')", "STRING"),
+            InvalidArgTestCase("filter_distinct(`some_symbol`)", "SYMBOL"),
+            InvalidArgTestCase("filter_distinct(`{{ '''a clob''' }}`)", "CLOB"),
+        )
+    }
+
+    @Test
+    fun invalidArityTest() = checkInvalidArity(funcName = "filter_distinct", maxArity = 1, minArity = 1)
+}