From 599a8abdee0f5374ebe0a14ebbf70bf7948b675a Mon Sep 17 00:00:00 2001 From: Francesco Capponi Date: Thu, 6 May 2021 18:13:12 -0400 Subject: [PATCH 01/37] Support long literals (#2) * Support long in SQL * Add comparison test * Add long support in PPL * Add PPL doc * Add PPL IT * Update doc * More UT Co-authored-by: Dai --- .../sql/ast/dsl/AstDSL.java | 4 ++ docs/experiment/ppl/general/datatypes.rst | 2 +- docs/user/general/datatypes.rst | 2 +- .../sql/ppl/DataTypeIT.java | 17 +++++++++ .../correctness/expressions/literals.txt | 4 ++ .../sql/ppl/parser/AstExpressionBuilder.java | 6 ++- .../ppl/parser/AstExpressionBuilderTest.java | 38 ++++++++++++++++--- .../sql/sql/parser/AstExpressionBuilder.java | 6 ++- .../sql/parser/AstExpressionBuilderTest.java | 25 ++++++++++++ 9 files changed, 95 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java index a1515a7479..8c15f71bd9 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java @@ -145,6 +145,10 @@ public static Literal intLiteral(Integer value) { return literal(value, DataType.INTEGER); } + public static Literal longLiteral(Long value) { + return literal(value, DataType.LONG); + } + public static Literal dateLiteral(String value) { return literal(value, DataType.DATE); } diff --git a/docs/experiment/ppl/general/datatypes.rst b/docs/experiment/ppl/general/datatypes.rst index 5182abd018..147b6b2e62 100644 --- a/docs/experiment/ppl/general/datatypes.rst +++ b/docs/experiment/ppl/general/datatypes.rst @@ -110,7 +110,7 @@ Notes: Not all the PPL Type has correspond Elasticsearch Type. e.g. data and tim Numeric Data Types ================== -TODO +Numeric values ranged from -2147483648 to +2147483647 are recognized as integer with type name ``INTEGER``. For others outside the range, ``LONG`` integer will be the data type after parsed. Date and Time Data Types diff --git a/docs/user/general/datatypes.rst b/docs/user/general/datatypes.rst index 768eb85064..b4fc179a43 100644 --- a/docs/user/general/datatypes.rst +++ b/docs/user/general/datatypes.rst @@ -125,7 +125,7 @@ Here are examples for NULL literal and expressions with NULL literal involved:: Numeric Data Types ================== -TODO +Numeric values ranged from -2147483648 to +2147483647 are recognized as integer with type name ``INTEGER``. For others outside the range, ``LONG`` integer will be the data type after parsed. Date and Time Data Types diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/DataTypeIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/DataTypeIT.java index e609bd93e3..16b93c290a 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/DataTypeIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/DataTypeIT.java @@ -64,4 +64,21 @@ public void test_nonnumeric_data_types() throws IOException { schema("nested_value", "array")); } + @Test + public void test_long_integer_data_type() throws IOException { + JSONObject result = executeQuery( + String.format("source=%s | eval " + + " int1 = 2147483647," + + " int2 = -2147483648," + + " long1 = 2147483648," + + " long2 = -2147483649 | " + + "fields int1, int2, long1, long2 ", + TEST_INDEX_DATATYPE_NUMERIC)); + verifySchema(result, + schema("int1", "integer"), + schema("int2", "integer"), + schema("long1", "long"), + schema("long2", "long")); + } + } diff --git a/integ-test/src/test/resources/correctness/expressions/literals.txt b/integ-test/src/test/resources/correctness/expressions/literals.txt index b70fa41e83..264aa459e9 100644 --- a/integ-test/src/test/resources/correctness/expressions/literals.txt +++ b/integ-test/src/test/resources/correctness/expressions/literals.txt @@ -5,3 +5,7 @@ true -4.567 -123,false 'ODFE' +2147483647 +-2147483648 +2147483648 +-2147483649 diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java index 4d1bf4a23c..a118f1deb6 100644 --- a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java @@ -260,7 +260,11 @@ public UnresolvedExpression visitStringLiteral(StringLiteralContext ctx) { @Override public UnresolvedExpression visitIntegerLiteral(IntegerLiteralContext ctx) { - return new Literal(Integer.valueOf(ctx.getText()), DataType.INTEGER); + long number = Long.parseLong(ctx.getText()); + if (Integer.MIN_VALUE <= number && number <= Integer.MAX_VALUE) { + return new Literal((int) number, DataType.INTEGER); + } + return new Literal(number, DataType.LONG); } @Override diff --git a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilderTest.java index 7f0f3badb2..81abf2fab2 100644 --- a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -36,6 +36,7 @@ import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intervalLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.let; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.longLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.not; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.nullLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.or; @@ -429,13 +430,40 @@ public void testStringLiteralExpr() { @Test public void testIntegerLiteralExpr() { - assertEqual("source=t a=1", + assertEqual("source=t a=1 b=-1", filter( relation("t"), - compare( - "=", - field("a"), - intLiteral(1) + and( + compare( + "=", + field("a"), + intLiteral(1) + ), + compare( + "=", + field("b"), + intLiteral(-1) + ) + ) + )); + } + + @Test + public void testLongLiteralExpr() { + assertEqual("source=t a=1234567890123 b=-1234567890123", + filter( + relation("t"), + and( + compare( + "=", + field("a"), + longLiteral(1234567890123L) + ), + compare( + "=", + field("b"), + longLiteral(-1234567890123L) + ) ) )); } diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java index 84e58d9535..1fefc0ddfb 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java @@ -255,7 +255,11 @@ public UnresolvedExpression visitString(StringContext ctx) { @Override public UnresolvedExpression visitSignedDecimal(SignedDecimalContext ctx) { - return AstDSL.intLiteral(Integer.valueOf(ctx.getText())); + long number = Long.parseLong(ctx.getText()); + if (Integer.MIN_VALUE <= number && number <= Integer.MAX_VALUE) { + return AstDSL.intLiteral((int) number); + } + return AstDSL.longLiteral(number); } @Override diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java index 8ff5c50ce6..b1ec56ce51 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -25,6 +25,7 @@ import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.function; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intervalLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.longLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.not; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.nullLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.or; @@ -74,6 +75,30 @@ public void canBuildIntegerLiteral() { intLiteral(123), buildExprAst("123") ); + assertEquals( + intLiteral(Integer.MAX_VALUE), + buildExprAst(String.valueOf(Integer.MAX_VALUE)) + ); + assertEquals( + intLiteral(Integer.MIN_VALUE), + buildExprAst(String.valueOf(Integer.MIN_VALUE)) + ); + } + + @Test + public void canBuildLongLiteral() { + assertEquals( + longLiteral(1234567890123L), + buildExprAst("1234567890123") + ); + assertEquals( + longLiteral(Integer.MAX_VALUE + 1L), + buildExprAst(String.valueOf(Integer.MAX_VALUE + 1L)) + ); + assertEquals( + longLiteral(Integer.MIN_VALUE - 1L), + buildExprAst(String.valueOf(Integer.MIN_VALUE - 1L)) + ); } @Test From 8b942a6529499dc8ff48e362509aeec1637b7c1c Mon Sep 17 00:00:00 2001 From: Francesco Capponi Date: Thu, 6 May 2021 18:13:35 -0400 Subject: [PATCH 02/37] Adding support to NOT REGEXP_QUERY, which is currently failing when run with "Negative operator [REGEXP] is not supported." (#1) --- .../sql/legacy/MethodQueryIT.java | 20 +++++++++++++++++++ .../sql/legacy/QueryIT.java | 16 +++++++++++++++ .../sql/legacy/domain/Condition.java | 4 +++- .../sql/legacy/query/maker/Maker.java | 3 ++- 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java index 6cb584d85c..862363df8c 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java @@ -91,6 +91,26 @@ public void scoreQueryTest() throws IOException { "{\"filter\":{\"match\":{\"address\":{\"query\":\"Street\""))); } + @Test + public void regexpQueryTest() throws IOException { + final String result = explainQuery(String.format(Locale.ROOT, + "SELECT * FROM %s WHERE address=REGEXP_QUERY('.*')", + TestsConstants.TEST_INDEX_ACCOUNT)); + Assert.assertThat(result, + containsString("{\"bool\":{\"must\":[{\"regexp\":" + + "{\"address\":{\"value\":\".*\",\"flags_value\":255,\"max_determinized_states\":10000,\"boost\":1.0}}}")); + } + + @Test + public void negativeRegexpQueryTest() throws IOException { + final String result = explainQuery(String.format(Locale.ROOT, + "SELECT * FROM %s WHERE NOT(address=REGEXP_QUERY('.*'))", + TestsConstants.TEST_INDEX_ACCOUNT)); + Assert.assertThat(result, + containsString("{\"bool\":{\"must_not\":[{\"regexp\":" + + "{\"address\":{\"value\":\".*\",\"flags_value\":255,\"max_determinized_states\":10000,\"boost\":1.0}}}")); + } + /** * wildcardQuery 是用通配符的方式查找某个term 　比如例子中 l*e means leae ltae .... * "wildcard": { "address" : { "wildcard" : "l*e" } } diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryIT.java index 1e634ff679..c929be4367 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryIT.java @@ -482,6 +482,22 @@ public void regexQueryTest() throws IOException { Assert.assertEquals(4, hitSource.getInt("age")); } + @Test + public void negativeRegexQueryTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE NOT(dog_name = REGEXP_QUERY('sn.*', 'INTERSECTION|COMPLEMENT|EMPTY', 10000))", + TestsConstants.TEST_INDEX_DOG)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, hits.length()); + + JSONObject hitSource = getSource(hits.getJSONObject(0)); + Assert.assertNotEquals("snoopy", hitSource.getString("dog_name")); + Assert.assertEquals("rex", hitSource.getString("dog_name")); + } + @Test public void doubleNotTest() throws IOException { JSONObject response1 = executeQuery( diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Condition.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Condition.java index 25ac0bc210..c915c945c7 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Condition.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Condition.java @@ -63,7 +63,8 @@ public enum OPERATOR { CHILDREN_COMPLEX, SCRIPT, NIN_TERMS, - NTERM; + NTERM, + NREGEXP; public static Map methodNameToOpear; @@ -136,6 +137,7 @@ public enum OPERATOR { negatives.put(IN, NIN); negatives.put(BETWEEN, NBETWEEN); negatives.put(NESTED_COMPLEX, NOT_EXISTS_NESTED_COMPLEX); + negatives.put(REGEXP, NREGEXP); } static { diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/maker/Maker.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/maker/Maker.java index 18caafa507..ae85aec094 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/maker/Maker.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/maker/Maker.java @@ -95,7 +95,7 @@ public abstract class Maker { private static final Set NOT_OPERATOR_SET = ImmutableSet.of( Condition.OPERATOR.N, Condition.OPERATOR.NIN, Condition.OPERATOR.ISN, Condition.OPERATOR.NBETWEEN, Condition.OPERATOR.NLIKE, Condition.OPERATOR.NIN_TERMS, Condition.OPERATOR.NTERM, - Condition.OPERATOR.NOT_EXISTS_NESTED_COMPLEX + Condition.OPERATOR.NOT_EXISTS_NESTED_COMPLEX, Condition.OPERATOR.NREGEXP ); protected Maker(Boolean isQuery) { @@ -217,6 +217,7 @@ private ToXContent make(Condition cond, String name, Object value) throws SqlPar toXContent = QueryBuilders.wildcardQuery(name, queryStr); break; case REGEXP: + case NREGEXP: Object[] values = (Object[]) value; RegexpQueryBuilder regexpQuery = QueryBuilders.regexpQuery(name, values[0].toString()); if (1 < values.length) { From 4a5f71244c583ef91da08da86a5a08397c781c1e Mon Sep 17 00:00:00 2001 From: Francesco Capponi Date: Sat, 22 May 2021 16:39:34 -0400 Subject: [PATCH 03/37] In support (#3) * Adding IN feature in new Engine * IN: fixing some CSV strange behavior that in the legacy engine COUNT is returning float, and some tests for the new engine were disabled for that --- .../sql/ast/dsl/AstDSL.java | 4 ++ .../sql/ast/expression/DataType.java | 1 + .../sql/ast/expression/In.java | 3 + .../sql/data/model/ExprValueUtils.java | 6 +- .../sql/expression/DSL.java | 21 +++++- .../function/BuiltinFunctionName.java | 1 + .../predicate/BinaryPredicateOperator.java | 28 ++++++++ .../sql/utils/OperatorUtils.java | 22 ++++++ .../sql/data/model/ExprValueUtilsTest.java | 15 +++- .../BinaryPredicateOperatorTest.java | 70 +++++++++++++++++- .../script/filter/FilterQueryBuilder.java | 2 + .../script/filter/lucene/TermsQuery.java | 39 ++++++++++ .../script/filter/FilterQueryBuilderTest.java | 39 ++++++++++ .../sql/legacy/CsvFormatResponseIT.java | 6 ++ .../sql/sql/InIT.java | 71 +++++++++++++++++++ sql/src/main/antlr/OpenDistroSQLParser.g4 | 9 +++ .../sql/sql/parser/AstExpressionBuilder.java | 17 +++++ .../sql/sql/antlr/SQLSyntaxParserTest.java | 7 ++ .../sql/parser/AstExpressionBuilderTest.java | 17 +++++ 19 files changed, 371 insertions(+), 7 deletions(-) create mode 100644 elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/lucene/TermsQuery.java create mode 100644 integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/InIT.java diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java index 8c15f71bd9..2c756faec1 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java @@ -161,6 +161,10 @@ public static Literal timestampLiteral(String value) { return literal(value, DataType.TIMESTAMP); } + public static Literal arrayLiteral(List value) { + return literal(value, DataType.ARRAY); + } + public static Literal doubleLiteral(Double value) { return literal(value, DataType.DOUBLE); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/DataType.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/DataType.java index 13e6b422bd..5b68d15caa 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/DataType.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/DataType.java @@ -37,6 +37,7 @@ public enum DataType { DATE(ExprCoreType.DATE), TIME(ExprCoreType.TIME), TIMESTAMP(ExprCoreType.TIMESTAMP), + ARRAY(ExprCoreType.ARRAY), INTERVAL(ExprCoreType.INTERVAL); @Getter diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/In.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/In.java index 365787780b..945bce841d 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/In.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/In.java @@ -28,7 +28,10 @@ * Params include the field expression and/or wildcard field expression, * nested field expression (@field). * And the values that the field is mapped to (@valueList). + * + * @deprecated use function ("in") instead */ +@Deprecated @Getter @ToString @EqualsAndHashCode(callSuper = false) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java index 976f2de8bb..50de610b28 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java @@ -88,7 +88,7 @@ public static ExprValue tupleValue(Map map) { /** * {@link ExprCollectionValue} constructor. */ - public static ExprValue collectionValue(List list) { + public static ExprValue collectionValue(List list) { List valueList = new ArrayList<>(); list.forEach(o -> valueList.add(fromObjectValue(o))); return new ExprCollectionValue(valueList); @@ -129,6 +129,10 @@ public static ExprValue fromObjectValue(Object o) { return stringValue((String) o); } else if (o instanceof Float) { return floatValue((Float) o); + } else if (o instanceof ExprValue) { + // since there is no primitive in Java for differentiating TIMESTAMP DATETIME and DATE + // we can allow passing a ExprValue that already contains this information + return (ExprValue) o; } else { throw new ExpressionEvaluationException("unsupported object " + o.getClass()); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java index 0d03ddc536..467499cdd7 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java @@ -26,8 +26,10 @@ import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; import com.amazon.opendistroforelasticsearch.sql.expression.window.ranking.RankingWindowFunction; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.List; import lombok.RequiredArgsConstructor; @RequiredArgsConstructor @@ -252,7 +254,7 @@ public FunctionExpression subtract(Expression... expressions) { public FunctionExpression multiply(Expression... expressions) { return function(BuiltinFunctionName.MULTIPLY, expressions); } - + public FunctionExpression adddate(Expression... expressions) { return function(BuiltinFunctionName.ADDDATE, expressions); } @@ -364,7 +366,7 @@ public FunctionExpression module(Expression... expressions) { public FunctionExpression substr(Expression... expressions) { return function(BuiltinFunctionName.SUBSTR, expressions); } - + public FunctionExpression substring(Expression... expressions) { return function(BuiltinFunctionName.SUBSTR, expressions); } @@ -588,4 +590,19 @@ public FunctionExpression castTimestamp(Expression value) { return (FunctionExpression) repository .compile(BuiltinFunctionName.CAST_TO_TIMESTAMP.getName(), Arrays.asList(value)); } + + /** + * Check that a field is contained in a set of values. + */ + public FunctionExpression in(Expression field, Expression... expressions) { + List where = new ArrayList<>(); + where.add(field); + where.addAll(Arrays.asList(expressions)); + + return function(BuiltinFunctionName.IN, where.toArray(new Expression[0])); + } + + public FunctionExpression not_in(Expression field, Expression... expressions) { + return not(in(field, expressions)); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java index 6b29c68da1..1e4c245b52 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java @@ -106,6 +106,7 @@ public enum BuiltinFunctionName { GTE(FunctionName.of(">=")), LIKE(FunctionName.of("like")), NOT_LIKE(FunctionName.of("not like")), + IN(FunctionName.of("in")), /** * Aggregation Function. diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java index d08c3fab8f..67e4fdd64a 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java @@ -19,9 +19,16 @@ import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.ARRAY; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATETIME; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; @@ -63,6 +70,7 @@ public static void register(BuiltinFunctionRepository repository) { repository.register(like()); repository.register(notLike()); repository.register(regexp()); + repository.register(in()); } /** @@ -262,6 +270,26 @@ private static FunctionResolver notLike() { STRING)); } + private static FunctionResolver in() { + return FunctionDSL.define(BuiltinFunctionName.IN.getName(), + FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::in), + BOOLEAN, INTEGER, ARRAY), + FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::in), + BOOLEAN, STRING, ARRAY), + FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::in), + BOOLEAN, LONG, ARRAY), + FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::in), + BOOLEAN, FLOAT, ARRAY), + FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::in), + BOOLEAN, DOUBLE, ARRAY), + FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::in), + BOOLEAN, DATE, ARRAY), + FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::in), + BOOLEAN, DATETIME, ARRAY), + FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::in), + BOOLEAN, TIMESTAMP, ARRAY)); + } + private static ExprValue lookupTableFunction(ExprValue arg1, ExprValue arg2, Table table) { if (table.contains(arg1, arg2)) { diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/OperatorUtils.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/OperatorUtils.java index d887d5c391..fa48fb6a57 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/OperatorUtils.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/OperatorUtils.java @@ -15,8 +15,10 @@ package com.amazon.opendistroforelasticsearch.sql.utils; +import com.amazon.opendistroforelasticsearch.sql.data.model.AbstractExprNumberValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprStringValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; import java.util.regex.Pattern; import lombok.experimental.UtilityClass; @@ -99,4 +101,24 @@ private static String patternToRegex(String patternString) { regex.append('$'); return regex.toString(); } + + + /** + * IN (..., ...) operator util. + * Expression { expr IN (collection of values..) } is to judge + * if expr is contained in a given collection. + */ + public static ExprBooleanValue in(ExprValue expr, ExprValue setOfValues) { + return ExprBooleanValue.of(isIn(expr, setOfValues)); + } + + private static boolean isIn(ExprValue expr, ExprValue setOfValues) { + if (expr instanceof AbstractExprNumberValue) { + return setOfValues.collectionValue().contains(expr.value()); + } else if (expr instanceof ExprStringValue) { + return setOfValues.collectionValue().contains(expr.stringValue()); + } else { + return setOfValues.collectionValue().contains(expr); + } + } } diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java index 4c97632958..66cf4fc9d2 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java @@ -215,13 +215,24 @@ public void invalidConvertExprValue(ExprValue value, Function assertThat(exception.getMessage(), Matchers.containsString("invalid")); } + // disabling test because in case of expr collections, we could pass ExprValues + // @Test + // public void unSupportedObject() { + // Exception exception = assertThrows(ExpressionEvaluationException.class, + // () -> ExprValueUtils.fromObjectValue(integerValue(1))); + // assertEquals( + // "unsupported object " + // + "class com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue", + // exception.getMessage()); + // } + @Test public void unSupportedObject() { Exception exception = assertThrows(ExpressionEvaluationException.class, - () -> ExprValueUtils.fromObjectValue(integerValue(1))); + () -> ExprValueUtils.fromObjectValue(new Object())); assertEquals( "unsupported object " - + "class com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue", + + "class java.lang.Object", exception.getMessage()); } diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java index aa7402142c..7efd0dca46 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java @@ -27,14 +27,17 @@ import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.booleanValue; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.fromObjectValue; -import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.missingValue; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATETIME; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP; import static com.amazon.opendistroforelasticsearch.sql.utils.ComparisonUtil.compare; import static com.amazon.opendistroforelasticsearch.sql.utils.OperatorUtils.matches; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; @@ -49,14 +52,15 @@ import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import com.amazon.opendistroforelasticsearch.sql.expression.ExpressionTestBase; import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; +import com.amazon.opendistroforelasticsearch.sql.utils.OperatorUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.sun.org.apache.xpath.internal.Arg; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.ObjectInputStream; @@ -832,4 +836,66 @@ public void compare_int_long() { FunctionExpression equal = dsl.equal(DSL.literal(1), DSL.literal(1L)); assertTrue(equal.valueOf(valueEnv()).booleanValue()); } + + private static Stream testInArguments() { + List arguments = + Arrays.asList(Arrays.asList(1, Arrays.asList(0, 2, 1, 3)), + Arrays.asList(1, Arrays.asList(2, 0)), Arrays.asList(1L, Arrays.asList(1L, 2L, 3L)), + Arrays.asList(2L, Arrays.asList(1L, 2L)), Arrays.asList(3F, Arrays.asList(1F, 2F)), + Arrays.asList(0F, Arrays.asList(1F, 2F)), Arrays.asList(1D, Arrays.asList(1D, 1D)), + Arrays.asList(1D, Arrays.asList(2D, 2D)), + Arrays.asList("b", Arrays.asList("a", "c")), + Arrays.asList("b", Arrays.asList("c", "a")), + Arrays.asList("a", Arrays.asList("a", "b")), + Arrays.asList("b", Arrays.asList("a", "b")), + Arrays.asList("c", Arrays.asList("a", "b")), + Arrays.asList("a", Arrays.asList("b", "c")), + Arrays.asList("a", Arrays.asList("a", "a")), + Arrays.asList("b", Arrays.asList("a", "a"))); + + Stream.Builder builder = Stream.builder(); + for (List argGroup : arguments) { + builder.add(Arguments.of(fromObjectValue(argGroup.get(0)), fromObjectValue(argGroup.get(1)))); + } + builder + .add(Arguments.of(fromObjectValue("2021-01-02", DATE), + fromObjectValue(Arrays.asList(fromObjectValue("2021-01-01", DATE), + fromObjectValue("2021-01-03", DATE))))) + .add(Arguments.of(fromObjectValue("2021-01-02", DATE), + fromObjectValue(Arrays.asList(fromObjectValue("2021-01-01", DATE), + fromObjectValue("2021-01-03", DATE))))) + .add(Arguments.of(fromObjectValue("2021-01-01 03:00:00", DATETIME), + fromObjectValue(Arrays.asList(fromObjectValue("2021-01-01 01:00:00", DATETIME), + fromObjectValue("3021-01-01 02:00:00", DATETIME))))) + .add(Arguments.of(fromObjectValue("2021-01-01 01:00:00", TIMESTAMP), + fromObjectValue(Arrays.asList(fromObjectValue("2021-01-01 01:00:00", TIMESTAMP), + fromObjectValue("3021-01-01 01:00:00", TIMESTAMP))))); + return builder.build(); + } + + @ParameterizedTest(name = "in({0}, ({1}))") + @MethodSource("testInArguments") + public void in(ExprValue field, ExprValue arrayOfArgs) { + FunctionExpression in = dsl.in( + DSL.literal(field), DSL.literal(arrayOfArgs)); + assertEquals(BOOLEAN, in.type()); + assertEquals(OperatorUtils.in(field, arrayOfArgs), in.valueOf(valueEnv())); + } + + @ParameterizedTest(name = "not in({0}, ({1}))") + @MethodSource("testInArguments") + public void not_in(ExprValue field, ExprValue arrayOfArgs) { + FunctionExpression notIn = dsl.not_in( + DSL.literal(field), DSL.literal(arrayOfArgs)); + assertEquals(BOOLEAN, notIn.type()); + assertEquals(!OperatorUtils.in(field, arrayOfArgs).booleanValue(), + notIn.valueOf(valueEnv()).booleanValue()); + } + + @Test + public void in_not_an_array() { + assertThrows(ExpressionEvaluationException.class, () -> + dsl.in(DSL.literal(1), DSL.literal("1"))); + } + } \ No newline at end of file diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilder.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilder.java index 4cc8be3512..af2dcf81e7 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilder.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilder.java @@ -24,6 +24,7 @@ import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.lucene.RangeQuery; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.lucene.RangeQuery.Comparison; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.lucene.TermQuery; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.lucene.TermsQuery; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.lucene.WildcardQuery; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.serialization.ExpressionSerializer; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; @@ -63,6 +64,7 @@ public class FilterQueryBuilder extends ExpressionNodeVisitor 1)"; + JSONObject results = executeQuery(sql); + Assert.assertThat(getTotalHits(results), equalTo(1)); + } + +} diff --git a/sql/src/main/antlr/OpenDistroSQLParser.g4 b/sql/src/main/antlr/OpenDistroSQLParser.g4 index 4f01c657c9..01967fbcdc 100644 --- a/sql/src/main/antlr/OpenDistroSQLParser.g4 +++ b/sql/src/main/antlr/OpenDistroSQLParser.g4 @@ -260,6 +260,7 @@ predicate | predicate IS nullNotnull #isNullPredicate | left=predicate NOT? LIKE right=predicate #likePredicate | left=predicate REGEXP right=predicate #regexpPredicate + | predicate NOT? IN LR_BRACKET arrayArgs? RR_BRACKET #inPredicate ; expressionAtom @@ -370,3 +371,11 @@ functionArg : expression ; +arrayArgs + : arrayArg (COMMA arrayArg)* + ; + +arrayArg + : expression + ; + diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java index 1fefc0ddfb..2fa59cfaef 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java @@ -18,9 +18,11 @@ import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.qualifiedName; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.stringLiteral; +import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.IN; import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.IS_NOT_NULL; import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.IS_NULL; import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.LIKE; +import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.NOT; import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.NOT_LIKE; import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.REGEXP; import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.BinaryComparisonPredicateContext; @@ -61,8 +63,10 @@ import com.amazon.opendistroforelasticsearch.sql.ast.expression.Case; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Cast; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Function; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.In; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Interval; import com.amazon.opendistroforelasticsearch.sql.ast.expression.IntervalUnit; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Literal; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Not; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Or; import com.amazon.opendistroforelasticsearch.sql.ast.expression.QualifiedName; @@ -233,6 +237,19 @@ public UnresolvedExpression visitRegexpPredicate(RegexpPredicateContext ctx) { Arrays.asList(visit(ctx.left), visit(ctx.right))); } + @Override + public UnresolvedExpression visitInPredicate(OpenDistroSQLParser.InPredicateContext ctx) { + UnresolvedExpression between = new Function(IN.getName().getFunctionName(), + Arrays.asList(visit(ctx.predicate()), AstDSL.arrayLiteral(ctx.arrayArgs().arrayArg() + .stream() + .map(this::visitArrayArg) + .map(unresolvedExpression -> ((Literal) unresolvedExpression).getValue()) + .collect(Collectors.toList())))); + + return ctx.NOT() == null ? between : + new Function(NOT.getName().getFunctionName(), Collections.singletonList(between)); + } + @Override public UnresolvedExpression visitAndExpression(AndExpressionContext ctx) { return new And(visit(ctx.left), visit(ctx.right)); diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java index 0f2605d7d6..d0dddf1142 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -131,6 +131,13 @@ public void canParseCaseStatement() { assertNotNull(parser.parse("SELECT CASE age WHEN 30 THEN 'age1' END FROM test")); } + @Test + public void canParseInStatement() { + assertNotNull(parser.parse("SELECT age FROM test WHERE age IN (1,30)")); + assertNotNull(parser.parse("SELECT age FROM test WHERE age NOT IN (1,30)")); + assertNotNull(parser.parse("SELECT age FROM test WHERE NOT (age IN (1,30))")); + } + @Test public void canNotParseAggregateFunctionWithWrongArgument() { assertThrows(SyntaxCheckException.class, () -> parser.parse("SELECT SUM() FROM test")); diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java index b1ec56ce51..4047f86ce1 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -18,11 +18,13 @@ import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.aggregate; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.and; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.arrayLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.booleanLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.caseWhen; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.dateLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.doubleLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.function; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.in; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intervalLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.longLiteral; @@ -49,6 +51,7 @@ import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLLexer; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser; import com.google.common.collect.ImmutableList; +import java.util.Arrays; import org.antlr.v4.runtime.CommonTokenStream; import org.apache.commons.lang3.tuple.ImmutablePair; import org.junit.jupiter.api.Test; @@ -283,6 +286,20 @@ public void canBuildLogicalExpression() { ); } + @Test + public void canBuildInPredicate() { + assertEquals(function("in", intLiteral(1), arrayLiteral(Arrays.asList(0,2,3,4))), + buildExprAst("1 in (0,2,3,4)")); + } + + @Test + public void canBuildNotInPredicate() { + assertEquals( + function("not", + function("in", intLiteral(1),arrayLiteral(Arrays.asList(0,2,3,4)))), + buildExprAst("1 not in (0,2,3,4)")); + } + @Test public void canBuildWindowFunction() { assertEquals( From f6d62d35dc0bcd4aaa2c990e82a0cdc8c6416636 Mon Sep 17 00:00:00 2001 From: Francesco Capponi Date: Mon, 31 May 2021 23:44:33 -0400 Subject: [PATCH 04/37] Date Autocast for new engine --- .../sql/data/model/ExprStringValue.java | 6 ++ .../sql/data/type/ExprCoreType.java | 4 + .../predicate/BinaryPredicateOperator.java | 77 ++++++++++--------- .../BinaryPredicateOperatorTest.java | 32 ++++++++ .../sql/utils/ComparisonUtil.java | 14 ++++ .../script/filter/FilterQueryBuilderTest.java | 60 +++++++++++++++ 6 files changed, 155 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java index 09761dcfe9..a0d7364bcd 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java @@ -18,6 +18,7 @@ import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; @@ -80,6 +81,11 @@ public LocalTime timeValue() { } } + @Override + public Instant timestampValue() { + return new ExprTimestampValue(value).timestampValue(); + } + @Override public String toString() { return String.format("\"%s\"", value); diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java index 7928dbb85f..50f1cd891c 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java @@ -131,4 +131,8 @@ public static List coreTypes() { public static List numberTypes() { return ImmutableList.of(INTEGER, LONG, FLOAT, DOUBLE); } + + public static List dateTypes() { + return ImmutableList.of(DATE, DATETIME, TIMESTAMP, TIME); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java index d08c3fab8f..b60179f6fc 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java @@ -25,16 +25,26 @@ import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionBuilder; import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionDSL; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName; import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionResolver; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionSignature; +import com.amazon.opendistroforelasticsearch.sql.expression.function.SerializableFunction; import com.amazon.opendistroforelasticsearch.sql.utils.OperatorUtils; import com.google.common.collect.ImmutableTable; import com.google.common.collect.Table; +import java.io.Serializable; +import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; import lombok.experimental.UtilityClass; +import org.apache.commons.lang3.tuple.Pair; + /** * The definition of binary predicate function @@ -192,53 +202,44 @@ private static FunctionResolver notEqual() { } private static FunctionResolver less() { - return FunctionDSL - .define(BuiltinFunctionName.LESS.getName(), ExprCoreType.coreTypes().stream() - .map(type -> FunctionDSL - .impl(FunctionDSL - .nullMissingHandling((v1, v2) -> ExprBooleanValue.of(v1.compareTo(v2) < 0)), - BOOLEAN, - type, type)) - .collect( - Collectors.toList())); + return FunctionDSL.define(BuiltinFunctionName.LESS.getName(), + getComparisonFunctions((Function & Serializable) value -> value < 0)); } private static FunctionResolver lte() { - return FunctionDSL - .define(BuiltinFunctionName.LTE.getName(), ExprCoreType.coreTypes().stream() - .map(type -> FunctionDSL - .impl( - FunctionDSL - .nullMissingHandling( - (v1, v2) -> ExprBooleanValue.of(v1.compareTo(v2) <= 0)), - BOOLEAN, - type, type)) - .collect( - Collectors.toList())); + return FunctionDSL.define(BuiltinFunctionName.LTE.getName(), + getComparisonFunctions((Function & Serializable) value -> value <= 0)); } private static FunctionResolver greater() { - return FunctionDSL - .define(BuiltinFunctionName.GREATER.getName(), ExprCoreType.coreTypes().stream() - .map(type -> FunctionDSL - .impl(FunctionDSL - .nullMissingHandling((v1, v2) -> ExprBooleanValue.of(v1.compareTo(v2) > 0)), - BOOLEAN, type, type)) - .collect( - Collectors.toList())); + return FunctionDSL.define(BuiltinFunctionName.GREATER.getName(), + getComparisonFunctions((Function & Serializable) value -> value > 0)); } private static FunctionResolver gte() { - return FunctionDSL - .define(BuiltinFunctionName.GTE.getName(), ExprCoreType.coreTypes().stream() - .map(type -> FunctionDSL - .impl( - FunctionDSL.nullMissingHandling( - (v1, v2) -> ExprBooleanValue.of(v1.compareTo(v2) >= 0)), - BOOLEAN, - type, type)) - .collect( - Collectors.toList())); + return FunctionDSL.define(BuiltinFunctionName.GTE.getName(), + getComparisonFunctions((Function & Serializable) value -> value >= 0)); + } + + private static List>> + getComparisonFunctions(Function f) { + List>> + comparisonFunctions = ExprCoreType.coreTypes() + .stream() + .map(type -> FunctionDSL.impl(FunctionDSL.nullMissingHandling( + (v1, v2) -> ExprBooleanValue.of(f.apply(v1.compareTo(v2)))), BOOLEAN, type, type)) + .collect(Collectors.toList()); + + // all string to date conversions + comparisonFunctions.addAll(ExprCoreType.dateTypes() + .stream() + .map(type -> FunctionDSL.impl(FunctionDSL.nullMissingHandling((v1, v2) -> { + // casting V2 from string to whatever type v1 is + ExprValue v2Casted = ExprValueUtils.fromObjectValue(v2.value(), (ExprCoreType) v1.type()); + return ExprBooleanValue.of(f.apply(v1.compareTo(v2Casted))); + }), BOOLEAN, type, STRING)) + .collect(Collectors.toList())); + return comparisonFunctions; } private static FunctionResolver like() { diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java index aa7402142c..42309faf0c 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java @@ -40,12 +40,16 @@ import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprByteValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprCollectionValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDateValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDatetimeValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDoubleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprFloatValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprLongValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprShortValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprStringValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimeValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; @@ -148,6 +152,34 @@ private static Stream testCompareValueArguments() { builder.add(Arguments.of(new ExprShortValue(1), new ExprShortValue(1))); builder.add(Arguments.of(new ExprShortValue(1), new ExprShortValue(2))); builder.add(Arguments.of(new ExprShortValue(2), new ExprShortValue(1))); + + builder.add(Arguments.of(new ExprDatetimeValue("2006-01-01 00:00:01"), + new ExprStringValue("2006-01-01 00:00:00"))); + builder.add(Arguments.of(new ExprDatetimeValue("2006-01-01 00:00:00"), + new ExprStringValue("2006-01-01 00:00:01"))); + builder.add(Arguments.of(new ExprDatetimeValue("2006-01-01 00:00:00"), + new ExprStringValue("2006-01-01 00:00:00"))); + + builder.add(Arguments.of(new ExprDateValue("2006-01-01"), + new ExprStringValue("2006-01-01"))); + builder.add(Arguments.of(new ExprDateValue("2006-01-01"), + new ExprStringValue("2006-01-02"))); + builder.add(Arguments.of(new ExprDateValue("2006-01-02"), + new ExprStringValue("2006-01-01"))); + + builder.add(Arguments.of(new ExprTimestampValue("2006-01-01 00:00:01"), + new ExprStringValue("2006-01-01 00:00:00"))); + builder.add(Arguments.of(new ExprTimestampValue("2006-01-01 00:00:00"), + new ExprStringValue("2006-01-01 00:00:01"))); + builder.add(Arguments.of(new ExprTimestampValue("2006-01-01 00:00:00"), + new ExprStringValue("2006-01-01 00:00:00"))); + + builder.add(Arguments.of(new ExprTimeValue("00:00:01"), + new ExprStringValue("00:00:00"))); + builder.add(Arguments.of(new ExprTimeValue("00:00:00"), + new ExprStringValue("00:00:01"))); + builder.add(Arguments.of(new ExprTimeValue("00:00:00"), + new ExprStringValue("00:00:00"))); return builder.build(); } diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/ComparisonUtil.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/ComparisonUtil.java index 6913ce6a50..26d1515f07 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/ComparisonUtil.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/ComparisonUtil.java @@ -22,13 +22,19 @@ import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getStringValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprByteValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDateValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDatetimeValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDoubleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprFloatValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprLongValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprShortValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprStringValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimeValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; public class ComparisonUtil { @@ -57,6 +63,14 @@ public static int compare(ExprValue v1, ExprValue v2) { return getDoubleValue(v1).compareTo(getDoubleValue(v2)); } else if (v1 instanceof ExprStringValue) { return getStringValue(v1).compareTo(getStringValue(v2)); + } else if (v1 instanceof ExprDateValue) { + return v1.dateValue().compareTo(v2.dateValue()); + } else if (v1 instanceof ExprDatetimeValue) { + return v1.datetimeValue().compareTo(v2.datetimeValue()); + } else if (v1 instanceof ExprTimeValue) { + return v1.timeValue().compareTo(v2.timeValue()); + } else if (v1 instanceof ExprTimestampValue) { + return v1.timestampValue().compareTo(v2.timestampValue()); } else { throw new ExpressionEvaluationException( String.format("%s instances are not comparable", v1.getClass().getSimpleName())); diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilderTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilderTest.java index 4de1956cdc..7a0a81bee7 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilderTest.java @@ -16,8 +16,12 @@ package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATETIME; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIME; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT_KEYWORD; import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.literal; import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.ref; @@ -26,19 +30,36 @@ import static org.mockito.Mockito.doAnswer; import com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprByteValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDoubleValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprFloatValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprLongValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprShortValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.serialization.ExpressionSerializer; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; import com.amazon.opendistroforelasticsearch.sql.expression.config.ExpressionConfig; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import java.util.Arrays; +import java.util.List; import java.util.Map; +import java.util.stream.Stream; import org.json.JSONObject; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayNameGeneration; import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -99,6 +120,45 @@ void should_build_range_query_for_comparison_expression() { buildQuery(expr))); } + private static List validDateAutocast() { + + return Arrays.asList( + Arguments.of(DATE, "2006-01-01"), + Arguments.of(TIME, "00:00:00"), + Arguments.of(TIMESTAMP, "2006-01-01 00:00:00"), + Arguments.of(DATETIME, "2006-01-01 00:00:00") + ); + } + + @ParameterizedTest + @MethodSource("validDateAutocast") + void shouldBuildRangeQueryForDateComparisonExpressionAutocast(ExprType exprType, + final String time) { + Expression[] params = {ref("date", exprType), literal(time)}; + + String serializedString = "\"" + time + "\""; + Map ranges = ImmutableMap.of( + dsl.less(params), new Object[]{null, serializedString, true, false}, + dsl.greater(params), new Object[]{serializedString, null, false, true}, + dsl.lte(params), new Object[]{null, serializedString, true, true}, + dsl.gte(params), new Object[]{serializedString, null, true, true}); + + ranges.forEach((expr, range) -> + assertJsonEquals( + "{\n" + + " \"range\" : {\n" + + " \"date\" : {\n" + + " \"from\" : " + range[0] + ",\n" + + " \"to\" : " + range[1] + ",\n" + + " \"include_lower\" : " + range[2] + ",\n" + + " \"include_upper\" : " + range[3] + ",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + "}", + buildQuery(expr))); + } + @Test void should_build_wildcard_query_for_like_expression() { assertJsonEquals( From a6e0935bdbcb825079febeb66aacdeaf4776572b Mon Sep 17 00:00:00 2001 From: Francesco Capponi Date: Tue, 1 Jun 2021 01:03:27 -0400 Subject: [PATCH 05/37] Date Autocast for new engine --- .../sql/data/model/ExprDatetimeValue.java | 36 +++++++-------- .../sql/data/model/ExprTimestampValue.java | 45 ++++++++++++++++--- .../sql/data/model/DateTimeValueTest.java | 24 +++++++--- .../script/filter/FilterQueryBuilderTest.java | 11 ----- 4 files changed, 72 insertions(+), 44 deletions(-) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDatetimeValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDatetimeValue.java index 2dd49383ac..1f4550efa1 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDatetimeValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDatetimeValue.java @@ -15,6 +15,8 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue.DATE_FORMATS_ALLOWED; + import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; @@ -36,31 +38,25 @@ public class ExprDatetimeValue extends AbstractExprValue { private final LocalDateTime datetime; - private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS; - private static final int MIN_FRACTION_SECONDS = 0; - private static final int MAX_FRACTION_SECONDS = 6; - - static { - FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder() - .appendPattern("yyyy-MM-dd HH:mm:ss") - .appendFraction( - ChronoField.MICRO_OF_SECOND, - MIN_FRACTION_SECONDS, - MAX_FRACTION_SECONDS, - true) - .toFormatter(); - } - /** * Constructor with datetime string as input. */ public ExprDatetimeValue(String datetime) { - try { - this.datetime = LocalDateTime.parse(datetime, FORMATTER_VARIABLE_MICROS); - } catch (DateTimeParseException e) { - throw new SemanticCheckException(String.format("datetime:%s in unsupported format, please " - + "use yyyy-MM-dd HH:mm:ss[.SSSSSS]", datetime)); + LocalDateTime localDateTime = null; + + for (DateTimeFormatter format : DATE_FORMATS_ALLOWED) { + try { + localDateTime = LocalDateTime.parse(datetime, format); + } catch (DateTimeParseException ignored) { + // ignored + } + } + if (localDateTime == null) { + throw new SemanticCheckException(String.format( + "datetime:%s in unsupported format, please " + "use yyyy-MM-dd HH:mm:ss[.SSSSSS]", + datetime)); } + this.datetime = localDateTime; } @Override diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java index 0c9c506569..870e8d1e58 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java @@ -30,6 +30,8 @@ import java.time.format.DateTimeParseException; import java.time.temporal.ChronoField; import java.time.temporal.ChronoUnit; +import java.util.Arrays; +import java.util.List; import java.util.Objects; import lombok.RequiredArgsConstructor; @@ -50,9 +52,13 @@ public class ExprTimestampValue extends AbstractExprValue { private final Instant timestamp; private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS; + private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS_WITH_T; + private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS_WITH_T_Z; + public static final List DATE_FORMATS_ALLOWED; private static final int MIN_FRACTION_SECONDS = 0; private static final int MAX_FRACTION_SECONDS = 6; + static { FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder() .appendPattern("yyyy-MM-dd HH:mm:ss") @@ -62,21 +68,46 @@ public class ExprTimestampValue extends AbstractExprValue { MAX_FRACTION_SECONDS, true) .toFormatter(); + FORMATTER_VARIABLE_MICROS_WITH_T = new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd'T'HH:mm:ss") + .appendFraction( + ChronoField.MICRO_OF_SECOND, + MIN_FRACTION_SECONDS, + MAX_FRACTION_SECONDS, + true) + .toFormatter(); + + FORMATTER_VARIABLE_MICROS_WITH_T_Z = new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd'T'HH:mm:ss'Z'") + .appendFraction( + ChronoField.MICRO_OF_SECOND, + MIN_FRACTION_SECONDS, + MAX_FRACTION_SECONDS, + true) + .toFormatter(); + DATE_FORMATS_ALLOWED = Arrays.asList( + FORMATTER_VARIABLE_MICROS, + FORMATTER_VARIABLE_MICROS_WITH_T, + FORMATTER_VARIABLE_MICROS_WITH_T_Z); } /** * Constructor. */ public ExprTimestampValue(String timestamp) { - try { - this.timestamp = LocalDateTime.parse(timestamp, FORMATTER_VARIABLE_MICROS) - .atZone(ZONE) - .toInstant(); - } catch (DateTimeParseException e) { - throw new SemanticCheckException(String.format("timestamp:%s in unsupported format, please " + Instant localTimestamp = null; + for (DateTimeFormatter format : DATE_FORMATS_ALLOWED) { + try { + localTimestamp = LocalDateTime.parse(timestamp, format).atZone(ZONE).toInstant(); + } catch (DateTimeParseException ignored) { + // ignored + } + } + if (localTimestamp == null) { + throw new SemanticCheckException(String.format("timestamp:%s in unsupported format, please " + "use yyyy-MM-dd HH:mm:ss[.SSSSSS]", timestamp)); } - + this.timestamp = localTimestamp; } @Override diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java index 294c21ec52..14dd2a1f77 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java @@ -113,9 +113,9 @@ public void timeInUnsupportedFormat() { public void timestampInUnsupportedFormat() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprTimestampValue("2020-07-07T01:01:01Z")); + () -> new ExprTimestampValue("2020-07-07 01:01:01+10")); assertEquals( - "timestamp:2020-07-07T01:01:01Z in unsupported format, " + "timestamp:2020-07-07 01:01:01+10 in unsupported format, " + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", exception.getMessage()); } @@ -124,9 +124,9 @@ public void timestampInUnsupportedFormat() { public void datetimeInUnsupportedFormat() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprDatetimeValue("2020-07-07T01:01:01Z")); + () -> new ExprDatetimeValue("2020-07-07 01:01:01+10")); assertEquals( - "datetime:2020-07-07T01:01:01Z in unsupported format, " + "datetime:2020-07-07 01:01:01+10 in unsupported format, " + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", exception.getMessage()); } @@ -140,11 +140,17 @@ public void stringDateTimeValue() { assertEquals(LocalTime.parse("19:44:00"), stringValue.timeValue()); assertEquals("\"2020-08-17 19:44:00\"", stringValue.toString()); + ExprValue stringValueTZ = new ExprStringValue("2020-08-17T19:44:00Z"); + assertEquals(LocalDateTime.parse("2020-08-17T19:44:00"), stringValueTZ.datetimeValue()); + assertEquals(LocalDate.parse("2020-08-17"), stringValueTZ.dateValue()); + assertEquals(LocalTime.parse("19:44:00"), stringValueTZ.timeValue()); + assertEquals("\"2020-08-17T19:44:00Z\"", stringValueTZ.toString()); + SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprStringValue("2020-07-07T01:01:01Z").datetimeValue()); + () -> new ExprStringValue("2020-07-07 01:01:01+10").datetimeValue()); assertEquals( - "datetime:2020-07-07T01:01:01Z in unsupported format, " + "datetime:2020-07-07 01:01:01+10 in unsupported format, " + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", exception.getMessage()); } @@ -211,6 +217,9 @@ public void timestampWithVariableMicroPrecision() { assertEquals(LocalTime.parse(timeWithMicros), timestampValue.timeValue()); String localDateTime = String.format("%sT%s", dateValue, timeWithMicros); assertEquals(LocalDateTime.parse(localDateTime), timestampValue.datetimeValue()); + + ExprValue timestampValueWithT = new ExprTimestampValue(timestampString); + assertEquals(LocalDateTime.parse(localDateTime), timestampValueWithT.datetimeValue()); } } @@ -232,6 +241,9 @@ public void datetimeWithVariableMicroPrecision() { assertEquals(LocalTime.parse(timeWithMicros), datetimeValue.timeValue()); String localDateTime = String.format("%sT%s", dateValue, timeWithMicros); assertEquals(LocalDateTime.parse(localDateTime), datetimeValue.datetimeValue()); + + ExprValue datetimeValueWithT = new ExprDatetimeValue(localDateTime); + assertEquals(LocalDateTime.parse(localDateTime), datetimeValueWithT.datetimeValue()); } } diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilderTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilderTest.java index 7a0a81bee7..a77aa2ff26 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilderTest.java @@ -30,26 +30,16 @@ import static org.mockito.Mockito.doAnswer; import com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils; -import com.amazon.opendistroforelasticsearch.sql.data.model.ExprByteValue; -import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDoubleValue; -import com.amazon.opendistroforelasticsearch.sql.data.model.ExprFloatValue; -import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue; -import com.amazon.opendistroforelasticsearch.sql.data.model.ExprLongValue; -import com.amazon.opendistroforelasticsearch.sql.data.model.ExprShortValue; -import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.serialization.ExpressionSerializer; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; import com.amazon.opendistroforelasticsearch.sql.expression.config.ExpressionConfig; -import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.stream.Stream; import org.json.JSONObject; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayNameGeneration; @@ -59,7 +49,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; From 4910e2d06362f45e097dcb343339c605588c7ea7 Mon Sep 17 00:00:00 2001 From: Francesco Capponi Date: Tue, 1 Jun 2021 01:08:54 -0400 Subject: [PATCH 06/37] Date Autocast for new engine --- .../storage/script/filter/FilterQueryBuilderTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilderTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilderTest.java index 4ed7def11e..6327af11c4 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilderTest.java @@ -16,9 +16,9 @@ package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.fromObjectValue; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATE; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATETIME; -import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.fromObjectValue; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIME; From 76248c78d744ca97b39a23ac3247e89c94e98987 Mon Sep 17 00:00:00 2001 From: Francesco Capponi Date: Mon, 7 Jun 2021 10:55:31 -0400 Subject: [PATCH 07/37] Adding field comparison in Legacy Engine --- .../sql/legacy/MethodQueryIT.java | 27 +++++++++++++++++ .../sql/legacy/domain/Where.java | 20 ++++++++++++- .../sql/legacy/parser/SqlParser.java | 2 +- .../sql/legacy/parser/WhereParser.java | 30 +++++++++++++++---- 4 files changed, 71 insertions(+), 8 deletions(-) diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java index 862363df8c..77c564337c 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.Locale; import org.junit.Assert; +import org.junit.Assume; import org.junit.Test; /** @@ -144,4 +145,30 @@ public void matchPhraseQueryTest() throws IOException { Assert.assertThat(result, containsString("{\"match_phrase\":{\"address\":{\"query\":\"671 Bristol Street\"")); } + + @Test + public void testFieldToFieldComparison() throws IOException { + Assume.assumeFalse(isNewQueryEngineEabled()); + String result = explainQuery("select * from " + TestsConstants.TEST_INDEX_ACCOUNT + + " where age > account_number"); + + Assert.assertThat(result, + containsString("{\"script\":{\"script\":{\"source\":\"doc['age'].value " + + "> doc['account_number'].value\",\"lang\":\"painless\"}")); + + } + + @Test + public void testFieldToFieldComparisonWithExtraClause() throws IOException { + Assume.assumeFalse(isNewQueryEngineEabled()); + String result = explainQuery("select * from " + TestsConstants.TEST_INDEX_ACCOUNT + + " where age > account_number AND age in (1)"); + + Assert.assertThat(result, + containsString("{\"script\":{\"script\":{\"source\":\"doc['age'].value " + + "> doc['account_number'].value\",\"lang\":\"painless\"}")); + Assert.assertThat(result, + containsString("{\"term\":{\"age\":{\"value\":1,\"boost\":1.0}}}")); + + } } diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Where.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Where.java index 8b7f436f76..d929ba8280 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Where.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Where.java @@ -31,16 +31,34 @@ public static Where newInstance() { return new Where(CONN.AND); } + public static Where newInstance(boolean isJoin) { + return new Where(CONN.AND, isJoin); + } + private LinkedList wheres = new LinkedList<>(); protected CONN conn; + protected final boolean isJoin; public Where(String connStr) { - this.conn = CONN.valueOf(connStr.toUpperCase()); + this(connStr, false); + } + + public Where(String connStr, boolean isJoin) { + this(CONN.valueOf(connStr.toUpperCase()), isJoin); } public Where(CONN conn) { + this(conn, false); + } + + public Where(CONN conn, boolean isJoin) { this.conn = conn; + this.isJoin=isJoin; + } + + public boolean isJoin() { + return isJoin; } public void addWhere(Where where) { diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SqlParser.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SqlParser.java index d296f66b0c..44ecc36daf 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SqlParser.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SqlParser.java @@ -448,7 +448,7 @@ private JoinSelect createBasicJoinSelectAccordingToTableSource(SQLJoinTableSourc throws SqlParseException { JoinSelect joinSelect = new JoinSelect(); if (joinTableSource.getCondition() != null) { - Where where = Where.newInstance(); + Where where = Where.newInstance(true); WhereParser whereParser = new WhereParser(this, joinTableSource.getCondition()); whereParser.parseWhere(joinTableSource.getCondition(), where); joinSelect.setConnectedWhere(where); diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/WhereParser.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/WhereParser.java index 73814fff0e..8e67aff653 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/WhereParser.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/WhereParser.java @@ -214,7 +214,7 @@ private void routeCond(SQLBinaryOpExpr bExpr, SQLExpr sub, Where where) throws S if (sub instanceof SQLBinaryOpExpr && !isCond((SQLBinaryOpExpr) sub)) { SQLBinaryOpExpr binarySub = (SQLBinaryOpExpr) sub; if (binarySub.getOperator().priority != bExpr.getOperator().priority) { - Where subWhere = new Where(bExpr.getOperator().name); + Where subWhere = new Where(bExpr.getOperator().name, where.isJoin()); where.addWhere(subWhere); parseWhere(binarySub, subWhere); } else { @@ -291,7 +291,7 @@ private void explainCond(String opear, SQLExpr expr, Where where) throws SqlPars condition = new Condition(Where.CONN.valueOf(opear), soExpr.getLeft().toString(), soExpr.getLeft(), soExpr.getOperator().name, parseValue(soExpr.getRight()), soExpr.getRight(), childrenType); } else { - SQLMethodInvokeExpr sqlMethodInvokeExpr = parseSQLBinaryOpExprWhoIsConditionInWhere(soExpr); + SQLMethodInvokeExpr sqlMethodInvokeExpr = parseSQLBinaryOpExprWhoIsConditionInWhere(soExpr, where.isJoin()); if (sqlMethodInvokeExpr == null) { condition = new Condition(Where.CONN.valueOf(opear), soExpr.getLeft().toString(), soExpr.getLeft(), soExpr.getOperator().name, parseValue(soExpr.getRight()), @@ -536,10 +536,20 @@ private MethodField parseSQLCastExprWithFunctionInWhere(SQLCastExpr soExpr) thro ); } - private SQLMethodInvokeExpr parseSQLBinaryOpExprWhoIsConditionInWhere(SQLBinaryOpExpr soExpr) + private SQLMethodInvokeExpr parseSQLBinaryOpExprWhoIsConditionInWhere(SQLBinaryOpExpr soExpr, + boolean join) throws SqlParseException { - if (bothSideAreNotFunction(soExpr) && bothSidesAreNotCast(soExpr)) { + if ( + // if one of the two side is a method, it has to be resolved as script + neitherSideIsFunction(soExpr) + // if one of the two side is a cast, it has to be resolved as script + && neitherSidesIsCast(soExpr) + // if both sides are field, we cannot use the QueryRange (which would be more efficient + // `field > integer` comparisons) but ES doesn't allow comparing two fields and + // we must use a SCRIPT + && (!bothSidesAreFieldIdentifier(soExpr) || join) + ) { return null; } @@ -608,14 +618,22 @@ private SQLMethodInvokeExpr parseSQLBinaryOpExprWhoIsConditionInWhere(SQLBinaryO } - private Boolean bothSideAreNotFunction(SQLBinaryOpExpr soExpr) { + private Boolean neitherSideIsFunction(SQLBinaryOpExpr soExpr) { return !(soExpr.getLeft() instanceof SQLMethodInvokeExpr || soExpr.getRight() instanceof SQLMethodInvokeExpr); } - private Boolean bothSidesAreNotCast(SQLBinaryOpExpr soExpr) { + private Boolean neitherSidesIsCast(SQLBinaryOpExpr soExpr) { return !(soExpr.getLeft() instanceof SQLCastExpr || soExpr.getRight() instanceof SQLCastExpr); } + private Boolean bothSidesAreFieldIdentifier(SQLBinaryOpExpr soExpr) { + return (soExpr.getLeft() instanceof SQLIdentifierExpr && soExpr.getRight() instanceof SQLIdentifierExpr) + // "missing" is a SQLIdentifier but not a Field Identifier. + // We might want to have a subclass for "missing" or for "fieldIdentifier" or + // change type altogether to avoid conflicts + && !"missing".equalsIgnoreCase(((SQLIdentifierExpr) soExpr.getRight()).getLowerName()); + } + private Object[] getMethodValuesWithSubQueries(SQLMethodInvokeExpr method) throws SqlParseException { List values = new ArrayList<>(); for (SQLExpr innerExpr : method.getParameters()) { From 3a934787f08f3a9bd080e6aeeb0fee09fae45f0a Mon Sep 17 00:00:00 2001 From: Francesco Capponi Date: Fri, 11 Jun 2021 23:49:10 -0400 Subject: [PATCH 08/37] Adding support for much more permissive date parsing for ExprTimestampValue creation (support of T or not T, autofill missing part with 0s) --- .../sql/data/model/ExprTimestampValue.java | 4 ++- .../sql/data/utils/ExprDateFormatters.java | 25 +++++++++++++++++-- .../sql/data/model/DateTimeValueTest.java | 8 +++--- .../value/ElasticsearchExprValueFactory.java | 16 +++--------- .../ElasticsearchExprValueFactoryTest.java | 11 ++++++-- 5 files changed, 42 insertions(+), 22 deletions(-) rename elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchDateFormatters.java => core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprDateFormatters.java (84%) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java index 0c9c506569..894a3e82ca 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java @@ -19,6 +19,7 @@ import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.data.utils.ExprDateFormatters; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; import java.time.Instant; import java.time.LocalDate; @@ -69,7 +70,8 @@ public class ExprTimestampValue extends AbstractExprValue { */ public ExprTimestampValue(String timestamp) { try { - this.timestamp = LocalDateTime.parse(timestamp, FORMATTER_VARIABLE_MICROS) + this.timestamp = LocalDateTime.parse(timestamp, + ExprDateFormatters.TOLERANT_PARSER_DATE_TIME_FORMATTER) .atZone(ZONE) .toInstant(); } catch (DateTimeParseException e) { diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchDateFormatters.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprDateFormatters.java similarity index 84% rename from elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchDateFormatters.java rename to core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprDateFormatters.java index c653c063ac..818eb71375 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchDateFormatters.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprDateFormatters.java @@ -15,7 +15,7 @@ * */ -package com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value; +package com.amazon.opendistroforelasticsearch.sql.data.utils; import static java.time.temporal.ChronoField.DAY_OF_MONTH; import static java.time.temporal.ChronoField.HOUR_OF_DAY; @@ -37,7 +37,7 @@ * Reference org.elasticsearch.common.time.DateFormatters. */ @UtilityClass -public class ElasticsearchDateFormatters { +public class ExprDateFormatters { public static final DateTimeFormatter TIME_ZONE_FORMATTER_NO_COLON = new DateTimeFormatterBuilder() @@ -73,7 +73,16 @@ public class ElasticsearchDateFormatters { new DateTimeFormatterBuilder() .append(STRICT_YEAR_MONTH_DAY_FORMATTER) .optionalStart() + + // otherwise use space + .optionalStart() + .appendLiteral(' ') + .optionalEnd() + // optional T + .optionalStart() .appendLiteral('T') + .optionalEnd() + .optionalStart() .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE) .optionalStart() @@ -82,13 +91,18 @@ public class ElasticsearchDateFormatters { .optionalStart() .appendLiteral(':') .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) + + // optional millis with dot .optionalStart() .appendFraction(NANO_OF_SECOND, 1, 9, true) .optionalEnd() + + // otherwise optional millis use with comma .optionalStart() .appendLiteral(',') .appendFraction(NANO_OF_SECOND, 1, 9, false) .optionalEnd() + .optionalEnd() .optionalEnd() .optionalStart() @@ -104,4 +118,11 @@ public class ElasticsearchDateFormatters { public static final DateTimeFormatter SQL_LITERAL_DATE_TIME_FORMAT = DateTimeFormatter .ofPattern("yyyy-MM-dd HH:mm:ss"); + + public static final DateTimeFormatter TOLERANT_PARSER_DATE_TIME_FORMATTER = + new DateTimeFormatterBuilder() + .appendOptional(STRICT_DATE_OPTIONAL_TIME_FORMATTER) + .appendOptional(SQL_LITERAL_DATE_TIME_FORMAT) + .appendOptional(STRICT_HOUR_MINUTE_SECOND_FORMATTER) + .toFormatter(); } diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java index 294c21ec52..071258ee21 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java @@ -113,9 +113,9 @@ public void timeInUnsupportedFormat() { public void timestampInUnsupportedFormat() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprTimestampValue("2020-07-07T01:01:01Z")); + () -> new ExprTimestampValue("2020-07-07T1:01:01Z")); assertEquals( - "timestamp:2020-07-07T01:01:01Z in unsupported format, " + "timestamp:2020-07-07T1:01:01Z in unsupported format, " + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", exception.getMessage()); } @@ -239,9 +239,9 @@ public void datetimeWithVariableMicroPrecision() { public void timestampOverMaxMicroPrecision() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprTimestampValue("2020-07-07 01:01:01.1234567")); + () -> new ExprTimestampValue("2020-07-07 01:01:01.12345678910")); assertEquals( - "timestamp:2020-07-07 01:01:01.1234567 in unsupported format, " + "timestamp:2020-07-07 01:01:01.12345678910 in unsupported format, " + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", exception.getMessage()); } diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java index 2fbae486d6..b0f768ce94 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java @@ -36,9 +36,6 @@ import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_IP; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT_KEYWORD; -import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchDateFormatters.SQL_LITERAL_DATE_TIME_FORMAT; -import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchDateFormatters.STRICT_DATE_OPTIONAL_TIME_FORMATTER; -import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchDateFormatters.STRICT_HOUR_MINUTE_SECOND_FORMATTER; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprByteValue; @@ -57,6 +54,7 @@ import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.data.utils.ExprDateFormatters; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.utils.Content; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.utils.ElasticsearchJsonContent; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.utils.ObjectContent; @@ -64,8 +62,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import java.time.Instant; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -87,13 +83,6 @@ public class ElasticsearchExprValueFactory { @Setter private Map typeMapping; - private static final DateTimeFormatter DATE_TIME_FORMATTER = - new DateTimeFormatterBuilder() - .appendOptional(SQL_LITERAL_DATE_TIME_FORMAT) - .appendOptional(STRICT_DATE_OPTIONAL_TIME_FORMATTER) - .appendOptional(STRICT_HOUR_MINUTE_SECOND_FORMATTER) - .toFormatter(); - private static final String TOP_PATH = ""; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -185,7 +174,8 @@ private ExprValue constructTimestamp(String value) { try { return new ExprTimestampValue( // Using Elasticsearch DateFormatters for now. - DateFormatters.from(DATE_TIME_FORMATTER.parse(value)).toInstant()); + DateFormatters.from(ExprDateFormatters.TOLERANT_PARSER_DATE_TIME_FORMATTER + .parse(value)).toInstant()); } catch (DateTimeParseException e) { throw new IllegalStateException( String.format( diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java index badf6afb57..a40921a64c 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java @@ -190,6 +190,12 @@ public void constructDate() { assertEquals( new ExprTimestampValue("2015-01-01 12:10:30"), tupleValue("{\"timestampV\":\"2015-01-01 12:10:30\"}").get("timestampV")); + assertEquals( + new ExprTimestampValue("2020-08-17T19:44:00.100500"), + tupleValue("{\"timestampV\":\"2020-08-17T19:44:00.100500\"}").get("timestampV")); + assertEquals( + new ExprTimestampValue("2020-08-17 19:44:00.100500"), + tupleValue("{\"timestampV\":\"2020-08-17 19:44:00.100500\"}").get("timestampV")); assertEquals( new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), tupleValue("{\"timestampV\":1420070400001}").get("timestampV")); @@ -221,9 +227,10 @@ public void constructDate() { public void constructDateFromUnsupportedFormatThrowException() { IllegalStateException exception = assertThrows( - IllegalStateException.class, () -> tupleValue("{\"timestampV\":\"2015-01-01 12:10\"}")); + IllegalStateException.class, () -> + tupleValue("{\"timestampV\":\"2015-01-01 1:10:10\"}")); assertEquals( - "Construct ExprTimestampValue from \"2015-01-01 12:10\" failed, " + "Construct ExprTimestampValue from \"2015-01-01 1:10:10\" failed, " + "unsupported date format.", exception.getMessage()); } From 0284f03be6c1cc8c7aae04682878ffb57929a743 Mon Sep 17 00:00:00 2001 From: Francesco Capponi Date: Sat, 19 Jun 2021 16:19:58 -0400 Subject: [PATCH 09/37] Legacy: field comparison, adding support to <> operator (behave as !=) --- .../sql/legacy/MethodQueryIT.java | 14 ++++++++++++++ .../sql/legacy/parser/WhereParser.java | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java index 77c564337c..b8a4de1756 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java @@ -171,4 +171,18 @@ public void testFieldToFieldComparisonWithExtraClause() throws IOException { containsString("{\"term\":{\"age\":{\"value\":1,\"boost\":1.0}}}")); } + + @Test + public void testFieldToFieldComparisonWithDifferentOperator() throws IOException { + Assume.assumeFalse(isNewQueryEngineEabled()); + String result = explainQuery("select * from " + TestsConstants.TEST_INDEX_ACCOUNT + + " where age <> account_number AND age in (1)"); + + Assert.assertThat(result, + containsString("{\"script\":{\"script\":{\"source\":\"doc['age'].value " + + "!= doc['account_number'].value\",\"lang\":\"painless\"}")); + Assert.assertThat(result, + containsString("{\"term\":{\"age\":{\"value\":1,\"boost\":1.0}}}")); + + } } diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/WhereParser.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/WhereParser.java index 8e67aff653..f7ef5432ac 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/WhereParser.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/WhereParser.java @@ -610,6 +610,10 @@ && neitherSidesIsCast(soExpr) operator = "=="; } + if (operator.equals("<>")) { + operator = "!="; + } + String finalStr = v1Dec + v2Dec + v1 + " " + operator + " " + v2; SQLMethodInvokeExpr scriptMethod = new SQLMethodInvokeExpr("script", null); From 19bedb26525eacaf1a22ec577dd77f46b82ecd5a Mon Sep 17 00:00:00 2001 From: Francesco Capponi Date: Wed, 30 Jun 2021 20:08:44 -0400 Subject: [PATCH 10/37] Fixing date after merge --- .../sql/data/model/ExprDatetimeValue.java | 25 ++++++------------- .../sql/data/model/DateTimeValueTest.java | 23 +++++------------ 2 files changed, 13 insertions(+), 35 deletions(-) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDatetimeValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDatetimeValue.java index 1f4550efa1..509e835242 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDatetimeValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDatetimeValue.java @@ -15,10 +15,9 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; -import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue.DATE_FORMATS_ALLOWED; - import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.data.utils.ExprDateFormatters; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; import com.google.common.base.Objects; import java.time.Instant; @@ -28,9 +27,7 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; -import java.time.temporal.ChronoField; import java.time.temporal.ChronoUnit; import lombok.RequiredArgsConstructor; @@ -42,21 +39,13 @@ public class ExprDatetimeValue extends AbstractExprValue { * Constructor with datetime string as input. */ public ExprDatetimeValue(String datetime) { - LocalDateTime localDateTime = null; - - for (DateTimeFormatter format : DATE_FORMATS_ALLOWED) { - try { - localDateTime = LocalDateTime.parse(datetime, format); - } catch (DateTimeParseException ignored) { - // ignored - } - } - if (localDateTime == null) { - throw new SemanticCheckException(String.format( - "datetime:%s in unsupported format, please " + "use yyyy-MM-dd HH:mm:ss[.SSSSSS]", - datetime)); + try { + this.datetime = LocalDateTime.parse(datetime, + ExprDateFormatters.TOLERANT_PARSER_DATE_TIME_FORMATTER); + } catch (DateTimeParseException e) { + throw new SemanticCheckException(String.format("datetime:%s in unsupported format, please " + + "use yyyy-MM-dd HH:mm:ss[.SSSSSS]", datetime)); } - this.datetime = localDateTime; } @Override diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java index 071258ee21..dd136b8c76 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java @@ -120,17 +120,6 @@ public void timestampInUnsupportedFormat() { exception.getMessage()); } - @Test - public void datetimeInUnsupportedFormat() { - SemanticCheckException exception = - assertThrows(SemanticCheckException.class, - () -> new ExprDatetimeValue("2020-07-07T01:01:01Z")); - assertEquals( - "datetime:2020-07-07T01:01:01Z in unsupported format, " - + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", - exception.getMessage()); - } - @Test public void stringDateTimeValue() { ExprValue stringValue = new ExprStringValue("2020-08-17 19:44:00"); @@ -142,9 +131,9 @@ public void stringDateTimeValue() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprStringValue("2020-07-07T01:01:01Z").datetimeValue()); + () -> new ExprStringValue("2020-07-07T01:01:01Z12345678").datetimeValue()); assertEquals( - "datetime:2020-07-07T01:01:01Z in unsupported format, " + "datetime:2020-07-07T01:01:01Z12345678 in unsupported format, " + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", exception.getMessage()); } @@ -250,9 +239,9 @@ public void timestampOverMaxMicroPrecision() { public void datetimeOverMaxMicroPrecision() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprDatetimeValue("2020-07-07 01:01:01.1234567")); + () -> new ExprDatetimeValue("2020-07-07 01:01:01.1234567890")); assertEquals( - "datetime:2020-07-07 01:01:01.1234567 in unsupported format, " + "datetime:2020-07-07 01:01:01.1234567890 in unsupported format, " + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", exception.getMessage()); } @@ -261,9 +250,9 @@ public void datetimeOverMaxMicroPrecision() { public void timeOverMaxMicroPrecision() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprTimeValue("01:01:01.1234567")); + () -> new ExprTimeValue("01:01:01.1234567891")); assertEquals( - "time:01:01:01.1234567 in unsupported format, please use HH:mm:ss[.SSSSSS]", + "time:01:01:01.1234567891 in unsupported format, please use HH:mm:ss[.SSSSSS]", exception.getMessage()); } } From e0928c464922a1191719c0c8282c96f05939e048 Mon Sep 17 00:00:00 2001 From: Francesco Capponi Date: Thu, 1 Jul 2021 19:32:12 -0400 Subject: [PATCH 11/37] Revert "In support (#3)" This reverts commit 4a5f7124 --- .../sql/ast/dsl/AstDSL.java | 4 -- .../sql/ast/expression/DataType.java | 1 - .../sql/ast/expression/In.java | 3 - .../sql/data/model/ExprValueUtils.java | 6 +- .../sql/expression/DSL.java | 21 +----- .../function/BuiltinFunctionName.java | 1 - .../predicate/BinaryPredicateOperator.java | 28 -------- .../sql/utils/OperatorUtils.java | 22 ------ .../sql/data/model/ExprValueUtilsTest.java | 15 +--- .../BinaryPredicateOperatorTest.java | 70 +----------------- .../script/filter/FilterQueryBuilder.java | 2 - .../script/filter/lucene/TermsQuery.java | 39 ---------- .../script/filter/FilterQueryBuilderTest.java | 38 ---------- .../sql/legacy/CsvFormatResponseIT.java | 6 -- .../sql/sql/InIT.java | 71 ------------------- sql/src/main/antlr/OpenDistroSQLParser.g4 | 9 --- .../sql/sql/parser/AstExpressionBuilder.java | 17 ----- .../sql/sql/antlr/SQLSyntaxParserTest.java | 7 -- .../sql/parser/AstExpressionBuilderTest.java | 17 ----- 19 files changed, 7 insertions(+), 370 deletions(-) delete mode 100644 elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/lucene/TermsQuery.java delete mode 100644 integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/InIT.java diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java index 2c756faec1..8c15f71bd9 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java @@ -161,10 +161,6 @@ public static Literal timestampLiteral(String value) { return literal(value, DataType.TIMESTAMP); } - public static Literal arrayLiteral(List value) { - return literal(value, DataType.ARRAY); - } - public static Literal doubleLiteral(Double value) { return literal(value, DataType.DOUBLE); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/DataType.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/DataType.java index 5b68d15caa..13e6b422bd 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/DataType.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/DataType.java @@ -37,7 +37,6 @@ public enum DataType { DATE(ExprCoreType.DATE), TIME(ExprCoreType.TIME), TIMESTAMP(ExprCoreType.TIMESTAMP), - ARRAY(ExprCoreType.ARRAY), INTERVAL(ExprCoreType.INTERVAL); @Getter diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/In.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/In.java index 945bce841d..365787780b 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/In.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/In.java @@ -28,10 +28,7 @@ * Params include the field expression and/or wildcard field expression, * nested field expression (@field). * And the values that the field is mapped to (@valueList). - * - * @deprecated use function ("in") instead */ -@Deprecated @Getter @ToString @EqualsAndHashCode(callSuper = false) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java index 50de610b28..976f2de8bb 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java @@ -88,7 +88,7 @@ public static ExprValue tupleValue(Map map) { /** * {@link ExprCollectionValue} constructor. */ - public static ExprValue collectionValue(List list) { + public static ExprValue collectionValue(List list) { List valueList = new ArrayList<>(); list.forEach(o -> valueList.add(fromObjectValue(o))); return new ExprCollectionValue(valueList); @@ -129,10 +129,6 @@ public static ExprValue fromObjectValue(Object o) { return stringValue((String) o); } else if (o instanceof Float) { return floatValue((Float) o); - } else if (o instanceof ExprValue) { - // since there is no primitive in Java for differentiating TIMESTAMP DATETIME and DATE - // we can allow passing a ExprValue that already contains this information - return (ExprValue) o; } else { throw new ExpressionEvaluationException("unsupported object " + o.getClass()); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java index 467499cdd7..0d03ddc536 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java @@ -26,10 +26,8 @@ import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; import com.amazon.opendistroforelasticsearch.sql.expression.window.ranking.RankingWindowFunction; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.List; import lombok.RequiredArgsConstructor; @RequiredArgsConstructor @@ -254,7 +252,7 @@ public FunctionExpression subtract(Expression... expressions) { public FunctionExpression multiply(Expression... expressions) { return function(BuiltinFunctionName.MULTIPLY, expressions); } - + public FunctionExpression adddate(Expression... expressions) { return function(BuiltinFunctionName.ADDDATE, expressions); } @@ -366,7 +364,7 @@ public FunctionExpression module(Expression... expressions) { public FunctionExpression substr(Expression... expressions) { return function(BuiltinFunctionName.SUBSTR, expressions); } - + public FunctionExpression substring(Expression... expressions) { return function(BuiltinFunctionName.SUBSTR, expressions); } @@ -590,19 +588,4 @@ public FunctionExpression castTimestamp(Expression value) { return (FunctionExpression) repository .compile(BuiltinFunctionName.CAST_TO_TIMESTAMP.getName(), Arrays.asList(value)); } - - /** - * Check that a field is contained in a set of values. - */ - public FunctionExpression in(Expression field, Expression... expressions) { - List where = new ArrayList<>(); - where.add(field); - where.addAll(Arrays.asList(expressions)); - - return function(BuiltinFunctionName.IN, where.toArray(new Expression[0])); - } - - public FunctionExpression not_in(Expression field, Expression... expressions) { - return not(in(field, expressions)); - } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java index 1e4c245b52..6b29c68da1 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java @@ -106,7 +106,6 @@ public enum BuiltinFunctionName { GTE(FunctionName.of(">=")), LIKE(FunctionName.of("like")), NOT_LIKE(FunctionName.of("not like")), - IN(FunctionName.of("in")), /** * Aggregation Function. diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java index d926a8132d..b60179f6fc 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java @@ -19,16 +19,9 @@ import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; -import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.ARRAY; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; -import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATE; -import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATETIME; -import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; -import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; -import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; -import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; @@ -80,7 +73,6 @@ public static void register(BuiltinFunctionRepository repository) { repository.register(like()); repository.register(notLike()); repository.register(regexp()); - repository.register(in()); } /** @@ -271,26 +263,6 @@ private static FunctionResolver notLike() { STRING)); } - private static FunctionResolver in() { - return FunctionDSL.define(BuiltinFunctionName.IN.getName(), - FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::in), - BOOLEAN, INTEGER, ARRAY), - FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::in), - BOOLEAN, STRING, ARRAY), - FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::in), - BOOLEAN, LONG, ARRAY), - FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::in), - BOOLEAN, FLOAT, ARRAY), - FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::in), - BOOLEAN, DOUBLE, ARRAY), - FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::in), - BOOLEAN, DATE, ARRAY), - FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::in), - BOOLEAN, DATETIME, ARRAY), - FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::in), - BOOLEAN, TIMESTAMP, ARRAY)); - } - private static ExprValue lookupTableFunction(ExprValue arg1, ExprValue arg2, Table table) { if (table.contains(arg1, arg2)) { diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/OperatorUtils.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/OperatorUtils.java index fa48fb6a57..d887d5c391 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/OperatorUtils.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/OperatorUtils.java @@ -15,10 +15,8 @@ package com.amazon.opendistroforelasticsearch.sql.utils; -import com.amazon.opendistroforelasticsearch.sql.data.model.AbstractExprNumberValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue; -import com.amazon.opendistroforelasticsearch.sql.data.model.ExprStringValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; import java.util.regex.Pattern; import lombok.experimental.UtilityClass; @@ -101,24 +99,4 @@ private static String patternToRegex(String patternString) { regex.append('$'); return regex.toString(); } - - - /** - * IN (..., ...) operator util. - * Expression { expr IN (collection of values..) } is to judge - * if expr is contained in a given collection. - */ - public static ExprBooleanValue in(ExprValue expr, ExprValue setOfValues) { - return ExprBooleanValue.of(isIn(expr, setOfValues)); - } - - private static boolean isIn(ExprValue expr, ExprValue setOfValues) { - if (expr instanceof AbstractExprNumberValue) { - return setOfValues.collectionValue().contains(expr.value()); - } else if (expr instanceof ExprStringValue) { - return setOfValues.collectionValue().contains(expr.stringValue()); - } else { - return setOfValues.collectionValue().contains(expr); - } - } } diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java index 66cf4fc9d2..4c97632958 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java @@ -215,24 +215,13 @@ public void invalidConvertExprValue(ExprValue value, Function assertThat(exception.getMessage(), Matchers.containsString("invalid")); } - // disabling test because in case of expr collections, we could pass ExprValues - // @Test - // public void unSupportedObject() { - // Exception exception = assertThrows(ExpressionEvaluationException.class, - // () -> ExprValueUtils.fromObjectValue(integerValue(1))); - // assertEquals( - // "unsupported object " - // + "class com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue", - // exception.getMessage()); - // } - @Test public void unSupportedObject() { Exception exception = assertThrows(ExpressionEvaluationException.class, - () -> ExprValueUtils.fromObjectValue(new Object())); + () -> ExprValueUtils.fromObjectValue(integerValue(1))); assertEquals( "unsupported object " - + "class java.lang.Object", + + "class com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue", exception.getMessage()); } diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java index 42257e3670..42309faf0c 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java @@ -27,17 +27,14 @@ import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.booleanValue; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.fromObjectValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.missingValue; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; -import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATE; -import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATETIME; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; -import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP; import static com.amazon.opendistroforelasticsearch.sql.utils.ComparisonUtil.compare; import static com.amazon.opendistroforelasticsearch.sql.utils.OperatorUtils.matches; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; @@ -56,15 +53,14 @@ import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; -import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import com.amazon.opendistroforelasticsearch.sql.expression.ExpressionTestBase; import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; -import com.amazon.opendistroforelasticsearch.sql.utils.OperatorUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import com.sun.org.apache.xpath.internal.Arg; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.ObjectInputStream; @@ -868,66 +864,4 @@ public void compare_int_long() { FunctionExpression equal = dsl.equal(DSL.literal(1), DSL.literal(1L)); assertTrue(equal.valueOf(valueEnv()).booleanValue()); } - - private static Stream testInArguments() { - List arguments = - Arrays.asList(Arrays.asList(1, Arrays.asList(0, 2, 1, 3)), - Arrays.asList(1, Arrays.asList(2, 0)), Arrays.asList(1L, Arrays.asList(1L, 2L, 3L)), - Arrays.asList(2L, Arrays.asList(1L, 2L)), Arrays.asList(3F, Arrays.asList(1F, 2F)), - Arrays.asList(0F, Arrays.asList(1F, 2F)), Arrays.asList(1D, Arrays.asList(1D, 1D)), - Arrays.asList(1D, Arrays.asList(2D, 2D)), - Arrays.asList("b", Arrays.asList("a", "c")), - Arrays.asList("b", Arrays.asList("c", "a")), - Arrays.asList("a", Arrays.asList("a", "b")), - Arrays.asList("b", Arrays.asList("a", "b")), - Arrays.asList("c", Arrays.asList("a", "b")), - Arrays.asList("a", Arrays.asList("b", "c")), - Arrays.asList("a", Arrays.asList("a", "a")), - Arrays.asList("b", Arrays.asList("a", "a"))); - - Stream.Builder builder = Stream.builder(); - for (List argGroup : arguments) { - builder.add(Arguments.of(fromObjectValue(argGroup.get(0)), fromObjectValue(argGroup.get(1)))); - } - builder - .add(Arguments.of(fromObjectValue("2021-01-02", DATE), - fromObjectValue(Arrays.asList(fromObjectValue("2021-01-01", DATE), - fromObjectValue("2021-01-03", DATE))))) - .add(Arguments.of(fromObjectValue("2021-01-02", DATE), - fromObjectValue(Arrays.asList(fromObjectValue("2021-01-01", DATE), - fromObjectValue("2021-01-03", DATE))))) - .add(Arguments.of(fromObjectValue("2021-01-01 03:00:00", DATETIME), - fromObjectValue(Arrays.asList(fromObjectValue("2021-01-01 01:00:00", DATETIME), - fromObjectValue("3021-01-01 02:00:00", DATETIME))))) - .add(Arguments.of(fromObjectValue("2021-01-01 01:00:00", TIMESTAMP), - fromObjectValue(Arrays.asList(fromObjectValue("2021-01-01 01:00:00", TIMESTAMP), - fromObjectValue("3021-01-01 01:00:00", TIMESTAMP))))); - return builder.build(); - } - - @ParameterizedTest(name = "in({0}, ({1}))") - @MethodSource("testInArguments") - public void in(ExprValue field, ExprValue arrayOfArgs) { - FunctionExpression in = dsl.in( - DSL.literal(field), DSL.literal(arrayOfArgs)); - assertEquals(BOOLEAN, in.type()); - assertEquals(OperatorUtils.in(field, arrayOfArgs), in.valueOf(valueEnv())); - } - - @ParameterizedTest(name = "not in({0}, ({1}))") - @MethodSource("testInArguments") - public void not_in(ExprValue field, ExprValue arrayOfArgs) { - FunctionExpression notIn = dsl.not_in( - DSL.literal(field), DSL.literal(arrayOfArgs)); - assertEquals(BOOLEAN, notIn.type()); - assertEquals(!OperatorUtils.in(field, arrayOfArgs).booleanValue(), - notIn.valueOf(valueEnv()).booleanValue()); - } - - @Test - public void in_not_an_array() { - assertThrows(ExpressionEvaluationException.class, () -> - dsl.in(DSL.literal(1), DSL.literal("1"))); - } - } \ No newline at end of file diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilder.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilder.java index af2dcf81e7..4cc8be3512 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilder.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilder.java @@ -24,7 +24,6 @@ import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.lucene.RangeQuery; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.lucene.RangeQuery.Comparison; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.lucene.TermQuery; -import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.lucene.TermsQuery; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.lucene.WildcardQuery; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.serialization.ExpressionSerializer; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; @@ -64,7 +63,6 @@ public class FilterQueryBuilder extends ExpressionNodeVisitor 1)"; - JSONObject results = executeQuery(sql); - Assert.assertThat(getTotalHits(results), equalTo(1)); - } - -} diff --git a/sql/src/main/antlr/OpenDistroSQLParser.g4 b/sql/src/main/antlr/OpenDistroSQLParser.g4 index 01967fbcdc..4f01c657c9 100644 --- a/sql/src/main/antlr/OpenDistroSQLParser.g4 +++ b/sql/src/main/antlr/OpenDistroSQLParser.g4 @@ -260,7 +260,6 @@ predicate | predicate IS nullNotnull #isNullPredicate | left=predicate NOT? LIKE right=predicate #likePredicate | left=predicate REGEXP right=predicate #regexpPredicate - | predicate NOT? IN LR_BRACKET arrayArgs? RR_BRACKET #inPredicate ; expressionAtom @@ -371,11 +370,3 @@ functionArg : expression ; -arrayArgs - : arrayArg (COMMA arrayArg)* - ; - -arrayArg - : expression - ; - diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java index 2fa59cfaef..1fefc0ddfb 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java @@ -18,11 +18,9 @@ import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.qualifiedName; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.stringLiteral; -import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.IN; import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.IS_NOT_NULL; import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.IS_NULL; import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.LIKE; -import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.NOT; import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.NOT_LIKE; import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.REGEXP; import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.BinaryComparisonPredicateContext; @@ -63,10 +61,8 @@ import com.amazon.opendistroforelasticsearch.sql.ast.expression.Case; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Cast; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Function; -import com.amazon.opendistroforelasticsearch.sql.ast.expression.In; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Interval; import com.amazon.opendistroforelasticsearch.sql.ast.expression.IntervalUnit; -import com.amazon.opendistroforelasticsearch.sql.ast.expression.Literal; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Not; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Or; import com.amazon.opendistroforelasticsearch.sql.ast.expression.QualifiedName; @@ -237,19 +233,6 @@ public UnresolvedExpression visitRegexpPredicate(RegexpPredicateContext ctx) { Arrays.asList(visit(ctx.left), visit(ctx.right))); } - @Override - public UnresolvedExpression visitInPredicate(OpenDistroSQLParser.InPredicateContext ctx) { - UnresolvedExpression between = new Function(IN.getName().getFunctionName(), - Arrays.asList(visit(ctx.predicate()), AstDSL.arrayLiteral(ctx.arrayArgs().arrayArg() - .stream() - .map(this::visitArrayArg) - .map(unresolvedExpression -> ((Literal) unresolvedExpression).getValue()) - .collect(Collectors.toList())))); - - return ctx.NOT() == null ? between : - new Function(NOT.getName().getFunctionName(), Collections.singletonList(between)); - } - @Override public UnresolvedExpression visitAndExpression(AndExpressionContext ctx) { return new And(visit(ctx.left), visit(ctx.right)); diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java index d0dddf1142..0f2605d7d6 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -131,13 +131,6 @@ public void canParseCaseStatement() { assertNotNull(parser.parse("SELECT CASE age WHEN 30 THEN 'age1' END FROM test")); } - @Test - public void canParseInStatement() { - assertNotNull(parser.parse("SELECT age FROM test WHERE age IN (1,30)")); - assertNotNull(parser.parse("SELECT age FROM test WHERE age NOT IN (1,30)")); - assertNotNull(parser.parse("SELECT age FROM test WHERE NOT (age IN (1,30))")); - } - @Test public void canNotParseAggregateFunctionWithWrongArgument() { assertThrows(SyntaxCheckException.class, () -> parser.parse("SELECT SUM() FROM test")); diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java index 4047f86ce1..b1ec56ce51 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -18,13 +18,11 @@ import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.aggregate; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.and; -import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.arrayLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.booleanLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.caseWhen; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.dateLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.doubleLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.function; -import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.in; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intervalLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.longLiteral; @@ -51,7 +49,6 @@ import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLLexer; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser; import com.google.common.collect.ImmutableList; -import java.util.Arrays; import org.antlr.v4.runtime.CommonTokenStream; import org.apache.commons.lang3.tuple.ImmutablePair; import org.junit.jupiter.api.Test; @@ -286,20 +283,6 @@ public void canBuildLogicalExpression() { ); } - @Test - public void canBuildInPredicate() { - assertEquals(function("in", intLiteral(1), arrayLiteral(Arrays.asList(0,2,3,4))), - buildExprAst("1 in (0,2,3,4)")); - } - - @Test - public void canBuildNotInPredicate() { - assertEquals( - function("not", - function("in", intLiteral(1),arrayLiteral(Arrays.asList(0,2,3,4)))), - buildExprAst("1 not in (0,2,3,4)")); - } - @Test public void canBuildWindowFunction() { assertEquals( From ea833349729ff073116159ec99c66310b03c181f Mon Sep 17 00:00:00 2001 From: Francesco Capponi Date: Thu, 8 Jul 2021 12:45:58 -0400 Subject: [PATCH 12/37] Allowing customizing the number of nested results through config --- .../sql/legacy/plugin/SqlSettings.java | 2 ++ .../rewriter/nestedfield/NestedFieldProjection.java | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlSettings.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlSettings.java index 346b4b8423..fe2da083ed 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlSettings.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlSettings.java @@ -45,6 +45,7 @@ public class SqlSettings { public static final String QUERY_ANALYSIS_ENABLED = "opendistro.sql.query.analysis.enabled"; public static final String QUERY_ANALYSIS_SEMANTIC_SUGGESTION = "opendistro.sql.query.analysis.semantic.suggestion"; public static final String QUERY_ANALYSIS_SEMANTIC_THRESHOLD = "opendistro.sql.query.analysis.semantic.threshold"; + public static final String QUERY_NESTED_LIMIT = "opendistro.sql.engine.nested.limit"; public static final String METRICS_ROLLING_WINDOW = "opendistro.sql.metrics.rollingwindow"; public static final String METRICS_ROLLING_INTERVAL = "opendistro.sql.metrics.rollinginterval"; @@ -58,6 +59,7 @@ public SqlSettings() { Map> settings = new HashMap<>(); settings.put(SQL_ENABLED, Setting.boolSetting(SQL_ENABLED, true, NodeScope, Dynamic)); settings.put(SQL_NEW_ENGINE_ENABLED, Setting.boolSetting(SQL_NEW_ENGINE_ENABLED, true, NodeScope, Dynamic)); + settings.put(QUERY_NESTED_LIMIT, Setting.intSetting(QUERY_NESTED_LIMIT, 3, NodeScope, Dynamic)); settings.put(QUERY_SLOWLOG, Setting.intSetting(QUERY_SLOWLOG, 2, NodeScope, Dynamic)); settings.put(QUERY_RESPONSE_FORMAT, Setting.simpleString(QUERY_RESPONSE_FORMAT, Format.JDBC.getFormatName(), NodeScope, Dynamic)); diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/NestedFieldProjection.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/NestedFieldProjection.java index 4ae0951fd0..e82eb9619f 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/NestedFieldProjection.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/NestedFieldProjection.java @@ -16,6 +16,7 @@ package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.nestedfield; import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.matchtoterm.VerificationException; import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import org.apache.lucene.search.join.ScoreMode; @@ -34,6 +35,7 @@ import java.util.function.Supplier; import java.util.stream.Stream; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_NESTED_LIMIT; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.mapping; import static java.util.stream.Collectors.toList; @@ -143,7 +145,11 @@ private List extractNestedQueries(QueryBuilder query) { private void buildInnerHit(List fieldNames, NestedQueryBuilder query) { query.innerHit(new InnerHitBuilder().setFetchSourceContext( new FetchSourceContext(true, fieldNames.toArray(new String[0]), null) - )); + ).setSize(this.queryNestedLimit())); + } + + private int queryNestedLimit() { + return LocalClusterState.state().getSettingValue(QUERY_NESTED_LIMIT); } /** From 6114478c9cbaac920d88d1167e6d7c54d394ab09 Mon Sep 17 00:00:00 2001 From: cip999 Date: Fri, 13 Aug 2021 18:19:05 +0200 Subject: [PATCH 13/37] Improved support of jdbc format for nested fields in SELECT statements --- .../sql/legacy/PrettyFormatResponseIT.java | 29 ++++ .../sql/legacy/SQLIntegTestCase.java | 7 +- .../sql/legacy/TestUtils.java | 5 + .../sql/legacy/TestsConstants.java | 1 + .../executor/format/SelectResultSet.java | 144 ++++++++---------- 5 files changed, 108 insertions(+), 78 deletions(-) diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java index 5039e9e61e..5f6378ff7a 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java @@ -76,6 +76,7 @@ protected void init() throws Exception { loadIndex(Index.PHRASE); loadIndex(Index.GAME_OF_THRONES); loadIndex(Index.NESTED); + loadIndex(Index.BOOKS); } @Override @@ -518,6 +519,34 @@ public void fieldOrderOther() throws IOException { testFieldOrder(expectedFields, expectedValues); } + @Test + public void selectSubSubFieldOfNestedField() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT nested(authors.info.name, authors), " + + "performance.revenue " + + "FROM %s", TestsConstants.TEST_INDEX_BOOKS)); + + List fields = Arrays.asList("authors.info.name", "performance.revenue"); + JSONArray dataRows = getDataRows(response); + assertContainsColumns(getSchema(response), fields); + assertContainsData(dataRows, fields); + assertEquals(3, dataRows.length()); + } + + @Test + public void selectMultipleNestedFields() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT nested(authors.info.name, authors), " + + "nested(performance.sells.year) " + + "FROM %s", TestsConstants.TEST_INDEX_BOOKS)); + + List fields = Arrays.asList("authors.info.name", "performance.sells.year"); + JSONArray dataRows = getDataRows(response); + assertContainsColumns(getSchema(response), fields); + assertContainsData(dataRows, fields); + assertEquals(4, dataRows.length()); + } + private void testFieldOrder(final String[] expectedFields, final Object[] expectedValues) throws IOException { diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java index 9a10d3f054..df10a4f1bf 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java @@ -41,6 +41,7 @@ import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getResponseBody; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getStringIndexMapping; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getWeblogsIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getBooksIndexMapping; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.isIndexExist; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.loadDataByRestClient; import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.CURSOR_CLOSE_ENDPOINT; @@ -556,7 +557,11 @@ public enum Index { DATA_TYPE_NONNUMERIC(TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC, "_doc", getDataTypeNonnumericIndexMapping(), - "src/test/resources/datatypes.json"); + "src/test/resources/datatypes.json"), + BOOKS(TestsConstants.TEST_INDEX_BOOKS, + "nestedType", + getBooksIndexMapping(), + "src/test/resources/books.json"); private final String name; private final String type; diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestUtils.java index 1bb43bfb8d..d7662cd494 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestUtils.java @@ -253,6 +253,11 @@ public static String getDataTypeNonnumericIndexMapping() { return getMappingFile(mappingFile); } + public static String getBooksIndexMapping() { + String mappingFile = "books_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static void loadBulk(Client client, String jsonPath, String defaultIndex) throws Exception { System.out.println(String.format("Loading file %s into elasticsearch cluster", jsonPath)); diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java index a4aec50d98..120de484e0 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java @@ -60,6 +60,7 @@ public class TestsConstants { public final static String TEST_INDEX_STRINGS = TEST_INDEX + "_strings"; public final static String TEST_INDEX_DATATYPE_NUMERIC = TEST_INDEX + "_datatypes_numeric"; public final static String TEST_INDEX_DATATYPE_NONNUMERIC = TEST_INDEX + "_datatypes_nonnumeric"; + public final static String TEST_INDEX_BOOKS = TEST_INDEX + "_nested"; public final static String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; public final static String TS_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java index a6f4dadeef..231236a5d4 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java @@ -579,6 +579,7 @@ private void extractData() { if (queryResult instanceof SearchHits) { SearchHits searchHits = (SearchHits) queryResult; + this.rows = new ArrayList<>(); this.rows = populateRows(searchHits); this.size = rows.size(); this.internalTotalHits = Optional.ofNullable(searchHits.getTotalHits()).map(th -> th.value).orElse(0L); @@ -643,14 +644,30 @@ private long rowsLeft(Integer fetchSize, Integer limit) { private List populateRows(SearchHits searchHits) { List rows = new ArrayList<>(); - Set newKeys = new HashSet<>(head); + List> rowsAsMap = populateRows(head, searchHits); + for (Map rowAsMap : rowsAsMap) { + rows.add(new DataRows.Row(rowAsMap)); + } + return rows; + } + + /** + * Returns a list of "flattened" rows as (path, value) hashmaps. + * The core idea is to flatten non-nested hits, and recur on inner hits + * by calling the method flatNestedField(), which in turn calls back + * populateRows. + */ + private List> populateRows(List keys, SearchHits searchHits) { + List> rows = new ArrayList<>(); for (SearchHit hit : searchHits) { Map rowSource = hit.getSourceAsMap(); - List result; + List> result = new ArrayList<>(); if (!isJoinQuery()) { // Row already flatten in source in join. And join doesn't support nested fields for now. - rowSource = flatRow(head, rowSource); + result = flatNestedField(keys, hit.getInnerHits()); + + rowSource = flatRow(keys, rowSource); rowSource.put(SCORE, hit.getScore()); for (Map.Entry field : hit.getFields().entrySet()) { @@ -659,13 +676,15 @@ private List populateRows(SearchHits searchHits) { if (formatType.equalsIgnoreCase(Format.JDBC.getFormatName())) { dateFieldFormatter.applyJDBCDateFormat(rowSource); } - result = flatNestedField(newKeys, rowSource, hit.getInnerHits()); + + for (Map row : result) { + row.putAll(rowSource); + } } else { if (formatType.equalsIgnoreCase(Format.JDBC.getFormatName())) { dateFieldFormatter.applyJDBCDateFormat(rowSource); } - result = new ArrayList<>(); - result.add(new DataRows.Row(rowSource)); + result.add(rowSource); } rows.addAll(result); @@ -781,6 +800,7 @@ private Map addNumericAggregation(List aggs, Map flatRow(List keys, Map row) { Map flattenedRow = new HashMap<>(); for (String key : keys) { @@ -813,94 +833,64 @@ private Map flatRow(List keys, Map row) } /** - * If innerHits associated with column name exists, flatten both the inner field name and the inner rows in it. - *

- * Sample input: - * newKeys = {'region', 'employees.age'}, row = {'region': 'US'} - * innerHits = employees: { - * hits: [{ - * source: { - * age: 26, - * firstname: 'Hank' - * } - * },{ - * source: { - * age: 30, - * firstname: 'John' - * } - * }] - * } + * Flattens the inner hits passed as argument. + * For each inner hit, iterates on its column names colName and computes a list of new keys. + * The new keys are the original keys that begin with colName + ".", stripped of that prefix. + * Then, calls populateRows on the new keys and the hits associated to colName. + * This also works for nested fields which contain another nested field as a subfield. */ - private List flatNestedField(Set newKeys, Map row, - Map innerHits) { - List result = new ArrayList<>(); - result.add(new DataRows.Row(row)); + private List> flatNestedField(List keys, Map innerHits) { + List> result = new ArrayList<>(); + result.add(new HashMap<>()); if (innerHits == null) { return result; } for (String colName : innerHits.keySet()) { - SearchHit[] colValue = innerHits.get(colName).getHits(); - doFlatNestedFieldName(colName, colValue, newKeys); - result = doFlatNestedFieldValue(colName, colValue, result); - } + List newKeys = new ArrayList<>(); + for (String key : keys) { + if (key.startsWith(colName) && !key.equals(colName)) { + newKeys.add(key.substring(colName.length() + 1)); + } + } - return result; - } + List> innerResult = populateRows(newKeys, innerHits.get(colName)); + for (Map innerRow : innerResult) { + Map row = new HashMap<>(); + for (String path : innerRow.keySet()) { + if (!path.equals(SCORE)) { + row.put(colName + "." + path, innerRow.get(path)); + } else { + row.put(path, innerRow.get(path)); + } + } + innerRow.clear(); + innerRow.putAll(row); + } - private void doFlatNestedFieldName(String colName, SearchHit[] colValue, Set keys) { - Map innerRow = colValue[0].getSourceAsMap(); - for (String field : innerRow.keySet()) { - String innerName = colName + "." + field; - keys.add(innerName); + // In the case of multiple sets of inner hits, returns all possible combinations of entries + result = cartesianProduct(result, innerResult); } - keys.remove(colName); + return result; } /** - * Do Cartesian Product between current outer row and inner rows by nested loop and remove original outer row. - *

- * Sample input: - * colName = 'employees', rows = [{region: 'US'}] - * colValue= [{ - * source: { - * age: 26, - * firstname: 'Hank' - * } - * },{ - * source: { - * age: 30, - * firstname: 'John' - * } - * }] - *

- * Return: - * [ - * {region:'US', employees.age:26, employees.firstname:'Hank'}, - * {region:'US', employees.age:30, employees.firstname:'John'} - * ] + * Performs the "cartesian product" between two sets of rows, + * which is the set of all possible unions of a row in rowsLeft and a row in rowsRight. */ - private List doFlatNestedFieldValue(String colName, SearchHit[] colValue, List rows) { - List result = new ArrayList<>(); - for (DataRows.Row row : rows) { - for (SearchHit hit : colValue) { - Map innerRow = hit.getSourceAsMap(); - Map copy = new HashMap<>(); - - for (String field : row.getContents().keySet()) { - copy.put(field, row.getData(field)); - } - for (String field : innerRow.keySet()) { - copy.put(colName + "." + field, innerRow.get(field)); - } - - copy.remove(colName); - result.add(new DataRows.Row(copy)); + List> cartesianProduct(List> rowsLeft, + List> rowsRight) { + List> result = new ArrayList<>(); + for (Map rowLeft : rowsLeft) { + for (Map rowRight : rowsRight) { + Map union = new HashMap<>(); + union.putAll(rowLeft); + union.putAll(rowRight); + result.add(union); } } - return result; } From 1a9d1aa477d72ffd1289a59e311ed7bf928328cb Mon Sep 17 00:00:00 2001 From: cip999 Date: Fri, 13 Aug 2021 18:20:23 +0200 Subject: [PATCH 14/37] Improved support of jdbc format for nested fields in SELECT statements --- integ-test/src/test/resources/books.json | 4 ++ .../indexDefinitions/books_index_mapping.json | 61 +++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 integ-test/src/test/resources/books.json create mode 100644 integ-test/src/test/resources/indexDefinitions/books_index_mapping.json diff --git a/integ-test/src/test/resources/books.json b/integ-test/src/test/resources/books.json new file mode 100644 index 0000000000..f6dd1c3286 --- /dev/null +++ b/integ-test/src/test/resources/books.json @@ -0,0 +1,4 @@ +{"index": {"_id": "1"}} +{"authors": [{"id": 8634, "info": {"name": "Andrea", "age": 22}}, {"id": 836, "info": {"name": "Carmen", "age": 19}}], "performance": {"revenue": 932.88, "sells": [{"year": 2018, "quantity": {"italy": 1000000, "abroad": 500000}}, {"year": 2020, "quantity": {"italy": 2001002, "abroad": 0}}]}} +{"index": {"_id": "2"}} +{"authors": [{"id": 8, "info": {"name": "Mark", "age": 80}}], "performance": {"revenue": 555.0, "sells": []}} diff --git a/integ-test/src/test/resources/indexDefinitions/books_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/books_index_mapping.json new file mode 100644 index 0000000000..2db565c72a --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/books_index_mapping.json @@ -0,0 +1,61 @@ +{ + "mappings": { + "properties": { + "authors": { + "type": "nested", + "properties": { + "id": { + "type": "long", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "info": { + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "age": { + "type": "long" + } + } + } + } + }, + "performance": { + "properties": { + "revenue": { + "type": "double" + }, + "sells": { + "type": "nested", + "properties": { + "year": { + "type": "long" + }, + "quantity": { + "properties": { + "italy": { + "type": "long" + }, + "abroad": { + "type": "long" + } + } + } + } + } + } + } + } + } +} From 880c258e154fe126313fe2285122158c16d7d303 Mon Sep 17 00:00:00 2001 From: cip999 Date: Sun, 15 Aug 2021 13:40:38 +0200 Subject: [PATCH 15/37] Now jdbc format takes into account limit (if it exists) when flattening rows --- .../sql/legacy/PrettyFormatResponseIT.java | 18 +++++++ .../executor/format/SelectResultSet.java | 50 +++++++++++++++---- 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java index 5f6378ff7a..c496bbf95b 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java @@ -547,6 +547,24 @@ public void selectMultipleNestedFields() throws IOException { assertEquals(4, dataRows.length()); } + @Test + public void nestedFieldWithLimit() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT nested(authors.info.name, authors) " + + "FROM %s LIMIT 1", TestsConstants.TEST_INDEX_BOOKS)); + assertEquals(1, getDataRows(response).length()); + } + + @Test + public void multipleNestedFieldsWithLimit() throws IOException, InterruptedException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT nested(authors.info.name, authors), " + + "nested(performance.sells.year)" + + "FROM %s LIMIT 3", TestsConstants.TEST_INDEX_BOOKS)); + assertEquals(3, getDataRows(response).length()); + } + + private void testFieldOrder(final String[] expectedFields, final Object[] expectedValues) throws IOException { diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java index 231236a5d4..dfdf53fb58 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java @@ -87,6 +87,7 @@ public class SelectResultSet extends ResultSet { private long size; private long totalHits; private long internalTotalHits; + private Integer rowCount; private List rows; private Cursor cursor; @@ -118,6 +119,9 @@ public SelectResultSet(Client client, this.schema = new Schema(indexName, typeName, columns); this.head = schema.getHeaders(); this.dateFieldFormatter = new DateFieldFormatter(indexName, columns, fieldAliasMap); + if (query instanceof Select) { + this.rowCount = ((Select) query).getRowCount(); + } extractData(); populateCursor(); @@ -579,7 +583,6 @@ private void extractData() { if (queryResult instanceof SearchHits) { SearchHits searchHits = (SearchHits) queryResult; - this.rows = new ArrayList<>(); this.rows = populateRows(searchHits); this.size = rows.size(); this.internalTotalHits = Optional.ofNullable(searchHits.getTotalHits()).map(th -> th.value).orElse(0L); @@ -644,7 +647,7 @@ private long rowsLeft(Integer fetchSize, Integer limit) { private List populateRows(SearchHits searchHits) { List rows = new ArrayList<>(); - List> rowsAsMap = populateRows(head, searchHits); + List> rowsAsMap = populateRows(head, searchHits, rowCount); for (Map rowAsMap : rowsAsMap) { rows.add(new DataRows.Row(rowAsMap)); } @@ -657,15 +660,23 @@ private List populateRows(SearchHits searchHits) { * by calling the method flatNestedField(), which in turn calls back * populateRows. */ - private List> populateRows(List keys, SearchHits searchHits) { + private List> populateRows(List keys, SearchHits searchHits, Integer rowsLeft) { List> rows = new ArrayList<>(); + for (SearchHit hit : searchHits) { + if (rowsLeft != null && rowsLeft == 0) { + return rows; + } + Map rowSource = hit.getSourceAsMap(); List> result = new ArrayList<>(); if (!isJoinQuery()) { // Row already flatten in source in join. And join doesn't support nested fields for now. - result = flatNestedField(keys, hit.getInnerHits()); + result = flatNestedField(keys, hit.getInnerHits(), rowsLeft); + if (rowsLeft != null) { + rowsLeft -= result.size(); + } rowSource = flatRow(keys, rowSource); rowSource.put(SCORE, hit.getScore()); @@ -685,6 +696,9 @@ private List> populateRows(List keys, SearchHits sea dateFieldFormatter.applyJDBCDateFormat(rowSource); } result.add(rowSource); + if (rowsLeft != null) { + rowsLeft--; + } } rows.addAll(result); @@ -839,7 +853,9 @@ private Map flatRow(List keys, Map row) * Then, calls populateRows on the new keys and the hits associated to colName. * This also works for nested fields which contain another nested field as a subfield. */ - private List> flatNestedField(List keys, Map innerHits) { + private List> flatNestedField(List keys, + Map innerHits, + Integer rowsLeft) { List> result = new ArrayList<>(); result.add(new HashMap<>()); @@ -855,7 +871,9 @@ private List> flatNestedField(List keys, Map> innerResult = populateRows(newKeys, innerHits.get(colName)); + List> innerResult = populateRows(newKeys, innerHits.get(colName), + (rowsLeft == null) ? null : + ((result.isEmpty() ? 0 : ratioCeil(rowsLeft, result.size())))); for (Map innerRow : innerResult) { Map row = new HashMap<>(); for (String path : innerRow.keySet()) { @@ -870,7 +888,7 @@ private List> flatNestedField(List keys, Map> flatNestedField(List keys, Map> cartesianProduct(List> rowsLeft, - List> rowsRight) { + private List> cartesianProduct(List> rowsLeft, + List> rowsRight, + Integer maxRows) { List> result = new ArrayList<>(); for (Map rowLeft : rowsLeft) { + if (maxRows != null && result.size() == maxRows) { + break; + } + for (Map rowRight : rowsRight) { + if (maxRows != null && result.size() == maxRows) { + break; + } + Map union = new HashMap<>(); union.putAll(rowLeft); union.putAll(rowRight); result.add(union); } } + return result; } + private int ratioCeil(int a, int b) { + return (a + b - 1) / b; + } + private Map addMap(String field, Object term) { Map data = new HashMap<>(); data.put(field, term); From 5072613f9e7f4aa1bce4119a103827593e31293f Mon Sep 17 00:00:00 2001 From: cip999 Date: Sun, 15 Aug 2021 16:43:54 +0200 Subject: [PATCH 16/37] Updated some variable names for clarity's sake --- .../sql/legacy/PrettyFormatResponseIT.java | 2 +- .../sql/legacy/SQLIntegTestCase.java | 2 +- .../sql/legacy/TestsConstants.java | 2 +- .../executor/format/SelectResultSet.java | 49 ++++++++++++------- 4 files changed, 35 insertions(+), 20 deletions(-) diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java index c496bbf95b..0cff5986ee 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java @@ -76,7 +76,7 @@ protected void init() throws Exception { loadIndex(Index.PHRASE); loadIndex(Index.GAME_OF_THRONES); loadIndex(Index.NESTED); - loadIndex(Index.BOOKS); + loadIndex(Index.BOOKS_NESTED_WITH_SUBPROPERTIES); } @Override diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java index df10a4f1bf..63f7e4b4f4 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java @@ -558,7 +558,7 @@ public enum Index { "_doc", getDataTypeNonnumericIndexMapping(), "src/test/resources/datatypes.json"), - BOOKS(TestsConstants.TEST_INDEX_BOOKS, + BOOKS_NESTED_WITH_SUBPROPERTIES(TestsConstants.TEST_INDEX_BOOKS, "nestedType", getBooksIndexMapping(), "src/test/resources/books.json"); diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java index 120de484e0..c8a184ce97 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java @@ -60,7 +60,7 @@ public class TestsConstants { public final static String TEST_INDEX_STRINGS = TEST_INDEX + "_strings"; public final static String TEST_INDEX_DATATYPE_NUMERIC = TEST_INDEX + "_datatypes_numeric"; public final static String TEST_INDEX_DATATYPE_NONNUMERIC = TEST_INDEX + "_datatypes_nonnumeric"; - public final static String TEST_INDEX_BOOKS = TEST_INDEX + "_nested"; + public final static String TEST_INDEX_BOOKS = TEST_INDEX + "_nested_subproperties"; public final static String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; public final static String TS_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java index dfdf53fb58..18ced20fa4 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java @@ -87,7 +87,7 @@ public class SelectResultSet extends ResultSet { private long size; private long totalHits; private long internalTotalHits; - private Integer rowCount; + private Integer limitRowCount; private List rows; private Cursor cursor; @@ -120,7 +120,7 @@ public SelectResultSet(Client client, this.head = schema.getHeaders(); this.dateFieldFormatter = new DateFieldFormatter(indexName, columns, fieldAliasMap); if (query instanceof Select) { - this.rowCount = ((Select) query).getRowCount(); + this.limitRowCount = ((Select) query).getRowCount(); } extractData(); @@ -647,7 +647,7 @@ private long rowsLeft(Integer fetchSize, Integer limit) { private List populateRows(SearchHits searchHits) { List rows = new ArrayList<>(); - List> rowsAsMap = populateRows(head, searchHits, rowCount); + List> rowsAsMap = populateRows(head, searchHits, limitRowCount); for (Map rowAsMap : rowsAsMap) { rows.add(new DataRows.Row(rowAsMap)); } @@ -659,12 +659,27 @@ private List populateRows(SearchHits searchHits) { * The core idea is to flatten non-nested hits, and recur on inner hits * by calling the method flatNestedField(), which in turn calls back * populateRows. + *

+ * Sample input: + * keys = {"authors.info.name"} (authors is nested) + * searchHits = { + * { + * key = "authors" + * innerHits = { + * {"info.name": "Andrea"}, + * {"info.name": "Carmen} + * } + * } + * } + * Sample output: + * rows = [{"authors.info.name": "Andrea"}, {"authors.info.name": "Carmen"}] + *

*/ - private List> populateRows(List keys, SearchHits searchHits, Integer rowsLeft) { + private List> populateRows(List keys, SearchHits searchHits, Integer remainingRows) { List> rows = new ArrayList<>(); for (SearchHit hit : searchHits) { - if (rowsLeft != null && rowsLeft == 0) { + if (remainingRows != null && remainingRows == 0) { return rows; } @@ -673,9 +688,9 @@ private List> populateRows(List keys, SearchHits sea if (!isJoinQuery()) { // Row already flatten in source in join. And join doesn't support nested fields for now. - result = flatNestedField(keys, hit.getInnerHits(), rowsLeft); - if (rowsLeft != null) { - rowsLeft -= result.size(); + result = flatNestedField(keys, hit.getInnerHits(), remainingRows); + if (remainingRows != null) { + remainingRows -= result.size(); } rowSource = flatRow(keys, rowSource); @@ -696,8 +711,8 @@ private List> populateRows(List keys, SearchHits sea dateFieldFormatter.applyJDBCDateFormat(rowSource); } result.add(rowSource); - if (rowsLeft != null) { - rowsLeft--; + if (remainingRows != null) { + remainingRows--; } } @@ -855,7 +870,7 @@ private Map flatRow(List keys, Map row) */ private List> flatNestedField(List keys, Map innerHits, - Integer rowsLeft) { + Integer remainingRows) { List> result = new ArrayList<>(); result.add(new HashMap<>()); @@ -872,8 +887,8 @@ private List> flatNestedField(List keys, } List> innerResult = populateRows(newKeys, innerHits.get(colName), - (rowsLeft == null) ? null : - ((result.isEmpty() ? 0 : ratioCeil(rowsLeft, result.size())))); + (remainingRows == null) ? null : + ((result.isEmpty() ? 0 : ratioCeil(remainingRows, result.size())))); for (Map innerRow : innerResult) { Map row = new HashMap<>(); for (String path : innerRow.keySet()) { @@ -888,7 +903,7 @@ private List> flatNestedField(List keys, } // In the case of multiple sets of inner hits, returns all possible combinations of entries - result = cartesianProduct(result, innerResult, rowsLeft); + result = cartesianProduct(result, innerResult, remainingRows); } return result; @@ -900,15 +915,15 @@ private List> flatNestedField(List keys, */ private List> cartesianProduct(List> rowsLeft, List> rowsRight, - Integer maxRows) { + Integer remainingRows) { List> result = new ArrayList<>(); for (Map rowLeft : rowsLeft) { - if (maxRows != null && result.size() == maxRows) { + if (remainingRows != null && result.size() == remainingRows) { break; } for (Map rowRight : rowsRight) { - if (maxRows != null && result.size() == maxRows) { + if (remainingRows != null && result.size() == remainingRows) { break; } From 501ab9f56b53f087aa4307d51af2891f92cdd5ab Mon Sep 17 00:00:00 2001 From: cip999 Date: Sun, 15 Aug 2021 17:03:49 +0200 Subject: [PATCH 17/37] Test commit --- .../sql/legacy/executor/format/SelectResultSet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java index 18ced20fa4..cae1bf065a 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java @@ -864,7 +864,7 @@ private Map flatRow(List keys, Map row) /** * Flattens the inner hits passed as argument. * For each inner hit, iterates on its column names colName and computes a list of new keys. - * The new keys are the original keys that begin with colName + ".", stripped of that prefix. + * The new keys are the original keys that begin with colName + "." stripped of that prefix. * Then, calls populateRows on the new keys and the hits associated to colName. * This also works for nested fields which contain another nested field as a subfield. */ From 7df68e458337667f5354e4f07b7574dbe4137df2 Mon Sep 17 00:00:00 2001 From: cip999 Date: Sun, 15 Aug 2021 17:11:27 +0200 Subject: [PATCH 18/37] Update actions to run on next LTS Ubuntu version --- .github/workflows/sql-cli-release-workflow.yml | 2 +- .github/workflows/sql-cli-test-and-build-workflow.yml | 2 +- .github/workflows/sql-jdbc-push-jdbc-maven.yml | 2 +- .github/workflows/sql-release-workflow.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sql-cli-release-workflow.yml b/.github/workflows/sql-cli-release-workflow.yml index 2959df51b6..baa70174fe 100644 --- a/.github/workflows/sql-cli-release-workflow.yml +++ b/.github/workflows/sql-cli-release-workflow.yml @@ -8,7 +8,7 @@ on: jobs: build: - runs-on: [ubuntu-16.04] + runs-on: [ubuntu-18.04] defaults: run: working-directory: sql-cli diff --git a/.github/workflows/sql-cli-test-and-build-workflow.yml b/.github/workflows/sql-cli-test-and-build-workflow.yml index 1af71516cd..69b2807ca7 100644 --- a/.github/workflows/sql-cli-test-and-build-workflow.yml +++ b/.github/workflows/sql-cli-test-and-build-workflow.yml @@ -5,7 +5,7 @@ on: [pull_request, push] jobs: build: - runs-on: [ubuntu-16.04] + runs-on: [ubuntu-18.04] defaults: run: working-directory: sql-cli diff --git a/.github/workflows/sql-jdbc-push-jdbc-maven.yml b/.github/workflows/sql-jdbc-push-jdbc-maven.yml index 53f2d5d391..4e5855c817 100644 --- a/.github/workflows/sql-jdbc-push-jdbc-maven.yml +++ b/.github/workflows/sql-jdbc-push-jdbc-maven.yml @@ -8,7 +8,7 @@ on: jobs: upload-jdbc-jar: - runs-on: [ubuntu-16.04] + runs-on: [ubuntu-18.04] defaults: run: working-directory: sql-jdbc diff --git a/.github/workflows/sql-release-workflow.yml b/.github/workflows/sql-release-workflow.yml index d2a5abcbb7..c487115f2a 100644 --- a/.github/workflows/sql-release-workflow.yml +++ b/.github/workflows/sql-release-workflow.yml @@ -12,7 +12,7 @@ jobs: java: [14] name: Build and Release SQL Plugin - runs-on: [ubuntu-16.04] + runs-on: [ubuntu-18.04] steps: - name: Checkout SQL From a17f9016aa31d8be24114b55ededc0ac6f8daf4f Mon Sep 17 00:00:00 2001 From: cip999 Date: Sun, 15 Aug 2021 17:29:04 +0200 Subject: [PATCH 19/37] Update actions to run on next LTS Ubuntu version --- .github/workflows/sql-cli-release-workflow.yml | 2 +- .github/workflows/sql-cli-test-and-build-workflow.yml | 2 +- .github/workflows/sql-jdbc-push-jdbc-maven.yml | 2 +- .github/workflows/sql-release-workflow.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sql-cli-release-workflow.yml b/.github/workflows/sql-cli-release-workflow.yml index 2959df51b6..baa70174fe 100644 --- a/.github/workflows/sql-cli-release-workflow.yml +++ b/.github/workflows/sql-cli-release-workflow.yml @@ -8,7 +8,7 @@ on: jobs: build: - runs-on: [ubuntu-16.04] + runs-on: [ubuntu-18.04] defaults: run: working-directory: sql-cli diff --git a/.github/workflows/sql-cli-test-and-build-workflow.yml b/.github/workflows/sql-cli-test-and-build-workflow.yml index 1af71516cd..69b2807ca7 100644 --- a/.github/workflows/sql-cli-test-and-build-workflow.yml +++ b/.github/workflows/sql-cli-test-and-build-workflow.yml @@ -5,7 +5,7 @@ on: [pull_request, push] jobs: build: - runs-on: [ubuntu-16.04] + runs-on: [ubuntu-18.04] defaults: run: working-directory: sql-cli diff --git a/.github/workflows/sql-jdbc-push-jdbc-maven.yml b/.github/workflows/sql-jdbc-push-jdbc-maven.yml index 53f2d5d391..4e5855c817 100644 --- a/.github/workflows/sql-jdbc-push-jdbc-maven.yml +++ b/.github/workflows/sql-jdbc-push-jdbc-maven.yml @@ -8,7 +8,7 @@ on: jobs: upload-jdbc-jar: - runs-on: [ubuntu-16.04] + runs-on: [ubuntu-18.04] defaults: run: working-directory: sql-jdbc diff --git a/.github/workflows/sql-release-workflow.yml b/.github/workflows/sql-release-workflow.yml index d2a5abcbb7..c487115f2a 100644 --- a/.github/workflows/sql-release-workflow.yml +++ b/.github/workflows/sql-release-workflow.yml @@ -12,7 +12,7 @@ jobs: java: [14] name: Build and Release SQL Plugin - runs-on: [ubuntu-16.04] + runs-on: [ubuntu-18.04] steps: - name: Checkout SQL From 953ad267d36b4649ac4264242aa6e268f61a2ca9 Mon Sep 17 00:00:00 2001 From: cip999 Date: Sun, 22 Aug 2021 00:39:00 +0200 Subject: [PATCH 20/37] Implemented "greatest" and "least" functions in old engine --- .../sql/legacy/SQLFunctionsIT.java | 15 +++++ .../types/function/ScalarFunction.java | 10 +++ .../sql/legacy/utils/SQLFunctions.java | 65 ++++++++++++++++++- 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java index fc9088a24a..aab2c20d54 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java @@ -17,6 +17,8 @@ package com.amazon.opendistroforelasticsearch.sql.legacy; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME; import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.hitAny; import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvDouble; import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvInt; @@ -68,6 +70,7 @@ protected void init() throws Exception { loadIndex(Index.BANK); loadIndex(Index.ONLINE); loadIndex(Index.DATE); + loadIndex(Index.DATETIME); } @Test @@ -877,6 +880,18 @@ public void literalMultiField() throws Exception { assertThat(hits[0].getFields(), allOf(hasValue(contains(1)), hasValue(contains(2)))); } + @Test + public void greatestWithAliasAndStrings() throws Exception { + JSONObject response = + executeJdbcRequest("SELECT greatest(firstname, lastname) AS max " + + "FROM " + TEST_INDEX_ACCOUNT + " LIMIT 3"); + + verifyDataRows(response, + rows("duke"), + rows("hattie"), + rows("nanette")); + } + private SearchHits query(String query) throws IOException { final String rsp = executeQueryWithStringOutput(query); diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ScalarFunction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ScalarFunction.java index 7ba1e0f618..1f74fbb147 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ScalarFunction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ScalarFunction.java @@ -60,9 +60,19 @@ public enum ScalarFunction implements TypeExpression { EXP(func(T(NUMBER)).to(T)), EXPM1(func(T(NUMBER)).to(T)), FLOOR(func(T(NUMBER)).to(T)), + GREATEST( + func(T(NUMBER), NUMBER).to(T), + func(T(STRING), STRING).to(T), + func(ESDataType.DATE, ESDataType.DATE).to(ESDataType.DATE) + ), IF(func(BOOLEAN, ES_TYPE, ES_TYPE).to(ES_TYPE)), IFNULL(func(ES_TYPE, ES_TYPE).to(ES_TYPE)), ISNULL(func(ES_TYPE).to(INTEGER)), + LEAST( + func(T(NUMBER), NUMBER).to(T), + func(T(STRING), STRING).to(T), + func(ESDataType.DATE, ESDataType.DATE).to(ESDataType.DATE) + ), LEFT(func(T(STRING), INTEGER).to(T)), LENGTH(func(STRING).to(INTEGER)), LN(func(T(NUMBER)).to(DOUBLE)), diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java index ba635522dd..752f7252fe 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java @@ -75,7 +75,7 @@ public class SQLFunctions { ); private static final Set binaryOperators = Sets.newHashSet( - "add", "multiply", "divide", "subtract", "modulus" + "add", "multiply", "divide", "subtract", "modulus", "greatest", "least" ); private static final Set dateFunctions = Sets.newHashSet( @@ -332,6 +332,13 @@ public Tuple function(String methodName, List paramers, functionStr = modulus((SQLExpr) paramers.get(0).value, (SQLExpr) paramers.get(1).value); break; + case "greatest": + functionStr = greatest((SQLExpr) paramers.get(0).value, (SQLExpr) paramers.get(1).value); + break; + case "least": + functionStr = least((SQLExpr) paramers.get(0).value, (SQLExpr) paramers.get(1).value); + break; + case "field": functionStr = field(Util.expr2Object((SQLExpr) paramers.get(0).value).toString()); break; @@ -596,6 +603,56 @@ private Tuple binaryOpertator(String methodName, String operator + def(name, extractName(a) + " " + operator + " " + extractName(b))); } + private Tuple greatest(SQLExpr a, SQLExpr b) { + String name = nextId("greatest"); + + String name_a = extractName(a); + String name_b = extractName(b); + + String value_a = getPropertyOrStringValue(a); + String value_b = getPropertyOrStringValue(b); + + String nameString_a = nextId("string_a"); + String nameString_b = nextId("string_b"); + + return new Tuple<>(name, + scriptDeclare(a) + scriptDeclare(b) + + "def " + name + ";" + + String.format("if (%s instanceof Number) ", name_a) + + String.format("{%s = (%s >= %s ? %s : %s);} ", name, name_a, name_b, name_a, name_b) + + "else {" + + def(nameString_a, convertToString(value_a)) + ";" + + def(nameString_b, convertToString(value_b)) + ";" + + String.format("%s = (%s.compareTo(%s) >= 0 ? %s : %s);}", + name, nameString_a, nameString_b, value_a, value_b) + + name + " = " + name); + } + + private Tuple least(SQLExpr a, SQLExpr b) { + String name = nextId("least"); + + String name_a = extractName(a); + String name_b = extractName(b); + + String value_a = getPropertyOrStringValue(a); + String value_b = getPropertyOrStringValue(b); + + String nameString_a = nextId("string_a"); + String nameString_b = nextId("string_b"); + + return new Tuple<>(name, + scriptDeclare(a) + scriptDeclare(b) + + "def " + name + ";" + + String.format("if (%s instanceof Number) ", name_a) + + String.format("{%s = (%s <= %s ? %s : %s);} ", name, name_a, name_b, name_a, name_b) + + "else {" + + def(nameString_a, convertToString(value_a)) + ";" + + def(nameString_b, convertToString(value_b)) + ";" + + String.format("%s = (%s.compareTo(%s) <= 0 ? %s : %s);}", + name, nameString_a, nameString_b, value_a, value_b) + + name + " = " + name); + } + private static boolean isProperty(SQLExpr expr) { return (expr instanceof SQLIdentifierExpr || expr instanceof SQLPropertyExpr || expr instanceof SQLVariantRefExpr); @@ -628,7 +685,6 @@ private static String getPropertyOrStringValue(SQLExpr expr) { } private static String scriptDeclare(SQLExpr a) { - if (isProperty(a) || a instanceof SQLNumericLiteralExpr) { return ""; } else { @@ -668,6 +724,11 @@ private static String convertType(SQLExpr script) { } + private static String convertToString(String name) { + return String.format("(%s instanceof String ? (String) %s : %s.toString())", + name, name, name); + } + private String getScriptText(MethodField field) { String content = ((SQLTextLiteralExpr) field.getParams().get(1).value).getText(); return content; From 83107e02df9ab4a0a15b6a040ac70a11de28a14c Mon Sep 17 00:00:00 2001 From: cip999 Date: Sun, 22 Aug 2021 16:36:32 +0200 Subject: [PATCH 21/37] Implemented "greatest" and "least" functions in old engine --- .../sql/legacy/SQLFunctionsIT.java | 4 +- .../sql/legacy/utils/SQLFunctions.java | 84 +++++++++++-------- 2 files changed, 51 insertions(+), 37 deletions(-) diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java index aab2c20d54..37a447dbbb 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java @@ -884,13 +884,13 @@ public void literalMultiField() throws Exception { public void greatestWithAliasAndStrings() throws Exception { JSONObject response = executeJdbcRequest("SELECT greatest(firstname, lastname) AS max " + - "FROM " + TEST_INDEX_ACCOUNT + " LIMIT 3"); + "FROM " + TEST_INDEX_ACCOUNT + " LIMIT 3"); verifyDataRows(response, rows("duke"), rows("hattie"), rows("nanette")); - } + } private SearchHits query(String query) throws IOException { final String rsp = executeQueryWithStringOutput(query); diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java index 752f7252fe..e036f160f9 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java @@ -606,51 +606,52 @@ private Tuple binaryOpertator(String methodName, String operator private Tuple greatest(SQLExpr a, SQLExpr b) { String name = nextId("greatest"); - String name_a = extractName(a); - String name_b = extractName(b); - - String value_a = getPropertyOrStringValue(a); - String value_b = getPropertyOrStringValue(b); - - String nameString_a = nextId("string_a"); - String nameString_b = nextId("string_b"); - - return new Tuple<>(name, - scriptDeclare(a) + scriptDeclare(b) - + "def " + name + ";" - + String.format("if (%s instanceof Number) ", name_a) - + String.format("{%s = (%s >= %s ? %s : %s);} ", name, name_a, name_b, name_a, name_b) - + "else {" - + def(nameString_a, convertToString(value_a)) + ";" - + def(nameString_b, convertToString(value_b)) + ";" - + String.format("%s = (%s.compareTo(%s) >= 0 ? %s : %s);}", - name, nameString_a, nameString_b, value_a, value_b) - + name + " = " + name); + return comparisonFunctionTemplate(a, b, name, ">"); } private Tuple least(SQLExpr a, SQLExpr b) { String name = nextId("least"); - String name_a = extractName(a); - String name_b = extractName(b); + return comparisonFunctionTemplate(a, b, name, "<"); + } - String value_a = getPropertyOrStringValue(a); - String value_b = getPropertyOrStringValue(b); + private Tuple comparisonFunctionTemplate(SQLExpr a, SQLExpr b, + String name, String comparisonOperator) { + String value_a = (a instanceof SQLTextLiteralExpr + ? getPropertyOrStringValue(a) : getPropertyOrValue(a)); + String value_b = (b instanceof SQLTextLiteralExpr + ? getPropertyOrStringValue(b) : getPropertyOrValue(b)); String nameString_a = nextId("string_a"); String nameString_b = nextId("string_b"); - return new Tuple<>(name, - scriptDeclare(a) + scriptDeclare(b) - + "def " + name + ";" - + String.format("if (%s instanceof Number) ", name_a) - + String.format("{%s = (%s <= %s ? %s : %s);} ", name, name_a, name_b, name_a, name_b) - + "else {" - + def(nameString_a, convertToString(value_a)) + ";" - + def(nameString_b, convertToString(value_b)) + ";" - + String.format("%s = (%s.compareTo(%s) <= 0 ? %s : %s);}", - name, nameString_a, nameString_b, value_a, value_b) - + name + " = " + name); + String numberComparison = String.format("%s = (%s %s %s ? %s : %s);", + name, value_a, comparisonOperator, value_b, value_a, value_b); + String stringOrDateComparison = def(nameString_a, convertToString(value_a)) + ";" + + def(nameString_b, convertToString(value_b)) + ";" + + String.format("%s = (%s.compareTo(%s) %s 0 ? %s : %s);", + name, nameString_a, nameString_b, comparisonOperator, value_a, value_b); + + String definition = "def " + name + ";" + + String.format("if (%s) {%s = %s;} ", checkIfNull(a), name, value_b) + + String.format("else if (%s) {%s = %s;} ", checkIfNull(b), name, value_a) + + "else {"; + + if (isProperty(a) && isProperty(b)) { + definition += String.format("if (%s instanceof Number) {", value_a) + + numberComparison + + "} else {" + + stringOrDateComparison + + "} "; + } else if (a instanceof SQLNumericLiteralExpr || b instanceof SQLNumericLiteralExpr) { + definition += numberComparison; + } else { + definition += stringOrDateComparison; + } + + definition += String.format("} %s = %s", name, name); + + return new Tuple<>(name, definition); } private static boolean isProperty(SQLExpr expr) { @@ -661,6 +662,8 @@ private static boolean isProperty(SQLExpr expr) { private static String getPropertyOrValue(SQLExpr expr) { if (isProperty(expr)) { return doc(expr) + ".value"; + } else if (expr instanceof SQLNullExpr) { + return "null"; } else { return exprString(expr); } @@ -724,6 +727,17 @@ private static String convertType(SQLExpr script) { } + private static String checkIfNull(SQLExpr expr) { + if (isProperty(expr)) { + return doc(expr) + ".size() == 0 || " + getPropertyOrValue(expr) + " == null"; + } else if (expr instanceof SQLNullExpr) { + return "0 == 0"; + } + else { + return "0 == 1"; + } + } + private static String convertToString(String name) { return String.format("(%s instanceof String ? (String) %s : %s.toString())", name, name, name); From 89a65673f8587bb162b10cf74769b3b20fc2b168 Mon Sep 17 00:00:00 2001 From: cip999 Date: Wed, 18 Aug 2021 16:01:53 +0200 Subject: [PATCH 22/37] Added "datetime_format" URL parameter to handle jdbj date and time formatting with new engine --- .../sql/data/model/ExprTimestampValue.java | 40 +++++++++++++++++-- .../sql/legacy/SQLIntegTestCase.java | 6 ++- .../sql/legacy/plugin/RestSQLQueryAction.java | 2 +- .../sql/legacy/plugin/RestSqlAction.java | 2 +- .../sql/legacy/plugin/RestSqlStatsAction.java | 2 +- .../sql/plugin/rest/RestPPLQueryAction.java | 2 +- .../sql/plugin/rest/RestPPLStatsAction.java | 2 +- .../sql/protocol/response/QueryResult.java | 11 +++++ .../format/JdbcResponseFormatter.java | 7 +++- .../sql/sql/domain/SQLQueryRequest.java | 5 +++ 10 files changed, 68 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java index 894a3e82ca..30c85ef0ac 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java @@ -46,14 +46,22 @@ public class ExprTimestampValue extends AbstractExprValue { /** * todo. only support timestamp in format yyyy-MM-dd HH:mm:ss. */ - private static final DateTimeFormatter FORMATTER_WITNOUT_NANO = DateTimeFormatter + private static final DateTimeFormatter FORMATTER_WITHOUT_NANO = DateTimeFormatter .ofPattern("yyyy-MM-dd HH:mm:ss"); + private static final DateTimeFormatter FORMATTER_WITHOUT_TIME = DateTimeFormatter + .ofPattern("yyyy-MM-dd"); private final Instant timestamp; + private String datetimeFormat; + private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS; private static final int MIN_FRACTION_SECONDS = 0; private static final int MAX_FRACTION_SECONDS = 6; + private static final String ALWAYS_INCLUDE_TIME = "always_include_time"; + private static final String NEVER_INCLUDE_TIME = "never_include_time"; + private static final String INCLUDE_TIME_WHEN_NONZERO = "include_time_when_nonzero"; + static { FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder() .appendPattern("yyyy-MM-dd HH:mm:ss") @@ -69,6 +77,8 @@ public class ExprTimestampValue extends AbstractExprValue { * Constructor. */ public ExprTimestampValue(String timestamp) { + this.datetimeFormat = ALWAYS_INCLUDE_TIME; + try { this.timestamp = LocalDateTime.parse(timestamp, ExprDateFormatters.TOLERANT_PARSER_DATE_TIME_FORMATTER) @@ -83,9 +93,29 @@ public ExprTimestampValue(String timestamp) { @Override public String value() { - return timestamp.getNano() == 0 ? FORMATTER_WITNOUT_NANO.withZone(ZONE) - .format(timestamp.truncatedTo(ChronoUnit.SECONDS)) - : FORMATTER_VARIABLE_MICROS.withZone(ZONE).format(timestamp); + switch (datetimeFormat) { + case NEVER_INCLUDE_TIME: + return valueWithoutTime(); + + case INCLUDE_TIME_WHEN_NONZERO: + LocalTime time = timeValue(); + return (time.getHour() == 0 && time.getMinute() == 0 && time.getSecond() == 0) ? + valueWithoutTime() : valueWithTime(); + + case ALWAYS_INCLUDE_TIME: + default: + return valueWithTime(); + } + } + + private String valueWithTime() { + return timestamp.getNano() == 0 ? FORMATTER_WITHOUT_NANO.withZone(ZONE) + .format(timestamp.truncatedTo(ChronoUnit.SECONDS)) + : FORMATTER_VARIABLE_MICROS.withZone(ZONE).format(timestamp); + } + + private String valueWithoutTime() { + return FORMATTER_WITHOUT_TIME.withZone(ZONE).format(timestamp); } @Override @@ -132,4 +162,6 @@ public boolean equal(ExprValue other) { public int hashCode() { return Objects.hashCode(timestamp); } + + public void setDatetimeFormat(String format) { this.datetimeFormat = format; } } diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java index 63f7e4b4f4..3edd35ecc4 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java @@ -223,8 +223,12 @@ protected Request getSqlCursorCloseRequest(String cursorRequest) { } protected String executeQuery(String query, String requestType) { + return executeQuery(query, requestType, "always_include_time"); + } + + protected String executeQuery(String query, String requestType, String datetimeFormat) { try { - String endpoint = "/_opendistro/_sql?format=" + requestType; + String endpoint = "/_opendistro/_sql?format=" + requestType + "&datetime_format=" + datetimeFormat; String requestBody = makeRequest(query); Request sqlRequest = new Request("POST", endpoint); diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java index f6fdc812bd..34d84146cf 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java @@ -172,7 +172,7 @@ private ResponseListener createQueryResponseListener(RestChannel } else if (format.equals(Format.RAW)) { formatter = new RawResponseFormatter(); } else { - formatter = new JdbcResponseFormatter(PRETTY); + formatter = new JdbcResponseFormatter(PRETTY, request.getDatetimeFormat()); } return new ResponseListener() { @Override diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java index 5eb3de5a0c..41e186019b 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java @@ -172,7 +172,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli @Override protected Set responseParams() { Set responseParams = new HashSet<>(super.responseParams()); - responseParams.addAll(Arrays.asList("sql", "flat", "separator", "_score", "_type", "_id", "newLine", "format", "sanitize")); + responseParams.addAll(Arrays.asList("sql", "flat", "separator", "_score", "_type", "_id", "newLine", "format", "datetime_format", "sanitize")); return responseParams; } diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlStatsAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlStatsAction.java index d4e5b6c500..4581586c14 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlStatsAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlStatsAction.java @@ -84,7 +84,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli @Override protected Set responseParams() { Set responseParams = new HashSet<>(super.responseParams()); - responseParams.addAll(Arrays.asList("sql", "flat", "separator", "_score", "_type", "_id", "newLine", "format", "sanitize")); + responseParams.addAll(Arrays.asList("sql", "flat", "separator", "_score", "_type", "_id", "newLine", "format", "datetime_format", "sanitize")); return responseParams; } diff --git a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLQueryAction.java b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLQueryAction.java index 75e07ca815..0ab50dbf7e 100644 --- a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLQueryAction.java +++ b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLQueryAction.java @@ -115,7 +115,7 @@ public String getName() { @Override protected Set responseParams() { Set responseParams = new HashSet<>(super.responseParams()); - responseParams.addAll(Arrays.asList("format", "sanitize")); + responseParams.addAll(Arrays.asList("format", "datetime_format", "sanitize")); return responseParams; } diff --git a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLStatsAction.java b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLStatsAction.java index c306f9ca93..1e12d22f29 100644 --- a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLStatsAction.java +++ b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLStatsAction.java @@ -85,7 +85,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli @Override protected Set responseParams() { Set responseParams = new HashSet<>(super.responseParams()); - responseParams.addAll(Arrays.asList("format", "sanitize")); + responseParams.addAll(Arrays.asList("format", "datetime_format", "sanitize")); return responseParams; } } diff --git a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java index 5deb7e0f56..8261f15ae0 100644 --- a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java +++ b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java @@ -16,8 +16,10 @@ package com.amazon.opendistroforelasticsearch.sql.protocol.response; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.Schema.Column; import java.util.Collection; @@ -75,6 +77,15 @@ public Iterator iterator() { .iterator(); } + public void setDatetimeFormat(String datetimeFormat) { + for (ExprValue exprValue : exprValues) { + for (ExprValue v : ExprValueUtils.getTupleValue(exprValue).values()) + if (v.type() == ExprCoreType.TIMESTAMP) { + ((ExprTimestampValue) v).setDatetimeFormat(datetimeFormat); + } + } + } + private String getColumnName(Column column) { return (column.getAlias() != null) ? column.getAlias() : column.getName(); } diff --git a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatter.java b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatter.java index 52f8c1d703..a2d203fbcb 100644 --- a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatter.java +++ b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatter.java @@ -36,12 +36,17 @@ */ public class JdbcResponseFormatter extends JsonResponseFormatter { - public JdbcResponseFormatter(Style style) { + private final String datetimeFormat; + + public JdbcResponseFormatter(Style style, String datetimeFormat) { super(style); + this.datetimeFormat = datetimeFormat; } @Override protected Object buildJsonObject(QueryResult response) { + response.setDatetimeFormat(datetimeFormat); + JdbcResponse.JdbcResponseBuilder json = JdbcResponse.builder(); // Fetch schema and data rows diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java index 7f024c1028..b5b0a6be89 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java @@ -42,6 +42,7 @@ public class SQLQueryRequest { private static final Set SUPPORTED_FIELDS = ImmutableSet.of( "query", "fetch_size", "parameters"); private static final String QUERY_PARAMS_FORMAT = "format"; + private static final String QUERY_PARAMS_DATETIMEFORMAT = "datetime_format"; private static final String QUERY_PARAMS_SANITIZE = "sanitize"; /** @@ -142,6 +143,10 @@ private String getFormat(Map params) { return "jdbc"; } + public String getDatetimeFormat() { + return params.getOrDefault(QUERY_PARAMS_DATETIMEFORMAT, "always_include_time"); + } + private boolean shouldSanitize(Map params) { if (params.containsKey(QUERY_PARAMS_SANITIZE)) { return Boolean.parseBoolean(params.get(QUERY_PARAMS_SANITIZE)); From 209664b7b303fe19d84f2f19f4110a066dfb0308 Mon Sep 17 00:00:00 2001 From: cip999 Date: Wed, 18 Aug 2021 16:02:47 +0200 Subject: [PATCH 23/37] Added "datetime_format" URL parameter to handle jdbj date and time formatting with new engine --- .../sql/sql/JdbcFormatIT.java | 16 ++++++++++++++-- integ-test/src/test/resources/datetime.json | 4 ++-- .../date_time_index_mapping.json | 2 +- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/JdbcFormatIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/JdbcFormatIT.java index 836cb5636c..b38b7b379f 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/JdbcFormatIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/JdbcFormatIT.java @@ -17,8 +17,8 @@ package com.amazon.opendistroforelasticsearch.sql.sql; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.*; import com.amazon.opendistroforelasticsearch.sql.legacy.SQLIntegTestCase; import org.json.JSONObject; @@ -29,6 +29,7 @@ public class JdbcFormatIT extends SQLIntegTestCase { @Override protected void init() throws Exception { loadIndex(Index.BANK); + loadIndex(Index.DATETIME); } @Test @@ -55,4 +56,15 @@ public void testAliasInSchema() { verifySchema(response, schema("account_number", "acc", "long")); } + @Test + public void testDatetimeFormat() { + String response = executeQuery( + "SELECT login_time FROM " + TEST_INDEX_DATE_TIME, + "jdbc", "include_time_when_nonzero"); + + assertTrue(response.contains("\"2015-01-01\"")); + assertTrue(response.contains("\"2015-01-01 12:10:30\"")); + assertTrue(response.contains("\"2020-04-08 06:10:30\"")); + } + } diff --git a/integ-test/src/test/resources/datetime.json b/integ-test/src/test/resources/datetime.json index 3898da61a7..8f7e5dbfbf 100644 --- a/integ-test/src/test/resources/datetime.json +++ b/integ-test/src/test/resources/datetime.json @@ -2,7 +2,7 @@ {"login_time":"2015-01-01"} {"index":{"_id":"2"}} {"login_time":"2015-01-01T12:10:30Z"} -{"index":{"_id":"3"}} -{"login_time":"1585882955"} +//{"index":{"_id":"3"}} +//{"login_time":"1585882955"} {"index":{"_id":"4"}} {"login_time":"2020-04-08T11:10:30+05:00"} diff --git a/integ-test/src/test/resources/indexDefinitions/date_time_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/date_time_index_mapping.json index 6f1d465a82..a0631a1260 100644 --- a/integ-test/src/test/resources/indexDefinitions/date_time_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/date_time_index_mapping.json @@ -1,7 +1,7 @@ { "mappings": { "properties": { - "birthday": { + "login_time": { "type": "date" } } From b412e9abe7cb21c6b447d9edf05881e2b41b4b82 Mon Sep 17 00:00:00 2001 From: cip999 Date: Wed, 18 Aug 2021 16:52:09 +0200 Subject: [PATCH 24/37] Added "datetime_format" URL parameter to handle jdbj date and time formatting with new engine --- .../sql/data/model/ExprTimestampValue.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java index 30c85ef0ac..4f2a12062f 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java @@ -99,8 +99,8 @@ public String value() { case INCLUDE_TIME_WHEN_NONZERO: LocalTime time = timeValue(); - return (time.getHour() == 0 && time.getMinute() == 0 && time.getSecond() == 0) ? - valueWithoutTime() : valueWithTime(); + return (time.getHour() == 0 && time.getMinute() == 0 && time.getSecond() == 0) + ? valueWithoutTime() : valueWithTime(); case ALWAYS_INCLUDE_TIME: default: @@ -163,5 +163,8 @@ public int hashCode() { return Objects.hashCode(timestamp); } - public void setDatetimeFormat(String format) { this.datetimeFormat = format; } + public void setDatetimeFormat(String format) { + this.datetimeFormat = format; + } + } From fdfec15cbd9ec30ff8dab6559b960ad58846c2de Mon Sep 17 00:00:00 2001 From: cip999 Date: Wed, 18 Aug 2021 17:12:46 +0200 Subject: [PATCH 25/37] Added "datetime_format" URL parameter to handle jdbj date and time formatting with new engine --- .../sql/sql/JdbcFormatIT.java | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/JdbcFormatIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/JdbcFormatIT.java index b38b7b379f..e3d3066a5b 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/JdbcFormatIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/JdbcFormatIT.java @@ -57,7 +57,29 @@ public void testAliasInSchema() { } @Test - public void testDatetimeFormat() { + public void testDatetimeFormat1() { + String response = executeQuery( + "SELECT login_time FROM " + TEST_INDEX_DATE_TIME, + "jdbc", "always_include_time"); + + assertTrue(response.contains("\"2015-01-01 00:00:00\"")); + assertTrue(response.contains("\"2015-01-01 12:10:30\"")); + assertTrue(response.contains("\"2020-04-08 06:10:30\"")); + } + + @Test + public void testDatetimeFormat2() { + String response = executeQuery( + "SELECT login_time FROM " + TEST_INDEX_DATE_TIME, + "jdbc", "never_include_time"); + + assertTrue(response.contains("\"2015-01-01\"")); + assertFalse(response.contains("\"2015-01-01 12:10:30\"")); + assertTrue(response.contains("\"2020-04-08\"")); + } + + @Test + public void testDatetimeFormat3() { String response = executeQuery( "SELECT login_time FROM " + TEST_INDEX_DATE_TIME, "jdbc", "include_time_when_nonzero"); From 39380c2e005dee3a8459ad7c28aaef0f37427045 Mon Sep 17 00:00:00 2001 From: cip999 Date: Fri, 20 Aug 2021 14:40:28 +0200 Subject: [PATCH 26/37] Added "datetime_format" URL parameter to handle jdbc date and time formatting with new engine --- .../sql/data/model/ExprTimestampValue.java | 8 +++---- .../sql/data/model/DateTimeValueTest.java | 23 +++++++++++++++++++ .../format/JdbcResponseFormatterTest.java | 3 ++- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java index 4f2a12062f..a3fae52453 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java @@ -58,9 +58,9 @@ public class ExprTimestampValue extends AbstractExprValue { private static final int MIN_FRACTION_SECONDS = 0; private static final int MAX_FRACTION_SECONDS = 6; - private static final String ALWAYS_INCLUDE_TIME = "always_include_time"; - private static final String NEVER_INCLUDE_TIME = "never_include_time"; - private static final String INCLUDE_TIME_WHEN_NONZERO = "include_time_when_nonzero"; + public static final String ALWAYS_INCLUDE_TIME = "always_include_time"; + public static final String NEVER_INCLUDE_TIME = "never_include_time"; + public static final String INCLUDE_TIME_WHEN_NONZERO = "include_time_when_nonzero"; static { FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder() @@ -99,7 +99,7 @@ public String value() { case INCLUDE_TIME_WHEN_NONZERO: LocalTime time = timeValue(); - return (time.getHour() == 0 && time.getMinute() == 0 && time.getSecond() == 0) + return (time.getHour() + time.getMinute() + time.getSecond() == 0) ? valueWithoutTime() : valueWithTime(); case ALWAYS_INCLUDE_TIME: diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java index dd136b8c76..dfd80ad2ed 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java @@ -17,6 +17,8 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue.INCLUDE_TIME_WHEN_NONZERO; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue.NEVER_INCLUDE_TIME; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIME; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP; @@ -255,4 +257,25 @@ public void timeOverMaxMicroPrecision() { "time:01:01:01.1234567891 in unsupported format, please use HH:mm:ss[.SSSSSS]", exception.getMessage()); } + + @Test + public void timestampValueNeverIncludeTime() { + ExprTimestampValue timestampValue = new ExprTimestampValue("2020-07-07 01:01:01"); + timestampValue.setDatetimeFormat(NEVER_INCLUDE_TIME); + + assertEquals("2020-07-07", timestampValue.value()); + } + + @Test + public void timestampValueIncludeTimeWhenNonzero() { + ExprTimestampValue firstTimestampValue = new ExprTimestampValue("2020-07-07 00:00:01"); + firstTimestampValue.setDatetimeFormat(INCLUDE_TIME_WHEN_NONZERO); + + ExprTimestampValue secondTimestampValue = new ExprTimestampValue("2020-07-07 00:00:00.291000"); + secondTimestampValue.setDatetimeFormat(INCLUDE_TIME_WHEN_NONZERO); + + assertEquals("2020-07-07 00:00:01", firstTimestampValue.value()); + assertEquals("2020-07-07", secondTimestampValue.value()); + } + } diff --git a/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatterTest.java b/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatterTest.java index e421016c00..cad2abda48 100644 --- a/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatterTest.java +++ b/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatterTest.java @@ -24,6 +24,7 @@ import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue.ALWAYS_INCLUDE_TIME; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT_KEYWORD; import static com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.Schema; @@ -47,7 +48,7 @@ @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) class JdbcResponseFormatterTest { - private final JdbcResponseFormatter formatter = new JdbcResponseFormatter(COMPACT); + private final JdbcResponseFormatter formatter = new JdbcResponseFormatter(COMPACT, ALWAYS_INCLUDE_TIME); @Test void format_response() { From 60da9ed76b8edc4f270e216a9a650921f2a525d5 Mon Sep 17 00:00:00 2001 From: cip999 Date: Fri, 20 Aug 2021 14:59:30 +0200 Subject: [PATCH 27/37] Added "datetime_format" URL parameter to handle jdbc date and time formatting with new engine --- .../sql/sql/JdbcFormatIT.java | 34 ------------------- integ-test/src/test/resources/datetime.json | 4 +-- 2 files changed, 2 insertions(+), 36 deletions(-) diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/JdbcFormatIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/JdbcFormatIT.java index e3d3066a5b..1515c65d46 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/JdbcFormatIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/JdbcFormatIT.java @@ -17,7 +17,6 @@ package com.amazon.opendistroforelasticsearch.sql.sql; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; -import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME; import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.*; import com.amazon.opendistroforelasticsearch.sql.legacy.SQLIntegTestCase; @@ -56,37 +55,4 @@ public void testAliasInSchema() { verifySchema(response, schema("account_number", "acc", "long")); } - @Test - public void testDatetimeFormat1() { - String response = executeQuery( - "SELECT login_time FROM " + TEST_INDEX_DATE_TIME, - "jdbc", "always_include_time"); - - assertTrue(response.contains("\"2015-01-01 00:00:00\"")); - assertTrue(response.contains("\"2015-01-01 12:10:30\"")); - assertTrue(response.contains("\"2020-04-08 06:10:30\"")); - } - - @Test - public void testDatetimeFormat2() { - String response = executeQuery( - "SELECT login_time FROM " + TEST_INDEX_DATE_TIME, - "jdbc", "never_include_time"); - - assertTrue(response.contains("\"2015-01-01\"")); - assertFalse(response.contains("\"2015-01-01 12:10:30\"")); - assertTrue(response.contains("\"2020-04-08\"")); - } - - @Test - public void testDatetimeFormat3() { - String response = executeQuery( - "SELECT login_time FROM " + TEST_INDEX_DATE_TIME, - "jdbc", "include_time_when_nonzero"); - - assertTrue(response.contains("\"2015-01-01\"")); - assertTrue(response.contains("\"2015-01-01 12:10:30\"")); - assertTrue(response.contains("\"2020-04-08 06:10:30\"")); - } - } diff --git a/integ-test/src/test/resources/datetime.json b/integ-test/src/test/resources/datetime.json index 8f7e5dbfbf..3898da61a7 100644 --- a/integ-test/src/test/resources/datetime.json +++ b/integ-test/src/test/resources/datetime.json @@ -2,7 +2,7 @@ {"login_time":"2015-01-01"} {"index":{"_id":"2"}} {"login_time":"2015-01-01T12:10:30Z"} -//{"index":{"_id":"3"}} -//{"login_time":"1585882955"} +{"index":{"_id":"3"}} +{"login_time":"1585882955"} {"index":{"_id":"4"}} {"login_time":"2020-04-08T11:10:30+05:00"} From ea616d882a0a47d86d6a2316074e465975d45aaf Mon Sep 17 00:00:00 2001 From: cip999 Date: Fri, 20 Aug 2021 15:32:44 +0200 Subject: [PATCH 28/37] Added "datetime_format" URL parameter to handle jdbc date and time formatting with new engine --- .../sql/data/model/ExprTimestampValue.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java index a3fae52453..77d04b97cd 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java @@ -52,7 +52,7 @@ public class ExprTimestampValue extends AbstractExprValue { .ofPattern("yyyy-MM-dd"); private final Instant timestamp; - private String datetimeFormat; + private String datetimeFormat = ALWAYS_INCLUDE_TIME; private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS; private static final int MIN_FRACTION_SECONDS = 0; @@ -77,8 +77,6 @@ public class ExprTimestampValue extends AbstractExprValue { * Constructor. */ public ExprTimestampValue(String timestamp) { - this.datetimeFormat = ALWAYS_INCLUDE_TIME; - try { this.timestamp = LocalDateTime.parse(timestamp, ExprDateFormatters.TOLERANT_PARSER_DATE_TIME_FORMATTER) From b6b925db86c0e488980ea398d85f86f15da74d8f Mon Sep 17 00:00:00 2001 From: cip999 Date: Fri, 20 Aug 2021 15:44:39 +0200 Subject: [PATCH 29/37] Added "datetime_format" URL parameter to handle jdbc date and time formatting with new engine --- .../sql/protocol/response/QueryResult.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java index 8261f15ae0..39b5441e07 100644 --- a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java +++ b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java @@ -79,10 +79,11 @@ public Iterator iterator() { public void setDatetimeFormat(String datetimeFormat) { for (ExprValue exprValue : exprValues) { - for (ExprValue v : ExprValueUtils.getTupleValue(exprValue).values()) + for (ExprValue v : ExprValueUtils.getTupleValue(exprValue).values()) { if (v.type() == ExprCoreType.TIMESTAMP) { ((ExprTimestampValue) v).setDatetimeFormat(datetimeFormat); } + } } } From 8fb065c86d27a333d07b80e30dacdec05872de08 Mon Sep 17 00:00:00 2001 From: cip999 Date: Fri, 20 Aug 2021 15:59:56 +0200 Subject: [PATCH 30/37] Added "datetime_format" URL parameter to handle jdbc date and time formatting with new engine --- .../sql/protocol/response/QueryResult.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java index 39b5441e07..67cb173638 100644 --- a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java +++ b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java @@ -77,6 +77,9 @@ public Iterator iterator() { .iterator(); } + /** + * Sets the date and time format for all inner elements of exprValues. + */ public void setDatetimeFormat(String datetimeFormat) { for (ExprValue exprValue : exprValues) { for (ExprValue v : ExprValueUtils.getTupleValue(exprValue).values()) { From 8c99a03335cc8d89a4002c29a41250a9cfd8cd16 Mon Sep 17 00:00:00 2001 From: cip999 Date: Fri, 20 Aug 2021 16:14:15 +0200 Subject: [PATCH 31/37] Added "datetime_format" URL parameter to handle jdbc date and time formatting with new engine --- .../protocol/response/format/JdbcResponseFormatterTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatterTest.java b/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatterTest.java index cad2abda48..7aef62e167 100644 --- a/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatterTest.java +++ b/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatterTest.java @@ -16,6 +16,7 @@ package com.amazon.opendistroforelasticsearch.sql.protocol.response.format; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue.ALWAYS_INCLUDE_TIME; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.stringValue; @@ -24,7 +25,6 @@ import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; -import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue.ALWAYS_INCLUDE_TIME; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT_KEYWORD; import static com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.Schema; @@ -48,7 +48,8 @@ @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) class JdbcResponseFormatterTest { - private final JdbcResponseFormatter formatter = new JdbcResponseFormatter(COMPACT, ALWAYS_INCLUDE_TIME); + private final JdbcResponseFormatter formatter = new JdbcResponseFormatter(COMPACT, + ALWAYS_INCLUDE_TIME); @Test void format_response() { From e892bda57a39bbb40abdc58d241a0162e067510d Mon Sep 17 00:00:00 2001 From: cip999 Date: Fri, 20 Aug 2021 17:31:17 +0200 Subject: [PATCH 32/37] Added "datetime_format" URL parameter to handle jdbc date and time formatting with new engine --- .../protocol/response/QueryResultTest.java | 44 +++++++++++++++++++ .../sql/sql/domain/SQLQueryRequestTest.java | 14 ++++++ 2 files changed, 58 insertions(+) diff --git a/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResultTest.java b/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResultTest.java index 5373650b25..945603e463 100644 --- a/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResultTest.java +++ b/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResultTest.java @@ -16,18 +16,26 @@ package com.amazon.opendistroforelasticsearch.sql.protocol.response; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue.INCLUDE_TIME_WHEN_NONZERO; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.tupleValue; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashMap; + import org.junit.jupiter.api.Test; class QueryResultTest { @@ -36,6 +44,10 @@ class QueryResultTest { new ExecutionEngine.Schema.Column("name", null, STRING), new ExecutionEngine.Schema.Column("age", null, INTEGER))); + private ExecutionEngine.Schema schemaTimestamp = new ExecutionEngine.Schema(ImmutableList.of( + new ExecutionEngine.Schema.Column("name", null, STRING), + new ExecutionEngine.Schema.Column("birthdate", null, TIMESTAMP))); + @Test void size() { @@ -125,4 +137,36 @@ void iterate() { } } + @Test + void setDateTimeFormatTest() { + LinkedHashMap valueMap1 = new LinkedHashMap<>(); + valueMap1.put("name", ExprValueUtils.fromObjectValue("John")); + valueMap1.put("birthdate", ExprValueUtils.fromObjectValue("1987-10-23 00:00:00", TIMESTAMP)); + + LinkedHashMap valueMap2 = new LinkedHashMap<>(); + valueMap2.put("name", ExprValueUtils.fromObjectValue("Andrea")); + valueMap2.put("birthdate", ExprValueUtils.fromObjectValue("1999-03-14 03:30:00", TIMESTAMP)); + + QueryResult response = new QueryResult( + schemaTimestamp, + Arrays.asList( + new ExprTupleValue(valueMap1), + new ExprTupleValue(valueMap2) + )); + + response.setDatetimeFormat(INCLUDE_TIME_WHEN_NONZERO); + + int i = 0; + for (Object[] objects : response) { + if (i == 0) { + assertArrayEquals(new Object[] {"John", "1987-10-23"}, objects); + } else if (i == 1) { + assertArrayEquals(new Object[] {"Andrea", "1999-03-14 03:30:00"}, objects); + } else { + fail("More rows returned than expected"); + } + i++; + } + } + } \ No newline at end of file diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java index b7ea848224..5d16757dad 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java @@ -23,6 +23,8 @@ import com.amazon.opendistroforelasticsearch.sql.protocol.response.format.Format; import com.google.common.collect.ImmutableMap; + +import java.util.HashMap; import java.util.Map; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -44,6 +46,18 @@ public void shouldSupportQueryWithJDBCFormat() { assertEquals(request.format(), Format.JDBC); } + @Test + public void shouldSupportQueryDatetimeFormat() { + Map params = new HashMap<>(); + params.put("format", "jdbc"); + params.put("datetime_format", "never_include_time"); + + SQLQueryRequest request = SQLQueryRequestBuilder.request("SELECT 1") + .params(params) + .build(); + assertEquals("never_include_time", request.getDatetimeFormat()); + } + @Test public void shouldSupportQueryWithQueryFieldOnly() { SQLQueryRequest request = From ccbbe4934f5aa0191e9aa1d5b78ee2d28710b348 Mon Sep 17 00:00:00 2001 From: cip999 Date: Tue, 24 Aug 2021 23:52:12 +0200 Subject: [PATCH 33/37] Massive refactoring of modules handling SQL functions to allow for nesting --- .../sql/legacy/SQLFunctionsIT.java | 15 +- .../types/function/ScalarFunction.java | 6 +- .../sql/legacy/parser/FieldMaker.java | 90 ++-- .../sql/legacy/utils/SQLFunctions.java | 458 +++++++++--------- .../sql/legacy/utils/Util.java | 6 +- .../legacy/unittest/DateFunctionsTest.java | 3 +- .../legacy/unittest/MathFunctionsTest.java | 52 +- .../legacy/unittest/QueryFunctionsTest.java | 10 +- .../legacy/unittest/StringOperatorsTest.java | 36 +- .../legacy/unittest/parser/SqlParserTest.java | 50 +- .../unittest/utils/SQLFunctionsTest.java | 8 +- 11 files changed, 382 insertions(+), 352 deletions(-) diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java index 37a447dbbb..8f6b828fbd 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java @@ -890,7 +890,20 @@ public void greatestWithAliasAndStrings() throws Exception { rows("duke"), rows("hattie"), rows("nanette")); - } + } + + @Test + public void complexNestedFunctions() throws Exception { + JSONObject response = + executeJdbcRequest("SELECT if(greatest(account_number / 2, age - 2) = 34, " + + "concat_ws(' ', firstname, lastname), 0) " + + "FROM " + TEST_INDEX_ACCOUNT + " LIMIT 3"); + + verifyDataRows(response, + rows(0), + rows("hattie bond"), + rows(0)); + } private SearchHits query(String query) throws IOException { final String rsp = executeQueryWithStringOutput(query); diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ScalarFunction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ScalarFunction.java index 1f74fbb147..997b052be8 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ScalarFunction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ScalarFunction.java @@ -50,8 +50,10 @@ public enum ScalarFunction implements TypeExpression { CURDATE(func().to(ESDataType.DATE)), DATE(func(ESDataType.DATE).to(ESDataType.DATE)), DATE_FORMAT( - func(ESDataType.DATE, STRING).to(STRING), - func(ESDataType.DATE, STRING, STRING).to(STRING) + func(ESDataType.DATE, STRING).to(STRING), + func(ESDataType.DATE, STRING, STRING).to(STRING), + func(T(STRING), STRING).to(STRING), + func(T(STRING), STRING, STRING).to(STRING) ), DAYOFMONTH(func(ESDataType.DATE).to(INTEGER)), DEGREES(func(T(NUMBER)).to(DOUBLE)), diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/FieldMaker.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/FieldMaker.java index f56b0126c6..378319bcd6 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/FieldMaker.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/FieldMaker.java @@ -18,19 +18,7 @@ import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLObject; import com.alibaba.druid.sql.ast.SQLSetQuantifier; -import com.alibaba.druid.sql.ast.expr.SQLAggregateExpr; -import com.alibaba.druid.sql.ast.expr.SQLAggregateOption; -import com.alibaba.druid.sql.ast.expr.SQLAllColumnExpr; -import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; -import com.alibaba.druid.sql.ast.expr.SQLCaseExpr; -import com.alibaba.druid.sql.ast.expr.SQLCastExpr; -import com.alibaba.druid.sql.ast.expr.SQLCharExpr; -import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; -import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr; -import com.alibaba.druid.sql.ast.expr.SQLNumericLiteralExpr; -import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; -import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; -import com.alibaba.druid.sql.ast.expr.SQLVariantRefExpr; +import com.alibaba.druid.sql.ast.expr.*; import com.alibaba.druid.sql.ast.statement.SQLSelectGroupByClause; import com.alibaba.druid.sql.ast.statement.SQLSelectItem; import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock; @@ -41,6 +29,7 @@ import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlFeatureNotImplementedException; import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.SQLDefinitionExpr; import com.amazon.opendistroforelasticsearch.sql.legacy.utils.SQLFunctions; import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import com.google.common.base.Strings; @@ -127,7 +116,7 @@ private Field makeFieldImpl(SQLExpr expr, String alias, String tableAlias) throw return makeMethodField(methodInvokeExpr.getMethodName(), methodInvokeExpr.getParameters(), null, alias, tableAlias, true); } else { - throw new SqlParseException("unknown field name : " + expr); + throw new SqlParseException("unknown field name: " + expr); } } @@ -165,15 +154,48 @@ private Field makeScriptMethodField(SQLBinaryOpExpr binaryExpr, String alias, St } params.add(new SQLCharExpr(scriptFieldAlias)); - Object left = getScriptValue(binaryExpr.getLeft()); - Object right = getScriptValue(binaryExpr.getRight()); - String script = String.format("%s %s %s", left, binaryExpr.getOperator().getName(), right); + Tuple left = processOperand(binaryExpr.getLeft(), tableAlias); + Tuple right = processOperand(binaryExpr.getRight(), tableAlias); + + String operator = binaryExpr.getOperator().getName(); + if (operator.equals("=")) { + operator = "=="; + } + + String script = String.format("%s %s %s", left.v1(), operator, right.v1()); params.add(new SQLCharExpr(script)); + params.add(left.v2()); + params.add(right.v2()); return makeMethodField("script", params, null, null, tableAlias, false); } + private Tuple processOperand(SQLExpr expr, String tableAlias) + throws SqlParseException { + if (expr instanceof SQLBinaryOpExpr) { + SQLBinaryOpExpr binaryOpExpr = (SQLBinaryOpExpr) expr; + if (SQLFunctions.isFunctionTranslatedToScript(binaryOpExpr.getOperator().toString())) { + SQLMethodInvokeExpr mExpr = makeBinaryMethodField(binaryOpExpr, null, false); + MethodField mField = makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), + null, null, tableAlias, false); + String varName = mField.getParams().get(0).toString(); + return new Tuple<>(varName, new SQLDefinitionExpr(varName, mField.getParams().get(1).toString())); + } else { + throw new SqlParseException("Invalid operand: " + + "valuable expression expected, found " + + binaryOpExpr.toString()); + } + } else if (expr instanceof SQLMethodInvokeExpr) { + SQLMethodInvokeExpr mExpr = (SQLMethodInvokeExpr) expr; + MethodField mField = makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), + null, null, tableAlias, false); + String varName = mField.getParams().get(0).toString(); + return new Tuple<>(varName, new SQLDefinitionExpr(varName, mField.getParams().get(1).toString())); + } else { + return new Tuple<>(getScriptValue(expr).toString(), new SQLNullExpr()); + } + } private static Field makeFilterMethodField(SQLMethodInvokeExpr filterMethod, String alias) throws SqlParseException { @@ -292,26 +314,22 @@ public MethodField makeMethodField(String name, List arguments, SQLAggr if (SQLFunctions.isFunctionTranslatedToScript(binaryOpExpr.getOperator().toString())) { SQLMethodInvokeExpr mExpr = makeBinaryMethodField(binaryOpExpr, alias, first); - MethodField abc = makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), + MethodField mField = makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), null, null, tableAlias, false); - paramers.add(new KVValue(abc.getParams().get(0).toString(), - new SQLCharExpr(abc.getParams().get(1).toString()))); + String varName = mField.getParams().get(0).toString(); + paramers.add(new KVValue(varName, + new SQLDefinitionExpr(varName, mField.getParams().get(1).toString()) + )); } else { - if (!binaryOpExpr.getOperator().getName().equals("=")) { - paramers.add(new KVValue("script", makeScriptMethodField(binaryOpExpr, null, tableAlias))); - } else { - SQLExpr right = binaryOpExpr.getRight(); - Object value = Util.expr2Object(right); - paramers.add(new KVValue(binaryOpExpr.getLeft().toString(), value)); - } + paramers.add(new KVValue("script", makeScriptMethodField(binaryOpExpr, null, tableAlias))); } } else if (object instanceof SQLMethodInvokeExpr) { SQLMethodInvokeExpr mExpr = (SQLMethodInvokeExpr) object; String methodName = mExpr.getMethodName().toLowerCase(); if (methodName.equals("script")) { - KVValue script = new KVValue("script", makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), - null, alias, tableAlias, true)); + KVValue script = new KVValue("script", makeMethodField(mExpr.getMethodName(), + mExpr.getParameters(), null, alias, tableAlias, true)); paramers.add(script); } else if (methodName.equals("nested") || methodName.equals("reverse_nested")) { NestedType nestedType = new NestedType(); @@ -332,19 +350,21 @@ public MethodField makeMethodField(String name, List arguments, SQLAggr paramers.add(new KVValue("children", childrenType)); } else if (SQLFunctions.isFunctionTranslatedToScript(methodName)) { //throw new SqlParseException("only support script/nested as inner functions"); - MethodField abc = makeMethodField(methodName, mExpr.getParameters(), null, null, tableAlias, false); - paramers.add(new KVValue(abc.getParams().get(0).toString(), - new SQLCharExpr(abc.getParams().get(1).toString()))); + MethodField mField = makeMethodField(methodName, mExpr.getParameters(), + null, null, tableAlias, false); + String varName = mField.getParams().get(0).toString(); + paramers.add(new KVValue(varName, + new SQLDefinitionExpr(varName, mField.getParams().get(1).toString()))); } else { throw new SqlParseException("only support script/nested/children as inner functions"); } } else if (object instanceof SQLCaseExpr) { String scriptCode = new CaseWhenParser((SQLCaseExpr) object, alias, tableAlias).parse(); - paramers.add(new KVValue("script", new SQLCharExpr(scriptCode))); + paramers.add(new KVValue("script", new SQLDefinitionExpr("script", scriptCode))); } else if (object instanceof SQLCastExpr) { String castName = sqlFunctions.nextId("cast"); List methodParameters = new ArrayList<>(); - methodParameters.add(new KVValue(((SQLCastExpr) object).getExpr().toString())); + methodParameters.add(new KVValue(((SQLCastExpr) object).getExpr())); String castType = ((SQLCastExpr) object).getDataType().getName(); String scriptCode = sqlFunctions.getCastScriptStatement(castName, castType, methodParameters); @@ -356,7 +376,7 @@ public MethodField makeMethodField(String name, List arguments, SQLAggr scriptCode += "; return " + castName; } methodParameters.add(new KVValue(scriptCode)); - paramers.add(new KVValue("script", new SQLCharExpr(scriptCode))); + paramers.add(new KVValue("script", new SQLDefinitionExpr("script", scriptCode))); } else if (object instanceof SQLAggregateExpr) { SQLObject parent = object.getParent(); SQLExpr source = (SQLExpr) parent.getAttribute("source"); diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java index e036f160f9..1f6a5f96a7 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java @@ -38,11 +38,8 @@ import com.google.common.collect.Sets; import org.elasticsearch.common.collect.Tuple; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; +import java.lang.reflect.Method; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -174,8 +171,7 @@ public Tuple function(String methodName, List paramers, functionStr = date_format( (SQLExpr) paramers.get(0).value, Util.expr2Object((SQLExpr) paramers.get(1).value).toString(), - paramers.size() > 2 ? Util.expr2Object((SQLExpr) paramers.get(2).value).toString() : null, - name); + paramers.size() < 3 ? null : Util.expr2Object((SQLExpr) paramers.get(2).value).toString()); break; case "year": @@ -260,7 +256,7 @@ public Tuple function(String methodName, List paramers, case "sinh": case "cosh": functionStr = mathSingleValueTemplate("Math." + methodName, methodName, - (SQLExpr) paramers.get(0).value, name); + (SQLExpr) paramers.get(0).value); break; case "rand": @@ -274,22 +270,22 @@ public Tuple function(String methodName, List paramers, case "cot": // ES does not support the function name cot functionStr = mathSingleValueTemplate("1 / Math.tan", methodName, - (SQLExpr) paramers.get(0).value, name); + (SQLExpr) paramers.get(0).value); break; case "sign": case "signum": methodName = "signum"; functionStr = mathSingleValueTemplate("Math." + methodName, methodName, - (SQLExpr) paramers.get(0).value, name); + (SQLExpr) paramers.get(0).value); break; case "pow": case "power": methodName = "pow"; functionStr = mathDoubleValueTemplate("Math." + methodName, methodName, - (SQLExpr) paramers.get(0).value, Util.expr2Object((SQLExpr) paramers.get(1).value).toString(), - name); + (SQLExpr) paramers.get(0).value, + Util.expr2Object((SQLExpr) paramers.get(1).value).toString()); break; case "atan2": @@ -304,14 +300,14 @@ public Tuple function(String methodName, List paramers, break; case "degrees": - functionStr = degrees((SQLExpr) paramers.get(0).value, name); + functionStr = degrees((SQLExpr) paramers.get(0).value); break; case "radians": - functionStr = radians((SQLExpr) paramers.get(0).value, name); + functionStr = radians((SQLExpr) paramers.get(0).value); break; case "trim": - functionStr = trim((SQLExpr) paramers.get(0).value, name); + functionStr = trim((SQLExpr) paramers.get(0).value); break; case "add": @@ -344,14 +340,14 @@ public Tuple function(String methodName, List paramers, break; case "log2": - functionStr = log(SQLUtils.toSQLExpr("2"), (SQLExpr) paramers.get(0).value, name); + functionStr = log(SQLUtils.toSQLExpr("2"), (SQLExpr) paramers.get(0).value); break; case "log10": functionStr = log10((SQLExpr) paramers.get(0).value); break; case "log": if (paramers.size() > 1) { - functionStr = log((SQLExpr) paramers.get(0).value, (SQLExpr) paramers.get(1).value, name); + functionStr = log((SQLExpr) paramers.get(0).value, (SQLExpr) paramers.get(1).value); } else { functionStr = ln((SQLExpr) paramers.get(0).value); } @@ -433,6 +429,8 @@ public Tuple cast(String castType, List paramers) throw public Tuple upper(SQLExpr field, String locale, String valueName) { String name = nextId("upper"); + scriptDeclare(field); + if (valueName == null) { return new Tuple<>(name, def(name, upper(getPropertyOrStringValue(field), locale))); } else { @@ -471,9 +469,9 @@ private static String exprString(SQLExpr expr) { private static String func(String methodName, boolean quotes, String... params) { if (quotes) { return methodName + "(" + quoteParams(params) + ")"; + } else { + return methodName + "(" + String.join(", ", params) + ")"; } - - return methodName + "(" + String.join(", ", params) + ")"; } /** @@ -488,16 +486,9 @@ private Tuple concat_ws(String split, List columns) { List result = Lists.newArrayList(); for (SQLExpr column : columns) { - String strColumn = exprString(column); - if (strColumn.startsWith("def ")) { - result.add(strColumn); - } else if (isProperty(column)) { - result.add("doc['" + strColumn + "'].value"); - } else { - result.add("'" + strColumn + "'"); - } - + result.add(scriptDeclare(column) + getPropertyOrStringValue(column)); } + return new Tuple<>(name, def(name, Joiner.on("+ " + split + " +").join(result))); } @@ -506,41 +497,75 @@ private Tuple concat_ws(String split, List columns) { public Tuple split(SQLExpr field, String pattern, int index, String valueName) { String name = nextId("split"); final String script; + if (valueName == null) { - script = def(name, - getPropertyOrValue(field) + "." + script = def(name, scriptDeclare(field) + + getPropertyOrStringValue(field) + "." + func("split", true, pattern) + "[" + index + "]"); } else { script = "; " + def(name, valueName + "." + func("split", true, pattern) + "[" + index + "]"); } + return new Tuple<>(name, script); } //split(Column expr, java.lang.String pattern) public Tuple split(SQLExpr field, String pattern, String valueName) { String name = nextId("split"); + if (valueName == null) { return new Tuple<>(name, - def(name, getPropertyOrValue(field) + "." + def(name, getPropertyOrStringValue(field) + "." + func("split", true, pattern))); } else { - return new Tuple<>(name, getPropertyOrValue(field) + "; " - + def(name, valueName + "." + func("split", true, pattern))); + return new Tuple<>(name, + "; " + def(name, valueName + "." + func("split", true, pattern))); } } - private Tuple date_format(SQLExpr field, String pattern, String zoneId, String valueName) { + private Tuple date_format(SQLExpr field, String pattern, String zoneId) { String name = nextId("date_format"); - if (valueName == null) { - return new Tuple<>(name, "def " + name + " = DateTimeFormatter.ofPattern('" + pattern + "').withZone(" - + (zoneId != null ? "ZoneId.of('" + zoneId + "')" : "ZoneId.of(\"UTC\")") - + ").format(Instant.ofEpochMilli(" + getPropertyOrValue(field) + ".toInstant().toEpochMilli()))"); - } else { - return new Tuple<>(name, exprString(field) + "; " - + "def " + name + " = new SimpleDateFormat('" + pattern + "').format(" - + "new Date(" + valueName + " - 8*1000*60*60))"); - } + String parsedDateName = nextId("parsed_date"); + + String value = getPropertyOrStringValue(field); + + String definition = scriptDeclare(field) + + "def formats = new ArrayList(); " + + "formats.add(\"yyyy-MM-dd\"); " + + "formats.add(\"yyyy-MM-dd HH:mm:ss\"); " + + "formats.add(\"yyyy-MM-dd HH:mm:ss.SSSSSS\"); " + + "formats.add(\"yyyy-MM-dd'T'HH:mm:ss.SSSSSS\"); " + + "formats.add(\"yyyy-MM-dd'T'HH:mm:ss'Z'\"); " + + "def " + parsedDateName + ";" + + "if (" + value + " instanceof String) {" + + "for (String format : formats) {" + + "try {" + parsedDateName + " = " + + "ZonedDateTime.parse(" + value + ", " + + "new DateTimeFormatterBuilder()" + + ".appendPattern(format)" + + ".parseDefaulting(ChronoField.HOUR_OF_DAY, 0)" + + ".parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)" + + ".parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)" + + ".parseDefaulting(ChronoField.NANO_OF_SECOND, 0)" + + ".toFormatter()" + + StringUtils.format(".withZone(ZoneId.of('%s')));} ", + zoneId != null ? zoneId : "UTC") + + "catch (DateTimeParseException e) {} " + + "if (" + parsedDateName + " != null) {break;} } " + + "if (" + parsedDateName + " == null) { " + + "throw new IllegalArgumentException(\"Date format not recognized\"); } } " + + "else {" + parsedDateName + " = " + value + ";} "; + + definition += def(name, StringUtils.format( + "DateTimeFormatter.ofPattern(\"%s\").withZone(%s).format" + + "(Instant.ofEpochMilli(%s.toInstant().toEpochMilli()))", + pattern, + zoneId != null ? "ZoneId.of('" + zoneId + "')" : "ZoneId.of('UTC')", + parsedDateName + )); + + return new Tuple<>(name, definition); } /** @@ -571,8 +596,8 @@ public Tuple add(SQLExpr a, SQLExpr b) { public Tuple assign(SQLExpr a) { String name = nextId("assign"); - return new Tuple<>(name, - def(name, extractName(a))); + return new Tuple<>(name, scriptDeclare(a) + + def(name, getPropertyOrValue(a))); } private Tuple modulus(SQLExpr a, SQLExpr b) { @@ -600,7 +625,8 @@ private Tuple binaryOpertator(String methodName, String operator String name = nextId(methodName); return new Tuple<>(name, scriptDeclare(a) + scriptDeclare(b) + convertType(a) + convertType(b) - + def(name, extractName(a) + " " + operator + " " + extractName(b))); + + def(name, + getPropertyOrValue(a) + " " + operator + " " + getPropertyOrValue(b))); } private Tuple greatest(SQLExpr a, SQLExpr b) { @@ -632,7 +658,8 @@ private Tuple comparisonFunctionTemplate(SQLExpr a, SQLExpr b, + String.format("%s = (%s.compareTo(%s) %s 0 ? %s : %s);", name, nameString_a, nameString_b, comparisonOperator, value_a, value_b); - String definition = "def " + name + ";" + String definition = scriptDeclare(a) + scriptDeclare(b) + + "def " + name + ";" + String.format("if (%s) {%s = %s;} ", checkIfNull(a), name, value_b) + String.format("else if (%s) {%s = %s;} ", checkIfNull(b), name, value_a) + "else {"; @@ -655,15 +682,18 @@ private Tuple comparisonFunctionTemplate(SQLExpr a, SQLExpr b, } private static boolean isProperty(SQLExpr expr) { - return (expr instanceof SQLIdentifierExpr || expr instanceof SQLPropertyExpr + return (expr instanceof SQLIdentifierExpr + || expr instanceof SQLPropertyExpr || expr instanceof SQLVariantRefExpr); } private static String getPropertyOrValue(SQLExpr expr) { - if (isProperty(expr)) { - return doc(expr) + ".value"; + if (expr instanceof SQLDefinitionExpr) { + return ((SQLDefinitionExpr) expr).getName(); } else if (expr instanceof SQLNullExpr) { return "null"; + } else if (isProperty(expr)) { + return StringUtils.format("(%s.size() == 0 ? null : %s.value)", doc(expr), doc(expr)); } else { return exprString(expr); } @@ -675,65 +705,50 @@ private static String getPropertyOrValue(String expr) { } else if (StringUtils.isNumeric(expr)) { return expr; } else { - return doc(expr) + ".value"; + return StringUtils.format("(%s.size() == 0 ? null : %s.value)", doc(expr), doc(expr)); } } private static String getPropertyOrStringValue(SQLExpr expr) { - if (isProperty(expr)) { - return doc(expr) + ".value"; - } else { + if (expr instanceof SQLTextLiteralExpr) { return "'" + exprString(expr) + "'"; - } - } - - private static String scriptDeclare(SQLExpr a) { - if (isProperty(a) || a instanceof SQLNumericLiteralExpr) { - return ""; } else { - return exprString(a) + ";"; + return getPropertyOrValue(expr); } } - private static String extractName(SQLExpr script) { - if (isProperty(script)) { - return doc(script) + ".value"; - } - String scriptStr = exprString(script); - String[] variance = scriptStr.split(";"); - String newScript = variance[variance.length - 1]; - if (newScript.trim().startsWith("def ")) { - //for now ,if variant is string,then change to double. - return newScript.trim().substring(4).split("=")[0].trim(); + private static String scriptDeclare(Object obj) { + if (obj instanceof SQLDefinitionExpr) { + return ((SQLDefinitionExpr) obj).getDefinition() + "; "; + } else if (obj instanceof MethodField) { + MethodField mField = (MethodField) obj; + SQLExpr left = (SQLExpr) mField.getParams().get(2).value; + SQLExpr right = (SQLExpr) mField.getParams().get(3).value; + return scriptDeclare(left) + scriptDeclare(right); } else { - return scriptStr; + return ""; } } //cast(year as int) private static String convertType(SQLExpr script) { - String[] variance = exprString(script).split(";"); - String newScript = variance[variance.length - 1]; - if (newScript.trim().startsWith("def ")) { - //for now ,if variant is string,then change to double. - String temp = newScript.trim().substring(4).split("=")[0].trim(); - - return " if( " + temp + " instanceof String) " + temp + "= Double.parseDouble(" + temp.trim() + "); "; - } else { + if (!(script instanceof SQLDefinitionExpr)) { return ""; + } else { + return StringUtils.format(" if(%1$s instanceof String) {%1$s = Double.parseDouble(%1$s);}", + ((SQLDefinitionExpr) script).getName()); } - - } - private static String checkIfNull(SQLExpr expr) { - if (isProperty(expr)) { - return doc(expr) + ".size() == 0 || " + getPropertyOrValue(expr) + " == null"; - } else if (expr instanceof SQLNullExpr) { + private static String checkIfNull(Object script) { + if (script instanceof SQLDefinitionExpr) { + return ((SQLDefinitionExpr) script).getName() + " == null"; + } else if (script instanceof SQLExpr && isProperty((SQLExpr) script)) { + return doc((SQLExpr) script) + ".size() == 0 || " + getPropertyOrValue((SQLExpr) script) + " == null"; + } else if (script instanceof SQLNullExpr) { return "0 == 0"; - } - else { + } else { return "0 == 1"; } } @@ -744,53 +759,48 @@ private static String convertToString(String name) { } private String getScriptText(MethodField field) { - String content = ((SQLTextLiteralExpr) field.getParams().get(1).value).getText(); - return content; + return ((SQLTextLiteralExpr) field.getParams().get(1).value).getText(); } /** * Using exprString() rather than getPropertyOrValue() for "base" since something like "Math.E" gets evaluated * incorrectly in getPropertyOrValue(), returning it as a doc value instead of the literal string */ - public Tuple log(SQLExpr base, SQLExpr field, String valueName) { + public Tuple log(SQLExpr base, SQLExpr field) { String name = nextId("log"); - String result; - if (valueName == null) { - result = def(name, func("Math.log", false, getPropertyOrValue(field)) - + "/" + func("Math.log", false, exprString(base))); - } else { - result = getPropertyOrValue(field) + "; " - + def(name, func("Math.log", false, valueName) + "/" - + func("Math.log", false, exprString(base))); - } - return new Tuple<>(name, result); + return new Tuple<>(name, scriptDeclare(base) + scriptDeclare(field) + + def(name, func("Math.log", false, getPropertyOrValue(field)) + + "/" + func("Math.log", false, getPropertyOrValue(base)))); } public Tuple log10(SQLExpr field) { String name = nextId("log10"); - return new Tuple<>(name, def(name, StringUtils.format("Math.log10(%s)", getPropertyOrValue(field)))); + return new Tuple<>(name, scriptDeclare(field) + + def(name, StringUtils.format("Math.log10(%s)", getPropertyOrValue(field)))); } public Tuple ln(SQLExpr field) { String name = nextId("ln"); - return new Tuple<>(name, def(name, StringUtils.format("Math.log(%s)", getPropertyOrValue(field)))); + return new Tuple<>(name, scriptDeclare(field) + + StringUtils.format("Math.log(%s)", getPropertyOrValue(field))); } - public Tuple trim(SQLExpr field, String valueName) { - return strSingleValueTemplate("trim", field, valueName); + public Tuple trim(SQLExpr field) { + return strSingleValueTemplate("trim", field); } - private Tuple degrees(SQLExpr field, String valueName) { - return mathSingleValueTemplate("Math.toDegrees", "degrees", field, valueName); + private Tuple degrees(SQLExpr field) { + return mathSingleValueTemplate("Math.toDegrees", "degrees", field); } - private Tuple radians(SQLExpr field, String valueName) { - return mathSingleValueTemplate("Math.toRadians", "radians", field, valueName); + private Tuple radians(SQLExpr field) { + return mathSingleValueTemplate("Math.toRadians", "radians", field); } - private Tuple rand(SQLExpr expr) { + private Tuple rand(SQLExpr field) { String name = nextId("rand"); - return new Tuple<>(name, def(name, format("new Random(%s).nextDouble()", getPropertyOrValue(expr)))); + return new Tuple<>(name, scriptDeclare(field) + + def(name, format("new Random(%s).nextDouble()", getPropertyOrValue(field)))); } private Tuple rand() { @@ -798,35 +808,24 @@ private Tuple rand() { return new Tuple<>(name, def(name, "new Random().nextDouble()")); } - private Tuple mathDoubleValueTemplate(String methodName, String fieldName, SQLExpr val1, - String val2, String valueName) { + private Tuple mathDoubleValueTemplate(String methodName, String fieldName, + SQLExpr val1, String val2) { String name = nextId(fieldName); - if (valueName == null) { - return new Tuple<>(name, def(name, func(methodName, false, getPropertyOrValue(val1), - getPropertyOrValue(val2)))); - } else { - return new Tuple<>(name, getPropertyOrValue(val1) + "; " - + def(name, func(methodName, false, valueName, getPropertyOrValue(val2)))); - } + return new Tuple<>(name, scriptDeclare(val1) + + def(name, func(methodName, false, getPropertyOrValue(val1), getPropertyOrValue(val2)))); } private Tuple mathDoubleValueTemplate(String methodName, String fieldName, SQLExpr val1, SQLExpr val2) { String name = nextId(fieldName); - return new Tuple<>(name, def(name, func(methodName, false, - getPropertyOrValue(val1), getPropertyOrValue(val2)))); + return new Tuple<>(name, scriptDeclare(val1) + scriptDeclare(val2) + + def(name, func(methodName, false, getPropertyOrValue(val1), getPropertyOrValue(val2)))); } - private Tuple mathSingleValueTemplate(String methodName, String fieldName, SQLExpr field, - String valueName) { + private Tuple mathSingleValueTemplate(String methodName, String fieldName, SQLExpr field) { String name = nextId(fieldName); - if (valueName == null) { - return new Tuple<>(name, def(name, func(methodName, false, getPropertyOrValue(field)))); - } else { - return new Tuple<>(name, getPropertyOrValue(field) + "; " - + def(name, func(methodName, false, valueName))); - } - + return new Tuple<>(name, scriptDeclare(field) + + def(name, func(methodName, false, getPropertyOrValue(field)))); } private Tuple mathConstantTemplate(String methodName, String fieldName) { @@ -834,15 +833,10 @@ private Tuple mathConstantTemplate(String methodName, String fie return new Tuple<>(name, def(name, methodName)); } - private Tuple strSingleValueTemplate(String methodName, SQLExpr field, String valueName) { + private Tuple strSingleValueTemplate(String methodName, SQLExpr field) { String name = nextId(methodName); - if (valueName == null) { - return new Tuple<>(name, def(name, getPropertyOrStringValue(field) + "." + func(methodName, false))); - } else { - return new Tuple<>(name, getPropertyOrStringValue(field) + "; " - + def(name, valueName + "." + func(methodName, false))); - } - + return new Tuple<>(name, scriptDeclare(field) + + def(name, getPropertyOrStringValue(field) + "." + func(methodName, false))); } // query: substring(Column expr, int pos, int len) @@ -852,11 +846,12 @@ public Tuple substring(SQLExpr field, int pos, int len) { String name = nextId("substring"); // start and end are 0-indexes int start = pos < 1 ? 0 : pos - 1; - return new Tuple<>(name, StringUtils.format( - "def end = (int) Math.min(%s + %s, %s.length()); " - + def(name, getPropertyOrStringValue(field) + "." - + func("substring", false, Integer.toString(start), "end")), - Integer.toString(start), Integer.toString(len), getPropertyOrStringValue(field) + return new Tuple<>(name, scriptDeclare(field) + + StringUtils.format( + "def end = (int) Math.min(%s + %s, %s.length()); " + + def(name, getPropertyOrStringValue(field) + "." + + func("substring", false, Integer.toString(start), "end")), + Integer.toString(start), Integer.toString(len), getPropertyOrStringValue(field) )); } @@ -870,12 +865,14 @@ private String upper(String property, String culture) { private Tuple length(SQLExpr field) { String name = nextId("length"); - return new Tuple<>(name, def(name, getPropertyOrStringValue(field) + ".length()")); + return new Tuple<>(name, scriptDeclare(field) + + def(name, getPropertyOrStringValue(field) + ".length()")); } private Tuple replace(SQLExpr field, String target, String replacement) { String name = nextId("replace"); - return new Tuple<>(name, def(name, getPropertyOrStringValue(field) + return new Tuple<>(name, scriptDeclare(field) + + def(name, getPropertyOrStringValue(field) + ".replace(" + target + "," + replacement + ")")); } @@ -885,167 +882,152 @@ private Tuple locate(String pattern, SQLExpr source, int start) String name = nextId("locate"); String docSource = getPropertyOrStringValue(source); start = start < 1 ? 0 : start - 1; - return new Tuple<>(name, def(name, StringUtils.format("%s.indexOf(%s,%d)+1", docSource, pattern, start))); + return new Tuple<>(name, scriptDeclare(source) + + def(name, StringUtils.format("%s.indexOf(%s, %d) + 1", docSource, pattern, start))); } private Tuple rtrim(SQLExpr field) { String name = nextId("rtrim"); String fieldString = getPropertyOrStringValue(field); - return new Tuple<>(name, StringUtils.format( - "int pos=%s.length()-1;" - + "while(pos >= 0 && Character.isWhitespace(%s.charAt(pos))) {pos --;} " - + def(name, "%s.substring(0, pos+1)"), - fieldString, fieldString, fieldString + return new Tuple<>(name, scriptDeclare(field) + + StringUtils.format( + "int pos = %s.length() - 1;" + + "while(pos >= 0 && Character.isWhitespace(%s.charAt(pos))) {pos--;} " + + def(name, "%s.substring(0, pos + 1)"), + fieldString, fieldString, fieldString )); } private Tuple ltrim(SQLExpr field) { String name = nextId("ltrim"); String fieldString = getPropertyOrStringValue(field); - return new Tuple<>(name, StringUtils.format( - "int pos=0;" - + "while(pos < %s.length() && Character.isWhitespace(%s.charAt(pos))) {pos ++;} " - + def(name, "%s.substring(pos, %s.length())"), - fieldString, fieldString, fieldString, fieldString + return new Tuple<>(name, scriptDeclare(field) + + StringUtils.format( + "int pos = 0;" + + "while(pos < %s.length() && Character.isWhitespace(%s.charAt(pos))) {pos++;} " + + def(name, "%s.substring(pos, %s.length())"), + fieldString, fieldString, fieldString, fieldString )); } private Tuple ascii(SQLExpr field) { String name = nextId("ascii"); - return new Tuple<>(name, def(name, "(int) " + getPropertyOrStringValue(field) + ".charAt(0)")); + return new Tuple<>(name, scriptDeclare(field) + + def(name, "(int) " + getPropertyOrStringValue(field) + ".charAt(0)")); } private Tuple left(SQLExpr expr, SQLExpr length) { String name = nextId("left"); - return new Tuple<>(name, StringUtils.format( - "def len = (int) Math.min(%s, %s.length()); def %s = %s.substring(0, len)", - exprString(length), getPropertyOrStringValue(expr), name, getPropertyOrStringValue(expr))); + return new Tuple<>(name, scriptDeclare(expr) + scriptDeclare(length) + + StringUtils.format( + "def len = (int) Math.min(%s, %s.length()); def %s = %s.substring(0, len)", + exprString(length), getPropertyOrStringValue(expr), name, getPropertyOrStringValue(expr) + )); } private Tuple right(SQLExpr expr, SQLExpr length) { String name = nextId("right"); - return new Tuple<>(name, StringUtils.format( - "def start = (int) Math.max(0, %s.length()-%s); def %s = %s.substring(start)", - getPropertyOrStringValue(expr), exprString(length), name, getPropertyOrStringValue(expr))); + return new Tuple<>(name, scriptDeclare(expr) + scriptDeclare(length) + + StringUtils.format( + "def start = (int) Math.max(0, %s.length()-%s); def %s = %s.substring(start)", + getPropertyOrStringValue(expr), exprString(length), name, getPropertyOrStringValue(expr) + )); } private Tuple date(SQLExpr field) { String name = nextId("date"); - return new Tuple<>(name, def(name, - "LocalDate.parse(" + getPropertyOrStringValue(field) + ".toString()," + return new Tuple<>(name, scriptDeclare(field) + + def(name, "LocalDate.parse(" + getPropertyOrStringValue(field) + ".toString(), " + "DateTimeFormatter.ISO_DATE_TIME)")); } private Tuple timestamp(SQLExpr field) { String name = nextId("timestamp"); - return new Tuple<>(name, def(name, - "DateTimeFormatter.ofPattern('yyyy-MM-dd HH:mm:ss').format(" + return new Tuple<>(name, scriptDeclare(field) + + def(name, "DateTimeFormatter.ofPattern('yyyy-MM-dd HH:mm:ss').format(" + "DateTimeFormatter.ISO_DATE_TIME.parse(" + getPropertyOrStringValue(field) + ".toString()))")); } private Tuple maketime(SQLExpr hr, SQLExpr min, SQLExpr sec) { String name = nextId("maketime"); - return new Tuple<>(name, def(name, StringUtils.format( - "LocalTime.of(%s, %s, %s).format(DateTimeFormatter.ofPattern('HH:mm:ss'))", - hr.toString(), min.toString(), sec.toString()))); + return new Tuple<>(name, scriptDeclare(hr) + scriptDeclare(min) + scriptDeclare(sec) + + def(name, StringUtils.format( + "LocalTime.of(%s, %s, %s).format(DateTimeFormatter.ofPattern('HH:mm:ss'))", + hr.toString(), min.toString(), sec.toString() + ))); } private Tuple now() { String name = nextId("now"); - return new Tuple<>(name, def(name, "new SimpleDateFormat('HH:mm:ss').format(System.currentTimeMillis())")); + return new Tuple<>(name, def(name, + "new SimpleDateFormat('HH:mm:ss').format(System.currentTimeMillis())")); } private Tuple curdate() { String name = nextId("curdate"); - return new Tuple<>(name, def(name, "new SimpleDateFormat('yyyy-MM-dd').format(System.currentTimeMillis())")); + return new Tuple<>(name, def(name, + "new SimpleDateFormat('yyyy-MM-dd').format(System.currentTimeMillis())")); } private Tuple ifFunc(List paramers) { - String expr1 = paramers.get(1).value.toString(); - String expr2 = paramers.get(2).value.toString(); String name = nextId("if"); - /** Input with null is regarded as false */ + SQLExpr expr1 = (SQLExpr) paramers.get(1).value; + SQLExpr expr2 = (SQLExpr) paramers.get(2).value; + + String value1 = (expr1 instanceof SQLTextLiteralExpr ? getPropertyOrStringValue(expr1) + : getPropertyOrValue(expr1)); + String value2 = (expr2 instanceof SQLTextLiteralExpr ? getPropertyOrStringValue(expr2) + : getPropertyOrValue(expr2)); + + String definition = scriptDeclare(expr1) + scriptDeclare(expr2); + + // Input with null is regarded as false if (paramers.get(0).value instanceof SQLNullExpr) { - return new Tuple<>(name, def(name, expr2)); - } - if (paramers.get(0).value instanceof MethodField) { + definition += def(name, value2); + } else if (paramers.get(0).value instanceof MethodField) { String condition = getScriptText((MethodField) paramers.get(0).value); - return new Tuple<>(name, "boolean cond = " + condition + ";" - + def(name, "cond ? " + expr1 + " : " + expr2)); + definition += scriptDeclare(paramers.get(0).value) + + "boolean cond = (" + condition + "); " + + def(name, "(cond ? " + value1 + " : " + value2 + ")"); } else if (paramers.get(0).value instanceof SQLBooleanExpr) { - Boolean condition = ((SQLBooleanExpr) paramers.get(0).value).getValue(); - if (condition) { - return new Tuple<>(name, def(name, expr1)); + if (((SQLBooleanExpr) paramers.get(0).value).getValue()) { + definition += def(name, value1); } else { - return new Tuple<>(name, def(name, expr2)); + definition += def(name, value2); } - } else { - /** - * Detailed explanation of cases that come here: - * the condition expression would be in the format of a=b: - * a is parsed as the key (String) of a KVValue (get from paramers.get(0)) - * and b is parsed as the value (Object) of this KVValue. - * - * Either a or b could be a column name, literal, or a number: - * - if isNumeric is true --> number - * - else if this string is single quoted --> literal - * - else --> column name - */ - String key = getPropertyOrValue(paramers.get(0).key); - String value = getPropertyOrValue(paramers.get(0).value.toString()); - String condition = key + " == " + value; - return new Tuple<>(name, "boolean cond = " + condition + ";" - + def(name, "cond ? " + expr1 + " : " + expr2)); } + + return new Tuple<>(name, definition); } - private Tuple ifnull(SQLExpr condition, SQLExpr expr) { + private Tuple ifnull(Object condition, SQLExpr expr) { String name = nextId("ifnull"); - if (condition instanceof SQLNullExpr) { - return new Tuple<>(name, def(name, expr.toString())); - } - if (isProperty(condition)) { - return new Tuple<>(name, def(name, doc(condition) + ".size()==0 ? " + expr.toString() + " : " - + getPropertyOrValue(condition))); - } else { - String condStr = Strings.isNullOrEmpty(condition.toString()) ? null : getPropertyOrStringValue(condition); - return new Tuple<>(name, def(name, condStr)); - } + + return new Tuple<>(name, scriptDeclare(condition) + scriptDeclare(expr) + + def(name, StringUtils.format( + "(%s ? %s : null)", + checkIfNull(condition), + (expr instanceof SQLTextLiteralExpr ? getPropertyOrStringValue(expr) : getPropertyOrValue(expr)) + ))); } private Tuple isnull(SQLExpr expr) { String name = nextId("isnull"); - if (expr instanceof SQLNullExpr) { - return new Tuple<>(name, def(name, "1")); - } - if (isProperty(expr)) { - return new Tuple<>(name, def(name, doc(expr) + ".size()==0 ? 1 : 0")); - } - // cases that return 1: - // expr is null || expr is math func but tends to throw "divided by zero" arithmetic exception - String resultStr = "0"; - if (Strings.isNullOrEmpty(expr.toString())) { - resultStr = "1"; - } - if (expr instanceof SQLCharExpr && this.generatedIds.size() > 1) { - // the expr is a math expression - String mathExpr = ((SQLCharExpr) expr).getText(); - return new Tuple<>(name, StringUtils.format( - "try {%s;} " - + "catch(ArithmeticException e) " - + "{return 1;} " - + "def %s=0", - mathExpr, name, name) - ); - } - return new Tuple<>(name, def(name, resultStr)); + + return new Tuple<>(name, scriptDeclare(expr) + def(name, + StringUtils.format("(%s ? 1 : 0)", checkIfNull(expr)))); } public String getCastScriptStatement(String name, String castType, List paramers) throws SqlParseException { - String castFieldName = String.format("doc['%s'].value", paramers.get(0).toString()); + SQLExpr expr = (SQLExpr) paramers.get(0).value; + String castFieldName = (expr instanceof SQLTextLiteralExpr ? getPropertyOrStringValue(expr) + : getPropertyOrValue(expr)); + + scriptDeclare(expr); + switch (StringUtils.toUpper(castType)) { case "INT": case "LONG": diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/Util.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/Util.java index 7c8bbbdf47..3f69c00bd3 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/Util.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/Util.java @@ -112,17 +112,17 @@ public static Object expr2Object(SQLExpr expr, String charWithQuote) { public static Object getScriptValue(SQLExpr expr) throws SqlParseException { if (expr instanceof SQLIdentifierExpr || expr instanceof SQLPropertyExpr || expr instanceof SQLVariantRefExpr) { - return "doc['" + expr.toString() + "'].value"; + return StringUtils.format("(doc['%1$s'].size() == 0 ? null : doc['%1$s'].value)", expr.toString()); } else if (expr instanceof SQLValuableExpr) { return ((SQLValuableExpr) expr).getValue(); } - throw new SqlParseException("could not parse sqlBinaryOpExpr need to be identifier/valuable got" + throw new SqlParseException("Could not parse sqlBinaryOpExpr: needs to be identifier/valuable, got" + expr.getClass().toString() + " with value:" + expr.toString()); } public static Object getScriptValueWithQuote(SQLExpr expr, String quote) throws SqlParseException { if (expr instanceof SQLIdentifierExpr || expr instanceof SQLPropertyExpr || expr instanceof SQLVariantRefExpr) { - return "doc['" + expr.toString() + "'].value"; + return StringUtils.format("(doc['%1$s'].size() == 0 ? null : doc['%1$s'].value)", expr.toString()); } else if (expr instanceof SQLCharExpr) { return quote + ((SQLCharExpr) expr).getValue() + quote; } else if (expr instanceof SQLIntegerExpr) { diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFunctionsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFunctionsTest.java index bc6d912c9f..34053356fa 100644 --- a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFunctionsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFunctionsTest.java @@ -154,7 +154,8 @@ public void date() { assertTrue( scriptContainsString( scriptField, - "LocalDate.parse(doc['creationDate'].value.toString(),DateTimeFormatter.ISO_DATE_TIME)")); + "LocalDate.parse((doc['creationDate'].size() == 0 " + + "? null : doc['creationDate'].value).toString()")); } @Test diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/MathFunctionsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/MathFunctionsTest.java index ed08283ea3..779d36c8d6 100644 --- a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/MathFunctionsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/MathFunctionsTest.java @@ -40,7 +40,7 @@ public void lowerCaseInSelect() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.abs(doc['age'].value)")); + "Math.abs((doc['age'].size() == 0 ? null : doc['age'].value))")); } @Test @@ -51,7 +51,7 @@ public void upperCaseInSelect() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.abs(doc['age'].value)")); + "Math.abs((doc['age'].size() == 0 ? null : doc['age'].value))")); } @Test @@ -63,7 +63,7 @@ public void lowerCaseInWhere() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.sqrt(doc['age'].value)")); + "Math.sqrt((doc['age'].size() == 0 ? null : doc['age'].value))")); assertTrue( CheckScriptContents.scriptHasPattern( scriptFilter, @@ -79,7 +79,7 @@ public void upperCaseInWhere() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.sqrt(doc['age'].value)")); + "Math.sqrt((doc['age'].size() == 0 ? null : doc['age'].value))")); assertTrue( CheckScriptContents.scriptHasPattern( scriptFilter, @@ -151,7 +151,7 @@ public void expm1WithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.expm1(doc['age'].value)")); + "Math.expm1((doc['age'].size() == 0 ? null : doc['age'].value))")); assertTrue( CheckScriptContents.scriptHasPattern( scriptFilter, @@ -184,7 +184,7 @@ public void degreesWithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.toDegrees(doc['age'].value)")); + "Math.toDegrees((doc['age'].size() == 0 ? null : doc['age'].value))")); } @Test @@ -206,7 +206,7 @@ public void radiansWithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.toRadians(doc['age'].value)")); + "Math.toRadians((doc['age'].size() == 0 ? null : doc['age'].value))")); } @Test @@ -228,7 +228,7 @@ public void sinWithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.toRadians(doc['age'].value)")); + "Math.toRadians((doc['age'].size() == 0 ? null : doc['age'].value))")); assertTrue( CheckScriptContents.scriptHasPattern( scriptField, @@ -258,7 +258,7 @@ public void atanWithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.atan(doc['age'].value)")); + "Math.atan((doc['age'].size() == 0 ? null : doc['age'].value))")); } @Test @@ -299,7 +299,7 @@ public void coshWithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.cosh(doc['age'].value)")); + "Math.cosh((doc['age'].size() == 0 ? null : doc['age'].value))")); } @Test @@ -320,13 +320,13 @@ public void powerWithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.pow(doc['age'].value, 2)")); + "Math.pow((doc['age'].size() == 0 ? null : doc['age'].value), 2)")); ScriptFilter scriptFilter = CheckScriptContents.getScriptFilterFromQuery(query, parser); assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.pow(doc['balance'].value, 3)")); + "Math.pow((doc['balance'].size() == 0 ? null : doc['balance'].value), 3)")); } @Test @@ -336,13 +336,13 @@ public void atan2WithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.atan2(doc['age'].value, 2)")); + "Math.atan2((doc['age'].size() == 0 ? null : doc['age'].value), 2)")); ScriptFilter scriptFilter = CheckScriptContents.getScriptFilterFromQuery(query, parser); assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.atan2(doc['balance'].value, 3)")); + "Math.atan2((doc['balance'].size() == 0 ? null : doc['balance'].value), 3)")); } @Test @@ -352,13 +352,13 @@ public void cotWithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "1 / Math.tan(doc['age'].value)")); + "1 / Math.tan((doc['age'].size() == 0 ? null : doc['age'].value))")); ScriptFilter scriptFilter = CheckScriptContents.getScriptFilterFromQuery(query, parser); assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "1 / Math.tan(doc['balance'].value)")); + "1 / Math.tan((doc['balance'].size() == 0 ? null : doc['balance'].value))")); } @Test @@ -367,13 +367,13 @@ public void signWithFunctionPropertyArgument() { ScriptField scriptField = CheckScriptContents.getScriptFieldFromQuery(query); assertTrue(CheckScriptContents.scriptContainsString( scriptField, - "Math.signum(doc['age'].value)")); + "Math.signum((doc['age'].size() == 0 ? null : doc['age'].value))")); ScriptFilter scriptFilter = CheckScriptContents.getScriptFilterFromQuery(query, parser); assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.signum(doc['balance'].value)")); + "Math.signum((doc['balance'].size() == 0 ? null : doc['balance'].value))")); } @Test @@ -383,13 +383,13 @@ public void logWithOneParam() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.log(doc['age'].value)")); + "Math.log((doc['age'].size() == 0 ? null : doc['age'].value))")); ScriptFilter scriptFilter = CheckScriptContents.getScriptFilterFromQuery(query, parser); assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.log(doc['age'].value)")); + "Math.log((doc['age'].size() == 0 ? null : doc['age'].value))")); } @Test @@ -399,13 +399,13 @@ public void logWithTwoParams() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.log(doc['age'].value)/Math.log(3)")); + "Math.log((doc['age'].size() == 0 ? null : doc['age'].value))/Math.log(3)")); ScriptFilter scriptFilter = CheckScriptContents.getScriptFilterFromQuery(query, parser); assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.log(doc['age'].value)/Math.log(3)")); + "Math.log((doc['age'].size() == 0 ? null : doc['age'].value))/Math.log(3)")); } @Test @@ -415,7 +415,7 @@ public void log10Test() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.log10(doc['age'].value)" + "Math.log10((doc['age'].size() == 0 ? null : doc['age'].value))" ) ); } @@ -427,13 +427,13 @@ public void lnTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.log(doc['age'].value)")); + "Math.log((doc['age'].size() == 0 ? null : doc['age'].value))")); ScriptFilter scriptFilter = CheckScriptContents.getScriptFilterFromQuery(query, parser); assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.log(doc['age'].value)")); + "Math.log((doc['age'].size() == 0 ? null : doc['age'].value))")); } @Test @@ -455,7 +455,7 @@ public void randWithOneParamTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "new Random(doc['age'].value).nextDouble()" + "new Random((doc['age'].size() == 0 ? null : doc['age'].value)).nextDouble()" ) ); } diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/QueryFunctionsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/QueryFunctionsTest.java index b939c8696e..7c818ce6c6 100644 --- a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/QueryFunctionsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/QueryFunctionsTest.java @@ -218,7 +218,7 @@ public void ifFunctionWithConditionStatement() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "boolean cond = doc['age'].value > 35;" + "boolean cond = ((doc['age'].size() == 0 ? null : doc['age'].value) > 35);" ) ); } @@ -230,7 +230,7 @@ public void ifFunctionWithEquationConditionStatement() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "boolean cond = doc['age'].value == 35;" + "boolean cond = ((doc['age'].size() == 0 ? null : doc['age'].value) == 35);" ) ); } @@ -242,7 +242,7 @@ public void ifFunctionWithConstantConditionStatement() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "boolean cond = 1 == 2;" + "boolean cond = (1 == 2);" ) ); } @@ -254,7 +254,7 @@ public void ifNull() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "doc['lastname'].size()==0" + "doc['lastname'].size() == 0" ) ); } @@ -266,7 +266,7 @@ public void isNullWithMathExpr() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "catch(ArithmeticException e)" + "add_1 == null" ) ); diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/StringOperatorsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/StringOperatorsTest.java index 0cad9e71e4..3c54647bbd 100644 --- a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/StringOperatorsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/StringOperatorsTest.java @@ -40,13 +40,13 @@ public void substringTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "doc['lastname'].value.substring(1, end)")); + "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).substring(1, end)")); ScriptFilter scriptFilter = CheckScriptContents.getScriptFilterFromQuery(query, parser); assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "doc['lastname'].value.substring(1, end)" + "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).substring(1, end)" ) ); } @@ -72,7 +72,7 @@ public void lengthTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "doc['lastname'].value.length()" + "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).length()" ) ); @@ -80,7 +80,7 @@ public void lengthTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "doc['lastname'].value.length()" + "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).length()" ) ); } @@ -94,7 +94,7 @@ public void replaceTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "doc['lastname'].value.replace('a','A')" + "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).replace('a','A')" ) ); @@ -102,7 +102,7 @@ public void replaceTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "doc['lastname'].value.replace('a','A')" + "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).replace('a','A')" ) ); } @@ -116,7 +116,7 @@ public void locateTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "doc['lastname'].value.indexOf('a',0)+1" + "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).indexOf('a', 0) + 1" ) ); @@ -124,7 +124,7 @@ public void locateTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "doc['lastname'].value.indexOf('a',0)+1" + "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).indexOf('a', 0) + 1" ) ); } @@ -138,7 +138,8 @@ public void ltrimTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Character.isWhitespace(doc['lastname'].value.charAt(pos))" + "Character.isWhitespace((doc['lastname'].size() == 0 " + + "? null : doc['lastname'].value).charAt(pos))" ) ); @@ -146,7 +147,8 @@ public void ltrimTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Character.isWhitespace(doc['lastname'].value.charAt(pos))" + "Character.isWhitespace((doc['lastname'].size() == 0 " + + "? null : doc['lastname'].value).charAt(pos))" ) ); } @@ -160,7 +162,8 @@ public void rtrimTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Character.isWhitespace(doc['lastname'].value.charAt(pos))" + "Character.isWhitespace((doc['lastname'].size() == 0 " + + "? null : doc['lastname'].value).charAt(pos))" ) ); @@ -168,7 +171,8 @@ public void rtrimTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Character.isWhitespace(doc['lastname'].value.charAt(pos))" + "Character.isWhitespace((doc['lastname'].size() == 0 " + + "? null : doc['lastname'].value).charAt(pos))" ) ); } @@ -182,7 +186,7 @@ public void asciiTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "(int) doc['lastname'].value.charAt(0)" + "(int) (doc['lastname'].size() == 0 ? null : doc['lastname'].value).charAt(0)" ) ); @@ -190,7 +194,7 @@ public void asciiTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "(int) doc['lastname'].value.charAt(0)" + "(int) (doc['lastname'].size() == 0 ? null : doc['lastname'].value).charAt(0)" ) ); } @@ -202,7 +206,7 @@ public void left() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "doc['lastname'].value.substring(0, len)" + "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).substring(0, len)" ) ); } @@ -214,7 +218,7 @@ public void right() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "doc['lastname'].value.substring(start)" + "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).substring(start)" ) ); } diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/SqlParserTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/SqlParserTest.java index 65077883fb..4b0c946bd1 100644 --- a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/SqlParserTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/SqlParserTest.java @@ -92,7 +92,8 @@ public void whereConditionLeftFunctionRightPropertyGreatTest() throws Exception Assert.assertTrue(((Condition) (where.getWheres().get(0))).getValue() instanceof ScriptFilter); ScriptFilter scriptFilter = (ScriptFilter) (((Condition) (where.getWheres().get(0))).getValue()); - Assert.assertTrue(scriptFilter.getScript().contains("doc['address'].value.split(' ')[0]")); + Assert.assertTrue(scriptFilter.getScript().contains( + "(doc['address'].size() == 0 ? null : doc['address'].value).split(' ')[0]")); Pattern pattern = Pattern.compile("floor_\\d+ > doc\\['b'].value"); java.util.regex.Matcher matcher = pattern.matcher(scriptFilter.getScript()); Assert.assertTrue(matcher.find()); @@ -176,7 +177,8 @@ public void whereConditionLeftFunctionRightFunctionEqualTest() throws Exception Assert.assertTrue((where.getWheres().size() == 1)); Assert.assertTrue(((Condition) (where.getWheres().get(0))).getValue() instanceof ScriptFilter); ScriptFilter scriptFilter = (ScriptFilter) (((Condition) (where.getWheres().get(0))).getValue()); - Assert.assertTrue(scriptFilter.getScript().contains("doc['address'].value.split(' ')[0]")); + Assert.assertTrue(scriptFilter.getScript().contains( + "(doc['address'].size() == 0 ? null : doc['address'].value).split(' ')[0]")); Pattern pattern = Pattern.compile("floor_\\d+ == floor_\\d+"); java.util.regex.Matcher matcher = pattern.matcher(scriptFilter.getScript()); Assert.assertTrue(matcher.find()); @@ -653,7 +655,8 @@ public void scriptFiledPlusLiteralTest() throws SqlParseException { MethodField scriptMethod = (MethodField) field; Assert.assertEquals("script", scriptMethod.getName().toLowerCase()); Assert.assertEquals(2, scriptMethod.getParams().size()); - Assert.assertTrue(scriptMethod.getParams().get(1).toString().contains("doc['field1'].value + 3")); + Assert.assertTrue(scriptMethod.getParams().get(1).toString().contains( + "(doc['field1'].size() == 0 ? null : doc['field1'].value) + 3")); } @Test @@ -669,7 +672,8 @@ public void scriptFieldPlusFieldTest() throws SqlParseException { Assert.assertEquals("script", scriptMethod.getName().toLowerCase()); Assert.assertEquals(2, scriptMethod.getParams().size()); Assert.assertTrue(scriptMethod.getParams().get(1).toString() - .contains("doc['field1'].value + doc['field2'].value")); + .contains("(doc['field1'].size() == 0 ? null : doc['field1'].value) " + + "+ (doc['field2'].size() == 0 ? null : doc['field2'].value)")); } @@ -721,9 +725,7 @@ public void implicitScriptOnAggregation() throws SqlParseException { MethodField avgMethodField = (MethodField) field; Assert.assertEquals("avg", avgMethodField.getName().toLowerCase()); Assert.assertEquals(1, avgMethodField.getParams().size()); - Assert.assertTrue(avgMethodField.getParams().get(0).value.toString().contains("doc['field1'].value")); - Assert.assertTrue(avgMethodField.getParams().get(0).value.toString().contains("doc['field2'].value")); - + Assert.assertTrue(avgMethodField.getParams().get(0).value.toString().contains("add_1")); } @Test @@ -1206,7 +1208,7 @@ public void caseWhenSwitchTest() { Assert.assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "doc['weather'].value=='Sunny'" + "(doc['weather'].size() == 0 ? null : doc['weather'].value)=='Sunny'" ) ); } @@ -1225,8 +1227,9 @@ public void castToIntTest() throws Exception { String alias = (String) methodField.getParams().get(0).value; String scriptCode = (String) methodField.getParams().get(1).value; Assert.assertEquals("cast_age",alias); - Assert.assertTrue(scriptCode.contains("doc['age'].value")); - Assert.assertTrue(scriptCode.contains("Double.parseDouble(doc['age'].value.toString()).intValue()")); + Assert.assertTrue(scriptCode.contains("(doc['age'].size() == 0 ? null : doc['age'].value)")); + Assert.assertTrue(scriptCode.contains("Double.parseDouble((doc['age'].size() == 0 " + + "? null : doc['age'].value).toString()).intValue()")); } @Test @@ -1243,8 +1246,9 @@ public void castToLongTest() throws Exception { String alias = (String) methodField.getParams().get(0).value; String scriptCode = (String) methodField.getParams().get(1).value; Assert.assertEquals("cast_insert_time",alias); - Assert.assertTrue(scriptCode.contains("doc['insert_time'].value")); - Assert.assertTrue(scriptCode.contains("Double.parseDouble(doc['insert_time'].value.toString()).longValue()")); + Assert.assertTrue(scriptCode.contains("(doc['insert_time'].size() == 0 ? null : doc['insert_time'].value)")); + Assert.assertTrue(scriptCode.contains("Double.parseDouble((doc['insert_time'].size() == 0 " + + "? null : doc['insert_time'].value).toString()).longValue()")); } @Test @@ -1261,8 +1265,9 @@ public void castToFloatTest() throws Exception { String alias = (String) methodField.getParams().get(0).value; String scriptCode = (String) methodField.getParams().get(1).value; Assert.assertEquals("cast_age",alias); - Assert.assertTrue(scriptCode.contains("doc['age'].value")); - Assert.assertTrue(scriptCode.contains("Double.parseDouble(doc['age'].value.toString()).floatValue()")); + Assert.assertTrue(scriptCode.contains("(doc['age'].size() == 0 ? null : doc['age'].value)")); + Assert.assertTrue(scriptCode.contains("Double.parseDouble((doc['age'].size() == 0 " + + "? null : doc['age'].value).toString()).floatValue()")); } @Test @@ -1279,8 +1284,9 @@ public void castToDoubleTest() throws Exception { String alias = (String) methodField.getParams().get(0).value; String scriptCode = (String) methodField.getParams().get(1).value; Assert.assertEquals("cast_age",alias); - Assert.assertTrue(scriptCode.contains("doc['age'].value")); - Assert.assertTrue(scriptCode.contains("Double.parseDouble(doc['age'].value.toString()).doubleValue()")); + Assert.assertTrue(scriptCode.contains("(doc['age'].size() == 0 ? null : doc['age'].value)")); + Assert.assertTrue(scriptCode.contains("Double.parseDouble((doc['age'].size() == 0 " + + "? null : doc['age'].value).toString()).doubleValue()")); } @Test @@ -1297,7 +1303,7 @@ public void castToStringTest() throws Exception { String alias = (String) methodField.getParams().get(0).value; String scriptCode = (String) methodField.getParams().get(1).value; Assert.assertEquals("cast_age",alias); - Assert.assertTrue(scriptCode.contains("doc['age'].value.toString()")); + Assert.assertTrue(scriptCode.contains("(doc['age'].size() == 0 ? null : doc['age'].value).toString()")); } @Test @@ -1314,9 +1320,10 @@ public void castToDateTimeTest() throws Exception { String alias = (String) methodField.getParams().get(0).value; String scriptCode = (String) methodField.getParams().get(1).value; Assert.assertEquals("cast_age",alias); - Assert.assertTrue(scriptCode.contains("doc['age'].value")); + Assert.assertTrue(scriptCode.contains("(doc['age'].size() == 0 ? null : doc['age'].value)")); Assert.assertTrue(scriptCode.contains("DateTimeFormatter.ofPattern(\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\").format(" - + "DateTimeFormatter.ISO_DATE_TIME.parse(doc['age'].value.toString()))")); + + "DateTimeFormatter.ISO_DATE_TIME.parse((doc['age'].size() == 0 " + + "? null : doc['age'].value).toString()))")); } @Test @@ -1331,8 +1338,9 @@ public void castToDoubleThenDivideTest() throws Exception { Assert.assertEquals("script",castField.getName()); String scriptCode = (String) methodField.getParams().get(1).value; - Assert.assertTrue(scriptCode.contains("doc['age'].value")); - Assert.assertTrue(scriptCode.contains("Double.parseDouble(doc['age'].value.toString()).doubleValue()")); + Assert.assertTrue(scriptCode.contains("(doc['age'].size() == 0 ? null : doc['age'].value)")); + Assert.assertTrue(scriptCode.contains("Double.parseDouble((doc['age'].size() == 0 " + + "? null : doc['age'].value).toString()).doubleValue()")); Assert.assertTrue(scriptCode.contains("/ 2")); } diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/SQLFunctionsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/SQLFunctionsTest.java index f8a664928b..e331e5734a 100644 --- a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/SQLFunctionsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/SQLFunctionsTest.java @@ -102,11 +102,11 @@ public void testCastReturnType() { @Test public void testCastIntStatementScript() throws SqlParseException { assertEquals( - "def result = (doc['age'].value instanceof boolean) " - + "? (doc['age'].value ? 1 : 0) " - + ": Double.parseDouble(doc['age'].value.toString()).intValue()", + "def result = ((doc['age'].size() == 0 ? null : doc['age'].value) instanceof boolean) " + + "? ((doc['age'].size() == 0 ? null : doc['age'].value) ? 1 : 0) " + + ": Double.parseDouble((doc['age'].size() == 0 ? null : doc['age'].value).toString()).intValue()", sqlFunctions.getCastScriptStatement( - "result", "int", Arrays.asList(new KVValue("age"))) + "result", "int", Arrays.asList(new KVValue(new SQLIdentifierExpr("age")))) ); } From 9f91648915cd8ad33b7442802d30ca1e456fc8f8 Mon Sep 17 00:00:00 2001 From: cip999 Date: Tue, 24 Aug 2021 23:58:26 +0200 Subject: [PATCH 34/37] Massive refactoring of modules handling SQL functions to allow for nesting --- .../sql/legacy/utils/SQLDefinitionExpr.java | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLDefinitionExpr.java diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLDefinitionExpr.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLDefinitionExpr.java new file mode 100644 index 0000000000..8da72a63c5 --- /dev/null +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLDefinitionExpr.java @@ -0,0 +1,73 @@ +package com.amazon.opendistroforelasticsearch.sql.legacy.utils; + +import com.alibaba.druid.sql.ast.SQLExprImpl; +import com.alibaba.druid.sql.ast.expr.SQLValuableExpr; +import com.alibaba.druid.sql.visitor.SQLASTVisitor; + +public class SQLDefinitionExpr extends SQLExprImpl implements SQLValuableExpr { + + private final String name; + + private final String definition; + + public SQLDefinitionExpr(String name, String definition) { + this.name = name; + this.definition = definition; + } + + public String getName() { + return name; + } + + public String getDefinition() { + return definition; + } + + @Override + public Object getValue() { + return this.getName(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + SQLDefinitionExpr other = (SQLDefinitionExpr) obj; + if (name == null) { + if (other.name != null) { + return false; + } + } else if (!name.equals(other.name)) { + return false; + } + if (definition == null) { + if (other.definition != null) { + return false; + } + } else if (!definition.equals(other.definition)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + ((definition == null) ? 0 : definition.hashCode()); + return result; + } + + @Override + protected void accept0(SQLASTVisitor visitor) { + + } +} From 27450a722f655d9e89a34bb7d8341e4930c85065 Mon Sep 17 00:00:00 2001 From: cip999 Date: Thu, 26 Aug 2021 00:18:27 +0200 Subject: [PATCH 35/37] Massive refactoring of modules handling SQL functions to allow for nesting --- .../sql/legacy/parser/FieldMaker.java | 12 ++++- .../sql/legacy/utils/SQLFunctions.java | 34 ++++++++---- .../sql/legacy/utils/Util.java | 14 +++-- .../legacy/unittest/DateFunctionsTest.java | 4 +- .../legacy/unittest/MathFunctionsTest.java | 52 +++++++++---------- .../legacy/unittest/QueryFunctionsTest.java | 4 +- .../legacy/unittest/StringOperatorsTest.java | 36 ++++++------- .../legacy/unittest/parser/SqlParserTest.java | 50 ++++++++---------- .../unittest/utils/SQLFunctionsTest.java | 8 +-- 9 files changed, 115 insertions(+), 99 deletions(-) diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/FieldMaker.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/FieldMaker.java index 378319bcd6..734dbf8276 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/FieldMaker.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/FieldMaker.java @@ -321,7 +321,15 @@ public MethodField makeMethodField(String name, List arguments, SQLAggr new SQLDefinitionExpr(varName, mField.getParams().get(1).toString()) )); } else { - paramers.add(new KVValue("script", makeScriptMethodField(binaryOpExpr, null, tableAlias))); + if (!SQLFunctions.isFunctionTranslatedToScript(name) + && binaryOpExpr.getOperator().getName().equals("=")) { + paramers.add(new KVValue( + binaryOpExpr.getLeft().toString(), + Util.expr2Object(binaryOpExpr.getRight()) + )); + } else { + paramers.add(new KVValue("script", makeScriptMethodField(binaryOpExpr, null, tableAlias))); + } } } else if (object instanceof SQLMethodInvokeExpr) { @@ -421,7 +429,7 @@ public MethodField makeMethodField(String name, List arguments, SQLAggr List tempParamers = new LinkedList<>(); for (KVValue temp : paramers) { if (temp.value instanceof SQLExpr) { - tempParamers.add(new KVValue(temp.key, Util.expr2Object((SQLExpr) temp.value))); + tempParamers.add(new KVValue(temp.key, Util.expr2ObjectOrDefinition((SQLExpr) temp.value))); } else { tempParamers.add(new KVValue(temp.key, temp.value)); } diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java index 1f6a5f96a7..77eba08e27 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java @@ -623,10 +623,20 @@ private Tuple divide(SQLExpr a, SQLExpr b) { private Tuple binaryOpertator(String methodName, String operator, SQLExpr a, SQLExpr b) { String name = nextId(methodName); - return new Tuple<>(name, - scriptDeclare(a) + scriptDeclare(b) + convertType(a) + convertType(b) - + def(name, - getPropertyOrValue(a) + " " + operator + " " + getPropertyOrValue(b))); + + if (methodName.equals("divide") || methodName.equals("modulus")) { + return new Tuple<>(name, + scriptDeclare(a) + scriptDeclare(b) + convertType(a) + convertType(b) + + def(name, StringUtils.format("((%2$s == 0 || %4$s || %5$s) ? null : %1$s %3$s %2$s)", + getPropertyOrValue(a), getPropertyOrValue(b), operator, + checkIfNull(a), checkIfNull(b)))); + } else { + return new Tuple<>(name, + scriptDeclare(a) + scriptDeclare(b) + convertType(a) + convertType(b) + + def(name, StringUtils.format("((%4$s || %5$s) ? null : %1$s %3$s %2$s)", + getPropertyOrValue(a), getPropertyOrValue(b), operator, + checkIfNull(a), checkIfNull(b)))); + } } private Tuple greatest(SQLExpr a, SQLExpr b) { @@ -660,7 +670,7 @@ private Tuple comparisonFunctionTemplate(SQLExpr a, SQLExpr b, String definition = scriptDeclare(a) + scriptDeclare(b) + "def " + name + ";" - + String.format("if (%s) {%s = %s;} ", checkIfNull(a), name, value_b) + + String.format("if (%s) {%s = ((%s) ? null : %s);} ", checkIfNull(a), name, checkIfNull(b), value_b) + String.format("else if (%s) {%s = %s;} ", checkIfNull(b), name, value_a) + "else {"; @@ -693,7 +703,7 @@ private static String getPropertyOrValue(SQLExpr expr) { } else if (expr instanceof SQLNullExpr) { return "null"; } else if (isProperty(expr)) { - return StringUtils.format("(%s.size() == 0 ? null : %s.value)", doc(expr), doc(expr)); + return doc(expr) + ".value"; } else { return exprString(expr); } @@ -705,7 +715,7 @@ private static String getPropertyOrValue(String expr) { } else if (StringUtils.isNumeric(expr)) { return expr; } else { - return StringUtils.format("(%s.size() == 0 ? null : %s.value)", doc(expr), doc(expr)); + return doc(expr) + ".value"; } } @@ -782,7 +792,7 @@ public Tuple log10(SQLExpr field) { public Tuple ln(SQLExpr field) { String name = nextId("ln"); return new Tuple<>(name, scriptDeclare(field) - + StringUtils.format("Math.log(%s)", getPropertyOrValue(field))); + + def(name, StringUtils.format("Math.log(%s)", getPropertyOrValue(field)))); } public Tuple trim(SQLExpr field) { @@ -1007,9 +1017,11 @@ private Tuple ifnull(Object condition, SQLExpr expr) { return new Tuple<>(name, scriptDeclare(condition) + scriptDeclare(expr) + def(name, StringUtils.format( - "(%s ? %s : null)", + "((%s) ? %s : %s)", checkIfNull(condition), - (expr instanceof SQLTextLiteralExpr ? getPropertyOrStringValue(expr) : getPropertyOrValue(expr)) + getPropertyOrStringValue(expr), + condition instanceof SQLExpr ? getPropertyOrStringValue((SQLExpr) condition) + : condition.toString() ))); } @@ -1017,7 +1029,7 @@ private Tuple isnull(SQLExpr expr) { String name = nextId("isnull"); return new Tuple<>(name, scriptDeclare(expr) + def(name, - StringUtils.format("(%s ? 1 : 0)", checkIfNull(expr)))); + StringUtils.format("((%s) ? 1 : 0)", checkIfNull(expr)))); } public String getCastScriptStatement(String name, String castType, List paramers) diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/Util.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/Util.java index 3f69c00bd3..587c72d7e6 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/Util.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/Util.java @@ -88,6 +88,14 @@ public static Object expr2Object(SQLExpr expr) { return expr2Object(expr, ""); } + public static Object expr2ObjectOrDefinition(SQLExpr expr) { + if (expr instanceof SQLDefinitionExpr) { + return ((SQLDefinitionExpr) expr).getDefinition(); + } else { + return expr2Object(expr, ""); + } + } + public static Object expr2Object(SQLExpr expr, String charWithQuote) { Object value = null; if (expr instanceof SQLNumericLiteralExpr) { @@ -112,7 +120,7 @@ public static Object expr2Object(SQLExpr expr, String charWithQuote) { public static Object getScriptValue(SQLExpr expr) throws SqlParseException { if (expr instanceof SQLIdentifierExpr || expr instanceof SQLPropertyExpr || expr instanceof SQLVariantRefExpr) { - return StringUtils.format("(doc['%1$s'].size() == 0 ? null : doc['%1$s'].value)", expr.toString()); + return StringUtils.format("doc['%s'].value", expr.toString()); } else if (expr instanceof SQLValuableExpr) { return ((SQLValuableExpr) expr).getValue(); } @@ -122,7 +130,7 @@ public static Object getScriptValue(SQLExpr expr) throws SqlParseException { public static Object getScriptValueWithQuote(SQLExpr expr, String quote) throws SqlParseException { if (expr instanceof SQLIdentifierExpr || expr instanceof SQLPropertyExpr || expr instanceof SQLVariantRefExpr) { - return StringUtils.format("(doc['%1$s'].size() == 0 ? null : doc['%1$s'].value)", expr.toString()); + return StringUtils.format("doc['%s'].value", expr.toString()); } else if (expr instanceof SQLCharExpr) { return quote + ((SQLCharExpr) expr).getValue() + quote; } else if (expr instanceof SQLIntegerExpr) { @@ -130,7 +138,7 @@ public static Object getScriptValueWithQuote(SQLExpr expr, String quote) throws } else if (expr instanceof SQLNumericLiteralExpr) { return ((SQLNumericLiteralExpr) expr).getNumber(); } else if (expr instanceof SQLNullExpr) { - return ((SQLNullExpr) expr).toString().toLowerCase(); + return expr.toString().toLowerCase(); } throw new SqlParseException("could not parse sqlBinaryOpExpr need to be identifier/valuable got" + expr.getClass().toString() + " with value:" + expr.toString()); diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFunctionsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFunctionsTest.java index 34053356fa..550882eb94 100644 --- a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFunctionsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFunctionsTest.java @@ -154,8 +154,8 @@ public void date() { assertTrue( scriptContainsString( scriptField, - "LocalDate.parse((doc['creationDate'].size() == 0 " - + "? null : doc['creationDate'].value).toString()")); + "LocalDate.parse(doc['creationDate'].value.toString(), " + + "DateTimeFormatter.ISO_DATE_TIME)")); } @Test diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/MathFunctionsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/MathFunctionsTest.java index 779d36c8d6..ed08283ea3 100644 --- a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/MathFunctionsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/MathFunctionsTest.java @@ -40,7 +40,7 @@ public void lowerCaseInSelect() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.abs((doc['age'].size() == 0 ? null : doc['age'].value))")); + "Math.abs(doc['age'].value)")); } @Test @@ -51,7 +51,7 @@ public void upperCaseInSelect() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.abs((doc['age'].size() == 0 ? null : doc['age'].value))")); + "Math.abs(doc['age'].value)")); } @Test @@ -63,7 +63,7 @@ public void lowerCaseInWhere() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.sqrt((doc['age'].size() == 0 ? null : doc['age'].value))")); + "Math.sqrt(doc['age'].value)")); assertTrue( CheckScriptContents.scriptHasPattern( scriptFilter, @@ -79,7 +79,7 @@ public void upperCaseInWhere() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.sqrt((doc['age'].size() == 0 ? null : doc['age'].value))")); + "Math.sqrt(doc['age'].value)")); assertTrue( CheckScriptContents.scriptHasPattern( scriptFilter, @@ -151,7 +151,7 @@ public void expm1WithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.expm1((doc['age'].size() == 0 ? null : doc['age'].value))")); + "Math.expm1(doc['age'].value)")); assertTrue( CheckScriptContents.scriptHasPattern( scriptFilter, @@ -184,7 +184,7 @@ public void degreesWithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.toDegrees((doc['age'].size() == 0 ? null : doc['age'].value))")); + "Math.toDegrees(doc['age'].value)")); } @Test @@ -206,7 +206,7 @@ public void radiansWithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.toRadians((doc['age'].size() == 0 ? null : doc['age'].value))")); + "Math.toRadians(doc['age'].value)")); } @Test @@ -228,7 +228,7 @@ public void sinWithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.toRadians((doc['age'].size() == 0 ? null : doc['age'].value))")); + "Math.toRadians(doc['age'].value)")); assertTrue( CheckScriptContents.scriptHasPattern( scriptField, @@ -258,7 +258,7 @@ public void atanWithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.atan((doc['age'].size() == 0 ? null : doc['age'].value))")); + "Math.atan(doc['age'].value)")); } @Test @@ -299,7 +299,7 @@ public void coshWithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.cosh((doc['age'].size() == 0 ? null : doc['age'].value))")); + "Math.cosh(doc['age'].value)")); } @Test @@ -320,13 +320,13 @@ public void powerWithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.pow((doc['age'].size() == 0 ? null : doc['age'].value), 2)")); + "Math.pow(doc['age'].value, 2)")); ScriptFilter scriptFilter = CheckScriptContents.getScriptFilterFromQuery(query, parser); assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.pow((doc['balance'].size() == 0 ? null : doc['balance'].value), 3)")); + "Math.pow(doc['balance'].value, 3)")); } @Test @@ -336,13 +336,13 @@ public void atan2WithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.atan2((doc['age'].size() == 0 ? null : doc['age'].value), 2)")); + "Math.atan2(doc['age'].value, 2)")); ScriptFilter scriptFilter = CheckScriptContents.getScriptFilterFromQuery(query, parser); assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.atan2((doc['balance'].size() == 0 ? null : doc['balance'].value), 3)")); + "Math.atan2(doc['balance'].value, 3)")); } @Test @@ -352,13 +352,13 @@ public void cotWithPropertyArgument() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "1 / Math.tan((doc['age'].size() == 0 ? null : doc['age'].value))")); + "1 / Math.tan(doc['age'].value)")); ScriptFilter scriptFilter = CheckScriptContents.getScriptFilterFromQuery(query, parser); assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "1 / Math.tan((doc['balance'].size() == 0 ? null : doc['balance'].value))")); + "1 / Math.tan(doc['balance'].value)")); } @Test @@ -367,13 +367,13 @@ public void signWithFunctionPropertyArgument() { ScriptField scriptField = CheckScriptContents.getScriptFieldFromQuery(query); assertTrue(CheckScriptContents.scriptContainsString( scriptField, - "Math.signum((doc['age'].size() == 0 ? null : doc['age'].value))")); + "Math.signum(doc['age'].value)")); ScriptFilter scriptFilter = CheckScriptContents.getScriptFilterFromQuery(query, parser); assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.signum((doc['balance'].size() == 0 ? null : doc['balance'].value))")); + "Math.signum(doc['balance'].value)")); } @Test @@ -383,13 +383,13 @@ public void logWithOneParam() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.log((doc['age'].size() == 0 ? null : doc['age'].value))")); + "Math.log(doc['age'].value)")); ScriptFilter scriptFilter = CheckScriptContents.getScriptFilterFromQuery(query, parser); assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.log((doc['age'].size() == 0 ? null : doc['age'].value))")); + "Math.log(doc['age'].value)")); } @Test @@ -399,13 +399,13 @@ public void logWithTwoParams() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.log((doc['age'].size() == 0 ? null : doc['age'].value))/Math.log(3)")); + "Math.log(doc['age'].value)/Math.log(3)")); ScriptFilter scriptFilter = CheckScriptContents.getScriptFilterFromQuery(query, parser); assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.log((doc['age'].size() == 0 ? null : doc['age'].value))/Math.log(3)")); + "Math.log(doc['age'].value)/Math.log(3)")); } @Test @@ -415,7 +415,7 @@ public void log10Test() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.log10((doc['age'].size() == 0 ? null : doc['age'].value))" + "Math.log10(doc['age'].value)" ) ); } @@ -427,13 +427,13 @@ public void lnTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Math.log((doc['age'].size() == 0 ? null : doc['age'].value))")); + "Math.log(doc['age'].value)")); ScriptFilter scriptFilter = CheckScriptContents.getScriptFilterFromQuery(query, parser); assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Math.log((doc['age'].size() == 0 ? null : doc['age'].value))")); + "Math.log(doc['age'].value)")); } @Test @@ -455,7 +455,7 @@ public void randWithOneParamTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "new Random((doc['age'].size() == 0 ? null : doc['age'].value)).nextDouble()" + "new Random(doc['age'].value).nextDouble()" ) ); } diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/QueryFunctionsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/QueryFunctionsTest.java index 7c818ce6c6..ee9492a48e 100644 --- a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/QueryFunctionsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/QueryFunctionsTest.java @@ -218,7 +218,7 @@ public void ifFunctionWithConditionStatement() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "boolean cond = ((doc['age'].size() == 0 ? null : doc['age'].value) > 35);" + "boolean cond = (doc['age'].value > 35);" ) ); } @@ -230,7 +230,7 @@ public void ifFunctionWithEquationConditionStatement() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "boolean cond = ((doc['age'].size() == 0 ? null : doc['age'].value) == 35);" + "boolean cond = (doc['age'].value == 35);" ) ); } diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/StringOperatorsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/StringOperatorsTest.java index 3c54647bbd..258be08892 100644 --- a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/StringOperatorsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/StringOperatorsTest.java @@ -40,13 +40,13 @@ public void substringTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).substring(1, end)")); + "doc['lastname'].value.substring(1, end)")); ScriptFilter scriptFilter = CheckScriptContents.getScriptFilterFromQuery(query, parser); assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).substring(1, end)" + "doc['lastname'].value.substring(1, end)" ) ); } @@ -72,7 +72,7 @@ public void lengthTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).length()" + "doc['lastname'].value.length()" ) ); @@ -80,7 +80,7 @@ public void lengthTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).length()" + "doc['lastname'].value.length()" ) ); } @@ -94,7 +94,7 @@ public void replaceTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).replace('a','A')" + "doc['lastname'].value.replace('a','A')" ) ); @@ -102,7 +102,7 @@ public void replaceTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).replace('a','A')" + "doc['lastname'].value.replace('a','A')" ) ); } @@ -116,7 +116,7 @@ public void locateTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).indexOf('a', 0) + 1" + "doc['lastname'].value.indexOf('a', 0) + 1" ) ); @@ -124,7 +124,7 @@ public void locateTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).indexOf('a', 0) + 1" + "doc['lastname'].value.indexOf('a', 0) + 1" ) ); } @@ -138,8 +138,7 @@ public void ltrimTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Character.isWhitespace((doc['lastname'].size() == 0 " - + "? null : doc['lastname'].value).charAt(pos))" + "Character.isWhitespace(doc['lastname'].value.charAt(pos))" ) ); @@ -147,8 +146,7 @@ public void ltrimTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Character.isWhitespace((doc['lastname'].size() == 0 " - + "? null : doc['lastname'].value).charAt(pos))" + "Character.isWhitespace(doc['lastname'].value.charAt(pos))" ) ); } @@ -162,8 +160,7 @@ public void rtrimTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "Character.isWhitespace((doc['lastname'].size() == 0 " - + "? null : doc['lastname'].value).charAt(pos))" + "Character.isWhitespace(doc['lastname'].value.charAt(pos))" ) ); @@ -171,8 +168,7 @@ public void rtrimTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "Character.isWhitespace((doc['lastname'].size() == 0 " - + "? null : doc['lastname'].value).charAt(pos))" + "Character.isWhitespace(doc['lastname'].value.charAt(pos))" ) ); } @@ -186,7 +182,7 @@ public void asciiTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "(int) (doc['lastname'].size() == 0 ? null : doc['lastname'].value).charAt(0)" + "(int) doc['lastname'].value.charAt(0)" ) ); @@ -194,7 +190,7 @@ public void asciiTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "(int) (doc['lastname'].size() == 0 ? null : doc['lastname'].value).charAt(0)" + "(int) doc['lastname'].value.charAt(0)" ) ); } @@ -206,7 +202,7 @@ public void left() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).substring(0, len)" + "doc['lastname'].value.substring(0, len)" ) ); } @@ -218,7 +214,7 @@ public void right() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "(doc['lastname'].size() == 0 ? null : doc['lastname'].value).substring(start)" + "doc['lastname'].value.substring(start)" ) ); } diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/SqlParserTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/SqlParserTest.java index 4b0c946bd1..65077883fb 100644 --- a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/SqlParserTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/SqlParserTest.java @@ -92,8 +92,7 @@ public void whereConditionLeftFunctionRightPropertyGreatTest() throws Exception Assert.assertTrue(((Condition) (where.getWheres().get(0))).getValue() instanceof ScriptFilter); ScriptFilter scriptFilter = (ScriptFilter) (((Condition) (where.getWheres().get(0))).getValue()); - Assert.assertTrue(scriptFilter.getScript().contains( - "(doc['address'].size() == 0 ? null : doc['address'].value).split(' ')[0]")); + Assert.assertTrue(scriptFilter.getScript().contains("doc['address'].value.split(' ')[0]")); Pattern pattern = Pattern.compile("floor_\\d+ > doc\\['b'].value"); java.util.regex.Matcher matcher = pattern.matcher(scriptFilter.getScript()); Assert.assertTrue(matcher.find()); @@ -177,8 +176,7 @@ public void whereConditionLeftFunctionRightFunctionEqualTest() throws Exception Assert.assertTrue((where.getWheres().size() == 1)); Assert.assertTrue(((Condition) (where.getWheres().get(0))).getValue() instanceof ScriptFilter); ScriptFilter scriptFilter = (ScriptFilter) (((Condition) (where.getWheres().get(0))).getValue()); - Assert.assertTrue(scriptFilter.getScript().contains( - "(doc['address'].size() == 0 ? null : doc['address'].value).split(' ')[0]")); + Assert.assertTrue(scriptFilter.getScript().contains("doc['address'].value.split(' ')[0]")); Pattern pattern = Pattern.compile("floor_\\d+ == floor_\\d+"); java.util.regex.Matcher matcher = pattern.matcher(scriptFilter.getScript()); Assert.assertTrue(matcher.find()); @@ -655,8 +653,7 @@ public void scriptFiledPlusLiteralTest() throws SqlParseException { MethodField scriptMethod = (MethodField) field; Assert.assertEquals("script", scriptMethod.getName().toLowerCase()); Assert.assertEquals(2, scriptMethod.getParams().size()); - Assert.assertTrue(scriptMethod.getParams().get(1).toString().contains( - "(doc['field1'].size() == 0 ? null : doc['field1'].value) + 3")); + Assert.assertTrue(scriptMethod.getParams().get(1).toString().contains("doc['field1'].value + 3")); } @Test @@ -672,8 +669,7 @@ public void scriptFieldPlusFieldTest() throws SqlParseException { Assert.assertEquals("script", scriptMethod.getName().toLowerCase()); Assert.assertEquals(2, scriptMethod.getParams().size()); Assert.assertTrue(scriptMethod.getParams().get(1).toString() - .contains("(doc['field1'].size() == 0 ? null : doc['field1'].value) " - + "+ (doc['field2'].size() == 0 ? null : doc['field2'].value)")); + .contains("doc['field1'].value + doc['field2'].value")); } @@ -725,7 +721,9 @@ public void implicitScriptOnAggregation() throws SqlParseException { MethodField avgMethodField = (MethodField) field; Assert.assertEquals("avg", avgMethodField.getName().toLowerCase()); Assert.assertEquals(1, avgMethodField.getParams().size()); - Assert.assertTrue(avgMethodField.getParams().get(0).value.toString().contains("add_1")); + Assert.assertTrue(avgMethodField.getParams().get(0).value.toString().contains("doc['field1'].value")); + Assert.assertTrue(avgMethodField.getParams().get(0).value.toString().contains("doc['field2'].value")); + } @Test @@ -1208,7 +1206,7 @@ public void caseWhenSwitchTest() { Assert.assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "(doc['weather'].size() == 0 ? null : doc['weather'].value)=='Sunny'" + "doc['weather'].value=='Sunny'" ) ); } @@ -1227,9 +1225,8 @@ public void castToIntTest() throws Exception { String alias = (String) methodField.getParams().get(0).value; String scriptCode = (String) methodField.getParams().get(1).value; Assert.assertEquals("cast_age",alias); - Assert.assertTrue(scriptCode.contains("(doc['age'].size() == 0 ? null : doc['age'].value)")); - Assert.assertTrue(scriptCode.contains("Double.parseDouble((doc['age'].size() == 0 " - + "? null : doc['age'].value).toString()).intValue()")); + Assert.assertTrue(scriptCode.contains("doc['age'].value")); + Assert.assertTrue(scriptCode.contains("Double.parseDouble(doc['age'].value.toString()).intValue()")); } @Test @@ -1246,9 +1243,8 @@ public void castToLongTest() throws Exception { String alias = (String) methodField.getParams().get(0).value; String scriptCode = (String) methodField.getParams().get(1).value; Assert.assertEquals("cast_insert_time",alias); - Assert.assertTrue(scriptCode.contains("(doc['insert_time'].size() == 0 ? null : doc['insert_time'].value)")); - Assert.assertTrue(scriptCode.contains("Double.parseDouble((doc['insert_time'].size() == 0 " - + "? null : doc['insert_time'].value).toString()).longValue()")); + Assert.assertTrue(scriptCode.contains("doc['insert_time'].value")); + Assert.assertTrue(scriptCode.contains("Double.parseDouble(doc['insert_time'].value.toString()).longValue()")); } @Test @@ -1265,9 +1261,8 @@ public void castToFloatTest() throws Exception { String alias = (String) methodField.getParams().get(0).value; String scriptCode = (String) methodField.getParams().get(1).value; Assert.assertEquals("cast_age",alias); - Assert.assertTrue(scriptCode.contains("(doc['age'].size() == 0 ? null : doc['age'].value)")); - Assert.assertTrue(scriptCode.contains("Double.parseDouble((doc['age'].size() == 0 " - + "? null : doc['age'].value).toString()).floatValue()")); + Assert.assertTrue(scriptCode.contains("doc['age'].value")); + Assert.assertTrue(scriptCode.contains("Double.parseDouble(doc['age'].value.toString()).floatValue()")); } @Test @@ -1284,9 +1279,8 @@ public void castToDoubleTest() throws Exception { String alias = (String) methodField.getParams().get(0).value; String scriptCode = (String) methodField.getParams().get(1).value; Assert.assertEquals("cast_age",alias); - Assert.assertTrue(scriptCode.contains("(doc['age'].size() == 0 ? null : doc['age'].value)")); - Assert.assertTrue(scriptCode.contains("Double.parseDouble((doc['age'].size() == 0 " - + "? null : doc['age'].value).toString()).doubleValue()")); + Assert.assertTrue(scriptCode.contains("doc['age'].value")); + Assert.assertTrue(scriptCode.contains("Double.parseDouble(doc['age'].value.toString()).doubleValue()")); } @Test @@ -1303,7 +1297,7 @@ public void castToStringTest() throws Exception { String alias = (String) methodField.getParams().get(0).value; String scriptCode = (String) methodField.getParams().get(1).value; Assert.assertEquals("cast_age",alias); - Assert.assertTrue(scriptCode.contains("(doc['age'].size() == 0 ? null : doc['age'].value).toString()")); + Assert.assertTrue(scriptCode.contains("doc['age'].value.toString()")); } @Test @@ -1320,10 +1314,9 @@ public void castToDateTimeTest() throws Exception { String alias = (String) methodField.getParams().get(0).value; String scriptCode = (String) methodField.getParams().get(1).value; Assert.assertEquals("cast_age",alias); - Assert.assertTrue(scriptCode.contains("(doc['age'].size() == 0 ? null : doc['age'].value)")); + Assert.assertTrue(scriptCode.contains("doc['age'].value")); Assert.assertTrue(scriptCode.contains("DateTimeFormatter.ofPattern(\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\").format(" - + "DateTimeFormatter.ISO_DATE_TIME.parse((doc['age'].size() == 0 " - + "? null : doc['age'].value).toString()))")); + + "DateTimeFormatter.ISO_DATE_TIME.parse(doc['age'].value.toString()))")); } @Test @@ -1338,9 +1331,8 @@ public void castToDoubleThenDivideTest() throws Exception { Assert.assertEquals("script",castField.getName()); String scriptCode = (String) methodField.getParams().get(1).value; - Assert.assertTrue(scriptCode.contains("(doc['age'].size() == 0 ? null : doc['age'].value)")); - Assert.assertTrue(scriptCode.contains("Double.parseDouble((doc['age'].size() == 0 " - + "? null : doc['age'].value).toString()).doubleValue()")); + Assert.assertTrue(scriptCode.contains("doc['age'].value")); + Assert.assertTrue(scriptCode.contains("Double.parseDouble(doc['age'].value.toString()).doubleValue()")); Assert.assertTrue(scriptCode.contains("/ 2")); } diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/SQLFunctionsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/SQLFunctionsTest.java index e331e5734a..f8a664928b 100644 --- a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/SQLFunctionsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/SQLFunctionsTest.java @@ -102,11 +102,11 @@ public void testCastReturnType() { @Test public void testCastIntStatementScript() throws SqlParseException { assertEquals( - "def result = ((doc['age'].size() == 0 ? null : doc['age'].value) instanceof boolean) " - + "? ((doc['age'].size() == 0 ? null : doc['age'].value) ? 1 : 0) " - + ": Double.parseDouble((doc['age'].size() == 0 ? null : doc['age'].value).toString()).intValue()", + "def result = (doc['age'].value instanceof boolean) " + + "? (doc['age'].value ? 1 : 0) " + + ": Double.parseDouble(doc['age'].value.toString()).intValue()", sqlFunctions.getCastScriptStatement( - "result", "int", Arrays.asList(new KVValue(new SQLIdentifierExpr("age")))) + "result", "int", Arrays.asList(new KVValue("age"))) ); } From 3d10353b5488ed4d7aeabf7be9c9c704c65771e7 Mon Sep 17 00:00:00 2001 From: cip999 Date: Thu, 26 Aug 2021 00:39:00 +0200 Subject: [PATCH 36/37] Massive refactoring of modules handling SQL functions to allow for nesting --- .../sql/legacy/utils/SQLFunctions.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java index 77eba08e27..42c889a0b4 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java @@ -1034,9 +1034,13 @@ private Tuple isnull(SQLExpr expr) { public String getCastScriptStatement(String name, String castType, List paramers) throws SqlParseException { - SQLExpr expr = (SQLExpr) paramers.get(0).value; - String castFieldName = (expr instanceof SQLTextLiteralExpr ? getPropertyOrStringValue(expr) - : getPropertyOrValue(expr)); + Object expr = paramers.get(0).value; + String castFieldName; + if (expr instanceof String) { + castFieldName = getPropertyOrValue((String) expr); + } else { + castFieldName = getPropertyOrValue((SQLExpr) expr); + } scriptDeclare(expr); From 6b7ef8bb0289884834af3edbef758d607d94c55a Mon Sep 17 00:00:00 2001 From: cip999 Date: Thu, 26 Aug 2021 10:40:46 +0200 Subject: [PATCH 37/37] Fixed bug in SQL comparison functions implementation --- .../sql/legacy/utils/SQLFunctions.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java index 42c889a0b4..0330b8086b 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java @@ -653,10 +653,8 @@ private Tuple least(SQLExpr a, SQLExpr b) { private Tuple comparisonFunctionTemplate(SQLExpr a, SQLExpr b, String name, String comparisonOperator) { - String value_a = (a instanceof SQLTextLiteralExpr - ? getPropertyOrStringValue(a) : getPropertyOrValue(a)); - String value_b = (b instanceof SQLTextLiteralExpr - ? getPropertyOrStringValue(b) : getPropertyOrValue(b)); + String value_a = getPropertyOrStringValue(a); + String value_b = getPropertyOrStringValue(b); String nameString_a = nextId("string_a"); String nameString_b = nextId("string_b"); @@ -674,7 +672,7 @@ private Tuple comparisonFunctionTemplate(SQLExpr a, SQLExpr b, + String.format("else if (%s) {%s = %s;} ", checkIfNull(b), name, value_a) + "else {"; - if (isProperty(a) && isProperty(b)) { + if (isPropertyOrDefinition(a) && isPropertyOrDefinition(b)) { definition += String.format("if (%s instanceof Number) {", value_a) + numberComparison + "} else {" @@ -697,6 +695,13 @@ private static boolean isProperty(SQLExpr expr) { || expr instanceof SQLVariantRefExpr); } + private static boolean isPropertyOrDefinition(SQLExpr expr) { + return (expr instanceof SQLIdentifierExpr + || expr instanceof SQLPropertyExpr + || expr instanceof SQLVariantRefExpr + || expr instanceof SQLDefinitionExpr); + } + private static String getPropertyOrValue(SQLExpr expr) { if (expr instanceof SQLDefinitionExpr) { return ((SQLDefinitionExpr) expr).getName();