diff --git a/.github/workflows/sql-cli-release-workflow.yml b/.github/workflows/sql-cli-release-workflow.yml index 2959df51b6..baa70174fe 100644 --- a/.github/workflows/sql-cli-release-workflow.yml +++ b/.github/workflows/sql-cli-release-workflow.yml @@ -8,7 +8,7 @@ on: jobs: build: - runs-on: [ubuntu-16.04] + runs-on: [ubuntu-18.04] defaults: run: working-directory: sql-cli diff --git a/.github/workflows/sql-cli-test-and-build-workflow.yml b/.github/workflows/sql-cli-test-and-build-workflow.yml index 1af71516cd..69b2807ca7 100644 --- a/.github/workflows/sql-cli-test-and-build-workflow.yml +++ b/.github/workflows/sql-cli-test-and-build-workflow.yml @@ -5,7 +5,7 @@ on: [pull_request, push] jobs: build: - runs-on: [ubuntu-16.04] + runs-on: [ubuntu-18.04] defaults: run: working-directory: sql-cli diff --git a/.github/workflows/sql-jdbc-push-jdbc-maven.yml b/.github/workflows/sql-jdbc-push-jdbc-maven.yml index 53f2d5d391..4e5855c817 100644 --- a/.github/workflows/sql-jdbc-push-jdbc-maven.yml +++ b/.github/workflows/sql-jdbc-push-jdbc-maven.yml @@ -8,7 +8,7 @@ on: jobs: upload-jdbc-jar: - runs-on: [ubuntu-16.04] + runs-on: [ubuntu-18.04] defaults: run: working-directory: sql-jdbc diff --git a/.github/workflows/sql-release-workflow.yml b/.github/workflows/sql-release-workflow.yml index d2a5abcbb7..c487115f2a 100644 --- a/.github/workflows/sql-release-workflow.yml +++ b/.github/workflows/sql-release-workflow.yml @@ -12,7 +12,7 @@ jobs: java: [14] name: Build and Release SQL Plugin - runs-on: [ubuntu-16.04] + runs-on: [ubuntu-18.04] steps: - name: Checkout SQL diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDatetimeValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDatetimeValue.java index 2dd49383ac..509e835242 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDatetimeValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDatetimeValue.java @@ -17,6 +17,7 @@ import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.data.utils.ExprDateFormatters; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; import com.google.common.base.Objects; import java.time.Instant; @@ -26,9 +27,7 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; -import java.time.temporal.ChronoField; import java.time.temporal.ChronoUnit; import lombok.RequiredArgsConstructor; @@ -36,27 +35,13 @@ public class ExprDatetimeValue extends AbstractExprValue { private final LocalDateTime datetime; - private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS; - private static final int MIN_FRACTION_SECONDS = 0; - private static final int MAX_FRACTION_SECONDS = 6; - - static { - FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder() - .appendPattern("yyyy-MM-dd HH:mm:ss") - .appendFraction( - ChronoField.MICRO_OF_SECOND, - MIN_FRACTION_SECONDS, - MAX_FRACTION_SECONDS, - true) - .toFormatter(); - } - /** * Constructor with datetime string as input. */ public ExprDatetimeValue(String datetime) { try { - this.datetime = LocalDateTime.parse(datetime, FORMATTER_VARIABLE_MICROS); + this.datetime = LocalDateTime.parse(datetime, + ExprDateFormatters.TOLERANT_PARSER_DATE_TIME_FORMATTER); } catch (DateTimeParseException e) { throw new SemanticCheckException(String.format("datetime:%s in unsupported format, please " + "use yyyy-MM-dd HH:mm:ss[.SSSSSS]", datetime)); diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java index 09761dcfe9..a0d7364bcd 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java @@ -18,6 +18,7 @@ import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; @@ -80,6 +81,11 @@ public LocalTime timeValue() { } } + @Override + public Instant timestampValue() { + return new ExprTimestampValue(value).timestampValue(); + } + @Override public String toString() { return String.format("\"%s\"", value); diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java index 0c9c506569..77d04b97cd 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java @@ -19,6 +19,7 @@ import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.data.utils.ExprDateFormatters; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; import java.time.Instant; import java.time.LocalDate; @@ -45,14 +46,22 @@ public class ExprTimestampValue extends AbstractExprValue { /** * todo. only support timestamp in format yyyy-MM-dd HH:mm:ss. */ - private static final DateTimeFormatter FORMATTER_WITNOUT_NANO = DateTimeFormatter + private static final DateTimeFormatter FORMATTER_WITHOUT_NANO = DateTimeFormatter .ofPattern("yyyy-MM-dd HH:mm:ss"); + private static final DateTimeFormatter FORMATTER_WITHOUT_TIME = DateTimeFormatter + .ofPattern("yyyy-MM-dd"); private final Instant timestamp; + private String datetimeFormat = ALWAYS_INCLUDE_TIME; + private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS; private static final int MIN_FRACTION_SECONDS = 0; private static final int MAX_FRACTION_SECONDS = 6; + public static final String ALWAYS_INCLUDE_TIME = "always_include_time"; + public static final String NEVER_INCLUDE_TIME = "never_include_time"; + public static final String INCLUDE_TIME_WHEN_NONZERO = "include_time_when_nonzero"; + static { FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder() .appendPattern("yyyy-MM-dd HH:mm:ss") @@ -69,7 +78,8 @@ public class ExprTimestampValue extends AbstractExprValue { */ public ExprTimestampValue(String timestamp) { try { - this.timestamp = LocalDateTime.parse(timestamp, FORMATTER_VARIABLE_MICROS) + this.timestamp = LocalDateTime.parse(timestamp, + ExprDateFormatters.TOLERANT_PARSER_DATE_TIME_FORMATTER) .atZone(ZONE) .toInstant(); } catch (DateTimeParseException e) { @@ -81,9 +91,29 @@ public ExprTimestampValue(String timestamp) { @Override public String value() { - return timestamp.getNano() == 0 ? FORMATTER_WITNOUT_NANO.withZone(ZONE) - .format(timestamp.truncatedTo(ChronoUnit.SECONDS)) - : FORMATTER_VARIABLE_MICROS.withZone(ZONE).format(timestamp); + switch (datetimeFormat) { + case NEVER_INCLUDE_TIME: + return valueWithoutTime(); + + case INCLUDE_TIME_WHEN_NONZERO: + LocalTime time = timeValue(); + return (time.getHour() + time.getMinute() + time.getSecond() == 0) + ? valueWithoutTime() : valueWithTime(); + + case ALWAYS_INCLUDE_TIME: + default: + return valueWithTime(); + } + } + + private String valueWithTime() { + return timestamp.getNano() == 0 ? FORMATTER_WITHOUT_NANO.withZone(ZONE) + .format(timestamp.truncatedTo(ChronoUnit.SECONDS)) + : FORMATTER_VARIABLE_MICROS.withZone(ZONE).format(timestamp); + } + + private String valueWithoutTime() { + return FORMATTER_WITHOUT_TIME.withZone(ZONE).format(timestamp); } @Override @@ -130,4 +160,9 @@ public boolean equal(ExprValue other) { public int hashCode() { return Objects.hashCode(timestamp); } + + public void setDatetimeFormat(String format) { + this.datetimeFormat = format; + } + } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java index 7928dbb85f..50f1cd891c 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java @@ -131,4 +131,8 @@ public static List coreTypes() { public static List numberTypes() { return ImmutableList.of(INTEGER, LONG, FLOAT, DOUBLE); } + + public static List dateTypes() { + return ImmutableList.of(DATE, DATETIME, TIMESTAMP, TIME); + } } diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchDateFormatters.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprDateFormatters.java similarity index 84% rename from elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchDateFormatters.java rename to core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprDateFormatters.java index c653c063ac..818eb71375 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchDateFormatters.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprDateFormatters.java @@ -15,7 +15,7 @@ * */ -package com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value; +package com.amazon.opendistroforelasticsearch.sql.data.utils; import static java.time.temporal.ChronoField.DAY_OF_MONTH; import static java.time.temporal.ChronoField.HOUR_OF_DAY; @@ -37,7 +37,7 @@ * Reference org.elasticsearch.common.time.DateFormatters. */ @UtilityClass -public class ElasticsearchDateFormatters { +public class ExprDateFormatters { public static final DateTimeFormatter TIME_ZONE_FORMATTER_NO_COLON = new DateTimeFormatterBuilder() @@ -73,7 +73,16 @@ public class ElasticsearchDateFormatters { new DateTimeFormatterBuilder() .append(STRICT_YEAR_MONTH_DAY_FORMATTER) .optionalStart() + + // otherwise use space + .optionalStart() + .appendLiteral(' ') + .optionalEnd() + // optional T + .optionalStart() .appendLiteral('T') + .optionalEnd() + .optionalStart() .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE) .optionalStart() @@ -82,13 +91,18 @@ public class ElasticsearchDateFormatters { .optionalStart() .appendLiteral(':') .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) + + // optional millis with dot .optionalStart() .appendFraction(NANO_OF_SECOND, 1, 9, true) .optionalEnd() + + // otherwise optional millis use with comma .optionalStart() .appendLiteral(',') .appendFraction(NANO_OF_SECOND, 1, 9, false) .optionalEnd() + .optionalEnd() .optionalEnd() .optionalStart() @@ -104,4 +118,11 @@ public class ElasticsearchDateFormatters { public static final DateTimeFormatter SQL_LITERAL_DATE_TIME_FORMAT = DateTimeFormatter .ofPattern("yyyy-MM-dd HH:mm:ss"); + + public static final DateTimeFormatter TOLERANT_PARSER_DATE_TIME_FORMATTER = + new DateTimeFormatterBuilder() + .appendOptional(STRICT_DATE_OPTIONAL_TIME_FORMATTER) + .appendOptional(SQL_LITERAL_DATE_TIME_FORMAT) + .appendOptional(STRICT_HOUR_MINUTE_SECOND_FORMATTER) + .toFormatter(); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java index d08c3fab8f..b60179f6fc 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java @@ -25,16 +25,26 @@ import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionBuilder; import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionDSL; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName; import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionResolver; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionSignature; +import com.amazon.opendistroforelasticsearch.sql.expression.function.SerializableFunction; import com.amazon.opendistroforelasticsearch.sql.utils.OperatorUtils; import com.google.common.collect.ImmutableTable; import com.google.common.collect.Table; +import java.io.Serializable; +import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; import lombok.experimental.UtilityClass; +import org.apache.commons.lang3.tuple.Pair; + /** * The definition of binary predicate function @@ -192,53 +202,44 @@ private static FunctionResolver notEqual() { } private static FunctionResolver less() { - return FunctionDSL - .define(BuiltinFunctionName.LESS.getName(), ExprCoreType.coreTypes().stream() - .map(type -> FunctionDSL - .impl(FunctionDSL - .nullMissingHandling((v1, v2) -> ExprBooleanValue.of(v1.compareTo(v2) < 0)), - BOOLEAN, - type, type)) - .collect( - Collectors.toList())); + return FunctionDSL.define(BuiltinFunctionName.LESS.getName(), + getComparisonFunctions((Function & Serializable) value -> value < 0)); } private static FunctionResolver lte() { - return FunctionDSL - .define(BuiltinFunctionName.LTE.getName(), ExprCoreType.coreTypes().stream() - .map(type -> FunctionDSL - .impl( - FunctionDSL - .nullMissingHandling( - (v1, v2) -> ExprBooleanValue.of(v1.compareTo(v2) <= 0)), - BOOLEAN, - type, type)) - .collect( - Collectors.toList())); + return FunctionDSL.define(BuiltinFunctionName.LTE.getName(), + getComparisonFunctions((Function & Serializable) value -> value <= 0)); } private static FunctionResolver greater() { - return FunctionDSL - .define(BuiltinFunctionName.GREATER.getName(), ExprCoreType.coreTypes().stream() - .map(type -> FunctionDSL - .impl(FunctionDSL - .nullMissingHandling((v1, v2) -> ExprBooleanValue.of(v1.compareTo(v2) > 0)), - BOOLEAN, type, type)) - .collect( - Collectors.toList())); + return FunctionDSL.define(BuiltinFunctionName.GREATER.getName(), + getComparisonFunctions((Function & Serializable) value -> value > 0)); } private static FunctionResolver gte() { - return FunctionDSL - .define(BuiltinFunctionName.GTE.getName(), ExprCoreType.coreTypes().stream() - .map(type -> FunctionDSL - .impl( - FunctionDSL.nullMissingHandling( - (v1, v2) -> ExprBooleanValue.of(v1.compareTo(v2) >= 0)), - BOOLEAN, - type, type)) - .collect( - Collectors.toList())); + return FunctionDSL.define(BuiltinFunctionName.GTE.getName(), + getComparisonFunctions((Function & Serializable) value -> value >= 0)); + } + + private static List>> + getComparisonFunctions(Function f) { + List>> + comparisonFunctions = ExprCoreType.coreTypes() + .stream() + .map(type -> FunctionDSL.impl(FunctionDSL.nullMissingHandling( + (v1, v2) -> ExprBooleanValue.of(f.apply(v1.compareTo(v2)))), BOOLEAN, type, type)) + .collect(Collectors.toList()); + + // all string to date conversions + comparisonFunctions.addAll(ExprCoreType.dateTypes() + .stream() + .map(type -> FunctionDSL.impl(FunctionDSL.nullMissingHandling((v1, v2) -> { + // casting V2 from string to whatever type v1 is + ExprValue v2Casted = ExprValueUtils.fromObjectValue(v2.value(), (ExprCoreType) v1.type()); + return ExprBooleanValue.of(f.apply(v1.compareTo(v2Casted))); + }), BOOLEAN, type, STRING)) + .collect(Collectors.toList())); + return comparisonFunctions; } private static FunctionResolver like() { diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java index 294c21ec52..dfd80ad2ed 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java @@ -17,6 +17,8 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue.INCLUDE_TIME_WHEN_NONZERO; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue.NEVER_INCLUDE_TIME; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIME; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP; @@ -113,20 +115,9 @@ public void timeInUnsupportedFormat() { public void timestampInUnsupportedFormat() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprTimestampValue("2020-07-07T01:01:01Z")); + () -> new ExprTimestampValue("2020-07-07T1:01:01Z")); assertEquals( - "timestamp:2020-07-07T01:01:01Z in unsupported format, " - + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", - exception.getMessage()); - } - - @Test - public void datetimeInUnsupportedFormat() { - SemanticCheckException exception = - assertThrows(SemanticCheckException.class, - () -> new ExprDatetimeValue("2020-07-07T01:01:01Z")); - assertEquals( - "datetime:2020-07-07T01:01:01Z in unsupported format, " + "timestamp:2020-07-07T1:01:01Z in unsupported format, " + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", exception.getMessage()); } @@ -142,9 +133,9 @@ public void stringDateTimeValue() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprStringValue("2020-07-07T01:01:01Z").datetimeValue()); + () -> new ExprStringValue("2020-07-07T01:01:01Z12345678").datetimeValue()); assertEquals( - "datetime:2020-07-07T01:01:01Z in unsupported format, " + "datetime:2020-07-07T01:01:01Z12345678 in unsupported format, " + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", exception.getMessage()); } @@ -239,9 +230,9 @@ public void datetimeWithVariableMicroPrecision() { public void timestampOverMaxMicroPrecision() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprTimestampValue("2020-07-07 01:01:01.1234567")); + () -> new ExprTimestampValue("2020-07-07 01:01:01.12345678910")); assertEquals( - "timestamp:2020-07-07 01:01:01.1234567 in unsupported format, " + "timestamp:2020-07-07 01:01:01.12345678910 in unsupported format, " + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", exception.getMessage()); } @@ -250,9 +241,9 @@ public void timestampOverMaxMicroPrecision() { public void datetimeOverMaxMicroPrecision() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprDatetimeValue("2020-07-07 01:01:01.1234567")); + () -> new ExprDatetimeValue("2020-07-07 01:01:01.1234567890")); assertEquals( - "datetime:2020-07-07 01:01:01.1234567 in unsupported format, " + "datetime:2020-07-07 01:01:01.1234567890 in unsupported format, " + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", exception.getMessage()); } @@ -261,9 +252,30 @@ public void datetimeOverMaxMicroPrecision() { public void timeOverMaxMicroPrecision() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprTimeValue("01:01:01.1234567")); + () -> new ExprTimeValue("01:01:01.1234567891")); assertEquals( - "time:01:01:01.1234567 in unsupported format, please use HH:mm:ss[.SSSSSS]", + "time:01:01:01.1234567891 in unsupported format, please use HH:mm:ss[.SSSSSS]", exception.getMessage()); } + + @Test + public void timestampValueNeverIncludeTime() { + ExprTimestampValue timestampValue = new ExprTimestampValue("2020-07-07 01:01:01"); + timestampValue.setDatetimeFormat(NEVER_INCLUDE_TIME); + + assertEquals("2020-07-07", timestampValue.value()); + } + + @Test + public void timestampValueIncludeTimeWhenNonzero() { + ExprTimestampValue firstTimestampValue = new ExprTimestampValue("2020-07-07 00:00:01"); + firstTimestampValue.setDatetimeFormat(INCLUDE_TIME_WHEN_NONZERO); + + ExprTimestampValue secondTimestampValue = new ExprTimestampValue("2020-07-07 00:00:00.291000"); + secondTimestampValue.setDatetimeFormat(INCLUDE_TIME_WHEN_NONZERO); + + assertEquals("2020-07-07 00:00:01", firstTimestampValue.value()); + assertEquals("2020-07-07", secondTimestampValue.value()); + } + } diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java index aa7402142c..42309faf0c 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java @@ -40,12 +40,16 @@ import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprByteValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprCollectionValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDateValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDatetimeValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDoubleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprFloatValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprLongValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprShortValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprStringValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimeValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; @@ -148,6 +152,34 @@ private static Stream testCompareValueArguments() { builder.add(Arguments.of(new ExprShortValue(1), new ExprShortValue(1))); builder.add(Arguments.of(new ExprShortValue(1), new ExprShortValue(2))); builder.add(Arguments.of(new ExprShortValue(2), new ExprShortValue(1))); + + builder.add(Arguments.of(new ExprDatetimeValue("2006-01-01 00:00:01"), + new ExprStringValue("2006-01-01 00:00:00"))); + builder.add(Arguments.of(new ExprDatetimeValue("2006-01-01 00:00:00"), + new ExprStringValue("2006-01-01 00:00:01"))); + builder.add(Arguments.of(new ExprDatetimeValue("2006-01-01 00:00:00"), + new ExprStringValue("2006-01-01 00:00:00"))); + + builder.add(Arguments.of(new ExprDateValue("2006-01-01"), + new ExprStringValue("2006-01-01"))); + builder.add(Arguments.of(new ExprDateValue("2006-01-01"), + new ExprStringValue("2006-01-02"))); + builder.add(Arguments.of(new ExprDateValue("2006-01-02"), + new ExprStringValue("2006-01-01"))); + + builder.add(Arguments.of(new ExprTimestampValue("2006-01-01 00:00:01"), + new ExprStringValue("2006-01-01 00:00:00"))); + builder.add(Arguments.of(new ExprTimestampValue("2006-01-01 00:00:00"), + new ExprStringValue("2006-01-01 00:00:01"))); + builder.add(Arguments.of(new ExprTimestampValue("2006-01-01 00:00:00"), + new ExprStringValue("2006-01-01 00:00:00"))); + + builder.add(Arguments.of(new ExprTimeValue("00:00:01"), + new ExprStringValue("00:00:00"))); + builder.add(Arguments.of(new ExprTimeValue("00:00:00"), + new ExprStringValue("00:00:01"))); + builder.add(Arguments.of(new ExprTimeValue("00:00:00"), + new ExprStringValue("00:00:00"))); return builder.build(); } diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/ComparisonUtil.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/ComparisonUtil.java index 6913ce6a50..26d1515f07 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/ComparisonUtil.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/ComparisonUtil.java @@ -22,13 +22,19 @@ import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getStringValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprByteValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDateValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDatetimeValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDoubleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprFloatValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprLongValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprShortValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprStringValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimeValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; public class ComparisonUtil { @@ -57,6 +63,14 @@ public static int compare(ExprValue v1, ExprValue v2) { return getDoubleValue(v1).compareTo(getDoubleValue(v2)); } else if (v1 instanceof ExprStringValue) { return getStringValue(v1).compareTo(getStringValue(v2)); + } else if (v1 instanceof ExprDateValue) { + return v1.dateValue().compareTo(v2.dateValue()); + } else if (v1 instanceof ExprDatetimeValue) { + return v1.datetimeValue().compareTo(v2.datetimeValue()); + } else if (v1 instanceof ExprTimeValue) { + return v1.timeValue().compareTo(v2.timeValue()); + } else if (v1 instanceof ExprTimestampValue) { + return v1.timestampValue().compareTo(v2.timestampValue()); } else { throw new ExpressionEvaluationException( String.format("%s instances are not comparable", v1.getClass().getSimpleName())); diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java index 2fbae486d6..b0f768ce94 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java @@ -36,9 +36,6 @@ import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_IP; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT_KEYWORD; -import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchDateFormatters.SQL_LITERAL_DATE_TIME_FORMAT; -import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchDateFormatters.STRICT_DATE_OPTIONAL_TIME_FORMATTER; -import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchDateFormatters.STRICT_HOUR_MINUTE_SECOND_FORMATTER; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprByteValue; @@ -57,6 +54,7 @@ import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.data.utils.ExprDateFormatters; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.utils.Content; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.utils.ElasticsearchJsonContent; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.utils.ObjectContent; @@ -64,8 +62,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import java.time.Instant; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -87,13 +83,6 @@ public class ElasticsearchExprValueFactory { @Setter private Map typeMapping; - private static final DateTimeFormatter DATE_TIME_FORMATTER = - new DateTimeFormatterBuilder() - .appendOptional(SQL_LITERAL_DATE_TIME_FORMAT) - .appendOptional(STRICT_DATE_OPTIONAL_TIME_FORMATTER) - .appendOptional(STRICT_HOUR_MINUTE_SECOND_FORMATTER) - .toFormatter(); - private static final String TOP_PATH = ""; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -185,7 +174,8 @@ private ExprValue constructTimestamp(String value) { try { return new ExprTimestampValue( // Using Elasticsearch DateFormatters for now. - DateFormatters.from(DATE_TIME_FORMATTER.parse(value)).toInstant()); + DateFormatters.from(ExprDateFormatters.TOLERANT_PARSER_DATE_TIME_FORMATTER + .parse(value)).toInstant()); } catch (DateTimeParseException e) { throw new IllegalStateException( String.format( diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java index badf6afb57..a40921a64c 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java @@ -190,6 +190,12 @@ public void constructDate() { assertEquals( new ExprTimestampValue("2015-01-01 12:10:30"), tupleValue("{\"timestampV\":\"2015-01-01 12:10:30\"}").get("timestampV")); + assertEquals( + new ExprTimestampValue("2020-08-17T19:44:00.100500"), + tupleValue("{\"timestampV\":\"2020-08-17T19:44:00.100500\"}").get("timestampV")); + assertEquals( + new ExprTimestampValue("2020-08-17 19:44:00.100500"), + tupleValue("{\"timestampV\":\"2020-08-17 19:44:00.100500\"}").get("timestampV")); assertEquals( new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), tupleValue("{\"timestampV\":1420070400001}").get("timestampV")); @@ -221,9 +227,10 @@ public void constructDate() { public void constructDateFromUnsupportedFormatThrowException() { IllegalStateException exception = assertThrows( - IllegalStateException.class, () -> tupleValue("{\"timestampV\":\"2015-01-01 12:10\"}")); + IllegalStateException.class, () -> + tupleValue("{\"timestampV\":\"2015-01-01 1:10:10\"}")); assertEquals( - "Construct ExprTimestampValue from \"2015-01-01 12:10\" failed, " + "Construct ExprTimestampValue from \"2015-01-01 1:10:10\" failed, " + "unsupported date format.", exception.getMessage()); } diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilderTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilderTest.java index 4de1956cdc..a77aa2ff26 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilderTest.java @@ -16,8 +16,12 @@ package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATETIME; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIME; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT_KEYWORD; import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.literal; import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.ref; @@ -26,12 +30,15 @@ import static org.mockito.Mockito.doAnswer; import com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.serialization.ExpressionSerializer; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; import com.amazon.opendistroforelasticsearch.sql.expression.config.ExpressionConfig; import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.List; import java.util.Map; import org.json.JSONObject; import org.junit.jupiter.api.BeforeEach; @@ -39,6 +46,9 @@ import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -99,6 +109,45 @@ void should_build_range_query_for_comparison_expression() { buildQuery(expr))); } + private static List validDateAutocast() { + + return Arrays.asList( + Arguments.of(DATE, "2006-01-01"), + Arguments.of(TIME, "00:00:00"), + Arguments.of(TIMESTAMP, "2006-01-01 00:00:00"), + Arguments.of(DATETIME, "2006-01-01 00:00:00") + ); + } + + @ParameterizedTest + @MethodSource("validDateAutocast") + void shouldBuildRangeQueryForDateComparisonExpressionAutocast(ExprType exprType, + final String time) { + Expression[] params = {ref("date", exprType), literal(time)}; + + String serializedString = "\"" + time + "\""; + Map ranges = ImmutableMap.of( + dsl.less(params), new Object[]{null, serializedString, true, false}, + dsl.greater(params), new Object[]{serializedString, null, false, true}, + dsl.lte(params), new Object[]{null, serializedString, true, true}, + dsl.gte(params), new Object[]{serializedString, null, true, true}); + + ranges.forEach((expr, range) -> + assertJsonEquals( + "{\n" + + " \"range\" : {\n" + + " \"date\" : {\n" + + " \"from\" : " + range[0] + ",\n" + + " \"to\" : " + range[1] + ",\n" + + " \"include_lower\" : " + range[2] + ",\n" + + " \"include_upper\" : " + range[3] + ",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + "}", + buildQuery(expr))); + } + @Test void should_build_wildcard_query_for_like_expression() { assertJsonEquals( diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java index 862363df8c..b8a4de1756 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.Locale; import org.junit.Assert; +import org.junit.Assume; import org.junit.Test; /** @@ -144,4 +145,44 @@ public void matchPhraseQueryTest() throws IOException { Assert.assertThat(result, containsString("{\"match_phrase\":{\"address\":{\"query\":\"671 Bristol Street\"")); } + + @Test + public void testFieldToFieldComparison() throws IOException { + Assume.assumeFalse(isNewQueryEngineEabled()); + String result = explainQuery("select * from " + TestsConstants.TEST_INDEX_ACCOUNT + + " where age > account_number"); + + Assert.assertThat(result, + containsString("{\"script\":{\"script\":{\"source\":\"doc['age'].value " + + "> doc['account_number'].value\",\"lang\":\"painless\"}")); + + } + + @Test + public void testFieldToFieldComparisonWithExtraClause() throws IOException { + Assume.assumeFalse(isNewQueryEngineEabled()); + String result = explainQuery("select * from " + TestsConstants.TEST_INDEX_ACCOUNT + + " where age > account_number AND age in (1)"); + + Assert.assertThat(result, + containsString("{\"script\":{\"script\":{\"source\":\"doc['age'].value " + + "> doc['account_number'].value\",\"lang\":\"painless\"}")); + Assert.assertThat(result, + containsString("{\"term\":{\"age\":{\"value\":1,\"boost\":1.0}}}")); + + } + + @Test + public void testFieldToFieldComparisonWithDifferentOperator() throws IOException { + Assume.assumeFalse(isNewQueryEngineEabled()); + String result = explainQuery("select * from " + TestsConstants.TEST_INDEX_ACCOUNT + + " where age <> account_number AND age in (1)"); + + Assert.assertThat(result, + containsString("{\"script\":{\"script\":{\"source\":\"doc['age'].value " + + "!= doc['account_number'].value\",\"lang\":\"painless\"}")); + Assert.assertThat(result, + containsString("{\"term\":{\"age\":{\"value\":1,\"boost\":1.0}}}")); + + } } diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java index 5039e9e61e..0cff5986ee 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java @@ -76,6 +76,7 @@ protected void init() throws Exception { loadIndex(Index.PHRASE); loadIndex(Index.GAME_OF_THRONES); loadIndex(Index.NESTED); + loadIndex(Index.BOOKS_NESTED_WITH_SUBPROPERTIES); } @Override @@ -518,6 +519,52 @@ public void fieldOrderOther() throws IOException { testFieldOrder(expectedFields, expectedValues); } + @Test + public void selectSubSubFieldOfNestedField() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT nested(authors.info.name, authors), " + + "performance.revenue " + + "FROM %s", TestsConstants.TEST_INDEX_BOOKS)); + + List fields = Arrays.asList("authors.info.name", "performance.revenue"); + JSONArray dataRows = getDataRows(response); + assertContainsColumns(getSchema(response), fields); + assertContainsData(dataRows, fields); + assertEquals(3, dataRows.length()); + } + + @Test + public void selectMultipleNestedFields() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT nested(authors.info.name, authors), " + + "nested(performance.sells.year) " + + "FROM %s", TestsConstants.TEST_INDEX_BOOKS)); + + List fields = Arrays.asList("authors.info.name", "performance.sells.year"); + JSONArray dataRows = getDataRows(response); + assertContainsColumns(getSchema(response), fields); + assertContainsData(dataRows, fields); + assertEquals(4, dataRows.length()); + } + + @Test + public void nestedFieldWithLimit() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT nested(authors.info.name, authors) " + + "FROM %s LIMIT 1", TestsConstants.TEST_INDEX_BOOKS)); + assertEquals(1, getDataRows(response).length()); + } + + @Test + public void multipleNestedFieldsWithLimit() throws IOException, InterruptedException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT nested(authors.info.name, authors), " + + "nested(performance.sells.year)" + + "FROM %s LIMIT 3", TestsConstants.TEST_INDEX_BOOKS)); + assertEquals(3, getDataRows(response).length()); + } + + private void testFieldOrder(final String[] expectedFields, final Object[] expectedValues) throws IOException { diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java index fc9088a24a..8f6b828fbd 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java @@ -17,6 +17,8 @@ package com.amazon.opendistroforelasticsearch.sql.legacy; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME; import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.hitAny; import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvDouble; import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvInt; @@ -68,6 +70,7 @@ protected void init() throws Exception { loadIndex(Index.BANK); loadIndex(Index.ONLINE); loadIndex(Index.DATE); + loadIndex(Index.DATETIME); } @Test @@ -877,6 +880,31 @@ public void literalMultiField() throws Exception { assertThat(hits[0].getFields(), allOf(hasValue(contains(1)), hasValue(contains(2)))); } + @Test + public void greatestWithAliasAndStrings() throws Exception { + JSONObject response = + executeJdbcRequest("SELECT greatest(firstname, lastname) AS max " + + "FROM " + TEST_INDEX_ACCOUNT + " LIMIT 3"); + + verifyDataRows(response, + rows("duke"), + rows("hattie"), + rows("nanette")); + } + + @Test + public void complexNestedFunctions() throws Exception { + JSONObject response = + executeJdbcRequest("SELECT if(greatest(account_number / 2, age - 2) = 34, " + + "concat_ws(' ', firstname, lastname), 0) " + + "FROM " + TEST_INDEX_ACCOUNT + " LIMIT 3"); + + verifyDataRows(response, + rows(0), + rows("hattie bond"), + rows(0)); + } + private SearchHits query(String query) throws IOException { final String rsp = executeQueryWithStringOutput(query); diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java index 9a10d3f054..3edd35ecc4 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java @@ -41,6 +41,7 @@ import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getResponseBody; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getStringIndexMapping; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getWeblogsIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getBooksIndexMapping; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.isIndexExist; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.loadDataByRestClient; import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.CURSOR_CLOSE_ENDPOINT; @@ -222,8 +223,12 @@ protected Request getSqlCursorCloseRequest(String cursorRequest) { } protected String executeQuery(String query, String requestType) { + return executeQuery(query, requestType, "always_include_time"); + } + + protected String executeQuery(String query, String requestType, String datetimeFormat) { try { - String endpoint = "/_opendistro/_sql?format=" + requestType; + String endpoint = "/_opendistro/_sql?format=" + requestType + "&datetime_format=" + datetimeFormat; String requestBody = makeRequest(query); Request sqlRequest = new Request("POST", endpoint); @@ -556,7 +561,11 @@ public enum Index { DATA_TYPE_NONNUMERIC(TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC, "_doc", getDataTypeNonnumericIndexMapping(), - "src/test/resources/datatypes.json"); + "src/test/resources/datatypes.json"), + BOOKS_NESTED_WITH_SUBPROPERTIES(TestsConstants.TEST_INDEX_BOOKS, + "nestedType", + getBooksIndexMapping(), + "src/test/resources/books.json"); private final String name; private final String type; diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestUtils.java index 1bb43bfb8d..d7662cd494 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestUtils.java @@ -253,6 +253,11 @@ public static String getDataTypeNonnumericIndexMapping() { return getMappingFile(mappingFile); } + public static String getBooksIndexMapping() { + String mappingFile = "books_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static void loadBulk(Client client, String jsonPath, String defaultIndex) throws Exception { System.out.println(String.format("Loading file %s into elasticsearch cluster", jsonPath)); diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java index a4aec50d98..c8a184ce97 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java @@ -60,6 +60,7 @@ public class TestsConstants { public final static String TEST_INDEX_STRINGS = TEST_INDEX + "_strings"; public final static String TEST_INDEX_DATATYPE_NUMERIC = TEST_INDEX + "_datatypes_numeric"; public final static String TEST_INDEX_DATATYPE_NONNUMERIC = TEST_INDEX + "_datatypes_nonnumeric"; + public final static String TEST_INDEX_BOOKS = TEST_INDEX + "_nested_subproperties"; public final static String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; public final static String TS_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/JdbcFormatIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/JdbcFormatIT.java index 836cb5636c..1515c65d46 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/JdbcFormatIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/JdbcFormatIT.java @@ -17,8 +17,7 @@ package com.amazon.opendistroforelasticsearch.sql.sql; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.*; import com.amazon.opendistroforelasticsearch.sql.legacy.SQLIntegTestCase; import org.json.JSONObject; @@ -29,6 +28,7 @@ public class JdbcFormatIT extends SQLIntegTestCase { @Override protected void init() throws Exception { loadIndex(Index.BANK); + loadIndex(Index.DATETIME); } @Test diff --git a/integ-test/src/test/resources/books.json b/integ-test/src/test/resources/books.json new file mode 100644 index 0000000000..f6dd1c3286 --- /dev/null +++ b/integ-test/src/test/resources/books.json @@ -0,0 +1,4 @@ +{"index": {"_id": "1"}} +{"authors": [{"id": 8634, "info": {"name": "Andrea", "age": 22}}, {"id": 836, "info": {"name": "Carmen", "age": 19}}], "performance": {"revenue": 932.88, "sells": [{"year": 2018, "quantity": {"italy": 1000000, "abroad": 500000}}, {"year": 2020, "quantity": {"italy": 2001002, "abroad": 0}}]}} +{"index": {"_id": "2"}} +{"authors": [{"id": 8, "info": {"name": "Mark", "age": 80}}], "performance": {"revenue": 555.0, "sells": []}} diff --git a/integ-test/src/test/resources/indexDefinitions/books_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/books_index_mapping.json new file mode 100644 index 0000000000..2db565c72a --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/books_index_mapping.json @@ -0,0 +1,61 @@ +{ + "mappings": { + "properties": { + "authors": { + "type": "nested", + "properties": { + "id": { + "type": "long", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "info": { + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "age": { + "type": "long" + } + } + } + } + }, + "performance": { + "properties": { + "revenue": { + "type": "double" + }, + "sells": { + "type": "nested", + "properties": { + "year": { + "type": "long" + }, + "quantity": { + "properties": { + "italy": { + "type": "long" + }, + "abroad": { + "type": "long" + } + } + } + } + } + } + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/date_time_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/date_time_index_mapping.json index 6f1d465a82..a0631a1260 100644 --- a/integ-test/src/test/resources/indexDefinitions/date_time_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/date_time_index_mapping.json @@ -1,7 +1,7 @@ { "mappings": { "properties": { - "birthday": { + "login_time": { "type": "date" } } diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ScalarFunction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ScalarFunction.java index 7ba1e0f618..997b052be8 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ScalarFunction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ScalarFunction.java @@ -50,8 +50,10 @@ public enum ScalarFunction implements TypeExpression { CURDATE(func().to(ESDataType.DATE)), DATE(func(ESDataType.DATE).to(ESDataType.DATE)), DATE_FORMAT( - func(ESDataType.DATE, STRING).to(STRING), - func(ESDataType.DATE, STRING, STRING).to(STRING) + func(ESDataType.DATE, STRING).to(STRING), + func(ESDataType.DATE, STRING, STRING).to(STRING), + func(T(STRING), STRING).to(STRING), + func(T(STRING), STRING, STRING).to(STRING) ), DAYOFMONTH(func(ESDataType.DATE).to(INTEGER)), DEGREES(func(T(NUMBER)).to(DOUBLE)), @@ -60,9 +62,19 @@ public enum ScalarFunction implements TypeExpression { EXP(func(T(NUMBER)).to(T)), EXPM1(func(T(NUMBER)).to(T)), FLOOR(func(T(NUMBER)).to(T)), + GREATEST( + func(T(NUMBER), NUMBER).to(T), + func(T(STRING), STRING).to(T), + func(ESDataType.DATE, ESDataType.DATE).to(ESDataType.DATE) + ), IF(func(BOOLEAN, ES_TYPE, ES_TYPE).to(ES_TYPE)), IFNULL(func(ES_TYPE, ES_TYPE).to(ES_TYPE)), ISNULL(func(ES_TYPE).to(INTEGER)), + LEAST( + func(T(NUMBER), NUMBER).to(T), + func(T(STRING), STRING).to(T), + func(ESDataType.DATE, ESDataType.DATE).to(ESDataType.DATE) + ), LEFT(func(T(STRING), INTEGER).to(T)), LENGTH(func(STRING).to(INTEGER)), LN(func(T(NUMBER)).to(DOUBLE)), diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Where.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Where.java index 8b7f436f76..d929ba8280 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Where.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Where.java @@ -31,16 +31,34 @@ public static Where newInstance() { return new Where(CONN.AND); } + public static Where newInstance(boolean isJoin) { + return new Where(CONN.AND, isJoin); + } + private LinkedList wheres = new LinkedList<>(); protected CONN conn; + protected final boolean isJoin; public Where(String connStr) { - this.conn = CONN.valueOf(connStr.toUpperCase()); + this(connStr, false); + } + + public Where(String connStr, boolean isJoin) { + this(CONN.valueOf(connStr.toUpperCase()), isJoin); } public Where(CONN conn) { + this(conn, false); + } + + public Where(CONN conn, boolean isJoin) { this.conn = conn; + this.isJoin=isJoin; + } + + public boolean isJoin() { + return isJoin; } public void addWhere(Where where) { diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java index a6f4dadeef..cae1bf065a 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java @@ -87,6 +87,7 @@ public class SelectResultSet extends ResultSet { private long size; private long totalHits; private long internalTotalHits; + private Integer limitRowCount; private List rows; private Cursor cursor; @@ -118,6 +119,9 @@ public SelectResultSet(Client client, this.schema = new Schema(indexName, typeName, columns); this.head = schema.getHeaders(); this.dateFieldFormatter = new DateFieldFormatter(indexName, columns, fieldAliasMap); + if (query instanceof Select) { + this.limitRowCount = ((Select) query).getRowCount(); + } extractData(); populateCursor(); @@ -643,14 +647,53 @@ private long rowsLeft(Integer fetchSize, Integer limit) { private List populateRows(SearchHits searchHits) { List rows = new ArrayList<>(); - Set newKeys = new HashSet<>(head); + List> rowsAsMap = populateRows(head, searchHits, limitRowCount); + for (Map rowAsMap : rowsAsMap) { + rows.add(new DataRows.Row(rowAsMap)); + } + return rows; + } + + /** + * Returns a list of "flattened" rows as (path, value) hashmaps. + * The core idea is to flatten non-nested hits, and recur on inner hits + * by calling the method flatNestedField(), which in turn calls back + * populateRows. + *

+ * Sample input: + * keys = {"authors.info.name"} (authors is nested) + * searchHits = { + * { + * key = "authors" + * innerHits = { + * {"info.name": "Andrea"}, + * {"info.name": "Carmen} + * } + * } + * } + * Sample output: + * rows = [{"authors.info.name": "Andrea"}, {"authors.info.name": "Carmen"}] + *

+ */ + private List> populateRows(List keys, SearchHits searchHits, Integer remainingRows) { + List> rows = new ArrayList<>(); + for (SearchHit hit : searchHits) { + if (remainingRows != null && remainingRows == 0) { + return rows; + } + Map rowSource = hit.getSourceAsMap(); - List result; + List> result = new ArrayList<>(); if (!isJoinQuery()) { // Row already flatten in source in join. And join doesn't support nested fields for now. - rowSource = flatRow(head, rowSource); + result = flatNestedField(keys, hit.getInnerHits(), remainingRows); + if (remainingRows != null) { + remainingRows -= result.size(); + } + + rowSource = flatRow(keys, rowSource); rowSource.put(SCORE, hit.getScore()); for (Map.Entry field : hit.getFields().entrySet()) { @@ -659,13 +702,18 @@ private List populateRows(SearchHits searchHits) { if (formatType.equalsIgnoreCase(Format.JDBC.getFormatName())) { dateFieldFormatter.applyJDBCDateFormat(rowSource); } - result = flatNestedField(newKeys, rowSource, hit.getInnerHits()); + + for (Map row : result) { + row.putAll(rowSource); + } } else { if (formatType.equalsIgnoreCase(Format.JDBC.getFormatName())) { dateFieldFormatter.applyJDBCDateFormat(rowSource); } - result = new ArrayList<>(); - result.add(new DataRows.Row(rowSource)); + result.add(rowSource); + if (remainingRows != null) { + remainingRows--; + } } rows.addAll(result); @@ -781,6 +829,7 @@ private Map addNumericAggregation(List aggs, Map flatRow(List keys, Map row) { Map flattenedRow = new HashMap<>(); for (String key : keys) { @@ -813,97 +862,85 @@ private Map flatRow(List keys, Map row) } /** - * If innerHits associated with column name exists, flatten both the inner field name and the inner rows in it. - *

- * Sample input: - * newKeys = {'region', 'employees.age'}, row = {'region': 'US'} - * innerHits = employees: { - * hits: [{ - * source: { - * age: 26, - * firstname: 'Hank' - * } - * },{ - * source: { - * age: 30, - * firstname: 'John' - * } - * }] - * } + * Flattens the inner hits passed as argument. + * For each inner hit, iterates on its column names colName and computes a list of new keys. + * The new keys are the original keys that begin with colName + "." stripped of that prefix. + * Then, calls populateRows on the new keys and the hits associated to colName. + * This also works for nested fields which contain another nested field as a subfield. */ - private List flatNestedField(Set newKeys, Map row, - Map innerHits) { - List result = new ArrayList<>(); - result.add(new DataRows.Row(row)); + private List> flatNestedField(List keys, + Map innerHits, + Integer remainingRows) { + List> result = new ArrayList<>(); + result.add(new HashMap<>()); if (innerHits == null) { return result; } for (String colName : innerHits.keySet()) { - SearchHit[] colValue = innerHits.get(colName).getHits(); - doFlatNestedFieldName(colName, colValue, newKeys); - result = doFlatNestedFieldValue(colName, colValue, result); - } + List newKeys = new ArrayList<>(); + for (String key : keys) { + if (key.startsWith(colName) && !key.equals(colName)) { + newKeys.add(key.substring(colName.length() + 1)); + } + } - return result; - } + List> innerResult = populateRows(newKeys, innerHits.get(colName), + (remainingRows == null) ? null : + ((result.isEmpty() ? 0 : ratioCeil(remainingRows, result.size())))); + for (Map innerRow : innerResult) { + Map row = new HashMap<>(); + for (String path : innerRow.keySet()) { + if (!path.equals(SCORE)) { + row.put(colName + "." + path, innerRow.get(path)); + } else { + row.put(path, innerRow.get(path)); + } + } + innerRow.clear(); + innerRow.putAll(row); + } - private void doFlatNestedFieldName(String colName, SearchHit[] colValue, Set keys) { - Map innerRow = colValue[0].getSourceAsMap(); - for (String field : innerRow.keySet()) { - String innerName = colName + "." + field; - keys.add(innerName); + // In the case of multiple sets of inner hits, returns all possible combinations of entries + result = cartesianProduct(result, innerResult, remainingRows); } - keys.remove(colName); + return result; } /** - * Do Cartesian Product between current outer row and inner rows by nested loop and remove original outer row. - *

- * Sample input: - * colName = 'employees', rows = [{region: 'US'}] - * colValue= [{ - * source: { - * age: 26, - * firstname: 'Hank' - * } - * },{ - * source: { - * age: 30, - * firstname: 'John' - * } - * }] - *

- * Return: - * [ - * {region:'US', employees.age:26, employees.firstname:'Hank'}, - * {region:'US', employees.age:30, employees.firstname:'John'} - * ] + * Performs the "cartesian product" between two sets of rows, + * which is the set of all possible unions of a row in rowsLeft and a row in rowsRight. */ - private List doFlatNestedFieldValue(String colName, SearchHit[] colValue, List rows) { - List result = new ArrayList<>(); - for (DataRows.Row row : rows) { - for (SearchHit hit : colValue) { - Map innerRow = hit.getSourceAsMap(); - Map copy = new HashMap<>(); - - for (String field : row.getContents().keySet()) { - copy.put(field, row.getData(field)); - } - for (String field : innerRow.keySet()) { - copy.put(colName + "." + field, innerRow.get(field)); + private List> cartesianProduct(List> rowsLeft, + List> rowsRight, + Integer remainingRows) { + List> result = new ArrayList<>(); + for (Map rowLeft : rowsLeft) { + if (remainingRows != null && result.size() == remainingRows) { + break; + } + + for (Map rowRight : rowsRight) { + if (remainingRows != null && result.size() == remainingRows) { + break; } - copy.remove(colName); - result.add(new DataRows.Row(copy)); + Map union = new HashMap<>(); + union.putAll(rowLeft); + union.putAll(rowRight); + result.add(union); } } return result; } + private int ratioCeil(int a, int b) { + return (a + b - 1) / b; + } + private Map addMap(String field, Object term) { Map data = new HashMap<>(); data.put(field, term); diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/FieldMaker.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/FieldMaker.java index f56b0126c6..734dbf8276 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/FieldMaker.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/FieldMaker.java @@ -18,19 +18,7 @@ import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLObject; import com.alibaba.druid.sql.ast.SQLSetQuantifier; -import com.alibaba.druid.sql.ast.expr.SQLAggregateExpr; -import com.alibaba.druid.sql.ast.expr.SQLAggregateOption; -import com.alibaba.druid.sql.ast.expr.SQLAllColumnExpr; -import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; -import com.alibaba.druid.sql.ast.expr.SQLCaseExpr; -import com.alibaba.druid.sql.ast.expr.SQLCastExpr; -import com.alibaba.druid.sql.ast.expr.SQLCharExpr; -import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; -import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr; -import com.alibaba.druid.sql.ast.expr.SQLNumericLiteralExpr; -import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; -import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; -import com.alibaba.druid.sql.ast.expr.SQLVariantRefExpr; +import com.alibaba.druid.sql.ast.expr.*; import com.alibaba.druid.sql.ast.statement.SQLSelectGroupByClause; import com.alibaba.druid.sql.ast.statement.SQLSelectItem; import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock; @@ -41,6 +29,7 @@ import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlFeatureNotImplementedException; import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.SQLDefinitionExpr; import com.amazon.opendistroforelasticsearch.sql.legacy.utils.SQLFunctions; import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import com.google.common.base.Strings; @@ -127,7 +116,7 @@ private Field makeFieldImpl(SQLExpr expr, String alias, String tableAlias) throw return makeMethodField(methodInvokeExpr.getMethodName(), methodInvokeExpr.getParameters(), null, alias, tableAlias, true); } else { - throw new SqlParseException("unknown field name : " + expr); + throw new SqlParseException("unknown field name: " + expr); } } @@ -165,15 +154,48 @@ private Field makeScriptMethodField(SQLBinaryOpExpr binaryExpr, String alias, St } params.add(new SQLCharExpr(scriptFieldAlias)); - Object left = getScriptValue(binaryExpr.getLeft()); - Object right = getScriptValue(binaryExpr.getRight()); - String script = String.format("%s %s %s", left, binaryExpr.getOperator().getName(), right); + Tuple left = processOperand(binaryExpr.getLeft(), tableAlias); + Tuple right = processOperand(binaryExpr.getRight(), tableAlias); + + String operator = binaryExpr.getOperator().getName(); + if (operator.equals("=")) { + operator = "=="; + } + + String script = String.format("%s %s %s", left.v1(), operator, right.v1()); params.add(new SQLCharExpr(script)); + params.add(left.v2()); + params.add(right.v2()); return makeMethodField("script", params, null, null, tableAlias, false); } + private Tuple processOperand(SQLExpr expr, String tableAlias) + throws SqlParseException { + if (expr instanceof SQLBinaryOpExpr) { + SQLBinaryOpExpr binaryOpExpr = (SQLBinaryOpExpr) expr; + if (SQLFunctions.isFunctionTranslatedToScript(binaryOpExpr.getOperator().toString())) { + SQLMethodInvokeExpr mExpr = makeBinaryMethodField(binaryOpExpr, null, false); + MethodField mField = makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), + null, null, tableAlias, false); + String varName = mField.getParams().get(0).toString(); + return new Tuple<>(varName, new SQLDefinitionExpr(varName, mField.getParams().get(1).toString())); + } else { + throw new SqlParseException("Invalid operand: " + + "valuable expression expected, found " + + binaryOpExpr.toString()); + } + } else if (expr instanceof SQLMethodInvokeExpr) { + SQLMethodInvokeExpr mExpr = (SQLMethodInvokeExpr) expr; + MethodField mField = makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), + null, null, tableAlias, false); + String varName = mField.getParams().get(0).toString(); + return new Tuple<>(varName, new SQLDefinitionExpr(varName, mField.getParams().get(1).toString())); + } else { + return new Tuple<>(getScriptValue(expr).toString(), new SQLNullExpr()); + } + } private static Field makeFilterMethodField(SQLMethodInvokeExpr filterMethod, String alias) throws SqlParseException { @@ -292,17 +314,21 @@ public MethodField makeMethodField(String name, List arguments, SQLAggr if (SQLFunctions.isFunctionTranslatedToScript(binaryOpExpr.getOperator().toString())) { SQLMethodInvokeExpr mExpr = makeBinaryMethodField(binaryOpExpr, alias, first); - MethodField abc = makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), + MethodField mField = makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), null, null, tableAlias, false); - paramers.add(new KVValue(abc.getParams().get(0).toString(), - new SQLCharExpr(abc.getParams().get(1).toString()))); + String varName = mField.getParams().get(0).toString(); + paramers.add(new KVValue(varName, + new SQLDefinitionExpr(varName, mField.getParams().get(1).toString()) + )); } else { - if (!binaryOpExpr.getOperator().getName().equals("=")) { - paramers.add(new KVValue("script", makeScriptMethodField(binaryOpExpr, null, tableAlias))); + if (!SQLFunctions.isFunctionTranslatedToScript(name) + && binaryOpExpr.getOperator().getName().equals("=")) { + paramers.add(new KVValue( + binaryOpExpr.getLeft().toString(), + Util.expr2Object(binaryOpExpr.getRight()) + )); } else { - SQLExpr right = binaryOpExpr.getRight(); - Object value = Util.expr2Object(right); - paramers.add(new KVValue(binaryOpExpr.getLeft().toString(), value)); + paramers.add(new KVValue("script", makeScriptMethodField(binaryOpExpr, null, tableAlias))); } } @@ -310,8 +336,8 @@ public MethodField makeMethodField(String name, List arguments, SQLAggr SQLMethodInvokeExpr mExpr = (SQLMethodInvokeExpr) object; String methodName = mExpr.getMethodName().toLowerCase(); if (methodName.equals("script")) { - KVValue script = new KVValue("script", makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), - null, alias, tableAlias, true)); + KVValue script = new KVValue("script", makeMethodField(mExpr.getMethodName(), + mExpr.getParameters(), null, alias, tableAlias, true)); paramers.add(script); } else if (methodName.equals("nested") || methodName.equals("reverse_nested")) { NestedType nestedType = new NestedType(); @@ -332,19 +358,21 @@ public MethodField makeMethodField(String name, List arguments, SQLAggr paramers.add(new KVValue("children", childrenType)); } else if (SQLFunctions.isFunctionTranslatedToScript(methodName)) { //throw new SqlParseException("only support script/nested as inner functions"); - MethodField abc = makeMethodField(methodName, mExpr.getParameters(), null, null, tableAlias, false); - paramers.add(new KVValue(abc.getParams().get(0).toString(), - new SQLCharExpr(abc.getParams().get(1).toString()))); + MethodField mField = makeMethodField(methodName, mExpr.getParameters(), + null, null, tableAlias, false); + String varName = mField.getParams().get(0).toString(); + paramers.add(new KVValue(varName, + new SQLDefinitionExpr(varName, mField.getParams().get(1).toString()))); } else { throw new SqlParseException("only support script/nested/children as inner functions"); } } else if (object instanceof SQLCaseExpr) { String scriptCode = new CaseWhenParser((SQLCaseExpr) object, alias, tableAlias).parse(); - paramers.add(new KVValue("script", new SQLCharExpr(scriptCode))); + paramers.add(new KVValue("script", new SQLDefinitionExpr("script", scriptCode))); } else if (object instanceof SQLCastExpr) { String castName = sqlFunctions.nextId("cast"); List methodParameters = new ArrayList<>(); - methodParameters.add(new KVValue(((SQLCastExpr) object).getExpr().toString())); + methodParameters.add(new KVValue(((SQLCastExpr) object).getExpr())); String castType = ((SQLCastExpr) object).getDataType().getName(); String scriptCode = sqlFunctions.getCastScriptStatement(castName, castType, methodParameters); @@ -356,7 +384,7 @@ public MethodField makeMethodField(String name, List arguments, SQLAggr scriptCode += "; return " + castName; } methodParameters.add(new KVValue(scriptCode)); - paramers.add(new KVValue("script", new SQLCharExpr(scriptCode))); + paramers.add(new KVValue("script", new SQLDefinitionExpr("script", scriptCode))); } else if (object instanceof SQLAggregateExpr) { SQLObject parent = object.getParent(); SQLExpr source = (SQLExpr) parent.getAttribute("source"); @@ -401,7 +429,7 @@ public MethodField makeMethodField(String name, List arguments, SQLAggr List tempParamers = new LinkedList<>(); for (KVValue temp : paramers) { if (temp.value instanceof SQLExpr) { - tempParamers.add(new KVValue(temp.key, Util.expr2Object((SQLExpr) temp.value))); + tempParamers.add(new KVValue(temp.key, Util.expr2ObjectOrDefinition((SQLExpr) temp.value))); } else { tempParamers.add(new KVValue(temp.key, temp.value)); } diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SqlParser.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SqlParser.java index d296f66b0c..44ecc36daf 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SqlParser.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SqlParser.java @@ -448,7 +448,7 @@ private JoinSelect createBasicJoinSelectAccordingToTableSource(SQLJoinTableSourc throws SqlParseException { JoinSelect joinSelect = new JoinSelect(); if (joinTableSource.getCondition() != null) { - Where where = Where.newInstance(); + Where where = Where.newInstance(true); WhereParser whereParser = new WhereParser(this, joinTableSource.getCondition()); whereParser.parseWhere(joinTableSource.getCondition(), where); joinSelect.setConnectedWhere(where); diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/WhereParser.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/WhereParser.java index 73814fff0e..f7ef5432ac 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/WhereParser.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/WhereParser.java @@ -214,7 +214,7 @@ private void routeCond(SQLBinaryOpExpr bExpr, SQLExpr sub, Where where) throws S if (sub instanceof SQLBinaryOpExpr && !isCond((SQLBinaryOpExpr) sub)) { SQLBinaryOpExpr binarySub = (SQLBinaryOpExpr) sub; if (binarySub.getOperator().priority != bExpr.getOperator().priority) { - Where subWhere = new Where(bExpr.getOperator().name); + Where subWhere = new Where(bExpr.getOperator().name, where.isJoin()); where.addWhere(subWhere); parseWhere(binarySub, subWhere); } else { @@ -291,7 +291,7 @@ private void explainCond(String opear, SQLExpr expr, Where where) throws SqlPars condition = new Condition(Where.CONN.valueOf(opear), soExpr.getLeft().toString(), soExpr.getLeft(), soExpr.getOperator().name, parseValue(soExpr.getRight()), soExpr.getRight(), childrenType); } else { - SQLMethodInvokeExpr sqlMethodInvokeExpr = parseSQLBinaryOpExprWhoIsConditionInWhere(soExpr); + SQLMethodInvokeExpr sqlMethodInvokeExpr = parseSQLBinaryOpExprWhoIsConditionInWhere(soExpr, where.isJoin()); if (sqlMethodInvokeExpr == null) { condition = new Condition(Where.CONN.valueOf(opear), soExpr.getLeft().toString(), soExpr.getLeft(), soExpr.getOperator().name, parseValue(soExpr.getRight()), @@ -536,10 +536,20 @@ private MethodField parseSQLCastExprWithFunctionInWhere(SQLCastExpr soExpr) thro ); } - private SQLMethodInvokeExpr parseSQLBinaryOpExprWhoIsConditionInWhere(SQLBinaryOpExpr soExpr) + private SQLMethodInvokeExpr parseSQLBinaryOpExprWhoIsConditionInWhere(SQLBinaryOpExpr soExpr, + boolean join) throws SqlParseException { - if (bothSideAreNotFunction(soExpr) && bothSidesAreNotCast(soExpr)) { + if ( + // if one of the two side is a method, it has to be resolved as script + neitherSideIsFunction(soExpr) + // if one of the two side is a cast, it has to be resolved as script + && neitherSidesIsCast(soExpr) + // if both sides are field, we cannot use the QueryRange (which would be more efficient + // `field > integer` comparisons) but ES doesn't allow comparing two fields and + // we must use a SCRIPT + && (!bothSidesAreFieldIdentifier(soExpr) || join) + ) { return null; } @@ -600,6 +610,10 @@ private SQLMethodInvokeExpr parseSQLBinaryOpExprWhoIsConditionInWhere(SQLBinaryO operator = "=="; } + if (operator.equals("<>")) { + operator = "!="; + } + String finalStr = v1Dec + v2Dec + v1 + " " + operator + " " + v2; SQLMethodInvokeExpr scriptMethod = new SQLMethodInvokeExpr("script", null); @@ -608,14 +622,22 @@ private SQLMethodInvokeExpr parseSQLBinaryOpExprWhoIsConditionInWhere(SQLBinaryO } - private Boolean bothSideAreNotFunction(SQLBinaryOpExpr soExpr) { + private Boolean neitherSideIsFunction(SQLBinaryOpExpr soExpr) { return !(soExpr.getLeft() instanceof SQLMethodInvokeExpr || soExpr.getRight() instanceof SQLMethodInvokeExpr); } - private Boolean bothSidesAreNotCast(SQLBinaryOpExpr soExpr) { + private Boolean neitherSidesIsCast(SQLBinaryOpExpr soExpr) { return !(soExpr.getLeft() instanceof SQLCastExpr || soExpr.getRight() instanceof SQLCastExpr); } + private Boolean bothSidesAreFieldIdentifier(SQLBinaryOpExpr soExpr) { + return (soExpr.getLeft() instanceof SQLIdentifierExpr && soExpr.getRight() instanceof SQLIdentifierExpr) + // "missing" is a SQLIdentifier but not a Field Identifier. + // We might want to have a subclass for "missing" or for "fieldIdentifier" or + // change type altogether to avoid conflicts + && !"missing".equalsIgnoreCase(((SQLIdentifierExpr) soExpr.getRight()).getLowerName()); + } + private Object[] getMethodValuesWithSubQueries(SQLMethodInvokeExpr method) throws SqlParseException { List values = new ArrayList<>(); for (SQLExpr innerExpr : method.getParameters()) { diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java index f6fdc812bd..34d84146cf 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java @@ -172,7 +172,7 @@ private ResponseListener createQueryResponseListener(RestChannel } else if (format.equals(Format.RAW)) { formatter = new RawResponseFormatter(); } else { - formatter = new JdbcResponseFormatter(PRETTY); + formatter = new JdbcResponseFormatter(PRETTY, request.getDatetimeFormat()); } return new ResponseListener() { @Override diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java index 5eb3de5a0c..41e186019b 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java @@ -172,7 +172,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli @Override protected Set responseParams() { Set responseParams = new HashSet<>(super.responseParams()); - responseParams.addAll(Arrays.asList("sql", "flat", "separator", "_score", "_type", "_id", "newLine", "format", "sanitize")); + responseParams.addAll(Arrays.asList("sql", "flat", "separator", "_score", "_type", "_id", "newLine", "format", "datetime_format", "sanitize")); return responseParams; } diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlStatsAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlStatsAction.java index d4e5b6c500..4581586c14 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlStatsAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlStatsAction.java @@ -84,7 +84,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli @Override protected Set responseParams() { Set responseParams = new HashSet<>(super.responseParams()); - responseParams.addAll(Arrays.asList("sql", "flat", "separator", "_score", "_type", "_id", "newLine", "format", "sanitize")); + responseParams.addAll(Arrays.asList("sql", "flat", "separator", "_score", "_type", "_id", "newLine", "format", "datetime_format", "sanitize")); return responseParams; } diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlSettings.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlSettings.java index 346b4b8423..fe2da083ed 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlSettings.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlSettings.java @@ -45,6 +45,7 @@ public class SqlSettings { public static final String QUERY_ANALYSIS_ENABLED = "opendistro.sql.query.analysis.enabled"; public static final String QUERY_ANALYSIS_SEMANTIC_SUGGESTION = "opendistro.sql.query.analysis.semantic.suggestion"; public static final String QUERY_ANALYSIS_SEMANTIC_THRESHOLD = "opendistro.sql.query.analysis.semantic.threshold"; + public static final String QUERY_NESTED_LIMIT = "opendistro.sql.engine.nested.limit"; public static final String METRICS_ROLLING_WINDOW = "opendistro.sql.metrics.rollingwindow"; public static final String METRICS_ROLLING_INTERVAL = "opendistro.sql.metrics.rollinginterval"; @@ -58,6 +59,7 @@ public SqlSettings() { Map> settings = new HashMap<>(); settings.put(SQL_ENABLED, Setting.boolSetting(SQL_ENABLED, true, NodeScope, Dynamic)); settings.put(SQL_NEW_ENGINE_ENABLED, Setting.boolSetting(SQL_NEW_ENGINE_ENABLED, true, NodeScope, Dynamic)); + settings.put(QUERY_NESTED_LIMIT, Setting.intSetting(QUERY_NESTED_LIMIT, 3, NodeScope, Dynamic)); settings.put(QUERY_SLOWLOG, Setting.intSetting(QUERY_SLOWLOG, 2, NodeScope, Dynamic)); settings.put(QUERY_RESPONSE_FORMAT, Setting.simpleString(QUERY_RESPONSE_FORMAT, Format.JDBC.getFormatName(), NodeScope, Dynamic)); diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/NestedFieldProjection.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/NestedFieldProjection.java index 4ae0951fd0..e82eb9619f 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/NestedFieldProjection.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/NestedFieldProjection.java @@ -16,6 +16,7 @@ package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.nestedfield; import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.matchtoterm.VerificationException; import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import org.apache.lucene.search.join.ScoreMode; @@ -34,6 +35,7 @@ import java.util.function.Supplier; import java.util.stream.Stream; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_NESTED_LIMIT; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.mapping; import static java.util.stream.Collectors.toList; @@ -143,7 +145,11 @@ private List extractNestedQueries(QueryBuilder query) { private void buildInnerHit(List fieldNames, NestedQueryBuilder query) { query.innerHit(new InnerHitBuilder().setFetchSourceContext( new FetchSourceContext(true, fieldNames.toArray(new String[0]), null) - )); + ).setSize(this.queryNestedLimit())); + } + + private int queryNestedLimit() { + return LocalClusterState.state().getSettingValue(QUERY_NESTED_LIMIT); } /** diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLDefinitionExpr.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLDefinitionExpr.java new file mode 100644 index 0000000000..8da72a63c5 --- /dev/null +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLDefinitionExpr.java @@ -0,0 +1,73 @@ +package com.amazon.opendistroforelasticsearch.sql.legacy.utils; + +import com.alibaba.druid.sql.ast.SQLExprImpl; +import com.alibaba.druid.sql.ast.expr.SQLValuableExpr; +import com.alibaba.druid.sql.visitor.SQLASTVisitor; + +public class SQLDefinitionExpr extends SQLExprImpl implements SQLValuableExpr { + + private final String name; + + private final String definition; + + public SQLDefinitionExpr(String name, String definition) { + this.name = name; + this.definition = definition; + } + + public String getName() { + return name; + } + + public String getDefinition() { + return definition; + } + + @Override + public Object getValue() { + return this.getName(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + SQLDefinitionExpr other = (SQLDefinitionExpr) obj; + if (name == null) { + if (other.name != null) { + return false; + } + } else if (!name.equals(other.name)) { + return false; + } + if (definition == null) { + if (other.definition != null) { + return false; + } + } else if (!definition.equals(other.definition)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + ((definition == null) ? 0 : definition.hashCode()); + return result; + } + + @Override + protected void accept0(SQLASTVisitor visitor) { + + } +} diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java index ba635522dd..0330b8086b 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java @@ -38,11 +38,8 @@ import com.google.common.collect.Sets; import org.elasticsearch.common.collect.Tuple; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; +import java.lang.reflect.Method; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -75,7 +72,7 @@ public class SQLFunctions { ); private static final Set binaryOperators = Sets.newHashSet( - "add", "multiply", "divide", "subtract", "modulus" + "add", "multiply", "divide", "subtract", "modulus", "greatest", "least" ); private static final Set dateFunctions = Sets.newHashSet( @@ -174,8 +171,7 @@ public Tuple function(String methodName, List paramers, functionStr = date_format( (SQLExpr) paramers.get(0).value, Util.expr2Object((SQLExpr) paramers.get(1).value).toString(), - paramers.size() > 2 ? Util.expr2Object((SQLExpr) paramers.get(2).value).toString() : null, - name); + paramers.size() < 3 ? null : Util.expr2Object((SQLExpr) paramers.get(2).value).toString()); break; case "year": @@ -260,7 +256,7 @@ public Tuple function(String methodName, List paramers, case "sinh": case "cosh": functionStr = mathSingleValueTemplate("Math." + methodName, methodName, - (SQLExpr) paramers.get(0).value, name); + (SQLExpr) paramers.get(0).value); break; case "rand": @@ -274,22 +270,22 @@ public Tuple function(String methodName, List paramers, case "cot": // ES does not support the function name cot functionStr = mathSingleValueTemplate("1 / Math.tan", methodName, - (SQLExpr) paramers.get(0).value, name); + (SQLExpr) paramers.get(0).value); break; case "sign": case "signum": methodName = "signum"; functionStr = mathSingleValueTemplate("Math." + methodName, methodName, - (SQLExpr) paramers.get(0).value, name); + (SQLExpr) paramers.get(0).value); break; case "pow": case "power": methodName = "pow"; functionStr = mathDoubleValueTemplate("Math." + methodName, methodName, - (SQLExpr) paramers.get(0).value, Util.expr2Object((SQLExpr) paramers.get(1).value).toString(), - name); + (SQLExpr) paramers.get(0).value, + Util.expr2Object((SQLExpr) paramers.get(1).value).toString()); break; case "atan2": @@ -304,14 +300,14 @@ public Tuple function(String methodName, List paramers, break; case "degrees": - functionStr = degrees((SQLExpr) paramers.get(0).value, name); + functionStr = degrees((SQLExpr) paramers.get(0).value); break; case "radians": - functionStr = radians((SQLExpr) paramers.get(0).value, name); + functionStr = radians((SQLExpr) paramers.get(0).value); break; case "trim": - functionStr = trim((SQLExpr) paramers.get(0).value, name); + functionStr = trim((SQLExpr) paramers.get(0).value); break; case "add": @@ -332,19 +328,26 @@ public Tuple function(String methodName, List paramers, functionStr = modulus((SQLExpr) paramers.get(0).value, (SQLExpr) paramers.get(1).value); break; + case "greatest": + functionStr = greatest((SQLExpr) paramers.get(0).value, (SQLExpr) paramers.get(1).value); + break; + case "least": + functionStr = least((SQLExpr) paramers.get(0).value, (SQLExpr) paramers.get(1).value); + break; + case "field": functionStr = field(Util.expr2Object((SQLExpr) paramers.get(0).value).toString()); break; case "log2": - functionStr = log(SQLUtils.toSQLExpr("2"), (SQLExpr) paramers.get(0).value, name); + functionStr = log(SQLUtils.toSQLExpr("2"), (SQLExpr) paramers.get(0).value); break; case "log10": functionStr = log10((SQLExpr) paramers.get(0).value); break; case "log": if (paramers.size() > 1) { - functionStr = log((SQLExpr) paramers.get(0).value, (SQLExpr) paramers.get(1).value, name); + functionStr = log((SQLExpr) paramers.get(0).value, (SQLExpr) paramers.get(1).value); } else { functionStr = ln((SQLExpr) paramers.get(0).value); } @@ -426,6 +429,8 @@ public Tuple cast(String castType, List paramers) throw public Tuple upper(SQLExpr field, String locale, String valueName) { String name = nextId("upper"); + scriptDeclare(field); + if (valueName == null) { return new Tuple<>(name, def(name, upper(getPropertyOrStringValue(field), locale))); } else { @@ -464,9 +469,9 @@ private static String exprString(SQLExpr expr) { private static String func(String methodName, boolean quotes, String... params) { if (quotes) { return methodName + "(" + quoteParams(params) + ")"; + } else { + return methodName + "(" + String.join(", ", params) + ")"; } - - return methodName + "(" + String.join(", ", params) + ")"; } /** @@ -481,16 +486,9 @@ private Tuple concat_ws(String split, List columns) { List result = Lists.newArrayList(); for (SQLExpr column : columns) { - String strColumn = exprString(column); - if (strColumn.startsWith("def ")) { - result.add(strColumn); - } else if (isProperty(column)) { - result.add("doc['" + strColumn + "'].value"); - } else { - result.add("'" + strColumn + "'"); - } - + result.add(scriptDeclare(column) + getPropertyOrStringValue(column)); } + return new Tuple<>(name, def(name, Joiner.on("+ " + split + " +").join(result))); } @@ -499,41 +497,75 @@ private Tuple concat_ws(String split, List columns) { public Tuple split(SQLExpr field, String pattern, int index, String valueName) { String name = nextId("split"); final String script; + if (valueName == null) { - script = def(name, - getPropertyOrValue(field) + "." + script = def(name, scriptDeclare(field) + + getPropertyOrStringValue(field) + "." + func("split", true, pattern) + "[" + index + "]"); } else { script = "; " + def(name, valueName + "." + func("split", true, pattern) + "[" + index + "]"); } + return new Tuple<>(name, script); } //split(Column expr, java.lang.String pattern) public Tuple split(SQLExpr field, String pattern, String valueName) { String name = nextId("split"); + if (valueName == null) { return new Tuple<>(name, - def(name, getPropertyOrValue(field) + "." + def(name, getPropertyOrStringValue(field) + "." + func("split", true, pattern))); } else { - return new Tuple<>(name, getPropertyOrValue(field) + "; " - + def(name, valueName + "." + func("split", true, pattern))); + return new Tuple<>(name, + "; " + def(name, valueName + "." + func("split", true, pattern))); } } - private Tuple date_format(SQLExpr field, String pattern, String zoneId, String valueName) { + private Tuple date_format(SQLExpr field, String pattern, String zoneId) { String name = nextId("date_format"); - if (valueName == null) { - return new Tuple<>(name, "def " + name + " = DateTimeFormatter.ofPattern('" + pattern + "').withZone(" - + (zoneId != null ? "ZoneId.of('" + zoneId + "')" : "ZoneId.of(\"UTC\")") - + ").format(Instant.ofEpochMilli(" + getPropertyOrValue(field) + ".toInstant().toEpochMilli()))"); - } else { - return new Tuple<>(name, exprString(field) + "; " - + "def " + name + " = new SimpleDateFormat('" + pattern + "').format(" - + "new Date(" + valueName + " - 8*1000*60*60))"); - } + String parsedDateName = nextId("parsed_date"); + + String value = getPropertyOrStringValue(field); + + String definition = scriptDeclare(field) + + "def formats = new ArrayList(); " + + "formats.add(\"yyyy-MM-dd\"); " + + "formats.add(\"yyyy-MM-dd HH:mm:ss\"); " + + "formats.add(\"yyyy-MM-dd HH:mm:ss.SSSSSS\"); " + + "formats.add(\"yyyy-MM-dd'T'HH:mm:ss.SSSSSS\"); " + + "formats.add(\"yyyy-MM-dd'T'HH:mm:ss'Z'\"); " + + "def " + parsedDateName + ";" + + "if (" + value + " instanceof String) {" + + "for (String format : formats) {" + + "try {" + parsedDateName + " = " + + "ZonedDateTime.parse(" + value + ", " + + "new DateTimeFormatterBuilder()" + + ".appendPattern(format)" + + ".parseDefaulting(ChronoField.HOUR_OF_DAY, 0)" + + ".parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)" + + ".parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)" + + ".parseDefaulting(ChronoField.NANO_OF_SECOND, 0)" + + ".toFormatter()" + + StringUtils.format(".withZone(ZoneId.of('%s')));} ", + zoneId != null ? zoneId : "UTC") + + "catch (DateTimeParseException e) {} " + + "if (" + parsedDateName + " != null) {break;} } " + + "if (" + parsedDateName + " == null) { " + + "throw new IllegalArgumentException(\"Date format not recognized\"); } } " + + "else {" + parsedDateName + " = " + value + ";} "; + + definition += def(name, StringUtils.format( + "DateTimeFormatter.ofPattern(\"%s\").withZone(%s).format" + + "(Instant.ofEpochMilli(%s.toInstant().toEpochMilli()))", + pattern, + zoneId != null ? "ZoneId.of('" + zoneId + "')" : "ZoneId.of('UTC')", + parsedDateName + )); + + return new Tuple<>(name, definition); } /** @@ -564,8 +596,8 @@ public Tuple add(SQLExpr a, SQLExpr b) { public Tuple assign(SQLExpr a) { String name = nextId("assign"); - return new Tuple<>(name, - def(name, extractName(a))); + return new Tuple<>(name, scriptDeclare(a) + + def(name, getPropertyOrValue(a))); } private Tuple modulus(SQLExpr a, SQLExpr b) { @@ -591,18 +623,91 @@ private Tuple divide(SQLExpr a, SQLExpr b) { private Tuple binaryOpertator(String methodName, String operator, SQLExpr a, SQLExpr b) { String name = nextId(methodName); - return new Tuple<>(name, - scriptDeclare(a) + scriptDeclare(b) + convertType(a) + convertType(b) - + def(name, extractName(a) + " " + operator + " " + extractName(b))); + + if (methodName.equals("divide") || methodName.equals("modulus")) { + return new Tuple<>(name, + scriptDeclare(a) + scriptDeclare(b) + convertType(a) + convertType(b) + + def(name, StringUtils.format("((%2$s == 0 || %4$s || %5$s) ? null : %1$s %3$s %2$s)", + getPropertyOrValue(a), getPropertyOrValue(b), operator, + checkIfNull(a), checkIfNull(b)))); + } else { + return new Tuple<>(name, + scriptDeclare(a) + scriptDeclare(b) + convertType(a) + convertType(b) + + def(name, StringUtils.format("((%4$s || %5$s) ? null : %1$s %3$s %2$s)", + getPropertyOrValue(a), getPropertyOrValue(b), operator, + checkIfNull(a), checkIfNull(b)))); + } + } + + private Tuple greatest(SQLExpr a, SQLExpr b) { + String name = nextId("greatest"); + + return comparisonFunctionTemplate(a, b, name, ">"); + } + + private Tuple least(SQLExpr a, SQLExpr b) { + String name = nextId("least"); + + return comparisonFunctionTemplate(a, b, name, "<"); + } + + private Tuple comparisonFunctionTemplate(SQLExpr a, SQLExpr b, + String name, String comparisonOperator) { + String value_a = getPropertyOrStringValue(a); + String value_b = getPropertyOrStringValue(b); + + String nameString_a = nextId("string_a"); + String nameString_b = nextId("string_b"); + + String numberComparison = String.format("%s = (%s %s %s ? %s : %s);", + name, value_a, comparisonOperator, value_b, value_a, value_b); + String stringOrDateComparison = def(nameString_a, convertToString(value_a)) + ";" + + def(nameString_b, convertToString(value_b)) + ";" + + String.format("%s = (%s.compareTo(%s) %s 0 ? %s : %s);", + name, nameString_a, nameString_b, comparisonOperator, value_a, value_b); + + String definition = scriptDeclare(a) + scriptDeclare(b) + + "def " + name + ";" + + String.format("if (%s) {%s = ((%s) ? null : %s);} ", checkIfNull(a), name, checkIfNull(b), value_b) + + String.format("else if (%s) {%s = %s;} ", checkIfNull(b), name, value_a) + + "else {"; + + if (isPropertyOrDefinition(a) && isPropertyOrDefinition(b)) { + definition += String.format("if (%s instanceof Number) {", value_a) + + numberComparison + + "} else {" + + stringOrDateComparison + + "} "; + } else if (a instanceof SQLNumericLiteralExpr || b instanceof SQLNumericLiteralExpr) { + definition += numberComparison; + } else { + definition += stringOrDateComparison; + } + + definition += String.format("} %s = %s", name, name); + + return new Tuple<>(name, definition); } private static boolean isProperty(SQLExpr expr) { - return (expr instanceof SQLIdentifierExpr || expr instanceof SQLPropertyExpr + return (expr instanceof SQLIdentifierExpr + || expr instanceof SQLPropertyExpr || expr instanceof SQLVariantRefExpr); } + private static boolean isPropertyOrDefinition(SQLExpr expr) { + return (expr instanceof SQLIdentifierExpr + || expr instanceof SQLPropertyExpr + || expr instanceof SQLVariantRefExpr + || expr instanceof SQLDefinitionExpr); + } + private static String getPropertyOrValue(SQLExpr expr) { - if (isProperty(expr)) { + if (expr instanceof SQLDefinitionExpr) { + return ((SQLDefinitionExpr) expr).getName(); + } else if (expr instanceof SQLNullExpr) { + return "null"; + } else if (isProperty(expr)) { return doc(expr) + ".value"; } else { return exprString(expr); @@ -620,102 +725,97 @@ private static String getPropertyOrValue(String expr) { } private static String getPropertyOrStringValue(SQLExpr expr) { - if (isProperty(expr)) { - return doc(expr) + ".value"; - } else { + if (expr instanceof SQLTextLiteralExpr) { return "'" + exprString(expr) + "'"; - } - } - - private static String scriptDeclare(SQLExpr a) { - - if (isProperty(a) || a instanceof SQLNumericLiteralExpr) { - return ""; } else { - return exprString(a) + ";"; + return getPropertyOrValue(expr); } } - private static String extractName(SQLExpr script) { - if (isProperty(script)) { - return doc(script) + ".value"; - } - String scriptStr = exprString(script); - String[] variance = scriptStr.split(";"); - String newScript = variance[variance.length - 1]; - if (newScript.trim().startsWith("def ")) { - //for now ,if variant is string,then change to double. - return newScript.trim().substring(4).split("=")[0].trim(); + private static String scriptDeclare(Object obj) { + if (obj instanceof SQLDefinitionExpr) { + return ((SQLDefinitionExpr) obj).getDefinition() + "; "; + } else if (obj instanceof MethodField) { + MethodField mField = (MethodField) obj; + SQLExpr left = (SQLExpr) mField.getParams().get(2).value; + SQLExpr right = (SQLExpr) mField.getParams().get(3).value; + return scriptDeclare(left) + scriptDeclare(right); } else { - return scriptStr; + return ""; } } //cast(year as int) private static String convertType(SQLExpr script) { - String[] variance = exprString(script).split(";"); - String newScript = variance[variance.length - 1]; - if (newScript.trim().startsWith("def ")) { - //for now ,if variant is string,then change to double. - String temp = newScript.trim().substring(4).split("=")[0].trim(); - - return " if( " + temp + " instanceof String) " + temp + "= Double.parseDouble(" + temp.trim() + "); "; - } else { + if (!(script instanceof SQLDefinitionExpr)) { return ""; + } else { + return StringUtils.format(" if(%1$s instanceof String) {%1$s = Double.parseDouble(%1$s);}", + ((SQLDefinitionExpr) script).getName()); } + } + private static String checkIfNull(Object script) { + if (script instanceof SQLDefinitionExpr) { + return ((SQLDefinitionExpr) script).getName() + " == null"; + } else if (script instanceof SQLExpr && isProperty((SQLExpr) script)) { + return doc((SQLExpr) script) + ".size() == 0 || " + getPropertyOrValue((SQLExpr) script) + " == null"; + } else if (script instanceof SQLNullExpr) { + return "0 == 0"; + } else { + return "0 == 1"; + } + } + private static String convertToString(String name) { + return String.format("(%s instanceof String ? (String) %s : %s.toString())", + name, name, name); } private String getScriptText(MethodField field) { - String content = ((SQLTextLiteralExpr) field.getParams().get(1).value).getText(); - return content; + return ((SQLTextLiteralExpr) field.getParams().get(1).value).getText(); } /** * Using exprString() rather than getPropertyOrValue() for "base" since something like "Math.E" gets evaluated * incorrectly in getPropertyOrValue(), returning it as a doc value instead of the literal string */ - public Tuple log(SQLExpr base, SQLExpr field, String valueName) { + public Tuple log(SQLExpr base, SQLExpr field) { String name = nextId("log"); - String result; - if (valueName == null) { - result = def(name, func("Math.log", false, getPropertyOrValue(field)) - + "/" + func("Math.log", false, exprString(base))); - } else { - result = getPropertyOrValue(field) + "; " - + def(name, func("Math.log", false, valueName) + "/" - + func("Math.log", false, exprString(base))); - } - return new Tuple<>(name, result); + return new Tuple<>(name, scriptDeclare(base) + scriptDeclare(field) + + def(name, func("Math.log", false, getPropertyOrValue(field)) + + "/" + func("Math.log", false, getPropertyOrValue(base)))); } public Tuple log10(SQLExpr field) { String name = nextId("log10"); - return new Tuple<>(name, def(name, StringUtils.format("Math.log10(%s)", getPropertyOrValue(field)))); + return new Tuple<>(name, scriptDeclare(field) + + def(name, StringUtils.format("Math.log10(%s)", getPropertyOrValue(field)))); } public Tuple ln(SQLExpr field) { String name = nextId("ln"); - return new Tuple<>(name, def(name, StringUtils.format("Math.log(%s)", getPropertyOrValue(field)))); + return new Tuple<>(name, scriptDeclare(field) + + def(name, StringUtils.format("Math.log(%s)", getPropertyOrValue(field)))); } - public Tuple trim(SQLExpr field, String valueName) { - return strSingleValueTemplate("trim", field, valueName); + public Tuple trim(SQLExpr field) { + return strSingleValueTemplate("trim", field); } - private Tuple degrees(SQLExpr field, String valueName) { - return mathSingleValueTemplate("Math.toDegrees", "degrees", field, valueName); + private Tuple degrees(SQLExpr field) { + return mathSingleValueTemplate("Math.toDegrees", "degrees", field); } - private Tuple radians(SQLExpr field, String valueName) { - return mathSingleValueTemplate("Math.toRadians", "radians", field, valueName); + private Tuple radians(SQLExpr field) { + return mathSingleValueTemplate("Math.toRadians", "radians", field); } - private Tuple rand(SQLExpr expr) { + private Tuple rand(SQLExpr field) { String name = nextId("rand"); - return new Tuple<>(name, def(name, format("new Random(%s).nextDouble()", getPropertyOrValue(expr)))); + return new Tuple<>(name, scriptDeclare(field) + + def(name, format("new Random(%s).nextDouble()", getPropertyOrValue(field)))); } private Tuple rand() { @@ -723,35 +823,24 @@ private Tuple rand() { return new Tuple<>(name, def(name, "new Random().nextDouble()")); } - private Tuple mathDoubleValueTemplate(String methodName, String fieldName, SQLExpr val1, - String val2, String valueName) { + private Tuple mathDoubleValueTemplate(String methodName, String fieldName, + SQLExpr val1, String val2) { String name = nextId(fieldName); - if (valueName == null) { - return new Tuple<>(name, def(name, func(methodName, false, getPropertyOrValue(val1), - getPropertyOrValue(val2)))); - } else { - return new Tuple<>(name, getPropertyOrValue(val1) + "; " - + def(name, func(methodName, false, valueName, getPropertyOrValue(val2)))); - } + return new Tuple<>(name, scriptDeclare(val1) + + def(name, func(methodName, false, getPropertyOrValue(val1), getPropertyOrValue(val2)))); } private Tuple mathDoubleValueTemplate(String methodName, String fieldName, SQLExpr val1, SQLExpr val2) { String name = nextId(fieldName); - return new Tuple<>(name, def(name, func(methodName, false, - getPropertyOrValue(val1), getPropertyOrValue(val2)))); + return new Tuple<>(name, scriptDeclare(val1) + scriptDeclare(val2) + + def(name, func(methodName, false, getPropertyOrValue(val1), getPropertyOrValue(val2)))); } - private Tuple mathSingleValueTemplate(String methodName, String fieldName, SQLExpr field, - String valueName) { + private Tuple mathSingleValueTemplate(String methodName, String fieldName, SQLExpr field) { String name = nextId(fieldName); - if (valueName == null) { - return new Tuple<>(name, def(name, func(methodName, false, getPropertyOrValue(field)))); - } else { - return new Tuple<>(name, getPropertyOrValue(field) + "; " - + def(name, func(methodName, false, valueName))); - } - + return new Tuple<>(name, scriptDeclare(field) + + def(name, func(methodName, false, getPropertyOrValue(field)))); } private Tuple mathConstantTemplate(String methodName, String fieldName) { @@ -759,15 +848,10 @@ private Tuple mathConstantTemplate(String methodName, String fie return new Tuple<>(name, def(name, methodName)); } - private Tuple strSingleValueTemplate(String methodName, SQLExpr field, String valueName) { + private Tuple strSingleValueTemplate(String methodName, SQLExpr field) { String name = nextId(methodName); - if (valueName == null) { - return new Tuple<>(name, def(name, getPropertyOrStringValue(field) + "." + func(methodName, false))); - } else { - return new Tuple<>(name, getPropertyOrStringValue(field) + "; " - + def(name, valueName + "." + func(methodName, false))); - } - + return new Tuple<>(name, scriptDeclare(field) + + def(name, getPropertyOrStringValue(field) + "." + func(methodName, false))); } // query: substring(Column expr, int pos, int len) @@ -777,11 +861,12 @@ public Tuple substring(SQLExpr field, int pos, int len) { String name = nextId("substring"); // start and end are 0-indexes int start = pos < 1 ? 0 : pos - 1; - return new Tuple<>(name, StringUtils.format( - "def end = (int) Math.min(%s + %s, %s.length()); " - + def(name, getPropertyOrStringValue(field) + "." - + func("substring", false, Integer.toString(start), "end")), - Integer.toString(start), Integer.toString(len), getPropertyOrStringValue(field) + return new Tuple<>(name, scriptDeclare(field) + + StringUtils.format( + "def end = (int) Math.min(%s + %s, %s.length()); " + + def(name, getPropertyOrStringValue(field) + "." + + func("substring", false, Integer.toString(start), "end")), + Integer.toString(start), Integer.toString(len), getPropertyOrStringValue(field) )); } @@ -795,12 +880,14 @@ private String upper(String property, String culture) { private Tuple length(SQLExpr field) { String name = nextId("length"); - return new Tuple<>(name, def(name, getPropertyOrStringValue(field) + ".length()")); + return new Tuple<>(name, scriptDeclare(field) + + def(name, getPropertyOrStringValue(field) + ".length()")); } private Tuple replace(SQLExpr field, String target, String replacement) { String name = nextId("replace"); - return new Tuple<>(name, def(name, getPropertyOrStringValue(field) + return new Tuple<>(name, scriptDeclare(field) + + def(name, getPropertyOrStringValue(field) + ".replace(" + target + "," + replacement + ")")); } @@ -810,167 +897,158 @@ private Tuple locate(String pattern, SQLExpr source, int start) String name = nextId("locate"); String docSource = getPropertyOrStringValue(source); start = start < 1 ? 0 : start - 1; - return new Tuple<>(name, def(name, StringUtils.format("%s.indexOf(%s,%d)+1", docSource, pattern, start))); + return new Tuple<>(name, scriptDeclare(source) + + def(name, StringUtils.format("%s.indexOf(%s, %d) + 1", docSource, pattern, start))); } private Tuple rtrim(SQLExpr field) { String name = nextId("rtrim"); String fieldString = getPropertyOrStringValue(field); - return new Tuple<>(name, StringUtils.format( - "int pos=%s.length()-1;" - + "while(pos >= 0 && Character.isWhitespace(%s.charAt(pos))) {pos --;} " - + def(name, "%s.substring(0, pos+1)"), - fieldString, fieldString, fieldString + return new Tuple<>(name, scriptDeclare(field) + + StringUtils.format( + "int pos = %s.length() - 1;" + + "while(pos >= 0 && Character.isWhitespace(%s.charAt(pos))) {pos--;} " + + def(name, "%s.substring(0, pos + 1)"), + fieldString, fieldString, fieldString )); } private Tuple ltrim(SQLExpr field) { String name = nextId("ltrim"); String fieldString = getPropertyOrStringValue(field); - return new Tuple<>(name, StringUtils.format( - "int pos=0;" - + "while(pos < %s.length() && Character.isWhitespace(%s.charAt(pos))) {pos ++;} " - + def(name, "%s.substring(pos, %s.length())"), - fieldString, fieldString, fieldString, fieldString + return new Tuple<>(name, scriptDeclare(field) + + StringUtils.format( + "int pos = 0;" + + "while(pos < %s.length() && Character.isWhitespace(%s.charAt(pos))) {pos++;} " + + def(name, "%s.substring(pos, %s.length())"), + fieldString, fieldString, fieldString, fieldString )); } private Tuple ascii(SQLExpr field) { String name = nextId("ascii"); - return new Tuple<>(name, def(name, "(int) " + getPropertyOrStringValue(field) + ".charAt(0)")); + return new Tuple<>(name, scriptDeclare(field) + + def(name, "(int) " + getPropertyOrStringValue(field) + ".charAt(0)")); } private Tuple left(SQLExpr expr, SQLExpr length) { String name = nextId("left"); - return new Tuple<>(name, StringUtils.format( - "def len = (int) Math.min(%s, %s.length()); def %s = %s.substring(0, len)", - exprString(length), getPropertyOrStringValue(expr), name, getPropertyOrStringValue(expr))); + return new Tuple<>(name, scriptDeclare(expr) + scriptDeclare(length) + + StringUtils.format( + "def len = (int) Math.min(%s, %s.length()); def %s = %s.substring(0, len)", + exprString(length), getPropertyOrStringValue(expr), name, getPropertyOrStringValue(expr) + )); } private Tuple right(SQLExpr expr, SQLExpr length) { String name = nextId("right"); - return new Tuple<>(name, StringUtils.format( - "def start = (int) Math.max(0, %s.length()-%s); def %s = %s.substring(start)", - getPropertyOrStringValue(expr), exprString(length), name, getPropertyOrStringValue(expr))); + return new Tuple<>(name, scriptDeclare(expr) + scriptDeclare(length) + + StringUtils.format( + "def start = (int) Math.max(0, %s.length()-%s); def %s = %s.substring(start)", + getPropertyOrStringValue(expr), exprString(length), name, getPropertyOrStringValue(expr) + )); } private Tuple date(SQLExpr field) { String name = nextId("date"); - return new Tuple<>(name, def(name, - "LocalDate.parse(" + getPropertyOrStringValue(field) + ".toString()," + return new Tuple<>(name, scriptDeclare(field) + + def(name, "LocalDate.parse(" + getPropertyOrStringValue(field) + ".toString(), " + "DateTimeFormatter.ISO_DATE_TIME)")); } private Tuple timestamp(SQLExpr field) { String name = nextId("timestamp"); - return new Tuple<>(name, def(name, - "DateTimeFormatter.ofPattern('yyyy-MM-dd HH:mm:ss').format(" + return new Tuple<>(name, scriptDeclare(field) + + def(name, "DateTimeFormatter.ofPattern('yyyy-MM-dd HH:mm:ss').format(" + "DateTimeFormatter.ISO_DATE_TIME.parse(" + getPropertyOrStringValue(field) + ".toString()))")); } private Tuple maketime(SQLExpr hr, SQLExpr min, SQLExpr sec) { String name = nextId("maketime"); - return new Tuple<>(name, def(name, StringUtils.format( - "LocalTime.of(%s, %s, %s).format(DateTimeFormatter.ofPattern('HH:mm:ss'))", - hr.toString(), min.toString(), sec.toString()))); + return new Tuple<>(name, scriptDeclare(hr) + scriptDeclare(min) + scriptDeclare(sec) + + def(name, StringUtils.format( + "LocalTime.of(%s, %s, %s).format(DateTimeFormatter.ofPattern('HH:mm:ss'))", + hr.toString(), min.toString(), sec.toString() + ))); } private Tuple now() { String name = nextId("now"); - return new Tuple<>(name, def(name, "new SimpleDateFormat('HH:mm:ss').format(System.currentTimeMillis())")); + return new Tuple<>(name, def(name, + "new SimpleDateFormat('HH:mm:ss').format(System.currentTimeMillis())")); } private Tuple curdate() { String name = nextId("curdate"); - return new Tuple<>(name, def(name, "new SimpleDateFormat('yyyy-MM-dd').format(System.currentTimeMillis())")); + return new Tuple<>(name, def(name, + "new SimpleDateFormat('yyyy-MM-dd').format(System.currentTimeMillis())")); } private Tuple ifFunc(List paramers) { - String expr1 = paramers.get(1).value.toString(); - String expr2 = paramers.get(2).value.toString(); String name = nextId("if"); - /** Input with null is regarded as false */ + SQLExpr expr1 = (SQLExpr) paramers.get(1).value; + SQLExpr expr2 = (SQLExpr) paramers.get(2).value; + + String value1 = (expr1 instanceof SQLTextLiteralExpr ? getPropertyOrStringValue(expr1) + : getPropertyOrValue(expr1)); + String value2 = (expr2 instanceof SQLTextLiteralExpr ? getPropertyOrStringValue(expr2) + : getPropertyOrValue(expr2)); + + String definition = scriptDeclare(expr1) + scriptDeclare(expr2); + + // Input with null is regarded as false if (paramers.get(0).value instanceof SQLNullExpr) { - return new Tuple<>(name, def(name, expr2)); - } - if (paramers.get(0).value instanceof MethodField) { + definition += def(name, value2); + } else if (paramers.get(0).value instanceof MethodField) { String condition = getScriptText((MethodField) paramers.get(0).value); - return new Tuple<>(name, "boolean cond = " + condition + ";" - + def(name, "cond ? " + expr1 + " : " + expr2)); + definition += scriptDeclare(paramers.get(0).value) + + "boolean cond = (" + condition + "); " + + def(name, "(cond ? " + value1 + " : " + value2 + ")"); } else if (paramers.get(0).value instanceof SQLBooleanExpr) { - Boolean condition = ((SQLBooleanExpr) paramers.get(0).value).getValue(); - if (condition) { - return new Tuple<>(name, def(name, expr1)); + if (((SQLBooleanExpr) paramers.get(0).value).getValue()) { + definition += def(name, value1); } else { - return new Tuple<>(name, def(name, expr2)); + definition += def(name, value2); } - } else { - /** - * Detailed explanation of cases that come here: - * the condition expression would be in the format of a=b: - * a is parsed as the key (String) of a KVValue (get from paramers.get(0)) - * and b is parsed as the value (Object) of this KVValue. - * - * Either a or b could be a column name, literal, or a number: - * - if isNumeric is true --> number - * - else if this string is single quoted --> literal - * - else --> column name - */ - String key = getPropertyOrValue(paramers.get(0).key); - String value = getPropertyOrValue(paramers.get(0).value.toString()); - String condition = key + " == " + value; - return new Tuple<>(name, "boolean cond = " + condition + ";" - + def(name, "cond ? " + expr1 + " : " + expr2)); } + + return new Tuple<>(name, definition); } - private Tuple ifnull(SQLExpr condition, SQLExpr expr) { + private Tuple ifnull(Object condition, SQLExpr expr) { String name = nextId("ifnull"); - if (condition instanceof SQLNullExpr) { - return new Tuple<>(name, def(name, expr.toString())); - } - if (isProperty(condition)) { - return new Tuple<>(name, def(name, doc(condition) + ".size()==0 ? " + expr.toString() + " : " - + getPropertyOrValue(condition))); - } else { - String condStr = Strings.isNullOrEmpty(condition.toString()) ? null : getPropertyOrStringValue(condition); - return new Tuple<>(name, def(name, condStr)); - } + + return new Tuple<>(name, scriptDeclare(condition) + scriptDeclare(expr) + + def(name, StringUtils.format( + "((%s) ? %s : %s)", + checkIfNull(condition), + getPropertyOrStringValue(expr), + condition instanceof SQLExpr ? getPropertyOrStringValue((SQLExpr) condition) + : condition.toString() + ))); } private Tuple isnull(SQLExpr expr) { String name = nextId("isnull"); - if (expr instanceof SQLNullExpr) { - return new Tuple<>(name, def(name, "1")); - } - if (isProperty(expr)) { - return new Tuple<>(name, def(name, doc(expr) + ".size()==0 ? 1 : 0")); - } - // cases that return 1: - // expr is null || expr is math func but tends to throw "divided by zero" arithmetic exception - String resultStr = "0"; - if (Strings.isNullOrEmpty(expr.toString())) { - resultStr = "1"; - } - if (expr instanceof SQLCharExpr && this.generatedIds.size() > 1) { - // the expr is a math expression - String mathExpr = ((SQLCharExpr) expr).getText(); - return new Tuple<>(name, StringUtils.format( - "try {%s;} " - + "catch(ArithmeticException e) " - + "{return 1;} " - + "def %s=0", - mathExpr, name, name) - ); - } - return new Tuple<>(name, def(name, resultStr)); + + return new Tuple<>(name, scriptDeclare(expr) + def(name, + StringUtils.format("((%s) ? 1 : 0)", checkIfNull(expr)))); } public String getCastScriptStatement(String name, String castType, List paramers) throws SqlParseException { - String castFieldName = String.format("doc['%s'].value", paramers.get(0).toString()); + Object expr = paramers.get(0).value; + String castFieldName; + if (expr instanceof String) { + castFieldName = getPropertyOrValue((String) expr); + } else { + castFieldName = getPropertyOrValue((SQLExpr) expr); + } + + scriptDeclare(expr); + switch (StringUtils.toUpper(castType)) { case "INT": case "LONG": diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/Util.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/Util.java index 7c8bbbdf47..587c72d7e6 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/Util.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/Util.java @@ -88,6 +88,14 @@ public static Object expr2Object(SQLExpr expr) { return expr2Object(expr, ""); } + public static Object expr2ObjectOrDefinition(SQLExpr expr) { + if (expr instanceof SQLDefinitionExpr) { + return ((SQLDefinitionExpr) expr).getDefinition(); + } else { + return expr2Object(expr, ""); + } + } + public static Object expr2Object(SQLExpr expr, String charWithQuote) { Object value = null; if (expr instanceof SQLNumericLiteralExpr) { @@ -112,17 +120,17 @@ public static Object expr2Object(SQLExpr expr, String charWithQuote) { public static Object getScriptValue(SQLExpr expr) throws SqlParseException { if (expr instanceof SQLIdentifierExpr || expr instanceof SQLPropertyExpr || expr instanceof SQLVariantRefExpr) { - return "doc['" + expr.toString() + "'].value"; + return StringUtils.format("doc['%s'].value", expr.toString()); } else if (expr instanceof SQLValuableExpr) { return ((SQLValuableExpr) expr).getValue(); } - throw new SqlParseException("could not parse sqlBinaryOpExpr need to be identifier/valuable got" + throw new SqlParseException("Could not parse sqlBinaryOpExpr: needs to be identifier/valuable, got" + expr.getClass().toString() + " with value:" + expr.toString()); } public static Object getScriptValueWithQuote(SQLExpr expr, String quote) throws SqlParseException { if (expr instanceof SQLIdentifierExpr || expr instanceof SQLPropertyExpr || expr instanceof SQLVariantRefExpr) { - return "doc['" + expr.toString() + "'].value"; + return StringUtils.format("doc['%s'].value", expr.toString()); } else if (expr instanceof SQLCharExpr) { return quote + ((SQLCharExpr) expr).getValue() + quote; } else if (expr instanceof SQLIntegerExpr) { @@ -130,7 +138,7 @@ public static Object getScriptValueWithQuote(SQLExpr expr, String quote) throws } else if (expr instanceof SQLNumericLiteralExpr) { return ((SQLNumericLiteralExpr) expr).getNumber(); } else if (expr instanceof SQLNullExpr) { - return ((SQLNullExpr) expr).toString().toLowerCase(); + return expr.toString().toLowerCase(); } throw new SqlParseException("could not parse sqlBinaryOpExpr need to be identifier/valuable got" + expr.getClass().toString() + " with value:" + expr.toString()); diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFunctionsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFunctionsTest.java index bc6d912c9f..550882eb94 100644 --- a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFunctionsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFunctionsTest.java @@ -154,7 +154,8 @@ public void date() { assertTrue( scriptContainsString( scriptField, - "LocalDate.parse(doc['creationDate'].value.toString(),DateTimeFormatter.ISO_DATE_TIME)")); + "LocalDate.parse(doc['creationDate'].value.toString(), " + + "DateTimeFormatter.ISO_DATE_TIME)")); } @Test diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/QueryFunctionsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/QueryFunctionsTest.java index b939c8696e..ee9492a48e 100644 --- a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/QueryFunctionsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/QueryFunctionsTest.java @@ -218,7 +218,7 @@ public void ifFunctionWithConditionStatement() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "boolean cond = doc['age'].value > 35;" + "boolean cond = (doc['age'].value > 35);" ) ); } @@ -230,7 +230,7 @@ public void ifFunctionWithEquationConditionStatement() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "boolean cond = doc['age'].value == 35;" + "boolean cond = (doc['age'].value == 35);" ) ); } @@ -242,7 +242,7 @@ public void ifFunctionWithConstantConditionStatement() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "boolean cond = 1 == 2;" + "boolean cond = (1 == 2);" ) ); } @@ -254,7 +254,7 @@ public void ifNull() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "doc['lastname'].size()==0" + "doc['lastname'].size() == 0" ) ); } @@ -266,7 +266,7 @@ public void isNullWithMathExpr() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "catch(ArithmeticException e)" + "add_1 == null" ) ); diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/StringOperatorsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/StringOperatorsTest.java index 0cad9e71e4..258be08892 100644 --- a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/StringOperatorsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/StringOperatorsTest.java @@ -116,7 +116,7 @@ public void locateTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptField, - "doc['lastname'].value.indexOf('a',0)+1" + "doc['lastname'].value.indexOf('a', 0) + 1" ) ); @@ -124,7 +124,7 @@ public void locateTest() { assertTrue( CheckScriptContents.scriptContainsString( scriptFilter, - "doc['lastname'].value.indexOf('a',0)+1" + "doc['lastname'].value.indexOf('a', 0) + 1" ) ); } diff --git a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLQueryAction.java b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLQueryAction.java index 75e07ca815..0ab50dbf7e 100644 --- a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLQueryAction.java +++ b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLQueryAction.java @@ -115,7 +115,7 @@ public String getName() { @Override protected Set responseParams() { Set responseParams = new HashSet<>(super.responseParams()); - responseParams.addAll(Arrays.asList("format", "sanitize")); + responseParams.addAll(Arrays.asList("format", "datetime_format", "sanitize")); return responseParams; } diff --git a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLStatsAction.java b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLStatsAction.java index c306f9ca93..1e12d22f29 100644 --- a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLStatsAction.java +++ b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLStatsAction.java @@ -85,7 +85,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli @Override protected Set responseParams() { Set responseParams = new HashSet<>(super.responseParams()); - responseParams.addAll(Arrays.asList("format", "sanitize")); + responseParams.addAll(Arrays.asList("format", "datetime_format", "sanitize")); return responseParams; } } diff --git a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java index 5deb7e0f56..67cb173638 100644 --- a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java +++ b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java @@ -16,8 +16,10 @@ package com.amazon.opendistroforelasticsearch.sql.protocol.response; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.Schema.Column; import java.util.Collection; @@ -75,6 +77,19 @@ public Iterator iterator() { .iterator(); } + /** + * Sets the date and time format for all inner elements of exprValues. + */ + public void setDatetimeFormat(String datetimeFormat) { + for (ExprValue exprValue : exprValues) { + for (ExprValue v : ExprValueUtils.getTupleValue(exprValue).values()) { + if (v.type() == ExprCoreType.TIMESTAMP) { + ((ExprTimestampValue) v).setDatetimeFormat(datetimeFormat); + } + } + } + } + private String getColumnName(Column column) { return (column.getAlias() != null) ? column.getAlias() : column.getName(); } diff --git a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatter.java b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatter.java index 52f8c1d703..a2d203fbcb 100644 --- a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatter.java +++ b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatter.java @@ -36,12 +36,17 @@ */ public class JdbcResponseFormatter extends JsonResponseFormatter { - public JdbcResponseFormatter(Style style) { + private final String datetimeFormat; + + public JdbcResponseFormatter(Style style, String datetimeFormat) { super(style); + this.datetimeFormat = datetimeFormat; } @Override protected Object buildJsonObject(QueryResult response) { + response.setDatetimeFormat(datetimeFormat); + JdbcResponse.JdbcResponseBuilder json = JdbcResponse.builder(); // Fetch schema and data rows diff --git a/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResultTest.java b/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResultTest.java index 5373650b25..945603e463 100644 --- a/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResultTest.java +++ b/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResultTest.java @@ -16,18 +16,26 @@ package com.amazon.opendistroforelasticsearch.sql.protocol.response; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue.INCLUDE_TIME_WHEN_NONZERO; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.tupleValue; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashMap; + import org.junit.jupiter.api.Test; class QueryResultTest { @@ -36,6 +44,10 @@ class QueryResultTest { new ExecutionEngine.Schema.Column("name", null, STRING), new ExecutionEngine.Schema.Column("age", null, INTEGER))); + private ExecutionEngine.Schema schemaTimestamp = new ExecutionEngine.Schema(ImmutableList.of( + new ExecutionEngine.Schema.Column("name", null, STRING), + new ExecutionEngine.Schema.Column("birthdate", null, TIMESTAMP))); + @Test void size() { @@ -125,4 +137,36 @@ void iterate() { } } + @Test + void setDateTimeFormatTest() { + LinkedHashMap valueMap1 = new LinkedHashMap<>(); + valueMap1.put("name", ExprValueUtils.fromObjectValue("John")); + valueMap1.put("birthdate", ExprValueUtils.fromObjectValue("1987-10-23 00:00:00", TIMESTAMP)); + + LinkedHashMap valueMap2 = new LinkedHashMap<>(); + valueMap2.put("name", ExprValueUtils.fromObjectValue("Andrea")); + valueMap2.put("birthdate", ExprValueUtils.fromObjectValue("1999-03-14 03:30:00", TIMESTAMP)); + + QueryResult response = new QueryResult( + schemaTimestamp, + Arrays.asList( + new ExprTupleValue(valueMap1), + new ExprTupleValue(valueMap2) + )); + + response.setDatetimeFormat(INCLUDE_TIME_WHEN_NONZERO); + + int i = 0; + for (Object[] objects : response) { + if (i == 0) { + assertArrayEquals(new Object[] {"John", "1987-10-23"}, objects); + } else if (i == 1) { + assertArrayEquals(new Object[] {"Andrea", "1999-03-14 03:30:00"}, objects); + } else { + fail("More rows returned than expected"); + } + i++; + } + } + } \ No newline at end of file diff --git a/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatterTest.java b/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatterTest.java index e421016c00..7aef62e167 100644 --- a/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatterTest.java +++ b/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatterTest.java @@ -16,6 +16,7 @@ package com.amazon.opendistroforelasticsearch.sql.protocol.response.format; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue.ALWAYS_INCLUDE_TIME; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.stringValue; @@ -47,7 +48,8 @@ @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) class JdbcResponseFormatterTest { - private final JdbcResponseFormatter formatter = new JdbcResponseFormatter(COMPACT); + private final JdbcResponseFormatter formatter = new JdbcResponseFormatter(COMPACT, + ALWAYS_INCLUDE_TIME); @Test void format_response() { diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java index 7f024c1028..b5b0a6be89 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java @@ -42,6 +42,7 @@ public class SQLQueryRequest { private static final Set SUPPORTED_FIELDS = ImmutableSet.of( "query", "fetch_size", "parameters"); private static final String QUERY_PARAMS_FORMAT = "format"; + private static final String QUERY_PARAMS_DATETIMEFORMAT = "datetime_format"; private static final String QUERY_PARAMS_SANITIZE = "sanitize"; /** @@ -142,6 +143,10 @@ private String getFormat(Map params) { return "jdbc"; } + public String getDatetimeFormat() { + return params.getOrDefault(QUERY_PARAMS_DATETIMEFORMAT, "always_include_time"); + } + private boolean shouldSanitize(Map params) { if (params.containsKey(QUERY_PARAMS_SANITIZE)) { return Boolean.parseBoolean(params.get(QUERY_PARAMS_SANITIZE)); diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java index b7ea848224..5d16757dad 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java @@ -23,6 +23,8 @@ import com.amazon.opendistroforelasticsearch.sql.protocol.response.format.Format; import com.google.common.collect.ImmutableMap; + +import java.util.HashMap; import java.util.Map; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -44,6 +46,18 @@ public void shouldSupportQueryWithJDBCFormat() { assertEquals(request.format(), Format.JDBC); } + @Test + public void shouldSupportQueryDatetimeFormat() { + Map params = new HashMap<>(); + params.put("format", "jdbc"); + params.put("datetime_format", "never_include_time"); + + SQLQueryRequest request = SQLQueryRequestBuilder.request("SELECT 1") + .params(params) + .build(); + assertEquals("never_include_time", request.getDatetimeFormat()); + } + @Test public void shouldSupportQueryWithQueryFieldOnly() { SQLQueryRequest request =