diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java index ab4b69440f9bff..4d9bda287d3874 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java @@ -115,6 +115,9 @@ public class EsTable extends Table { // Periodically pull es metadata private EsMetaStateTracker esMetaStateTracker; + // column name -> elasticsearch field data type + private Map column2typeMap = new HashMap<>(); + public EsTable() { super(TableType.ELASTICSEARCH); } @@ -340,6 +343,6 @@ public void syncTableMetaData() { } public List genColumnsFromEs() { - return EsUtil.genColumnsFromEs(client, indexName, mappingType, false); + return EsUtil.genColumnsFromEs(client, indexName, mappingType, false, column2typeMap); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalTable.java index 464fb79b8da262..d06e07d1fab042 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalTable.java @@ -26,7 +26,9 @@ import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * Elasticsearch external table. @@ -34,6 +36,7 @@ public class EsExternalTable extends ExternalTable { private EsTable esTable; + private Map column2typeMap = new HashMap<>(); /** * Create elasticsearch external table. @@ -78,9 +81,11 @@ public TTableDescriptor toThrift() { @Override public List initSchema() { EsRestClient restClient = ((EsExternalCatalog) catalog).getEsRestClient(); - return EsUtil.genColumnsFromEs(restClient, name, null, ((EsExternalCatalog) catalog).enableMappingEsId()); + return EsUtil.genColumnsFromEs(restClient, name, null, ((EsExternalCatalog) catalog).enableMappingEsId(), + column2typeMap); } + private EsTable toEsTable() { List schema = getFullSchema(); EsExternalCatalog esCatalog = (EsExternalCatalog) catalog; @@ -98,6 +103,7 @@ private EsTable toEsTable() { esTable.setHosts(String.join(",", esCatalog.getNodes())); esTable.syncTableMetaData(); esTable.setIncludeHiddenIndex(esCatalog.enableIncludeHiddenIndex()); + esTable.setColumn2typeMap(column2typeMap); return esTable; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java index 0178d9784ecad8..fcf95f87837ab1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java @@ -189,18 +189,18 @@ public static ObjectNode getMappingProps(String sourceIndex, String indexMapping * Add mappingEsId config in es external catalog. **/ public static List genColumnsFromEs(EsRestClient client, String indexName, String mappingType, - boolean mappingEsId) { + boolean mappingEsId, Map column2typeMap) { String mapping = client.getMapping(indexName); ObjectNode mappings = getMapping(mapping); // Get array_fields while removing _meta property. List arrayFields = new ArrayList<>(); ObjectNode rootSchema = getRootSchema(mappings, mappingType, arrayFields); - return genColumnsFromEs(indexName, mappingType, rootSchema, mappingEsId, arrayFields); + return genColumnsFromEs(indexName, mappingType, rootSchema, mappingEsId, arrayFields, column2typeMap); } @VisibleForTesting public static List genColumnsFromEs(String indexName, String mappingType, ObjectNode rootSchema, - boolean mappingEsId, List arrayFields) { + boolean mappingEsId, List arrayFields, Map column2typeMap) { List columns = new ArrayList<>(); if (mappingEsId) { Column column = new Column(); @@ -220,7 +220,8 @@ public static List genColumnsFromEs(String indexName, String mappingType while (iterator.hasNext()) { String fieldName = iterator.next(); ObjectNode fieldValue = (ObjectNode) mappingProps.get(fieldName); - Column column = parseEsField(fieldName, replaceFieldAlias(mappingProps, fieldValue), arrayFields); + Column column = parseEsField(fieldName, replaceFieldAlias(mappingProps, fieldValue), arrayFields, + column2typeMap); columns.add(column); } return columns; @@ -245,7 +246,8 @@ private static ObjectNode replaceFieldAlias(ObjectNode mappingProps, ObjectNode return fieldValue; } - private static Column parseEsField(String fieldName, ObjectNode fieldValue, List arrayFields) { + private static Column parseEsField(String fieldName, ObjectNode fieldValue, List arrayFields, + Map column2typeMap) { Column column = new Column(); column.setName(fieldName); column.setIsKey(true); @@ -256,6 +258,7 @@ private static Column parseEsField(String fieldName, ObjectNode fieldValue, List if (fieldValue.has("type")) { String typeStr = fieldValue.get("type").asText(); column.setComment("Elasticsearch type is " + typeStr); + column2typeMap.put(fieldName, typeStr); // reference https://www.elastic.co/guide/en/elasticsearch/reference/8.3/sql-data-types.html switch (typeStr) { case "null": @@ -298,15 +301,17 @@ private static Column parseEsField(String fieldName, ObjectNode fieldValue, List type = ScalarType.createStringType(); break; case "nested": - case "object": type = Type.JSONB; break; default: type = Type.UNSUPPORTED; } } else { + // When there is no explicit type in mapping, it indicates this type is an `object` in Elasticsearch. + // reference: https://www.elastic.co/guide/en/elasticsearch/reference/current/object.html type = Type.JSONB; - column.setComment("Elasticsearch no type"); + column.setComment("Elasticsearch type is object"); + column2typeMap.put(fieldName, "object"); } if (arrayFields.contains(fieldName)) { column.setType(ArrayType.create(type, true)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java index 518f6d93bbd2f1..b549f379bb4457 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java @@ -70,14 +70,14 @@ public final class QueryBuilders { * Generate dsl from compound expr. **/ private static QueryBuilder toCompoundEsDsl(Expr expr, List notPushDownList, - Map fieldsContext, BuilderOptions builderOptions) { + Map fieldsContext, BuilderOptions builderOptions, Map column2typeMap) { CompoundPredicate compoundPredicate = (CompoundPredicate) expr; switch (compoundPredicate.getOp()) { case AND: { QueryBuilder left = toEsDsl(compoundPredicate.getChild(0), notPushDownList, fieldsContext, - builderOptions); + builderOptions, column2typeMap); QueryBuilder right = toEsDsl(compoundPredicate.getChild(1), notPushDownList, fieldsContext, - builderOptions); + builderOptions, column2typeMap); if (left != null && right != null) { return QueryBuilders.boolQuery().must(left).must(right); } @@ -86,9 +86,9 @@ private static QueryBuilder toCompoundEsDsl(Expr expr, List notPushDownLis case OR: { int beforeSize = notPushDownList.size(); QueryBuilder left = toEsDsl(compoundPredicate.getChild(0), notPushDownList, fieldsContext, - builderOptions); + builderOptions, column2typeMap); QueryBuilder right = toEsDsl(compoundPredicate.getChild(1), notPushDownList, fieldsContext, - builderOptions); + builderOptions, column2typeMap); int afterSize = notPushDownList.size(); if (left != null && right != null) { return QueryBuilders.boolQuery().should(left).should(right); @@ -101,7 +101,7 @@ private static QueryBuilder toCompoundEsDsl(Expr expr, List notPushDownLis } case NOT: { QueryBuilder child = toEsDsl(compoundPredicate.getChild(0), notPushDownList, fieldsContext, - builderOptions); + builderOptions, column2typeMap); if (child != null) { return QueryBuilders.boolQuery().mustNot(child); } @@ -122,10 +122,10 @@ private static Expr exprWithoutCast(Expr expr) { return expr; } - public static QueryBuilder toEsDsl(Expr expr) { + public static QueryBuilder toEsDsl(Expr expr, Map column2typeMap) { return toEsDsl(expr, new ArrayList<>(), new HashMap<>(), BuilderOptions.builder().likePushDown(Boolean.parseBoolean(EsResource.LIKE_PUSH_DOWN_DEFAULT_VALUE)) - .build()); + .build(), column2typeMap); } private static TExprOpcode flipOpCode(TExprOpcode opCode) { @@ -185,32 +185,44 @@ private static QueryBuilder parseIsNullPredicate(Expr expr, String column) { return QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(column)); } - private static QueryBuilder parseLikePredicate(Expr expr, String column) { - LikePredicate likePredicate = (LikePredicate) expr; - if (likePredicate.getOp().equals(Operator.LIKE)) { - char[] chars = likePredicate.getChild(1).getStringValue().toCharArray(); - // example of translation : - // abc_123 ===> abc?123 - // abc%ykz ===> abc*123 - // %abc123 ===> *abc123 - // _abc123 ===> ?abc123 - // \\_abc1 ===> \\_abc1 - // abc\\_123 ===> abc\\_123 - // abc\\%123 ===> abc\\%123 - // NOTE. user must input sql like 'abc\\_123' or 'abc\\%ykz' - for (int i = 0; i < chars.length; i++) { - if (chars[i] == '_' || chars[i] == '%') { - if (i == 0) { - chars[i] = (chars[i] == '_') ? '?' : '*'; - } else if (chars[i - 1] != '\\') { - chars[i] = (chars[i] == '_') ? '?' : '*'; - } - } + private static QueryBuilder parseLikeExpression(Expr expr, String column) { + String pattern; + if (expr instanceof LikePredicate) { + LikePredicate likePredicate = (LikePredicate) expr; + if (!likePredicate.getOp().equals(Operator.LIKE)) { + return QueryBuilders.wildcardQuery(column, likePredicate.getChild(1).getStringValue()); + } + pattern = likePredicate.getChild(1).getStringValue(); + } else if (expr instanceof FunctionCallExpr) { + FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr; + String fnName = functionCallExpr.getFnName().getFunction(); + if (!fnName.equalsIgnoreCase("like")) { + return QueryBuilders.wildcardQuery(column, functionCallExpr.getChild(1).getStringValue()); } - return QueryBuilders.wildcardQuery(column, new String(chars)); + pattern = functionCallExpr.getChild(1).getStringValue(); } else { - return QueryBuilders.wildcardQuery(column, likePredicate.getChild(1).getStringValue()); + throw new IllegalArgumentException("Unsupported expression type"); + } + char[] chars = pattern.toCharArray(); + // example of translation : + // abc_123 ===> abc?123 + // abc%ykz ===> abc*123 + // %abc123 ===> *abc123 + // _abc123 ===> ?abc123 + // \\_abc1 ===> \\_abc1 + // abc\\_123 ===> abc\\_123 + // abc\\%123 ===> abc\\%123 + // NOTE. user must input sql like 'abc\\_123' or 'abc\\%ykz' + for (int i = 0; i < chars.length; i++) { + if (chars[i] == '_' || chars[i] == '%') { + if (i == 0) { + chars[i] = (chars[i] == '_') ? '?' : '*'; + } else if (chars[i - 1] != '\\') { + chars[i] = (chars[i] == '_') ? '?' : '*'; + } + } } + return QueryBuilders.wildcardQuery(column, new String(chars)); } private static QueryBuilder parseInPredicate(Expr expr, String column, boolean needDateCompat) { @@ -257,18 +269,18 @@ private static String getColumnFromExpr(Expr expr) { * Doris expr to es dsl. **/ public static QueryBuilder toEsDsl(Expr expr, List notPushDownList, Map fieldsContext, - BuilderOptions builderOptions) { + BuilderOptions builderOptions, Map column2typeMap) { if (expr == null) { return null; } // esquery functionCallExpr will be rewritten to castExpr in where clause rewriter, // so we get the functionCallExpr here. if (expr instanceof CastExpr) { - return toEsDsl(expr.getChild(0), notPushDownList, fieldsContext, builderOptions); + return toEsDsl(expr.getChild(0), notPushDownList, fieldsContext, builderOptions, column2typeMap); } // CompoundPredicate, `between` also converted to CompoundPredicate. if (expr instanceof CompoundPredicate) { - return toCompoundEsDsl(expr, notPushDownList, fieldsContext, builderOptions); + return toCompoundEsDsl(expr, notPushDownList, fieldsContext, builderOptions, column2typeMap); } TExprOpcode opCode = expr.getOpcode(); boolean isFlip = false; @@ -287,6 +299,7 @@ public static QueryBuilder toEsDsl(Expr expr, List notPushDownList, Map needCompatDateFields = builderOptions.getNeedCompatDateFields(); boolean needDateCompat = needCompatDateFields != null && needCompatDateFields.contains(column); @@ -313,24 +326,29 @@ public static QueryBuilder toEsDsl(Expr expr, List notPushDownList, Map notPushDownList = new ArrayList<>(); + if (table.getColumn2typeMap() == null) { + table.genColumnsFromEs(); + } for (Expr expr : conjuncts) { QueryBuilder queryBuilder = QueryBuilders.toEsDsl(expr, notPushDownList, fieldsContext, BuilderOptions.builder().likePushDown(table.isLikePushDown()) - .needCompatDateFields(table.needCompatDateFields()).build()); + .needCompatDateFields(table.needCompatDateFields()).build(), + table.getColumn2typeMap()); if (queryBuilder != null) { hasFilter = true; boolQueryBuilder.must(queryBuilder); diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java index f5b3133fbf7a09..8e184001e35bad 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; /** @@ -221,7 +222,7 @@ public void testEs8Mapping() throws IOException, URISyntaxException { public void testDateType() throws IOException, URISyntaxException { ObjectNode testDateFormat = EsUtil.getRootSchema( EsUtil.getMapping(loadJsonFromFile("data/es/test_date_format.json")), null, new ArrayList<>()); - List parseColumns = EsUtil.genColumnsFromEs("test_date_format", null, testDateFormat, false, new ArrayList<>()); + List parseColumns = EsUtil.genColumnsFromEs("test_date_format", null, testDateFormat, false, new ArrayList<>(), new HashMap<>()); Assertions.assertEquals(8, parseColumns.size()); for (Column column : parseColumns) { String name = column.getName(); @@ -254,7 +255,7 @@ public void testDateType() throws IOException, URISyntaxException { public void testFieldAlias() throws IOException, URISyntaxException { ObjectNode testFieldAlias = EsUtil.getRootSchema( EsUtil.getMapping(loadJsonFromFile("data/es/test_field_alias.json")), null, new ArrayList<>()); - List parseColumns = EsUtil.genColumnsFromEs("test_field_alias", null, testFieldAlias, true, new ArrayList<>()); + List parseColumns = EsUtil.genColumnsFromEs("test_field_alias", null, testFieldAlias, true, new ArrayList<>(), new HashMap<>()); Assertions.assertEquals("datetimev2(0)", parseColumns.get(2).getType().toSql()); Assertions.assertEquals("text", parseColumns.get(4).getType().toSql()); } @@ -264,7 +265,7 @@ public void testComplexType() throws IOException, URISyntaxException { ObjectNode testFieldAlias = EsUtil.getRootSchema( EsUtil.getMapping(loadJsonFromFile("data/es/es6_dynamic_complex_type.json")), null, new ArrayList<>()); List columns = EsUtil.genColumnsFromEs("test_complex_type", "complex_type", testFieldAlias, true, - new ArrayList<>()); + new ArrayList<>(), new HashMap<>()); Assertions.assertEquals(3, columns.size()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java index 64f69bb4644041..e5788dc3e8594a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java @@ -70,30 +70,32 @@ public void testBinaryPredicateConvertEsDsl() { Expr ltExpr = new BinaryPredicate(Operator.LT, k1, intLiteral); Expr gtExpr = new BinaryPredicate(Operator.GT, k1, intLiteral); Expr efnExpr = new BinaryPredicate(Operator.EQ_FOR_NULL, new SlotRef(null, "k1"), new IntLiteral(3)); - Assertions.assertEquals("{\"term\":{\"k1\":3}}", QueryBuilders.toEsDsl(eqExpr).toJson()); + Map column2typeMap = new HashMap<>(); + Assertions.assertEquals("{\"term\":{\"k1\":3}}", QueryBuilders.toEsDsl(eqExpr, column2typeMap).toJson()); Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}", - QueryBuilders.toEsDsl(neExpr).toJson()); - Assertions.assertEquals("{\"range\":{\"k1\":{\"lte\":3}}}", QueryBuilders.toEsDsl(leExpr).toJson()); - Assertions.assertEquals("{\"range\":{\"k1\":{\"gte\":3}}}", QueryBuilders.toEsDsl(geExpr).toJson()); - Assertions.assertEquals("{\"range\":{\"k1\":{\"lt\":3}}}", QueryBuilders.toEsDsl(ltExpr).toJson()); - Assertions.assertEquals("{\"range\":{\"k1\":{\"gt\":3}}}", QueryBuilders.toEsDsl(gtExpr).toJson()); - Assertions.assertEquals("{\"term\":{\"k1\":3}}", QueryBuilders.toEsDsl(efnExpr).toJson()); + QueryBuilders.toEsDsl(neExpr, column2typeMap).toJson()); + Assertions.assertEquals("{\"range\":{\"k1\":{\"lte\":3}}}", QueryBuilders.toEsDsl(leExpr, column2typeMap).toJson()); + Assertions.assertEquals("{\"range\":{\"k1\":{\"gte\":3}}}", QueryBuilders.toEsDsl(geExpr, column2typeMap).toJson()); + Assertions.assertEquals("{\"range\":{\"k1\":{\"lt\":3}}}", QueryBuilders.toEsDsl(ltExpr, column2typeMap).toJson()); + Assertions.assertEquals("{\"range\":{\"k1\":{\"gt\":3}}}", QueryBuilders.toEsDsl(gtExpr, column2typeMap).toJson()); + Assertions.assertEquals("{\"term\":{\"k1\":3}}", QueryBuilders.toEsDsl(efnExpr, column2typeMap).toJson()); SlotRef k2 = new SlotRef(null, "k2"); Expr dateTimeLiteral = new StringLiteral("2023-02-19 22:00:00"); Expr dateTimeEqExpr = new BinaryPredicate(Operator.EQ, k2, dateTimeLiteral); Assertions.assertEquals("{\"term\":{\"k2\":\"2023-02-19T22:00:00.000+08:00\"}}", QueryBuilders.toEsDsl(dateTimeEqExpr, new ArrayList<>(), new HashMap<>(), - BuilderOptions.builder().needCompatDateFields(Lists.newArrayList("k2")).build()).toJson()); + BuilderOptions.builder().needCompatDateFields(Lists.newArrayList("k2")).build(), + new HashMap<>()).toJson()); SlotRef k3 = new SlotRef(null, "k3"); Expr stringLiteral = new StringLiteral(""); Expr stringNeExpr = new BinaryPredicate(Operator.NE, k3, stringLiteral); Assertions.assertEquals("{\"bool\":{\"must\":{\"exists\":{\"field\":\"k3\"}},\"must_not\":{\"term\":{\"k3\":\"\"}}}}", - QueryBuilders.toEsDsl(stringNeExpr).toJson()); + QueryBuilders.toEsDsl(stringNeExpr, column2typeMap).toJson()); stringLiteral = new StringLiteral("message"); stringNeExpr = new BinaryPredicate(Operator.NE, k3, stringLiteral); Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k3\":\"message\"}}}}", - QueryBuilders.toEsDsl(stringNeExpr).toJson()); + QueryBuilders.toEsDsl(stringNeExpr, column2typeMap).toJson()); } @Test @@ -102,6 +104,7 @@ public void testCompoundPredicateConvertEsDsl() { IntLiteral intLiteral1 = new IntLiteral(3); SlotRef k2 = new SlotRef(null, "k2"); IntLiteral intLiteral2 = new IntLiteral(5); + Map column2typeMap = new HashMap<>(); BinaryPredicate binaryPredicate1 = new BinaryPredicate(Operator.EQ, k1, intLiteral1); BinaryPredicate binaryPredicate2 = new BinaryPredicate(Operator.GT, k2, intLiteral2); CompoundPredicate andPredicate = new CompoundPredicate(CompoundPredicate.Operator.AND, binaryPredicate1, @@ -110,21 +113,22 @@ public void testCompoundPredicateConvertEsDsl() { binaryPredicate2); CompoundPredicate notPredicate = new CompoundPredicate(CompoundPredicate.Operator.NOT, binaryPredicate1, null); Assertions.assertEquals("{\"bool\":{\"must\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}", - QueryBuilders.toEsDsl(andPredicate).toJson()); + QueryBuilders.toEsDsl(andPredicate, column2typeMap).toJson()); Assertions.assertEquals("{\"bool\":{\"should\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}", - QueryBuilders.toEsDsl(orPredicate).toJson()); + QueryBuilders.toEsDsl(orPredicate, column2typeMap).toJson()); Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}", - QueryBuilders.toEsDsl(notPredicate).toJson()); + QueryBuilders.toEsDsl(notPredicate, column2typeMap).toJson()); } @Test public void testIsNullPredicateConvertEsDsl() { SlotRef k1 = new SlotRef(null, "k1"); + Map column2typeMap = new HashMap<>(); IsNullPredicate isNullPredicate = new IsNullPredicate(k1, false); IsNullPredicate isNotNullPredicate = new IsNullPredicate(k1, true); Assertions.assertEquals("{\"bool\":{\"must_not\":{\"exists\":{\"field\":\"k1\"}}}}", - QueryBuilders.toEsDsl(isNullPredicate).toJson()); - Assertions.assertEquals("{\"exists\":{\"field\":\"k1\"}}", QueryBuilders.toEsDsl(isNotNullPredicate).toJson()); + QueryBuilders.toEsDsl(isNullPredicate, column2typeMap).toJson()); + Assertions.assertEquals("{\"exists\":{\"field\":\"k1\"}}", QueryBuilders.toEsDsl(isNotNullPredicate, column2typeMap).toJson()); } @Test @@ -136,13 +140,28 @@ public void testLikePredicateConvertEsDsl() { LikePredicate likePredicate1 = new LikePredicate(LikePredicate.Operator.LIKE, k1, stringLiteral1); LikePredicate regexPredicate = new LikePredicate(LikePredicate.Operator.REGEXP, k1, stringLiteral2); LikePredicate likePredicate2 = new LikePredicate(LikePredicate.Operator.LIKE, k1, stringLiteral3); - Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", QueryBuilders.toEsDsl(likePredicate1).toJson()); - Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", QueryBuilders.toEsDsl(regexPredicate).toJson()); - Assertions.assertEquals("{\"wildcard\":{\"k1\":\"1?2\"}}", QueryBuilders.toEsDsl(likePredicate2).toJson()); + Map column2typeMap = new HashMap<>(); + column2typeMap.put("k1", "keyword"); + Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", QueryBuilders.toEsDsl(likePredicate1, column2typeMap).toJson()); + Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", QueryBuilders.toEsDsl(regexPredicate, column2typeMap).toJson()); + Assertions.assertEquals("{\"wildcard\":{\"k1\":\"1?2\"}}", QueryBuilders.toEsDsl(likePredicate2, column2typeMap).toJson()); List notPushDownList = new ArrayList<>(); Assertions.assertNull(QueryBuilders.toEsDsl(likePredicate2, notPushDownList, new HashMap<>(), - BuilderOptions.builder().likePushDown(false).build())); + BuilderOptions.builder().likePushDown(false).build(), column2typeMap)); Assertions.assertFalse(notPushDownList.isEmpty()); + + // like can not push down + column2typeMap.clear(); + notPushDownList.clear(); + column2typeMap.put("k1", "text"); + Assertions.assertNull(QueryBuilders.toEsDsl(likePredicate1, notPushDownList, new HashMap<>(), + BuilderOptions.builder().likePushDown(true).build(), column2typeMap)); + Assertions.assertNull(QueryBuilders.toEsDsl(likePredicate2, notPushDownList, new HashMap<>(), + BuilderOptions.builder().likePushDown(true).build(), column2typeMap)); + Assertions.assertNull(QueryBuilders.toEsDsl(regexPredicate, notPushDownList, new HashMap<>(), + BuilderOptions.builder().likePushDown(true).build(), column2typeMap)); + + Assertions.assertEquals(3, notPushDownList.size()); } @Test @@ -155,9 +174,10 @@ public void testInPredicateConvertEsDsl() { intLiterals.add(intLiteral2); InPredicate isInPredicate = new InPredicate(k1, intLiterals, false); InPredicate isNotInPredicate = new InPredicate(k1, intLiterals, true); - Assertions.assertEquals("{\"terms\":{\"k1\":[3,5]}}", QueryBuilders.toEsDsl(isInPredicate).toJson()); + Map column2typeMap = new HashMap<>(); + Assertions.assertEquals("{\"terms\":{\"k1\":[3,5]}}", QueryBuilders.toEsDsl(isInPredicate, column2typeMap).toJson()); Assertions.assertEquals("{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}", - QueryBuilders.toEsDsl(isNotInPredicate).toJson()); + QueryBuilders.toEsDsl(isNotInPredicate, column2typeMap).toJson()); } @Test @@ -165,11 +185,12 @@ public void testFunctionCallConvertEsDsl() { SlotRef k1 = new SlotRef(null, "k1"); String str = "{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}"; StringLiteral stringLiteral = new StringLiteral(str); + Map column2typeMap = new HashMap<>(); List exprs = new ArrayList<>(); exprs.add(k1); exprs.add(stringLiteral); FunctionCallExpr functionCallExpr = new FunctionCallExpr("esquery", exprs); - Assertions.assertEquals(str, QueryBuilders.toEsDsl(functionCallExpr).toJson()); + Assertions.assertEquals(str, QueryBuilders.toEsDsl(functionCallExpr, column2typeMap).toJson()); SlotRef k2 = new SlotRef(null, "k2"); IntLiteral intLiteral = new IntLiteral(5); @@ -178,7 +199,44 @@ public void testFunctionCallConvertEsDsl() { functionCallExpr); Assertions.assertEquals( "{\"bool\":{\"must\":[{\"term\":{\"k2\":5}},{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}]}}", - QueryBuilders.toEsDsl(compoundPredicate).toJson()); + QueryBuilders.toEsDsl(compoundPredicate, column2typeMap).toJson()); + } + + @Test + public void testLikeFunctionCallConvertEsDsl() { + SlotRef k1 = new SlotRef(null, "k1"); + String str1 = "text%"; + StringLiteral stringLiteral1 = new StringLiteral(str1); + Map column2typeMap = new HashMap<>(); + column2typeMap.put("k1", "keyword"); + List exprs1 = new ArrayList<>(); + exprs1.add(k1); + exprs1.add(stringLiteral1); + FunctionCallExpr likeFunctionCallExpr = new FunctionCallExpr("like", exprs1); + Assertions.assertEquals("{\"wildcard\":{\"k1\":\"text*\"}}", + QueryBuilders.toEsDsl(likeFunctionCallExpr, column2typeMap).toJson()); + + SlotRef k2 = new SlotRef(null, "k2"); + String str2 = "text$"; + StringLiteral stringLiteral2 = new StringLiteral(str2); + List exprs2 = new ArrayList<>(); + exprs2.add(k2); + exprs2.add(stringLiteral2); + FunctionCallExpr regexFunctionCallExpr = new FunctionCallExpr("regexp", exprs2); + Assertions.assertEquals("{\"wildcard\":{\"k2\":\"text$\"}}", + QueryBuilders.toEsDsl(regexFunctionCallExpr, column2typeMap).toJson()); + + + column2typeMap.clear(); + List notPushDownList = new ArrayList<>(); + column2typeMap.put("k1", "text"); + Assertions.assertNull(QueryBuilders.toEsDsl(likeFunctionCallExpr, notPushDownList, new HashMap<>(), + BuilderOptions.builder().likePushDown(true).build(), column2typeMap)); + Assertions.assertEquals("{\"wildcard\":{\"k2\":\"text$\"}}", + QueryBuilders.toEsDsl(regexFunctionCallExpr, notPushDownList, new HashMap<>(), + BuilderOptions.builder().likePushDown(true).build(), column2typeMap).toJson()); + + Assertions.assertEquals(1, notPushDownList.size()); } @Test @@ -188,8 +246,10 @@ public void testCastConvertEsDsl() { BinaryPredicate castPredicate = new BinaryPredicate(Operator.EQ, castExpr, new IntLiteral(3)); List notPushDownList = new ArrayList<>(); Map fieldsContext = new HashMap<>(); + Map col2typeMap = new HashMap<>(); BuilderOptions builderOptions = BuilderOptions.builder().likePushDown(true).build(); - Assertions.assertNull(QueryBuilders.toEsDsl(castPredicate, notPushDownList, fieldsContext, builderOptions)); + Assertions.assertNull(QueryBuilders.toEsDsl(castPredicate, notPushDownList, fieldsContext, builderOptions, + col2typeMap)); Assertions.assertEquals(1, notPushDownList.size()); SlotRef k2 = new SlotRef(null, "k2"); @@ -198,7 +258,7 @@ public void testCastConvertEsDsl() { CompoundPredicate compoundPredicate = new CompoundPredicate(CompoundPredicate.Operator.OR, castPredicate, eqPredicate); - QueryBuilders.toEsDsl(compoundPredicate, notPushDownList, fieldsContext, builderOptions); + QueryBuilders.toEsDsl(compoundPredicate, notPushDownList, fieldsContext, builderOptions, col2typeMap); Assertions.assertEquals(3, notPushDownList.size()); SlotRef k3 = new SlotRef(null, "k3"); @@ -206,7 +266,7 @@ public void testCastConvertEsDsl() { CastExpr castDoubleExpr = new CastExpr(Type.DOUBLE, k3); BinaryPredicate castDoublePredicate = new BinaryPredicate(Operator.GE, castDoubleExpr, new FloatLiteral(3.0, Type.DOUBLE)); - QueryBuilders.toEsDsl(castDoublePredicate, notPushDownList, fieldsContext, builderOptions); + QueryBuilders.toEsDsl(castDoublePredicate, notPushDownList, fieldsContext, builderOptions, col2typeMap); Assertions.assertEquals(3, notPushDownList.size()); SlotRef k4 = new SlotRef(null, "k4"); @@ -214,25 +274,29 @@ public void testCastConvertEsDsl() { CastExpr castFloatExpr = new CastExpr(Type.FLOAT, k4); BinaryPredicate castFloatPredicate = new BinaryPredicate(Operator.GE, new FloatLiteral(3.0, Type.FLOAT), castFloatExpr); - QueryBuilders.QueryBuilder queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions); + QueryBuilders.QueryBuilder queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, + fieldsContext, builderOptions, col2typeMap); Assertions.assertEquals("{\"range\":{\"k4\":{\"lte\":3.0}}}", queryBuilder.toJson()); Assertions.assertEquals(3, notPushDownList.size()); castFloatPredicate = new BinaryPredicate(Operator.LE, new FloatLiteral(3.0, Type.FLOAT), castFloatExpr); - queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions); + queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions, + col2typeMap); Assertions.assertEquals("{\"range\":{\"k4\":{\"gte\":3.0}}}", queryBuilder.toJson()); Assertions.assertEquals(3, notPushDownList.size()); castFloatPredicate = new BinaryPredicate(Operator.LT, new FloatLiteral(3.0, Type.FLOAT), castFloatExpr); - queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions); + queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions, + col2typeMap); Assertions.assertEquals("{\"range\":{\"k4\":{\"gt\":3.0}}}", queryBuilder.toJson()); Assertions.assertEquals(3, notPushDownList.size()); castFloatPredicate = new BinaryPredicate(Operator.GT, new FloatLiteral(3.0, Type.FLOAT), castFloatExpr); - queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions); + queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions, + col2typeMap); Assertions.assertEquals("{\"range\":{\"k4\":{\"lt\":3.0}}}", queryBuilder.toJson()); Assertions.assertEquals(3, notPushDownList.size()); } diff --git a/regression-test/data/external_table_p0/es/test_es_query.out b/regression-test/data/external_table_p0/es/test_es_query.out index 6d761babed7428..628fe9772e938e 100644 --- a/regression-test/data/external_table_p0/es/test_es_query.out +++ b/regression-test/data/external_table_p0/es/test_es_query.out @@ -20,6 +20,10 @@ I'm not null or empty I'm not null or empty -- !sql06 -- +\N +\N +\N + I'm not null or empty -- !sql07 -- @@ -220,6 +224,17 @@ Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] -- !sql_5_28 -- value1 value2 +-- !sql_5_29 -- +string1 text#1 +string2 text2 +string3 text3_4*5 + +-- !sql_5_30 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string_ignore_above_10 text_ignore_above_10 + -- !sql_6_02 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 @@ -349,6 +364,17 @@ Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] -- !sql_6_28 -- value1 value2 +-- !sql_6_29 -- +string1 text#1 +string2 text2 +string3 text3_4*5 + +-- !sql_6_30 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string_ignore_above_10 text_ignore_above_10 + -- !sql_7_02 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 @@ -517,6 +543,19 @@ Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] +-- !sql_7_35 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string4 text3_4*5 + +-- !sql_7_36 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string4 text3_4*5 +string_ignore_above_10 text_ignore_above_10 + -- !sql_7_50 -- value1 value2 @@ -685,6 +724,19 @@ Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] +-- !sql_8_33 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string4 text3_4*5 + +-- !sql_8_34 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string4 text3_4*5 +string_ignore_above_10 text_ignore_above_10 + -- !sql01 -- ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] @@ -706,6 +758,10 @@ I'm not null or empty I'm not null or empty -- !sql06 -- +\N +\N +\N + I'm not null or empty -- !sql07 -- @@ -906,6 +962,17 @@ Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] -- !sql_5_28 -- value1 value2 +-- !sql_5_29 -- +string1 text#1 +string2 text2 +string3 text3_4*5 + +-- !sql_5_30 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string_ignore_above_10 text_ignore_above_10 + -- !sql_6_02 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 @@ -1035,6 +1102,17 @@ Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] -- !sql_6_28 -- value1 value2 +-- !sql_6_29 -- +string1 text#1 +string2 text2 +string3 text3_4*5 + +-- !sql_6_30 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string_ignore_above_10 text_ignore_above_10 + -- !sql_7_02 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 @@ -1203,6 +1281,19 @@ Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] +-- !sql_7_35 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string4 text3_4*5 + +-- !sql_7_36 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string4 text3_4*5 +string_ignore_above_10 text_ignore_above_10 + -- !sql_7_50 -- value1 value2 @@ -1371,3 +1462,16 @@ Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] +-- !sql_8_33 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string4 text3_4*5 + +-- !sql_8_34 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string4 text3_4*5 +string_ignore_above_10 text_ignore_above_10 + diff --git a/regression-test/suites/external_table_p0/es/test_es_query.groovy b/regression-test/suites/external_table_p0/es/test_es_query.groovy index 796a6656c13c92..08dbc0a94eec19 100644 --- a/regression-test/suites/external_table_p0/es/test_es_query.groovy +++ b/regression-test/suites/external_table_p0/es/test_es_query.groovy @@ -214,6 +214,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql_5_26 """select test6 from test2;""" order_qt_sql_5_27 """select * from composite_type_array order by name;""" order_qt_sql_5_28 """select * from test3_20231005;""" + order_qt_sql_5_29 """select test1, test2 from test1 where test1 like 'string%';""" + order_qt_sql_5_30 """select test1, test2 from test1 where test2 like 'text%';""" sql """switch test_es_query_es6""" // order_qt_sql_6_01 """show tables""" @@ -244,6 +246,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql_6_26 """select test6 from test2;""" order_qt_sql_6_27 """select * from composite_type_array order by name;""" order_qt_sql_6_28 """select * from test3_20231005;""" + order_qt_sql_6_29 """select test1, test2 from test1 where test1 like 'string%';""" + order_qt_sql_6_30 """select test1, test2 from test1 where test2 like 'text%';""" List> tables6N = sql """show tables""" boolean notContainHide = true @@ -299,6 +303,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql_7_32 """select test6 from test1;""" order_qt_sql_7_33 """select test6 from test2;""" order_qt_sql_7_34 """select * from composite_type_array order by name;""" + order_qt_sql_7_35 """select test1, test2 from test1 where test1 like 'string%';""" + order_qt_sql_7_36 """select test1, test2 from test1 where test2 like 'text%';""" List> tables7N = sql """show tables""" boolean notContainHide7 = true @@ -354,7 +360,9 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql_8_30 """select test6 from test1;""" order_qt_sql_8_31 """select test6 from test2;""" order_qt_sql_8_32 """select * from composite_type_array order by name;""" - + order_qt_sql_8_33 """select test1, test2 from test1 where test1 like 'string%';""" + order_qt_sql_8_34 """select test1, test2 from test1 where test2 like 'text%';""" + } sql """set enable_es_shard_scroll=true"""