diff --git a/src/main/java/io/milvus/v2/service/vector/VectorService.java b/src/main/java/io/milvus/v2/service/vector/VectorService.java index 9c4be6cc1..e3a719506 100644 --- a/src/main/java/io/milvus/v2/service/vector/VectorService.java +++ b/src/main/java/io/milvus/v2/service/vector/VectorService.java @@ -41,6 +41,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; public class VectorService extends BaseService { @@ -224,12 +225,17 @@ public DeleteResp delete(MilvusServiceGrpc.MilvusServiceBlockingStub blockingStu if (request.getFilter() == null) { request.setFilter(vectorUtils.getExprById(respR.getPrimaryFieldName(), request.getIds())); } - DeleteRequest deleteRequest = DeleteRequest.newBuilder() + DeleteRequest.Builder builder = DeleteRequest.newBuilder() .setCollectionName(request.getCollectionName()) .setPartitionName(request.getPartitionName()) - .setExpr(request.getFilter()) - .build(); - MutationResult response = blockingStub.delete(deleteRequest); + .setExpr(request.getFilter()); + if (request.getFilter() != null && !request.getFilter().isEmpty()) { + Map filterTemplateValues = request.getFilterTemplateValues(); + filterTemplateValues.forEach((key, value)->{ + builder.putExprTemplateValues(key, vectorUtils.deduceTemplateValue(value)); + }); + } + MutationResult response = blockingStub.delete(builder.build()); rpcUtils.handleResponse(title, response.getStatus()); GTsDict.getInstance().updateCollectionTs(request.getCollectionName(), response.getTimestamp()); return DeleteResp.builder() diff --git a/src/main/java/io/milvus/v2/service/vector/request/DeleteReq.java b/src/main/java/io/milvus/v2/service/vector/request/DeleteReq.java index 07cc76951..ed6c16af7 100644 --- a/src/main/java/io/milvus/v2/service/vector/request/DeleteReq.java +++ b/src/main/java/io/milvus/v2/service/vector/request/DeleteReq.java @@ -23,7 +23,9 @@ import lombok.Data; import lombok.experimental.SuperBuilder; +import java.util.HashMap; import java.util.List; +import java.util.Map; @Data @SuperBuilder @@ -33,4 +35,15 @@ public class DeleteReq { private String partitionName = ""; private String filter; private List ids; + + // Expression template, to improve expression parsing performance in complicated list + // Assume user has a filter = "pk > 3 and city in ["beijing", "shanghai", ......] + // The long list of city will increase the time cost to parse this expression. + // So, we provide exprTemplateValues for this purpose, user can set filter like this: + // filter = "pk > {age} and city in {city}" + // filterTemplateValues = Map{"age": 3, "city": List{"beijing", "shanghai", ......}} + // Valid value of this map can be: + // Boolean, Long, Double, String, List, List, List, List + @Builder.Default + private Map filterTemplateValues = new HashMap<>(); } diff --git a/src/main/java/io/milvus/v2/service/vector/request/QueryReq.java b/src/main/java/io/milvus/v2/service/vector/request/QueryReq.java index ae1aeb988..7a3ad5c44 100644 --- a/src/main/java/io/milvus/v2/service/vector/request/QueryReq.java +++ b/src/main/java/io/milvus/v2/service/vector/request/QueryReq.java @@ -24,9 +24,7 @@ import lombok.Data; import lombok.experimental.SuperBuilder; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.util.*; @Data @SuperBuilder @@ -42,4 +40,15 @@ public class QueryReq { private ConsistencyLevel consistencyLevel = null; private long offset; private long limit; + + // Expression template, to improve expression parsing performance in complicated list + // Assume user has a filter = "pk > 3 and city in ["beijing", "shanghai", ......] + // The long list of city will increase the time cost to parse this expression. + // So, we provide exprTemplateValues for this purpose, user can set filter like this: + // filter = "pk > {age} and city in {city}" + // filterTemplateValues = Map{"age": 3, "city": List{"beijing", "shanghai", ......}} + // Valid value of this map can be: + // Boolean, Long, Double, String, List, List, List, List + @Builder.Default + private Map filterTemplateValues = new HashMap<>(); } diff --git a/src/main/java/io/milvus/v2/service/vector/request/SearchReq.java b/src/main/java/io/milvus/v2/service/vector/request/SearchReq.java index 339adf8a2..b120c1924 100644 --- a/src/main/java/io/milvus/v2/service/vector/request/SearchReq.java +++ b/src/main/java/io/milvus/v2/service/vector/request/SearchReq.java @@ -57,4 +57,15 @@ public class SearchReq { private ConsistencyLevel consistencyLevel = null; private boolean ignoreGrowing; private String groupByFieldName; + + // Expression template, to improve expression parsing performance in complicated list + // Assume user has a filter = "pk > 3 and city in ["beijing", "shanghai", ......] + // The long list of city will increase the time cost to parse this expression. + // So, we provide exprTemplateValues for this purpose, user can set filter like this: + // filter = "pk > {age} and city in {city}" + // filterTemplateValues = Map{"age": 3, "city": List{"beijing", "shanghai", ......}} + // Valid value of this map can be: + // Boolean, Long, Double, String, List, List, List, List + @Builder.Default + private Map filterTemplateValues = new HashMap<>(); } diff --git a/src/main/java/io/milvus/v2/utils/VectorUtils.java b/src/main/java/io/milvus/v2/utils/VectorUtils.java index 0f0e86ef1..51f6f76f6 100644 --- a/src/main/java/io/milvus/v2/utils/VectorUtils.java +++ b/src/main/java/io/milvus/v2/utils/VectorUtils.java @@ -20,6 +20,7 @@ package io.milvus.v2.utils; import com.google.protobuf.ByteString; +import com.sun.org.apache.xpath.internal.operations.Bool; import io.milvus.common.utils.GTsDict; import io.milvus.common.utils.JsonUtils; import io.milvus.v2.common.ConsistencyLevel; @@ -44,6 +45,12 @@ public QueryRequest ConvertToGrpcQueryRequest(QueryReq request){ .addAllPartitionNames(request.getPartitionNames()) .addAllOutputFields(request.getOutputFields()) .setExpr(request.getFilter()); + if (request.getFilter() != null && !request.getFilter().isEmpty()) { + Map filterTemplateValues = request.getFilterTemplateValues(); + filterTemplateValues.forEach((key, value)->{ + builder.putExprTemplateValues(key, deduceTemplateValue(value)); + }); + } // a new parameter from v2.2.9, if user didn't specify consistency level, set this parameter to true if (request.getConsistencyLevel() == null) { @@ -180,6 +187,10 @@ public SearchRequest ConvertToGrpcSearchRequest(SearchReq request) { builder.setDslType(DslType.BoolExprV1); if (request.getFilter() != null && !request.getFilter().isEmpty()) { builder.setDsl(request.getFilter()); + Map filterTemplateValues = request.getFilterTemplateValues(); + filterTemplateValues.forEach((key, value)->{ + builder.putExprTemplateValues(key, deduceTemplateValue(value)); + }); } long guaranteeTimestamp = getGuaranteeTimestamp(request.getConsistencyLevel(), request.getCollectionName()); @@ -195,6 +206,58 @@ public SearchRequest ConvertToGrpcSearchRequest(SearchReq request) { return builder.build(); } + public static TemplateValue deduceTemplateValue(Object value) { + if (value instanceof Boolean) { + return TemplateValue.newBuilder() + .setBoolVal((Boolean)value) + .setType(DataType.Bool) + .build(); + } else if (value instanceof Long) { + return TemplateValue.newBuilder() + .setInt64Val((Long)value) + .setType(DataType.Int64) + .build(); + } else if (value instanceof Double) { + return TemplateValue.newBuilder() + .setFloatVal((Double)value) + .setType(DataType.Double) + .build(); + } else if (value instanceof String) { + return TemplateValue.newBuilder() + .setStringVal((String)value) + .setType(DataType.VarChar) + .build(); + } else if (value instanceof List) { + // TemplateArrayValue and TemplateValue can nest each other + // The element_type of TemplateArrayValue is deduced by its elements: + // 1. if all the elements are the same type, element_type is the first element's type + // 2. if not the same type, element_type is DataType.JSON + List array = (List)value; + TemplateArrayValue.Builder builder = TemplateArrayValue.newBuilder(); + DataType lastType = DataType.UNRECOGNIZED; + boolean sameType = true; + for (Object obj : array) { + TemplateValue tv = deduceTemplateValue(obj); + builder.addArray(tv); + if (lastType == DataType.UNRECOGNIZED) { + lastType = tv.getType(); + } else if (lastType != tv.getType()) { + sameType = false; + } + DataType arrayType = sameType ? lastType : DataType.JSON; + builder.setElementType(arrayType); + builder.setSameType(sameType); + } + + return TemplateValue.newBuilder() + .setArrayVal(builder.build()) + .setType(builder.getElementType()) + .build(); + } else { + throw new ParamException("Unsupported value type for expression template."); + } + } + public static SearchRequest convertAnnSearchParam(@NonNull AnnSearchReq annSearchReq, ConsistencyLevel consistencyLevel) { SearchRequest.Builder builder = SearchRequest.newBuilder(); diff --git a/src/main/milvus-proto b/src/main/milvus-proto index 32b0311b3..13fc3509e 160000 --- a/src/main/milvus-proto +++ b/src/main/milvus-proto @@ -1 +1 @@ -Subproject commit 32b0311b378567df2a3d93dde7b5cc64fd8b7349 +Subproject commit 13fc3509ee0b9bc0ccacf05f6dac2171208ee6ec