Support expression template values

Signed-off-by: yhmo <[email protected]>
milvus-io · Oct 25, 2024 · d0f4082 · d0f4082
1 parent 3d6de38
commit d0f4082
Show file tree

Hide file tree

Showing 6 changed files with 103 additions and 8 deletions.
diff --git a/src/main/java/io/milvus/v2/service/vector/VectorService.java b/src/main/java/io/milvus/v2/service/vector/VectorService.java
@@ -41,6 +41,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
 public class VectorService extends BaseService {
@@ -224,12 +225,17 @@ public DeleteResp delete(MilvusServiceGrpc.MilvusServiceBlockingStub blockingStu
         if (request.getFilter() == null) {
             request.setFilter(vectorUtils.getExprById(respR.getPrimaryFieldName(), request.getIds()));
         }
-        DeleteRequest deleteRequest = DeleteRequest.newBuilder()
+        DeleteRequest.Builder builder = DeleteRequest.newBuilder()
                 .setCollectionName(request.getCollectionName())
                 .setPartitionName(request.getPartitionName())
-                .setExpr(request.getFilter())
-                .build();
-        MutationResult response = blockingStub.delete(deleteRequest);
+                .setExpr(request.getFilter());
+        if (request.getFilter() != null && !request.getFilter().isEmpty()) {
+            Map<String, Object> filterTemplateValues = request.getFilterTemplateValues();
+            filterTemplateValues.forEach((key, value)->{
+                builder.putExprTemplateValues(key, vectorUtils.deduceTemplateValue(value));
+            });
+        }
+        MutationResult response = blockingStub.delete(builder.build());
         rpcUtils.handleResponse(title, response.getStatus());
         GTsDict.getInstance().updateCollectionTs(request.getCollectionName(), response.getTimestamp());
         return DeleteResp.builder()

diff --git a/src/main/java/io/milvus/v2/service/vector/request/DeleteReq.java b/src/main/java/io/milvus/v2/service/vector/request/DeleteReq.java
@@ -23,7 +23,9 @@
 import lombok.Data;
 import lombok.experimental.SuperBuilder;
 
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 @Data
 @SuperBuilder
@@ -33,4 +35,15 @@ public class DeleteReq {
     private String partitionName = "";
     private String filter;
     private List<Object> ids;
+
+    // Expression template, to improve expression parsing performance in complicated list
+    // Assume user has a filter = "pk > 3 and city in ["beijing", "shanghai", ......]
+    // The long list of city will increase the time cost to parse this expression.
+    // So, we provide exprTemplateValues for this purpose, user can set filter like this:
+    //     filter = "pk > {age} and city in {city}"
+    //     filterTemplateValues = Map{"age": 3, "city": List<String>{"beijing", "shanghai", ......}}
+    // Valid value of this map can be:
+    //     Boolean, Long, Double, String, List<Boolean>, List<Long>, List<Double>, List<String>
+    @Builder.Default
+    private Map<String, Object> filterTemplateValues = new HashMap<>();
 }
diff --git a/src/main/java/io/milvus/v2/service/vector/request/QueryReq.java b/src/main/java/io/milvus/v2/service/vector/request/QueryReq.java
@@ -24,9 +24,7 @@
 import lombok.Data;
 import lombok.experimental.SuperBuilder;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
+import java.util.*;
 
 @Data
 @SuperBuilder
@@ -42,4 +40,15 @@ public class QueryReq {
     private ConsistencyLevel consistencyLevel = null;
     private long offset;
     private long limit;
+
+    // Expression template, to improve expression parsing performance in complicated list
+    // Assume user has a filter = "pk > 3 and city in ["beijing", "shanghai", ......]
+    // The long list of city will increase the time cost to parse this expression.
+    // So, we provide exprTemplateValues for this purpose, user can set filter like this:
+    //     filter = "pk > {age} and city in {city}"
+    //     filterTemplateValues = Map{"age": 3, "city": List<String>{"beijing", "shanghai", ......}}
+    // Valid value of this map can be:
+    //     Boolean, Long, Double, String, List<Boolean>, List<Long>, List<Double>, List<String>
+    @Builder.Default
+    private Map<String, Object> filterTemplateValues = new HashMap<>();
 }
diff --git a/src/main/java/io/milvus/v2/service/vector/request/SearchReq.java b/src/main/java/io/milvus/v2/service/vector/request/SearchReq.java
@@ -57,4 +57,15 @@ public class SearchReq {
     private ConsistencyLevel consistencyLevel = null;
     private boolean ignoreGrowing;
     private String groupByFieldName;
+
+    // Expression template, to improve expression parsing performance in complicated list
+    // Assume user has a filter = "pk > 3 and city in ["beijing", "shanghai", ......]
+    // The long list of city will increase the time cost to parse this expression.
+    // So, we provide exprTemplateValues for this purpose, user can set filter like this:
+    //     filter = "pk > {age} and city in {city}"
+    //     filterTemplateValues = Map{"age": 3, "city": List<String>{"beijing", "shanghai", ......}}
+    // Valid value of this map can be:
+    //     Boolean, Long, Double, String, List<Boolean>, List<Long>, List<Double>, List<String>
+    @Builder.Default
+    private Map<String, Object> filterTemplateValues = new HashMap<>();
 }
diff --git a/src/main/java/io/milvus/v2/utils/VectorUtils.java b/src/main/java/io/milvus/v2/utils/VectorUtils.java
@@ -20,6 +20,7 @@
 package io.milvus.v2.utils;
 
 import com.google.protobuf.ByteString;
+import com.sun.org.apache.xpath.internal.operations.Bool;
 import io.milvus.common.utils.GTsDict;
 import io.milvus.common.utils.JsonUtils;
 import io.milvus.v2.common.ConsistencyLevel;
@@ -44,6 +45,12 @@ public QueryRequest ConvertToGrpcQueryRequest(QueryReq request){
                 .addAllPartitionNames(request.getPartitionNames())
                 .addAllOutputFields(request.getOutputFields())
                 .setExpr(request.getFilter());
+        if (request.getFilter() != null && !request.getFilter().isEmpty()) {
+            Map<String, Object> filterTemplateValues = request.getFilterTemplateValues();
+            filterTemplateValues.forEach((key, value)->{
+                builder.putExprTemplateValues(key, deduceTemplateValue(value));
+            });
+        }
 
         // a new parameter from v2.2.9, if user didn't specify consistency level, set this parameter to true
         if (request.getConsistencyLevel() == null) {
@@ -180,6 +187,10 @@ public SearchRequest ConvertToGrpcSearchRequest(SearchReq request) {
         builder.setDslType(DslType.BoolExprV1);
         if (request.getFilter() != null && !request.getFilter().isEmpty()) {
             builder.setDsl(request.getFilter());
+            Map<String, Object> filterTemplateValues = request.getFilterTemplateValues();
+            filterTemplateValues.forEach((key, value)->{
+                builder.putExprTemplateValues(key, deduceTemplateValue(value));
+            });
         }
 
         long guaranteeTimestamp = getGuaranteeTimestamp(request.getConsistencyLevel(), request.getCollectionName());
@@ -195,6 +206,51 @@ public SearchRequest ConvertToGrpcSearchRequest(SearchReq request) {
         return builder.build();
     }
 
+    public static TemplateValue deduceTemplateValue(Object value) {
+        if (value instanceof Boolean) {
+            return TemplateValue.newBuilder()
+                    .setBoolVal((Boolean)value)
+                    .setType(DataType.Bool)
+                    .build();
+        } else if (value instanceof Long) {
+           return TemplateValue.newBuilder()
+                   .setInt64Val((Long)value)
+                   .setType(DataType.Int64)
+                   .build();
+        } else if (value instanceof Double) {
+            return TemplateValue.newBuilder()
+                    .setFloatVal((Double)value)
+                    .setType(DataType.Double)
+                    .build();
+        } else if (value instanceof String) {
+            return TemplateValue.newBuilder()
+                    .setStringVal((String)value)
+                    .setType(DataType.VarChar)
+                    .build();
+        } else if (value instanceof List) {
+            List<Object> array = (List<Object>)value;
+            TemplateArrayValue.Builder builder = TemplateArrayValue.newBuilder();
+            DataType lastType = DataType.UNRECOGNIZED;
+            boolean sameType = true;
+            for (Object obj : array) {
+                TemplateValue tv = deduceTemplateValue(obj);
+                builder.addArray(tv);
+                if (lastType == DataType.UNRECOGNIZED) {
+                    lastType = tv.getType();
+                } else if (lastType != tv.getType()) {
+                    sameType = false;
+                }
+            }
+            DataType arrayType = sameType ? lastType : DataType.JSON;
+            return TemplateValue.newBuilder()
+                    .setArrayVal(builder.build())
+                    .setType(arrayType)
+                    .build();
+        } else {
+            throw new ParamException("Unsupported value type for expression template.");
+        }
+    }
+
     public static SearchRequest convertAnnSearchParam(@NonNull AnnSearchReq annSearchReq,
                                                       ConsistencyLevel consistencyLevel) {
         SearchRequest.Builder builder = SearchRequest.newBuilder();

diff --git a/src/main/milvus-proto b/src/main/milvus-proto