Optimize (#231)

* fix gatewat swagger * fix gateway * add log stream * update log stream * update log stream * update log stream * update log stream * update log stream * update log stream * update log stream * update log stream * fix ftl * fix label_field * fix mode * update log stream * update log stream * Optimize (#236) * add model log * rm model log * fix template --------- Co-authored-by: rk2b-fenghaolun <[email protected]>
primihub · Jan 10, 2024 · 56f854e · 56f854e
1 parent 8470ecd
commit 56f854e
Show file tree

Hide file tree

Showing 25 changed files with 244 additions and 107 deletions.
diff --git a/primihub-sdk/src/main/resources/templates/hetero_fitTransform.ftl b/primihub-sdk/src/main/resources/templates/hetero_fitTransform.ftl
@@ -15,7 +15,7 @@
   "role_params": {
     "Bob": {
       "data_set": "${label_dataset}",
-      "selected_column": null,
+      "selected_column": ${label_field0},
       "id": "id",
       "label": "y",
       "preprocess_column": null,
@@ -45,7 +45,7 @@
     },
     "Charlie": {
       "data_set": "${guest_dataset}",
-      "selected_column": null,
+      "selected_column": ${label_field1},
       "id": "id",
       "preprocess_column": null,
       "preprocess_dataset_id": "${new_guest_dataset}",

diff --git a/primihub-sdk/src/main/resources/templates/hetero_lr.ftl b/primihub-sdk/src/main/resources/templates/hetero_lr.ftl
@@ -11,6 +11,7 @@
 		"method": "${encryption!"Plaintext"}",
 		"process": "train",
 		"task_name": "VFL_logistic_regression_plaintext_train",
+        "psi": null,
 		"learning_rate": ${learningRate!1},
 		"alpha": ${alpha!0.0001},
 		"epoch": ${epoch!10},
@@ -21,15 +22,15 @@
 	"role_params": {
 		"Bob": {
 			"data_set": "${label_dataset}",
-			"selected_column": null,
+			"selected_column": ${label_field0},
 			"id": "id",
 			"label": "y",
 			"model_path": "${hostModelFileName}",
 			"metric_path": "${indicatorFileName}"
 		},
 		"Charlie": {
 			"data_set": "${guest_dataset}",
-			"selected_column": null,
+			"selected_column": ${label_field1},
 			"id": "id",
 			"model_path": "${guestModelFileName}"
 		},

diff --git a/primihub-sdk/src/main/resources/templates/hetero_lr_infer.ftl b/primihub-sdk/src/main/resources/templates/hetero_lr_infer.ftl
@@ -8,6 +8,7 @@
 	"common_params": {
 		"model": "${model}",
 		"process": "predict",
+        "psi": null,
 		"task_name": "VFL_logistic_regression_predict"
 	},
 	"role_params": {

diff --git a/primihub-sdk/src/main/resources/templates/hetero_xgb.ftl b/primihub-sdk/src/main/resources/templates/hetero_xgb.ftl
@@ -3,13 +3,12 @@
         "guest": [
             "Charlie"
         ],
-        "host": [
-            "Bob"
-        ]
+        "host": "Bob"
     },
     "common_params": {
         "model": "HeteroXGB",
         "task_name": "train",
+        "psi": null,
         "num_tree": ${numTree!5},
         "max_depth": ${maxDepth!5},
         "reg_lambda": ${regLambda!1},
@@ -36,7 +35,7 @@
             "data_set": "${label_dataset}",
             "id": "id",
             "secure_bits": 112,
-            "selected_column": null,
+            "selected_column": ${label_field0},
             "objective": "linear",
             "base_score": 0.5,
             "amplify_ratio": 8,
@@ -48,7 +47,7 @@
             "data_set": "${guest_dataset}",
             "id": "id",
             "model_path": "${guestModelFileName}",
-            "selected_column": null,
+            "selected_column": ${label_field1},
             "lookup_table": "${guestLookupTable}",
             "label": null,
             "batch_size": 5

diff --git a/primihub-sdk/src/main/resources/templates/hetero_xgb_infer.ftl b/primihub-sdk/src/main/resources/templates/hetero_xgb_infer.ftl
@@ -3,21 +3,20 @@
 		"guest": [
 			"Charlie"
 		],
-		"host": [
-			"Bob"
-		]
+		"host": "Bob"
 	},
 	"common_params": {
 		"model": "HeteroXGBInfer",
 		"task_name": "predict",
+        "psi": null,
 		"metric_path": "${indicatorFileName}",
 		"model_pred": "${predictFileName}"
 	},
 	"role_params": {
 		"Bob": {
 			"data_set": "${label_dataset}",
 			"id": "id",
-			"selected_column": null,
+			"selected_column": ${label_field0},
 			"label": "y",
 			"lookup_table": "${hostLookupTable}",
 			"model_path": "${hostModelFileName}"
@@ -26,7 +25,7 @@
 			"data_set": "${guest_dataset}",
 			"id": "id",
 			"model_path": "${guestModelFileName}",
-			"selected_column": null,
+			"selected_column": ${label_field1},
 			"lookup_table": "${guestLookupTable}",
 			"label": null
 		}

diff --git a/primihub-sdk/src/main/resources/templates/homo_fitTransform.ftl b/primihub-sdk/src/main/resources/templates/homo_fitTransform.ftl
@@ -12,7 +12,7 @@
         "FL_type": "H",
         "task_name": "HFL_simpleimpute_fit_transform",
         "task": "classification",
-        "selected_column": null,
+        "selected_column": ${label_field0},
         "id": "id",
         "label": "y",
         "preprocess_column": null,

diff --git a/primihub-sdk/src/main/resources/templates/homo_lr.ftl b/primihub-sdk/src/main/resources/templates/homo_lr.ftl
@@ -21,7 +21,7 @@
 		"batch_size": ${batchSize!100},
 		"global_epoch": ${globalEpoch!100},
 		"local_epoch": ${localEpoch!1},
-		"selected_column": null,
+		"selected_column": ${label_field0},
 		"id": "${id!"id"}",
 		"label": "${label!"y"}",
 		"print_metrics": ${printMetrics!false?c}

diff --git a/primihub-sdk/src/main/resources/templates/homo_nn_binary.ftl b/primihub-sdk/src/main/resources/templates/homo_nn_binary.ftl
@@ -21,7 +21,7 @@
 	  "batch_size": ${batchSize!100},
 	  "global_epoch": ${globalEpoch!100},
 	  "local_epoch": ${localEpoch!1},
-	  "selected_column": null,
+	  "selected_column": ${label_field0},
 	  "id": "${id!"id"}",
 	  "label": "${label!"y"}",
 	  "print_metrics": ${printMetrics!false?c}

diff --git a/...ce/application/src/main/java/com/primihub/application/controller/data/TaskController.java b/...ce/application/src/main/java/com/primihub/application/controller/data/TaskController.java
@@ -95,28 +95,6 @@ public BaseResultEntity cancelPsiTask(String taskId){
         return  dataTaskService.cancelTask(taskId);
     }
 
-    /**
-     * 用于创建连接
-     */
-    @GetMapping(value = "sseConnect/{taskId}",produces = MediaType.TEXT_EVENT_STREAM_VALUE)
-    public SseEmitter connect(@PathVariable String taskId,Integer all) {
-        if (all==null) {
-            all = 0;
-        }
-        return dataTaskService.connectSseTask(taskId,all);
-    }
-
-    /**
-     * 关闭连接
-     */
-    @GetMapping("sseClose/{taskId}")
-    public BaseResultEntity close(@PathVariable("taskId") String taskId) {
-        if (StringUtils.isBlank(taskId)) {
-            return BaseResultEntity.failure(BaseResultEnum.LACK_OF_PARAM,"taskId");
-        }
-        dataTaskService.removeSseTask(taskId);
-        return BaseResultEntity.success();
-    }
 
     @GetMapping("getTaskLogInfo")
     public BaseResultEntity getTaskLogInfo(Long taskId){

diff --git a/...lication/src/main/java/com/primihub/application/controller/share/ShareDataController.java b/...lication/src/main/java/com/primihub/application/controller/share/ShareDataController.java
@@ -15,13 +15,12 @@
 import com.primihub.biz.service.sys.SysOrganService;
 import com.primihub.biz.service.test.TestService;
 import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.MediaType;
 import org.springframework.util.StringUtils;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.bind.annotation.*;
 
 import java.util.List;
 import java.util.Map;
@@ -45,6 +44,7 @@ public class ShareDataController {
     @Autowired
     private ShareService shareService;
 
+    @ApiOperation(value = "通信检测",httpMethod = "POST",consumes = MediaType.APPLICATION_JSON_VALUE)
     @PostMapping("/healthConnection")
     public BaseResultEntity healthConnection(@RequestBody Object time){
         log.info("healthConnection - {}",time);
@@ -114,6 +114,15 @@ public BaseResultEntity batchSaveTestDataSet(@RequestBody List<DataSet> dataSets
         return testService.batchSaveTestDataSet(dataSets);
     }
 
+    @ApiOperation(value = "网关通信检测",httpMethod = "POST",consumes = MediaType.APPLICATION_JSON_VALUE)
+    @PostMapping("verifyGateway")
+    public BaseResultEntity verifyGatewayConnection(@RequestBody String uniqueIdentification){
+        if (org.apache.commons.lang3.StringUtils.isBlank(uniqueIdentification)){
+            return BaseResultEntity.failure(BaseResultEnum.LACK_OF_PARAM,"uniqueIdentification");
+        }
+        return sysOrganService.verifyGatewayConnection(uniqueIdentification);
+    }
+
 
 
 }
diff --git a/...pplication/src/main/java/com/primihub/application/controller/stream/StreamController.java b/...pplication/src/main/java/com/primihub/application/controller/stream/StreamController.java
@@ -0,0 +1,54 @@
+package com.primihub.application.controller.stream;
+
+import com.primihub.biz.entity.base.BaseResultEntity;
+import com.primihub.biz.entity.base.BaseResultEnum;
+import com.primihub.biz.service.data.DataTaskService;
+import io.swagger.annotations.Api;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
+
+
+
+@Api(value = "流传输接口",tags = "流传输接口")
+@RequestMapping("stream")
+@RestController
+@Slf4j
+public class StreamController {
+
+    @Autowired
+    private DataTaskService dataTaskService;
+
+    /**
+     * 用于创建连接
+     */
+    @GetMapping(value = "sseTaskConnect/{taskId}",produces = MediaType.TEXT_EVENT_STREAM_VALUE)
+    public SseEmitter connect(@PathVariable String taskId) {
+//        return dataTaskService.connectSseTask1(taskId);
+        return dataTaskService.connectSseTask(taskId);
+    }
+
+//    @GetMapping(value = "send/{taskId}")
+//    public String send(@PathVariable String taskId){
+//        dataTaskService.send(taskId);
+//        return "su";
+//    }
+
+    /**
+     * 关闭连接
+     */
+    @GetMapping("sseTaskClose/{taskId}")
+    public BaseResultEntity close(@PathVariable("taskId") String taskId) {
+        if (StringUtils.isBlank(taskId)) {
+            return BaseResultEntity.failure(BaseResultEnum.LACK_OF_PARAM,"taskId");
+        }
+        dataTaskService.removeSseTask(taskId);
+        return BaseResultEntity.success();
+    }
+}
diff --git a/primihub-service/biz/src/main/java/com/primihub/biz/constant/DataConstant.java b/primihub-service/biz/src/main/java/com/primihub/biz/constant/DataConstant.java
@@ -16,6 +16,7 @@ public class DataConstant {
     public final static String FREEMARKER_PYTHON_HETERO_LR_INFER_PATH= "hetero_lr_infer.ftl";
     public final static String FREEMARKER_PYTHON_HOMO_NN_BINARY_PATH= "homo_nn_binary.ftl";
     public final static String FREEMARKER_PYTHON_HOMO_NN_BINARY_INFER_PATH= "homo_nn_binary_infer.ftl";
+    public final static String FREEMARKER_PYTHON_HETERO_FITTRANSFORM_PATH= "hetero_fitTransform.ftl";
 
     public final static String FREEMARKER_PYTHON_EXCEPTION_PATH= "exception.ftl";
     public final static String FREEMARKER_PYTHON_DATA_ALIGN_PATH= "data_align.ftl";
@@ -30,6 +31,7 @@ public class DataConstant {
             FREEMARKER_PYTHON_HOMO_NN_BINARY_INFER_PATH,
             FREEMARKER_PYTHON_EXCEPTION_PATH,
             FREEMARKER_PYTHON_DATA_ALIGN_PATH,
+            FREEMARKER_PYTHON_HETERO_FITTRANSFORM_PATH
     };
     // python dataset host、guest
     public final static String PYTHON_LABEL_DATASET = "label_dataset";

diff --git a/primihub-service/biz/src/main/java/com/primihub/biz/constant/RedisKeyConstant.java b/primihub-service/biz/src/main/java/com/primihub/biz/constant/RedisKeyConstant.java
@@ -17,4 +17,6 @@ public class RedisKeyConstant {
 
     public final static String PIR_TASK_DATA_SET = "pir:t:d:s:<taskId>";
     public final static String PIR_TASK_DATA_LIST = "pir:t:d:l:<taskId>";
+
+    public final static String ORGAN_VERIFY_GATEWAY_UUID = "organ_verify_gateway_uuid";
 }
diff --git a/primihub-service/biz/src/main/java/com/primihub/biz/entity/sys/config/LokiConfig.java b/primihub-service/biz/src/main/java/com/primihub/biz/entity/sys/config/LokiConfig.java
@@ -7,4 +7,6 @@ public class LokiConfig {
     private String address;
     private String job;
     private String container;
+    private String namespace;
+    private String app;
 }
diff --git a/...in/java/com/primihub/biz/repository/primaryredis/sys/SysCommonPrimaryRedisRepository.java b/...in/java/com/primihub/biz/repository/primaryredis/sys/SysCommonPrimaryRedisRepository.java
@@ -52,4 +52,8 @@ public String getKey(String key){
         return primaryStringRedisTemplate.opsForValue().get(key);
     }
 
+    public void setValue(String key,String value){
+        primaryStringRedisTemplate.opsForValue().set(key,value);
+    }
+
 }