From c2f36077757a640c1c1d80617e88d524de6fb0af Mon Sep 17 00:00:00 2001 From: rk2b-fenghaolun <1120560948@qq.com> Date: Mon, 8 Apr 2024 10:27:34 +0800 Subject: [PATCH 1/4] fix FitTransform null error --- .../impl/FitTransformComponentTaskServiceImpl.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/primihub-service/biz/src/main/java/com/primihub/biz/service/data/component/impl/FitTransformComponentTaskServiceImpl.java b/primihub-service/biz/src/main/java/com/primihub/biz/service/data/component/impl/FitTransformComponentTaskServiceImpl.java index 803437e25..9435194b9 100644 --- a/primihub-service/biz/src/main/java/com/primihub/biz/service/data/component/impl/FitTransformComponentTaskServiceImpl.java +++ b/primihub-service/biz/src/main/java/com/primihub/biz/service/data/component/impl/FitTransformComponentTaskServiceImpl.java @@ -1,5 +1,6 @@ package com.primihub.biz.service.data.component.impl; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.primihub.biz.config.base.BaseConfiguration; import com.primihub.biz.config.base.ComponentsConfiguration; @@ -55,10 +56,12 @@ public BaseResultEntity runTask(DataComponentReq req, ComponentTaskReq taskReq) List ids = taskReq.getFusionResourceList().stream().map(data -> data.get("resourceId").toString()).collect(Collectors.toList()); List newest = taskReq.getNewest(); log.info("ids:{}", ids); + log.info("newest:{}", JSON.toJSONString(newest)); String path = baseConfiguration.getRunModelFileUrlDirPrefix()+taskReq.getDataTask().getTaskIdName() + File.separator + "fitTransform"; Map fitTransformEntityMap = getGrpcComponentDataSetMap(taskReq.getFusionResourceList(),path); fitTransformEntityMap.remove(taskReq.getFreemarkerMap().get(DataConstant.PYTHON_ARBITER_DATASET)); log.info("fitTransform-1:{}", JSONObject.toJSONString(fitTransformEntityMap)); + Map oldResourceIdMap = new HashMap<>(); if (newest!=null && newest.size()!=0){ ids = new ArrayList<>(); for (ModelDerivationDto modelDerivationDto : newest) { @@ -68,17 +71,17 @@ public BaseResultEntity runTask(DataComponentReq req, ComponentTaskReq taskReq) } log.info("newids:{}", ids); log.info("fitTransform-2:{}", JSONObject.toJSONString(fitTransformEntityMap)); + oldResourceIdMap = newest.stream().collect(Collectors.toMap(ModelDerivationDto::getOriginalResourceId, Function.identity())); } - Map oldResourceIdMap = newest.stream().collect(Collectors.toMap(ModelDerivationDto::getOriginalResourceId, Function.identity())); try { GrpcComponentDto labelDatasetDto = fitTransformEntityMap.get(taskReq.getFreemarkerMap().get(DataConstant.PYTHON_LABEL_DATASET)); - if (labelDatasetDto == null) { + if (labelDatasetDto == null && !oldResourceIdMap.isEmpty()) { labelDatasetDto = fitTransformEntityMap.get(oldResourceIdMap.get(taskReq.getFreemarkerMap().get(DataConstant.PYTHON_LABEL_DATASET)).getNewResourceId()); } taskReq.getFreemarkerMap().put("new_"+DataConstant.PYTHON_LABEL_DATASET,labelDatasetDto.getNewDataSetId()); taskReq.getFreemarkerMap().put("new_"+DataConstant.PYTHON_LABEL_DATASET+"_path",labelDatasetDto.getOutputFilePath()); GrpcComponentDto guestDatasetDto = fitTransformEntityMap.get(taskReq.getFreemarkerMap().get(DataConstant.PYTHON_GUEST_DATASET)); - if (guestDatasetDto == null) { + if (guestDatasetDto == null && !oldResourceIdMap.isEmpty()) { guestDatasetDto = fitTransformEntityMap.get(oldResourceIdMap.get(taskReq.getFreemarkerMap().get(DataConstant.PYTHON_GUEST_DATASET)).getNewResourceId()); } taskReq.getFreemarkerMap().put("new_"+DataConstant.PYTHON_GUEST_DATASET,guestDatasetDto.getNewDataSetId()); From b40aeef7634b02f087a33f798f4f43449746443a Mon Sep 17 00:00:00 2001 From: rk2b-fenghaolun <1120560948@qq.com> Date: Mon, 8 Apr 2024 13:22:28 +0800 Subject: [PATCH 2/4] fix FitTransform --- .../component/impl/FitTransformComponentTaskServiceImpl.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/primihub-service/biz/src/main/java/com/primihub/biz/service/data/component/impl/FitTransformComponentTaskServiceImpl.java b/primihub-service/biz/src/main/java/com/primihub/biz/service/data/component/impl/FitTransformComponentTaskServiceImpl.java index 9435194b9..25d212e34 100644 --- a/primihub-service/biz/src/main/java/com/primihub/biz/service/data/component/impl/FitTransformComponentTaskServiceImpl.java +++ b/primihub-service/biz/src/main/java/com/primihub/biz/service/data/component/impl/FitTransformComponentTaskServiceImpl.java @@ -73,13 +73,14 @@ public BaseResultEntity runTask(DataComponentReq req, ComponentTaskReq taskReq) log.info("fitTransform-2:{}", JSONObject.toJSONString(fitTransformEntityMap)); oldResourceIdMap = newest.stream().collect(Collectors.toMap(ModelDerivationDto::getOriginalResourceId, Function.identity())); } + log.info("freemarkerMap", JSON.toJSONString(taskReq.getFreemarkerMap())); try { GrpcComponentDto labelDatasetDto = fitTransformEntityMap.get(taskReq.getFreemarkerMap().get(DataConstant.PYTHON_LABEL_DATASET)); if (labelDatasetDto == null && !oldResourceIdMap.isEmpty()) { labelDatasetDto = fitTransformEntityMap.get(oldResourceIdMap.get(taskReq.getFreemarkerMap().get(DataConstant.PYTHON_LABEL_DATASET)).getNewResourceId()); } - taskReq.getFreemarkerMap().put("new_"+DataConstant.PYTHON_LABEL_DATASET,labelDatasetDto.getNewDataSetId()); - taskReq.getFreemarkerMap().put("new_"+DataConstant.PYTHON_LABEL_DATASET+"_path",labelDatasetDto.getOutputFilePath()); + taskReq.getFreemarkerMap().put("new_"+DataConstant.PYTHON_LABEL_DATASET, labelDatasetDto.getNewDataSetId()); + taskReq.getFreemarkerMap().put("new_"+DataConstant.PYTHON_LABEL_DATASET+"_path", labelDatasetDto.getOutputFilePath()); GrpcComponentDto guestDatasetDto = fitTransformEntityMap.get(taskReq.getFreemarkerMap().get(DataConstant.PYTHON_GUEST_DATASET)); if (guestDatasetDto == null && !oldResourceIdMap.isEmpty()) { guestDatasetDto = fitTransformEntityMap.get(oldResourceIdMap.get(taskReq.getFreemarkerMap().get(DataConstant.PYTHON_GUEST_DATASET)).getNewResourceId()); From 45831c5bbadf5c65af99723d175c74801b36c351 Mon Sep 17 00:00:00 2001 From: rk2b-fenghaolun <1120560948@qq.com> Date: Mon, 8 Apr 2024 13:37:34 +0800 Subject: [PATCH 3/4] fix component execute order --- .../biz/service/data/DataAsyncService.java | 3 +- .../biz/service/data/DataModelService.java | 40 +++++++++++++++++-- .../FitTransformComponentTaskServiceImpl.java | 2 +- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/primihub-service/biz/src/main/java/com/primihub/biz/service/data/DataAsyncService.java b/primihub-service/biz/src/main/java/com/primihub/biz/service/data/DataAsyncService.java index 57002f062..864cf9108 100644 --- a/primihub-service/biz/src/main/java/com/primihub/biz/service/data/DataAsyncService.java +++ b/primihub-service/biz/src/main/java/com/primihub/biz/service/data/DataAsyncService.java @@ -171,10 +171,11 @@ public void runModelTask(ComponentTaskReq req) { dataComponent.setComponentState(2); req.getDataModelTask().setComponentJson(JSONObject.toJSONString(req.getDataComponents())); dataModelPrRepository.updateDataModelTask(req.getDataModelTask()); - dataComponent.setComponentState(1); executeBeanMethod(false, dataComponentReqMap.get(dataComponent.getComponentCode()), req); if (req.getDataTask().getTaskState().equals(TaskStateEnum.FAIL.getStateType())) { dataComponent.setComponentState(3); + } else { + dataComponent.setComponentState(1); } dataComponent.setEndTime(System.currentTimeMillis()); req.getDataModelTask().setComponentJson(JSONObject.toJSONString(req.getDataComponents())); diff --git a/primihub-service/biz/src/main/java/com/primihub/biz/service/data/DataModelService.java b/primihub-service/biz/src/main/java/com/primihub/biz/service/data/DataModelService.java index a94aa0b1c..1154b7e39 100644 --- a/primihub-service/biz/src/main/java/com/primihub/biz/service/data/DataModelService.java +++ b/primihub-service/biz/src/main/java/com/primihub/biz/service/data/DataModelService.java @@ -381,21 +381,43 @@ public BaseResultEntity runTaskModel(Long modelId,Long userId) { } ComponentTaskReq taskReq = new ComponentTaskReq(dataModel); DataModelAndComponentReq modelComponentReq = taskReq.getModelComponentReq(); + List modelComponents = modelComponentReq.getModelComponents(); + List zComponentCodeList= new ArrayList<>(); + Map modelComponentMap = new HashMap<>(); + if (modelComponents != null && !modelComponents.isEmpty()) { + modelComponentMap = modelComponents.stream().collect(Collectors.toMap(DataComponentReq::getComponentCode, Function.identity())); + // 将组件中的`componentCode`按照组件顺序进行排序 + List zComponentList = modelComponents.stream().filter(dataComponentReq -> dataComponentReq.getInput().isEmpty()).collect(Collectors.toList()); + zComponentCodeList= new ArrayList<>(); + if (zComponentList.isEmpty()) { + return BaseResultEntity.failure(BaseResultEnum.DATA_RUN_TASK_FAIL,"未查询到模型组件信息"); + } + // 开头的code + zComponentCodeList.add(zComponentList.get(0).getComponentCode()); + + // 递归方法，排序componentCode + sortComponent(zComponentList.get(0).getOutput(), zComponentCodeList, modelComponentMap); + log.info("sort后的组件顺序:{}", zComponentCodeList); + } else { + return BaseResultEntity.failure(BaseResultEnum.DATA_RUN_TASK_FAIL,"未查询到模型组件信息"); + } if (modelComponentReq==null) { + log.info("{}-{}", BaseResultEnum.DATA_RUN_TASK_FAIL.getMessage(), "组件解析失败"); return BaseResultEntity.failure(BaseResultEnum.DATA_RUN_TASK_FAIL,"组件解析失败"); } - Set mandatorySet = componentsConfiguration.getModelComponents().stream() + /*Set mandatorySet = componentsConfiguration.getModelComponents().stream() .filter(modelComponent -> modelComponent.getIsMandatory() == 0) .map(ModelComponent::getComponentCode) - .collect(Collectors.toSet()); + .collect(Collectors.toSet());*/ // Set reqSet = modelComponentReq.getModelComponents().stream().map(DataComponentReq::getComponentCode).collect(Collectors.toSet()); // mandatorySet.removeAll(reqSet); // if (!mandatorySet.isEmpty()) { // return BaseResultEntity.failure(BaseResultEnum.DATA_RUN_TASK_FAIL,"缺少必选组件,请检查组件"); // } - for (DataComponentReq modelComponent : modelComponentReq.getModelComponents()) { - BaseResultEntity baseResultEntity = dataAsyncService.executeBeanMethod(true, modelComponent, taskReq); + for (String componentCode : zComponentCodeList) { + BaseResultEntity baseResultEntity = dataAsyncService.executeBeanMethod(true, modelComponentMap.get(componentCode), taskReq); if (baseResultEntity.getCode()!=0){ + log.error("{}-[{}]", "组件执行结果失败", JSONObject.toJSONString(baseResultEntity)); return baseResultEntity; } } @@ -424,6 +446,16 @@ public BaseResultEntity runTaskModel(Long modelId,Long userId) { return BaseResultEntity.success(returnMap); } + private void sortComponent(List dataComponentRelationReqs, List zComponentCodeList, Map componentMap) { + if (dataComponentRelationReqs.isEmpty()) { + return; + } + DataComponentRelationReq dataComponentRelationReq = dataComponentRelationReqs.get(0); + zComponentCodeList.add(dataComponentRelationReq.getComponentCode()); + DataComponentReq dataComponentReq = componentMap.get(dataComponentRelationReq.getComponentCode()); + sortComponent(dataComponentReq.getOutput(), zComponentCodeList, componentMap); + } + public BaseResultEntity restartTaskModel(Long taskId) { DataTask dataTask = dataTaskRepository.selectDataTaskByTaskId(taskId); if (dataTask==null) { diff --git a/primihub-service/biz/src/main/java/com/primihub/biz/service/data/component/impl/FitTransformComponentTaskServiceImpl.java b/primihub-service/biz/src/main/java/com/primihub/biz/service/data/component/impl/FitTransformComponentTaskServiceImpl.java index 25d212e34..3fc5876a9 100644 --- a/primihub-service/biz/src/main/java/com/primihub/biz/service/data/component/impl/FitTransformComponentTaskServiceImpl.java +++ b/primihub-service/biz/src/main/java/com/primihub/biz/service/data/component/impl/FitTransformComponentTaskServiceImpl.java @@ -73,7 +73,7 @@ public BaseResultEntity runTask(DataComponentReq req, ComponentTaskReq taskReq) log.info("fitTransform-2:{}", JSONObject.toJSONString(fitTransformEntityMap)); oldResourceIdMap = newest.stream().collect(Collectors.toMap(ModelDerivationDto::getOriginalResourceId, Function.identity())); } - log.info("freemarkerMap", JSON.toJSONString(taskReq.getFreemarkerMap())); + log.info("freemarkerMap:{}", JSON.toJSONString(taskReq.getFreemarkerMap())); try { GrpcComponentDto labelDatasetDto = fitTransformEntityMap.get(taskReq.getFreemarkerMap().get(DataConstant.PYTHON_LABEL_DATASET)); if (labelDatasetDto == null && !oldResourceIdMap.isEmpty()) { From 511f921152d9387edaca5140bb5938be88fd1473 Mon Sep 17 00:00:00 2001 From: rk2b-fenghaolun <1120560948@qq.com> Date: Mon, 8 Apr 2024 14:04:45 +0800 Subject: [PATCH 4/4] fix FitTransform task --- .../component/impl/FitTransformComponentTaskServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/primihub-service/biz/src/main/java/com/primihub/biz/service/data/component/impl/FitTransformComponentTaskServiceImpl.java b/primihub-service/biz/src/main/java/com/primihub/biz/service/data/component/impl/FitTransformComponentTaskServiceImpl.java index 3fc5876a9..108e1d956 100644 --- a/primihub-service/biz/src/main/java/com/primihub/biz/service/data/component/impl/FitTransformComponentTaskServiceImpl.java +++ b/primihub-service/biz/src/main/java/com/primihub/biz/service/data/component/impl/FitTransformComponentTaskServiceImpl.java @@ -76,13 +76,13 @@ public BaseResultEntity runTask(DataComponentReq req, ComponentTaskReq taskReq) log.info("freemarkerMap:{}", JSON.toJSONString(taskReq.getFreemarkerMap())); try { GrpcComponentDto labelDatasetDto = fitTransformEntityMap.get(taskReq.getFreemarkerMap().get(DataConstant.PYTHON_LABEL_DATASET)); - if (labelDatasetDto == null && !oldResourceIdMap.isEmpty()) { + if (labelDatasetDto == null && oldResourceIdMap.size() > 0) { labelDatasetDto = fitTransformEntityMap.get(oldResourceIdMap.get(taskReq.getFreemarkerMap().get(DataConstant.PYTHON_LABEL_DATASET)).getNewResourceId()); } taskReq.getFreemarkerMap().put("new_"+DataConstant.PYTHON_LABEL_DATASET, labelDatasetDto.getNewDataSetId()); taskReq.getFreemarkerMap().put("new_"+DataConstant.PYTHON_LABEL_DATASET+"_path", labelDatasetDto.getOutputFilePath()); GrpcComponentDto guestDatasetDto = fitTransformEntityMap.get(taskReq.getFreemarkerMap().get(DataConstant.PYTHON_GUEST_DATASET)); - if (guestDatasetDto == null && !oldResourceIdMap.isEmpty()) { + if (guestDatasetDto == null && oldResourceIdMap.size() > 0) { guestDatasetDto = fitTransformEntityMap.get(oldResourceIdMap.get(taskReq.getFreemarkerMap().get(DataConstant.PYTHON_GUEST_DATASET)).getNewResourceId()); } taskReq.getFreemarkerMap().put("new_"+DataConstant.PYTHON_GUEST_DATASET,guestDatasetDto.getNewDataSetId());