From f50d94928d50dbf203e539df9c67566cb67aac28 Mon Sep 17 00:00:00 2001 From: CharlieYan <16526279+CharlieYan24@users.noreply.github.com> Date: Thu, 15 Jun 2023 23:34:09 +0800 Subject: [PATCH 1/3] fix: reuse engine failed #4645 --- .../linkis/manager/label/score/DefaultNodeLabelScorer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/DefaultNodeLabelScorer.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/DefaultNodeLabelScorer.java index 76218ee78d..80f4cf2e2c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/DefaultNodeLabelScorer.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/DefaultNodeLabelScorer.java @@ -82,7 +82,8 @@ public Integer apply(Feature t, Integer count) { ftCounts.compute(label.getFeature(), countFunction); return Pair.of(String.valueOf(label.getId()), label); }) - .collect(Collectors.toMap(Pair::getKey, Pair::getRight)); + .distinct() + .collect(Collectors.toMap(Pair::getKey, Pair::getRight, (a, b) -> a)); for (Map.Entry> entry : outNodeDegree.entrySet()) { ServiceInstance node = entry.getKey(); From 1977e50515770eede010b05c676fdc05b71abf61 Mon Sep 17 00:00:00 2001 From: CharlieYan <16526279+CharlieYan24@users.noreply.github.com> Date: Sun, 18 Jun 2023 23:24:46 +0800 Subject: [PATCH 2/3] [AONE-50093744] fix: deplicate key execption --- .../manager/am/manager/DefaultEngineNodeManager.java | 6 +++++- .../manager/am/service/em/DefaultEMInfoService.java | 4 +++- .../am/service/engine/DefaultEngineInfoService.java | 5 ++++- .../am/service/impl/ECResourceInfoServiceImpl.java | 6 +++++- .../manager/label/score/DefaultNodeLabelScorer.java | 7 ++++--- .../label/service/impl/DefaultNodeLabelService.java | 6 +++++- .../linkis/manager/rm/restful/RMMonitorRest.java | 12 ++++++++++-- .../impl/DefaultResourceLabelPersistence.java | 4 +++- 8 files changed, 39 insertions(+), 11 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java index 4a0eb457c1..049bf1cafb 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java @@ -89,7 +89,11 @@ public List listEngines(String user) { List nodeMetrics = nodeMetricManagerPersistence.getNodeMetrics(nodes); Map metricses = nodeMetrics.stream() - .collect(Collectors.toMap(m -> m.getServiceInstance().toString(), m -> m)); + .collect( + Collectors.toMap( + m -> m.getServiceInstance().toString(), + m -> m, + (existingValue, newValue) -> newValue)); nodes.forEach( node -> { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.java index 1f03d63609..800e155037 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.java @@ -88,7 +88,9 @@ public EMNode[] getAllEM() { Map resourceInfoMap = resourceInfo.getResourceInfo().stream() - .collect(Collectors.toMap(r -> r.getServiceInstance(), r -> r)); + .collect( + Collectors.toMap( + r -> r.getServiceInstance(), r -> r, (existingValue, newValue) -> newValue)); return instances.stream() .map(emNodeManager::getEM) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineInfoService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineInfoService.java index 6b021d76ae..5fb31615a3 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineInfoService.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineInfoService.java @@ -79,7 +79,10 @@ private List getEngineNodes(List nodes) { Map resourceInfoMap = rmNodes.stream() .collect( - Collectors.toMap(entry -> entry.getServiceInstance().toString(), entry -> entry)); + Collectors.toMap( + entry -> entry.getServiceInstance().toString(), + entry -> entry, + (existingValue, newValue) -> newValue)); nodes.forEach( node -> { RMNode rmNode = resourceInfoMap.get(node.getServiceInstance().toString()); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java index d44f0062a1..edb399d679 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java @@ -118,7 +118,11 @@ public List> getECResourceInfoList( // map k:v---> instanceName：PersistencerEcNodeInfo Map persistencerEcNodeInfoMap = ecNodesInfo.stream() - .collect(Collectors.toMap(PersistencerEcNodeInfo::getInstance, item -> item)); + .collect( + Collectors.toMap( + PersistencerEcNodeInfo::getInstance, + item -> item, + (existingValue, newValue) -> newValue)); List instanceList = ecNodesInfo.stream().map(e -> e.getInstance()).collect(Collectors.toList()); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/DefaultNodeLabelScorer.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/DefaultNodeLabelScorer.java index 80f4cf2e2c..b085a50ba0 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/DefaultNodeLabelScorer.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/DefaultNodeLabelScorer.java @@ -82,8 +82,7 @@ public Integer apply(Feature t, Integer count) { ftCounts.compute(label.getFeature(), countFunction); return Pair.of(String.valueOf(label.getId()), label); }) - .distinct() - .collect(Collectors.toMap(Pair::getKey, Pair::getRight, (a, b) -> a)); + .collect(Collectors.toMap(Pair::getKey, Pair::getRight); for (Map.Entry> entry : outNodeDegree.entrySet()) { ServiceInstance node = entry.getKey(); @@ -191,7 +190,9 @@ public static Map> normalizeAndOutp labelScoreServiceInstance.setScore(scoreCalculate); return Pair.of(labelScoreServiceInstance, outNodeDegree.get(node)); }) - .collect(Collectors.toMap(Pair::getKey, Pair::getValue)); + .collect( + Collectors.toMap( + Pair::getKey, Pair::getValue, (existingValue, newValue) -> newValue)); rawOutput .keySet() diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.java index f3ef3f32bb..7cbe9eec71 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.java @@ -207,7 +207,11 @@ public void removeLabelsFromNode(ServiceInstance instance, List> labels List labelList = labelManagerPersistence.getLabelByServiceInstance(instance); Map dbLabels = labelList.stream() - .collect(Collectors.toMap(PersistenceLabel::getLabelKey, Function.identity())); + .collect( + Collectors.toMap( + PersistenceLabel::getLabelKey, + Function.identity(), + (existingValue, newValue) -> newValue)); List labelIds = labels.stream() diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/restful/RMMonitorRest.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/restful/RMMonitorRest.java index b685a59493..324995b391 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/restful/RMMonitorRest.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/restful/RMMonitorRest.java @@ -736,7 +736,11 @@ private EngineNode[] getEngineNodes(String user, boolean withResource) { List nodeMetrics = nodeMetricManagerPersistence.getNodeMetrics(nodes); Map metrics = nodeMetrics.stream() - .collect(Collectors.toMap(m -> m.getServiceInstance().toString(), m -> m)); + .collect( + Collectors.toMap( + m -> m.getServiceInstance().toString(), + m -> m, + (existingValue, newValue) -> newValue)); Map configurationMap = new HashMap<>(); @@ -825,7 +829,11 @@ public Map> getEngineNodesByUserList( Map metrics = nodeMetricManagerPersistence.getNodeMetrics(engineNodesList).stream() - .collect(Collectors.toMap(m -> m.getServiceInstance().toString(), Function.identity())); + .collect( + Collectors.toMap( + m -> m.getServiceInstance().toString(), + Function.identity(), + (existingValue, newValue) -> newValue)); Map>> labelsMap = nodeLabelService.getNodeLabelsByInstanceList( diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultResourceLabelPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultResourceLabelPersistence.java index 5a010372eb..abcdf6519f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultResourceLabelPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultResourceLabelPersistence.java @@ -180,7 +180,9 @@ public void removeResourceByLabels(List labels) { Map> keyValueMaps = blankIds.stream() .map(PersistenceUtils::entryToTunple) - .collect(Collectors.toMap(Tunple::getKey, Tunple::getValue)); + .collect( + Collectors.toMap( + Tunple::getKey, Tunple::getValue, (existingValue, newValue) -> newValue)); // labelManagerMapper.batchDeleteResourceByLabelKeyValuesMaps(keyValueMaps); } } From 9e7492c4043e0e6f286fc7bf615a5613433176b0 Mon Sep 17 00:00:00 2001 From: CharlieYan <16526279+CharlieYan24@users.noreply.github.com> Date: Sun, 18 Jun 2023 23:28:32 +0800 Subject: [PATCH 3/3] fix: queresource search failed 4660 --- .../linkis/manager/label/score/DefaultNodeLabelScorer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/DefaultNodeLabelScorer.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/DefaultNodeLabelScorer.java index b085a50ba0..4a5657b800 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/DefaultNodeLabelScorer.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/DefaultNodeLabelScorer.java @@ -82,7 +82,7 @@ public Integer apply(Feature t, Integer count) { ftCounts.compute(label.getFeature(), countFunction); return Pair.of(String.valueOf(label.getId()), label); }) - .collect(Collectors.toMap(Pair::getKey, Pair::getRight); + .collect(Collectors.toMap(Pair::getKey, Pair::getRight)); for (Map.Entry> entry : outNodeDegree.entrySet()) { ServiceInstance node = entry.getKey();