feat: 定时任务兼容不合法主机 #3305

src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/CollectionUtil.java

@@ -116,4 +116,26 @@ public static <E, K, V> Map<K, V> convertToMap(List<E> entityCollection,
         }
         return map;
     }
+
+    public static <T> ArrayList<T> mergeToArrayList(List<T> list1, List<T> list2) {
+        ArrayList<T> mergeList;
+
+        boolean isList1NotEmpty = CollectionUtils.isNotEmpty(list1);
+        boolean isList2NotEmpty = CollectionUtils.isNotEmpty(list2);
+
+        if (isList1NotEmpty && isList2NotEmpty) {
+            mergeList = new ArrayList<>(list1.size() + list2.size());
+            mergeList.addAll(list1);
+            mergeList.addAll(list2);
+            return mergeList;
+        } else if (isList1NotEmpty) {
+            mergeList = new ArrayList<>(list1);
+            return mergeList;
+        } else if (isList2NotEmpty) {
+            mergeList = new ArrayList<>(list2);
+            return mergeList;
+        } else {
+            return new ArrayList<>(0);
+        }
+    }
 }

src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/HostCompositeKey.java

@@ -0,0 +1,107 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available.
+ *
+ * Copyright (C) 2021 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License.
+ *
+ * License for BK-JOB蓝鲸智云作业平台:
+ * --------------------------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+ * to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+ * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+package com.tencent.bk.job.common.model;
+
+import com.tencent.bk.job.common.model.dto.HostDTO;
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.Objects;
+
+/**
+ * 主机复合 KEY，用于主机的多种表达方式
+ */
+@Getter
+@Setter
+public class HostCompositeKey {
+    /**
+     * Key 类型
+     */
+    private final HostCompositeKeyType keyType;
+
+    /**
+     * 主机唯一 key（目前支持 hostId/cloudIp 两种）
+     */
+    private final String key;
+
+    public HostCompositeKey(HostCompositeKeyType keyType, String key) {
+        this.keyType = keyType;
+        this.key = key;
+    }
+
+    public static HostCompositeKey ofHost(HostDTO host) {
+        if (host.getHostId() != null) {
+            // 优先使用 hostId
+            return new HostCompositeKey(HostCompositeKeyType.HOST_ID, String.valueOf(host.getHostId()));
+        } else if (host.toCloudIp() != null) {
+            // 没有 hostId, 使用管控区域 ID + ipv4
+            return new HostCompositeKey(HostCompositeKeyType.CLOUD_IP, host.toCloudIp());
+        } else {
+            throw new IllegalArgumentException("Invalid host, both hostId or cloudIp are empty");
+        }
+    }
+
+
+    @Getter
+    public enum HostCompositeKeyType {
+        /**
+         * HostId 作为 KEY
+         */
+        HOST_ID(1),
+        /**
+         * 管控区域 ID+ ipv4 作为 KEY
+         */
+        CLOUD_IP(2);
+
+        private final int value;
+
+        HostCompositeKeyType(int value) {
+            this.value = value;
+        }
+
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        HostCompositeKey that = (HostCompositeKey) o;
+        if (this.getKeyType() != that.getKeyType()) {
+            return false;
+        }
+
+        return keyType == ((HostCompositeKey) o).getKeyType() && key.equals(that.getKey());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(keyType, key);
+    }
+
+    @Override
+    public String toString() {
+        return keyType.getValue() + ":" + key;
+    }
+}

src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostDTO.java

@@ -28,6 +28,7 @@
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.tencent.bk.job.common.annotation.PersistenceObject;
+import com.tencent.bk.job.common.model.HostCompositeKey;
 import com.tencent.bk.job.common.model.openapi.v4.OpenApiHostDTO;
 import com.tencent.bk.job.common.model.vo.CloudAreaInfoVO;
 import com.tencent.bk.job.common.model.vo.HostInfoVO;
@@ -269,12 +270,8 @@ public HostDTO clone() {
      * @return 主机KEY
      */
     @JsonIgnore
-    public String getUniqueKey() {
-        if (hostId != null) {
-            return "HOST_ID:" + hostId;
-        } else {
-            return "HOST_IP:" + toCloudIp();
-        }
+    public HostCompositeKey getUniqueKey() {
+        return HostCompositeKey.ofHost(this);
     }
 
     /**

src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java

@@ -118,7 +118,7 @@ public void testBatchSaveAgentTasks() {
         agentTask2.setEndTime(1572858331000L);
         agentTask2.setTotalTime(1000L);
         agentTask2.setErrorCode(88);
-        agentTask2.setStatus(ExecuteObjectTaskStatusEnum.HOST_NOT_EXIST);
+        agentTask2.setStatus(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT);
         agentTaskList.add(agentTask2);
 
         fileAgentTaskDAO.batchSaveAgentTasks(agentTaskList);
@@ -156,7 +156,7 @@ public void testBatchSaveAgentTasks() {
         assertThat(agentTask2Return.getEndTime()).isEqualTo(1572858331000L);
         assertThat(agentTask2Return.getTotalTime()).isEqualTo(1000L);
         assertThat(agentTask2Return.getErrorCode()).isEqualTo(88);
-        assertThat(agentTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.HOST_NOT_EXIST);
+        assertThat(agentTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT);
     }
 
     @Test
@@ -193,7 +193,7 @@ public void testBatchUpdateAgentTasks() {
         agentTask2.setEndTime(1572858331000L);
         agentTask2.setTotalTime(1000L);
         agentTask2.setErrorCode(88);
-        agentTask2.setStatus(ExecuteObjectTaskStatusEnum.HOST_NOT_EXIST);
+        agentTask2.setStatus(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT);
         agentTaskList.add(agentTask2);
 
         fileAgentTaskDAO.batchUpdateAgentTasks(agentTaskList);
@@ -227,7 +227,7 @@ public void testBatchUpdateAgentTasks() {
         assertThat(agentTask2Return.getEndTime()).isEqualTo(1572858331000L);
         assertThat(agentTask2Return.getTotalTime()).isEqualTo(1000L);
         assertThat(agentTask2Return.getErrorCode()).isEqualTo(88);
-        assertThat(agentTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.HOST_NOT_EXIST);
+        assertThat(agentTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT);
     }
 
     @Test

src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImplIntegrationTest.java

@@ -119,7 +119,7 @@ public void testBatchSaveTasks() {
         executeObjectTask2.setEndTime(1572858331000L);
         executeObjectTask2.setTotalTime(1000L);
         executeObjectTask2.setErrorCode(88);
-        executeObjectTask2.setStatus(ExecuteObjectTaskStatusEnum.HOST_NOT_EXIST);
+        executeObjectTask2.setStatus(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT);
         executeObjectTaskList.add(executeObjectTask2);
 
         fileExecuteObjectTaskDAO.batchSaveTasks(executeObjectTaskList);
@@ -157,7 +157,7 @@ public void testBatchSaveTasks() {
         assertThat(executeObjectTask2Return.getEndTime()).isEqualTo(1572858331000L);
         assertThat(executeObjectTask2Return.getTotalTime()).isEqualTo(1000L);
         assertThat(executeObjectTask2Return.getErrorCode()).isEqualTo(88);
-        assertThat(executeObjectTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.HOST_NOT_EXIST);
+        assertThat(executeObjectTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT);
     }
 
     @Test
@@ -194,7 +194,7 @@ public void testBatchUpdateAgentTasks() {
         executeObjectTask2.setEndTime(1572858331000L);
         executeObjectTask2.setTotalTime(1000L);
         executeObjectTask2.setErrorCode(88);
-        executeObjectTask2.setStatus(ExecuteObjectTaskStatusEnum.HOST_NOT_EXIST);
+        executeObjectTask2.setStatus(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT);
         executeObjectTaskList.add(executeObjectTask2);
 
         fileExecuteObjectTaskDAO.batchUpdateTasks(executeObjectTaskList);
@@ -226,7 +226,7 @@ public void testBatchUpdateAgentTasks() {
         assertThat(executeObjectTask2Return.getEndTime()).isEqualTo(1572858331000L);
         assertThat(executeObjectTask2Return.getTotalTime()).isEqualTo(1000L);
         assertThat(executeObjectTask2Return.getErrorCode()).isEqualTo(88);
-        assertThat(executeObjectTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.HOST_NOT_EXIST);
+        assertThat(executeObjectTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT);
     }
 
     @Test

src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java

@@ -121,7 +121,7 @@ public void testBatchSaveAgentTasks() {
         agentTask2.setEndTime(1572858331000L);
         agentTask2.setTotalTime(1000L);
         agentTask2.setErrorCode(88);
-        agentTask2.setStatus(ExecuteObjectTaskStatusEnum.HOST_NOT_EXIST);
+        agentTask2.setStatus(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT);
         agentTask2.setTag("bb");
         agentTask2.setExitCode(2);
         agentTaskList.add(agentTask2);
@@ -159,7 +159,7 @@ public void testBatchSaveAgentTasks() {
         assertThat(agentTask2Return.getEndTime()).isEqualTo(1572858331000L);
         assertThat(agentTask2Return.getTotalTime()).isEqualTo(1000L);
         assertThat(agentTask2Return.getErrorCode()).isEqualTo(88);
-        assertThat(agentTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.HOST_NOT_EXIST);
+        assertThat(agentTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT);
         assertThat(agentTask2Return.getTag()).isEqualTo("bb");
         assertThat(agentTask2Return.getExitCode()).isEqualTo(2);
     }
@@ -199,7 +199,7 @@ public void testBatchUpdateAgentTasks() {
         agentTask2.setEndTime(1572858331000L);
         agentTask2.setTotalTime(1000L);
         agentTask2.setErrorCode(88);
-        agentTask2.setStatus(ExecuteObjectTaskStatusEnum.HOST_NOT_EXIST);
+        agentTask2.setStatus(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT);
         agentTask2.setTag("bb");
         agentTask2.setExitCode(2);
         agentTaskList.add(agentTask2);
@@ -235,7 +235,7 @@ public void testBatchUpdateAgentTasks() {
         assertThat(agentTask2Return.getEndTime()).isEqualTo(1572858331000L);
         assertThat(agentTask2Return.getTotalTime()).isEqualTo(1000L);
         assertThat(agentTask2Return.getErrorCode()).isEqualTo(88);
-        assertThat(agentTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.HOST_NOT_EXIST);
+        assertThat(agentTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT);
         assertThat(agentTask2Return.getTag()).isEqualTo("bb");
         assertThat(agentTask2Return.getExitCode()).isEqualTo(2);
     }

src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImplIntegrationTest.java

@@ -122,7 +122,7 @@ public void testBatchSaveTasks() {
         executeObjectTask2.setEndTime(1572858331000L);
         executeObjectTask2.setTotalTime(1000L);
         executeObjectTask2.setErrorCode(88);
-        executeObjectTask2.setStatus(ExecuteObjectTaskStatusEnum.HOST_NOT_EXIST);
+        executeObjectTask2.setStatus(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT);
         executeObjectTask2.setTag("bb");
         executeObjectTask2.setExitCode(2);
         executeObjectTaskList.add(executeObjectTask2);
@@ -162,7 +162,7 @@ public void testBatchSaveTasks() {
         assertThat(executeObjectTask2Return.getEndTime()).isEqualTo(1572858331000L);
         assertThat(executeObjectTask2Return.getTotalTime()).isEqualTo(1000L);
         assertThat(executeObjectTask2Return.getErrorCode()).isEqualTo(88);
-        assertThat(executeObjectTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.HOST_NOT_EXIST);
+        assertThat(executeObjectTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT);
         assertThat(executeObjectTask2Return.getTag()).isEqualTo("bb");
         assertThat(executeObjectTask2Return.getExitCode()).isEqualTo(2);
     }
@@ -202,7 +202,7 @@ public void testBatchUpdateTasks() {
         executeObjectTask2.setEndTime(1572858331000L);
         executeObjectTask2.setTotalTime(1000L);
         executeObjectTask2.setErrorCode(88);
-        executeObjectTask2.setStatus(ExecuteObjectTaskStatusEnum.HOST_NOT_EXIST);
+        executeObjectTask2.setStatus(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT);
         executeObjectTask2.setTag("bb");
         executeObjectTask2.setExitCode(2);
         executeObjectTaskList.add(executeObjectTask2);
@@ -238,7 +238,7 @@ public void testBatchUpdateTasks() {
         assertThat(executeObjectTask2Return.getEndTime()).isEqualTo(1572858331000L);
         assertThat(executeObjectTask2Return.getTotalTime()).isEqualTo(1000L);
         assertThat(executeObjectTask2Return.getErrorCode()).isEqualTo(88);
-        assertThat(executeObjectTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.HOST_NOT_EXIST);
+        assertThat(executeObjectTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT);
         assertThat(executeObjectTask2Return.getTag()).isEqualTo("bb");
         assertThat(executeObjectTask2Return.getExitCode()).isEqualTo(2);
     }

src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/ExecuteObjectTaskStatusEnum.java

@@ -37,9 +37,9 @@ public enum ExecuteObjectTaskStatusEnum {
      */
     AGENT_ERROR(1),
     /**
-     * 无效主机
+     * 无效执行对象
      */
-    HOST_NOT_EXIST(2),
+    INVALID_EXECUTE_OBJECT(2),
     /**
      * 上次已成功
      */

src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskStartCommand.java

@@ -31,7 +31,6 @@
 import com.tencent.bk.job.execute.common.constants.RunStatusEnum;
 import com.tencent.bk.job.execute.config.JobExecuteConfig;
 import com.tencent.bk.job.execute.engine.EngineDependentServiceHolder;
-import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum;
 import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor;
 import com.tencent.bk.job.execute.engine.listener.event.EventSource;
 import com.tencent.bk.job.execute.engine.listener.event.StepEvent;
@@ -59,7 +58,6 @@
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.util.StopWatch;
 
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -102,9 +100,9 @@ public abstract class AbstractGseTaskStartCommand extends AbstractGseTaskCommand
      */
     protected Map<String, TaskVariableDTO> globalVariables = new HashMap<>();
     /**
-     * 执行对象任务列表
+     * 目标执行对象任务列表(全量，包含非法的任务)
      */
-    protected List<ExecuteObjectTask> executeObjectTasks;
+    protected List<ExecuteObjectTask> targetExecuteObjectTasks;
 
 
     AbstractGseTaskStartCommand(EngineDependentServiceHolder engineDependentServiceHolder,
@@ -225,33 +223,19 @@ private boolean startGseTaskIfNotAvailable(StopWatch watch) {
     }
 
     private void initExecuteObjectTasks() {
-        this.executeObjectTasks = executeObjectTaskService.listTasksByGseTaskId(stepInstance, gseTask.getId());
-        updateUninstalledExecuteObjectTasks(this.executeObjectTasks);
+        targetExecuteObjectTasks =
+            executeObjectTaskService.listTasksByGseTaskId(stepInstance, gseTask.getId())
+                .stream()
+                .filter(ExecuteObjectTask::isTarget)
+                .collect(Collectors.toList());
 
-        executeObjectTasks.stream()
-            .filter(ExecuteObjectTask::isTarget)
-            .filter(executeObjectTask -> !executeObjectTask.getExecuteObject().isAgentIdEmpty())
+        targetExecuteObjectTasks.stream()
+            .filter(executeObjectTask -> executeObjectTask.getExecuteObject().isExecutable())
             .forEach(executeObjectTask ->
                 this.targetExecuteObjectTaskMap.put(
                     executeObjectTask.getExecuteObject().toExecuteObjectGseKey(), executeObjectTask));
     }
 
-    private void updateUninstalledExecuteObjectTasks(Collection<ExecuteObjectTask> executeObjectTasks) {
-        List<ExecuteObjectTask> invalidExecuteObjectTasks = executeObjectTasks.stream()
-            .filter(executeObjectTask -> executeObjectTask.getExecuteObject().isAgentIdEmpty())
-            .collect(Collectors.toList());
-        if (CollectionUtils.isNotEmpty(invalidExecuteObjectTasks)) {
-            log.warn("{} contains invalid execute object tasks: {}", gseTaskInfo, invalidExecuteObjectTasks);
-            invalidExecuteObjectTasks.forEach(executeObjectTask -> {
-                executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.AGENT_NOT_INSTALLED);
-                executeObjectTask.setStartTime(System.currentTimeMillis());
-                executeObjectTask.setEndTime(System.currentTimeMillis());
-                executeObjectTask.calculateTotalTime();
-            });
-            executeObjectTaskService.batchUpdateTasks(executeObjectTasks);
-        }
-    }
-
     private void initVariables() {
         if (taskInstance.isPlanInstance()) {
             List<TaskVariableDTO> taskVariables =