From 5e836457ccc1f26b2e20ee26640ffd5950ac001d Mon Sep 17 00:00:00 2001 From: justinwwhuang Date: Thu, 25 Apr 2024 09:49:09 +0800 Subject: [PATCH] [INLONG-10059][Agent] Fix the exception of installation package comparison (#10062) --- .../inlong/agent/installer/ModuleManager.java | 112 ++++++++++++++++-- .../java/installer/TestModuleManager.java | 8 +- .../pojo/agent/installer/ConfigResult.java | 5 +- .../service/core/impl/AgentServiceImpl.java | 3 +- 4 files changed, 107 insertions(+), 21 deletions(-) diff --git a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java index dd438ed02a0..db0dda79747 100755 --- a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java +++ b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java @@ -28,6 +28,7 @@ import org.apache.inlong.common.pojo.agent.installer.ConfigResult; import org.apache.inlong.common.pojo.agent.installer.ModuleConfig; import org.apache.inlong.common.pojo.agent.installer.ModuleStateEnum; +import org.apache.inlong.common.pojo.agent.installer.PackageConfig; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -54,6 +55,7 @@ import java.security.NoSuchAlgorithmException; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; @@ -78,6 +80,7 @@ public class ModuleManager extends AbstractDaemon { public static final int DOWNLOAD_PACKAGE_READ_BUFF_SIZE = 1024 * 1024; public static final String LOCAL_CONFIG_FILE = "modules.json"; private static final Logger LOGGER = LoggerFactory.getLogger(ModuleManager.class); + public static final int MAX_MODULE_SIZE = 10; private final InstallerConfiguration conf; private final String confPath; private final BlockingQueue configQueue; @@ -112,11 +115,8 @@ private boolean requiredKeys(InstallerConfiguration conf) { } public void submitConfig(ConfigResult config) { - if (config == null) { - return; - } - if (config.getModuleList().isEmpty()) { - LOGGER.error("module list should not be empty!"); + if (!isConfigValid(config)) { + LOGGER.error("config is invalid !"); return; } configQueue.clear(); @@ -127,6 +127,94 @@ public void submitConfig(ConfigResult config) { configQueue.add(config); } + private boolean isConfigValid(ConfigResult config) { + if (config == null) { + LOGGER.error("config is null!"); + return false; + } + if (config.getMd5() == null) { + LOGGER.error("modules md5 should not be null!"); + return false; + } + if (config.getModuleList().isEmpty()) { + LOGGER.error("module list should not be empty!"); + return false; + } + if (config.getModuleList().size() > MAX_MODULE_SIZE) { + LOGGER.error("module list {} over size {}!", config.getModuleList().size(), MAX_MODULE_SIZE); + return false; + } + for (int i = 0; i < config.getModuleList().size(); i++) { + if (!isModuleConfigValid(config.getModuleList().get(i))) { + return false; + } + } + return true; + } + + private boolean isModuleConfigValid(ModuleConfig module) { + if (module == null) { + LOGGER.error("module should not be null!"); + return false; + } + if (module.getMd5() == null) { + LOGGER.error("module md5 should not be null!"); + return false; + } + if (module.getName() == null) { + LOGGER.error("module name should not be null!"); + return false; + } + if (module.getVersion() == null) { + LOGGER.error("module version should not be null!"); + return false; + } + if (module.getInstallCommand() == null) { + LOGGER.error("module install cmd should not be null!"); + return false; + } + if (module.getStartCommand() == null) { + LOGGER.error("module start cmd should not be null!"); + return false; + } + if (module.getStopCommand() == null) { + LOGGER.error("module stop cmd should not be null!"); + return false; + } + if (module.getCheckCommand() == null) { + LOGGER.error("module check cmd should not be null!"); + return false; + } + if (!isPackageConfigValid(module.getPackageConfig())) { + return false; + } + return true; + } + + private boolean isPackageConfigValid(PackageConfig packageConfig) { + if (packageConfig == null) { + LOGGER.error("module package config should not be null!"); + return false; + } + if (packageConfig.getMd5() == null) { + LOGGER.error("package md5 should not be null!"); + return false; + } + if (packageConfig.getFileName() == null) { + LOGGER.error("package file name should not be null!"); + return false; + } + if (packageConfig.getDownloadUrl() == null) { + LOGGER.error("package url should not be null!"); + return false; + } + if (packageConfig.getStoragePath() == null) { + LOGGER.error("package save path should not be null!"); + return false; + } + return true; + } + public String getCurrentMd5() { return currentMd5; } @@ -190,7 +278,7 @@ public void saveToLocalFile(String confPath) { File jsonPath = new File(temp.getPath() + "/" + LOCAL_CONFIG_FILE); try (BufferedWriter writer = new BufferedWriter( new OutputStreamWriter(new FileOutputStream(jsonPath), StandardCharsets.UTF_8))) { - String curConfig = GSON.toJson(ConfigResult.builder().md5(currentMd5).moduleNum(currentModules.size()) + String curConfig = GSON.toJson(ConfigResult.builder().md5(currentMd5) .moduleList(currentModules.values().stream().collect(Collectors.toList())).build()); writer.write(curConfig); writer.flush(); @@ -235,7 +323,8 @@ private void checkModules() { installModule(module); saveModuleState(module.getId(), ModuleStateEnum.INSTALLED); } else { - LOGGER.info("check module {} package failed, change stated to new, will download package again", + LOGGER.info( + "check module {} package failed, change stated to new, will download package again", module.getName()); saveModuleState(module.getId(), ModuleStateEnum.NEW); } @@ -425,7 +514,8 @@ private boolean isProcessAllStarted(ModuleConfig module) { } private boolean downloadModule(ModuleConfig module) { - LOGGER.info("download module {} begin with url {}", module.getId(), module.getPackageConfig().getDownloadUrl()); + LOGGER.info("download module {} begin with url {}", module.getId(), + module.getPackageConfig().getDownloadUrl()); try { URL url = new URL(module.getPackageConfig().getDownloadUrl()); URLConnection conn = url.openConnection(); @@ -433,7 +523,8 @@ private boolean downloadModule(ModuleConfig module) { authHeader.forEach((k, v) -> { conn.setRequestProperty(k, v); }); - String path = module.getPackageConfig().getStoragePath() + "/" + module.getPackageConfig().getFileName(); + String path = + module.getPackageConfig().getStoragePath() + "/" + module.getPackageConfig().getFileName(); try (InputStream inputStream = conn.getInputStream(); FileOutputStream outputStream = new FileOutputStream(path)) { LOGGER.info("save path {}", path); @@ -461,7 +552,7 @@ private boolean downloadModule(ModuleConfig module) { private boolean isPackageDownloaded(ModuleConfig module) { String path = module.getPackageConfig().getStoragePath() + "/" + module.getPackageConfig().getFileName(); String fileMd5 = calcFileMd5(path); - if (fileMd5.equals(module.getPackageConfig().getMd5())) { + if (Objects.equals(fileMd5, module.getPackageConfig().getMd5())) { return true; } else { LOGGER.error("md5 not match! fileMd5 {} moduleMd5 {}", fileMd5, module.getPackageConfig().getMd5()); @@ -502,7 +593,6 @@ private static String calcFileMd5(String path) { ret = new String(Hex.encodeHex(md.digest())); } catch (NoSuchAlgorithmException e) { LOGGER.error("calc file md5 NoSuchAlgorithmException", e); - } catch (IOException e) { LOGGER.error("calc file md5 IOException", e); } diff --git a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java index 719727852b0..e5ccbd8cd3f 100755 --- a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java +++ b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java @@ -174,11 +174,11 @@ private ConfigResult getConfig() { configs.add(getModuleConfig(1, "inlong-agent", "inlong-agent-md5-185454", "1.0", 1, "cd ~/inlong-agent/bin;sh agent.sh start", "cd ~/inlong-agent/bin;sh agent.sh stop", "ps aux | grep core.AgentMain | grep java | grep -v grep | awk '{print $2}'", - "cd ~/inlong-agent/bin;sh agent.sh stop;rm -rf ~/inlong-agent/;mkdir ~/inlong-agent;cd /tmp;tar -xzvf agent-release-1.12.0-SNAPSHOT-bin.tar.gz -C ~/inlong-agent;cd ~/inlong-agent/bin;sh agent.sh start", - "echo empty uninstall cmd", "agent-release-1.12.0-SNAPSHOT-bin.tar.gz", - "http://11.151.252.111:8083/inlong/manager/openapi/agent/download/agent-release-1.12.0-SNAPSHOT-bin.tar.gz", + "cd ~/inlong-agent/bin;sh agent.sh stop;rm -rf ~/inlong-agent/;mkdir ~/inlong-agent;cd /tmp;tar -xzvf agent-release-1.13.0-SNAPSHOT-bin.tar.gz -C ~/inlong-agent;cd ~/inlong-agent/bin;sh agent.sh start", + "echo empty uninstall cmd", "agent-release-1.13.0-SNAPSHOT-bin.tar.gz", + "http://11.151.252.111:8083/inlong/manager/openapi/agent/download/agent-release-1.13.0-SNAPSHOT-bin.tar.gz", NEW_MD5)); - return ConfigResult.builder().moduleList(configs).moduleNum(1).md5("config-result-md5-193603").build(); + return ConfigResult.builder().moduleList(configs).md5("config-result-md5-193603").build(); } private ModuleConfig getModuleConfig(int id, String name, String md5, String version, Integer procNum, diff --git a/inlong-common/src/main/java/org/apache/inlong/common/pojo/agent/installer/ConfigResult.java b/inlong-common/src/main/java/org/apache/inlong/common/pojo/agent/installer/ConfigResult.java index fcbbd0f5c1c..14e504d2111 100644 --- a/inlong-common/src/main/java/org/apache/inlong/common/pojo/agent/installer/ConfigResult.java +++ b/inlong-common/src/main/java/org/apache/inlong/common/pojo/agent/installer/ConfigResult.java @@ -42,10 +42,7 @@ public class ConfigResult { * The md5 of the config result */ private String md5; - /** - * Number of module - */ - private Integer moduleNum; + /** * The list of module config list */ diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AgentServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AgentServiceImpl.java index 9eba178179d..09729662468 100644 --- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AgentServiceImpl.java +++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AgentServiceImpl.java @@ -844,8 +844,7 @@ private ConfigResult loadModuleConfigs(ConfigRequest request) { String jsonStr = GSON.toJson(configs); String configMd5 = DigestUtils.md5Hex(jsonStr); - ConfigResult configResult = ConfigResult.builder().moduleList(configs).moduleNum(configs.size()) - .md5(configMd5) + ConfigResult configResult = ConfigResult.builder().moduleList(configs).md5(configMd5) .code(InstallerCode.SUCCESS) .build(); LOGGER.info("success load module config, size = {}", configResult.getModuleList().size());