Skip to content

Commit

Permalink
[INLONG-10059][Agent] Fix the exception of installation package compa…
Browse files Browse the repository at this point in the history 
…rison (apache#10062)
  • Loading branch information
@justinwwhuang
justinwwhuang authored Apr 25, 2024
1 parent 8be68c5 commit 5e83645
Show file tree
Hide file tree
Showing 4 changed files with 107 additions and 21 deletions.
112 changes: 101 additions & 11 deletions ...-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.inlong.common.pojo.agent.installer.ConfigResult;
import org.apache.inlong.common.pojo.agent.installer.ModuleConfig;
import org.apache.inlong.common.pojo.agent.installer.ModuleStateEnum;
import org.apache.inlong.common.pojo.agent.installer.PackageConfig;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
Expand All @@ -54,6 +55,7 @@
import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
Expand All @@ -78,6 +80,7 @@ public class ModuleManager extends AbstractDaemon {
public static final int DOWNLOAD_PACKAGE_READ_BUFF_SIZE = 1024 * 1024;
public static final String LOCAL_CONFIG_FILE = "modules.json";
private static final Logger LOGGER = LoggerFactory.getLogger(ModuleManager.class);
public static final int MAX_MODULE_SIZE = 10;
private final InstallerConfiguration conf;
private final String confPath;
private final BlockingQueue<ConfigResult> configQueue;
Expand Down Expand Up @@ -112,11 +115,8 @@ private boolean requiredKeys(InstallerConfiguration conf) {
}

public void submitConfig(ConfigResult config) {
if (config == null) {
return;
}
if (config.getModuleList().isEmpty()) {
LOGGER.error("module list should not be empty!");
if (!isConfigValid(config)) {
LOGGER.error("config is invalid !");
return;
}
configQueue.clear();
Expand All @@ -127,6 +127,94 @@ public void submitConfig(ConfigResult config) {
configQueue.add(config);
}

private boolean isConfigValid(ConfigResult config) {
if (config == null) {
LOGGER.error("config is null!");
return false;
}
if (config.getMd5() == null) {
LOGGER.error("modules md5 should not be null!");
return false;
}
if (config.getModuleList().isEmpty()) {
LOGGER.error("module list should not be empty!");
return false;
}
if (config.getModuleList().size() > MAX_MODULE_SIZE) {
LOGGER.error("module list {} over size {}!", config.getModuleList().size(), MAX_MODULE_SIZE);
return false;
}
for (int i = 0; i < config.getModuleList().size(); i++) {
if (!isModuleConfigValid(config.getModuleList().get(i))) {
return false;
}
}
return true;
}

private boolean isModuleConfigValid(ModuleConfig module) {
if (module == null) {
LOGGER.error("module should not be null!");
return false;
}
if (module.getMd5() == null) {
LOGGER.error("module md5 should not be null!");
return false;
}
if (module.getName() == null) {
LOGGER.error("module name should not be null!");
return false;
}
if (module.getVersion() == null) {
LOGGER.error("module version should not be null!");
return false;
}
if (module.getInstallCommand() == null) {
LOGGER.error("module install cmd should not be null!");
return false;
}
if (module.getStartCommand() == null) {
LOGGER.error("module start cmd should not be null!");
return false;
}
if (module.getStopCommand() == null) {
LOGGER.error("module stop cmd should not be null!");
return false;
}
if (module.getCheckCommand() == null) {
LOGGER.error("module check cmd should not be null!");
return false;
}
if (!isPackageConfigValid(module.getPackageConfig())) {
return false;
}
return true;
}

private boolean isPackageConfigValid(PackageConfig packageConfig) {
if (packageConfig == null) {
LOGGER.error("module package config should not be null!");
return false;
}
if (packageConfig.getMd5() == null) {
LOGGER.error("package md5 should not be null!");
return false;
}
if (packageConfig.getFileName() == null) {
LOGGER.error("package file name should not be null!");
return false;
}
if (packageConfig.getDownloadUrl() == null) {
LOGGER.error("package url should not be null!");
return false;
}
if (packageConfig.getStoragePath() == null) {
LOGGER.error("package save path should not be null!");
return false;
}
return true;
}

public String getCurrentMd5() {
return currentMd5;
}
Expand Down Expand Up @@ -190,7 +278,7 @@ public void saveToLocalFile(String confPath) {
File jsonPath = new File(temp.getPath() + "/" + LOCAL_CONFIG_FILE);
try (BufferedWriter writer = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(jsonPath), StandardCharsets.UTF_8))) {
String curConfig = GSON.toJson(ConfigResult.builder().md5(currentMd5).moduleNum(currentModules.size())
String curConfig = GSON.toJson(ConfigResult.builder().md5(currentMd5)
.moduleList(currentModules.values().stream().collect(Collectors.toList())).build());
writer.write(curConfig);
writer.flush();
Expand Down Expand Up @@ -235,7 +323,8 @@ private void checkModules() {
installModule(module);
saveModuleState(module.getId(), ModuleStateEnum.INSTALLED);
} else {
LOGGER.info("check module {} package failed, change stated to new, will download package again",
LOGGER.info(
"check module {} package failed, change stated to new, will download package again",
module.getName());
saveModuleState(module.getId(), ModuleStateEnum.NEW);
}
Expand Down Expand Up @@ -425,15 +514,17 @@ private boolean isProcessAllStarted(ModuleConfig module) {
}

private boolean downloadModule(ModuleConfig module) {
LOGGER.info("download module {} begin with url {}", module.getId(), module.getPackageConfig().getDownloadUrl());
LOGGER.info("download module {} begin with url {}", module.getId(),
module.getPackageConfig().getDownloadUrl());
try {
URL url = new URL(module.getPackageConfig().getDownloadUrl());
URLConnection conn = url.openConnection();
Map<String, String> authHeader = httpManager.getAuthHeader();
authHeader.forEach((k, v) -> {
conn.setRequestProperty(k, v);
});
String path = module.getPackageConfig().getStoragePath() + "/" + module.getPackageConfig().getFileName();
String path =
module.getPackageConfig().getStoragePath() + "/" + module.getPackageConfig().getFileName();
try (InputStream inputStream = conn.getInputStream();
FileOutputStream outputStream = new FileOutputStream(path)) {
LOGGER.info("save path {}", path);
Expand Down Expand Up @@ -461,7 +552,7 @@ private boolean downloadModule(ModuleConfig module) {
private boolean isPackageDownloaded(ModuleConfig module) {
String path = module.getPackageConfig().getStoragePath() + "/" + module.getPackageConfig().getFileName();
String fileMd5 = calcFileMd5(path);
if (fileMd5.equals(module.getPackageConfig().getMd5())) {
if (Objects.equals(fileMd5, module.getPackageConfig().getMd5())) {
return true;
} else {
LOGGER.error("md5 not match! fileMd5 {} moduleMd5 {}", fileMd5, module.getPackageConfig().getMd5());
Expand Down Expand Up @@ -502,7 +593,6 @@ private static String calcFileMd5(String path) {
ret = new String(Hex.encodeHex(md.digest()));
} catch (NoSuchAlgorithmException e) {
LOGGER.error("calc file md5 NoSuchAlgorithmException", e);

} catch (IOException e) {
LOGGER.error("calc file md5 IOException", e);
}
Expand Down
8 changes: 4 additions & 4 deletions inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,11 +174,11 @@ private ConfigResult getConfig() {
configs.add(getModuleConfig(1, "inlong-agent", "inlong-agent-md5-185454", "1.0", 1,
"cd ~/inlong-agent/bin;sh agent.sh start", "cd ~/inlong-agent/bin;sh agent.sh stop",
"ps aux | grep core.AgentMain | grep java | grep -v grep | awk '{print $2}'",
"cd ~/inlong-agent/bin;sh agent.sh stop;rm -rf ~/inlong-agent/;mkdir ~/inlong-agent;cd /tmp;tar -xzvf agent-release-1.12.0-SNAPSHOT-bin.tar.gz -C ~/inlong-agent;cd ~/inlong-agent/bin;sh agent.sh start",
"echo empty uninstall cmd", "agent-release-1.12.0-SNAPSHOT-bin.tar.gz",
"http://11.151.252.111:8083/inlong/manager/openapi/agent/download/agent-release-1.12.0-SNAPSHOT-bin.tar.gz",
"cd ~/inlong-agent/bin;sh agent.sh stop;rm -rf ~/inlong-agent/;mkdir ~/inlong-agent;cd /tmp;tar -xzvf agent-release-1.13.0-SNAPSHOT-bin.tar.gz -C ~/inlong-agent;cd ~/inlong-agent/bin;sh agent.sh start",
"echo empty uninstall cmd", "agent-release-1.13.0-SNAPSHOT-bin.tar.gz",
"http://11.151.252.111:8083/inlong/manager/openapi/agent/download/agent-release-1.13.0-SNAPSHOT-bin.tar.gz",
NEW_MD5));
return ConfigResult.builder().moduleList(configs).moduleNum(1).md5("config-result-md5-193603").build();
return ConfigResult.builder().moduleList(configs).md5("config-result-md5-193603").build();
}

private ModuleConfig getModuleConfig(int id, String name, String md5, String version, Integer procNum,
Expand Down
5 changes: 1 addition & 4 deletions inlong-common/src/main/java/org/apache/inlong/common/pojo/agent/installer/ConfigResult.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,7 @@ public class ConfigResult {
* The md5 of the config result
*/
private String md5;
/**
* Number of module
*/
private Integer moduleNum;

/**
* The list of module config list
*/
Expand Down
3 changes: 1 addition & 2 deletions ...r-service/src/main/java/org/apache/inlong/manager/service/core/impl/AgentServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -844,8 +844,7 @@ private ConfigResult loadModuleConfigs(ConfigRequest request) {
String jsonStr = GSON.toJson(configs);
String configMd5 = DigestUtils.md5Hex(jsonStr);

ConfigResult configResult = ConfigResult.builder().moduleList(configs).moduleNum(configs.size())
.md5(configMd5)
ConfigResult configResult = ConfigResult.builder().moduleList(configs).md5(configMd5)
.code(InstallerCode.SUCCESS)
.build();
LOGGER.info("success load module config, size = {}", configResult.getModuleList().size());
Expand Down

0 comments on commit 5e83645

Please sign in to comment.