Merge branch 'main' into 1.7

core/app_config/AppConfig.cpp

@@ -54,14 +54,7 @@ void AppConfig::LoadAddrConfig(const Json::Value& confJson) {
             string host = configServerAddress[0];
             int32_t port = atoi(configServerAddress[1].c_str());
 
-            std::string exception;
-            // regular expressions to verify ip
-            boost::regex reg_ip
-                = boost::regex("(?:(?:1[0-9][0-9]\.)|(?:2[0-4][0-9]\.)|(?:25[0-5]\.)|(?:[1-9][0-9]\.)|(?:[0-9]\.)){3}(?"
-                               ":(?:1[0-9][0-9])|(?:2[0-4][0-9])|(?:25[0-5])|(?:[1-9][0-9])|(?:[0-9]))");  
-            if (!BoostRegexMatch(host.c_str(), reg_ip, exception))
-                LOG_WARNING(sLogger, ("ilogtail_configserver_address", "parse fail")("exception", exception));
-            else if (port < 1 || port > 65535)
+            if (port < 1 || port > 65535)
                 LOG_WARNING(sLogger, ("ilogtail_configserver_address", "illegal port")("port", port));
             else
                 mConfigServerAddresses.push_back(ConfigServerAddress(host, port));

core/app_config/AppConfigBase.cpp

@@ -149,6 +149,7 @@ DECLARE_FLAG_INT32(polling_dir_first_watch_timeout);
 DECLARE_FLAG_INT32(polling_file_first_watch_timeout);
 DECLARE_FLAG_INT32(modify_check_interval);
 DECLARE_FLAG_INT32(ignore_file_modify_timeout);
+DEFINE_FLAG_STRING(host_path_blacklist, "host path matches substring in blacklist will be ignored", "");
 
 
 namespace logtail {
@@ -720,6 +721,21 @@ void AppConfigBase::LoadResourceConf(const Json::Value& confJson) {
         }
     }
 
+    if (!STRING_FLAG(host_path_blacklist).empty()) {
+#ifdef _MSC_VER
+        static const std::string delim = ";";
+#else
+        static const std::string delim = ":";
+#endif
+        auto blacklist = SplitString(TrimString(STRING_FLAG(host_path_blacklist)), delim);
+        for (const auto& s : blacklist) {
+            auto s1 = TrimString(s);
+            if (!s1.empty()) {
+                mHostPathBlacklist.emplace_back(std::move(s1));
+            }
+        }
+    }
+
     if (!LoadInt32Parameter(mSendDataPort, confJson, "data_server_port", "ALIYUN_LOGTAIL_DATA_SERVER_PORT")) {
         mSendDataPort = INT32_FLAG(data_server_port);
     }
@@ -1224,4 +1240,12 @@ void AppConfigBase::SetLogtailSysConfDir(const std::string& dirPath) {
                                                                                  mUserRemoteYamlConfigDirPath));
 }
 
+bool AppConfigBase::IsHostPathMatchBlacklist(const string& dirPath) const {
+    for (auto& dp : mHostPathBlacklist) {
+        if (dirPath.find(dp) != std::string::npos) {
+            return true;
+        }
+    }
+    return false;
+}
 } // namespace logtail

core/app_config/AppConfigBase.h

@@ -137,6 +137,8 @@ class AppConfigBase {
     //   local time to adjust logs' time automatically.
     bool mEnableLogTimeAutoAdjust = false;
 
+    std::vector<std::string> mHostPathBlacklist;
+
     /**
      * @brief Load ConfigServer, DataServer and network interface
      *
@@ -370,6 +372,8 @@ class AppConfigBase {
 
     inline bool EnableLogTimeAutoAdjust() const { return mEnableLogTimeAutoAdjust; }
 
+    bool IsHostPathMatchBlacklist(const std::string& dirPath) const;
+
 #ifdef APSARA_UNIT_TEST_MAIN
     friend class SenderUnittest;
     friend class ConfigUpdatorUnittest;

core/config_manager/ConfigManager.cpp

@@ -305,7 +305,7 @@ ConfigManager::SendHeartbeat(const AppConfig::ConfigServerAddress& configServerA
     } catch (const sdk::LOGException& e) {
         LOG_WARNING(
             sLogger,
-            ("SendHeartBeat", "fail")("reqBody", reqBody)("errCode", e.GetErrorCode())("errMsg", e.GetMessage()));
+            ("SendHeartBeat", "fail")("reqBody", reqBody)("errCode", e.GetErrorCode())("errMsg", e.GetMessage())("host", configServerAddress.host)("port", configServerAddress.port));
         return emptyResult;
     }
 }

core/config_manager/ConfigManagerBase.cpp

@@ -1149,6 +1149,10 @@ bool ConfigManagerBase::LoadJsonConfig(const Json::Value& jsonRoot, bool localFl
 // if checkTimeout, will not register the dir which is timeout
 // if not checkTimeout, will register the dir which is timeout and add it to the timeout list
 bool ConfigManagerBase::RegisterHandlersRecursively(const std::string& path, Config* config, bool checkTimeout) {
+    if (AppConfig::GetInstance()->IsHostPathMatchBlacklist(path)) {
+        LOG_INFO(sLogger, ("ignore path matching host path blacklist", path));
+        return false;
+    }
     bool result = false;
     if (checkTimeout && config->IsTimeout(path))
         return result;
@@ -1285,6 +1289,10 @@ bool ConfigManagerBase::RegisterHandlers() {
 }
 
 void ConfigManagerBase::RegisterWildcardPath(Config* config, const string& path, int32_t depth) {
+    if (AppConfig::GetInstance()->IsHostPathMatchBlacklist(path)) {
+        LOG_INFO(sLogger, ("ignore path matching host path blacklist", path));
+        return;
+    }
     bool finish;
     if ((depth + 1) == ((int)config->mWildcardPaths.size() - 1))
         finish = true;
@@ -1455,6 +1463,10 @@ bool ConfigManagerBase::RegisterDirectory(const std::string& source, const std::
 }
 
 bool ConfigManagerBase::RegisterHandlersWithinDepth(const std::string& path, Config* config, int depth) {
+    if (AppConfig::GetInstance()->IsHostPathMatchBlacklist(path)) {
+        LOG_INFO(sLogger, ("ignore path matching host path blacklist", path));
+        return false;
+    }
     if (depth <= 0) {
         DirCheckPointPtr dirCheckPoint;
         if (CheckPointManager::Instance()->GetDirCheckPoint(path, dirCheckPoint) == false)
@@ -1498,6 +1510,10 @@ bool ConfigManagerBase::RegisterHandlersWithinDepth(const std::string& path, Con
 
 // path not terminated by '/', path already registered
 bool ConfigManagerBase::RegisterDescendants(const string& path, Config* config, int withinDepth) {
+    if (AppConfig::GetInstance()->IsHostPathMatchBlacklist(path)) {
+        LOG_INFO(sLogger, ("ignore path matching host path blacklist", path));
+        return false;
+    }
     if (withinDepth <= 0) {
         return true;
     }

core/controller/EventDispatcherBase.cpp

@@ -161,6 +161,10 @@ EventDispatcherBase::~EventDispatcherBase() {
 }
 
 bool EventDispatcherBase::RegisterEventHandler(const char* path, Config* config, EventHandler*& handler) {
+    if (AppConfig::GetInstance()->IsHostPathMatchBlacklist(path)) {
+        LOG_INFO(sLogger, ("ignore path matching host path blacklist", path));
+        return false;
+    }
     // @todo
     // if this path belong to many config, if register one config with max_depth 0, then it will register fail
     if (!config->WithinMaxDepth(path)) {

core/polling/PollingDirFile.cpp

@@ -20,6 +20,7 @@
 #include <Shlwapi.h>
 #endif
 #include <sys/stat.h>
+#include "app_config/AppConfig.h"
 #include "common/Flags.h"
 #include "common/StringTools.h"
 #include "common/ErrorUtil.h"
@@ -342,6 +343,10 @@ bool PollingDirFile::PollingNormalConfigPath(
         return false;
 
     string dirPath = obj.empty() ? srcPath : PathJoin(srcPath, obj);
+    if (AppConfig::GetInstance()->IsHostPathMatchBlacklist(dirPath)) {
+        LOG_INFO(sLogger, ("ignore path matching host path blacklist", dirPath));
+        return false;
+    }
     bool isNewDirectory = false;
     if (!CheckAndUpdateDirMatchCache(dirPath, statBuf, isNewDirectory))
         return true;
@@ -464,6 +469,10 @@ bool PollingDirFile::PollingNormalConfigPath(
 // corresponding value in mConstWildcardPaths, call PollingNormalConfigPath or call
 // PollingWildcardConfigPath recursively.
 bool PollingDirFile::PollingWildcardConfigPath(const Config* pConfig, const string& dirPath, int depth) {
+    if (AppConfig::GetInstance()->IsHostPathMatchBlacklist(dirPath)) {
+        LOG_INFO(sLogger, ("ignore path matching host path blacklist", dirPath));
+        return false;
+    }
     auto const wildcardPathSize = static_cast<int>(pConfig->mWildcardPaths.size());
     if (depth - wildcardPathSize > pConfig->mMaxDepth)
         return false;

docs/cn/configuration/system-config.md

@@ -13,6 +13,7 @@
 | `config_update_interval`    | Int | 本地配置热加载的更新间隔，单位为秒。<br>**注意：此参数仅对社区版有效。**  |
 | `data_server_port`    | Int |<p>用于控制 `flusher_sls` 往 `SLS` 发送的协议类型。</p> <p>取值范围：443（默认），表示使用 `HTTPS` 协议发送；80表示使用 `HTTP` 协议发送。</p><p>如果使用`SLS`内网域名写入，建议使用`HTTP`协议发送，提高传输性能。</p> |
 | `send_running_status`    | Bool | 为了更好的了解 `iLogtail` 的使用情况，以便做出更有针对性的发展规划，`iLogtail` 会上报一些脱敏后的运行统计信息。您也可以手动关闭此开关。                                              |
+| `host_path_blacklist` | String | 全局主机路径黑名单，黑名单为子串匹配，Linux下多个子串以:分隔，Windows下以;分隔。比如禁止采集NAS挂载，可以配置为`/volumes/kubernetes.io~csi/nas-`。
 
 ### 典型配置
 

scripts/gen_build_scripts.sh

@@ -60,7 +60,7 @@ ram_limit_nproc=\$((ram_size / 1024 / 768))
 EOF
 
   if [ $EXPORT_GO_ENVS ]; then
-    envs=($(go env | grep -E 'GOPRIVATE=".+"|GOPROXY=".+"'))
+    envs=($(go env | grep -E 'GOPRIVATE=(".+"|'\''.+'\'')|GOPROXY=(".+"|'\''.+'\'')'))
     for v in ${envs[@]}; do
       echo "go env -w $v" >> $BUILD_SCRIPT_FILE
     done