feat: support host monitor

core/CMakeLists.txt

@@ -124,6 +124,7 @@ set(SUB_DIRECTORIES_LIST
         prometheus prometheus/labels prometheus/schedulers prometheus/async
         ebpf ebpf/observer ebpf/security ebpf/handler
         parser sls_control sdk
+        host_monitor host_monitor/collector
         )
 if (LINUX)
     if (ENABLE_ENTERPRISE)

core/app_config/AppConfig.cpp

@@ -164,6 +164,7 @@ DEFINE_FLAG_STRING(loong_collector_operator_service, "loong collector operator s
 DEFINE_FLAG_INT32(loong_collector_operator_service_port, "loong collector operator service port", 8888);
 DEFINE_FLAG_INT32(loong_collector_k8s_meta_service_port, "loong collector operator service port", 9000);
 DEFINE_FLAG_STRING(_pod_name_, "agent pod name", "");
+DEFINE_FLAG_INT32(process_collect_silent_count, "number of process scanned between a sleep", 1000);
 
 DEFINE_FLAG_STRING(app_info_file, "", "app_info.json");
 DEFINE_FLAG_STRING(crash_stack_file_name, "crash stack back trace file name", "backtrace.dat");

core/application/Application.cpp

@@ -14,6 +14,8 @@
 
 #include "application/Application.h"
 
+#include "timer/Timer.h"
+
 #ifndef LOGTAIL_NO_TC_MALLOC
 #include <gperftools/malloc_extension.h>
 #endif
@@ -263,9 +265,9 @@ void Application::Start() { // GCOVR_EXCL_START
     }
 
     ProcessorRunner::GetInstance()->Init();
+    Timer::GetInstance()->Init();
 
-    time_t curTime = 0, lastConfigCheckTime = 0, lastUpdateMetricTime = 0,
-           lastCheckTagsTime = 0, lastQueueGCTime = 0;
+    time_t curTime = 0, lastConfigCheckTime = 0, lastUpdateMetricTime = 0, lastCheckTagsTime = 0, lastQueueGCTime = 0;
 #ifndef LOGTAIL_NO_TC_MALLOC
     time_t lastTcmallocReleaseMemTime = 0;
 #endif

core/common/FileSystemUtil.cpp

@@ -141,6 +141,50 @@ bool ReadFileContent(const std::string& fileName, std::string& content, uint32_t
     return true;
 }
 
+int GetLines(std::istream& is,
+             bool enableEmptyLine,
+             const std::function<void(const std::string&)>& pushBack,
+             std::string* errorMessage) {
+    std::string line;
+    // 此处必须判断eof，具体原因参见:
+    // https://stackoverflow.com/questions/40561482/getline-throws-basic-iosclear-exception-after-reading-the-last-line
+    while (!is.eof() && std::getline(is, line)) {
+        if (enableEmptyLine || !line.empty()) {
+            pushBack(line);
+        }
+    }
+    return 0;
+}
+
+int GetLines(const std::filesystem::path& filename,
+             bool enableEmptyLine,
+             const std::function<void(const std::string&)>& pushBack,
+             std::string* errorMessage) {
+    int ret = 0;
+    std::ifstream fin;
+    try {
+        fin.exceptions(std::ifstream::failbit | std::ifstream::badbit);
+        fin.open(filename.string(), std::ios_base::in);
+        fin.exceptions(std::ifstream::goodbit);
+        GetLines(fin, enableEmptyLine, pushBack, errorMessage);
+        fin.close();
+    } catch (const std::exception& fail) {
+        if (errorMessage != nullptr) {
+            LOG_ERROR(sLogger, ("open file fail", filename)("errno", strerror(errno)));
+            ret = -1;
+        }
+        fin.close();
+    }
+    return ret;
+}
+
+int GetFileLines(const std::filesystem::path& filename,
+                 std::vector<std::string>& res,
+                 bool enableEmptyLine,
+                 std::string* errorMessage) {
+    return GetLines(filename, enableEmptyLine, [&res](const std::string& s) { res.push_back(s); }, errorMessage);
+}
+
 bool OverwriteFile(const std::string& fileName, const std::string& content) {
     FILE* pFile = fopen(fileName.c_str(), "w");
     if (pFile == NULL) {

core/common/FileSystemUtil.h

@@ -26,6 +26,8 @@
 #elif defined(_MSC_VER)
 #include <Windows.h>
 #endif
+#include <filesystem>
+
 #include "DevInode.h"
 #include "ErrorUtil.h"
 #include "LogtailCommonFlags.h"
@@ -87,6 +89,19 @@ void TrimLastSeperator(std::string& path);
 // ReadFileContent reads all content of @fileName to @content.
 bool ReadFileContent(const std::string& fileName, std::string& content, uint32_t maxFileSize = 8192);
 
+int GetLines(std::istream& is,
+             bool enableEmptyLine,
+             const std::function<void(const std::string&)>& pushBack,
+             std::string* errorMessage);
+int GetLines(const std::filesystem::path& filename,
+             bool enableEmptyLine,
+             const std::function<void(const std::string&)>& pushBack,
+             std::string* errorMessage);
+int GetFileLines(const std::filesystem::path& filename,
+                 std::vector<std::string>& res,
+                 bool enableEmptyLine = true,
+                 std::string* errorMessage = nullptr);
+
 // OverwriteFile overwrides @fileName with @content.
 bool OverwriteFile(const std::string& fileName, const std::string& content);
 

core/common/MachineInfoUtil.cpp

@@ -15,6 +15,11 @@
 #include "MachineInfoUtil.h"
 
 #include <string.h>
+
+#include <boost/filesystem.hpp>
+
+#include "AppConfig.h"
+#include "common/UUIDUtil.h"
 #if defined(__linux__)
 #include <arpa/inet.h>
 #include <ifaddrs.h>
@@ -40,8 +45,10 @@
 
 #include "FileSystemUtil.h"
 #include "StringTools.h"
+#include "common/FileSystemUtil.h"
 #include "logger/Logger.h"
 
+DEFINE_FLAG_STRING(agent_host_id, "", "");
 
 #if defined(_MSC_VER)
 typedef LONG NTSTATUS, *PNTSTATUS;
@@ -496,6 +503,92 @@ size_t FetchECSMetaCallback(char* buffer, size_t size, size_t nmemb, std::string
     return sizes;
 }
 
+std::string GetSerialNumberFromEcsAssist(const std::string& machineIdFile) {
+    std::string sn;
+    if (CheckExistance(machineIdFile)) {
+        if (!ReadFileContent(machineIdFile, sn)) {
+            return "";
+        }
+    }
+    return sn;
+}
+
+static std::string GetEcsAssistMachineIdFile() {
+#if defined(WIN32)
+    return "C:\\ProgramData\\aliyun\\assist\\hybrid\\machine-id";
+#else
+    return "/usr/local/share/aliyun-assist/hybrid/machine-id";
+#endif
+}
+
+std::string GetSerialNumberFromEcsAssist() {
+    return GetSerialNumberFromEcsAssist(GetEcsAssistMachineIdFile());
+}
+
+std::string RandomHostid() {
+    static std::string hostId = CalculateRandomUUID();
+    return hostId;
+}
+
+const std::string& GetLocalHostId() {
+    static std::string fileName = AppConfig::GetInstance()->GetLoongcollectorConfDir() + PATH_SEPARATOR + "host_id";
+    static std::string hostId;
+    if (!hostId.empty()) {
+        return hostId;
+    }
+    if (CheckExistance(fileName)) {
+        if (!ReadFileContent(fileName, hostId)) {
+            hostId = "";
+        }
+    }
+    if (hostId.empty()) {
+        hostId = RandomHostid();
+
+        LOG_INFO(sLogger, ("save hostId file to local file system, hostId", hostId));
+        int fd = open(fileName.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0755);
+        if (fd == -1) {
+            int savedErrno = errno;
+            if (savedErrno != EEXIST) {
+                LOG_ERROR(sLogger, ("save hostId file fail", fileName)("errno", strerror(savedErrno)));
+            }
+        } else {
+            // 文件成功创建,现在写入hostId
+            ssize_t written = write(fd, hostId.c_str(), hostId.length());
+            if (written == static_cast<ssize_t>(hostId.length())) {
+                LOG_INFO(sLogger, ("hostId saved successfully to", fileName));
+            } else {
+                int writeErrno = errno;
+                LOG_ERROR(sLogger, ("Failed to write hostId to file", fileName)("errno", strerror(writeErrno)));
+            }
+            close(fd);
+        }
+    }
+    return hostId;
+}
+
+std::string FetchHostId() {
+    static std::string hostId;
+    if (!hostId.empty()) {
+        return hostId;
+    }
+    hostId = STRING_FLAG(agent_host_id);
+    if (!hostId.empty()) {
+        return hostId;
+    }
+    ECSMeta meta = FetchECSMeta();
+    hostId = meta.instanceID;
+    if (!hostId.empty()) {
+        return hostId;
+    }
+    hostId = GetSerialNumberFromEcsAssist();
+    if (!hostId.empty()) {
+        return hostId;
+    }
+    hostId = GetLocalHostId();
+
+    return hostId;
+}
+
 ECSMeta FetchECSMeta() {
     CURL* curl;
     for (size_t retryTimes = 1; retryTimes <= 5; retryTimes++) {

core/common/MachineInfoUtil.h

@@ -15,10 +15,11 @@
  */
 
 #pragma once
+#include <rapidjson/document.h>
+#include <rapidjson/rapidjson.h>
+
 #include <string>
 #include <unordered_set>
-#include <rapidjson/rapidjson.h>
-#include <rapidjson/document.h>
 
 namespace logtail {
 
@@ -37,8 +38,9 @@ std::string GetHostIp(const std::string& intf = "");
 void GetAllPids(std::unordered_set<int32_t>& pids);
 bool GetKernelInfo(std::string& kernelRelease, int64_t& kernelVersion);
 bool GetRedHatReleaseInfo(std::string& os, int64_t& osVersion, std::string bashPath = "");
-bool IsDigitsDotsHostname(const char *hostname);
+bool IsDigitsDotsHostname(const char* hostname);
 ECSMeta FetchECSMeta();
+std::string FetchHostId();
 
 // GetAnyAvailableIP walks through all interfaces (AF_INET) to find an available IP.
 // Priority:

core/common/StringTools.cpp

@@ -363,4 +363,12 @@ void RemoveFilePathTrailingSlash(std::string& filePath) {
     filePath = path.string();
 }
 
+bool IsInt(const char* sz) {
+    bool ok = (sz != nullptr && *sz != '\0');
+    for (auto* it = sz; ok && *it; ++it) {
+        ok = (0 != std::isdigit(*it));
+    }
+    return ok;
+}
+
 } // namespace logtail

core/common/StringTools.h

@@ -146,6 +146,12 @@ bool NormalizeTopicRegFormat(std::string& regStr);
 
 void RemoveFilePathTrailingSlash(std::string& path);
 
+bool IsInt(const char* sz);
+
+inline bool IsInt(const std::string& str) {
+    return IsInt(str.c_str());
+}
+
 #if defined(_MSC_VER)
 // TODO: Test it.
 #define FNM_PATHNAME 0

core/common/timer/Timer.cpp

@@ -23,6 +23,9 @@ namespace logtail {
 void Timer::Init() {
     {
         lock_guard<mutex> lock(mThreadRunningMux);
+        if (mIsThreadRunning) {
+            return;
+        }
         mIsThreadRunning = true;
     }
     mThreadRes = async(launch::async, &Timer::Run, this);
@@ -31,6 +34,9 @@ void Timer::Init() {
 void Timer::Stop() {
     {
         lock_guard<mutex> lock(mThreadRunningMux);
+        if (!mIsThreadRunning) {
+            return;
+        }
         mIsThreadRunning = false;
     }
     mCV.notify_one();

core/common/timer/Timer.h

@@ -34,11 +34,21 @@ struct TimerEventCompare {
 
 class Timer {
 public:
+    Timer(const Timer&) = delete;
+    Timer(Timer&&) = delete;
+    Timer& operator=(const Timer&) = delete;
+    Timer& operator=(Timer&&) = delete;
+    ~Timer() = default;
+    static Timer* GetInstance() {
+        static Timer sInstance;
+        return &sInstance;
+    }
     void Init();
     void Stop();
     void PushEvent(std::unique_ptr<TimerEvent>&& e);
 
 private:
+    Timer() = default;
     void Run();
 
     mutable std::mutex mQueueMux;
@@ -53,6 +63,7 @@ class Timer {
 #ifdef APSARA_UNIT_TEST_MAIN
     friend class TimerUnittest;
     friend class ScrapeSchedulerUnittest;
+    friend class HostMonitorInputRunnerUnittest;
 #endif
 };
 

core/common/timer/TimerEvent.h

@@ -29,6 +29,7 @@ class TimerEvent {
     virtual bool Execute() = 0;
 
     std::chrono::steady_clock::time_point GetExecTime() const { return mExecTime; }
+    void SetExecTime(std::chrono::steady_clock::time_point nextExecTime) { mExecTime = nextExecTime; }
 
 private:
     std::chrono::steady_clock::time_point mExecTime;

core/constants/EntityConstants.cpp

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2024 iLogtail Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "EntityConstants.h"
+
+namespace logtail {
+
+const std::string DEFAULT_ENV_KEY_HOST_TYPE = "HOST_TYPE";
+const std::string DEFAULT_ENV_VALUE_ECS = "ecs";
+const std::string DEFAULT_ENV_VALUE_HOST = "host";
+const std::string DEFAULT_CONTENT_KEY_ENTITY_TYPE = "__entity_type__";
+const std::string DEFAULT_CONTENT_KEY_ENTITY_ID = "__entity_id__";
+const std::string DEFAULT_CONTENT_KEY_DOMAIN = "__domain__";
+const std::string DEFAULT_CONTENT_VALUE_DOMAIN_ACS = "acs";
+const std::string DEFAULT_CONTENT_VALUE_DOMAIN_INFRA = "infra";
+const std::string DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME = "__first_observed_time__";
+const std::string DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME = "__last_observed_time__";
+const std::string DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS = "__keep_alive_seconds__";
+const std::string DEFAULT_CONTENT_KEY_METHOD = "__method__";
+const std::string DEFAULT_CONTENT_VALUE_METHOD_UPDATE = "update";
+const std::string DEFAULT_CONTENT_VALUE_METHOD_EXPIRE = "expire";
+
+// for process entity
+const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS = "process";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_PID = "process_pid";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_PPID = "process_ppid";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_USER = "process_user";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_COMM = "process_comm";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME = "process_create_time";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_CWD = "process_cwd";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_BINARY = "process_binary";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS = "process_arguments";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE = "process_language";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID = "process_container_id";
+} // namespace logtail