feat: support host monitor

core/CMakeLists.txt

@@ -122,6 +122,7 @@ set(SUB_DIRECTORIES_LIST
         prometheus prometheus/labels prometheus/schedulers prometheus/async
         ebpf ebpf/observer ebpf/security ebpf/handler
         parser sls_control sdk
+        host_monitor host_monitor/collector
         )
 if (LINUX)
     if (ENABLE_ENTERPRISE)

core/app_config/AppConfig.cpp

@@ -159,6 +159,7 @@ DEFINE_FLAG_STRING(loong_collector_operator_service, "loong collector operator s
 DEFINE_FLAG_INT32(loong_collector_operator_service_port, "loong collector operator service port", 8888);
 DEFINE_FLAG_INT32(loong_collector_k8s_meta_service_port, "loong collector operator service port", 9000);
 DEFINE_FLAG_STRING(_pod_name_, "agent pod name", "");
+DEFINE_FLAG_INT32(process_collect_silent_count, "number of process scanned between a sleep", 1000);
 
 DEFINE_FLAG_STRING(app_info_file, "", "app_info.json");
 DEFINE_FLAG_STRING(crash_stack_file_name, "crash stack back trace file name", "backtrace.dat");

core/common/FileSystemUtil.cpp

@@ -141,6 +141,50 @@ bool ReadFileContent(const std::string& fileName, std::string& content, uint32_t
     return true;
 }
 
+int GetLines(std::istream& is,
+             bool enableEmptyLine,
+             const std::function<void(const std::string&)>& pushBack,
+             std::string* errorMessage) {
+    std::string line;
+    // 此处必须判断eof，具体原因参见:
+    // https://stackoverflow.com/questions/40561482/getline-throws-basic-iosclear-exception-after-reading-the-last-line
+    while (!is.eof() && std::getline(is, line)) {
+        if (enableEmptyLine || !line.empty()) {
+            pushBack(line);
+        }
+    }
+    return 0;
+}
+
+int GetLines(const bfs::path& filename,
+             bool enableEmptyLine,
+             const std::function<void(const std::string&)>& pushBack,
+             std::string* errorMessage) {
+    int ret = 0;
+    std::ifstream fin;
+    try {
+        fin.exceptions(std::ifstream::failbit | std::ifstream::badbit);
+        fin.open(filename.string(), std::ios_base::in);
+        fin.exceptions(std::ifstream::goodbit);
+        GetLines(fin, enableEmptyLine, pushBack, errorMessage);
+        fin.close();
+    } catch (const std::exception& fail) {
+        if (errorMessage != nullptr) {
+            LOG_ERROR(sLogger, ("open file fail", filename)("errno", strerror(errno)));
+            ret = -1;
+        }
+        fin.close();
+    }
+    return ret;
+}
+
+int GetFileLines(const bfs::path& filename,
+                 std::vector<std::string>& res,
+                 bool enableEmptyLine,
+                 std::string* errorMessage) {
+    return GetLines(filename, enableEmptyLine, [&res](const std::string& s) { res.push_back(s); }, errorMessage);
+}
+
 bool OverwriteFile(const std::string& fileName, const std::string& content) {
     FILE* pFile = fopen(fileName.c_str(), "w");
     if (pFile == NULL) {

core/common/FileSystemUtil.h

@@ -26,10 +26,14 @@
 #elif defined(_MSC_VER)
 #include <Windows.h>
 #endif
+#include <boost/filesystem.hpp>
+
 #include "DevInode.h"
 #include "ErrorUtil.h"
 #include "LogtailCommonFlags.h"
 
+namespace bfs = boost::filesystem;
+
 // Filesystem utility.
 namespace logtail {
 
@@ -87,6 +91,19 @@ void TrimLastSeperator(std::string& path);
 // ReadFileContent reads all content of @fileName to @content.
 bool ReadFileContent(const std::string& fileName, std::string& content, uint32_t maxFileSize = 8192);
 
+int GetLines(std::istream& is,
+             bool enableEmptyLine,
+             const std::function<void(const std::string&)>& pushBack,
+             std::string* errorMessage);
+int GetLines(const bfs::path& filename,
+             bool enableEmptyLine,
+             const std::function<void(const std::string&)>& pushBack,
+             std::string* errorMessage);
+int GetFileLines(const bfs::path& filename,
+                 std::vector<std::string>& res,
+                 bool enableEmptyLine = true,
+                 std::string* errorMessage = nullptr);
+
 // OverwriteFile overwrides @fileName with @content.
 bool OverwriteFile(const std::string& fileName, const std::string& content);
 

core/common/StringTools.cpp

@@ -363,4 +363,12 @@ void RemoveFilePathTrailingSlash(std::string& filePath) {
     filePath = path.string();
 }
 
+bool IsInt(const char* sz) {
+    bool ok = (sz != nullptr && *sz != '\0');
+    for (auto* it = reinterpret_cast<const unsigned char*>(sz); ok && *it; ++it) {
+        ok = (0 != std::isdigit(*it));
+    }
+    return ok;
+}
+
 } // namespace logtail

core/common/StringTools.h

@@ -146,6 +146,12 @@ bool NormalizeTopicRegFormat(std::string& regStr);
 
 void RemoveFilePathTrailingSlash(std::string& path);
 
+bool IsInt(const char* sz);
+
+inline bool IsInt(const std::string& str) {
+    return IsInt(str.c_str());
+}
+
 #if defined(_MSC_VER)
 // TODO: Test it.
 #define FNM_PATHNAME 0

core/common/common.cmake

@@ -29,7 +29,7 @@ list(APPEND THIS_SOURCE_FILES_LIST ${XX_HASH_SOURCE_FILES})
 # add memory in common
 list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/memory/SourceBuffer.h)
 list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/http/AsynCurlRunner.cpp ${CMAKE_SOURCE_DIR}/common/http/Curl.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpResponse.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpRequest.cpp)
-list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/timer/Timer.cpp ${CMAKE_SOURCE_DIR}/common/timer/HttpRequestTimerEvent.cpp)
+list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/timer/Timer.cpp ${CMAKE_SOURCE_DIR}/common/timer/HttpRequestTimerEvent.cpp ${CMAKE_SOURCE_DIR}/common/timer/HostMonitorTimerEvent.cpp)
 list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/compression/Compressor.cpp ${CMAKE_SOURCE_DIR}/common/compression/CompressorFactory.cpp ${CMAKE_SOURCE_DIR}/common/compression/LZ4Compressor.cpp ${CMAKE_SOURCE_DIR}/common/compression/ZstdCompressor.cpp)
 # remove several files in common
 list(REMOVE_ITEM THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/BoostRegexValidator.cpp ${CMAKE_SOURCE_DIR}/common/GetUUID.cpp)

core/common/timer/HostMonitorTimerEvent.cpp

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2024 iLogtail Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "HostMonitorTimerEvent.h"
+
+#include "HostMonitorInputRunner.h"
+
+namespace logtail {
+
+bool HostMonitorTimerEvent::IsValid() const {
+    return HostMonitorInputRunner::GetInstance()->IsCollectTaskValid(mConfigName, mCollectorName);
+}
+
+bool HostMonitorTimerEvent::Execute() {
+    HostMonitorInputRunner::GetInstance()->ScheduleOnce(this);
+    return true;
+}
+
+} // namespace logtail

core/common/timer/HostMonitorTimerEvent.h

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2024 iLogtail Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <chrono>
+#include <string>
+
+#include "QueueKey.h"
+#include "timer/TimerEvent.h"
+
+namespace logtail {
+
+class HostMonitorTimerEvent : public TimerEvent {
+public:
+    HostMonitorTimerEvent(std::chrono::steady_clock::time_point execTime,
+                          size_t interval,
+                          std::string configName,
+                          std::string collectorName,
+                          QueueKey processQueueKey)
+        : TimerEvent(execTime),
+          mConfigName(std::move(configName)),
+          mCollectorName(collectorName),
+          mProcessQueueKey(processQueueKey),
+          mInputIdx(0) {
+        mInterval = std::chrono::seconds(interval);
+    }
+
+    bool IsValid() const override;
+    bool Execute() override;
+
+    const std::string GetConfigName() const { return mConfigName; }
+    const std::string GetCollectorName() const { return mCollectorName; }
+    const QueueKey GetProcessQueueKey() const { return mProcessQueueKey; }
+    int GetInputIndex() const { return mInputIdx; }
+    const std::chrono::seconds GetInterval() const { return mInterval; }
+    void ResetForNextExec() { SetExecTime(GetExecTime() + mInterval); }
+
+private:
+    std::string mConfigName;
+    std::string mCollectorName;
+    QueueKey mProcessQueueKey;
+    int mInputIdx;
+    std::chrono::seconds mInterval;
+};
+
+} // namespace logtail

core/common/timer/Timer.h

@@ -53,6 +53,7 @@ class Timer {
 #ifdef APSARA_UNIT_TEST_MAIN
     friend class TimerUnittest;
     friend class ScrapeSchedulerUnittest;
+    friend class HostMonitorInputRunnerUnittest;
 #endif
 };
 

core/common/timer/TimerEvent.h

@@ -29,6 +29,7 @@ class TimerEvent {
     virtual bool Execute() = 0;
 
     std::chrono::steady_clock::time_point GetExecTime() const { return mExecTime; }
+    void SetExecTime(std::chrono::steady_clock::time_point nextExecTime) { mExecTime = nextExecTime; }
 
 private:
     std::chrono::steady_clock::time_point mExecTime;

core/constants/EntityConstants.cpp

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2024 iLogtail Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "EntityConstants.h"
+
+namespace logtail {
+
+const std::string DEFAULT_ENV_KEY_HOST_TYPE = "HOST_TYPE";
+const std::string DEFAULT_ENV_VALUE_ECS = "ecs";
+const std::string DEFAULT_ENV_VALUE_HOST = "host";
+const std::string DEFAULT_CONTENT_KEY_ENTITY_TYPE = "__entity_type__";
+const std::string DEFAULT_CONTENT_KEY_ENTITY_ID = "__entity_id__";
+const std::string DEFAULT_CONTENT_KEY_DOMAIN = "__domain__";
+const std::string DEFAULT_CONTENT_VALUE_DOMAIN_ACS = "acs";
+const std::string DEFAULT_CONTENT_VALUE_DOMAIN_INFRA = "infra";
+const std::string DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME = "__first_observed_time__";
+const std::string DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME = "__last_observed_time__";
+const std::string DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS = "__keep_alive_seconds__";
+const std::string DEFAULT_CONTENT_KEY_METHOD = "__method__";
+const std::string DEFAULT_CONTENT_VALUE_METHOD_UPDATE = "update";
+const std::string DEFAULT_CONTENT_VALUE_METHOD_EXPIRE = "expire";
+
+// for process entity
+const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS = "process";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_PID = "process_pid";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_PPID = "process_ppid";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_USER = "process_user";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_COMM = "process_comm";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME = "process_create_time";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_CWD = "process_cwd";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_BINARY = "process_binary";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS = "process_arguments";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE = "process_language";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID = "process_container_id";
+} // namespace logtail

core/constants/EntityConstants.h

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2024 iLogtail Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string>
+
+namespace logtail {
+
+extern const std::string DEFAULT_ENV_KEY_HOST_TYPE;
+extern const std::string DEFAULT_ENV_VALUE_ECS;
+extern const std::string DEFAULT_ENV_VALUE_HOST;
+extern const std::string DEFAULT_CONTENT_KEY_ENTITY_TYPE;
+extern const std::string DEFAULT_CONTENT_KEY_ENTITY_ID;
+extern const std::string DEFAULT_CONTENT_KEY_DOMAIN;
+extern const std::string DEFAULT_CONTENT_VALUE_DOMAIN_ACS;
+extern const std::string DEFAULT_CONTENT_VALUE_DOMAIN_INFRA;
+extern const std::string DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME;
+extern const std::string DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME;
+extern const std::string DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS;
+extern const std::string DEFAULT_CONTENT_KEY_METHOD;
+extern const std::string DEFAULT_CONTENT_VALUE_METHOD_UPDATE;
+extern const std::string DEFAULT_CONTENT_VALUE_METHOD_EXPIRE;
+
+// for process entity
+extern const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_PID;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_PPID;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_USER;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_COMM;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_CWD;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_BINARY;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID;
+} // namespace logtail

core/host_monitor/Constants.cpp

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2024 iLogtail Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Constants.h"
+
+
+namespace logtail {
+
+#ifndef APSARA_UNIT_TEST_MAIN
+const bfs::path PROCESS_DIR = "/proc";
+#else
+bfs::path PROCESS_DIR = "/proc";
+#endif
+
+const bfs::path PROCESS_STAT = "stat";
+
+} // namespace logtail