feat: 采集器 corefile 支持正则匹配 --story=116561788 (#250)

TencentBlueKing · Apr 2, 2024 · 563b187 · 563b187
1 parent 0bdaf01
commit 563b187
Show file tree

Hide file tree

Showing 19 changed files with 198 additions and 105 deletions.
diff --git a/pkg/bkmonitorbeat/VERSION b/pkg/bkmonitorbeat/VERSION
@@ -1 +1 @@
-v3.29.x
+v3.30.x
diff --git a/pkg/bkmonitorbeat/configs/exceptionbeat.go b/pkg/bkmonitorbeat/configs/exceptionbeat.go
@@ -38,6 +38,7 @@ type ExceptionBeatConfig struct {
 	DiskRoBlackList        []string      `config:"disk_ro_black_list"`
 	CoreFileReportGap      time.Duration `config:"corefile_report_gap"`
 	CoreFilePattern        string        `config:"corefile_pattern"`
+	CoreFileMatchRegex     string        `config:"corefile_match_regex"`
 }
 
 var DefaultExceptionBeatConfig = ExceptionBeatConfig{

diff --git a/pkg/bkmonitorbeat/script/generate_support_files/write_project_yaml.sh b/pkg/bkmonitorbeat/script/generate_support_files/write_project_yaml.sh
@@ -59,8 +59,11 @@ config_templates:
               items:
                 title: interface
                 type: string
-            corefile_pattern_path:
-              title: corefile_pattern_path
+            corefile_pattern:
+              title: corefile_pattern
+              type: string
+            corefile_match_regex:
+              title: corefile_match_regex
               type: string
   - plugin_version: "*"
     name: bkmonitorbeat_prometheus.conf

diff --git a/pkg/bkmonitorbeat/script/generate_support_files/write_root_conf_tpl.sh b/pkg/bkmonitorbeat/script/generate_support_files/write_root_conf_tpl.sh
@@ -204,8 +204,11 @@ EOF
     check_oom_interval: 10
     used_max_disk_space_percent: 95
     free_min_disk_space: 10
-{%- if extra_vars is defined and extra_vars.corefile_pattern_path is defined %}
-    corefile_pattern: {{ extra_vars.corefile_pattern_path or '' }}
+{%- if extra_vars is defined and extra_vars.corefile_pattern is defined %}
+    corefile_pattern: {{ extra_vars.corefile_pattern or '' }}
+{%- endif %}
+{%- if extra_vars is defined and extra_vars.corefile_match_regex is defined %}
+    corefile_match_regex: {{ extra_vars.corefile_match_regex or '' }}
 {%- endif %}
 EOF
   cat <<EOF >> "$path"

diff --git a/pkg/bkmonitorbeat/support-files/templates/freebsd/x86_64/etc/bkmonitorbeat.conf.tpl b/pkg/bkmonitorbeat/support-files/templates/freebsd/x86_64/etc/bkmonitorbeat.conf.tpl
@@ -129,8 +129,11 @@ bkmonitorbeat:
     check_oom_interval: 10
     used_max_disk_space_percent: 95
     free_min_disk_space: 10
-{%- if extra_vars is defined and extra_vars.corefile_pattern_path is defined %}
-    corefile_pattern: {{ extra_vars.corefile_pattern_path or '' }}
+{%- if extra_vars is defined and extra_vars.corefile_pattern is defined %}
+    corefile_pattern: {{ extra_vars.corefile_pattern or '' }}
+{%- endif %}
+{%- if extra_vars is defined and extra_vars.corefile_match_regex is defined %}
+    corefile_match_regex: {{ extra_vars.corefile_match_regex or '' }}
 {%- endif %}
   # 进程采集：同步 CMDB 进程配置文件到 bkmonitorbeat 子任务文件夹下
   procconf_task:

diff --git a/pkg/bkmonitorbeat/support-files/templates/freebsd/x86_64/project.yaml b/pkg/bkmonitorbeat/support-files/templates/freebsd/x86_64/project.yaml
@@ -31,8 +31,11 @@ config_templates:
               items:
                 title: interface
                 type: string
-            corefile_pattern_path:
-              title: corefile_pattern_path
+            corefile_pattern:
+              title: corefile_pattern
+              type: string
+            corefile_match_regex:
+              title: corefile_match_regex
               type: string
   - plugin_version: "*"
     name: bkmonitorbeat_prometheus.conf

diff --git a/pkg/bkmonitorbeat/support-files/templates/linux/aarch64/etc/bkmonitorbeat.conf.tpl b/pkg/bkmonitorbeat/support-files/templates/linux/aarch64/etc/bkmonitorbeat.conf.tpl
@@ -129,8 +129,11 @@ bkmonitorbeat:
     check_oom_interval: 10
     used_max_disk_space_percent: 95
     free_min_disk_space: 10
-{%- if extra_vars is defined and extra_vars.corefile_pattern_path is defined %}
-    corefile_pattern: {{ extra_vars.corefile_pattern_path or '' }}
+{%- if extra_vars is defined and extra_vars.corefile_pattern is defined %}
+    corefile_pattern: {{ extra_vars.corefile_pattern or '' }}
+{%- endif %}
+{%- if extra_vars is defined and extra_vars.corefile_match_regex is defined %}
+    corefile_match_regex: {{ extra_vars.corefile_match_regex or '' }}
 {%- endif %}
   # 进程采集：同步 CMDB 进程配置文件到 bkmonitorbeat 子任务文件夹下
   procconf_task:

diff --git a/pkg/bkmonitorbeat/support-files/templates/linux/aarch64/project.yaml b/pkg/bkmonitorbeat/support-files/templates/linux/aarch64/project.yaml
@@ -31,8 +31,11 @@ config_templates:
               items:
                 title: interface
                 type: string
-            corefile_pattern_path:
-              title: corefile_pattern_path
+            corefile_pattern:
+              title: corefile_pattern
+              type: string
+            corefile_match_regex:
+              title: corefile_match_regex
               type: string
   - plugin_version: "*"
     name: bkmonitorbeat_prometheus.conf

diff --git a/pkg/bkmonitorbeat/support-files/templates/linux/x86/etc/bkmonitorbeat.conf.tpl b/pkg/bkmonitorbeat/support-files/templates/linux/x86/etc/bkmonitorbeat.conf.tpl
@@ -129,8 +129,11 @@ bkmonitorbeat:
     check_oom_interval: 10
     used_max_disk_space_percent: 95
     free_min_disk_space: 10
-{%- if extra_vars is defined and extra_vars.corefile_pattern_path is defined %}
-    corefile_pattern: {{ extra_vars.corefile_pattern_path or '' }}
+{%- if extra_vars is defined and extra_vars.corefile_pattern is defined %}
+    corefile_pattern: {{ extra_vars.corefile_pattern or '' }}
+{%- endif %}
+{%- if extra_vars is defined and extra_vars.corefile_match_regex is defined %}
+    corefile_match_regex: {{ extra_vars.corefile_match_regex or '' }}
 {%- endif %}
   # 进程采集：同步 CMDB 进程配置文件到 bkmonitorbeat 子任务文件夹下
   procconf_task:

diff --git a/pkg/bkmonitorbeat/support-files/templates/linux/x86/project.yaml b/pkg/bkmonitorbeat/support-files/templates/linux/x86/project.yaml
@@ -31,8 +31,11 @@ config_templates:
               items:
                 title: interface
                 type: string
-            corefile_pattern_path:
-              title: corefile_pattern_path
+            corefile_pattern:
+              title: corefile_pattern
+              type: string
+            corefile_match_regex:
+              title: corefile_match_regex
               type: string
   - plugin_version: "*"
     name: bkmonitorbeat_prometheus.conf

diff --git a/pkg/bkmonitorbeat/support-files/templates/linux/x86_64/etc/bkmonitorbeat.conf.tpl b/pkg/bkmonitorbeat/support-files/templates/linux/x86_64/etc/bkmonitorbeat.conf.tpl
@@ -129,8 +129,11 @@ bkmonitorbeat:
     check_oom_interval: 10
     used_max_disk_space_percent: 95
     free_min_disk_space: 10
-{%- if extra_vars is defined and extra_vars.corefile_pattern_path is defined %}
-    corefile_pattern: {{ extra_vars.corefile_pattern_path or '' }}
+{%- if extra_vars is defined and extra_vars.corefile_pattern is defined %}
+    corefile_pattern: {{ extra_vars.corefile_pattern or '' }}
+{%- endif %}
+{%- if extra_vars is defined and extra_vars.corefile_match_regex is defined %}
+    corefile_match_regex: {{ extra_vars.corefile_match_regex or '' }}
 {%- endif %}
   # 进程采集：同步 CMDB 进程配置文件到 bkmonitorbeat 子任务文件夹下
   procconf_task:

diff --git a/pkg/bkmonitorbeat/support-files/templates/linux/x86_64/project.yaml b/pkg/bkmonitorbeat/support-files/templates/linux/x86_64/project.yaml
@@ -31,8 +31,11 @@ config_templates:
               items:
                 title: interface
                 type: string
-            corefile_pattern_path:
-              title: corefile_pattern_path
+            corefile_pattern:
+              title: corefile_pattern
+              type: string
+            corefile_match_regex:
+              title: corefile_match_regex
               type: string
   - plugin_version: "*"
     name: bkmonitorbeat_prometheus.conf

diff --git a/pkg/bkmonitorbeat/support-files/templates/windows/x86/etc/bkmonitorbeat.conf.tpl b/pkg/bkmonitorbeat/support-files/templates/windows/x86/etc/bkmonitorbeat.conf.tpl
@@ -129,8 +129,11 @@ bkmonitorbeat:
     check_oom_interval: 10
     used_max_disk_space_percent: 95
     free_min_disk_space: 10
-{%- if extra_vars is defined and extra_vars.corefile_pattern_path is defined %}
-    corefile_pattern: {{ extra_vars.corefile_pattern_path or '' }}
+{%- if extra_vars is defined and extra_vars.corefile_pattern is defined %}
+    corefile_pattern: {{ extra_vars.corefile_pattern or '' }}
+{%- endif %}
+{%- if extra_vars is defined and extra_vars.corefile_match_regex is defined %}
+    corefile_match_regex: {{ extra_vars.corefile_match_regex or '' }}
 {%- endif %}
   # 进程采集：同步 CMDB 进程配置文件到 bkmonitorbeat 子任务文件夹下
   procconf_task:

diff --git a/pkg/bkmonitorbeat/support-files/templates/windows/x86/project.yaml b/pkg/bkmonitorbeat/support-files/templates/windows/x86/project.yaml
@@ -31,8 +31,11 @@ config_templates:
               items:
                 title: interface
                 type: string
-            corefile_pattern_path:
-              title: corefile_pattern_path
+            corefile_pattern:
+              title: corefile_pattern
+              type: string
+            corefile_match_regex:
+              title: corefile_match_regex
               type: string
   - plugin_version: "*"
     name: bkmonitorbeat_prometheus.conf

diff --git a/pkg/bkmonitorbeat/support-files/templates/windows/x86_64/etc/bkmonitorbeat.conf.tpl b/pkg/bkmonitorbeat/support-files/templates/windows/x86_64/etc/bkmonitorbeat.conf.tpl
@@ -129,8 +129,11 @@ bkmonitorbeat:
     check_oom_interval: 10
     used_max_disk_space_percent: 95
     free_min_disk_space: 10
-{%- if extra_vars is defined and extra_vars.corefile_pattern_path is defined %}
-    corefile_pattern: {{ extra_vars.corefile_pattern_path or '' }}
+{%- if extra_vars is defined and extra_vars.corefile_pattern is defined %}
+    corefile_pattern: {{ extra_vars.corefile_pattern or '' }}
+{%- endif %}
+{%- if extra_vars is defined and extra_vars.corefile_match_regex is defined %}
+    corefile_match_regex: {{ extra_vars.corefile_match_regex or '' }}
 {%- endif %}
   # 进程采集：同步 CMDB 进程配置文件到 bkmonitorbeat 子任务文件夹下
   procconf_task:

diff --git a/pkg/bkmonitorbeat/support-files/templates/windows/x86_64/project.yaml b/pkg/bkmonitorbeat/support-files/templates/windows/x86_64/project.yaml
@@ -31,8 +31,11 @@ config_templates:
               items:
                 title: interface
                 type: string
-            corefile_pattern_path:
-              title: corefile_pattern_path
+            corefile_pattern:
+              title: corefile_pattern
+              type: string
+            corefile_match_regex:
+              title: corefile_match_regex
               type: string
   - plugin_version: "*"
     name: bkmonitorbeat_prometheus.conf

diff --git a/pkg/bkmonitorbeat/tasks/exceptionbeat/collector/corefile/collector_unix.go b/pkg/bkmonitorbeat/tasks/exceptionbeat/collector/corefile/collector_unix.go
@@ -14,6 +14,7 @@ package corefile
 
 import (
 	"context"
+	"regexp"
 	"time"
 
 	"github.com/fsnotify/fsnotify"
@@ -41,6 +42,7 @@ type CoreFileCollector struct {
 	done                    chan bool
 	state                   int
 	coreFilePattern         string
+	matchRegx               *regexp.Regexp
 	corePath                string
 	pattern                 string
 	patternArr              [][]string
@@ -78,6 +80,14 @@ func (c *CoreFileCollector) Start(ctx context.Context, e chan<- define.Event, co
 	}
 	c.reportTimeInfo = make(map[string]*ReportInfo)
 	c.coreFilePattern = conf.CoreFilePattern
+	if conf.CoreFileMatchRegex != "" {
+		r, err := regexp.Compile(conf.CoreFileMatchRegex)
+		if err != nil {
+			logger.Errorf("faield to compile regex pattern(%s), err: %v", conf.CoreFileMatchRegex, err)
+		} else {
+			c.matchRegx = r
+		}
+	}
 
 	logger.Infof("CoreFileColletor start success with config data_id->[%d] report_gap->[%s]", c.dataid, c.reportTimeGap)
 	go c.statistic(ctx, e)

diff --git a/pkg/bkmonitorbeat/tasks/exceptionbeat/collector/corefile/utils_unix.go b/pkg/bkmonitorbeat/tasks/exceptionbeat/collector/corefile/utils_unix.go
@@ -488,9 +488,20 @@ func (c *CoreFileCollector) parseDimensions(groups []regexGroup) beat.MapStr {
 	return dimensions
 }
 
+func (c *CoreFileCollector) fillDimension(filePath string) (beat.MapStr, bool) {
+	m, ok := c.fillDimensionV0(filePath)
+	// 正则匹配模式下，不需要关心是否 dimensions 能否匹配到
+	if c.matchRegx != nil {
+		matched := c.matchRegx.MatchString(filePath)
+		return m, matched
+	}
+
+	return m, ok
+}
+
 // fillDimension: 填充维度信息到dimensions当中，如果解析失败，那么直接返回dimensions，不对其中的任何内容进行修改
 // 返回内容表示是否可以按照正则正常解析；如果正则解析失败的，很可能是用户自己瞎写的文件，不应该触发告警
-func (c *CoreFileCollector) fillDimension(filePath string) (beat.MapStr, bool) {
+func (c *CoreFileCollector) fillDimensionV0(filePath string) (beat.MapStr, bool) {
 
 	// 获取core file文件名
 	fileName, errFileName := c.getCoreFileName(filePath)