diff --git a/pkg/bkmonitorbeat/VERSION b/pkg/bkmonitorbeat/VERSION index 4daf47660..2a8468074 100644 --- a/pkg/bkmonitorbeat/VERSION +++ b/pkg/bkmonitorbeat/VERSION @@ -1 +1 @@ -v3.39.x +v3.40.x diff --git a/pkg/bkmonitorbeat/dockerfile/Dockerfile b/pkg/bkmonitorbeat/dockerfile/Dockerfile index 9ba2038a0..0c94a30d1 100644 --- a/pkg/bkmonitorbeat/dockerfile/Dockerfile +++ b/pkg/bkmonitorbeat/dockerfile/Dockerfile @@ -2,6 +2,7 @@ FROM centos:7 VOLUME /data RUN yum clean all && rm -f /var/lib/rpm/__db* && rpm --rebuilddb +RUN curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.tencent.com/repo/centos7_base.repo RUN yum install -y iproute strace tcpdump RUN mkdir -p /data/bkmonitorbeat/config/child_configs diff --git a/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/collector.go b/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/collector.go index 30a411fad..edb59964c 100644 --- a/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/collector.go +++ b/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/collector.go @@ -17,8 +17,10 @@ import ( "io" "net/http" "os" + "strings" "sync" "time" + "unicode" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" @@ -72,6 +74,7 @@ type MetricSet struct { remoteClient *http.Client workers int disableCustomTimestamp bool + normalizeMetricName bool remoteRelabelCache []*relabel.Config MetricRelabelRemote string MetricRelabelConfigs []*relabel.Config @@ -90,6 +93,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { TempFilePattern string `config:"temp_file_pattern"` Workers int `config:"workers"` DisableCustomTimestamp bool `config:"disable_custom_timestamp"` + NormalizeMetricName bool `config:"normalize_metric_name"` }{} if err := base.Module().UnpackConfig(&config); err != nil { @@ -149,6 +153,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { DimensionReplace: config.DimensionReplace, MetricRelabelConfigs: relabels, disableCustomTimestamp: config.DisableCustomTimestamp, + normalizeMetricName: config.NormalizeMetricName, workers: config.Workers, }, nil } @@ -325,6 +330,10 @@ func (m *MetricSet) getEventsFromReader(metricsReader io.ReadCloser, cleanup fun return eventChan } +func normalizeName(s string) string { + return strings.Join(strings.FieldsFunc(s, func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '_' }), "_") +} + func (m *MetricSet) produceEvents(line string, timestamp int64) ([]common.MapStr, *diffKey, error) { if len(line) <= 0 || line[0] == '#' { return nil, nil, nil @@ -340,6 +349,10 @@ func (m *MetricSet) produceEvents(line string, timestamp int64) ([]common.MapStr return nil, nil, upErr } + if m.normalizeMetricName { + promEvent.Key = normalizeName(promEvent.Key) + } + // 生成事件 var events []common.MapStr event, dk := m.getEventFromPromEvent(&promEvent) diff --git a/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/collector_test.go b/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/collector_test.go index 63be69df2..f77e68624 100644 --- a/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/collector_test.go +++ b/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/collector_test.go @@ -31,14 +31,15 @@ func TestGetEventFromPromEvent(t *testing.T) { } mb := &MetricSet{ - deltaKeys: deltaKeys, - lastDeltaMetrics: lastDeltaMetrics, + deltaKeys: deltaKeys, + lastDeltaMetrics: lastDeltaMetrics, + normalizeMetricName: true, } lines1 := ` metric1{label1="value1"} 10 metric2{label1="value2"} 11 -metric3{label1="value3"} 12 +metric3:foo:bar{label1="value3"} 12 ` ch := mb.getEventsFromReader(io.NopCloser(bytes.NewBufferString(lines1)), func() {}, true) expected := []common.MapStr{ @@ -55,16 +56,16 @@ metric3{label1="value3"} 12 }, }, { - "key": "metric3", + "key": "metric3_foo_bar", "labels": common.MapStr{ "label1": "value3", }, "value": float64(12), }, { - "key": "bkm_gather_up", + "key": "bkm_metricbeat_endpoint_up", "labels": common.MapStr{ - "bkm_up_code": "0", + "code": "0", }, "value": float64(1), }, @@ -109,9 +110,9 @@ metric4label1"value3"} 22 "value": float64(22), }, { - "key": "bkm_gather_up", + "key": "bkm_metricbeat_endpoint_up", "labels": common.MapStr{ - "bkm_up_code": "2502", + "code": "2502", }, "value": float64(1), }, @@ -129,7 +130,6 @@ metric4label1"value3"} 22 failedMetric := newFailReader(define.BeatMetricBeatConnOuterError) ch = mb.getEventsFromReader(failedMetric, func() {}, false) for msg := range ch { - fmt.Printf("aaa: %v", msg) + fmt.Printf("msg: %v", msg) } - } diff --git a/pkg/collector/dockerfile/collector/Dockerfile b/pkg/collector/dockerfile/collector/Dockerfile index 60e67ac9c..4cc3b5d1a 100644 --- a/pkg/collector/dockerfile/collector/Dockerfile +++ b/pkg/collector/dockerfile/collector/Dockerfile @@ -1,8 +1,9 @@ FROM centos:7 VOLUME /data -#RUN yum clean all && rm -f /var/lib/rpm/__db* && rpm --rebuilddb -#RUN yum install -y iproute strace tcpdump +RUN yum clean all && rm -f /var/lib/rpm/__db* && rpm --rebuilddb +RUN curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.tencent.com/repo/centos7_base.repo +RUN yum install -y iproute strace tcpdump RUN mkdir -p /data/hostid && mkdir -p /data/pid && mkdir -p /data/logs && mkdir -p /data/data RUN mkdir -p /data/collector diff --git a/pkg/collector/dockerfile/sidecar/Dockerfile b/pkg/collector/dockerfile/sidecar/Dockerfile index 0fc005644..6fb557eba 100644 --- a/pkg/collector/dockerfile/sidecar/Dockerfile +++ b/pkg/collector/dockerfile/sidecar/Dockerfile @@ -1,8 +1,9 @@ FROM centos:7 VOLUME /data -#RUN yum clean all && rm -f /var/lib/rpm/__db* && rpm --rebuilddb -#RUN yum install -y iproute strace tcpdump +RUN yum clean all && rm -f /var/lib/rpm/__db* && rpm --rebuilddb +RUN curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.tencent.com/repo/centos7_base.repo +RUN yum install -y iproute strace tcpdump RUN mkdir -p /data/sidecar COPY sidecar /data/sidecar/sidecar diff --git a/pkg/collector/internal/testkits/assert.go b/pkg/collector/internal/testkits/assert.go index 63f2edf3e..3b6fc9b0e 100644 --- a/pkg/collector/internal/testkits/assert.go +++ b/pkg/collector/internal/testkits/assert.go @@ -35,6 +35,15 @@ func AssertAttrsFoundStringVal(t *testing.T, attrs pcommon.Map, key, val string) assert.Equal(t, val, v.AsString()) } +func AssertAttrsStringVal(t *testing.T, attrs pcommon.Map, key, val string) { + var s string + v, ok := attrs.Get(key) + if ok { + s = v.AsString() + } + assert.Equal(t, val, s) +} + func AssertAttrsFoundIntVal(t *testing.T, attrs pcommon.Map, key string, val int) { v, ok := attrs.Get(key) assert.True(t, ok) diff --git a/pkg/collector/processor/servicediscover/factory_test.go b/pkg/collector/processor/servicediscover/factory_test.go index 5ebefc06c..765fe724c 100644 --- a/pkg/collector/processor/servicediscover/factory_test.go +++ b/pkg/collector/processor/servicediscover/factory_test.go @@ -394,9 +394,24 @@ processor: testCases := []struct { name string httpUrl string + attrs map[string]string }{ - {"MatchSuccess", "http://example:19100/benchmark/2024"}, - {"MatchFailed", "http://example:19100/benchmark-1/2024"}, + { + name: "Success", + httpUrl: "http://example:19100/benchmark/2024", + attrs: map[string]string{ + "http.route": "GET:/benchmark/{uuid}", + "peer.service": "example:19100", + }, + }, + { + name: "Failed", + httpUrl: "http://example:19100/benchmark-1/2024", + attrs: map[string]string{ + "http.route": "", + "peer.service": "", + }, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -421,19 +436,12 @@ processor: data = record.Data.(ptrace.Traces) foreach.Spans(data.ResourceSpans(), func(span ptrace.Span) { - if tc.name == "MatchSuccess" { - testkits.AssertAttrsFoundStringVal(t, span.Attributes(), "http.route", "GET:/benchmark/{uuid}") - testkits.AssertAttrsFoundStringVal(t, span.Attributes(), "peer.service", "example:19100") - } - if tc.name == "MatchFailed" { - testkits.AssertAttrsNotFound(t, span.Attributes(), "http.route") - testkits.AssertAttrsNotFound(t, span.Attributes(), "peer.service") + for k, v := range tc.attrs { + testkits.AssertAttrsStringVal(t, span.Attributes(), k, v) } }) }) - } - } func TestTracesRegexMatchedWithSpanName(t *testing.T) { diff --git a/pkg/operator/common/kits/kits.go b/pkg/operator/common/kits/kits.go index 2a393c374..990556f48 100644 --- a/pkg/operator/common/kits/kits.go +++ b/pkg/operator/common/kits/kits.go @@ -29,3 +29,7 @@ func CheckIfSystemResource(items map[string]string) bool { func CheckIfForwardLocalhost(items map[string]string) bool { return stringToBool(items, "forwardLocalhost") } + +func CheckIfNormalizeMetricName(items map[string]string) bool { + return stringToBool(items, "normalizeMetricName") +} diff --git a/pkg/operator/dockerfile/operator/Dockerfile b/pkg/operator/dockerfile/operator/Dockerfile index 592fed7d7..7889ec873 100644 --- a/pkg/operator/dockerfile/operator/Dockerfile +++ b/pkg/operator/dockerfile/operator/Dockerfile @@ -2,6 +2,7 @@ FROM centos:7 VOLUME /data RUN yum clean all && rm -f /var/lib/rpm/__db* && rpm --rebuilddb +RUN curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.tencent.com/repo/centos7_base.repo RUN yum install -y iproute strace tcpdump RUN mkdir -p /data/bkmonitor-operator/config diff --git a/pkg/operator/dockerfile/reloader/Dockerfile b/pkg/operator/dockerfile/reloader/Dockerfile index 8491e70e3..99b9f86cb 100644 --- a/pkg/operator/dockerfile/reloader/Dockerfile +++ b/pkg/operator/dockerfile/reloader/Dockerfile @@ -2,6 +2,7 @@ FROM centos:7 VOLUME /data RUN yum clean all && rm -f /var/lib/rpm/__db* && rpm --rebuilddb +RUN curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.tencent.com/repo/centos7_base.repo RUN yum install -y iproute strace tcpdump RUN mkdir -p /data/reloader/config diff --git a/pkg/operator/operator/discover/discover.go b/pkg/operator/operator/discover/discover.go index 4439b855b..ca86142e0 100644 --- a/pkg/operator/operator/discover/discover.go +++ b/pkg/operator/operator/discover/discover.go @@ -124,6 +124,7 @@ type BaseParams struct { Client kubernetes.Interface RelabelRule string RelabelIndex string + NormalizeMetricName bool Name string KubeConfig string Namespaces []string @@ -408,6 +409,7 @@ func (d *BaseDiscover) makeMetricTarget(lbls, origLabels labels.Labels, namespac metricTarget.TaskType = taskType metricTarget.RelabelRule = d.RelabelRule metricTarget.RelabelIndex = d.RelabelIndex + metricTarget.NormalizeMetricName = d.NormalizeMetricName return metricTarget, nil } diff --git a/pkg/operator/operator/operator.go b/pkg/operator/operator/operator.go index a6b0294c4..c65c0e649 100644 --- a/pkg/operator/operator/operator.go +++ b/pkg/operator/operator/operator.go @@ -567,6 +567,7 @@ func (c *Operator) createServiceMonitorDiscovers(serviceMonitor *promv1.ServiceM Client: c.client, RelabelRule: serviceMonitor.Annotations[annotationRelabelRule], RelabelIndex: serviceMonitor.Annotations[annotationRelabelIndex], + NormalizeMetricName: kits.CheckIfNormalizeMetricName(serviceMonitor.Annotations), AnnotationMatchSelector: parseSelector(serviceMonitor.Annotations[annotationMonitorMatchSelector]), AnnotationDropSelector: parseSelector(serviceMonitor.Annotations[annotationMonitorDropSelector]), Name: monitorMeta.ID(), @@ -770,6 +771,7 @@ func (c *Operator) createPodMonitorDiscovers(podMonitor *promv1.PodMonitor) []di Client: c.client, RelabelRule: podMonitor.Annotations[annotationRelabelRule], RelabelIndex: podMonitor.Annotations[annotationRelabelIndex], + NormalizeMetricName: kits.CheckIfNormalizeMetricName(podMonitor.Annotations), AnnotationMatchSelector: parseSelector(podMonitor.Annotations[annotationMonitorMatchSelector]), AnnotationDropSelector: parseSelector(podMonitor.Annotations[annotationMonitorDropSelector]), Name: monitorMeta.ID(), diff --git a/pkg/operator/operator/target/metric.go b/pkg/operator/operator/target/metric.go index da9806e00..52dc3c183 100644 --- a/pkg/operator/operator/target/metric.go +++ b/pkg/operator/operator/target/metric.go @@ -56,6 +56,7 @@ type MetricTarget struct { Meta define.MonitorMeta RelabelRule string RelabelIndex string + NormalizeMetricName bool Address string NodeName string Scheme string @@ -149,6 +150,7 @@ func (t *MetricTarget) YamlBytes() ([]byte, error) { module = append(module, *remoteRelabel) } module = append(module, yaml.MapItem{Key: "disable_custom_timestamp", Value: t.DisableCustomTimestamp}) + module = append(module, yaml.MapItem{Key: "normalize_metric_name", Value: t.NormalizeMetricName}) address := t.Address if !strings.HasPrefix(address, "http://") && !strings.HasPrefix(address, "https://") {