From d8159f7221a4ae7bfae14bb108c973aab9e2513b Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Mon, 13 Nov 2023 01:34:22 -0700 Subject: [PATCH] chore(parsers): Add additional parser benchmarks (#14276) (cherry picked from commit 6fd41686d35110bc7b5611b040a4543428a115b3) --- plugins/parsers/dropwizard/parser_test.go | 60 +++++++++++++++++ .../parsers/form_urlencoded/parser_test.go | 51 +++++++++++++-- plugins/parsers/grok/parser_test.go | 55 ++++++++++++++++ plugins/parsers/logfmt/parser_test.go | 54 ++++++++++++++++ plugins/parsers/nagios/parser_test.go | 50 +++++++++++++++ plugins/parsers/opentsdb/parser_test.go | 48 ++++++++++++++ plugins/parsers/prometheus/parser_test.go | 64 ++++++++++++++++++- plugins/parsers/value/parser_test.go | 32 ++++++++++ plugins/parsers/wavefront/parser_test.go | 50 +++++++++++++++ 9 files changed, 457 insertions(+), 7 deletions(-) diff --git a/plugins/parsers/dropwizard/parser_test.go b/plugins/parsers/dropwizard/parser_test.go index e8eb56c87b28a..b0cf817b99d4d 100644 --- a/plugins/parsers/dropwizard/parser_test.go +++ b/plugins/parsers/dropwizard/parser_test.go @@ -589,3 +589,63 @@ func TestDropWizard(t *testing.T) { }) } } + +const benchmarkData = `{ + "version": "3.0.0", + "gauges" : { + "benchmark,tags_host=myhost,tags_sdkver=3.11.5,tags_platform=python": { + "value": 5.0 + }, + "benchmark,tags_host=myhost,tags_sdkver=3.11.4,tags_platform=python": { + "value": 4.0 + } + } +} +` + +func TestBenchmarkData(t *testing.T) { + plugin := &Parser{} + require.NoError(t, plugin.Init()) + + expected := []telegraf.Metric{ + metric.New( + "benchmark", + map[string]string{ + "metric_type": "gauge", + "tags_host": "myhost", + "tags_platform": "python", + "tags_sdkver": "3.11.5", + }, + map[string]interface{}{ + "value": 5.0, + }, + time.Unix(0, 0), + ), + metric.New( + "benchmark", + map[string]string{ + "metric_type": "gauge", + "tags_host": "myhost", + "tags_platform": "python", + "tags_sdkver": "3.11.4", + }, + map[string]interface{}{ + "value": 4.0, + }, + time.Unix(0, 0), + ), + } + + actual, err := plugin.Parse([]byte(benchmarkData)) + require.NoError(t, err) + testutil.RequireMetricsEqual(t, expected, actual, testutil.IgnoreTime()) +} + +func BenchmarkParsing(b *testing.B) { + plugin := &Parser{} + require.NoError(b, plugin.Init()) + + for n := 0; n < b.N; n++ { + _, _ = plugin.Parse([]byte(benchmarkData)) + } +} diff --git a/plugins/parsers/form_urlencoded/parser_test.go b/plugins/parsers/form_urlencoded/parser_test.go index 45daadad078ee..01819ba024a5b 100644 --- a/plugins/parsers/form_urlencoded/parser_test.go +++ b/plugins/parsers/form_urlencoded/parser_test.go @@ -2,7 +2,11 @@ package form_urlencoded import ( "testing" + "time" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/metric" + "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/require" ) @@ -35,14 +39,14 @@ func TestParseLineValidFormData(t *testing.T) { MetricName: "form_urlencoded_test", } - metric, err := parser.ParseLine(validFormData) + metrics, err := parser.ParseLine(validFormData) require.NoError(t, err) - require.Equal(t, "form_urlencoded_test", metric.Name()) - require.Equal(t, map[string]string{}, metric.Tags()) + require.Equal(t, "form_urlencoded_test", metrics.Name()) + require.Equal(t, map[string]string{}, metrics.Tags()) require.Equal(t, map[string]interface{}{ "field1": float64(42), "field2": float64(69), - }, metric.Fields()) + }, metrics.Fields()) } func TestParseValidFormDataWithTags(t *testing.T) { @@ -170,3 +174,42 @@ func TestParseInvalidFormDataEmptyString(t *testing.T) { require.NoError(t, err) require.Empty(t, metrics) } + +const benchmarkData = `tags_host=myhost&tags_platform=python&tags_sdkver=3.11.5&value=5` + +func TestBenchmarkData(t *testing.T) { + plugin := &Parser{ + MetricName: "benchmark", + TagKeys: []string{"tags_host", "tags_platform", "tags_sdkver"}, + } + + expected := []telegraf.Metric{ + metric.New( + "benchmark", + map[string]string{ + "tags_host": "myhost", + "tags_platform": "python", + "tags_sdkver": "3.11.5", + }, + map[string]interface{}{ + "value": 5.0, + }, + time.Unix(0, 0), + ), + } + + actual, err := plugin.Parse([]byte(benchmarkData)) + require.NoError(t, err) + testutil.RequireMetricsEqual(t, expected, actual, testutil.IgnoreTime()) +} + +func BenchmarkParsing(b *testing.B) { + plugin := &Parser{ + MetricName: "benchmark", + TagKeys: []string{"source", "tags_platform", "tags_sdkver"}, + } + + for n := 0; n < b.N; n++ { + _, _ = plugin.Parse([]byte(benchmarkData)) + } +} diff --git a/plugins/parsers/grok/parser_test.go b/plugins/parsers/grok/parser_test.go index 785ecca24e7f4..3df2c40177c7e 100644 --- a/plugins/parsers/grok/parser_test.go +++ b/plugins/parsers/grok/parser_test.go @@ -1184,3 +1184,58 @@ func TestMultilineNilMetric(t *testing.T) { require.NoError(t, err) require.Empty(t, actual) } + +const benchmarkData = `benchmark 5 1653643421 source=myhost tags_platform=python tags_sdkver=3.11.5 +benchmark 4 1653643422 source=myhost tags_platform=python tags_sdkver=3.11.4 +` + +func TestBenchmarkData(t *testing.T) { + plugin := &Parser{ + //nolint:lll // conditionally long lines allowed + Patterns: []string{"%{WORD:measurement:measurement} %{NUMBER:value:float} %{NUMBER:timestamp:ts-epoch} source=%{WORD:source:tag} tags_platform=%{WORD:tags_platform:tag} tags_sdkver=%{GREEDYDATA:tags_sdkver:tag}"}, + } + require.NoError(t, plugin.Init()) + + expected := []telegraf.Metric{ + metric.New( + "benchmark", + map[string]string{ + "source": "myhost", + "tags_platform": "python", + "tags_sdkver": "3.11.5", + }, + map[string]interface{}{ + "value": 5.0, + }, + time.Unix(1653643421, 0), + ), + metric.New( + "benchmark", + map[string]string{ + "source": "myhost", + "tags_platform": "python", + "tags_sdkver": "3.11.4", + }, + map[string]interface{}{ + "value": 4.0, + }, + time.Unix(1653643422, 0), + ), + } + + actual, err := plugin.Parse([]byte(benchmarkData)) + require.NoError(t, err) + testutil.RequireMetricsEqual(t, expected, actual) +} + +func BenchmarkParsing(b *testing.B) { + plugin := &Parser{ + //nolint:lll // conditionally long lines allowed + Patterns: []string{"%{WORD:measurement:measurement} %{NUMBER:value:float} %{NUMBER:timestamp:ts-epoch} source=%{WORD:source:tag} tags_platform=%{WORD:tags_platform:tag} tags_sdkver=%{GREEDYDATA:tags_sdkver:tag}"}, + } + require.NoError(b, plugin.Init()) + + for n := 0; n < b.N; n++ { + _, _ = plugin.Parse([]byte(benchmarkData)) + } +} diff --git a/plugins/parsers/logfmt/parser_test.go b/plugins/parsers/logfmt/parser_test.go index 8b70291519ce5..f3f1bf155d59a 100644 --- a/plugins/parsers/logfmt/parser_test.go +++ b/plugins/parsers/logfmt/parser_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/require" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/metric" "github.com/influxdata/telegraf/testutil" ) @@ -282,3 +283,56 @@ func TestTags(t *testing.T) { }) } } + +const benchmarkData = `tags_host=myhost tags_platform=python tags_sdkver=3.11.5 value=5 +tags_host=myhost tags_platform=python tags_sdkver=3.11.4 value=4 +` + +func TestBenchmarkData(t *testing.T) { + plugin := &Parser{ + TagKeys: []string{"tags_host", "tags_platform", "tags_sdkver"}, + } + require.NoError(t, plugin.Init()) + + expected := []telegraf.Metric{ + metric.New( + "", + map[string]string{ + "tags_host": "myhost", + "tags_platform": "python", + "tags_sdkver": "3.11.5", + }, + map[string]interface{}{ + "value": 5, + }, + time.Unix(0, 0), + ), + metric.New( + "", + map[string]string{ + "tags_host": "myhost", + "tags_platform": "python", + "tags_sdkver": "3.11.4", + }, + map[string]interface{}{ + "value": 4, + }, + time.Unix(0, 0), + ), + } + + actual, err := plugin.Parse([]byte(benchmarkData)) + require.NoError(t, err) + testutil.RequireMetricsEqual(t, expected, actual, testutil.IgnoreTime()) +} + +func BenchmarkParsing(b *testing.B) { + plugin := &Parser{ + TagKeys: []string{"tags_host", "tags_platform", "tags_sdkver"}, + } + require.NoError(b, plugin.Init()) + + for n := 0; n < b.N; n++ { + _, _ = plugin.Parse([]byte(benchmarkData)) + } +} diff --git a/plugins/parsers/nagios/parser_test.go b/plugins/parsers/nagios/parser_test.go index dd3dd98d51537..23c71d070937b 100644 --- a/plugins/parsers/nagios/parser_test.go +++ b/plugins/parsers/nagios/parser_test.go @@ -524,3 +524,53 @@ func TestParseThreshold(t *testing.T) { require.Equal(t, tests[i].eErr, err) } } + +const benchmarkData = `DISK OK - free space: / 3326 MB (56%); | /=2643MB;5948;5958;0;5968 +/ 15272 MB (77%); +/boot 68 MB (69%); +` + +func TestBenchmarkData(t *testing.T) { + plugin := &Parser{} + + expected := []telegraf.Metric{ + metric.New( + "nagios", + map[string]string{ + "perfdata": "/", + "unit": "MB", + }, + map[string]interface{}{ + "critical_gt": 5958.0, + "critical_lt": 0.0, + "min": 0.0, + "max": 5968.0, + "value": 2643.0, + "warning_gt": 5948.0, + "warning_lt": 0.0, + }, + time.Unix(0, 0), + ), + metric.New( + "nagios_state", + map[string]string{}, + map[string]interface{}{ + "long_service_output": "/ 15272 MB (77%);\n/boot 68 MB (69%);", + "service_output": "DISK OK - free space: / 3326 MB (56%);", + }, + time.Unix(0, 0), + ), + } + + actual, err := plugin.Parse([]byte(benchmarkData)) + require.NoError(t, err) + testutil.RequireMetricsEqual(t, expected, actual, testutil.IgnoreTime()) +} + +func BenchmarkParsing(b *testing.B) { + plugin := &Parser{} + + for n := 0; n < b.N; n++ { + _, _ = plugin.Parse([]byte(benchmarkData)) + } +} diff --git a/plugins/parsers/opentsdb/parser_test.go b/plugins/parsers/opentsdb/parser_test.go index 6c3fea7255d31..e10a9b5af6a4e 100644 --- a/plugins/parsers/opentsdb/parser_test.go +++ b/plugins/parsers/opentsdb/parser_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/metric" "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/require" ) @@ -306,3 +307,50 @@ func TestParse_DefaultTags(t *testing.T) { }) } } + +const benchmarkData = `put benchmark_a 1653643420 4 tags_host=myhost tags_platform=python tags_sdkver=3.11.4 +put benchmark_b 1653643420 5 tags_host=myhost tags_platform=python tags_sdkver=3.11.5 +` + +func TestBenchmarkData(t *testing.T) { + plugin := &Parser{} + + expected := []telegraf.Metric{ + metric.New( + "benchmark_a", + map[string]string{ + "tags_host": "myhost", + "tags_platform": "python", + "tags_sdkver": "3.11.4", + }, + map[string]interface{}{ + "value": 4.0, + }, + time.Unix(1653643420, 0), + ), + metric.New( + "benchmark_b", + map[string]string{ + "tags_host": "myhost", + "tags_platform": "python", + "tags_sdkver": "3.11.5", + }, + map[string]interface{}{ + "value": 5.0, + }, + time.Unix(1653643420, 0), + ), + } + + actual, err := plugin.Parse([]byte(benchmarkData)) + require.NoError(t, err) + testutil.RequireMetricsEqual(t, expected, actual) +} + +func BenchmarkParsing(b *testing.B) { + plugin := &Parser{} + + for n := 0; n < b.N; n++ { + _, _ = plugin.Parse([]byte(benchmarkData)) + } +} diff --git a/plugins/parsers/prometheus/parser_test.go b/plugins/parsers/prometheus/parser_test.go index af9c626d5d220..3c232337ef926 100644 --- a/plugins/parsers/prometheus/parser_test.go +++ b/plugins/parsers/prometheus/parser_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/metric" "github.com/influxdata/telegraf/plugins/parsers/prometheus/common" "github.com/influxdata/telegraf/testutil" ) @@ -390,10 +391,10 @@ test_counter{label="test"} 1 %d ) parser := Parser{IgnoreTimestamp: true} - metric, _ := parser.ParseLine(metricsWithTimestamps) + m, _ := parser.ParseLine(metricsWithTimestamps) - testutil.RequireMetricEqual(t, expected, metric, testutil.IgnoreTime(), testutil.SortMetrics()) - require.WithinDuration(t, time.Now(), metric.Time(), 5*time.Second) + testutil.RequireMetricEqual(t, expected, m, testutil.IgnoreTime(), testutil.SortMetrics()) + require.WithinDuration(t, time.Now(), m.Time(), 5*time.Second) } func parse(buf []byte) ([]telegraf.Metric, error) { @@ -647,3 +648,60 @@ func TestHistogramInfBucketPresence(t *testing.T) { testutil.RequireMetricsEqual(t, expected, metrics, testutil.IgnoreTime(), testutil.SortMetrics()) } + +const benchmarkData = ` +# HELP benchmark_a Test metric for benchmarking +# TYPE benchmark_a gauge +benchmark_a{source="myhost",tags_platform="python",tags_sdkver="3.11.5"} 5 1653643420000 + +# HELP benchmark_b Test metric for benchmarking +# TYPE benchmark_b gauge +benchmark_b{source="myhost",tags_platform="python",tags_sdkver="3.11.4"} 4 1653643420000 +` + +func TestBenchmarkData(t *testing.T) { + plugin := &Parser{ + IgnoreTimestamp: false, + } + + expected := []telegraf.Metric{ + metric.New( + "prometheus", + map[string]string{ + "source": "myhost", + "tags_platform": "python", + "tags_sdkver": "3.11.5", + }, + map[string]interface{}{ + "benchmark_a": 5.0, + }, + time.Unix(1653643420, 0), + telegraf.Gauge, + ), + metric.New( + "prometheus", + map[string]string{ + "source": "myhost", + "tags_platform": "python", + "tags_sdkver": "3.11.4", + }, + map[string]interface{}{ + "benchmark_b": 4.0, + }, + time.Unix(1653643420, 0), + telegraf.Gauge, + ), + } + + actual, err := plugin.Parse([]byte(benchmarkData)) + require.NoError(t, err) + testutil.RequireMetricsEqual(t, expected, actual) +} + +func BenchmarkParsing(b *testing.B) { + plugin := &Parser{} + + for n := 0; n < b.N; n++ { + _, _ = plugin.Parse([]byte(benchmarkData)) + } +} diff --git a/plugins/parsers/value/parser_test.go b/plugins/parsers/value/parser_test.go index 13d45e5b85160..65e9edbc2b0be 100644 --- a/plugins/parsers/value/parser_test.go +++ b/plugins/parsers/value/parser_test.go @@ -4,6 +4,7 @@ import ( "testing" "time" + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/metric" "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/require" @@ -298,3 +299,34 @@ func TestInvalidDatatype(t *testing.T) { } require.ErrorContains(t, parser.Init(), "unknown datatype") } + +const benchmarkData = `5` + +func TestBenchmarkData(t *testing.T) { + plugin := &Parser{} + require.NoError(t, plugin.Init()) + + expected := []telegraf.Metric{ + metric.New( + "", + map[string]string{}, + map[string]interface{}{ + "value": 5, + }, + time.Unix(0, 0), + ), + } + + actual, err := plugin.Parse([]byte(benchmarkData)) + require.NoError(t, err) + testutil.RequireMetricsEqual(t, expected, actual, testutil.IgnoreTime()) +} + +func BenchmarkParsing(b *testing.B) { + plugin := &Parser{} + require.NoError(b, plugin.Init()) + + for n := 0; n < b.N; n++ { + _, _ = plugin.Parse([]byte(benchmarkData)) + } +} diff --git a/plugins/parsers/wavefront/parser_test.go b/plugins/parsers/wavefront/parser_test.go index 503cc1779d1bb..4ae7d7f07420d 100644 --- a/plugins/parsers/wavefront/parser_test.go +++ b/plugins/parsers/wavefront/parser_test.go @@ -8,6 +8,7 @@ import ( "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/metric" + "github.com/influxdata/telegraf/testutil" ) func TestParse(t *testing.T) { @@ -289,3 +290,52 @@ func TestParseDefaultTags(t *testing.T) { ) require.EqualValues(t, parsedMetrics[0], testMetric) } + +const benchmarkData = `benchmark 5 1653643420 source="myhost" tags_platform="python" tags_sdkver="3.11.5" +benchmark 4 1653643420 source="myhost" tags_platform="python" tags_sdkver="3.11.4" +` + +func TestBenchmarkData(t *testing.T) { + plugin := &Parser{} + require.NoError(t, plugin.Init()) + + expected := []telegraf.Metric{ + metric.New( + "benchmark", + map[string]string{ + "source": "myhost", + "tags_platform": "python", + "tags_sdkver": "3.11.5", + }, + map[string]interface{}{ + "value": 5.0, + }, + time.Unix(1653643420, 0), + ), + metric.New( + "benchmark", + map[string]string{ + "source": "myhost", + "tags_platform": "python", + "tags_sdkver": "3.11.4", + }, + map[string]interface{}{ + "value": 4.0, + }, + time.Unix(1653643420, 0), + ), + } + + actual, err := plugin.Parse([]byte(benchmarkData)) + require.NoError(t, err) + testutil.RequireMetricsEqual(t, expected, actual) +} + +func BenchmarkParsing(b *testing.B) { + plugin := &Parser{} + require.NoError(b, plugin.Init()) + + for n := 0; n < b.N; n++ { + _, _ = plugin.Parse([]byte(benchmarkData)) + } +}