diff --git a/.chloggen/clickhouse-add-client-info.yaml b/.chloggen/clickhouse-add-client-info.yaml new file mode 100644 index 000000000000..ccc37300811f --- /dev/null +++ b/.chloggen/clickhouse-add-client-info.yaml @@ -0,0 +1,28 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: clickhouseexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add client info to queries + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [34915] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: This change adds client product info to the system.query_log for more insight on where queries originate + + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/exporter/clickhouseexporter/README.md b/exporter/clickhouseexporter/README.md index aa8d8ea9808d..ae0e54feae07 100644 --- a/exporter/clickhouseexporter/README.md +++ b/exporter/clickhouseexporter/README.md @@ -290,6 +290,11 @@ Connection options: - `compress` (default = lz4): Controls the compression algorithm. Valid options: `none` (disabled), `zstd`, `lz4` (default), `gzip`, `deflate`, `br`, `true` (lz4). Ignored if `compress` is set in the `endpoint` or `connection_params`. - `async_insert` (default = true): Enables [async inserts](https://clickhouse.com/docs/en/optimize/asynchronous-inserts). Ignored if async inserts are configured in the `endpoint` or `connection_params`. Async inserts may still be overridden server-side. +Additional DSN features: + +The underlying `clickhouse-go` module offers additional configuration. These can be set in the exporter's `endpoint` or `connection_params` config values. +- `client_info_product` Must be in `productName/version` format with comma separated entries. By default the exporter will append its binary build information. You can use this information to track the origin of `INSERT` statements in the `system.query_log` table. + ClickHouse tables: - `logs_table_name` (default = otel_logs): The table name for logs. diff --git a/exporter/clickhouseexporter/config.go b/exporter/clickhouseexporter/config.go index 05ddf2e09903..197a2c3cd795 100644 --- a/exporter/clickhouseexporter/config.go +++ b/exporter/clickhouseexporter/config.go @@ -20,6 +20,8 @@ import ( // Config defines configuration for Elastic exporter. type Config struct { + collectorVersionResolver collectorVersionResolver + TimeoutSettings exporterhelper.TimeoutConfig `mapstructure:",squash"` configretry.BackOffConfig `mapstructure:"retry_on_failure"` QueueSettings exporterhelper.QueueConfig `mapstructure:"sending_queue"` @@ -147,6 +149,15 @@ func (cfg *Config) buildDSN() (string, error) { queryParams.Set("compress", cfg.Compress) } + productInfo := queryParams.Get("client_info_product") + binaryProductInfo := fmt.Sprintf("%s/%s", "otelcol", cfg.collectorVersionResolver.GetVersion()) + if productInfo == "" { + productInfo = binaryProductInfo + } else { + productInfo = fmt.Sprintf("%s,%s", productInfo, binaryProductInfo) + } + queryParams.Set("client_info_product", productInfo) + // Use database from config if not specified in path, or if config is not default. if dsnURL.Path == "" || cfg.Database != defaultDatabase { dsnURL.Path = cfg.Database diff --git a/exporter/clickhouseexporter/config_test.go b/exporter/clickhouseexporter/config_test.go index b3167ed52fc7..675411bcdefd 100644 --- a/exporter/clickhouseexporter/config_test.go +++ b/exporter/clickhouseexporter/config_test.go @@ -32,6 +32,7 @@ func TestLoadConfig(t *testing.T) { require.NoError(t, err) defaultCfg := createDefaultConfig() + defaultCfg.(*Config).collectorVersionResolver = newDefaultTestCollectorVersionResolver() defaultCfg.(*Config).Endpoint = defaultEndpoint storageID := component.MustNewIDWithName("file_storage", "clickhouse") @@ -47,14 +48,15 @@ func TestLoadConfig(t *testing.T) { { id: component.NewIDWithName(metadata.Type, "full"), expected: &Config{ - Endpoint: defaultEndpoint, - Database: "otel", - Username: "foo", - Password: "bar", - TTL: 72 * time.Hour, - LogsTableName: "otel_logs", - TracesTableName: "otel_traces", - CreateSchema: true, + collectorVersionResolver: newDefaultTestCollectorVersionResolver(), + Endpoint: defaultEndpoint, + Database: "otel", + Username: "foo", + Password: "bar", + TTL: 72 * time.Hour, + LogsTableName: "otel_logs", + TracesTableName: "otel_traces", + CreateSchema: true, TimeoutSettings: exporterhelper.TimeoutConfig{ Timeout: 5 * time.Second, }, @@ -89,6 +91,7 @@ func TestLoadConfig(t *testing.T) { t.Run(tt.id.String(), func(t *testing.T) { factory := NewFactory() cfg := factory.CreateDefaultConfig() + cfg.(*Config).collectorVersionResolver = newDefaultTestCollectorVersionResolver() sub, err := cm.Sub(tt.id.String()) require.NoError(t, err) @@ -282,7 +285,7 @@ func TestConfig_buildDSN(t *testing.T) { wantChOptions: ChOptions{ Secure: false, }, - want: "clickhouse://127.0.0.1:9000/default?async_insert=true&compress=lz4", + want: "clickhouse://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "Support tcp scheme", @@ -292,7 +295,7 @@ func TestConfig_buildDSN(t *testing.T) { wantChOptions: ChOptions{ Secure: false, }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "prefers database name from config over from DSN", @@ -305,7 +308,7 @@ func TestConfig_buildDSN(t *testing.T) { wantChOptions: ChOptions{ Secure: false, }, - want: "clickhouse://foo:bar@127.0.0.1:9000/otel?async_insert=true&compress=lz4", + want: "clickhouse://foo:bar@127.0.0.1:9000/otel?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "use database name from DSN if not set in config", @@ -317,7 +320,7 @@ func TestConfig_buildDSN(t *testing.T) { wantChOptions: ChOptions{ Secure: false, }, - want: "clickhouse://foo:bar@127.0.0.1:9000/otel?async_insert=true&compress=lz4", + want: "clickhouse://foo:bar@127.0.0.1:9000/otel?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "invalid config", @@ -337,7 +340,7 @@ func TestConfig_buildDSN(t *testing.T) { wantChOptions: ChOptions{ Secure: true, }, - want: "https://127.0.0.1:9000/default?async_insert=true&compress=lz4&secure=true", + want: "https://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4&secure=true", }, { name: "Preserve query parameters", @@ -347,7 +350,7 @@ func TestConfig_buildDSN(t *testing.T) { wantChOptions: ChOptions{ Secure: true, }, - want: "clickhouse://127.0.0.1:9000/default?async_insert=true&compress=lz4&foo=bar&secure=true", + want: "clickhouse://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4&foo=bar&secure=true", }, { name: "Parse clickhouse settings", @@ -359,7 +362,7 @@ func TestConfig_buildDSN(t *testing.T) { DialTimeout: 30 * time.Second, Compress: clickhouse.CompressionBrotli, }, - want: "https://127.0.0.1:9000/default?async_insert=true&compress=br&dial_timeout=30s&secure=true", + want: "https://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=br&dial_timeout=30s&secure=true", }, { name: "Should respect connection parameters", @@ -370,7 +373,7 @@ func TestConfig_buildDSN(t *testing.T) { wantChOptions: ChOptions{ Secure: true, }, - want: "clickhouse://127.0.0.1:9000/default?async_insert=true&compress=lz4&foo=bar&secure=true", + want: "clickhouse://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4&foo=bar&secure=true", }, { name: "support replace database in DSN with config to override database", @@ -378,21 +381,21 @@ func TestConfig_buildDSN(t *testing.T) { Endpoint: "tcp://127.0.0.1:9000/otel", Database: "override", }, - want: "tcp://127.0.0.1:9000/override?async_insert=true&compress=lz4", + want: "tcp://127.0.0.1:9000/override?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "when config option is missing, preserve async_insert false in DSN", fields: fields{ Endpoint: "tcp://127.0.0.1:9000?async_insert=false", }, - want: "tcp://127.0.0.1:9000/default?async_insert=false&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=false&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "when config option is missing, preserve async_insert true in DSN", fields: fields{ Endpoint: "tcp://127.0.0.1:9000?async_insert=true", }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "ignore config option when async_insert is present in connection params as false", @@ -402,7 +405,7 @@ func TestConfig_buildDSN(t *testing.T) { AsyncInsert: &configTrue, }, - want: "tcp://127.0.0.1:9000/default?async_insert=false&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=false&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "ignore config option when async_insert is present in connection params as true", @@ -412,7 +415,7 @@ func TestConfig_buildDSN(t *testing.T) { AsyncInsert: &configFalse, }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "ignore config option when async_insert is present in DSN as false", @@ -421,7 +424,7 @@ func TestConfig_buildDSN(t *testing.T) { AsyncInsert: &configTrue, }, - want: "tcp://127.0.0.1:9000/default?async_insert=false&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=false&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "use async_insert true config option when it is not present in DSN", @@ -430,7 +433,7 @@ func TestConfig_buildDSN(t *testing.T) { AsyncInsert: &configTrue, }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "use async_insert false config option when it is not present in DSN", @@ -439,7 +442,7 @@ func TestConfig_buildDSN(t *testing.T) { AsyncInsert: &configFalse, }, - want: "tcp://127.0.0.1:9000/default?async_insert=false&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=false&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "set async_insert to true when not present in config or DSN", @@ -447,7 +450,7 @@ func TestConfig_buildDSN(t *testing.T) { Endpoint: "tcp://127.0.0.1:9000", }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "connection_params takes priority over endpoint and async_insert option.", @@ -457,7 +460,7 @@ func TestConfig_buildDSN(t *testing.T) { AsyncInsert: &configFalse, }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "use compress br config option when it is not present in DSN", @@ -466,7 +469,7 @@ func TestConfig_buildDSN(t *testing.T) { Compress: "br", }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=br", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=br", }, { name: "set compress to lz4 when not present in config or DSN", @@ -474,7 +477,7 @@ func TestConfig_buildDSN(t *testing.T) { Endpoint: "tcp://127.0.0.1:9000", }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "connection_params takes priority over endpoint and compress option.", @@ -483,12 +486,29 @@ func TestConfig_buildDSN(t *testing.T) { ConnectionParams: map[string]string{"compress": "br"}, Compress: "lz4", }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=br", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=br", + }, + { + name: "include default otel product info in DSN", + fields: fields{ + Endpoint: "tcp://127.0.0.1:9000", + }, + + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", + }, + { + name: "correctly append default product info when value is included in DSN", + fields: fields{ + Endpoint: "tcp://127.0.0.1:9000?client_info_product=customProductInfo%2Fv1.2.3", + }, + + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=customProductInfo%2Fv1.2.3%2Cotelcol%2Ftest&compress=lz4", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := createDefaultConfig().(*Config) + cfg.collectorVersionResolver = newDefaultTestCollectorVersionResolver() mergeConfigWithFields(cfg, tt.fields) dsn, err := cfg.buildDSN() diff --git a/exporter/clickhouseexporter/factory.go b/exporter/clickhouseexporter/factory.go index 4c545d5a9fb8..3be99bf366aa 100644 --- a/exporter/clickhouseexporter/factory.go +++ b/exporter/clickhouseexporter/factory.go @@ -32,6 +32,8 @@ func NewFactory() exporter.Factory { func createDefaultConfig() component.Config { return &Config{ + collectorVersionResolver: newBinaryCollectorVersionResolver(), + TimeoutSettings: exporterhelper.NewDefaultTimeoutConfig(), QueueSettings: exporterhelper.NewDefaultQueueConfig(), BackOffConfig: configretry.NewDefaultBackOffConfig(), diff --git a/exporter/clickhouseexporter/version_info.go b/exporter/clickhouseexporter/version_info.go new file mode 100644 index 000000000000..80086e2c5dd9 --- /dev/null +++ b/exporter/clickhouseexporter/version_info.go @@ -0,0 +1,38 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package clickhouseexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter" + +import ( + "runtime" + "runtime/debug" +) + +type collectorVersionResolver interface { + // GetVersion returns the collector build information for use in query tracking. + // Version should not include any slashes. + GetVersion() string +} + +// binaryCollectorVersionResolver will use the Go binary to detect the collector version. +type binaryCollectorVersionResolver struct { + version string +} + +func newBinaryCollectorVersionResolver() *binaryCollectorVersionResolver { + resolver := binaryCollectorVersionResolver{} + + osInformation := runtime.GOOS[:3] + "-" + runtime.GOARCH + resolver.version = "unknown-" + osInformation + + info, ok := debug.ReadBuildInfo() + if ok && info.Main.Version != "" { + resolver.version = info.Main.Version + "-" + osInformation + } + + return &resolver +} + +func (r *binaryCollectorVersionResolver) GetVersion() string { + return r.version +} diff --git a/exporter/clickhouseexporter/version_info_test.go b/exporter/clickhouseexporter/version_info_test.go new file mode 100644 index 000000000000..6ac66cb3ce3b --- /dev/null +++ b/exporter/clickhouseexporter/version_info_test.go @@ -0,0 +1,17 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package clickhouseexporter + +// testCollectorVersionResolver will return a constant value for the collector version. +type testCollectorVersionResolver struct { + version string +} + +func newDefaultTestCollectorVersionResolver() *testCollectorVersionResolver { + return &testCollectorVersionResolver{version: "test"} +} + +func (r *testCollectorVersionResolver) GetVersion() string { + return r.version +}