diff --git a/.chloggen/elasticsearch-remove-dedot-dedup.yaml b/.chloggen/elasticsearch-remove-dedot-dedup.yaml new file mode 100644 index 000000000000..6384d2f6bd0b --- /dev/null +++ b/.chloggen/elasticsearch-remove-dedot-dedup.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: exporter/elasticsearch + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Remove Dedot and Dedup configs, which were deprecated since v0.104.0 + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [37091] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/.chloggen/elasticsearch-remove-index.yaml b/.chloggen/elasticsearch-remove-index.yaml new file mode 100644 index 000000000000..9ae7e795be51 --- /dev/null +++ b/.chloggen/elasticsearch-remove-index.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: exporter/elasticsearch + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Remove Index config, which was deprecated since v0.60.0 + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [37091] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/.chloggen/elasticsearch-remove-max-requests.yaml b/.chloggen/elasticsearch-remove-max-requests.yaml new file mode 100644 index 000000000000..951823812ae4 --- /dev/null +++ b/.chloggen/elasticsearch-remove-max-requests.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: exporter/elasticsearch + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Remove MaxRequests config, which was deprecated since v0.112.0 + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [37091] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/exporter/elasticsearchexporter/README.md b/exporter/elasticsearchexporter/README.md index 13ecfa53507d..1383247ee92e 100644 --- a/exporter/elasticsearchexporter/README.md +++ b/exporter/elasticsearchexporter/README.md @@ -115,9 +115,6 @@ Telemetry data will be written to signal specific data streams by default: logs to `logs-generic-default`, metrics to `metrics-generic-default`, and traces to `traces-generic-default`. This can be customised through the following settings: -- `index` (DEPRECATED, please use `logs_index` for logs, `metrics_index` for metrics, `traces_index` for traces): The [index] or [data stream] name to publish events to. - The default value is `logs-generic-default`. - - `logs_index`: The [index] or [data stream] name to publish events to. The default value is `logs-generic-default` - `logs_dynamic_index` (optional): uses resource, scope, or log record attributes to dynamically construct index name. @@ -161,7 +158,7 @@ behaviours, which may be configured through the following settings: - `data_stream.dataset` will always be appended with `.otel`. It is recommended to use with `*_dynamic_index.enabled: true` to route documents to data stream `${data_stream.type}-${data_stream.dataset}-${data_stream.namespace}`. - Span events are stored in separate documents. They will be routed with `data_stream.type` set to `logs` if `traces_dynamic_index::enabled` is `true`. - - `raw`: Omit the `Attributes.` string prefixed to field names for log and + - `raw`: Omit the `Attributes.` string prefixed to field names for log and span attributes as well as omit the `Events.` string prefixed to field names for span events. - `bodymap`: Provides fine-grained control over the final documents to be ingested. @@ -169,12 +166,6 @@ behaviours, which may be configured through the following settings: It works only for logs where the log record body is a map. Each LogRecord body is serialized to JSON as-is and becomes a separate document for ingestion. If the log record body is not a map, the exporter will log a warning and drop the log record. - - `dedup` (DEPRECATED). This configuration is deprecated and non-operational, - and will be removed in the future. Object keys are always deduplicated to - avoid Elasticsearch rejecting documents. - - `dedot` (default=true; DEPRECATED, in future dedotting will always be enabled - for ECS mode, and never for other modes): When enabled attributes with `.` - will be split into proper json objects. #### ECS mapping mode @@ -203,7 +194,6 @@ The behaviour of this bulk indexing can be configured with the following setting - `interval` (default=30s): Write buffer flush time limit. - `retry`: Elasticsearch bulk request retry settings - `enabled` (default=true): Enable/Disable request retry on error. Failed requests are retried with exponential backoff. - - `max_requests` (DEPRECATED, use retry::max_retries instead): Number of HTTP request retries including the initial attempt. If used, `retry::max_retries` will be set to `max_requests - 1`. - `max_retries` (default=2): Number of HTTP request retries. To disable retries, set `retry::enabled` to `false` instead of setting `max_retries` to `0`. - `initial_interval` (default=100ms): Initial waiting time if a HTTP request failed. - `max_interval` (default=1m): Max waiting time if a HTTP request failed. @@ -361,4 +351,4 @@ When sending high traffic of metrics to a TSDB metrics data stream, e.g. using O This will be fixed in a future version of Elasticsearch. A possible workaround would be to use a transform processor to truncate the timestamp, but this will cause duplicate data to be dropped silently. -However, if `@timestamp` precision is not the problem, check your metrics pipeline setup for misconfiguration that causes an actual violation of the [single writer principle](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#single-writer). \ No newline at end of file +However, if `@timestamp` precision is not the problem, check your metrics pipeline setup for misconfiguration that causes an actual violation of the [single writer principle](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#single-writer). diff --git a/exporter/elasticsearchexporter/config.go b/exporter/elasticsearchexporter/config.go index 0835396d928f..cf911a77d126 100644 --- a/exporter/elasticsearchexporter/config.go +++ b/exporter/elasticsearchexporter/config.go @@ -17,7 +17,6 @@ import ( "go.opentelemetry.io/collector/config/configopaque" "go.opentelemetry.io/collector/exporter/exporterbatcher" "go.opentelemetry.io/collector/exporter/exporterhelper" - "go.uber.org/zap" ) // Config defines configuration for Elastic exporter. @@ -38,14 +37,6 @@ type Config struct { // NumWorkers configures the number of workers publishing bulk requests. NumWorkers int `mapstructure:"num_workers"` - // Index configures the index, index alias, or data stream name events should be indexed in. - // - // https://www.elastic.co/guide/en/elasticsearch/reference/current/indices.html - // https://www.elastic.co/guide/en/elasticsearch/reference/current/data-streams.html - // - // Deprecated: `index` is deprecated and replaced with `logs_index`. - Index string `mapstructure:"index"` - // This setting is required when logging pipelines used. LogsIndex string `mapstructure:"logs_index"` // fall back to pure LogsIndex, if 'elasticsearch.index.prefix' or 'elasticsearch.index.suffix' are not found in resource or attribute (prio: resource > attribute) @@ -170,10 +161,6 @@ type RetrySettings struct { // Enabled allows users to disable retry without having to comment out all settings. Enabled bool `mapstructure:"enabled"` - // MaxRequests configures how often an HTTP request is attempted before it is assumed to be failed. - // Deprecated: use MaxRetries instead. - MaxRequests int `mapstructure:"max_requests"` - // MaxRetries configures how many times an HTTP request is retried. MaxRetries int `mapstructure:"max_retries"` @@ -190,18 +177,6 @@ type RetrySettings struct { type MappingsSettings struct { // Mode configures the field mappings. Mode string `mapstructure:"mode"` - - // Dedup is non-operational, and will be removed in the future. - // - // Deprecated: [v0.104.0] deduplication is always enabled, and cannot be - // disabled. Disabling deduplication is not meaningful, as Elasticsearch - // will always reject documents with duplicate JSON object keys. - Dedup *bool `mapstructure:"dedup,omitempty"` - - // Deprecated: [v0.104.0] dedotting will always be applied for ECS mode - // in future, and never for other modes. Elasticsearch's "dot_expander" - // Ingest processor may be used as an alternative for non-ECS modes. - Dedot bool `mapstructure:"dedot"` } type MappingMode int @@ -278,12 +253,6 @@ func (cfg *Config) Validate() error { return errors.New("compression must be one of [none, gzip]") } - if cfg.Retry.MaxRequests != 0 && cfg.Retry.MaxRetries != 0 { - return errors.New("must not specify both retry::max_requests and retry::max_retries") - } - if cfg.Retry.MaxRequests < 0 { - return errors.New("retry::max_requests should be non-negative") - } if cfg.Retry.MaxRetries < 0 { return errors.New("retry::max_retries should be non-negative") } @@ -369,17 +338,3 @@ func parseCloudID(input string) (*url.URL, error) { func (cfg *Config) MappingMode() MappingMode { return mappingModes[cfg.Mapping.Mode] } - -func handleDeprecatedConfig(cfg *Config, logger *zap.Logger) { - if cfg.Mapping.Dedup != nil { - logger.Warn("dedup is deprecated, and is always enabled") - } - if cfg.Mapping.Dedot && cfg.MappingMode() != MappingECS || !cfg.Mapping.Dedot && cfg.MappingMode() == MappingECS { - logger.Warn("dedot has been deprecated: in the future, dedotting will always be performed in ECS mode only") - } - if cfg.Retry.MaxRequests != 0 { - cfg.Retry.MaxRetries = cfg.Retry.MaxRequests - 1 - // Do not set cfg.Retry.Enabled = false if cfg.Retry.MaxRequest = 1 to avoid breaking change on behavior - logger.Warn("retry::max_requests has been deprecated, and will be removed in a future version. Use retry::max_retries instead.") - } -} diff --git a/exporter/elasticsearchexporter/config_test.go b/exporter/elasticsearchexporter/config_test.go index b83beb3e91ba..5918685e8acb 100644 --- a/exporter/elasticsearchexporter/config_test.go +++ b/exporter/elasticsearchexporter/config_test.go @@ -61,7 +61,6 @@ func TestConfig(t *testing.T) { QueueSize: exporterhelper.NewDefaultQueueConfig().QueueSize, }, Endpoints: []string{"https://elastic.example.com:9200"}, - Index: "", LogsIndex: "logs-generic-default", LogsDynamicIndex: DynamicIndexSetting{ Enabled: false, @@ -104,8 +103,7 @@ func TestConfig(t *testing.T) { RetryOnStatus: []int{http.StatusTooManyRequests, http.StatusInternalServerError}, }, Mapping: MappingsSettings{ - Mode: "none", - Dedot: true, + Mode: "none", }, LogstashFormat: LogstashFormatSettings{ Enabled: false, @@ -133,7 +131,6 @@ func TestConfig(t *testing.T) { QueueSize: exporterhelper.NewDefaultQueueConfig().QueueSize, }, Endpoints: []string{"http://localhost:9200"}, - Index: "", LogsIndex: "my_log_index", LogsDynamicIndex: DynamicIndexSetting{ Enabled: false, @@ -176,8 +173,7 @@ func TestConfig(t *testing.T) { RetryOnStatus: []int{http.StatusTooManyRequests, http.StatusInternalServerError}, }, Mapping: MappingsSettings{ - Mode: "none", - Dedot: true, + Mode: "none", }, LogstashFormat: LogstashFormatSettings{ Enabled: false, @@ -205,7 +201,6 @@ func TestConfig(t *testing.T) { QueueSize: exporterhelper.NewDefaultQueueConfig().QueueSize, }, Endpoints: []string{"http://localhost:9200"}, - Index: "", LogsIndex: "logs-generic-default", LogsDynamicIndex: DynamicIndexSetting{ Enabled: false, @@ -248,8 +243,7 @@ func TestConfig(t *testing.T) { RetryOnStatus: []int{http.StatusTooManyRequests, http.StatusInternalServerError}, }, Mapping: MappingsSettings{ - Mode: "none", - Dedot: true, + Mode: "none", }, LogstashFormat: LogstashFormatSettings{ Enabled: false, @@ -284,14 +278,6 @@ func TestConfig(t *testing.T) { cfg.CloudID = "foo:YmFyLmNsb3VkLmVzLmlvJGFiYzEyMyRkZWY0NTY=" }), }, - { - id: component.NewIDWithName(metadata.Type, "deprecated_index"), - configFile: "config.yaml", - expected: withDefaultConfig(func(cfg *Config) { - cfg.Endpoints = []string{"https://elastic.example.com:9200"} - cfg.Index = "my_log_index" - }), - }, { id: component.NewIDWithName(metadata.Type, "confighttp_endpoint"), configFile: "config.yaml", @@ -417,14 +403,6 @@ func TestConfig_Validate(t *testing.T) { }), err: `compression must be one of [none, gzip]`, }, - "both max_retries and max_requests specified": { - config: withDefaultConfig(func(cfg *Config) { - cfg.Endpoints = []string{"http://test:9200"} - cfg.Retry.MaxRetries = 1 - cfg.Retry.MaxRequests = 1 - }), - err: `must not specify both retry::max_requests and retry::max_retries`, - }, } for name, tt := range tests { diff --git a/exporter/elasticsearchexporter/exporter.go b/exporter/elasticsearchexporter/exporter.go index ebd3800858a2..655e56636d73 100644 --- a/exporter/elasticsearchexporter/exporter.go +++ b/exporter/elasticsearchexporter/exporter.go @@ -44,8 +44,10 @@ func newExporter( index string, dynamicIndex bool, ) *elasticsearchExporter { + dedot := cfg.MappingMode() == MappingECS + model := &encodeModel{ - dedot: cfg.Mapping.Dedot, + dedot: dedot, mode: cfg.MappingMode(), } diff --git a/exporter/elasticsearchexporter/exporter_test.go b/exporter/elasticsearchexporter/exporter_test.go index 5554c089e02b..cbaa5c3cc25c 100644 --- a/exporter/elasticsearchexporter/exporter_test.go +++ b/exporter/elasticsearchexporter/exporter_test.go @@ -219,7 +219,6 @@ func TestExporterLogs(t *testing.T) { exporter := newTestLogsExporter(t, server.URL, func(cfg *Config) { cfg.Mapping.Mode = "ecs" - cfg.Mapping.Dedot = true }) logs := newLogsWithAttributes( map[string]any{"attr.key": "value"}, @@ -230,29 +229,6 @@ func TestExporterLogs(t *testing.T) { rec.WaitItems(1) }) - t.Run("publish with dedup", func(t *testing.T) { - rec := newBulkRecorder() - server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { - assert.JSONEq(t, `{"@timestamp":"1970-01-01T00:00:00.000000000Z","Scope":{"name":"","value":"value","version":""},"SeverityNumber":0,"TraceFlags":0}`, string(docs[0].Document)) - rec.Record(docs) - return itemsAllOK(docs) - }) - - exporter := newTestLogsExporter(t, server.URL, func(cfg *Config) { - cfg.Mapping.Mode = "raw" - // dedup is the default - }) - logs := newLogsWithAttributes( - // Scope collides with the top-level "Scope" field, - // so will be removed during deduplication. - map[string]any{"Scope": "value"}, - nil, - nil, - ) - mustSendLogs(t, exporter, logs) - rec.WaitItems(1) - }) - t.Run("publish with headers", func(t *testing.T) { done := make(chan struct{}, 1) server := newESTestServerBulkHandlerFunc(t, func(w http.ResponseWriter, r *http.Request) { @@ -427,7 +403,7 @@ func TestExporterLogs(t *testing.T) { body: func() pcommon.Value { return pcommon.NewValueStr("foo") }(), - wantDocument: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","attributes":{"attr.foo":"attr.foo.value"},"data_stream":{"dataset":"attr.dataset.otel","namespace":"resource.attribute.namespace","type":"logs"},"dropped_attributes_count":0,"observed_timestamp":"1970-01-01T00:00:00.000000000Z","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"},"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0},"severity_number":0,"body":{"text":"foo"}}`), + wantDocument: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","attributes.attr.foo":"attr.foo.value","data_stream.dataset":"attr.dataset.otel","data_stream.namespace":"resource.attribute.namespace","data_stream.type":"logs","dropped_attributes_count":0,"observed_timestamp":"1970-01-01T00:00:00.000000000Z","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"},"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0},"severity_number":0,"body.text":"foo"}`), }, { body: func() pcommon.Value { @@ -438,7 +414,7 @@ func TestExporterLogs(t *testing.T) { m.PutEmptyMap("inner").PutStr("foo", "bar") return vm }(), - wantDocument: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","attributes":{"attr.foo":"attr.foo.value"},"data_stream":{"dataset":"attr.dataset.otel","namespace":"resource.attribute.namespace","type":"logs"},"dropped_attributes_count":0,"observed_timestamp":"1970-01-01T00:00:00.000000000Z","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"},"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0},"severity_number":0,"body":{"flattened":{"true":true,"false":false,"inner":{"foo":"bar"}}}}`), + wantDocument: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","attributes.attr.foo":"attr.foo.value","data_stream.dataset":"attr.dataset.otel","data_stream.namespace":"resource.attribute.namespace","data_stream.type":"logs","dropped_attributes_count":0,"observed_timestamp":"1970-01-01T00:00:00.000000000Z","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"},"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0},"severity_number":0,"body.flattened.true":true,"body.flattened.false":false,"body.flattened.inner.foo":"bar"}`), }, { body: func() pcommon.Value { @@ -450,7 +426,7 @@ func TestExporterLogs(t *testing.T) { return vm }(), isEvent: true, - wantDocument: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","attributes":{"attr.foo":"attr.foo.value","event.name":"foo"},"data_stream":{"dataset":"attr.dataset.otel","namespace":"resource.attribute.namespace","type":"logs"},"dropped_attributes_count":0,"observed_timestamp":"1970-01-01T00:00:00.000000000Z","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"},"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0},"severity_number":0,"body":{"structured":{"true":true,"false":false,"inner":{"foo":"bar"}}}}`), + wantDocument: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","attributes.attr.foo":"attr.foo.value","attributes.event.name":"foo","data_stream.dataset":"attr.dataset.otel","data_stream.namespace":"resource.attribute.namespace","data_stream.type":"logs","dropped_attributes_count":0,"observed_timestamp":"1970-01-01T00:00:00.000000000Z","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"},"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0},"severity_number":0,"body.structured.true":true,"body.structured.false":false,"body.structured.inner.foo":"bar"}`), }, { body: func() pcommon.Value { @@ -461,7 +437,7 @@ func TestExporterLogs(t *testing.T) { s.AppendEmpty().SetEmptyMap().PutStr("foo", "bar") return vs }(), - wantDocument: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","attributes":{"attr.foo":"attr.foo.value"},"data_stream":{"dataset":"attr.dataset.otel","namespace":"resource.attribute.namespace","type":"logs"},"dropped_attributes_count":0,"observed_timestamp":"1970-01-01T00:00:00.000000000Z","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"},"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0},"severity_number":0,"body":{"flattened":{"value":["foo",false,{"foo":"bar"}]}}}`), + wantDocument: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","attributes.attr.foo":"attr.foo.value","data_stream.dataset":"attr.dataset.otel","data_stream.namespace":"resource.attribute.namespace","data_stream.type":"logs","dropped_attributes_count":0,"observed_timestamp":"1970-01-01T00:00:00.000000000Z","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"},"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0},"severity_number":0,"body.flattened.value":["foo",false,{"foo":"bar"}]}`), }, { body: func() pcommon.Value { @@ -473,7 +449,7 @@ func TestExporterLogs(t *testing.T) { return vs }(), isEvent: true, - wantDocument: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","attributes":{"attr.foo":"attr.foo.value","event.name":"foo"},"data_stream":{"dataset":"attr.dataset.otel","namespace":"resource.attribute.namespace","type":"logs"},"dropped_attributes_count":0,"observed_timestamp":"1970-01-01T00:00:00.000000000Z","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"},"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0},"severity_number":0,"body":{"structured":{"value":["foo",false,{"foo":"bar"}]}}}`), + wantDocument: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","attributes.attr.foo":"attr.foo.value","attributes.event.name":"foo","data_stream.dataset":"attr.dataset.otel","data_stream.namespace":"resource.attribute.namespace","data_stream.type":"logs","dropped_attributes_count":0,"observed_timestamp":"1970-01-01T00:00:00.000000000Z","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"},"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0},"severity_number":0,"body.structured.value":["foo",false,{"foo":"bar"}]}`), }, } { rec := newBulkRecorder() @@ -642,22 +618,20 @@ func TestExporterLogs(t *testing.T) { resp[i].Status = http.StatusOK var idxInfo struct { - Attributes struct { - Idx int - } + Idx int `json:"attributes.idx"` } if err := json.Unmarshal(doc.Document, &idxInfo); err != nil { panic(err) } - if idxInfo.Attributes.Idx == retryIdx { + if idxInfo.Idx == retryIdx { if attempts[retryIdx] == 0 { resp[i].Status = http.StatusTooManyRequests } else { defer wg.Done() } } - attempts[idxInfo.Attributes.Idx]++ + attempts[idxInfo.Idx]++ } return resp, nil }) @@ -701,7 +675,7 @@ func TestExporterLogs(t *testing.T) { assert.Len(t, rec.Items(), 1) doc := rec.Items()[0].Document - assert.JSONEq(t, `{"some.record.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `attributes`).Raw) + assert.JSONEq(t, `["foo","bar"]`, gjson.GetBytes(doc, `attributes\.some\.record\.attribute`).Raw) assert.JSONEq(t, `{"some.scope.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `scope.attributes`).Raw) assert.JSONEq(t, `{"some.resource.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `resource.attributes`).Raw) }) @@ -730,7 +704,8 @@ func TestExporterLogs(t *testing.T) { rec.WaitItems(1) doc := rec.Items()[0].Document - assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `attributes`).Raw) + assert.JSONEq(t, `"a"`, gjson.GetBytes(doc, `attributes\.a`).Raw) + assert.JSONEq(t, `"a.b"`, gjson.GetBytes(doc, `attributes\.a\.b`).Raw) assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `scope.attributes`).Raw) assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `resource.attributes`).Raw) }) @@ -1166,19 +1141,19 @@ func TestExporterMetrics(t *testing.T) { expected := []itemRequest{ { Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.metric.foo":"histogram"}}}`), - Document: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"metric.foo":{"counts":[1,2,3,4],"values":[0.5,1.5,2.5,3.0]}},"resource":{"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0}}`), + Document: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","data_stream.dataset":"generic.otel","data_stream.namespace":"default","data_stream.type":"metrics","metrics.metric.foo":{"counts":[1,2,3,4],"values":[0.5,1.5,2.5,3.0]},"resource":{"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0}}`), }, { Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.metric.foo":"histogram"}}}`), - Document: []byte(`{"@timestamp":"1970-01-01T01:00:00.000000000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"metric.foo":{"counts":[4,5,6,7],"values":[2.0,4.5,5.5,6.0]}},"resource":{"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0}}`), + Document: []byte(`{"@timestamp":"1970-01-01T01:00:00.000000000Z","data_stream.dataset":"generic.otel","data_stream.namespace":"default","data_stream.type":"metrics","metrics.metric.foo":{"counts":[4,5,6,7],"values":[2.0,4.5,5.5,6.0]},"resource":{"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0}}`), }, { Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.metric.sum":"gauge_double"}}}`), - Document: []byte(`{"@timestamp":"1970-01-01T01:00:00.000000000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"metric.sum":1.5},"resource":{"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0},"start_timestamp":"1970-01-01T02:00:00.000000000Z"}`), + Document: []byte(`{"@timestamp":"1970-01-01T01:00:00.000000000Z","data_stream.dataset":"generic.otel","data_stream.namespace":"default","data_stream.type":"metrics","metrics.metric.sum":1.5,"resource":{"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0},"start_timestamp":"1970-01-01T02:00:00.000000000Z"}`), }, { Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.metric.summary":"summary"}}}`), - Document: []byte(`{"@timestamp":"1970-01-01T03:00:00.000000000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"metric.summary":{"sum":1.5,"value_count":1}},"resource":{"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0},"start_timestamp":"1970-01-01T03:00:00.000000000Z"}`), + Document: []byte(`{"@timestamp":"1970-01-01T03:00:00.000000000Z","data_stream.dataset":"generic.otel","data_stream.namespace":"default","data_stream.type":"metrics","metrics.metric.summary":{"sum":1.5,"value_count":1},"resource":{"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0},"start_timestamp":"1970-01-01T03:00:00.000000000Z"}`), }, } @@ -1208,7 +1183,7 @@ func TestExporterMetrics(t *testing.T) { assert.Len(t, rec.Items(), 1) doc := rec.Items()[0].Document - assert.JSONEq(t, `{"some.record.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `attributes`).Raw) + assert.JSONEq(t, `["foo","bar"]`, gjson.GetBytes(doc, `attributes\.some\.record\.attribute`).Raw) assert.JSONEq(t, `{"some.scope.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `scope.attributes`).Raw) assert.JSONEq(t, `{"some.resource.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `resource.attributes`).Raw) }) @@ -1247,7 +1222,7 @@ func TestExporterMetrics(t *testing.T) { expected := []itemRequest{ { Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.sum":"gauge_long","metrics.summary":"summary"}}}`), - Document: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","_doc_count":10,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"sum":0,"summary":{"sum":1.0,"value_count":10}},"resource":{"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0}}`), + Document: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","_doc_count":10,"data_stream.dataset":"generic.otel","data_stream.namespace":"default","data_stream.type":"metrics","metrics.sum":0,"metrics.summary":{"sum":1.0,"value_count":10},"resource":{"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0}}`), }, } @@ -1297,11 +1272,11 @@ func TestExporterMetrics(t *testing.T) { expected := []itemRequest{ { Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.histogram.summary":"summary"}}}`), - Document: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","_doc_count":10,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"histogram.summary":{"sum":1.0,"value_count":10}},"resource":{"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0}}`), + Document: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","_doc_count":10,"data_stream.dataset":"generic.otel","data_stream.namespace":"default","data_stream.type":"metrics","metrics.histogram.summary":{"sum":1.0,"value_count":10},"resource":{"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0}}`), }, { Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.exphistogram.summary":"summary"}}}`), - Document: []byte(`{"@timestamp":"1970-01-01T01:00:00.000000000Z","_doc_count":10,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"exphistogram.summary":{"sum":1.0,"value_count":10}},"resource":{"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0}}`), + Document: []byte(`{"@timestamp":"1970-01-01T01:00:00.000000000Z","_doc_count":10,"data_stream.dataset":"generic.otel","data_stream.namespace":"default","data_stream.type":"metrics","metrics.exphistogram.summary":{"sum":1.0,"value_count":10},"resource":{"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0}}`), }, } @@ -1340,7 +1315,7 @@ func TestExporterMetrics(t *testing.T) { expected := []itemRequest{ { Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.foo.bar":"gauge_long","metrics.foo":"gauge_long","metrics.foo.bar.baz":"gauge_long"}}}`), - Document: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"foo":0,"foo.bar":0,"foo.bar.baz":0},"resource":{"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0}}`), + Document: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","data_stream.dataset":"generic.otel","data_stream.namespace":"default","data_stream.type":"metrics","metrics.foo":0,"metrics.foo.bar":0,"metrics.foo.bar.baz":0,"resource":{"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0}}`), }, } @@ -1371,7 +1346,8 @@ func TestExporterMetrics(t *testing.T) { rec.WaitItems(1) doc := rec.Items()[0].Document - assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `attributes`).Raw) + assert.JSONEq(t, `"a"`, gjson.GetBytes(doc, `attributes\.a`).Raw) + assert.JSONEq(t, `"a.b"`, gjson.GetBytes(doc, `attributes\.a\.b`).Raw) assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `scope.attributes`).Raw) assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `resource.attributes`).Raw) }) @@ -1625,11 +1601,11 @@ func TestExporterTraces(t *testing.T) { expected := []itemRequest{ { Action: []byte(`{"create":{"_index":"traces-generic.otel-default"}}`), - Document: []byte(`{"@timestamp":"1970-01-01T01:00:00.000000000Z","attributes":{"attr.foo":"attr.bar"},"data_stream":{"dataset":"generic.otel","namespace":"default","type":"traces"},"dropped_attributes_count":2,"dropped_events_count":3,"dropped_links_count":4,"duration":3600000000000,"kind":"Unspecified","links":[{"attributes":{"link.attr.foo":"link.attr.bar"},"dropped_attributes_count":11,"span_id":"","trace_id":"","trace_state":"bar"}],"name":"name","resource":{"attributes":{"resource.foo":"resource.bar"},"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0},"status":{"code":"Unset"},"trace_state":"foo"}`), + Document: []byte(`{"@timestamp":"1970-01-01T01:00:00.000000000Z","attributes.attr.foo":"attr.bar","data_stream.dataset":"generic.otel","data_stream.namespace":"default","data_stream.type":"traces","dropped_attributes_count":2,"dropped_events_count":3,"dropped_links_count":4,"duration":3600000000000,"kind":"Unspecified","links":[{"attributes":{"link.attr.foo":"link.attr.bar"},"dropped_attributes_count":11,"span_id":"","trace_id":"","trace_state":"bar"}],"name":"name","resource":{"attributes":{"resource.foo":"resource.bar"},"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0},"status.code":"Unset","trace_state":"foo"}`), }, { Action: []byte(`{"create":{"_index":"logs-generic.otel-default"}}`), - Document: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","attributes":{"event.attr.foo":"event.attr.bar","event.name":"exception"},"data_stream":{"dataset":"generic.otel","namespace":"default","type":"logs"},"dropped_attributes_count":1,"resource":{"attributes":{"resource.foo":"resource.bar"},"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0}}`), + Document: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","attributes.event.attr.foo":"event.attr.bar","attributes.event.name":"exception","data_stream.dataset":"generic.otel","data_stream.namespace":"default","data_stream.type":"logs","dropped_attributes_count":1,"resource":{"attributes":{"resource.foo":"resource.bar"},"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0}}`), }, } @@ -1665,7 +1641,7 @@ func TestExporterTraces(t *testing.T) { assert.Len(t, rec.Items(), 2) for _, item := range rec.Items() { doc := item.Document - assert.JSONEq(t, `{"some.record.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `attributes`).Raw) + assert.JSONEq(t, `["foo","bar"]`, gjson.GetBytes(doc, `attributes\.some\.record\.attribute`).Raw) assert.JSONEq(t, `{"some.scope.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `scope.attributes`).Raw) assert.JSONEq(t, `{"some.resource.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `resource.attributes`).Raw) } @@ -1695,7 +1671,8 @@ func TestExporterTraces(t *testing.T) { rec.WaitItems(1) doc := rec.Items()[0].Document - assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `attributes`).Raw) + assert.JSONEq(t, `"a"`, gjson.GetBytes(doc, `attributes\.a`).Raw) + assert.JSONEq(t, `"a.b"`, gjson.GetBytes(doc, `attributes\.a\.b`).Raw) assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `scope.attributes`).Raw) assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `resource.attributes`).Raw) }) diff --git a/exporter/elasticsearchexporter/factory.go b/exporter/elasticsearchexporter/factory.go index 4783d430196a..f3ee459729b7 100644 --- a/exporter/elasticsearchexporter/factory.go +++ b/exporter/elasticsearchexporter/factory.go @@ -50,7 +50,6 @@ func createDefaultConfig() component.Config { return &Config{ QueueSettings: qs, ClientConfig: httpClientConfig, - Index: "", LogsIndex: defaultLogsIndex, LogsDynamicIndex: DynamicIndexSetting{ Enabled: false, @@ -73,8 +72,7 @@ func createDefaultConfig() component.Config { }, }, Mapping: MappingsSettings{ - Mode: "none", - Dedot: true, + Mode: "none", }, LogstashFormat: LogstashFormatSettings{ Enabled: false, @@ -111,14 +109,7 @@ func createLogsExporter( ) (exporter.Logs, error) { cf := cfg.(*Config) - index := cf.LogsIndex - if cf.Index != "" { - set.Logger.Warn("index option are deprecated and replaced with logs_index and traces_index.") - index = cf.Index - } - handleDeprecatedConfig(cf, set.Logger) - - exporter := newExporter(cf, set, index, cf.LogsDynamicIndex.Enabled) + exporter := newExporter(cf, set, cf.LogsIndex, cf.LogsDynamicIndex.Enabled) return exporterhelper.NewLogs( ctx, @@ -135,7 +126,6 @@ func createMetricsExporter( cfg component.Config, ) (exporter.Metrics, error) { cf := cfg.(*Config) - handleDeprecatedConfig(cf, set.Logger) exporter := newExporter(cf, set, cf.MetricsIndex, cf.MetricsDynamicIndex.Enabled) @@ -153,7 +143,6 @@ func createTracesExporter(ctx context.Context, cfg component.Config, ) (exporter.Traces, error) { cf := cfg.(*Config) - handleDeprecatedConfig(cf, set.Logger) exporter := newExporter(cf, set, cf.TracesIndex, cf.TracesDynamicIndex.Enabled) diff --git a/exporter/elasticsearchexporter/factory_test.go b/exporter/elasticsearchexporter/factory_test.go index e24dca1af5e6..62cef7d772ec 100644 --- a/exporter/elasticsearchexporter/factory_test.go +++ b/exporter/elasticsearchexporter/factory_test.go @@ -11,8 +11,6 @@ import ( "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/exporter/exportertest" - "go.uber.org/zap" - "go.uber.org/zap/zaptest/observer" ) func TestCreateDefaultConfig(t *testing.T) { @@ -60,92 +58,3 @@ func TestFactory_CreateTraces(t *testing.T) { require.NoError(t, exporter.Shutdown(context.Background())) } - -func TestFactory_CreateLogsAndTracesExporterWithDeprecatedIndexOption(t *testing.T) { - factory := NewFactory() - cfg := withDefaultConfig(func(cfg *Config) { - cfg.Endpoints = []string{"http://test:9200"} - cfg.Index = "test_index" - }) - params := exportertest.NewNopSettings() - logsExporter, err := factory.CreateLogs(context.Background(), params, cfg) - require.NoError(t, err) - require.NotNil(t, logsExporter) - require.NoError(t, logsExporter.Shutdown(context.Background())) - - tracesExporter, err := factory.CreateTraces(context.Background(), params, cfg) - require.NoError(t, err) - require.NotNil(t, tracesExporter) - require.NoError(t, tracesExporter.Shutdown(context.Background())) -} - -func TestFactory_DedupDeprecated(t *testing.T) { - factory := NewFactory() - cfg := withDefaultConfig(func(cfg *Config) { - dedup := false - cfg.Endpoint = "http://testing.invalid:9200" - cfg.Mapping.Dedup = &dedup - cfg.Mapping.Dedot = false // avoid dedot warnings - }) - - loggerCore, logObserver := observer.New(zap.WarnLevel) - set := exportertest.NewNopSettings() - set.Logger = zap.New(loggerCore) - - logsExporter, err := factory.CreateLogs(context.Background(), set, cfg) - require.NoError(t, err) - require.NoError(t, logsExporter.Shutdown(context.Background())) - - tracesExporter, err := factory.CreateTraces(context.Background(), set, cfg) - require.NoError(t, err) - require.NoError(t, tracesExporter.Shutdown(context.Background())) - - metricsExporter, err := factory.CreateMetrics(context.Background(), set, cfg) - require.NoError(t, err) - require.NoError(t, metricsExporter.Shutdown(context.Background())) - - records := logObserver.AllUntimed() - assert.Len(t, records, 3) - assert.Equal(t, "dedup is deprecated, and is always enabled", records[0].Message) - assert.Equal(t, "dedup is deprecated, and is always enabled", records[1].Message) - assert.Equal(t, "dedup is deprecated, and is always enabled", records[2].Message) -} - -func TestFactory_DedotDeprecated(t *testing.T) { - loggerCore, logObserver := observer.New(zap.WarnLevel) - set := exportertest.NewNopSettings() - set.Logger = zap.New(loggerCore) - - cfgNoDedotECS := withDefaultConfig(func(cfg *Config) { - cfg.Endpoint = "http://testing.invalid:9200" - cfg.Mapping.Dedot = false - cfg.Mapping.Mode = "ecs" - }) - - cfgDedotRaw := withDefaultConfig(func(cfg *Config) { - cfg.Endpoint = "http://testing.invalid:9200" - cfg.Mapping.Dedot = true - cfg.Mapping.Mode = "raw" - }) - - for _, cfg := range []*Config{cfgNoDedotECS, cfgDedotRaw} { - factory := NewFactory() - logsExporter, err := factory.CreateLogs(context.Background(), set, cfg) - require.NoError(t, err) - require.NoError(t, logsExporter.Shutdown(context.Background())) - - tracesExporter, err := factory.CreateTraces(context.Background(), set, cfg) - require.NoError(t, err) - require.NoError(t, tracesExporter.Shutdown(context.Background())) - - metricsExporter, err := factory.CreateMetrics(context.Background(), set, cfg) - require.NoError(t, err) - require.NoError(t, metricsExporter.Shutdown(context.Background())) - } - - records := logObserver.AllUntimed() - assert.Len(t, records, 6) - for _, record := range records { - assert.Equal(t, "dedot has been deprecated: in the future, dedotting will always be performed in ECS mode only", record.Message) - } -} diff --git a/exporter/elasticsearchexporter/integrationtest/datareceiver.go b/exporter/elasticsearchexporter/integrationtest/datareceiver.go index bf237f86524b..4a94a54dcdad 100644 --- a/exporter/elasticsearchexporter/integrationtest/datareceiver.go +++ b/exporter/elasticsearchexporter/integrationtest/datareceiver.go @@ -152,7 +152,7 @@ func (es *esDataReceiver) GenConfigYAMLStr() string { enabled: true initial_interval: 100ms max_interval: 1s - max_requests: 10000`, + max_retries: 10000`, es.endpoint, TestLogsIndex, TestMetricsIndex, TestTracesIndex, ) diff --git a/exporter/elasticsearchexporter/utils_test.go b/exporter/elasticsearchexporter/utils_test.go index fc320b36f073..1f4ef330000d 100644 --- a/exporter/elasticsearchexporter/utils_test.go +++ b/exporter/elasticsearchexporter/utils_test.go @@ -39,11 +39,15 @@ func itemRequestsSortFunc(a, b itemRequest) int { } func assertRecordedItems(t *testing.T, expected []itemRequest, recorder *bulkRecorder, assertOrder bool) { // nolint:unparam + t.Helper() + recorder.WaitItems(len(expected)) assertItemRequests(t, expected, recorder.Items(), assertOrder) } func assertItemRequests(t *testing.T, expected, actual []itemRequest, assertOrder bool) { // nolint:unparam + t.Helper() + expectedItems := expected actualItems := actual if !assertOrder {