[etcd] Use prometheus metricset for etcd v3 metrics (#8438)

* add docker healthcheck

* remove vars from test

* use prometheus metricset

* update docs

* bump package version
change input type
improve titles and descriptions

* fix sample eventt

* disable v2 data streams by default

* add additional v3 metrics

* fix etcd policy template

* update docs

* add fields unit type
remove duplicate field

* update docs

* add hosts global var
bump to 0.7.0

* Update packages/etcd/_dev/build/docs/README.md

Co-authored-by: Agi K Thomas <[email protected]>

* update docs

* set ecs version in pipeline

* update dashboard

* Update packages/etcd/changelog.yml

Co-authored-by: Richa Talwar <[email protected]>

* bump kibana version

* change migration version

* export with 8.7

* reoder lenses

* replace dashboard image

* fix ecs version in pipeline

* update fields description

* update sample event

* update docs

---------

Co-authored-by: Agi K Thomas <[email protected]>
Co-authored-by: Richa Talwar <[email protected]>

packages/etcd/_dev/build/docs/README.md

@@ -1,17 +1,21 @@
 # etcd Integration
 
-This integration is used to collect metrics from [etcd v2 and v3 servers](https://etcd.io/).
-This integration periodically fetches metrics from [etcd monitoring server APIs](https://etcd.io/docs/v3.1/op-guide/monitoring/). 
+This integration is used to collect metrics from [etcd v2 and v3 instances](https://etcd.io/).
+
+It periodically fetches metrics from [etcd metrics APIs](https://etcd.io/docs/v3.1/op-guide/monitoring/). 
 
 ## Compatibility
 
-The etcd package was tested with etcd 3.5.1.
+The etcd package was tested with etcd `3.5.x`.
 
 ## Metrics
 
-When using etcd v2, metrics are collected using etcd v2 API. When using v3, metrics are retrieved from the /metrics endpoint.
+For etcd v2, metrics are collected through the etcd v2 APIs, whereas for v3, they are fetched from the `/metrics` endpoint.
+
+When using v3, datasets are bundled within `metrics` data stream, while for v2, available datasets include `leader`, `self`, and `store`.
+
+The ETCD v2 APIs are not enabled by default. However, you can enable ETCD v2 APIs when using ETCD v3 and above by utilizing the --enable-v2 flag, provided it is supported.
 
-When using v3, datasets are bundled into `metrics`. When using v2, datasets available are `leader`, `self` and `store`.
 
 ### metrics
 

packages/etcd/_dev/deploy/docker/Dockerfile

@@ -1,2 +1,3 @@
 ARG SERVICE_VERSION=${SERVICE_VERSION:-3.5.1}
 FROM bitnami/etcd:${SERVICE_VERSION}
+HEALTHCHECK --interval=15s --retries=90 CMD curl -f http://localhost:2379/metrics

packages/etcd/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.7.0"
+  changes:
+    - description: Switch to Prometheus metricset, add ECS fields, dimensions mappings and metric types.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8438
 - version: 0.6.1
   changes:
     - description: Fix incorrect field mappings in leader datastream

packages/etcd/data_stream/leader/manifest.yml

@@ -19,3 +19,4 @@ streams:
         default: /metrics
     title: etcd leader metrics
     description: Collect etcd leader metrics
+    enabled: false

packages/etcd/data_stream/metrics/_dev/test/system/test-default-config.yml

@@ -1,6 +1,3 @@
 vars:
   hosts:
-    - http://{{Hostname}}:2379
-data_stream:
-  vars:
-    server_status_path: /metrics
+    - http://{{Hostname}}:2379/metrics

packages/etcd/data_stream/metrics/agent/stream/stream.yml.hbs

@@ -1,9 +1,13 @@
-metricsets: ["metrics"]
+metricsets: ["collector"]
 hosts:
 {{#each hosts}}
   - {{this}}
 {{/each}}
+metrics_filters.include:
+{{#each metrics_filters.include}}
+    - {{this}}
+{{/each}}
+metrics_path: {{metrics_path}}
 period: {{period}}
-{{#if server_status_path}}
-server_status_path: {{server_status_path}}
-{{/if}}
+use_types: true
+rate_counters: false

packages/etcd/data_stream/metrics/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,133 @@
+---
+description: Pipeline for processing etcd v3 metrics.
+processors:
+  - set:
+      field: event.module
+      value: etcd
+  - set:
+      field: ecs.version
+      value: "8.5.1"
+  - rename:
+      field: prometheus.etcd_mvcc_db_total_size_in_bytes.value
+      target_field: etcd.disk.mvcc_db_total_size.bytes
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_disk_wal_fsync_duration_seconds.histogram
+      target_field: etcd.disk.wal_fsync_duration_seconds.histogram
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_disk_backend_commit_duration_seconds.histogram
+      target_field: etcd.disk.backend_commit_duration_seconds.histogram
+      ignore_missing: true
+  - rename:
+      field: prometheus.go_memstats_alloc_bytes.value
+      target_field: etcd.memory.go_memstats_alloc.bytes
+      ignore_missing: true
+  - rename:
+      field: prometheus.go_memstats_alloc_bytes_total.counter
+      target_field: etcd.memory.go_memstats_alloc.total.bytes
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_network_client_grpc_sent_bytes_total.counter
+      target_field: etcd.network.client_grpc_sent.bytes
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_network_client_grpc_received_bytes_total.counter
+      target_field: etcd.network.client_grpc_received.bytes
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_server_has_leader.value
+      target_field: etcd.server.has_leader.count
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_server_leader_changes_seen_total.counter
+      target_field: etcd.server.leader_changes.count
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_server_proposals_committed_total.value
+      target_field: etcd.server.proposals_committed.count
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_server_proposals_pending.value
+      target_field: etcd.server.proposals_pending.count
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_server_proposals_failed_total.counter
+      target_field: etcd.server.proposals_failed.count
+      ignore_missing: true
+  - rename:
+      field: prometheus.grpc_server_started_total.counter
+      target_field: etcd.server.grpc_started.count
+      ignore_missing: true
+  - rename:
+      field: prometheus.grpc_server_handled_total.counter
+      target_field: etcd.server.grpc_handled.count
+      ignore_missing: true
+  - rename:
+      field: prometheus.process_start_time_seconds.value
+      target_field: etcd.process_start_time.sec
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_network_peer_round_trip_time_seconds.histogram
+      target_field: etcd.network.peer_round_trip_time_seconds.histogram
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_server_proposals_applied_total.value
+      target_field: etcd.server.proposals_applied_total
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_network_peer_received_bytes_total.counter
+      target_field: etcd.network.peer_received_bytes_total
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_network_peer_sent_bytes_total.counter
+      target_field: etcd.network.peer_sent_bytes_total
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_network_peer_sent_failures_total.counter
+      target_field: etcd.network.peer_sent_failures_total
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_network_peer_received_failures_total.counter
+      target_field: etcd.network.peer_received_failures_total
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_debugging_store_writes_total.counter
+      target_field: etcd.store.writes_total
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_debugging_store_expires_total.counter
+      target_field: etcd.store.expires_total
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_debugging_store_reads_total.counter
+      target_field: etcd.store.reads_total
+      ignore_missing: true
+  - rename:
+      field: prometheus.etcd_debugging_store_watchers.value
+      target_field: etcd.store.watchers
+      ignore_missing: true
+  - rename:
+      field: prometheus.labels
+      target_field: etcd.labels
+      ignore_missing: true
+      ignore_failure: true
+  - fingerprint:
+      fields: ["etcd.labels"]
+      target_field: "etcd.labels.fingerprint"
+      ignore_missing: true
+  - remove:
+      field: prometheus
+      ignore_missing: true
+  - set:
+      field: event.kind
+      value: pipeline_error
+      if: ctx.error?.message != null
+on_failure:
+  - set:
+      field: error.message
+      value: "{{{_ingest.on_failure_message}}}"
+  - append:
+      field: event.kind
+      value: pipeline_error
+      allow_duplicates: false

packages/etcd/data_stream/metrics/fields/ecs.yml

@@ -1,8 +1,35 @@
-- external: ecs
-  name: host.ip
-- external: ecs
-  name: ecs.version
-- external: ecs
-  name: service.address
-- external: ecs
-  name: service.type
+- name: ecs.version
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: event.kind
+  external: ecs
+- name: service.type
+  external: ecs
+- name: service.address
+  dimension: true
+  external: ecs
+- name: agent.id
+  dimension: true
+  external: ecs
+- name: cloud.account.id
+  dimension: true
+  external: ecs
+- name: cloud.region
+  dimension: true
+  external: ecs
+- name: cloud.availability_zone
+  dimension: true
+  external: ecs
+- name: cloud.instance.id
+  dimension: true
+  external: ecs
+- name: cloud.provider
+  dimension: true
+  external: ecs
+- name: container.id
+  dimension: true
+  external: ecs
+- name: host.name
+  dimension: true
+  external: ecs