From aa5da2847957fa62687e0b4e7777c86fe7c05f6c Mon Sep 17 00:00:00 2001 From: Kevin Lacabane Date: Fri, 11 Aug 2023 19:12:55 +0200 Subject: [PATCH] [elasticsearch] support api_key (#36274) * support api_key in es module * changelog entry * add unit test * fix errorf call * update documentation * mage update * x-pack mage update --- CHANGELOG.next.asciidoc | 1 + .../docs/modules/elasticsearch.asciidoc | 1 + metricbeat/metricbeat.reference.yml | 1 + .../elasticsearch/_meta/config-xpack.yml | 1 + .../elasticsearch/_meta/config.reference.yml | 1 + .../module/elasticsearch/_meta/config.yml | 1 + metricbeat/module/elasticsearch/metricset.go | 19 ++++++-- .../module/elasticsearch/node/node_test.go | 45 +++++++++++++++++++ .../elasticsearch-xpack.yml.disabled | 1 + .../modules.d/elasticsearch.yml.disabled | 1 + x-pack/metricbeat/metricbeat.reference.yml | 1 + 11 files changed, 69 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 00540f65a43..74d8df56ee3 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -187,6 +187,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415 - Resolve statsd module's prematurely halting of metrics parsing upon encountering an invalid packet. {pull}35075[35075] - Fix the gap in fetching forecast API metrics at the end of each month for Azure billing module {pull}36142[36142] - Add option in SQL module to execute queries for all dbs. {pull}35688[35688] +- Add support for api_key authentication in elasticsearch module {pull}36274[36274] *Osquerybeat* diff --git a/metricbeat/docs/modules/elasticsearch.asciidoc b/metricbeat/docs/modules/elasticsearch.asciidoc index 54ddd22a762..109f615f073 100644 --- a/metricbeat/docs/modules/elasticsearch.asciidoc +++ b/metricbeat/docs/modules/elasticsearch.asciidoc @@ -71,6 +71,7 @@ metricbeat.modules: hosts: ["http://localhost:9200"] #username: "elastic" #password: "changeme" + #api_key: "foo:bar" #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] #index_recovery.active_only: true diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 2a184fdb0bf..eabdcc8e918 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -282,6 +282,7 @@ metricbeat.modules: hosts: ["http://localhost:9200"] #username: "elastic" #password: "changeme" + #api_key: "foo:bar" #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] #index_recovery.active_only: true diff --git a/metricbeat/module/elasticsearch/_meta/config-xpack.yml b/metricbeat/module/elasticsearch/_meta/config-xpack.yml index 4406ba7c050..0cdbe68aa79 100644 --- a/metricbeat/module/elasticsearch/_meta/config-xpack.yml +++ b/metricbeat/module/elasticsearch/_meta/config-xpack.yml @@ -4,4 +4,5 @@ hosts: ["http://localhost:9200"] #username: "user" #password: "secret" + #api_key: "foo:bar" diff --git a/metricbeat/module/elasticsearch/_meta/config.reference.yml b/metricbeat/module/elasticsearch/_meta/config.reference.yml index 199d28a0aea..1aa1d583b72 100644 --- a/metricbeat/module/elasticsearch/_meta/config.reference.yml +++ b/metricbeat/module/elasticsearch/_meta/config.reference.yml @@ -12,6 +12,7 @@ hosts: ["http://localhost:9200"] #username: "elastic" #password: "changeme" + #api_key: "foo:bar" #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] #index_recovery.active_only: true diff --git a/metricbeat/module/elasticsearch/_meta/config.yml b/metricbeat/module/elasticsearch/_meta/config.yml index 4a9dde67ad6..2ca3677cbf2 100644 --- a/metricbeat/module/elasticsearch/_meta/config.yml +++ b/metricbeat/module/elasticsearch/_meta/config.yml @@ -6,3 +6,4 @@ hosts: ["http://localhost:9200"] #username: "user" #password: "secret" + #api_key: "foo:bar" diff --git a/metricbeat/module/elasticsearch/metricset.go b/metricbeat/module/elasticsearch/metricset.go index 363a9634f05..508ac68431c 100644 --- a/metricbeat/module/elasticsearch/metricset.go +++ b/metricbeat/module/elasticsearch/metricset.go @@ -18,6 +18,7 @@ package elasticsearch import ( + "encoding/base64" "encoding/json" "errors" "fmt" @@ -90,19 +91,29 @@ func NewMetricSet(base mb.BaseMetricSet, servicePath string) (*MetricSet, error) return nil, err } - http.SetHeaderDefault(productorigin.Header, productorigin.Beats) - config := struct { - Scope Scope `config:"scope"` - XPackEnabled bool `config:"xpack.enabled"` + Scope Scope `config:"scope"` + XPackEnabled bool `config:"xpack.enabled"` + ApiKey string `config:"api_key"` }{ Scope: ScopeNode, XPackEnabled: false, + ApiKey: "", } if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } + http.SetHeaderDefault(productorigin.Header, productorigin.Beats) + + if config.ApiKey != "" { + hostData := base.HostData() + if hostData.User != "" || hostData.Password != "" { + return nil, fmt.Errorf("cannot set both api_key and username/password") + } + http.SetHeader("Authorization", "ApiKey "+base64.StdEncoding.EncodeToString([]byte(config.ApiKey))) + } + ms := &MetricSet{ base, servicePath, diff --git a/metricbeat/module/elasticsearch/node/node_test.go b/metricbeat/module/elasticsearch/node/node_test.go index 4a49072f294..32f91bc8c60 100644 --- a/metricbeat/module/elasticsearch/node/node_test.go +++ b/metricbeat/module/elasticsearch/node/node_test.go @@ -20,6 +20,7 @@ package node import ( + "encoding/base64" "io/ioutil" "net/http" "net/http/httptest" @@ -78,4 +79,48 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) }) } + + t.Run("with api key", func(t *testing.T) { + apiKey := "foo:bar" + expectedHeader := "ApiKey " + base64.StdEncoding.EncodeToString([]byte(apiKey)) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.RequestURI { + case "/_nodes/_local": + apiKey := r.Header.Get("Authorization") + if apiKey != expectedHeader { + t.Errorf("expected api key to be %s but got %s", expectedHeader, apiKey) + } + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte([]byte("{}"))) + + case "/": + apiKey := r.Header.Get("Authorization") + if apiKey != expectedHeader { + t.Errorf("expected api key to be %s but got %s", expectedHeader, apiKey) + } + + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json") + w.Write([]byte("{}")) + + default: + t.FailNow() + } + + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "elasticsearch", + "metricsets": []string{"node"}, + "hosts": []string{server.URL}, + "api_key": "foo:bar", + } + reporter := &mbtest.CapturingReporterV2{} + + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + metricSet.Fetch(reporter) + }) } diff --git a/metricbeat/modules.d/elasticsearch-xpack.yml.disabled b/metricbeat/modules.d/elasticsearch-xpack.yml.disabled index d89c8b5d29b..b69fe6b87f9 100644 --- a/metricbeat/modules.d/elasticsearch-xpack.yml.disabled +++ b/metricbeat/modules.d/elasticsearch-xpack.yml.disabled @@ -7,4 +7,5 @@ hosts: ["http://localhost:9200"] #username: "user" #password: "secret" + #api_key: "foo:bar" diff --git a/metricbeat/modules.d/elasticsearch.yml.disabled b/metricbeat/modules.d/elasticsearch.yml.disabled index aadd41d5946..33983e4ac14 100644 --- a/metricbeat/modules.d/elasticsearch.yml.disabled +++ b/metricbeat/modules.d/elasticsearch.yml.disabled @@ -9,3 +9,4 @@ hosts: ["http://localhost:9200"] #username: "user" #password: "secret" + #api_key: "foo:bar" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 3d3fa585ff0..fa15aca7fb6 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -525,6 +525,7 @@ metricbeat.modules: hosts: ["http://localhost:9200"] #username: "elastic" #password: "changeme" + #api_key: "foo:bar" #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] #index_recovery.active_only: true