[AWS] Use namespace for GetListMetrics when exists (#41022)

* Use namespace for GetListMetrics when exists
elastic · Oct 8, 2024 · 36327a4 · 36327a4
1 parent bc242ae
commit 36327a4
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 6 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -202,6 +202,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Remove excessive info-level logs in cgroups setup {pull}40491[40491]
 - Add missing ECS Cloud fields in GCP `metrics` metricset when using `exclude_labels: true` {issue}40437[40437] {pull}40467[40467]
 - Add AWS OwningAccount support for cross account monitoring {issue}40570[40570] {pull}40691[40691]
+- Use namespace for GetListMetrics when exists in AWS {pull}41022[41022]
 - Fix http server helper SSL config. {pull}39405[39405]
 
 *Osquerybeat*

diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc
@@ -329,7 +329,8 @@ GetMetricData max page size: 100, based on https://docs.aws.amazon.com/AmazonClo
 | IAM ListAccountAliases | 1 | Once on startup
 | STS GetCallerIdentity | 1 | Once on startup
 | EC2 DescribeRegions| 1 | Once on startup
-| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per collection period
+| CloudWatch ListMetrics without specifying namespace in configuration | Total number of results / ListMetrics max page size | Per region per collection period
+| CloudWatch ListMetrics with specific namespaces in configuration | Total number of results / ListMetrics max page size * number of unique namespaces | Per region per collection period
 | CloudWatch GetMetricData | Total number of results / GetMetricData max page size | Per region per namespace per collection period
 |===
 `billing`, `ebs`, `elb`, `sns`, `usage` and `lambda` are the same as `cloudwatch` metricset.

diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc
@@ -317,7 +317,8 @@ GetMetricData max page size: 100, based on https://docs.aws.amazon.com/AmazonClo
 | IAM ListAccountAliases | 1 | Once on startup
 | STS GetCallerIdentity | 1 | Once on startup
 | EC2 DescribeRegions| 1 | Once on startup
-| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per collection period
+| CloudWatch ListMetrics without specifying namespace in configuration | Total number of results / ListMetrics max page size | Per region per collection period
+| CloudWatch ListMetrics with specific namespaces in configuration | Total number of results / ListMetrics max page size * number of unique namespaces | Per region per collection period
 | CloudWatch GetMetricData | Total number of results / GetMetricData max page size | Per region per namespace per collection period
 |===
 `billing`, `ebs`, `elb`, `sns`, `usage` and `lambda` are the same as `cloudwatch` metricset.

diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go
@@ -179,10 +179,22 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 			continue
 		}
 
-		// retrieve all the details for all the metrics available in the current region
-		listMetricsOutput, err := aws.GetListMetricsOutput("*", regionName, m.Period, m.IncludeLinkedAccounts, m.OwningAccount, m.MonitoringAccountID, svcCloudwatch)
-		if err != nil {
-			m.Logger().Errorf("Error while retrieving the list of metrics for region %s: %w", regionName, err)
+		// retrieve all the details for all the metrics available in the current region when no namespace is specified
+		// otherwise only retrieve metrics from the specific namespaces from the config
+		var listMetricsOutput []aws.MetricWithID
+		if len(namespaceDetailTotal) == 0 {
+			listMetricsOutput, err = aws.GetListMetricsOutput("*", regionName, m.Period, m.IncludeLinkedAccounts, m.OwningAccount, m.MonitoringAccountID, svcCloudwatch)
+			if err != nil {
+				m.Logger().Errorf("Error while retrieving the list of metrics for region %s and namespace %s: %w", regionName, "*", err)
+			}
+		} else {
+			for namespace := range namespaceDetailTotal {
+				listMetricsOutputPerNamespace, err := aws.GetListMetricsOutput(namespace, regionName, m.Period, m.IncludeLinkedAccounts, m.OwningAccount, m.MonitoringAccountID, svcCloudwatch)
+				if err != nil {
+					m.Logger().Errorf("Error while retrieving the list of metrics for region %s and namespace %s: %w", regionName, namespace, err)
+				}
+				listMetricsOutput = append(listMetricsOutput, listMetricsOutputPerNamespace...)
+			}
 		}
 
 		if len(listMetricsOutput) == 0 {