Add support for 1-second Storage Resolution in the AWS EMF Exporter (#…

…36057)

This change implements the capability for users of the AWS EMF Exporter
to specify which metrics they would like to have sent to CloudWatch with
a 1 second Storage Resolution. The EMF Exporter now explicitly states
the Storage Resolution for each metric as 60 seconds, the previous
implicit default, so there is no behavior change. If the user specifies
a metric to have 1 second resolution it will be sent to CloudWatch EMF
with the Storage Resolution set accordingly.

#### Description
Previously the AWS EMF Exporter sent metric data into CloudWatch without
specifying the storage resolution. CloudWatch would then default to a 60
second storage resolution, even if metrics are sent more frequently than
every 60 seconds. This would confuse users when they try to apply
functions like AVG, SUM, MAX, or MIN to their metrics with a period of 5
seconds. The function would be applied by CloudWatch to 60 seconds worth
of data and produced unexpected results and confusion for the user. This
commit makes this 60 second resolution explicit in the messages sent to
CloudWatch by the EMF Exporter and also gives the user the option to
specify a more granular 1 second resolution per metric in the
configuration file of the AWS EMF Exporter.

#### Link to tracking issue
Fixes #29506 

#### Testing
Added tests to verify that config file parsing validates a metric
descriptor that specifies either a valid unit, valid storage resolution,
or both and rejects other invalid metric descriptors.
Added tests that the translation from metric data to CW EMF carries a
storage resolution with it, defaulting to a value of 60 (current
behavior) if no storage resolution valid is explicitly set in the
configuration.

#### Documentation
Comments added in the code but have not updated the README.md pending
acceptance of the PR.

.chloggen/awsemf_storageresolution.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: awsemfexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add support for 1 second metric resolution in CloudWatch Embedded Metrics Format based on metric attributes
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [29506]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: 
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

exporter/awsemfexporter/README.md

@@ -85,6 +85,14 @@ This exporter follows default credential resolution for the
 Follow the [guidelines](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html) for the 
 credential configuration.
 
+## Metric Attributes
+By setting attributes on your metrics you can change how individual metrics are sent to CloudWatch. Attributes can be set in code or using components like the [Attribute Processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/attributesprocessor).
+
+The AWS EMF Exporter will interpret the following metric attributes to change how it publishes metrics to CloudWatch:
+
+| Attribute Name    | Description                                                            | Default |
+| :---------------- | :--------------------------------------------------------------------- | ------- |
+| `aws.emf.storage_resolution` | This attribute should be set to an integer value of `1` or `60`. When sending the metric value to CloudWatch use the specified storage resolution value. CloudWatch currently supports a storage resolution of `1` or `60` to indicate 1 second or 60 second resolution. | `aws.emf.storage_resolution = 60` |
 
 ## Configuration Examples
 

exporter/awsemfexporter/metric_translator.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"reflect"
+	"strconv"
 	"time"
 
 	"go.opentelemetry.io/collector/pdata/pmetric"
@@ -29,6 +30,9 @@ const (
 	prometheusReceiver        = "prometheus"
 	attributeReceiver         = "receiver"
 	fieldPrometheusMetricType = "prom_metric_type"
+
+	// metric attributes for AWS EMF, not to be treated as metric labels
+	emfStorageResolutionAttribute = "aws.emf.storage_resolution"
 )
 
 var fieldPrometheusTypes = map[pmetric.MetricType]string{
@@ -45,10 +49,16 @@ type cWMetrics struct {
 	fields       map[string]any
 }
 
+type cWMetricInfo struct {
+	Name              string
+	Unit              string
+	StorageResolution int
+}
+
 type cWMeasurement struct {
 	Namespace  string
 	Dimensions [][]string
-	Metrics    []map[string]string
+	Metrics    []cWMetricInfo
 }
 
 type cWMetricStats struct {
@@ -156,7 +166,7 @@ func (mt metricTranslator) translateOTelToGroupedMetric(rm pmetric.ResourceMetri
 
 // translateGroupedMetricToCWMetric converts Grouped Metric format to CloudWatch Metric format.
 func translateGroupedMetricToCWMetric(groupedMetric *groupedMetric, config *Config) *cWMetrics {
-	labels := groupedMetric.labels
+	labels := filterAWSEMFAttributes(groupedMetric.labels)
 	fieldsLength := len(labels) + len(groupedMetric.metrics)
 
 	isPrometheusMetric := groupedMetric.metadata.receiver == prometheusReceiver
@@ -198,7 +208,7 @@ func translateGroupedMetricToCWMetric(groupedMetric *groupedMetric, config *Conf
 
 // groupedMetricToCWMeasurement creates a single CW Measurement from a grouped metric.
 func groupedMetricToCWMeasurement(groupedMetric *groupedMetric, config *Config) cWMeasurement {
-	labels := groupedMetric.labels
+	labels := filterAWSEMFAttributes(groupedMetric.labels)
 	dimensionRollupOption := config.DimensionRollupOption
 
 	// Create a dimension set containing list of label names
@@ -208,6 +218,7 @@ func groupedMetricToCWMeasurement(groupedMetric *groupedMetric, config *Config)
 		dimSet[idx] = labelName
 		idx++
 	}
+
 	dimensions := [][]string{dimSet}
 
 	// Apply single/zero dimension rollup to labels
@@ -228,14 +239,20 @@ func groupedMetricToCWMeasurement(groupedMetric *groupedMetric, config *Config)
 	// Add on rolled-up dimensions
 	dimensions = append(dimensions, rollupDimensionArray...)
 
-	metrics := make([]map[string]string, len(groupedMetric.metrics))
+	metrics := make([]cWMetricInfo, len(groupedMetric.metrics))
 	idx = 0
 	for metricName, metricInfo := range groupedMetric.metrics {
-		metrics[idx] = map[string]string{
-			"Name": metricName,
+		metrics[idx] = cWMetricInfo{
+			Name:              metricName,
+			StorageResolution: 60,
 		}
 		if metricInfo.unit != "" {
-			metrics[idx]["Unit"] = metricInfo.unit
+			metrics[idx].Unit = metricInfo.unit
+		}
+		if storRes, ok := groupedMetric.labels[emfStorageResolutionAttribute]; ok {
+			if storResInt, err := strconv.Atoi(storRes); err == nil {
+				metrics[idx].StorageResolution = storResInt
+			}
 		}
 		idx++
 	}
@@ -250,7 +267,7 @@ func groupedMetricToCWMeasurement(groupedMetric *groupedMetric, config *Config)
 // groupedMetricToCWMeasurementsWithFilters filters the grouped metric using the given list of metric
 // declarations and returns the corresponding list of CW Measurements.
 func groupedMetricToCWMeasurementsWithFilters(groupedMetric *groupedMetric, config *Config) (cWMeasurements []cWMeasurement) {
-	labels := groupedMetric.labels
+	labels := filterAWSEMFAttributes(groupedMetric.labels)
 
 	// Filter metric declarations by labels
 	metricDeclarations := make([]*MetricDeclaration, 0, len(config.MetricDeclarations))
@@ -278,7 +295,7 @@ func groupedMetricToCWMeasurementsWithFilters(groupedMetric *groupedMetric, conf
 	// Group metrics by matched metric declarations
 	type metricDeclarationGroup struct {
 		metricDeclIdxList []int
-		metrics           []map[string]string
+		metrics           []cWMetricInfo
 	}
 
 	metricDeclGroups := make(map[string]*metricDeclarationGroup)
@@ -299,19 +316,25 @@ func groupedMetricToCWMeasurementsWithFilters(groupedMetric *groupedMetric, conf
 			continue
 		}
 
-		metric := map[string]string{
-			"Name": metricName,
+		metric := cWMetricInfo{
+			Name:              metricName,
+			StorageResolution: 60,
 		}
 		if metricInfo.unit != "" {
-			metric["Unit"] = metricInfo.unit
+			metric.Unit = metricInfo.unit
+		}
+		if storRes, ok := groupedMetric.labels[emfStorageResolutionAttribute]; ok {
+			if storResInt, err := strconv.Atoi(storRes); err == nil {
+				metric.StorageResolution = storResInt
+			}
 		}
 		metricDeclKey := fmt.Sprint(metricDeclIdx)
 		if group, ok := metricDeclGroups[metricDeclKey]; ok {
 			group.metrics = append(group.metrics, metric)
 		} else {
 			metricDeclGroups[metricDeclKey] = &metricDeclarationGroup{
 				metricDeclIdxList: metricDeclIdx,
-				metrics:           []map[string]string{metric},
+				metrics:           []cWMetricInfo{metric},
 			}
 		}
 	}
@@ -465,3 +488,14 @@ func translateGroupedMetricToEmf(groupedMetric *groupedMetric, config *Config, d
 
 	return event, nil
 }
+
+func filterAWSEMFAttributes(labels map[string]string) map[string]string {
+	// remove any labels that are attributes specific to AWS EMF Exporter
+	filteredLabels := make(map[string]string)
+	for labelName := range labels {
+		if labelName != emfStorageResolutionAttribute {
+			filteredLabels[labelName] = labels[labelName]
+		}
+	}
+	return filteredLabels
+}