Merge remote-tracking branch 'upstream/main'

.chloggen/metricsgeneration_match_attrs.yaml

@@ -0,0 +1,28 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: metricsgenerationprocessor
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Introduce functionality to only do metric calculations on data points whose attributes match
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [35425]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  This functionality can be enabled by the `metricsgeneration.MatchAttributes` feature gate, which is disabled by default.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

.github/CODEOWNERS

@@ -126,7 +126,7 @@ extension/sumologicextension/                                       @open-teleme
 internal/aws/                                                       @open-telemetry/collector-contrib-approvers @Aneurysm9 @mxiamxia
 internal/collectd/                                                  @open-telemetry/collector-contrib-approvers @atoulme
 internal/coreinternal/                                              @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers
-internal/docker/                                                    @open-telemetry/collector-contrib-approvers @jamesmoessis
+internal/docker/                                                    @open-telemetry/collector-contrib-approvers @jamesmoessis @MovieStoreGuy
 internal/exp/metrics/                                               @open-telemetry/collector-contrib-approvers @sh0rez @RichieSams
 internal/filter/                                                    @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers
 internal/grpcutil/                                                  @open-telemetry/collector-contrib-approvers @jmacd @moh-osman3 @lquerel

cmd/opampsupervisor/.gitignore

@@ -1,2 +1,3 @@
 effective.yaml
 agent.log
+*.dat

cmd/opampsupervisor/supervisor/supervisor_test.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"strings"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -1154,6 +1155,7 @@ service:
 		assert.True(t, configChanged)
 
 		got := s.agentConfigOwnMetricsSection.Load().(string)
+		got = strings.ReplaceAll(got, "\r\n", "\n")
 
 		// replace the port because that changes on each run
 		portRegex := regexp.MustCompile(":[0-9]{5}")
@@ -1320,6 +1322,7 @@ service:
 		assert.Equal(t, remoteCfg.String(), s.remoteConfig.String())
 
 		gotMergedConfig := s.cfgState.Load().(*configState).mergedConfig
+		gotMergedConfig = strings.ReplaceAll(gotMergedConfig, "\r\n", "\n")
 		// replace random port numbers
 		portRegex := regexp.MustCompile(":[0-9]{5}")
 		replacedMergedConfig := portRegex.ReplaceAll([]byte(gotMergedConfig), []byte(":55555"))
@@ -1363,8 +1366,9 @@ service:
 
 	require.NoError(t, s.createTemplates())
 
-	noopConfig, err := s.composeNoopConfig()
+	noopConfigBytes, err := s.composeNoopConfig()
+	noopConfig := strings.ReplaceAll(string(noopConfigBytes), "\r\n", "\n")
 
 	require.NoError(t, err)
-	require.Equal(t, expectedConfig, string(noopConfig))
+	require.Equal(t, expectedConfig, noopConfig)
 }

exporter/googlecloudpubsubexporter/go.mod

@@ -13,12 +13,12 @@ require (
 	go.opentelemetry.io/collector/exporter v0.111.1-0.20241008154146-ea48c09c31ae
 	go.opentelemetry.io/collector/pdata v1.17.1-0.20241008154146-ea48c09c31ae
 	go.uber.org/zap v1.27.0
-	google.golang.org/api v0.200.0
+	google.golang.org/api v0.201.0
 	google.golang.org/grpc v1.67.1
 )
 
 require (
-	cloud.google.com/go v0.115.1 // indirect
+	cloud.google.com/go v0.116.0 // indirect
 	cloud.google.com/go/auth v0.9.8 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
 	cloud.google.com/go/compute/metadata v0.5.2 // indirect

internal/docker/metadata.yaml

@@ -1,4 +1,4 @@
 status:
   codeowners:
-    active: [jamesmoessis, moviestoreguy]
+    active: [jamesmoessis, MovieStoreGuy]
     emeritus: [rmfitzpatrick]

processor/metricsgenerationprocessor/README.md

@@ -16,13 +16,24 @@
 
 ## Description
 
-The metrics generation processor (`experimental_metricsgenerationprocessor`) can be used to create new metrics using existing metrics following a given rule. This processor currently supports the following two approaches for creating a new metric.
+The metrics generation processor (`experimental_metricsgenerationprocessor`) can be used to create new metrics using existing metrics following a given rule. This processor currently supports the following two rule types for creating a new metric.
 
-1. It can create a new metric from two existing metrics by applying one of the following arithmetic operations: add, subtract, multiply, divide, or percent. One use case is to calculate the `pod.memory.utilization` metric like the following equation-
+1. `calculate`: It can create a new metric from two existing metrics by applying one of the following arithmetic operations: add, subtract, multiply, divide, or percent. One use case is to calculate the `pod.memory.utilization` metric like the following equation-
 `pod.memory.utilization` = (`pod.memory.usage.bytes` / `node.memory.limit`)
-1. It can create a new metric by scaling the value of an existing metric with a given constant number. One use case is to convert `pod.memory.usage` metric values from Megabytes to Bytes (multiply the existing metric's value by 1,048,576)
+1. `scale`: It can create a new metric by scaling the value of an existing metric with a given constant number. One use case is to convert `pod.memory.usage` metric values from Megabytes to Bytes (multiply the existing metric's value by 1,048,576)
 
-Note: The created metric's type is inherited from the metric configured as `metric1`.
+## `calculate` Rule Functionality
+
+There are some specific behaviors of the `calculate` metric generation rule that users may want to be aware of:
+
+- The created metric will have the same type as the metric configured as `metric1`.
+- If no valid data points are calculated for the metric being created, it will not be created.
+  This ensures the processor is not emitting new metrics that are empty.
+- Users may want to have metric calculations done on data points whose overlapping attributes match. To enable this
+  behavior, please enable the feature gate `metricsgeneration.MatchAttributes`. This feature gate is disabled
+  by default, meaning the value used for `metric2` during the calculations is simply the first data point's value.
+  Refer to [documentation](https://github.com/open-telemetry/opentelemetry-collector/blob/main/featuregate/README.md)
+  for more information on how to enable and disable feature gates.
 
 ## Configuration
 

processor/metricsgenerationprocessor/go.mod

@@ -10,6 +10,7 @@ require (
 	go.opentelemetry.io/collector/confmap v1.17.1-0.20241008154146-ea48c09c31ae
 	go.opentelemetry.io/collector/consumer v0.111.1-0.20241008154146-ea48c09c31ae
 	go.opentelemetry.io/collector/consumer/consumertest v0.111.1-0.20241008154146-ea48c09c31ae
+	go.opentelemetry.io/collector/featuregate v1.17.1-0.20241008154146-ea48c09c31ae
 	go.opentelemetry.io/collector/pdata v1.17.1-0.20241008154146-ea48c09c31ae
 	go.opentelemetry.io/collector/processor v0.111.1-0.20241008154146-ea48c09c31ae
 	go.uber.org/goleak v1.3.0
@@ -24,6 +25,7 @@ require (
 	github.com/go-viper/mapstructure/v2 v2.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/hashicorp/go-version v1.7.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/knadh/koanf/providers/confmap v0.1.0 // indirect

processor/metricsgenerationprocessor/processor.go

@@ -5,12 +5,22 @@ package metricsgenerationprocessor // import "github.com/open-telemetry/opentele
 
 import (
 	"context"
+	"fmt"
 
 	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/featuregate"
 	"go.opentelemetry.io/collector/pdata/pmetric"
 	"go.uber.org/zap"
 )
 
+var matchAttributes = featuregate.GlobalRegistry().MustRegister(
+	"metricsgeneration.MatchAttributes",
+	featuregate.StageAlpha,
+	featuregate.WithRegisterDescription("When enabled, the metric calculations will only be done between data points whose attributes match."),
+	featuregate.WithRegisterReferenceURL("https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/35425"),
+	featuregate.WithRegisterFromVersion("v0.112.0"),
+)
+
 type metricsGenerationProcessor struct {
 	rules  []internalRule
 	logger *zap.Logger
@@ -47,25 +57,41 @@ func (mgp *metricsGenerationProcessor) processMetrics(_ context.Context, md pmet
 		nameToMetricMap := getNameToMetricMap(rm)
 
 		for _, rule := range mgp.rules {
-			operand2 := float64(0)
 			_, ok := nameToMetricMap[rule.metric1]
 			if !ok {
 				mgp.logger.Debug("Missing first metric", zap.String("metric_name", rule.metric1))
 				continue
 			}
 
-			if rule.ruleType == string(calculate) {
+			switch rule.ruleType {
+			case string(calculate):
+				// Operation type is validated during config validation, but this adds extra validation as a safety net
+				ot := OperationType(rule.operation)
+				if !ot.isValid() {
+					mgp.logger.Debug(fmt.Sprintf("Invalid operation type '%s' specified for rule: %s. This rule is skipped.", rule.operation, rule.name))
+					continue
+				}
+
 				metric2, ok := nameToMetricMap[rule.metric2]
 				if !ok {
 					mgp.logger.Debug("Missing second metric", zap.String("metric_name", rule.metric2))
 					continue
 				}
-				operand2 = getMetricValue(metric2)
 
-			} else if rule.ruleType == string(scale) {
-				operand2 = rule.scaleBy
+				if matchAttributes.IsEnabled() {
+					generateCalculatedMetrics(rm, metric2, rule, mgp.logger)
+				} else {
+					// When matching metric attributes isn't required the value of the first data point of metric2 is
+					// used for all calculations. The resulting logic is the same as generating a new metric from
+					// a scalar.
+					generateScalarMetrics(rm, getMetricValue(metric2), rule, mgp.logger)
+				}
+			case string(scale):
+				generateScalarMetrics(rm, rule.scaleBy, rule, mgp.logger)
+			default:
+				mgp.logger.Error(fmt.Sprintf("Invalid rule type configured: '%s'. This rule is skipped.", rule.ruleType))
+				continue
 			}
-			generateMetrics(rm, operand2, rule, mgp.logger)
 		}
 	}
 	return md, nil