Add support for transaction_name_groups and `use_path_as_transactio…

…n_name` (#2331)

Add new config options and their implementation.

docs/configuration.asciidoc

@@ -1074,6 +1074,28 @@ NOTE: All errors that are captured during a request to an ignored URL are still
 
 NOTE: Changing this configuration will overwrite the default value.
 
+[float]
+[[config-transaction-name-groups]]
+==== `TransactionNameGroups` (added[1.27.0])
+
+<<dynamic-configuration, image:./images/dynamic-config.svg[] >>
+
+With this option, you can group transaction names that contain dynamic parts with a wildcard expression. For example, the pattern `GET /user/*/cart` would consolidate transactions, such as `GET /users/42/cart` and `GET /users/73/cart` into a single transaction name `GET /users/*/cart`, hence reducing the transaction name cardinality.
+
+This option supports the wildcard `*`, which matches zero or more characters. Examples: `GET /foo/*/bar/*/baz*``, `*foo*`. Matching is case insensitive by default. Prepending an element with (?-i) makes the matching case sensitive.
+
+[options="header"]
+|============
+| Environment variable name     | IConfiguration or Web.config key
+| `ELASTIC_APM_TRANSACTION_NAME_GROUPS` | `ElasticApm:TransactionNameGroups`
+|============
+
+[options="header"]
+|============
+| Default                 | Type
+| `<none>`                  | String
+|============
+
 [float]
 [[config-use-elastic-apm-traceparent-header]]
 ==== `UseElasticTraceparentHeader` (added[1.3.0])
@@ -1094,6 +1116,31 @@ When this setting is `true`, the agent also adds the header `elasticapm-tracepar
 | `true`                  | Boolean
 |============
 
+[float]
+[[config-use-path-as-transaction-name]]
+==== `UsePathAsTransactionName` (added[1.27.0])
+
+<<dynamic-configuration, image:./images/dynamic-config.svg[] >>
+
+If set to `true`, transaction names of unsupported or partially-supported frameworks will be in the form of `$method $path` instead of just `$method unknown route`.
+
+WARNING: If your URLs contain path parameters like `/user/$userId`,
+you should be very careful when enabling this flag,
+as it can lead to an explosion of transaction groups.
+Take a look at the <<config-transaction-name-groups,`TransactionNameGroups`>> option on how to mitigate this problem by grouping URLs together.
+
+[options="header"]
+|============
+| Environment variable name                    | IConfiguration or Web.config key
+| `ELASTIC_APM_USE_PATH_AS_TRANSACTION_NAME` | `ElasticApm:UsePathAsTransactionName`
+|============
+
+[options="header"]
+|============
+| Default                 | Type
+| `true`                  | Boolean
+|============
+
 [float]
 [[config-use-windows-credentials]]
 ==== `UseWindowsCredentials`
@@ -1368,10 +1415,12 @@ you must instead set the `LogLevel` for the internal APM logger under the `Loggi
 | <<config-stack-trace-limit,`StackTraceLimit`>> | Yes | Stacktrace, Performance
 | <<config-trace-context-ignore-sampled-false,`TraceContextIgnoreSampledFalse`>> | No | Core
 | <<config-transaction-ignore-urls,`TransactionIgnoreUrls`>>  | Yes | HTTP, Performance
+| <<config-transaction-name-groups,`TransactionNameGroups`>>  | Yes | HTTP
 | <<config-transaction-max-spans,`TransactionMaxSpans`>>  | Yes | Core, Performance
 | <<config-transaction-sample-rate,`TransactionSampleRate`>> | Yes | Core, Performance
 | <<config-trace-continuation-strategy,`TraceContinuationStrategy`>> | Yes | HTTP, Performance
 | <<config-use-elastic-apm-traceparent-header,`UseElasticTraceparentHeader`>> | No | HTTP
+| <<config-use-path-as-transaction-name,`UsePathAsTransactionName`>>  | Yes | HTTP
 | <<config-use-windows-credentials,`UseWindowsCredentials`>> | No | Reporter
 | <<config-verify-server-cert,`VerifyServerCert`>> | No | Reporter
 

src/Elastic.Apm/BackendComm/CentralConfig/CentralConfiguration.cs

@@ -54,6 +54,8 @@ public CentralConfiguration(IApmLogger logger, CentralConfigPayload configPayloa
 				GetSimpleConfigurationValue(DynamicConfigurationOption.ExitSpanMinDuration, ParseExitSpanMinDuration);
 			TraceContinuationStrategy =
 				GetConfigurationValue(DynamicConfigurationOption.TraceContinuationStrategy, ParseTraceContinuationStrategy);
+			TransactionNameGroups = GetConfigurationValue(DynamicConfigurationOption.TransactionNameGroups, ParseTransactionNameGroups);
+			UsePathAsTransactionName = GetSimpleConfigurationValue(DynamicConfigurationOption.UsePathAsTransactionName, ParseUsePathAsTransactionName);
 		}
 
 		public string Description => $"Central configuration (ETag: `{ETag}')";
@@ -92,10 +94,14 @@ public CentralConfiguration(IApmLogger logger, CentralConfigPayload configPayloa
 
 		internal IReadOnlyList<WildcardMatcher> TransactionIgnoreUrls { get; private set; }
 
+		internal IReadOnlyCollection<WildcardMatcher> TransactionNameGroups { get; private set; }
+
 		internal int? TransactionMaxSpans { get; private set; }
 
 		internal double? TransactionSampleRate { get; private set; }
 
+		internal bool? UsePathAsTransactionName { get; private set; }
+
 		private CentralConfigurationKeyValue BuildKv(DynamicConfigurationOption option, string value) => new(option, value, Description);
 
 		private T GetConfigurationValue<T>(DynamicConfigurationOption option, Func<ConfigurationKeyValue, T> parser)

src/Elastic.Apm/BackendComm/CentralConfig/DynamicConfigurationOption.cs

@@ -30,8 +30,10 @@ internal enum DynamicConfigurationOption
 		StackTraceLimit = ConfigurationOption.StackTraceLimit,
 		TraceContinuationStrategy = ConfigurationOption.TraceContinuationStrategy,
 		TransactionIgnoreUrls = ConfigurationOption.TransactionIgnoreUrls,
+		TransactionNameGroups = ConfigurationOption.TransactionNameGroups,
 		TransactionMaxSpans = ConfigurationOption.TransactionMaxSpans,
 		TransactionSampleRate = ConfigurationOption.TransactionSampleRate,
+		UsePathAsTransactionName = ConfigurationOption.UsePathAsTransactionName
 	}
 
 	internal static class DynamicConfigurationExtensions
@@ -64,6 +66,8 @@ public static ConfigurationOption ToConfigurationOption(this DynamicConfiguratio
 				TransactionIgnoreUrls => ConfigurationOption.TransactionIgnoreUrls,
 				TransactionMaxSpans => ConfigurationOption.TransactionMaxSpans,
 				TransactionSampleRate => ConfigurationOption.TransactionSampleRate,
+				TransactionNameGroups => ConfigurationOption.TransactionNameGroups,
+				UsePathAsTransactionName => ConfigurationOption.UsePathAsTransactionName,
 				_ => throw new ArgumentOutOfRangeException(nameof(dynamicOption), dynamicOption, null)
 			};
 
@@ -75,7 +79,7 @@ public static ConfigurationOption ToConfigurationOption(this DynamicConfiguratio
 				ConfigurationOption.CaptureHeaders => CaptureHeaders,
 				ConfigurationOption.ExitSpanMinDuration => ExitSpanMinDuration,
 				ConfigurationOption.IgnoreMessageQueues => IgnoreMessageQueues,
-				ConfigurationOption.LogLevel => DynamicConfigurationOption.LogLevel,
+				ConfigurationOption.LogLevel => LogLevel,
 				ConfigurationOption.Recording => Recording,
 				ConfigurationOption.SanitizeFieldNames => SanitizeFieldNames,
 				ConfigurationOption.SpanCompressionEnabled => SpanCompressionEnabled,
@@ -90,6 +94,8 @@ public static ConfigurationOption ToConfigurationOption(this DynamicConfiguratio
 				ConfigurationOption.TransactionIgnoreUrls => TransactionIgnoreUrls,
 				ConfigurationOption.TransactionMaxSpans => TransactionMaxSpans,
 				ConfigurationOption.TransactionSampleRate => TransactionSampleRate,
+				ConfigurationOption.TransactionNameGroups => TransactionNameGroups,
+				ConfigurationOption.UsePathAsTransactionName => UsePathAsTransactionName,
 				_ => null
 			};
 
@@ -101,7 +107,7 @@ public static string ToJsonKey(this DynamicConfigurationOption dynamicOption) =>
 				CaptureHeaders => "capture_headers",
 				ExitSpanMinDuration => "exit_span_min_duration",
 				IgnoreMessageQueues => "ignore_message_queues",
-				DynamicConfigurationOption.LogLevel => "log_level",
+				LogLevel => "log_level",
 				Recording => "recording",
 				SanitizeFieldNames => "sanitize_field_names",
 				SpanCompressionEnabled => "span_compression_enabled",
@@ -116,6 +122,8 @@ public static string ToJsonKey(this DynamicConfigurationOption dynamicOption) =>
 				TransactionIgnoreUrls => "transaction_ignore_urls",
 				TransactionMaxSpans => "transaction_max_spans",
 				TransactionSampleRate => "transaction_sample_rate",
+				TransactionNameGroups => "transaction_name_groups",
+				UsePathAsTransactionName => "use_path_as_transaction_name",
 				_ => throw new ArgumentOutOfRangeException(nameof(dynamicOption), dynamicOption, null)
 			};
 	}

src/Elastic.Apm/BackendComm/CentralConfig/RuntimeConfigurationSnapshot.cs

@@ -117,12 +117,17 @@ public ConfigurationKeyValue Lookup(ConfigurationOption option) =>
 		public IReadOnlyList<WildcardMatcher> TransactionIgnoreUrls =>
 			_dynamicConfiguration?.TransactionIgnoreUrls ?? _mainConfiguration.TransactionIgnoreUrls;
 
+		public IReadOnlyCollection<WildcardMatcher> TransactionNameGroups =>
+			 _dynamicConfiguration?.TransactionNameGroups ?? _mainConfiguration.TransactionNameGroups;
+
 		public int TransactionMaxSpans => _dynamicConfiguration?.TransactionMaxSpans ?? _mainConfiguration.TransactionMaxSpans;
 
 		public double TransactionSampleRate => _dynamicConfiguration?.TransactionSampleRate ?? _mainConfiguration.TransactionSampleRate;
 
 		public bool UseElasticTraceparentHeader => _mainConfiguration.UseElasticTraceparentHeader;
 
+		public bool UsePathAsTransactionName => _dynamicConfiguration?.UsePathAsTransactionName ?? _mainConfiguration.UsePathAsTransactionName;
+
 		public bool VerifyServerCert => _mainConfiguration.VerifyServerCert;
 	}
 }

src/Elastic.Apm/Config/AbstractConfigurationReader.cs

@@ -483,6 +483,29 @@ protected IReadOnlyList<WildcardMatcher> ParseTransactionIgnoreUrls(Configuratio
 			}
 		}
 
+		protected IReadOnlyCollection<WildcardMatcher> ParseTransactionNameGroups(ConfigurationKeyValue kv)
+		{
+			if (kv?.Value == null)
+				return DefaultValues.TransactionNameGroups;
+
+			try
+			{
+				_logger?.Trace()?.Log("Try parsing TransactionNameGroups, values: {TransactionNameGroups}", kv.Value);
+				var transactionNameGroups = kv.Value.Split(',').Where(n => !string.IsNullOrEmpty(n)).ToList();
+
+				var retVal = new List<WildcardMatcher>(transactionNameGroups.Count);
+				foreach (var item in transactionNameGroups)
+					retVal.Add(WildcardMatcher.ValueOf(item.Trim()));
+				return retVal;
+			}
+			catch (Exception e)
+			{
+				_logger?.Error()
+					?.LogException(e, "Failed parsing TransactionNameGroups, values in the config: {TransactionNameGroupsValues}", kv.Value);
+				return DefaultValues.TransactionNameGroups;
+			}
+		}
+
 		protected bool ParseSpanCompressionEnabled(ConfigurationKeyValue kv)
 		{
 			if (kv == null || string.IsNullOrEmpty(kv.Value))
@@ -980,6 +1003,9 @@ protected int ParseTransactionMaxSpans(ConfigurationKeyValue kv)
 			return DefaultValues.TransactionMaxSpans;
 		}
 
+		protected bool ParseUsePathAsTransactionName(ConfigurationKeyValue kv) =>
+			ParseBoolOption(kv, DefaultValues.UsePathAsTransactionName, "UsePathAsTransactionName");
+
 		internal static bool IsMsOrElastic(byte[] array)
 		{
 			var elasticToken = new byte[] { 174, 116, 0, 210, 193, 137, 207, 34 };

src/Elastic.Apm/Config/ConfigConsts.cs

@@ -51,6 +51,7 @@ public static class DefaultValues
 			public const double TransactionSampleRate = 1.0;
 			public const string UnknownServiceName = "unknown-" + Consts.AgentName + "-service";
 			public const bool UseElasticTraceparentHeader = true;
+			public const bool UsePathAsTransactionName = true;
 			public const bool VerifyServerCert = true;
 			public const string TraceContinuationStrategy = "continue";
 
@@ -77,6 +78,8 @@ public static class DefaultValues
 
 			public static readonly IReadOnlyList<WildcardMatcher> TransactionIgnoreUrls;
 
+			public static readonly IReadOnlyCollection<WildcardMatcher> TransactionNameGroups = new List<WildcardMatcher>().AsReadOnly();
+
 			static DefaultValues()
 			{
 				var sanitizeFieldNames = new List<WildcardMatcher>();

src/Elastic.Apm/Config/ConfigurationOption.cs

@@ -90,10 +90,14 @@ public enum ConfigurationOption
 		TransactionIgnoreUrls,
 		/// <inheritdoc cref="IConfigurationReader.TransactionMaxSpans"/>
 		TransactionMaxSpans,
+		/// <inheritdoc cref="IConfigurationReader.TransactionNameGroups"/>
+		TransactionNameGroups,
 		/// <inheritdoc cref="IConfigurationReader.TransactionSampleRate"/>
 		TransactionSampleRate,
 		/// <inheritdoc cref="IConfigurationReader.UseElasticTraceparentHeader"/>
 		UseElasticTraceparentHeader,
+		/// <inheritdoc cref="IConfigurationReader.UsePathAsTransactionName"/>
+		UsePathAsTransactionName,
 		/// <inheritdoc cref="IConfigurationReader.VerifyServerCert"/>
 		VerifyServerCert,
 		/// <inheritdoc cref="IConfigurationReader.ServerUrls"/>
@@ -180,11 +184,13 @@ public static string ToEnvironmentVariable(this ConfigurationOption option) =>
 				TraceContinuationStrategy => EnvPrefix + "TRACE_CONTINUATION_STRATEGY",
 				TransactionIgnoreUrls => EnvPrefix + "TRANSACTION_IGNORE_URLS",
 				TransactionMaxSpans => EnvPrefix + "TRANSACTION_MAX_SPANS",
+				TransactionNameGroups => EnvPrefix + "TRANSACTION_NAME_GROUPS",
 				TransactionSampleRate => EnvPrefix + "TRANSACTION_SAMPLE_RATE",
 				UseElasticTraceparentHeader => EnvPrefix + "USE_ELASTIC_TRACEPARENT_HEADER",
+				UsePathAsTransactionName => EnvPrefix + "USE_PATH_AS_TRANSACTION_NAME",
 				VerifyServerCert => EnvPrefix + "VERIFY_SERVER_CERT",
 				FullFrameworkConfigurationReaderType => EnvPrefix + "FULL_FRAMEWORK_CONFIGURATION_READER_TYPE",
-				_ => throw new System.ArgumentOutOfRangeException(nameof(option), option, null)
+				_ => throw new ArgumentOutOfRangeException(nameof(option), option, null)
 			};
 
 		public static string ToConfigKey(this ConfigurationOption option) =>
@@ -229,14 +235,16 @@ public static string ToConfigKey(this ConfigurationOption option) =>
 				TraceContinuationStrategy => KeyPrefix + nameof(TraceContinuationStrategy),
 				TransactionIgnoreUrls => KeyPrefix + nameof(TransactionIgnoreUrls),
 				TransactionMaxSpans => KeyPrefix + nameof(TransactionMaxSpans),
+				TransactionNameGroups => KeyPrefix + nameof(TransactionNameGroups),
 				TransactionSampleRate => KeyPrefix + nameof(TransactionSampleRate),
 				UseElasticTraceparentHeader => KeyPrefix + nameof(UseElasticTraceparentHeader),
+				UsePathAsTransactionName => KeyPrefix + nameof(UsePathAsTransactionName),
 				VerifyServerCert => KeyPrefix + nameof(VerifyServerCert),
 				ServerUrls => KeyPrefix + nameof(ServerUrls),
 				SpanFramesMinDuration => KeyPrefix + nameof(SpanFramesMinDuration),
 				TraceContextIgnoreSampledFalse => KeyPrefix + nameof(TraceContextIgnoreSampledFalse),
 				FullFrameworkConfigurationReaderType => KeyPrefix + nameof(FullFrameworkConfigurationReaderType),
-				_ => throw new System.ArgumentOutOfRangeException(nameof(option), option, null)
+				_ => throw new ArgumentOutOfRangeException(nameof(option), option, null)
 			};
 	}
 }

src/Elastic.Apm/Config/FallbackToEnvironmentConfigurationBase.cs

@@ -130,10 +130,14 @@ IConfigurationEnvironmentValueProvider environmentValueProvider
 				ParseTraceContinuationStrategy(Lookup(ConfigurationOption.TraceContinuationStrategy));
 			TransactionIgnoreUrls =
 				ParseTransactionIgnoreUrls(Lookup(ConfigurationOption.TransactionIgnoreUrls));
+			TransactionNameGroups =
+				ParseTransactionNameGroups(Lookup(ConfigurationOption.TransactionNameGroups));
 			TransactionMaxSpans = ParseTransactionMaxSpans(Lookup(ConfigurationOption.TransactionMaxSpans));
 			TransactionSampleRate = ParseTransactionSampleRate(Lookup(ConfigurationOption.TransactionSampleRate));
 			UseElasticTraceparentHeader =
 				ParseUseElasticTraceparentHeader(Lookup(ConfigurationOption.UseElasticTraceparentHeader));
+			UsePathAsTransactionName =
+				ParseUsePathAsTransactionName(Lookup(ConfigurationOption.UsePathAsTransactionName));
 			VerifyServerCert = ParseVerifyServerCert(Lookup(ConfigurationOption.VerifyServerCert));
 
 			var urlConfig = Lookup(ConfigurationOption.ServerUrl);
@@ -237,12 +241,16 @@ public ConfigurationKeyValue Lookup(ConfigurationOption option) =>
 
 		public IReadOnlyList<WildcardMatcher> TransactionIgnoreUrls { get; }
 
+		public IReadOnlyCollection<WildcardMatcher> TransactionNameGroups { get; }
+
 		public int TransactionMaxSpans { get; }
 
 		public double TransactionSampleRate { get; }
 
 		public bool UseElasticTraceparentHeader { get; }
 
+		public bool UsePathAsTransactionName { get; }
+
 		public bool VerifyServerCert { get; }
 
 		public bool OpenTelemetryBridgeEnabled { get; }

src/Elastic.Apm/Config/IConfiguration.cs

@@ -12,5 +12,4 @@ namespace Elastic.Apm.Config
 	public interface IConfiguration : IConfigurationReader
 	{
 	}
-
 }

src/Elastic.Apm/Config/IConfigurationReader.cs

@@ -374,6 +374,24 @@ public interface IConfigurationReader : IConfigurationDescription, IConfiguratio
 		/// </summary>
 		IReadOnlyList<WildcardMatcher> TransactionIgnoreUrls { get; }
 
+		/// <summary>
+		/// A list of patterns to be used to group incoming HTTP server transactions by matching names contain dynamic parts
+		/// to a more suitable route name.
+		/// </summary>
+		/// <remarks>
+		/// This setting can be particularly useful in scenarios where the APM agent is unable to determine a suitable
+		/// transaction name for a request. For example, in ASP.NET Core, we can leverage the routing information to
+		/// provide sensible transaction names and avoid high-cardinality. In other frameworks, such as WCF, there is no
+		/// such suitable available. In that case, the APM agent will use the request path in the transaction name, which
+		/// can lead to a high-cardinality problem. By using this setting, you can group similar transactions together.
+		/// <para>
+		///	For example, the pattern '<c>GET /user/*/cart</c>' would consolidate transactions, such as `GET /users/42/cart` and
+		///	'GET /users/73/cart' into a single transaction name 'GET /users/*/cart', hence reducing the transaction
+		///	name cardinality."
+		/// </para>
+		/// </remarks>
+		IReadOnlyCollection<WildcardMatcher> TransactionNameGroups { get; }
+
 		/// <summary>
 		/// The number of spans that are recorded per transaction.
 		/// <list type="bullet">
@@ -408,6 +426,12 @@ public interface IConfigurationReader : IConfigurationDescription, IConfiguratio
 		/// </summary>
 		bool UseElasticTraceparentHeader { get; }
 
+		/// <summary>
+		/// If <c>true</c>, the default, the agent will use the path of the incoming HTTP request as the transaction name in situations
+		/// when a more accurate route name cannot be determined from route data or request headers.
+		/// </summary>
+		bool UsePathAsTransactionName { get; }
+
 		/// <summary>
 		/// The agent verifies the server's certificate if an HTTPS connection to the APM server is used.
 		/// Verification can be disabled by setting to <c>false</c>.

src/Elastic.Apm/Helpers/WildcardMatcher.cs

@@ -86,7 +86,6 @@ public static WildcardMatcher ValueOf(string wildcardString)
 			return new CompoundWildcardMatcher(matcher, matchers);
 		}
 
-
 		/// <summary>
 		/// Returns <code>true</code>, if any of the matchers match the provided string.
 		/// </summary>

src/Elastic.Apm/Model/Transaction.cs

@@ -697,7 +697,28 @@ public void End()
 		}
 
 		if (IsSampled || _apmServerInfo?.Version < new ElasticVersion(8, 0, 0, string.Empty))
+		{
+			// Apply any transaction name groups
+			if (Configuration.TransactionNameGroups.Count > 0)
+			{
+				var matched = WildcardMatcher.AnyMatch(Configuration.TransactionNameGroups, Name, null);
+				if (matched is not null)
+				{
+					var matchedTransactionNameGroup = matched.GetMatcher();
+
+					if (!string.IsNullOrEmpty(matchedTransactionNameGroup))
+					{
+						_logger?.Trace()?.Log("Transaction name '{TransactionName}' matched transaction " +
+							"name group '{TransactionNameGroup}' from configuration",
+								Name, matchedTransactionNameGroup);
+
+						Name = matchedTransactionNameGroup;
+					}
+				}
+			}
+
 			_sender.QueueTransaction(this);
+		}
 		else
 		{
 			_logger?.Debug()