Add support for transaction_name_groups and use_path_as_transaction_name

docs/configuration.asciidoc

@@ -1074,6 +1074,26 @@ NOTE: All errors that are captured during a request to an ignored URL are still
 
 NOTE: Changing this configuration will overwrite the default value.
 
+[float]
+[[config-transaction-name-groups]]
+==== `TransactionNameGroups` (added[1.27.0])
+
+With this option, you can group transaction names that contain dynamic parts with a wildcard expression. For example, the pattern `GET /user/*/cart` would consolidate transactions, such as 'GET /users/42/cart' and 'GET /users/73/cart' into a single transaction name 'GET /users/*/cart', hence reducing the transaction name cardinality.
+
+This option supports the wildcard *, which matches zero or more characters. Examples: GET /foo/*/bar/*/baz*, *foo*. Matching is case insensitive by default. Prepending an element with (?-i) makes the matching case sensitive.
+
+[options="header"]
+|============
+| Environment variable name     | IConfiguration or Web.config key
+| `ELASTIC_APM_TRANSACTION_NAME_GROUPS` | `ElasticApm:TransactionNameGroups`
+|============
+
+[options="header"]
+|============
+| Default                 | Type
+| `<none>`                  | String
+|============
+
 [float]
 [[config-use-elastic-apm-traceparent-header]]
 ==== `UseElasticTraceparentHeader` (added[1.3.0])
@@ -1094,6 +1114,30 @@ When this setting is `true`, the agent also adds the header `elasticapm-tracepar
 | `true`                  | Boolean
 |============
 
+[float]
+[[config-use-path-as-transaction-name]]
+==== `UsePathAsTransactionName` (added[1.27.0])
+
+If set to `true`,
+transaction names of unsupported or partially-supported frameworks will be in the form of `$method $path` instead of just `$method unknown route`.
+
+WARNING: If your URLs contain path parameters like `/user/$userId`,
+you should be very careful when enabling this flag,
+as it can lead to an explosion of transaction groups.
+Take a look at the <<config-transaction-name-groups,`TransactionNameGroups`>> option on how to mitigate this problem by grouping URLs together.
+
+[options="header"]
+|============
+| Environment variable name                    | IConfiguration or Web.config key
+| `ELASTIC_APM_USE_PATH_AS_TRANSACTION_NAME` | `ElasticApm:UsePathAsTransactionName`
+|============
+
+[options="header"]
+|============
+| Default                 | Type
+| `true`                  | Boolean
+|============
+
 [float]
 [[config-use-windows-credentials]]
 ==== `UseWindowsCredentials`
@@ -1368,10 +1412,12 @@ you must instead set the `LogLevel` for the internal APM logger under the `Loggi
 | <<config-stack-trace-limit,`StackTraceLimit`>> | Yes | Stacktrace, Performance
 | <<config-trace-context-ignore-sampled-false,`TraceContextIgnoreSampledFalse`>> | No | Core
 | <<config-transaction-ignore-urls,`TransactionIgnoreUrls`>>  | Yes | HTTP, Performance
+| <<config-transaction-name-groups,`TransactionNameGroups`>>  | No | HTTP
 | <<config-transaction-max-spans,`TransactionMaxSpans`>>  | Yes | Core, Performance
 | <<config-transaction-sample-rate,`TransactionSampleRate`>> | Yes | Core, Performance
 | <<config-trace-continuation-strategy,`TraceContinuationStrategy`>> | Yes | HTTP, Performance
 | <<config-use-elastic-apm-traceparent-header,`UseElasticTraceparentHeader`>> | No | HTTP
+| <<config-use-path-as-transaction-name,`UsePathAsTransactionName`>>  | No | HTTP
 | <<config-use-windows-credentials,`UseWindowsCredentials`>> | No | Reporter
 | <<config-verify-server-cert,`VerifyServerCert`>> | No | Reporter
 

src/Elastic.Apm/BackendComm/CentralConfig/RuntimeConfigurationSnapshot.cs

@@ -117,12 +117,17 @@ public ConfigurationKeyValue Lookup(ConfigurationOption option) =>
 		public IReadOnlyList<WildcardMatcher> TransactionIgnoreUrls =>
 			_dynamicConfiguration?.TransactionIgnoreUrls ?? _mainConfiguration.TransactionIgnoreUrls;
 
+		public IReadOnlyCollection<WildcardMatcher> TransactionNameGroups =>
+			_mainConfiguration.TransactionNameGroups;
+
 		public int TransactionMaxSpans => _dynamicConfiguration?.TransactionMaxSpans ?? _mainConfiguration.TransactionMaxSpans;
 
 		public double TransactionSampleRate => _dynamicConfiguration?.TransactionSampleRate ?? _mainConfiguration.TransactionSampleRate;
 
 		public bool UseElasticTraceparentHeader => _mainConfiguration.UseElasticTraceparentHeader;
 
+		public bool UsePathAsTransactionName => _mainConfiguration.UsePathAsTransactionName;
+
 		public bool VerifyServerCert => _mainConfiguration.VerifyServerCert;
 	}
 }

src/Elastic.Apm/Config/AbstractConfigurationReader.cs

@@ -483,6 +483,29 @@ protected IReadOnlyList<WildcardMatcher> ParseTransactionIgnoreUrls(Configuratio
 			}
 		}
 
+		protected IReadOnlyCollection<WildcardMatcher> ParseTransactionNameGroups(ConfigurationKeyValue kv)
+		{
+			if (kv?.Value == null)
+				return DefaultValues.TransactionNameGroups;
+
+			try
+			{
+				_logger?.Trace()?.Log("Try parsing TransactionNameGroups, values: {TransactionNameGroups}", kv.Value);
+				var transactionNameGroups = kv.Value.Split(',').Where(n => !string.IsNullOrEmpty(n)).ToList();
+
+				var retVal = new List<WildcardMatcher>(transactionNameGroups.Count);
+				foreach (var item in transactionNameGroups)
+					retVal.Add(WildcardMatcher.ValueOf(item.Trim()));
+				return retVal;
+			}
+			catch (Exception e)
+			{
+				_logger?.Error()
+					?.LogException(e, "Failed parsing TransactionNameGroups, values in the config: {TransactionNameGroupsValues}", kv.Value);
+				return DefaultValues.TransactionNameGroups;
+			}
+		}
+
 		protected bool ParseSpanCompressionEnabled(ConfigurationKeyValue kv)
 		{
 			if (kv == null || string.IsNullOrEmpty(kv.Value))
@@ -980,6 +1003,9 @@ protected int ParseTransactionMaxSpans(ConfigurationKeyValue kv)
 			return DefaultValues.TransactionMaxSpans;
 		}
 
+		protected bool ParseUsePathAsTransactionName(ConfigurationKeyValue kv) =>
+			ParseBoolOption(kv, DefaultValues.UsePathAsTransactionName, "UsePathAsTransactionName");
+
 		internal static bool IsMsOrElastic(byte[] array)
 		{
 			var elasticToken = new byte[] { 174, 116, 0, 210, 193, 137, 207, 34 };

src/Elastic.Apm/Config/ConfigConsts.cs

@@ -51,6 +51,7 @@ public static class DefaultValues
 			public const double TransactionSampleRate = 1.0;
 			public const string UnknownServiceName = "unknown-" + Consts.AgentName + "-service";
 			public const bool UseElasticTraceparentHeader = true;
+			public const bool UsePathAsTransactionName = true;
 			public const bool VerifyServerCert = true;
 			public const string TraceContinuationStrategy = "continue";
 
@@ -77,6 +78,8 @@ public static class DefaultValues
 
 			public static readonly IReadOnlyList<WildcardMatcher> TransactionIgnoreUrls;
 
+			public static readonly IReadOnlyCollection<WildcardMatcher> TransactionNameGroups = new List<WildcardMatcher>().AsReadOnly();
+
 			static DefaultValues()
 			{
 				var sanitizeFieldNames = new List<WildcardMatcher>();

src/Elastic.Apm/Config/ConfigurationOption.cs

@@ -90,10 +90,14 @@ public enum ConfigurationOption
 		TransactionIgnoreUrls,
 		/// <inheritdoc cref="IConfigurationReader.TransactionMaxSpans"/>
 		TransactionMaxSpans,
+		/// <inheritdoc cref="IConfigurationReader.TransactionNameGroups"/>
+		TransactionNameGroups,
 		/// <inheritdoc cref="IConfigurationReader.TransactionSampleRate"/>
 		TransactionSampleRate,
 		/// <inheritdoc cref="IConfigurationReader.UseElasticTraceparentHeader"/>
 		UseElasticTraceparentHeader,
+		/// <inheritdoc cref="IConfigurationReader.UsePathAsTransactionName"/>
+		UsePathAsTransactionName,
 		/// <inheritdoc cref="IConfigurationReader.VerifyServerCert"/>
 		VerifyServerCert,
 		/// <inheritdoc cref="IConfigurationReader.ServerUrls"/>
@@ -180,11 +184,13 @@ public static string ToEnvironmentVariable(this ConfigurationOption option) =>
 				TraceContinuationStrategy => EnvPrefix + "TRACE_CONTINUATION_STRATEGY",
 				TransactionIgnoreUrls => EnvPrefix + "TRANSACTION_IGNORE_URLS",
 				TransactionMaxSpans => EnvPrefix + "TRANSACTION_MAX_SPANS",
+				TransactionNameGroups => EnvPrefix + "TRANSACTION_NAME_GROUPS",
 				TransactionSampleRate => EnvPrefix + "TRANSACTION_SAMPLE_RATE",
 				UseElasticTraceparentHeader => EnvPrefix + "USE_ELASTIC_TRACEPARENT_HEADER",
+				UsePathAsTransactionName => EnvPrefix + "USE_PATH_AS_TRANSACTION_NAME",
 				VerifyServerCert => EnvPrefix + "VERIFY_SERVER_CERT",
 				FullFrameworkConfigurationReaderType => EnvPrefix + "FULL_FRAMEWORK_CONFIGURATION_READER_TYPE",
-				_ => throw new System.ArgumentOutOfRangeException(nameof(option), option, null)
+				_ => throw new ArgumentOutOfRangeException(nameof(option), option, null)
 			};
 
 		public static string ToConfigKey(this ConfigurationOption option) =>
@@ -229,14 +235,16 @@ public static string ToConfigKey(this ConfigurationOption option) =>
 				TraceContinuationStrategy => KeyPrefix + nameof(TraceContinuationStrategy),
 				TransactionIgnoreUrls => KeyPrefix + nameof(TransactionIgnoreUrls),
 				TransactionMaxSpans => KeyPrefix + nameof(TransactionMaxSpans),
+				TransactionNameGroups => KeyPrefix + nameof(TransactionNameGroups),
 				TransactionSampleRate => KeyPrefix + nameof(TransactionSampleRate),
 				UseElasticTraceparentHeader => KeyPrefix + nameof(UseElasticTraceparentHeader),
+				UsePathAsTransactionName => KeyPrefix + nameof(UsePathAsTransactionName),
 				VerifyServerCert => KeyPrefix + nameof(VerifyServerCert),
 				ServerUrls => KeyPrefix + nameof(ServerUrls),
 				SpanFramesMinDuration => KeyPrefix + nameof(SpanFramesMinDuration),
 				TraceContextIgnoreSampledFalse => KeyPrefix + nameof(TraceContextIgnoreSampledFalse),
 				FullFrameworkConfigurationReaderType => KeyPrefix + nameof(FullFrameworkConfigurationReaderType),
-				_ => throw new System.ArgumentOutOfRangeException(nameof(option), option, null)
+				_ => throw new ArgumentOutOfRangeException(nameof(option), option, null)
 			};
 	}
 }

src/Elastic.Apm/Config/FallbackToEnvironmentConfigurationBase.cs

@@ -130,10 +130,14 @@ IConfigurationEnvironmentValueProvider environmentValueProvider
 				ParseTraceContinuationStrategy(Lookup(ConfigurationOption.TraceContinuationStrategy));
 			TransactionIgnoreUrls =
 				ParseTransactionIgnoreUrls(Lookup(ConfigurationOption.TransactionIgnoreUrls));
+			TransactionNameGroups =
+				ParseTransactionNameGroups(Lookup(ConfigurationOption.TransactionNameGroups));
 			TransactionMaxSpans = ParseTransactionMaxSpans(Lookup(ConfigurationOption.TransactionMaxSpans));
 			TransactionSampleRate = ParseTransactionSampleRate(Lookup(ConfigurationOption.TransactionSampleRate));
 			UseElasticTraceparentHeader =
 				ParseUseElasticTraceparentHeader(Lookup(ConfigurationOption.UseElasticTraceparentHeader));
+			UsePathAsTransactionName =
+				ParseUsePathAsTransactionName(Lookup(ConfigurationOption.UsePathAsTransactionName));
 			VerifyServerCert = ParseVerifyServerCert(Lookup(ConfigurationOption.VerifyServerCert));
 
 			var urlConfig = Lookup(ConfigurationOption.ServerUrl);
@@ -237,12 +241,16 @@ public ConfigurationKeyValue Lookup(ConfigurationOption option) =>
 
 		public IReadOnlyList<WildcardMatcher> TransactionIgnoreUrls { get; }
 
+		public IReadOnlyCollection<WildcardMatcher> TransactionNameGroups { get; }
+
 		public int TransactionMaxSpans { get; }
 
 		public double TransactionSampleRate { get; }
 
 		public bool UseElasticTraceparentHeader { get; }
 
+		public bool UsePathAsTransactionName { get; }
+
 		public bool VerifyServerCert { get; }
 
 		public bool OpenTelemetryBridgeEnabled { get; }

src/Elastic.Apm/Config/IConfiguration.cs

@@ -12,5 +12,4 @@ namespace Elastic.Apm.Config
 	public interface IConfiguration : IConfigurationReader
 	{
 	}
-
 }

src/Elastic.Apm/Config/IConfigurationReader.cs

@@ -374,6 +374,24 @@ public interface IConfigurationReader : IConfigurationDescription, IConfiguratio
 		/// </summary>
 		IReadOnlyList<WildcardMatcher> TransactionIgnoreUrls { get; }
 
+		/// <summary>
+		/// A list of patterns to be used to group incoming HTTP server transactions by matching names contain dynamic parts
+		/// to a more suitable route name.
+		/// </summary>
+		/// <remarks>
+		/// This setting can be particularly useful in scenarios where the APM agent is unable to determine a suitable
+		/// transaction name for a request. For example, in ASP.NET Core, we can leverage the routing information to
+		/// provide sensible transaction names and avoid high-cardinality. In other frameworks, such as WCF, there is no
+		/// such suitable available. In that case, the APM agent will use the request path in the transaction name, which
+		/// can lead to a high-cardinality problem. By using this setting, you can group similar transactions together.
+		/// <para>
+		///	For example, the pattern '<c>GET /user/*/cart</c>' would consolidate transactions, such as `GET /users/42/cart` and
+		///	'GET /users/73/cart' into a single transaction name 'GET /users/*/cart', hence reducing the transaction
+		///	name cardinality."
+		/// </para>
+		/// </remarks>
+		IReadOnlyCollection<WildcardMatcher> TransactionNameGroups { get; }
+
 		/// <summary>
 		/// The number of spans that are recorded per transaction.
 		/// <list type="bullet">
@@ -408,6 +426,12 @@ public interface IConfigurationReader : IConfigurationDescription, IConfiguratio
 		/// </summary>
 		bool UseElasticTraceparentHeader { get; }
 
+		/// <summary>
+		/// If <c>true</c>, the default, the agent will use the path of the incoming HTTP request as the transaction name in situations
+		/// when a more accurate route name cannot be determined from route data or request headers.
+		/// </summary>
+		bool UsePathAsTransactionName { get; }
+
 		/// <summary>
 		/// The agent verifies the server's certificate if an HTTPS connection to the APM server is used.
 		/// Verification can be disabled by setting to <c>false</c>.

src/Elastic.Apm/Helpers/WildcardMatcher.cs

@@ -86,7 +86,6 @@ public static WildcardMatcher ValueOf(string wildcardString)
 			return new CompoundWildcardMatcher(matcher, matchers);
 		}
 
-
 		/// <summary>
 		/// Returns <code>true</code>, if any of the matchers match the provided string.
 		/// </summary>

src/Elastic.Apm/Model/Transaction.cs

@@ -697,7 +697,28 @@ public void End()
 		}
 
 		if (IsSampled || _apmServerInfo?.Version < new ElasticVersion(8, 0, 0, string.Empty))
+		{
+			// Apply any transaction name groups
+			if (Configuration.TransactionNameGroups.Count > 0)
+			{
+				var matched = WildcardMatcher.AnyMatch(Configuration.TransactionNameGroups, Name, null);
+				if (matched is not null)
+				{
+					var matchedTransactionNameGroup = matched.GetMatcher();
+
+					if (!string.IsNullOrEmpty(matchedTransactionNameGroup))
+					{
+						_logger?.Trace()?.Log("Transaction name '{TransactionName}' matched transaction " +
+							"name group '{TransactionNameGroup}' from configuration",
+								Name, matchedTransactionNameGroup);
+
+						Name = matchedTransactionNameGroup;
+					}
+				}
+			}
+
 			_sender.QueueTransaction(this);
+		}
 		else
 		{
 			_logger?.Debug()

src/integrations/Elastic.Apm.AspNetCore/WebRequestTransactionCreator.cs

@@ -36,8 +36,12 @@ internal static ITransaction StartTransactionAsync(HttpContext context, IApmLogg
 					return null;
 				}
 
+				// For completeness we set the initial transaction name based on the config.
+				// I don't believe there are any valid scenarios where this will not be overwritten later.
 				ITransaction transaction;
-				var transactionName = $"{context.Request.Method} {context.Request.Path}";
+				var transactionName = configuration?.UsePathAsTransactionName ?? ConfigConsts.DefaultValues.UsePathAsTransactionName
+					? $"{context.Request.Method} {context.Request.Path}"
+					: $"{context.Request.Method} unknown route";
 
 				var containsTraceParentHeader =
 					context.Request.Headers.TryGetValue(TraceContext.TraceParentHeaderName, out var traceParentHeader);

src/integrations/Elastic.Apm.AspNetFullFramework/ElasticApmModule.cs

@@ -348,7 +348,10 @@ private void ProcessBeginRequest(object sender)
 				return;
 			}
 
-			var transactionName = $"{request.HttpMethod} {request.Unvalidated.Path}";
+			// Set the initial transaction name based on the request path, if enabled in configuration (default is true).
+			var transactionName = Agent.Instance.Configuration.UsePathAsTransactionName
+				? $"{request.HttpMethod} {request.Unvalidated.Path}"
+				: $"{request.HttpMethod} unknown route";
 
 			var distributedTracingData = ExtractIncomingDistributedTracingData(request);
 			ITransaction transaction;

test/Elastic.Apm.Feature.Tests/TestConfiguration.cs

@@ -66,8 +66,10 @@ public class TestConfiguration : IConfiguration
 		public string TraceContinuationStrategy { get; } = DefaultValues.TraceContinuationStrategy;
 		public IReadOnlyList<WildcardMatcher> TransactionIgnoreUrls { get; set; } = DefaultValues.TransactionIgnoreUrls;
 		public int TransactionMaxSpans { get; set; } = DefaultValues.TransactionMaxSpans;
+		public IReadOnlyCollection<WildcardMatcher> TransactionNameGroups { get; set; } = DefaultValues.TransactionNameGroups;
 		public double TransactionSampleRate { get; set; } = DefaultValues.TransactionSampleRate;
 		public bool UseElasticTraceparentHeader { get; set; } = DefaultValues.UseElasticTraceparentHeader;
+		public bool UsePathAsTransactionName { get; set; } = DefaultValues.UsePathAsTransactionName;
 		public bool VerifyServerCert { get; set; } = DefaultValues.VerifyServerCert;
 		public bool OpenTelemetryBridgeEnabled { get; set; }
 

test/Elastic.Apm.Tests.Utilities/MockConfiguration.cs

@@ -68,7 +68,9 @@ public MockConfiguration(IApmLogger logger = null,
 			string spanCompressionEnabled = null,
 			string spanCompressionExactMatchMaxDuration = null,
 			string spanCompressionSameKindMaxDuration = null,
-			string traceContinuationStrategy = null
+			string traceContinuationStrategy = null,
+			string transactionNameGroups = null,
+			string usePathAsTransactionName = null
 		) : base(
 			logger,
 			new ConfigurationDefaults { DebugName = nameof(MockConfiguration) },
@@ -116,9 +118,11 @@ public MockConfiguration(IApmLogger logger = null,
 				ConfigurationOption.TraceContextIgnoreSampledFalse => traceContextIgnoreSampledFalse,
 				ConfigurationOption.TraceContinuationStrategy => traceContinuationStrategy,
 				ConfigurationOption.TransactionIgnoreUrls => transactionIgnoreUrls,
+				ConfigurationOption.TransactionNameGroups => transactionNameGroups,
 				ConfigurationOption.TransactionMaxSpans => transactionMaxSpans,
 				ConfigurationOption.TransactionSampleRate => transactionSampleRate,
 				ConfigurationOption.UseElasticTraceparentHeader => useElasticTraceparentHeader,
+				ConfigurationOption.UsePathAsTransactionName => usePathAsTransactionName,
 				ConfigurationOption.VerifyServerCert => verifyServerCert,
 				ConfigurationOption.FullFrameworkConfigurationReaderType => null,
 				_ => throw new Exception($"{nameof(MockConfiguration)} does not have implementation for configuration : {key}")

test/Elastic.Apm.Tests/ConstructorTests.cs

@@ -149,8 +149,13 @@ private class LogConfiguration : IConfiguration, IConfigurationDescription
 		public bool UseElasticTraceparentHeader => ConfigConsts.DefaultValues.UseElasticTraceparentHeader;
 
 		public int TransactionMaxSpans => ConfigConsts.DefaultValues.TransactionMaxSpans;
-		// ReSharper restore UnassignedGetOnlyAutoProperty
 
+		public IReadOnlyCollection<WildcardMatcher> TransactionNameGroups =>
+			ConfigConsts.DefaultValues.TransactionNameGroups;
+
+		public bool UsePathAsTransactionName => ConfigConsts.DefaultValues.UsePathAsTransactionName;
+
+		// ReSharper restore UnassignedGetOnlyAutoProperty
 		public ConfigurationKeyValue Lookup(ConfigurationOption option) => null;
 	}
 }