[Exception Replay] Added test suite for ASP.NET Core (#5821)

## Summary of changes
Added the infrastructure of integration testing for Exception Replay in
ASP.NET Core.

To add a test, one needs to simply create a class that inherits from
`IRun` / `IAsyncRun` and decorated with `ExceptionReplayTestData`
attribute, providing to the attribute the amount of snapshots
anticipated both in default configuration and full stack trace capture
configuration.

This PR also introduces the renaming of the configs of Exception Replay
from Exception Debugging. In the early days we used to call Exception
Replay internally Exception Debugging, thus some of the configs held
this name. This PR renamed those leftovers to Exception Replay,
including the enablement config of `DD_EXCEPTION_REPLAY_ENABLED`.
[Introduced a PR to
`dd-go`](DataDog/dd-go#146180) too where these
configs were added to the `config_norm_rules.json` file.

## Reason for change
Laying the groundwork for integration testing of Exception Replay.

## Implementation details
The testing infra bootstraps the web app, queries via reflection all the
tests (classes inheriting from `IRun`/`IAsyncRun` and decorated with
`ExceptionReplayTestData` attribute) and list them. Upon executing a
test, a request will hit an endpoint of `RunTest` with the name of the
test to run (FQN of the class name). The crafted test must throw an
exception to be considered valid.
For a more complex test scenarios, more endpoints could be introduced.

Every test runs in 4 flavors:
- With Exception Replay while Dynamic Instrumentation is **enabled**,
default stack trace capturing limit.
- With Exception Replay while Dynamic Instrumentation is **enabled**,
full stack trace capturing limit.
- With Exception Replay while Dynamic Instrumentation is **disabled**,
default stack trace capturing limit.
- With Exception Replay while Dynamic Instrumentation is **disabled**,
full stack trace capturing limit.

## Test coverage
This whole PR 🤓. I've added two tests that utilize the infra:
- `ExceptionWithNonSupportedFramesTest`
- `RecursiveExceptionTest`

## Other details
Fixes DEBUG-2681, DEBUG-2732, DEBUG-1642.

Datadog.Trace.Debugger.slnf

@@ -20,6 +20,7 @@
       "tracer\\test\\Datadog.Trace.TestHelpers\\Datadog.Trace.TestHelpers.csproj",
       "tracer\\test\\Datadog.Trace.Tests\\Datadog.Trace.Tests.csproj",
       "tracer\\test\\Datadog.Tracer.Native.Tests\\Datadog.Tracer.Native.Tests.vcxproj",
+      "tracer\\test\\test-applications\\debugger\\Samples.Debugger.AspNetCore5\\Samples.Debugger.AspNetCore5.csproj",
       "tracer\\test\\test-applications\\debugger\\Samples.Probes\\Samples.Probes.csproj",
       "tracer\\test\\test-applications\\debugger\\dependency-libs\\Samples.Probes.External\\Samples.Probes.External.csproj",
       "tracer\\test\\test-applications\\debugger\\dependency-libs\\Samples.Probes.TestRuns\\Samples.Probes.TestRuns.csproj",

tracer/build/_build/Build.Steps.Debugger.cs

@@ -1,3 +1,4 @@
+using System.Linq;
 using Nuke.Common;
 using Nuke.Common.ProjectModel;
 using Nuke.Common.Tooling;
@@ -27,6 +28,8 @@ partial class Build
 
     Project DebuggerSamples => Solution.GetProject(Projects.DebuggerSamples);
 
+    Project ExceptionReplaySamples => Solution.GetProject(Projects.ExceptionReplaySamples);
+
     Project DebuggerSamplesTestRuns => Solution.GetProject(Projects.DebuggerSamplesTestRuns);
 
     Project DebuggerUnreferencedExternal => Solution.GetProject(Projects.DebuggerUnreferencedExternal);
@@ -68,6 +71,7 @@ partial class Build
 
     Target CompileDebuggerIntegrationTestsSamples => _ => _
         .Unlisted()
+        .DependsOn(HackForMissingMsBuildLocation)
         .DependsOn(CompileDebuggerIntegrationTestsDependencies)
         .Requires(() => Framework)
         .Requires(() => MonitoringHomeDirectory != null)
@@ -77,6 +81,11 @@ partial class Build
         {
             DotnetBuild(DebuggerSamples, framework: Framework);
 
+            if (ExceptionReplaySamples.TryGetTargetFrameworks().Contains(Framework))
+            {
+                DotnetBuild(ExceptionReplaySamples, framework: Framework);
+            }
+
             if (!IsWin)
             {
                 // The sample helper in the test library assumes that the sample has
@@ -86,6 +95,15 @@ partial class Build
                     .SetConfiguration(BuildConfiguration)
                     .SetNoWarnDotNetCore3()
                     .SetProject(DebuggerSamples));
+
+                if (ExceptionReplaySamples.TryGetTargetFrameworks().Contains(Framework))
+                {
+                    DotNetPublish(x => x
+                                      .SetFramework(Framework)
+                                      .SetConfiguration(BuildConfiguration)
+                                      .SetNoWarnDotNetCore3()
+                                      .SetProject(ExceptionReplaySamples));
+                }
             }
         });
 

tracer/build/_build/Projects.cs

@@ -32,6 +32,7 @@ public static class Projects
 
     public const string DebuggerIntegrationTests = "Datadog.Trace.Debugger.IntegrationTests";
     public const string DebuggerSamples = "Samples.Probes";
+    public const string ExceptionReplaySamples = "Samples.Debugger.AspNetCore5";
     public const string DebuggerSamplesTestRuns = "Samples.Probes.TestRuns";
     public const string DebuggerUnreferencedExternal = "Samples.Probes.Unreferenced.External";
 

tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs

@@ -449,7 +449,7 @@ private static void InitializeTracer(Stopwatch sw)
                     }
                     else
                     {
-                        Log.Information("Exception Debugging is disabled. To enable it, please set DD_EXCEPTION_DEBUGGING_ENABLED environment variable to 'true'.");
+                        Log.Information("Exception Replay is disabled. To enable it, please set DD_EXCEPTION_REPLAY_ENABLED environment variable to '1'/'true'.");
                     }
                 }
                 catch (Exception ex)

tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.Debugger.cs

@@ -3,6 +3,7 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>
 
+using System;
 using Datadog.Trace.Debugger;
 using Datadog.Trace.Debugger.ExceptionAutoInstrumentation;
 
@@ -105,38 +106,46 @@ internal static class Debugger
             public const string RedactedTypes = "DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES";
 
             /// <summary>
-            /// Configuration key for enabling or disabling Exception Debugging.
+            /// The old configuration key for enabling or disabling Exception Replay.
             /// Default value is false (disabled).
             /// </summary>
-            /// <seealso cref="ExceptionDebuggingSettings.Enabled"/>
+            /// <seealso cref="ExceptionReplaySettings.Enabled"/>
+            [Obsolete]
             public const string ExceptionDebuggingEnabled = "DD_EXCEPTION_DEBUGGING_ENABLED";
 
+            /// <summary>
+            /// Configuration key for enabling or disabling Exception Replay.
+            /// Default value is false (disabled).
+            /// </summary>
+            /// <seealso cref="ExceptionReplaySettings.Enabled"/>
+            public const string ExceptionReplayEnabled = "DD_EXCEPTION_REPLAY_ENABLED";
+
             /// <summary>
             /// Configuration key for the maximum number of frames in a call stack we would like to capture values for.
             /// </summary>
-            /// <seealso cref="ExceptionDebuggingSettings.MaximumFramesToCapture"/>
-            public const string ExceptionDebuggingMaxFramesToCapture = "DD_EXCEPTION_DEBUGGING_MAX_FRAMES_TO_CAPTURE";
+            /// <seealso cref="ExceptionReplaySettings.MaximumFramesToCapture"/>
+            public const string ExceptionReplayMaxFramesToCapture = "DD_EXCEPTION_REPLAY_MAX_FRAMES_TO_CAPTURE";
 
             /// <summary>
             /// Configuration key to enable capturing the variables of all the frames in exception call stack.
             /// Default value is false.
             /// </summary>
-            /// <seealso cref="ExceptionDebuggingSettings.CaptureFullCallStack"/>
-            public const string ExceptionDebuggingCaptureFullCallStackEnabled = "DD_EXCEPTION_DEBUGGING_CAPTURE_FULL_CALLSTACK_ENABLED";
+            /// <seealso cref="ExceptionReplaySettings.CaptureFullCallStack"/>
+            public const string ExceptionReplayCaptureFullCallStackEnabled = "DD_EXCEPTION_REPLAY_CAPTURE_FULL_CALLSTACK_ENABLED";
 
             /// <summary>
             /// Configuration key for the interval used to track exceptions
             /// Default value is <c>1</c>h.
             /// </summary>
-            /// <seealso cref="ExceptionDebuggingSettings.RateLimit"/>
-            public const string RateLimitSeconds = "DD_EXCEPTION_DEBUGGING_RATE_LIMIT_SECONDS";
+            /// <seealso cref="ExceptionReplaySettings.RateLimit"/>
+            public const string RateLimitSeconds = "DD_EXCEPTION_REPLAY_RATE_LIMIT_SECONDS";
 
             /// <summary>
-            /// Configuration key for setting the maximum number of exceptions to be analyzed by Exception Debugging within a 1-second time interval.
+            /// Configuration key for setting the maximum number of exceptions to be analyzed by Exception Replay within a 1-second time interval.
             /// Default value is <c>100</c>.
             /// </summary>
-            /// <seealso cref="ExceptionDebuggingSettings.MaxExceptionAnalysisLimit"/>
-            public const string MaxExceptionAnalysisLimit = "DD_EXCEPTION_DEBUGGING_MAX_EXCEPTION_ANALYSIS_LIMIT";
+            /// <seealso cref="ExceptionReplaySettings.MaxExceptionAnalysisLimit"/>
+            public const string MaxExceptionAnalysisLimit = "DD_EXCEPTION_REPLAY_MAX_EXCEPTION_ANALYSIS_LIMIT";
         }
     }
 }

tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionCaseInstrumentationManager.cs

@@ -115,7 +115,20 @@ private static List<MethodUniqueIdentifier> GetMethodsToRejit(ParticipatingFrame
 
         internal static void Revert(ExceptionCase @case)
         {
-            Log.Information("Reverting {ExceptionCase}", @case);
+            if (@case.Probes == null || @case.Processors == null)
+            {
+                Log.Information("Received empty @case, nothing to revert.");
+                return;
+            }
+
+            try
+            {
+                Log.Information("Reverting {ExceptionCase}", @case);
+            }
+            catch (Exception ex)
+            {
+                Log.Error(ex, "Failed to log an exception case while reverting...");
+            }
 
             foreach (var probe in @case.Probes)
             {

tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebugging.cs

@@ -21,16 +21,16 @@ internal class ExceptionDebugging
     {
         internal static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(ExceptionDebugging));
 
-        private static ExceptionDebuggingSettings? _settings;
+        private static ExceptionReplaySettings? _settings;
         private static int _firstInitialization = 1;
         private static bool _isDisabled;
 
         private static SnapshotUploader? _uploader;
         private static SnapshotSink? _snapshotSink;
 
-        public static ExceptionDebuggingSettings Settings
+        public static ExceptionReplaySettings Settings
         {
-            get => LazyInitializer.EnsureInitialized(ref _settings, ExceptionDebuggingSettings.FromDefaultSource)!;
+            get => LazyInitializer.EnsureInitialized(ref _settings, ExceptionReplaySettings.FromDefaultSource)!;
             private set => _settings = value;
         }
 

tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebuggingProbe.cs

@@ -65,9 +65,14 @@ private bool ShouldInstrument()
             return false;
         }
 
+        /// <summary>
+        /// In .NET 6+ there's a bug that prevents Rejit-related APIs to work properly when Edit and Continue feature is turned on.
+        /// See https://github.com/dotnet/runtime/issues/91963 for addiitonal details.
+        /// </summary>
         private bool CheckIfMethodMayBeOmittedFromCallStack()
         {
-            return FrameworkDescription.Instance.IsCoreClr() && RuntimeHelper.IsNetOnward(6) && Method.Method.DeclaringType?.Assembly != null && RuntimeHelper.IsModuleDebugCompiled(Method.Method.DeclaringType.Assembly);
+            return ExceptionTrackManager.IsEditAndContinueFeatureEnabled &&
+                   FrameworkDescription.Instance.IsCoreClr() && RuntimeHelper.IsNetOnward(6) && Method.Method.DeclaringType?.Assembly != null && RuntimeHelper.IsModuleDebugCompiled(Method.Method.DeclaringType.Assembly);
         }
 
         private void ProcessCase(ExceptionCase @case)

tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionRelatedFrames.cs

@@ -46,15 +46,15 @@ public virtual IEnumerable<ParticipatingFrame> GetAllFlattenedFrames()
                 // For reference, see the following test: ExceptionCaughtAndRethrownAsInnerTest.
 
                 var firstFrame = Frames?.FirstOrDefault();
-                var lastFrameOfInner = InnerFrame.Frames?.FirstOrDefault();
+                var lastFrameOfInner = InnerFrame.Frames?.Reverse().FirstOrDefault();
 
                 var skipDuplicatedMethod = 0;
                 if (lastFrameOfInner?.Method == firstFrame?.Method)
                 {
                     skipDuplicatedMethod = 1;
                 }
 
-                foreach (var frame in InnerFrame.GetAllFlattenedFrames().Skip(skipDuplicatedMethod))
+                foreach (var frame in InnerFrame.GetAllFlattenedFrames().Reverse().Skip(skipDuplicatedMethod).Reverse())
                 {
                     yield return frame;
                 }

tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebuggingSettings.cs → tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplaySettings.cs

@@ -1,4 +1,4 @@
-// <copyright file="ExceptionDebuggingSettings.cs" company="Datadog">
+// <copyright file="ExceptionReplaySettings.cs" company="Datadog">
 // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>
@@ -15,23 +15,25 @@
 
 namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation
 {
-    internal class ExceptionDebuggingSettings
+    internal class ExceptionReplaySettings
     {
         public const int DefaultMaxFramesToCapture = 4;
         public const int DefaultRateLimitSeconds = 60 * 60; // 1 hour
         public const int DefaultMaxExceptionAnalysisLimit = 100;
 
-        public ExceptionDebuggingSettings(IConfigurationSource? source, IConfigurationTelemetry telemetry)
+        public ExceptionReplaySettings(IConfigurationSource? source, IConfigurationTelemetry telemetry)
         {
             source ??= NullConfigurationSource.Instance;
             var config = new ConfigurationBuilder(source, telemetry);
 
-            Enabled = config.WithKeys(ConfigurationKeys.Debugger.ExceptionDebuggingEnabled).AsBool(false);
+#pragma warning disable CS0612 // Type or member is obsolete
+            Enabled = config.WithKeys(ConfigurationKeys.Debugger.ExceptionDebuggingEnabled, ConfigurationKeys.Debugger.ExceptionReplayEnabled).AsBool(false);
+#pragma warning restore CS0612 // Type or member is obsolete
 
-            CaptureFullCallStack = config.WithKeys(ConfigurationKeys.Debugger.ExceptionDebuggingCaptureFullCallStackEnabled).AsBool(false);
+            CaptureFullCallStack = config.WithKeys(ConfigurationKeys.Debugger.ExceptionReplayCaptureFullCallStackEnabled).AsBool(false);
 
             var maximumFramesToCapture = config
-                                        .WithKeys(ConfigurationKeys.Debugger.ExceptionDebuggingMaxFramesToCapture)
+                                        .WithKeys(ConfigurationKeys.Debugger.ExceptionReplayMaxFramesToCapture)
                                         .AsInt32(DefaultMaxFramesToCapture, maxDepth => maxDepth > 0)
                                         .Value;
 
@@ -60,12 +62,12 @@ public ExceptionDebuggingSettings(IConfigurationSource? source, IConfigurationTe
 
         public int MaxExceptionAnalysisLimit { get; }
 
-        public static ExceptionDebuggingSettings FromSource(IConfigurationSource source, IConfigurationTelemetry telemetry)
+        public static ExceptionReplaySettings FromSource(IConfigurationSource source, IConfigurationTelemetry telemetry)
         {
-            return new ExceptionDebuggingSettings(source, telemetry);
+            return new ExceptionReplaySettings(source, telemetry);
         }
 
-        public static ExceptionDebuggingSettings FromDefaultSource()
+        public static ExceptionReplaySettings FromDefaultSource()
         {
             return FromSource(GlobalConfigurationSource.Instance, TelemetryFactory.Config);
         }

tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionTrackManager.cs

@@ -20,6 +20,7 @@
 using Datadog.Trace.Debugger.Snapshots;
 using Datadog.Trace.Debugger.Symbols;
 using Datadog.Trace.Logging;
+using Datadog.Trace.Util;
 using Datadog.Trace.VendoredMicrosoftCode.System.Buffers;
 using Datadog.Trace.Vendors.Serilog.Events;
 using ProbeInfo = Datadog.Trace.Debugger.Expressions.ProbeInfo;
@@ -45,6 +46,8 @@ internal class ExceptionTrackManager
         private static Task? _exceptionProcessorTask;
         private static bool _isInitialized;
 
+        internal static bool IsEditAndContinueFeatureEnabled { get; private set; }
+
         private static async Task StartExceptionProcessingAsync(CancellationToken cancellationToken)
         {
             while (!cancellationToken.IsCancellationRequested)
@@ -189,7 +192,7 @@ private static void ProcessException(Exception exception, int normalizedExHash,
 
             var exceptionTypes = new HashSet<Type>();
             var currentFrame = allParticipatingFrames;
-            var iterationLimit = 5;
+            var iterationLimit = 10;
 
             while (iterationLimit-- >= 0 && currentFrame != null)
             {
@@ -443,8 +446,8 @@ private static string GetNoCaptureReason(ParticipatingFrame frame, ExceptionDebu
             {
                 if (probe.MayBeOmittedFromCallStack)
                 {
-                    // Frame is non-optimized in .NET 6+.
-                    noCaptureReason = $"The method {frame.Method.GetFullyQualifiedName()} could not be captured because it resides in a module that is not optimized. In .NET 6 and later versions, the code must be compiled with optimizations enabled.";
+                    // The process is spawned with `COMPLUS_ForceEnc` & the module of the method is non-optimized.
+                    noCaptureReason = $"The method {frame.Method.GetFullyQualifiedName()} could not be captured because the process is spawned with Edit and Continue feature turned on and the module is compiled as Debug. Set the environment variable `COMPLUS_ForceEnc` to `0`. For further info, visit: https://github.com/dotnet/runtime/issues/91963.";
                 }
                 else if (probe.ProbeStatus == Status.ERROR)
                 {
@@ -522,10 +525,20 @@ public static void Initialize()
             _exceptionProcessorTask = Task.Factory.StartNew(
                                                async () => await StartExceptionProcessingAsync(Cts.Token).ConfigureAwait(false), TaskCreationOptions.LongRunning)
                                           .Unwrap();
-
+            IsEditAndContinueFeatureEnabled = IsEnCFeatureEnabled();
             _isInitialized = true;
         }
 
+        /// <summary>
+        /// In .NET 6+ there's a bug that prevents Rejit-related APIs to work properly when Edit and Continue feature is turned on.
+        /// See https://github.com/dotnet/runtime/issues/91963 for additional details.
+        /// </summary>
+        private static bool IsEnCFeatureEnabled()
+        {
+            var encEnabled = EnvironmentHelpers.GetEnvironmentVariable("COMPLUS_ForceEnc");
+            return !string.IsNullOrEmpty(encEnabled) && (encEnabled == "1" || encEnabled == "true");
+        }
+
         public static bool IsSupportedExceptionType(Type ex) =>
             ex != typeof(BadImageFormatException) &&
             ex != typeof(InvalidProgramException) &&

tracer/src/Datadog.Trace/Debugger/Expressions/MethodScopeMembers.cs

@@ -23,6 +23,7 @@ internal MethodScopeMembers(int numberOfLocals, int numberOfArguments)
         }
 
         Members = ArrayPool<ScopeMember>.Shared.Rent(_initialSize);
+        Array.Clear(Members, 0, Members.Length);
         Exception = null;
         Return = default;
         InvocationTarget = default;
@@ -55,7 +56,7 @@ internal void Dispose()
     {
         if (Members != null)
         {
-            ArrayPool<ScopeMember>.Shared.Return(Members, false);
+            ArrayPool<ScopeMember>.Shared.Return(Members);
         }
     }
 }

tracer/src/Datadog.Tracer.Native/debugger_environment_variables.h

@@ -17,8 +17,9 @@ namespace environment
     // Determines if the Dynamic Instrumentation (aka live debugger) is enabled.
     const WSTRING debugger_enabled = WStr("DD_DYNAMIC_INSTRUMENTATION_ENABLED");
 
-    // Determines if the Exception Debugging product is enabled.
-    const WSTRING exception_debugging_enabled = WStr("DD_EXCEPTION_DEBUGGING_ENABLED");
+    // Determines if the Exception Replay product is enabled.
+    const WSTRING exception_debugging_enabled = WStr("DD_EXCEPTION_DEBUGGING_ENABLED"); // Old name
+    const WSTRING exception_replay_enabled = WStr("DD_EXCEPTION_REPLAY_ENABLED");
 
 } // namespace environment
 } // namespace debugger

tracer/src/Datadog.Tracer.Native/debugger_environment_variables_util.cpp

@@ -9,9 +9,16 @@ bool IsDebuggerEnabled()
     CheckIfTrue(GetEnvironmentValue(environment::debugger_enabled));
 }
 
-bool IsExceptionDebuggingEnabled()
+bool IsExceptionReplayEnabled()
 {
-    CheckIfTrue(GetEnvironmentValue(environment::exception_debugging_enabled));
+    static int sValue = -1;
+    if (sValue == -1)
+    {
+        const auto old_exception_replay_flag = GetEnvironmentValue(environment::exception_debugging_enabled);
+        const auto new_exception_replay_flag = GetEnvironmentValue(environment::exception_replay_enabled);
+        sValue = (IsTrue(old_exception_replay_flag) || IsTrue(new_exception_replay_flag)) ? 1 : 0;
+    }
+    return sValue == 1;
 }
 
 bool IsDebuggerInstrumentAllEnabled()

tracer/src/Datadog.Tracer.Native/debugger_environment_variables_util.h

@@ -9,7 +9,7 @@ namespace debugger
 {
 
 bool IsDebuggerEnabled();
-bool IsExceptionDebuggingEnabled();
+bool IsExceptionReplayEnabled();
 bool IsDebuggerInstrumentAllEnabled();
 
 } // namespace debugger

tracer/src/Datadog.Tracer.Native/debugger_probes_instrumentation_requester.cpp

@@ -197,7 +197,7 @@ DebuggerProbesInstrumentationRequester::DebuggerProbesInstrumentationRequester(
     m_work_offloader(work_offloader),
     m_fault_tolerant_method_duplicator(fault_tolerant_method_duplicator)
 {
-    is_debugger_or_exception_debugging_enabled = IsDebuggerEnabled() || IsExceptionDebuggingEnabled();
+    is_debugger_or_exception_debugging_enabled = IsDebuggerEnabled() || IsExceptionReplayEnabled();
 }
 
 void DebuggerProbesInstrumentationRequester::RemoveProbes(debugger::DebuggerRemoveProbesDefinition* removeProbes,