Add cloud symweb support (#4848)

Add back -mi (add symweb symbol server), -nocache and -interface options
to setsymbolserver command.

Add back --internal-server support to dotnet-symbol.

Clean up the HttpSymbolStore.cs - fix HttpResponse not being properly
disposed. Removed the separate authenticated client instance/var.

Fix dotnet-symbol for some heap dumps. Some didn't have coreclr.dll's
debug directory contents. Catch error and continue downloading files.

eng/Versions.props

@@ -27,6 +27,7 @@
   <PropertyGroup>
     <!-- Opt-in/out repo features -->
     <UsingToolXliff>false</UsingToolXliff>
+    <AzureIdentityVersion>1.12.0</AzureIdentityVersion>
     <!-- Uncomment this line to use the custom version of roslyn as needed. -->
     <!-- <UsingToolMicrosoftNetCompilers Condition="'$(DotNetBuildSourceOnly)' != 'true'">true</UsingToolMicrosoftNetCompilers> -->
     <!-- CoreFX -->
@@ -48,6 +49,7 @@
     <SystemBuffersVersion>4.5.1</SystemBuffersVersion>
     <SystemMemoryVersion>4.5.5</SystemMemoryVersion>
     <SystemRuntimeLoaderVersion>4.3.0</SystemRuntimeLoaderVersion>
+    <SystemThreadingTasksExtensionsVersion>4.5.4</SystemThreadingTasksExtensionsVersion>
     <SystemTextEncodingsWebVersion>8.0.0</SystemTextEncodingsWebVersion>
     <SystemTextJsonVersion>8.0.4</SystemTextJsonVersion>
     <XUnitAbstractionsVersion>2.0.3</XUnitAbstractionsVersion>

src/Microsoft.Diagnostics.DebugServices.Implementation/Microsoft.Diagnostics.DebugServices.Implementation.csproj

@@ -21,6 +21,7 @@
     <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
     <PackageReference Include="System.Runtime.Loader" Version="$(SystemRuntimeLoaderVersion)" />
     <PackageReference Include="System.Text.Json" Version="$(SystemTextJsonVersion)" />
+    <PackageReference Include="System.Threading.Tasks.Extensions" Version="$(SystemThreadingTasksExtensionsVersion)" />
   </ItemGroup>
 
   <ItemGroup>

src/Microsoft.Diagnostics.DebugServices.Implementation/SymbolService.cs

@@ -7,11 +7,15 @@
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Net.Http.Headers;
 using System.Reflection.Metadata;
 using System.Reflection.PortableExecutable;
 using System.Runtime.InteropServices;
 using System.Text;
 using System.Threading;
+using System.Threading.Tasks;
+using Azure.Core;
+using Azure.Identity;
 using Microsoft.FileFormats;
 using Microsoft.FileFormats.ELF;
 using Microsoft.FileFormats.MachO;
@@ -32,6 +36,9 @@ public class SymbolService : ISymbolService
         /// Symbol server URLs
         /// </summary>
         public const string MsdlSymbolServer = "https://msdl.microsoft.com/download/symbols/";
+        public const string SymwebSymbolServer = "https://symweb.azurefd.net/";
+
+        private static string _symwebHost = new Uri(SymwebSymbolServer).Host;
 
         private readonly IHost _host;
         private string _defaultSymbolCache;
@@ -207,9 +214,20 @@ void ParseServer(int start)
                     }
                     if (symbolServerPath != null)
                     {
-                        if (!AddSymbolServer(symbolServerPath: symbolServerPath.Trim()))
+                        symbolServerPath = symbolServerPath.Trim();
+                        if (IsSymweb(symbolServerPath))
                         {
-                            return false;
+                            if (!AddSymwebSymbolServer(includeInteractiveCredentials: false))
+                            {
+                                return false;
+                            }
+                        }
+                        else
+                        {
+                            if (!AddSymbolServer(symbolServerPath))
+                            {
+                                return false;
+                            }
                         }
                     }
                     foreach (string symbolCachePath in symbolCachePaths.Reverse<string>())
@@ -226,19 +244,89 @@ void ParseServer(int start)
             return true;
         }
 
+        /// <summary>
+        /// Add the cloud symweb symbol server with authentication.
+        /// </summary>
+        /// <param name="includeInteractiveCredentials">specifies whether credentials requiring user interaction will be included in the default authentication flow</param>
+        /// <param name="timeoutInMinutes">symbol server timeout in minutes (optional uses <see cref="DefaultTimeout"/> if null)</param>
+        /// <param name="retryCount">number of retries (optional uses <see cref="DefaultRetryCount"/> if null)</param>
+        /// <returns>if false, failure</returns>
+        public bool AddSymwebSymbolServer(
+            bool includeInteractiveCredentials = false,
+            int? timeoutInMinutes = null,
+            int? retryCount = null)
+        {
+            TokenCredential tokenCredential = new DefaultAzureCredential(includeInteractiveCredentials);
+            AccessToken accessToken;
+            async ValueTask<AuthenticationHeaderValue> authenticationFunc(CancellationToken token)
+            {
+                try
+                {
+                    if (accessToken.ExpiresOn <= DateTimeOffset.UtcNow.AddMinutes(2))
+                    {
+                        accessToken = await tokenCredential.GetTokenAsync(new TokenRequestContext(["api://af9e1c69-e5e9-4331-8cc5-cdf93d57bafa/.default"]), token).ConfigureAwait(false);
+                    }
+                    return new AuthenticationHeaderValue("Bearer", accessToken.Token);
+                }
+                catch (Exception ex) when (ex is CredentialUnavailableException or AuthenticationFailedException)
+                {
+                    Trace.TraceError($"AddSymwebSymbolServer: {ex}");
+                    return null;
+                }
+            }
+            return AddSymbolServer(SymwebSymbolServer, timeoutInMinutes, retryCount, authenticationFunc);
+        }
+
+        /// <summary>
+        /// Add symbol server to search path. The server URL can be the cloud symweb.
+        /// </summary>
+        /// <param name="accessToken">PAT or access token</param>
+        /// <param name="symbolServerPath">symbol server url (optional, uses <see cref="DefaultSymbolPath"/> if null)</param>
+        /// <param name="timeoutInMinutes">symbol server timeout in minutes (optional uses <see cref="DefaultTimeout"/> if null)</param>
+        /// <param name="retryCount">number of retries (optional uses <see cref="DefaultRetryCount"/> if null)</param>
+        /// <returns>if false, failure</returns>
+        public bool AddAuthenticatedSymbolServer(
+            string accessToken,
+            string symbolServerPath = null,
+            int? timeoutInMinutes = null,
+            int? retryCount = null)
+        {
+            if (accessToken == null)
+            {
+                throw new ArgumentNullException(nameof(accessToken));
+            }
+            AuthenticationHeaderValue authenticationValue = new("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($":{accessToken}")));
+            return AddSymbolServer(symbolServerPath, timeoutInMinutes, retryCount, (_) => new ValueTask<AuthenticationHeaderValue>(authenticationValue));
+        }
+
         /// <summary>
         /// Add symbol server to search path.
         /// </summary>
         /// <param name="symbolServerPath">symbol server url (optional, uses <see cref="DefaultSymbolPath"/> if null)</param>
-        /// <param name="authToken">PAT for secure symbol server (optional)</param>
         /// <param name="timeoutInMinutes">symbol server timeout in minutes (optional uses <see cref="DefaultTimeout"/> if null)</param>
         /// <param name="retryCount">number of retries (optional uses <see cref="DefaultRetryCount"/> if null)</param>
         /// <returns>if false, failure</returns>
         public bool AddSymbolServer(
             string symbolServerPath = null,
-            string authToken = null,
             int? timeoutInMinutes = null,
             int? retryCount = null)
+        {
+            return AddSymbolServer(symbolServerPath, timeoutInMinutes, retryCount, authenticationFunc: null);
+        }
+
+        /// <summary>
+        /// Add symbol server to search path.
+        /// </summary>
+        /// <param name="symbolServerPath">symbol server url (optional, uses <see cref="DefaultSymbolPath"/> if null)</param>
+        /// <param name="timeoutInMinutes">symbol server timeout in minutes (optional uses <see cref="DefaultTimeout"/> if null)</param>
+        /// <param name="retryCount">number of retries (optional uses <see cref="DefaultRetryCount"/> if null)</param>
+        /// <param name="authenticationFunc">function that returns the authentication value for a request</param>
+        /// <returns>if false, failure</returns>
+        public bool AddSymbolServer(
+            string symbolServerPath,
+            int? timeoutInMinutes,
+            int? retryCount,
+            Func<CancellationToken, ValueTask<AuthenticationHeaderValue>> authenticationFunc)
         {
             // Add symbol server URL if exists
             symbolServerPath ??= DefaultSymbolPath;
@@ -260,9 +348,11 @@ public bool AddSymbolServer(
                 if (!IsDuplicateSymbolStore<HttpSymbolStore>(store, (httpSymbolStore) => uri.Equals(httpSymbolStore.Uri)))
                 {
                     // Create http symbol server store
-                    HttpSymbolStore httpSymbolStore = new(Tracer.Instance, store, uri, personalAccessToken: authToken);
-                    httpSymbolStore.Timeout = TimeSpan.FromMinutes(timeoutInMinutes.GetValueOrDefault(DefaultTimeout));
-                    httpSymbolStore.RetryCount = retryCount.GetValueOrDefault(DefaultRetryCount);
+                    HttpSymbolStore httpSymbolStore = new(Tracer.Instance, store, uri, authenticationFunc)
+                    {
+                        Timeout = TimeSpan.FromMinutes(timeoutInMinutes.GetValueOrDefault(DefaultTimeout)),
+                        RetryCount = retryCount.GetValueOrDefault(DefaultRetryCount)
+                    };
                     SetSymbolStore(httpSymbolStore);
                 }
             }
@@ -953,6 +1043,23 @@ public override string ToString()
             return sb.ToString();
         }
 
+        /// <summary>
+        /// Returns true if cloud symweb server
+        /// </summary>
+        /// <param name="server"></param>
+        private static bool IsSymweb(string server)
+        {
+            try
+            {
+                Uri uri = new(server);
+                return uri.Host.Equals(_symwebHost, StringComparison.OrdinalIgnoreCase);
+            }
+            catch (Exception ex) when (ex is UriFormatException or InvalidOperationException)
+            {
+                return false;
+            }
+        }
+
         /// <summary>
         /// Attempts to download/retrieve from cache the key.
         /// </summary>

src/Microsoft.Diagnostics.DebugServices/ISymbolService.cs

@@ -53,15 +53,43 @@ public interface ISymbolService
         /// <returns>if false, error parsing symbol path</returns>
         bool ParseSymbolPath(string symbolPath);
 
+        /// <summary>
+        /// Add the cloud symweb symbol server with authentication.
+        /// </summary>
+        /// <param name="includeInteractiveCredentials">specifies whether credentials requiring user interaction will be included in the default authentication flow</param>
+        /// <param name="timeoutInMinutes">symbol server timeout in minutes (optional uses <see cref="DefaultTimeout"/> if null)</param>
+        /// <param name="retryCount">number of retries (optional uses <see cref="DefaultRetryCount"/> if null)</param>
+        /// <returns>if false, failure</returns>
+        public bool AddSymwebSymbolServer(
+            bool includeInteractiveCredentials = false,
+            int? timeoutInMinutes = null,
+            int? retryCount = null);
+
+        /// <summary>
+        /// Add symbol server to search path with a PAT.
+        /// </summary>
+        /// <param name="accessToken">PAT or access token</param>
+        /// <param name="symbolServerPath">symbol server url (optional, uses <see cref="DefaultSymbolPath"/> if null)</param>
+        /// <param name="timeoutInMinutes">symbol server timeout in minutes (optional uses <see cref="DefaultTimeout"/> if null)</param>
+        /// <param name="retryCount">number of retries (optional uses <see cref="DefaultRetryCount"/> if null)</param>
+        /// <returns>if false, failure</returns>
+        public bool AddAuthenticatedSymbolServer(
+            string accessToken,
+            string symbolServerPath = null,
+            int? timeoutInMinutes = null,
+            int? retryCount = null);
+
         /// <summary>
         /// Add symbol server to search path.
         /// </summary>
         /// <param name="symbolServerPath">symbol server url (optional, uses <see cref="DefaultSymbolPath"/> if null)</param>
-        /// <param name="authToken">PAT for secure symbol server (optional)</param>
-        /// <param name="timeoutInMinutes">symbol server timeout in minutes (optional, uses <see cref="DefaultTimeout"/> if null)</param>
-        /// <param name="retryCount">number of retries (optional, uses <see cref="DefaultRetryCount"/> if null)</param>
+        /// <param name="timeoutInMinutes">symbol server timeout in minutes (optional uses <see cref="DefaultTimeout"/> if null)</param>
+        /// <param name="retryCount">number of retries (optional uses <see cref="DefaultRetryCount"/> if null)</param>
         /// <returns>if false, failure</returns>
-        bool AddSymbolServer(string symbolServerPath = null, string authToken = null, int? timeoutInMinutes = null, int? retryCount = null);
+        public bool AddSymbolServer(
+            string symbolServerPath = null,
+            int? timeoutInMinutes = null,
+            int? retryCount = null);
 
         /// <summary>
         /// Add cache path to symbol search path

src/Microsoft.Diagnostics.ExtensionCommands/Host/SetSymbolServerCommand.cs

@@ -1,6 +1,9 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System;
+using System.Net.Http.Headers;
+using System.Text;
 using Microsoft.Diagnostics.DebugServices;
 
 namespace Microsoft.Diagnostics.ExtensionCommands
@@ -18,7 +21,13 @@ public class SetSymbolServerCommand : CommandBase
         [Option(Name = "--ms", Aliases = new string[] { "-ms" }, Help = "Use the public Microsoft symbol server.")]
         public bool MicrosoftSymbolServer { get; set; }
 
-        [Option(Name = "--disable", Aliases = new string[] { "-disable" }, Help = "Clear or disable symbol download support.")]
+        [Option(Name = "--mi", Aliases = new string[] { "-mi" }, Help = "Use the internal symweb symbol server.")]
+        public bool InternalSymbolServer { get; set; }
+
+        [Option(Name = "--interactive", Aliases = new string[] { "-interactive" }, Help = "Allows user interaction will be included in the authentication flow.")]
+        public bool Interactive { get; set; }
+
+        [Option(Name = "--disable", Aliases = new string[] { "-disable", "-clear" }, Help = "Clear or disable symbol download support.")]
         public bool Disable { get; set; }
 
         [Option(Name = "--reset", Aliases = new string[] { "-reset" }, Help = "Reset the HTTP symbol servers clearing any cached failures.")]
@@ -27,6 +36,9 @@ public class SetSymbolServerCommand : CommandBase
         [Option(Name = "--cache", Aliases = new string[] { "-cache" }, Help = "Specify a symbol cache directory.")]
         public string Cache { get; set; }
 
+        [Option(Name = "--nocache", Aliases = new string[] { "-nocache" }, Help = "Do not automatically add the default cache before a server.")]
+        public bool NoCache { get; set; }
+
         [Option(Name = "--directory", Aliases = new string[] { "-directory" }, Help = "Specify a directory to search for symbols.")]
         public string Directory { get; set; }
 
@@ -47,9 +59,13 @@ public class SetSymbolServerCommand : CommandBase
 
         public override void Invoke()
         {
-            if (MicrosoftSymbolServer && !string.IsNullOrEmpty(SymbolServerUrl))
+            if (MicrosoftSymbolServer && InternalSymbolServer)
             {
-                throw new DiagnosticsException("Cannot have -ms option and a symbol server path");
+                throw new DiagnosticsException("Cannot have both -ms and -mi options");
+            }
+            if ((MicrosoftSymbolServer || InternalSymbolServer) && !string.IsNullOrEmpty(SymbolServerUrl))
+            {
+                throw new DiagnosticsException("Cannot have -ms or -mi option and a symbol server path");
             }
             if (Disable)
             {
@@ -59,13 +75,24 @@ public override void Invoke()
             {
                 SymbolService.Reset();
             }
-            if (MicrosoftSymbolServer || !string.IsNullOrEmpty(SymbolServerUrl))
+            if (MicrosoftSymbolServer || InternalSymbolServer || !string.IsNullOrEmpty(SymbolServerUrl))
             {
-                if (string.IsNullOrEmpty(Cache))
+                if (string.IsNullOrEmpty(Cache) && !NoCache)
                 {
                     Cache = SymbolService.DefaultSymbolCache;
                 }
-                SymbolService.AddSymbolServer(SymbolServerUrl, AccessToken, Timeout, RetryCount);
+                if (InternalSymbolServer)
+                {
+                    SymbolService.AddSymwebSymbolServer(includeInteractiveCredentials: Interactive, Timeout, RetryCount);
+                }
+                else if (AccessToken is not null)
+                {
+                    SymbolService.AddAuthenticatedSymbolServer(AccessToken, SymbolServerUrl, Timeout, RetryCount);
+                }
+                else
+                {
+                    SymbolService.AddSymbolServer(SymbolServerUrl, Timeout, RetryCount);
+                }
             }
             if (!string.IsNullOrEmpty(Cache))
             {

src/Microsoft.SymbolStore/KeyGenerators/PEFileKeyGenerator.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
@@ -61,7 +62,7 @@ public override IEnumerable<SymbolStoreKey> GetKeys(KeyTypeFlags flags)
                     {
                         pdbs = _peFile.Pdbs.ToArray();
                     }
-                    catch (InvalidVirtualAddressException ex)
+                    catch (Exception ex) when (ex is InvalidVirtualAddressException || ex is BadInputFormatException)
                     {
                         Tracer.Error("Reading PDB records for {0}: {1}", _path, ex.Message);
                     }

src/Microsoft.SymbolStore/Microsoft.SymbolStore.csproj

@@ -15,10 +15,7 @@
 
   <ItemGroup>
     <PackageReference Include="System.Reflection.Metadata" Version="$(SystemReflectionMetadataVersion)" />
-<!--
-    <PackageReference Condition="'$(TargetFramework)' != 'net462'" Include="System.Reflection.Metadata" Version="$(SystemReflectionMetadataVersion)" />
-    <PackageReference Condition="'$(TargetFramework)' == 'net462'" Include="System.Reflection.Metadata" Version="1.6.0" />
--->
+    <PackageReference Include="System.Threading.Tasks.Extensions" Version="$(SystemThreadingTasksExtensionsVersion)" />
   </ItemGroup>
 
   <ItemGroup>