ClientTelemetry : Adds logic to call client config in every 10 minutes

Microsoft.Azure.Cosmos/src/Handler/DiagnosticsHandlerHelper.cs

@@ -6,6 +6,7 @@ namespace Microsoft.Azure.Cosmos.Handler
 {
     using System;
     using System.Collections.Generic;
+    using System.Threading.Tasks;
     using Documents.Rntbd;
     using Microsoft.Azure.Cosmos.Core.Trace;
 
@@ -21,16 +22,14 @@ internal class DiagnosticsHandlerHelper
         private const string Telemetrykey = "telemetry";
 
         public static readonly TimeSpan DiagnosticsRefreshInterval = TimeSpan.FromSeconds(10);
-        private static readonly SystemUsageRecorder DiagnosticSystemUsageRecorder = new SystemUsageRecorder(
+        private static readonly TimeSpan ClientTelemetryRefreshInterval = TimeSpan.FromSeconds(5);
+
+        // Need to reset it in Tests hence kept it non-readonly.
+        private static SystemUsageRecorder DiagnosticSystemUsageRecorder = new SystemUsageRecorder(
             identifier: Diagnostickey,
             historyLength: 6,
             refreshInterval: DiagnosticsHandlerHelper.DiagnosticsRefreshInterval);
-
-        private static readonly TimeSpan ClientTelemetryRefreshInterval = TimeSpan.FromSeconds(5);
-        private static readonly SystemUsageRecorder TelemetrySystemUsageRecorder = new SystemUsageRecorder(
-            identifier: Telemetrykey,
-            historyLength: 120,
-            refreshInterval: DiagnosticsHandlerHelper.ClientTelemetryRefreshInterval);
+        private static SystemUsageRecorder TelemetrySystemUsageRecorder = null;
 
         /// <summary>
         /// Singleton to make sure only one instance of DiagnosticHandlerHelper is there.
@@ -62,13 +61,13 @@ public static void Refresh(bool isClientTelemetryEnabled)
         {
             if (isClientTelemetryEnabled != DiagnosticsHandlerHelper.isTelemetryMonitoringEnabled)
             {
-                DiagnosticsHandlerHelper.Instance.StopSystemMonitor();
+                DiagnosticsHandlerHelper tempInstance = DiagnosticsHandlerHelper.Instance;
 
-                // Update telemetry flag
                 DiagnosticsHandlerHelper.isTelemetryMonitoringEnabled = isClientTelemetryEnabled;
-
-                // Create new instance, it will start a new system monitor job
                 DiagnosticsHandlerHelper.Instance = new DiagnosticsHandlerHelper();
+
+                // Stopping the monitor is a blocking call so we do it in a separate thread
+                _ = Task.Run(() => tempInstance.StopSystemMonitor());
             }
         }
 
@@ -102,8 +101,18 @@ private DiagnosticsHandlerHelper()
 
                 if (DiagnosticsHandlerHelper.isTelemetryMonitoringEnabled)
                 {
+                    // re-initialize a fresh telemetry recorder when feature is switched on
+                    DiagnosticsHandlerHelper.TelemetrySystemUsageRecorder = new SystemUsageRecorder(
+                                                                                   identifier: Telemetrykey,
+                                                                                   historyLength: 120,
+                                                                                   refreshInterval: DiagnosticsHandlerHelper.ClientTelemetryRefreshInterval);
+
                     recorders.Add(DiagnosticsHandlerHelper.TelemetrySystemUsageRecorder);
                 }
+                else
+                {
+                    DiagnosticsHandlerHelper.TelemetrySystemUsageRecorder = null;
+                }
 
                 this.systemUsageMonitor = SystemUsageMonitor.CreateAndStart(recorders);
 
@@ -154,7 +163,7 @@ public SystemUsageHistory GetClientTelemetrySystemHistory()
 
             try
             {
-                return DiagnosticsHandlerHelper.TelemetrySystemUsageRecorder.Data;
+                return DiagnosticsHandlerHelper.TelemetrySystemUsageRecorder?.Data;
             }
             catch (Exception ex)
             {

Microsoft.Azure.Cosmos/src/Telemetry/TelemetryToServiceHelper.cs

@@ -21,7 +21,8 @@ internal class TelemetryToServiceHelper : IDisposable
     {
         private ITelemetryCollector collector = new TelemetryCollectorNoOp();
 
-        internal static int DefaultBackgroundRefreshClientConfigTimeIntervalInMS = (int)TimeSpan.FromMinutes(10).TotalMilliseconds;
+        internal static TimeSpan DefaultBackgroundRefreshClientConfigTimeInterval 
+            = TimeSpan.FromMinutes(10);
 
         private readonly AuthorizationTokenProvider cosmosAuthorization;
         private readonly CosmosHttpClient httpClient;
@@ -73,7 +74,12 @@ public static TelemetryToServiceHelper CreateAndInitializeClientConfigAndTelemet
             }
 
             TelemetryToServiceHelper helper = new TelemetryToServiceHelper(
-                clientId, connectionPolicy, cosmosAuthorization, httpClient, serviceEndpoint, globalEndpointManager, cancellationTokenSource);
+                clientId: clientId, 
+                connectionPolicy: connectionPolicy, 
+                cosmosAuthorization: cosmosAuthorization, 
+                httpClient: httpClient, serviceEndpoint, 
+                globalEndpointManager: globalEndpointManager, 
+                cancellationTokenSource: cancellationTokenSource);
 
             _ = helper.RetrieveConfigAndInitiateTelemetryAsync(); // Let it run in backgroud
 
@@ -86,15 +92,26 @@ private async Task RetrieveConfigAndInitiateTelemetryAsync()
             try
             {
                 Uri serviceEndpointWithPath = new Uri(this.serviceEnpoint + Paths.ClientConfigPathSegment);
-
-                TryCatch<AccountClientConfiguration> databaseAccountClientConfigs = await this.GetDatabaseAccountClientConfigAsync(this.cosmosAuthorization, this.httpClient, serviceEndpointWithPath);
-                if (databaseAccountClientConfigs.Succeeded)
-                {
-                    this.InitializeClientTelemetry(databaseAccountClientConfigs.Result);
-                }
-                else if (!this.cancellationTokenSource.IsCancellationRequested)
+                while (!this.cancellationTokenSource.IsCancellationRequested)
                 {
-                    DefaultTrace.TraceWarning($"Exception while calling client config " + databaseAccountClientConfigs.Exception.ToString());
+                    TryCatch<AccountClientConfiguration> databaseAccountClientConfigs = await this.GetDatabaseAccountClientConfigAsync(
+                        cosmosAuthorization: this.cosmosAuthorization,
+                        httpClient: this.httpClient, 
+                        clientConfigEndpoint: serviceEndpointWithPath);
+
+                    if (databaseAccountClientConfigs.Succeeded)
+                    {
+                        this.InitializeClientTelemetry(
+                            clientConfig: databaseAccountClientConfigs.Result);
+                    }
+                    else if (!this.cancellationTokenSource.IsCancellationRequested)
+                    {
+                        DefaultTrace.TraceWarning($"Exception while calling client config " + databaseAccountClientConfigs.Exception.ToString());
+                    }
+
+                    await Task.Delay(
+                        delay: TelemetryToServiceHelper.DefaultBackgroundRefreshClientConfigTimeInterval,
+                        cancellationToken: this.cancellationTokenSource.Token);
                 }
             }
             catch (Exception ex)
@@ -125,14 +142,28 @@ await cosmosAuthorization.AddAuthorizationHeaderAsync(
                         timeoutPolicy: HttpTimeoutPolicyControlPlaneRead.Instance,
                         clientSideRequestStatistics: null,
                         cancellationToken: default))
-                    using (DocumentServiceResponse documentServiceResponse = await ClientExtensions.ParseResponseAsync(responseMessage))
                     {
-                        return TryCatch<AccountClientConfiguration>.FromResult(CosmosResource.FromStream<AccountClientConfiguration>(documentServiceResponse));
+                        // It means feature flag is off at gateway, then log the exception and retry after defined interval.
+                        // If feature flag is OFF at gateway, SDK won't refresh the latest state of the flag.
+                        if (responseMessage.StatusCode == System.Net.HttpStatusCode.BadRequest)
+                        {
+                            throw new InvalidOperationException("Client Config API is not enabled at compute gateway.");
+                        }
+
+                        using (DocumentServiceResponse documentServiceResponse = await ClientExtensions.ParseResponseAsync(responseMessage))
+                        {
+                            return TryCatch<AccountClientConfiguration>.FromResult(CosmosResource.FromStream<AccountClientConfiguration>(documentServiceResponse));
+                        }
                     }
                 }
+                catch (ObjectDisposedException)
+                {
+                    throw new OperationCanceledException($"Client is being disposed for {clientConfigEndpoint} at {DateTime.UtcNow}");
+                }
                 catch (Exception ex)
                 {
-                    DefaultTrace.TraceWarning($"Exception while calling client config " + ex.StackTrace);
+                    // Do not log if exception is due to client dispose.
+                    DefaultTrace.TraceWarning($"Exception while calling client config " + ex);
                     return TryCatch<AccountClientConfiguration>.FromException(ex);
                 }
             }
@@ -153,10 +184,16 @@ public bool IsClientTelemetryJobRunning()
         /// </summary>
         private void InitializeClientTelemetry(AccountClientConfiguration clientConfig)
         {
+            // If state of the job is same as state of the flag, then no need to do anything.
+            if (clientConfig.IsClientTelemetryEnabled() == this.IsClientTelemetryJobRunning()) 
+            {
+                return;
+            }
+
             DiagnosticsHandlerHelper.Refresh(clientConfig.IsClientTelemetryEnabled());
 
             if (clientConfig.IsClientTelemetryEnabled())
-            {
+            { 
                 try
                 {
                     this.clientTelemetry = ClientTelemetry.CreateAndStartBackgroundTelemetry(
@@ -180,6 +217,11 @@ private void InitializeClientTelemetry(AccountClientConfiguration clientConfig)
                     this.connectionPolicy.EnableClientTelemetry = false;
                 }
             }
+            else
+            {
+                this.StopClientTelemetry();
+                DefaultTrace.TraceVerbose("Client Telemetry Disabled.");    
+            }
         }
 
         public void Dispose()
@@ -197,6 +239,8 @@ private void StopClientTelemetry()
 
             this.clientTelemetry?.Dispose();
             this.clientTelemetry = null;
+
+            DiagnosticsHandlerHelper.Refresh(isClientTelemetryEnabled: false);
         }
     }
 }