From f440f1b175558895da7836c3f2bd77fc87595a0d Mon Sep 17 00:00:00 2001 From: Maya-Painter Date: Thu, 20 Jul 2023 20:00:38 -0700 Subject: [PATCH] initial commit --- .../src/Diagnostics/CosmosDiagnostics.cs | 10 ++ .../src/Diagnostics/CosmosTraceDiagnostics.cs | 32 ++++ .../Core/Metrics/BackendAccumulatedMetrics.cs | 140 ++++++++++++++++++ .../Core/Metrics/RuntimeExecutionTimes.cs | 6 +- .../Metrics/BackendAccumulatedMetricsTests.cs | 13 ++ 5 files changed, 198 insertions(+), 3 deletions(-) create mode 100644 Microsoft.Azure.Cosmos/src/Query/Core/Metrics/BackendAccumulatedMetrics.cs create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/Metrics/BackendAccumulatedMetricsTests.cs diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnostics.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnostics.cs index cb3ffe6c04..4c753b0b27 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnostics.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnostics.cs @@ -43,6 +43,16 @@ public virtual int GetFailedRequestCount() throw new NotImplementedException($"{nameof(CosmosDiagnostics)}.{nameof(GetFailedRequestCount)}"); } + /// + /// This represents the accumulated backend query metrics for the request. + /// + /// The string represemation of the query metrics dictionary. + public virtual BackendAccumulatedMetrics GetQueryMetrics() + { + // Default implementation avoids breaking change for users upgrading. + throw new NotImplementedException($"{nameof(CosmosDiagnostics)}.{nameof(GetQueryMetrics)}"); + } + /// /// Gets the string field instance in the Azure CosmosDB database service. /// diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosTraceDiagnostics.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosTraceDiagnostics.cs index 07e340f0ac..5c610de166 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosTraceDiagnostics.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosTraceDiagnostics.cs @@ -9,6 +9,7 @@ namespace Microsoft.Azure.Cosmos.Diagnostics using System.Linq; using System.Text; using Microsoft.Azure.Cosmos.Json; + using Microsoft.Azure.Cosmos.Query.Core.Metrics; using Microsoft.Azure.Cosmos.Tracing; using Microsoft.Azure.Cosmos.Tracing.TraceData; using static Microsoft.Azure.Cosmos.Tracing.TraceData.ClientSideRequestStatisticsTraceDatum; @@ -49,11 +50,42 @@ public override TimeSpan GetClientElapsedTime() return this.Value?.Summary?.RegionsContacted; } + public override BackendAccumulatedMetrics GetQueryMetrics() + { + QueryMetrics queryMetrics = QueryMetrics.Empty; + this.WalkTraceTreeForQueryMetrics(this.Value, ref queryMetrics); + return new BackendAccumulatedMetrics(queryMetrics); + } + internal bool IsGoneExceptionHit() { return this.WalkTraceTreeForGoneException(this.Value); } + private bool WalkTraceTreeForQueryMetrics(ITrace currentTrace, ref QueryMetrics queryMetrics) + { + if (currentTrace == null) + { + return false; + } + + foreach (object datums in currentTrace.Data.Values) + { + if (datums is QueryMetricsTraceDatum queryMetricsTraceDatum) + { + queryMetrics += queryMetricsTraceDatum.QueryMetrics; + return true; + } + } + + foreach (ITrace childTrace in currentTrace.Children) + { + this.WalkTraceTreeForQueryMetrics(childTrace, ref queryMetrics); + } + + return false; + } + private bool WalkTraceTreeForGoneException(ITrace currentTrace) { if (currentTrace == null) diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Metrics/BackendAccumulatedMetrics.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Metrics/BackendAccumulatedMetrics.cs new file mode 100644 index 0000000000..b377e6b713 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Metrics/BackendAccumulatedMetrics.cs @@ -0,0 +1,140 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ +namespace Microsoft.Azure.Cosmos +{ + using System; + using Microsoft.Azure.Cosmos.Query.Core.Metrics; + + /// + /// Exposed metrics received for queries from the backend. + /// + public sealed class BackendAccumulatedMetrics + { + internal BackendAccumulatedMetrics(QueryMetrics queryMetrics) + { + this.RetrievedDocumentCount = queryMetrics.BackendMetrics.RetrievedDocumentCount; + this.RetrievedDocumentSize = queryMetrics.BackendMetrics.RetrievedDocumentSize; + this.OutputDocumentCount = queryMetrics.BackendMetrics.OutputDocumentCount; + this.OutputDocumentSize = queryMetrics.BackendMetrics.OutputDocumentSize; + this.IndexHitRatio = queryMetrics.BackendMetrics.IndexHitRatio; + this.TotalTime = queryMetrics.BackendMetrics.TotalTime; + this.QueryCompilationTime = queryMetrics.BackendMetrics.QueryPreparationTimes.QueryCompilationTime; + this.LogicalPlanBuildTime = queryMetrics.BackendMetrics.QueryPreparationTimes.LogicalPlanBuildTime; + this.PhysicalPlanBuildTime = queryMetrics.BackendMetrics.QueryPreparationTimes.PhysicalPlanBuildTime; + this.QueryOptimizationTime = queryMetrics.BackendMetrics.QueryPreparationTimes.QueryOptimizationTime; + this.IndexLookupTime = queryMetrics.BackendMetrics.IndexLookupTime; + this.DocumentLoadTime = queryMetrics.BackendMetrics.DocumentLoadTime; + this.VMExecutionTime = queryMetrics.BackendMetrics.VMExecutionTime; + this.SystemFunctionExecutionTime = queryMetrics.BackendMetrics.RuntimeExecutionTimes.SystemFunctionExecutionTime; + this.UserDefinedFunctionExecutionTime = queryMetrics.BackendMetrics.RuntimeExecutionTimes.UserDefinedFunctionExecutionTime; + this.QueryEngineExecutionTime = queryMetrics.BackendMetrics.RuntimeExecutionTimes.QueryEngineExecutionTime; + this.DocumentWriteTime = queryMetrics.BackendMetrics.DocumentWriteTime; + } + + /// + /// Gets the total query time in the Azure Cosmos database service. + /// + public TimeSpan TotalTime { get; } + + /// + /// Gets the query compile time in the Azure DocumentDB database service. + /// + public TimeSpan QueryCompilationTime { get; } + + /// + /// Gets the query logical plan build time in the Azure DocumentDB database service. + /// + public TimeSpan LogicalPlanBuildTime { get; } + + /// + /// Gets the query physical plan build time in the Azure DocumentDB database service. + /// + public TimeSpan PhysicalPlanBuildTime { get; } + + /// + /// Gets the query optimization time in the Azure DocumentDB database service. + /// + public TimeSpan QueryOptimizationTime { get; } + + /// + /// Gets the number of documents retrieved during query in the Azure Cosmos database service. + /// + public long RetrievedDocumentCount { get; } + + /// + /// Gets the size of documents retrieved in bytes during query in the Azure Cosmos DB service. + /// + public long RetrievedDocumentSize { get; } + + /// + /// Gets the number of documents returned by query in the Azure Cosmos DB service. + /// + public long OutputDocumentCount { get; } + + /// + /// Gets the size of documents outputted in bytes during query in the Azure Cosmos database service. + /// + public long OutputDocumentSize { get; } + + /// + /// Gets the query QueryPreparationTimes in the Azure Cosmos database service. + /// + public TimeSpan QueryPreparationTimes { get; } + + /// + /// Gets the query index lookup time in the Azure Cosmos database service. + /// + public TimeSpan IndexLookupTime { get; } + + /// + /// Gets the document loading time during query in the Azure Cosmos database service. + /// + public TimeSpan DocumentLoadTime { get; } + + /// + /// Gets the query runtime execution times during query in the Azure Cosmos database service. + /// + public TimeSpan RuntimeExecutionTimes { get; } + + /// + /// Gets the output writing/serializing time during query in the Azure Cosmos database service. + /// + public TimeSpan DocumentWriteTime { get; } + + /// + /// Gets the index hit ratio by query in the Azure Cosmos database service. + /// + public double IndexHitRatio { get; } + + /// + /// Gets the VMExecution Time. + /// + public TimeSpan VMExecutionTime { get; } + + /// + /// Gets the query system function execution time in the Azure Cosmos DB service. + /// + public TimeSpan SystemFunctionExecutionTime { get; } + + /// + /// Gets the query user defined function execution time in the Azure Cosmos DB service. + /// + public TimeSpan UserDefinedFunctionExecutionTime { get; } + + /// + /// Gets the total query runtime execution time in the Azure Cosmos DB service. + /// + public TimeSpan QueryEngineExecutionTime { get; } + + /// + /// String representation of the QueryBackendMetrics. + /// + /// Metric text + public override string ToString() + { + return $"totalExecutionTimeInMs={this.TotalTime.TotalMilliseconds};queryCompileTimeInMs={this.QueryCompilationTime.TotalMilliseconds};queryLogicalPlanBuildTimeInMs={this.LogicalPlanBuildTime.TotalMilliseconds};queryPhysicalPlanBuildTimeInMs={this.PhysicalPlanBuildTime.TotalMilliseconds};queryOptimizationTimeInMs={this.QueryOptimizationTime.TotalMilliseconds};indexLookupTimeInMs={this.IndexLookupTime.TotalMilliseconds};documentLoadTimeInMs={this.DocumentLoadTime.TotalMilliseconds};systemFunctionExecuteTimeInMs={this.SystemFunctionExecutionTime.TotalMilliseconds};userFunctionExecuteTimeInMs={this.UserDefinedFunctionExecutionTime.TotalMilliseconds};retrievedDocumentCount={this.RetrievedDocumentCount};retrievedDocumentSize={this.RetrievedDocumentSize};outputDocumentCount={this.OutputDocumentCount};outputDocumentSize={this.OutputDocumentSize};writeOutputTimeInMs={this.DocumentWriteTime.TotalMilliseconds};indexUtilizationRatio={this.IndexHitRatio}"; + } + + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Metrics/RuntimeExecutionTimes.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Metrics/RuntimeExecutionTimes.cs index 0fd8dc22f0..9a4dadc6f3 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Metrics/RuntimeExecutionTimes.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Metrics/RuntimeExecutionTimes.cs @@ -62,9 +62,9 @@ public Accumulator(TimeSpan queryEngineExecutionTime, TimeSpan systemFunctionExe this.UserDefinedFunctionExecutionTime = userDefinedFunctionExecutionTimes; } - public TimeSpan QueryEngineExecutionTime { get; set; } - public TimeSpan SystemFunctionExecutionTime { get; set; } - public TimeSpan UserDefinedFunctionExecutionTime { get; set; } + public TimeSpan QueryEngineExecutionTime { get; } + public TimeSpan SystemFunctionExecutionTime { get; } + public TimeSpan UserDefinedFunctionExecutionTime { get; } public Accumulator Accumulate(RuntimeExecutionTimes runtimeExecutionTimes) { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/Metrics/BackendAccumulatedMetricsTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/Metrics/BackendAccumulatedMetricsTests.cs new file mode 100644 index 0000000000..7b05c1fd71 --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/Metrics/BackendAccumulatedMetricsTests.cs @@ -0,0 +1,13 @@ +namespace Microsoft.Azure.Cosmos.Tests.Query.Metrics +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + + internal class BackendAccumulatedMetricsTests + { + //TODO + } +}