Skip to content

Commit

Permalink
first draft
Browse files Browse the repository at this point in the history
  • Loading branch information
@sourabh1007
sourabh1007 committed Oct 27, 2023
1 parent 083df87 commit 2a6af07
Show file tree
Hide file tree
Showing 4 changed files with 59 additions and 53 deletions.
2 changes: 1 addition & 1 deletion Microsoft.Azure.Cosmos/src/Telemetry/ClientTelemetryHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ private static List<SystemInfo> RecordSystemUsage(
SystemUsageHistory systemUsageHistory,
bool isDirectConnectionMode)
{
if (systemUsageHistory.Values == null)
if (systemUsageHistory.Values == null || systemUsageHistory.Values.Count == 0)
{
return null;
}
Expand Down
52 changes: 33 additions & 19 deletions Microsoft.Azure.Cosmos/src/Telemetry/ClientTelemetryPayloadWriter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,17 +38,28 @@ public static async Task SerializedPayloadChunksAsync(
foreach (KeyValuePair<OperationInfo, (LongConcurrentHistogram latency, LongConcurrentHistogram requestcharge)> entry in operationInfoSnapshot)
{
long lengthNow = stringBuilder.Length;

OperationInfo payloadForLatency = entry.Key;
payloadForLatency.MetricInfo = new MetricInfo(ClientTelemetryOptions.RequestLatencyName, ClientTelemetryOptions.RequestLatencyUnit);
payloadForLatency.SetAggregators(entry.Value.latency, ClientTelemetryOptions.TicksToMsFactor);

OperationInfo payloadForRequestCharge = payloadForLatency.Copy();
payloadForRequestCharge.MetricInfo = new MetricInfo(ClientTelemetryOptions.RequestChargeName, ClientTelemetryOptions.RequestChargeUnit);
payloadForRequestCharge.SetAggregators(entry.Value.requestcharge, ClientTelemetryOptions.HistogramPrecisionFactor);

string latencyMetrics = JsonConvert.SerializeObject(payloadForLatency);
string requestChargeMetrics = JsonConvert.SerializeObject(payloadForRequestCharge);

// Latency Metrics is not captured then don't add it in payload
string latencyMetrics = string.Empty;
if (entry.Value.latency != null && entry.Value.latency.TotalCount > 0)
{
OperationInfo payloadForLatency = entry.Key;
payloadForLatency.MetricInfo = new MetricInfo(ClientTelemetryOptions.RequestLatencyName, ClientTelemetryOptions.RequestLatencyUnit);
payloadForLatency.SetAggregators(entry.Value.latency, ClientTelemetryOptions.TicksToMsFactor);

latencyMetrics = JsonConvert.SerializeObject(payloadForLatency);
}

// RequestCharge Metrics is not captured then don't add it in payload
string requestChargeMetrics = string.Empty;
if (entry.Value.requestcharge != null && entry.Value.requestcharge.TotalCount > 0)
{
OperationInfo payloadForRequestCharge = entry.Key;
payloadForRequestCharge.MetricInfo = new MetricInfo(ClientTelemetryOptions.RequestChargeName, ClientTelemetryOptions.RequestChargeUnit);
payloadForRequestCharge.SetAggregators(entry.Value.requestcharge, ClientTelemetryOptions.HistogramPrecisionFactor);

requestChargeMetrics = JsonConvert.SerializeObject(payloadForRequestCharge);
}

int thisSectionLength = latencyMetrics.Length + requestChargeMetrics.Length;
if (lengthNow + thisSectionLength > ClientTelemetryOptions.PayloadSizeThreshold)
Expand All @@ -64,7 +75,6 @@ public static async Task SerializedPayloadChunksAsync(
writer.WriteRawValue(latencyMetrics);
writer.WriteRawValue(requestChargeMetrics);
}

}
writer.WriteEndArray();

Expand All @@ -76,14 +86,18 @@ public static async Task SerializedPayloadChunksAsync(
foreach (KeyValuePair<CacheRefreshInfo, LongConcurrentHistogram> entry in cacheRefreshInfoSnapshot)
{
long lengthNow = stringBuilder.Length;

CacheRefreshInfo payloadForLatency = entry.Key;
payloadForLatency.MetricInfo = new MetricInfo(ClientTelemetryOptions.RequestLatencyName, ClientTelemetryOptions.RequestLatencyUnit);
payloadForLatency.SetAggregators(entry.Value, ClientTelemetryOptions.TicksToMsFactor);

string latencyMetrics = JsonConvert.SerializeObject(payloadForLatency);
string cacheLatencyMetrics = string.Empty;
if (entry.Value != null && entry.Value.TotalCount > 0)
{
CacheRefreshInfo payloadForLatency = entry.Key;
payloadForLatency.MetricInfo = new MetricInfo(ClientTelemetryOptions.RequestLatencyName, ClientTelemetryOptions.RequestLatencyUnit);
payloadForLatency.SetAggregators(entry.Value, ClientTelemetryOptions.TicksToMsFactor);

if (lengthNow + latencyMetrics.Length > ClientTelemetryOptions.PayloadSizeThreshold)
cacheLatencyMetrics = JsonConvert.SerializeObject(payloadForLatency);
}

if (lengthNow + cacheLatencyMetrics.Length > ClientTelemetryOptions.PayloadSizeThreshold)
{
writer.WriteEndArray();
writer.WriteEndObject();
Expand All @@ -93,7 +107,7 @@ public static async Task SerializedPayloadChunksAsync(
writer = ClientTelemetryPayloadWriter.GetWriterWithSectionStartTag(stringBuilder, properties, "cacheRefreshInfo");
}

writer.WriteRawValue(latencyMetrics);
writer.WriteRawValue(cacheLatencyMetrics);
}
writer.WriteEndArray();

Expand Down
30 changes: 16 additions & 14 deletions Microsoft.Azure.Cosmos/src/Telemetry/NetworkDataRecorder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,23 +24,13 @@ public void Record(List<StoreResponseStatistics> storeResponseStatistics, string
{
if (NetworkDataRecorder.IsStatusCodeNotExcluded((int)storeStatistics.StoreResult.StatusCode, (int)storeStatistics.StoreResult.SubStatusCode))
{
if (NetworkDataRecorder.IsUserOrServerError((int)storeStatistics.StoreResult.StatusCode))
{
RequestInfo requestInfo = this.CreateRequestInfo(storeStatistics, databaseId, containerId);
LongConcurrentHistogram latencyHist = this.RequestInfoErrorBucket.GetOrAdd(requestInfo, x => new LongConcurrentHistogram(ClientTelemetryOptions.RequestLatencyMin,
ClientTelemetryOptions.RequestLatencyMax,
ClientTelemetryOptions.RequestLatencyPrecision));
latencyHist.RecordValue(storeStatistics.RequestLatency.Ticks);
RequestInfo requestInfo = this.CreateRequestInfo(storeStatistics, databaseId, containerId);
ConcurrentDictionary<RequestInfo, LongConcurrentHistogram> bucket = NetworkDataRecorder.IsUserOrServerError((int)storeStatistics.StoreResult.StatusCode) ? this.RequestInfoErrorBucket : this.RequestInfoHighLatencyBucket;

}
else
{
RequestInfo requestInfo = this.CreateRequestInfo(storeStatistics, databaseId, containerId);
LongConcurrentHistogram latencyHist = this.RequestInfoHighLatencyBucket.GetOrAdd(requestInfo, x => new LongConcurrentHistogram(ClientTelemetryOptions.RequestLatencyMin,
LongConcurrentHistogram latencyHist = bucket.GetOrAdd(requestInfo, x => new LongConcurrentHistogram(ClientTelemetryOptions.RequestLatencyMin,
ClientTelemetryOptions.RequestLatencyMax,
ClientTelemetryOptions.RequestLatencyPrecision));
latencyHist.RecordValue(storeStatistics.RequestLatency.Ticks);
}
latencyHist.RecordValue(storeStatistics.RequestLatency.Ticks);
}
}
}
Expand All @@ -53,6 +43,12 @@ ConcurrentDictionary<RequestInfo, LongConcurrentHistogram> requestInfoErrorList
List<RequestInfo> allRequests = new List<RequestInfo>();
foreach (KeyValuePair<RequestInfo, LongConcurrentHistogram> entry in requestInfoErrorList)
{
// No metrics available to record
if (entry.Value == null || entry.Value.TotalCount == 0)
{
continue;
}

MetricInfo metricInfo = new MetricInfo(ClientTelemetryOptions.RequestLatencyName, ClientTelemetryOptions.RequestLatencyUnit);
metricInfo.SetAggregators(entry.Value, ClientTelemetryOptions.TicksToMsFactor);

Expand All @@ -73,6 +69,12 @@ ConcurrentDictionary<RequestInfo, LongConcurrentHistogram> requestInfoHighLatenc
List<RequestInfo> allRequests = new List<RequestInfo>();
foreach (KeyValuePair<RequestInfo, LongConcurrentHistogram> entry in requestInfoHighLatencyList)
{
// No metrics available to record
if (entry.Value == null || entry.Value.TotalCount == 0)
{
continue;
}

MetricInfo metricInfo = new MetricInfo(ClientTelemetryOptions.RequestLatencyName, ClientTelemetryOptions.RequestLatencyUnit);
metricInfo.SetAggregators(entry.Value, ClientTelemetryOptions.TicksToMsFactor);

Expand Down
28 changes: 9 additions & 19 deletions Microsoft.Azure.Cosmos/src/Telemetry/TelemetrySystemUsage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,8 @@ public static SystemInfo GetCpuInfo(IReadOnlyCollection<SystemUsageLoad> systemU
}
}

if (histogram.TotalCount > 0)
{
systemInfo.SetAggregators(histogram, ClientTelemetryOptions.HistogramPrecisionFactor);
}

systemInfo.SetAggregators(histogram, ClientTelemetryOptions.HistogramPrecisionFactor);

return systemInfo;
}

Expand All @@ -70,10 +67,7 @@ public static SystemInfo GetMemoryRemainingInfo(IReadOnlyCollection<SystemUsageL
}
}

if (histogram.TotalCount > 0)
{
systemInfo.SetAggregators(histogram, ClientTelemetryOptions.KbToMbFactor);
}
systemInfo.SetAggregators(histogram, ClientTelemetryOptions.KbToMbFactor);

return systemInfo;
}
Expand All @@ -99,10 +93,7 @@ public static SystemInfo GetAvailableThreadsInfo(IReadOnlyCollection<SystemUsage
}
}

if (histogram.TotalCount > 0)
{
systemInfo.SetAggregators(histogram);
}
systemInfo.SetAggregators(histogram);

return systemInfo;
}
Expand Down Expand Up @@ -153,10 +144,7 @@ public static SystemInfo GetThreadWaitIntervalInMs(IReadOnlyCollection<SystemUsa
}
}

if (histogram.TotalCount > 0)
{
systemInfo.SetAggregators(histogram, ClientTelemetryOptions.TicksToMsFactor);
}
systemInfo.SetAggregators(histogram, ClientTelemetryOptions.TicksToMsFactor);

return systemInfo;
}
Expand Down Expand Up @@ -189,11 +177,13 @@ public static SystemInfo GetTcpConnectionCount(IReadOnlyCollection<SystemUsageLo
}
}

if (histogram.TotalCount > 0)
if (histogram.TotalCount == 0)
{
systemInfo.SetAggregators(histogram);
return null;
}

systemInfo.SetAggregators(histogram);

return systemInfo;
}
}
Expand Down

0 comments on commit 2a6af07

Please sign in to comment.