add discovery latency logging

ad-astra-video · Jan 20, 2023 · 064c2ff · 064c2ff
1 parent c227285
commit 064c2ff
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 0 deletions.
diff --git a/discovery/discovery.go b/discovery/discovery.go
@@ -94,7 +94,13 @@ func (o *orchestratorPool) GetOrchestrators(ctx context.Context, numOrchestrator
 		return caps.CompatibleWith(info.Capabilities)
 	}
 	getOrchInfo := func(ctx context.Context, od common.OrchestratorDescriptor, infoCh chan common.OrchestratorDescriptor, errCh chan error) {
+		started := time.Now()
 		info, err := serverGetOrchInfo(ctx, o.bcast, od.LocalInfo.URL)
+		if err == nil {
+			if monitor.Enabled {
+				monitor.LogDiscoveryLatency(ctx, info.GetTicketParams().Recipient, time.Since(started).Seconds())
+			}
+		}
 		if err == nil && isCompatible(info) {
 			od.RemoteInfo = info
 			infoCh <- od

diff --git a/monitor/census.go b/monitor/census.go
@@ -129,6 +129,7 @@ type (
 		mStreamEnded                  *stats.Int64Measure
 		mMaxSessions                  *stats.Int64Measure
 		mCurrentSessions              *stats.Int64Measure
+		mDiscoveryLatency             *stats.Float64Measure
 		mDiscoveryError               *stats.Int64Measure
 		mTranscodeRetried             *stats.Int64Measure
 		mTranscodersNumber            *stats.Int64Measure
@@ -287,6 +288,7 @@ func InitCensus(nodeType NodeType, version string) {
 	census.mStreamEnded = stats.Int64("stream_ended_total", "StreamEnded", "tot")
 	census.mMaxSessions = stats.Int64("max_sessions_total", "MaxSessions", "tot")
 	census.mCurrentSessions = stats.Int64("current_sessions_total", "Number of currently transcoded streams", "tot")
+	census.mDiscoveryLatency = stats.Float64("discovery_latency", "Time to complete discovery request", "tot")
 	census.mDiscoveryError = stats.Int64("discovery_errors_total", "Number of discover errors", "tot")
 	census.mTranscodeRetried = stats.Int64("transcode_retried", "Number of times segment transcode was retried", "tot")
 	census.mTranscodersNumber = stats.Int64("transcoders_number", "Number of transcoders currently connected to orchestrator", "tot")
@@ -634,6 +636,13 @@ func InitCensus(nodeType NodeType, version string) {
 			TagKeys:     baseTags,
 			Aggregation: view.LastValue(),
 		},
+		{
+			Name:        "discovery_latency",
+			Measure:     census.mDiscoveryLatency,
+			Description: "Time to complete discovery request in seconds",
+			TagKeys:     append([]tag.Key{census.kOrchestratorURI}, baseTags...),
+			Aggregation: view.LastValue(),
+		},
 		{
 			Name:        "discovery_errors_total",
 			Measure:     census.mDiscoveryError,
@@ -963,6 +972,13 @@ func manifestIDTagAndIP(ctx context.Context, others ...tag.Mutator) []tag.Mutato
 	return others
 }
 
+func LogDiscoveryLatency(ctx context.Context, address []byte, latency float64) {
+	if err := stats.RecordWithTags(census.ctx,
+		[]tag.Mutator{tag.Insert(census.kOrchestratorAddress, common.BytesToAddress(address).Hex())},
+		census.mDiscoveryLatency.M(latency)); err != nil {
+			clog.Errorf(ctx, "Error recording metrics err=%q", err)
+	}
+}
 // LogDiscoveryError records discovery error
 func LogDiscoveryError(ctx context.Context, uri, code string) {
 	if strings.Contains(code, "OrchestratorCapped") {