diff --git a/discovery/discovery.go b/discovery/discovery.go index d8a4b78a87..82452df854 100644 --- a/discovery/discovery.go +++ b/discovery/discovery.go @@ -94,7 +94,13 @@ func (o *orchestratorPool) GetOrchestrators(ctx context.Context, numOrchestrator return caps.CompatibleWith(info.Capabilities) } getOrchInfo := func(ctx context.Context, od common.OrchestratorDescriptor, infoCh chan common.OrchestratorDescriptor, errCh chan error) { + started := time.Now() info, err := serverGetOrchInfo(ctx, o.bcast, od.LocalInfo.URL) + if err == nil { + if monitor.Enabled { + monitor.LogDiscoveryLatency(ctx, info.GetTicketParams().Recipient, time.Since(started).Seconds()) + } + } if err == nil && isCompatible(info) { od.RemoteInfo = info infoCh <- od diff --git a/monitor/census.go b/monitor/census.go index 989898f568..ba42208862 100644 --- a/monitor/census.go +++ b/monitor/census.go @@ -129,6 +129,7 @@ type ( mStreamEnded *stats.Int64Measure mMaxSessions *stats.Int64Measure mCurrentSessions *stats.Int64Measure + mDiscoveryLatency *stats.Float64Measure mDiscoveryError *stats.Int64Measure mTranscodeRetried *stats.Int64Measure mTranscodersNumber *stats.Int64Measure @@ -287,6 +288,7 @@ func InitCensus(nodeType NodeType, version string) { census.mStreamEnded = stats.Int64("stream_ended_total", "StreamEnded", "tot") census.mMaxSessions = stats.Int64("max_sessions_total", "MaxSessions", "tot") census.mCurrentSessions = stats.Int64("current_sessions_total", "Number of currently transcoded streams", "tot") + census.mDiscoveryLatency = stats.Float64("discovery_latency", "Time to complete discovery request", "tot") census.mDiscoveryError = stats.Int64("discovery_errors_total", "Number of discover errors", "tot") census.mTranscodeRetried = stats.Int64("transcode_retried", "Number of times segment transcode was retried", "tot") census.mTranscodersNumber = stats.Int64("transcoders_number", "Number of transcoders currently connected to orchestrator", "tot") @@ -634,6 +636,13 @@ func InitCensus(nodeType NodeType, version string) { TagKeys: baseTags, Aggregation: view.LastValue(), }, + { + Name: "discovery_latency", + Measure: census.mDiscoveryLatency, + Description: "Time to complete discovery request in seconds", + TagKeys: append([]tag.Key{census.kOrchestratorURI}, baseTags...), + Aggregation: view.LastValue(), + }, { Name: "discovery_errors_total", Measure: census.mDiscoveryError, @@ -963,6 +972,13 @@ func manifestIDTagAndIP(ctx context.Context, others ...tag.Mutator) []tag.Mutato return others } +func LogDiscoveryLatency(ctx context.Context, address []byte, latency float64) { + if err := stats.RecordWithTags(census.ctx, + []tag.Mutator{tag.Insert(census.kOrchestratorAddress, common.BytesToAddress(address).Hex())}, + census.mDiscoveryLatency.M(latency)); err != nil { + clog.Errorf(ctx, "Error recording metrics err=%q", err) + } +} // LogDiscoveryError records discovery error func LogDiscoveryError(ctx context.Context, uri, code string) { if strings.Contains(code, "OrchestratorCapped") {