Skip to content

Commit

Permalink
Improve loss detection in case of senders of loss
Browse files Browse the repository at this point in the history
  • Loading branch information
@streamer45
streamer45 committed Oct 7, 2024
1 parent 0fb55c3 commit 9ee3933
Show file tree
Hide file tree
Showing 8 changed files with 100 additions and 32 deletions.
35 changes: 29 additions & 6 deletions lib/rtc_monitor.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
});
};
import { EventEmitter } from 'events';
import { newRTCLocalInboundStats, newRTCLocalOutboundStats, newRTCRemoteInboundStats, newRTCCandidatePairStats } from './rtc_stats';
import { newRTCLocalInboundStats, newRTCLocalOutboundStats, newRTCRemoteInboundStats, newRTCRemoteOutboundStats, newRTCCandidatePairStats } from './rtc_stats';
export const mosThreshold = 3.5;
export class RTCMonitor extends EventEmitter {
constructor(cfg) {
Expand All @@ -28,6 +28,7 @@ export class RTCMonitor extends EventEmitter {
lastLocalIn: {},
lastLocalOut: {},
lastRemoteIn: {},
lastRemoteOut: {},
};
}
start() {
Expand All @@ -37,7 +38,7 @@ export class RTCMonitor extends EventEmitter {
this.logger.logDebug('RTCMonitor: starting');
this.intervalID = setInterval(this.gatherStats, this.cfg.monitorInterval);
}
getLocalInQualityStats(localIn) {
getLocalInQualityStats(localIn, remoteOut) {
const stats = {};
let totalTime = 0;
let totalPacketsReceived = 0;
Expand All @@ -53,7 +54,23 @@ export class RTCMonitor extends EventEmitter {
}
const tsDiff = stat.timestamp - this.stats.lastLocalIn[ssrc].timestamp;
const receivedDiff = stat.packetsReceived - this.stats.lastLocalIn[ssrc].packetsReceived;
const lostDiff = stat.packetsLost - this.stats.lastLocalIn[ssrc].packetsLost;
// Tracking loss on the receiving end is a bit more tricky because packets are
// forwarded without much modification by the server so if the sender is having issues, these are
// propagated to the receiver side which may believe it's having problems as a consequence.
//
// What we want to know instead is whether the local side is having issues on the
// server -> receiver path rather than sender -> server -> receiver one.
// To do this we check for any mismatches in packets sent by the remote and packets
// received by us.
//
// Note: it's expected for local.packetsReceived to be slightly higher than remote.packetsSent
// since reports are generated at different times, with the local one likely being more time-accurate.
//
// Having remote.packetsSent higher than local.packetsReceived is instead a fairly good sign
// some packets have been lost in transit.
const potentiallyLost = remoteOut[ssrc].packetsSent - stat.packetsReceived;
const prevPotentiallyLost = this.stats.lastRemoteOut[ssrc].packetsSent - this.stats.lastLocalIn[ssrc].packetsReceived;
const lostDiff = prevPotentiallyLost >= 0 && potentiallyLost > prevPotentiallyLost ? potentiallyLost - prevPotentiallyLost : 0;
totalTime += tsDiff;
totalPacketsReceived += receivedDiff;
totalPacketsLost += lostDiff;
Expand Down Expand Up @@ -88,7 +105,7 @@ export class RTCMonitor extends EventEmitter {
totalTime += tsDiff;
totalRemoteJitter += stat.jitter;
totalRTT += stat.roundTripTime;
totalLossRate = stat.fractionLost;
totalLossRate += stat.fractionLost;
totalRemoteStats++;
}
if (totalRemoteStats > 0) {
Expand All @@ -103,6 +120,7 @@ export class RTCMonitor extends EventEmitter {
const localIn = {};
const localOut = {};
const remoteIn = {};
const remoteOut = {};
let candidate;
reports.forEach((report) => {
// Collect necessary stats to make further calculations:
Expand All @@ -123,6 +141,9 @@ export class RTCMonitor extends EventEmitter {
if (report.type === 'remote-inbound-rtp' && report.kind === 'audio') {
remoteIn[report.ssrc] = newRTCRemoteInboundStats(report);
}
if (report.type === 'remote-outbound-rtp' && report.kind === 'audio') {
remoteOut[report.ssrc] = newRTCRemoteOutboundStats(report);
}
});
if (!candidate) {
this.logger.logDebug('RTCMonitor: no valid candidate was found');
Expand All @@ -135,14 +156,15 @@ export class RTCMonitor extends EventEmitter {
transportLatency = (candidate.currentRoundTripTime * 1000) / 2;
}
// Step 2: if receiving any stream, calculate average jitter and loss rate using local stats.
const localInStats = this.getLocalInQualityStats(localIn);
const localInStats = this.getLocalInQualityStats(localIn, remoteOut);
// Step 3: if sending any stream, calculate average latency, jitter and
// loss rate using remote stats.
const remoteInStats = this.getRemoteInQualityStats(remoteIn, localOut);
// Step 4: cache current stats for calculating deltas on next iteration.
this.stats.lastLocalIn = Object.assign({}, localIn);
this.stats.lastLocalOut = Object.assign({}, localOut);
this.stats.lastRemoteIn = Object.assign({}, remoteIn);
this.stats.lastRemoteOut = Object.assign({}, remoteOut);
if (typeof transportLatency === 'undefined' && typeof remoteInStats.avgLatency === 'undefined') {
transportLatency = this.peer.getRTT() / 2;
}
Expand All @@ -160,7 +182,7 @@ export class RTCMonitor extends EventEmitter {
// Step 5 (or the magic step): calculate MOS (Mean Opinion Score)
const mos = this.calculateMOS(latency, jitter, lossRate);
this.emit('mos', mos);
this.peer.handleMetrics(lossRate, this.peer.getRTT(), jitter);
this.peer.handleMetrics(lossRate, jitter / 1000);
this.logger.logDebug(`RTCMonitor: MOS --> ${mos}`);
}
calculateMOS(latency, jitter, lossRate) {
Expand Down Expand Up @@ -198,6 +220,7 @@ export class RTCMonitor extends EventEmitter {
lastLocalIn: {},
lastLocalOut: {},
lastRemoteIn: {},
lastRemoteOut: {},
};
}
}
2 changes: 1 addition & 1 deletion lib/rtc_peer.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ export declare class RTCPeer extends EventEmitter {
replaceTrack(oldTrackID: string, newTrack: MediaStreamTrack | null): void;
removeTrack(trackID: string): void;
getStats(): Promise<RTCStatsReport>;
handleMetrics(lossRate: number, rtt: number, jitter: number): void;
handleMetrics(lossRate: number, jitter: number): void;
static getVideoCodec(mimeType: string): Promise<RTCRtpCodecCapability | null>;
destroy(): void;
}
6 changes: 3 additions & 3 deletions lib/rtc_peer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,13 +341,13 @@ export class RTCPeer extends EventEmitter {
}
return this.pc.getStats(null);
}
handleMetrics(lossRate, rtt, jitter) {
handleMetrics(lossRate, jitter) {
try {
if (lossRate >= 0) {
this.dc.send(encodeDCMsg(this.enc, DCMessageType.LossRate, lossRate));
}
if (rtt > 0) {
this.dc.send(encodeDCMsg(this.enc, DCMessageType.RoundTripTime, rtt));
if (this.rtt > 0) {
this.dc.send(encodeDCMsg(this.enc, DCMessageType.RoundTripTime, this.rtt));
}
if (jitter > 0) {
this.dc.send(encodeDCMsg(this.enc, DCMessageType.Jitter, jitter));
Expand Down
6 changes: 6 additions & 0 deletions lib/rtc_stats.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@ export declare function newRTCRemoteInboundStats(report: any): {
jitter: any;
roundTripTime: any;
};
export declare function newRTCRemoteOutboundStats(report: any): {
timestamp: any;
kind: any;
packetsSent: any;
bytesSent: any;
};
export declare function newRTCCandidatePairStats(report: any, reports: RTCStatsReport): RTCCandidatePairStats;
export declare function parseSSRCStats(reports: RTCStatsReport): SSRCStats;
export declare function parseICEStats(reports: RTCStatsReport): ICEStats;
Expand Down
15 changes: 9 additions & 6 deletions lib/rtc_stats.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,14 @@ export function newRTCRemoteInboundStats(report) {
roundTripTime: report.roundTripTime,
};
}
export function newRTCRemoteOutboundStats(report) {
return {
timestamp: report.timestamp,
kind: report.kind,
packetsSent: report.packetsSent,
bytesSent: report.bytesSent,
};
}
export function newRTCCandidatePairStats(report, reports) {
let local;
let remote;
Expand Down Expand Up @@ -88,12 +96,7 @@ export function parseSSRCStats(reports) {
stats[report.ssrc].remote.in = newRTCRemoteInboundStats(report);
break;
case 'remote-outbound-rtp':
stats[report.ssrc].remote.out = {
timestamp: report.timestamp,
kind: report.kind,
packetsSent: report.packetsSent,
bytesSent: report.bytesSent,
};
stats[report.ssrc].remote.out = newRTCRemoteOutboundStats(report);
break;
}
});
Expand Down
46 changes: 39 additions & 7 deletions src/rtc_monitor.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import {EventEmitter} from 'events';

import {Logger, RTCMonitorConfig, RTCLocalInboundStats, RTCRemoteInboundStats, RTCLocalOutboundStats, RTCCandidatePairStats} from './types';
import {newRTCLocalInboundStats, newRTCLocalOutboundStats, newRTCRemoteInboundStats, newRTCCandidatePairStats} from './rtc_stats';
import {Logger, RTCMonitorConfig, RTCLocalInboundStats, RTCRemoteInboundStats, RTCRemoteOutboundStats, RTCLocalOutboundStats, RTCCandidatePairStats} from './types';
import {newRTCLocalInboundStats, newRTCLocalOutboundStats, newRTCRemoteInboundStats, newRTCRemoteOutboundStats, newRTCCandidatePairStats} from './rtc_stats';
import {RTCPeer} from './rtc_peer';

export const mosThreshold = 3.5;
Expand All @@ -18,10 +18,15 @@ type RemoteInboundStatsMap = {
[key: string]: RTCRemoteInboundStats,
};

type RemoteOutboundStatsMap = {
[key: string]: RTCRemoteOutboundStats,
}

type MonitorStatsSample = {
lastLocalIn: LocalInboundStatsMap,
lastLocalOut: LocalOutboundStatsMap,
lastRemoteIn: RemoteInboundStatsMap,
lastRemoteOut: RemoteOutboundStatsMap,
};

type CallQualityStats = {
Expand All @@ -48,6 +53,7 @@ export class RTCMonitor extends EventEmitter {
lastLocalIn: {},
lastLocalOut: {},
lastRemoteIn: {},
lastRemoteOut: {},
};
}

Expand All @@ -69,7 +75,7 @@ export class RTCMonitor extends EventEmitter {
});
};

private getLocalInQualityStats(localIn: LocalInboundStatsMap) {
private getLocalInQualityStats(localIn: LocalInboundStatsMap, remoteOut: RemoteOutboundStatsMap) {
const stats: CallQualityStats = {};

let totalTime = 0;
Expand All @@ -89,7 +95,24 @@ export class RTCMonitor extends EventEmitter {

const tsDiff = stat.timestamp - this.stats.lastLocalIn[ssrc].timestamp;
const receivedDiff = stat.packetsReceived - this.stats.lastLocalIn[ssrc].packetsReceived;
const lostDiff = stat.packetsLost - this.stats.lastLocalIn[ssrc].packetsLost;

// Tracking loss on the receiving end is a bit more tricky because packets are
// forwarded without much modification by the server so if the sender is having issues, these are
// propagated to the receiver side which may believe it's having problems as a consequence.
//
// What we want to know instead is whether the local side is having issues on the
// server -> receiver path rather than sender -> server -> receiver one.
// To do this we check for any mismatches in packets sent by the remote and packets
// received by us.
//
// Note: it's expected for local.packetsReceived to be slightly higher than remote.packetsSent
// since reports are generated at different times, with the local one likely being more time-accurate.
//
// Having remote.packetsSent higher than local.packetsReceived is instead a fairly good sign
// some packets have been lost in transit.
const potentiallyLost = remoteOut[ssrc].packetsSent - stat.packetsReceived;
const prevPotentiallyLost = this.stats.lastRemoteOut[ssrc].packetsSent - this.stats.lastLocalIn[ssrc].packetsReceived;
const lostDiff = prevPotentiallyLost >= 0 && potentiallyLost > prevPotentiallyLost ? potentiallyLost - prevPotentiallyLost : 0;

totalTime += tsDiff;
totalPacketsReceived += receivedDiff;
Expand Down Expand Up @@ -131,7 +154,7 @@ export class RTCMonitor extends EventEmitter {
totalTime += tsDiff;
totalRemoteJitter += stat.jitter;
totalRTT += stat.roundTripTime;
totalLossRate = stat.fractionLost;
totalLossRate += stat.fractionLost;
totalRemoteStats++;
}

Expand All @@ -149,6 +172,7 @@ export class RTCMonitor extends EventEmitter {
const localIn: LocalInboundStatsMap = {};
const localOut: LocalOutboundStatsMap = {};
const remoteIn: RemoteInboundStatsMap = {};
const remoteOut: RemoteOutboundStatsMap = {};
let candidate: RTCCandidatePairStats | undefined;
reports.forEach((report: any) => {
// Collect necessary stats to make further calculations:
Expand All @@ -173,6 +197,10 @@ export class RTCMonitor extends EventEmitter {
if (report.type === 'remote-inbound-rtp' && report.kind === 'audio') {
remoteIn[report.ssrc] = newRTCRemoteInboundStats(report);
}

if (report.type === 'remote-outbound-rtp' && report.kind === 'audio') {
remoteOut[report.ssrc] = newRTCRemoteOutboundStats(report);
}
});

if (!candidate) {
Expand All @@ -189,7 +217,7 @@ export class RTCMonitor extends EventEmitter {
}

// Step 2: if receiving any stream, calculate average jitter and loss rate using local stats.
const localInStats = this.getLocalInQualityStats(localIn);
const localInStats = this.getLocalInQualityStats(localIn, remoteOut);

// Step 3: if sending any stream, calculate average latency, jitter and
// loss rate using remote stats.
Expand All @@ -205,6 +233,9 @@ export class RTCMonitor extends EventEmitter {
this.stats.lastRemoteIn = {
...remoteIn,
};
this.stats.lastRemoteOut = {
...remoteOut,
};

if (typeof transportLatency === 'undefined' && typeof remoteInStats.avgLatency === 'undefined') {
transportLatency = this.peer.getRTT() / 2;
Expand All @@ -227,7 +258,7 @@ export class RTCMonitor extends EventEmitter {
// Step 5 (or the magic step): calculate MOS (Mean Opinion Score)
const mos = this.calculateMOS(latency!, jitter, lossRate);
this.emit('mos', mos);
this.peer.handleMetrics(lossRate, this.peer.getRTT(), jitter);
this.peer.handleMetrics(lossRate, jitter / 1000);
this.logger.logDebug(`RTCMonitor: MOS --> ${mos}`);
}

Expand Down Expand Up @@ -273,6 +304,7 @@ export class RTCMonitor extends EventEmitter {
lastLocalIn: {},
lastLocalOut: {},
lastRemoteIn: {},
lastRemoteOut: {},
};
}
}
6 changes: 3 additions & 3 deletions src/rtc_peer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -378,13 +378,13 @@ export class RTCPeer extends EventEmitter {
return this.pc.getStats(null);
}

public handleMetrics(lossRate: number, rtt: number, jitter: number) {
public handleMetrics(lossRate: number, jitter: number) {
try {
if (lossRate >= 0) {
this.dc.send(encodeDCMsg(this.enc, DCMessageType.LossRate, lossRate));
}
if (rtt > 0) {
this.dc.send(encodeDCMsg(this.enc, DCMessageType.RoundTripTime, rtt));
if (this.rtt > 0) {
this.dc.send(encodeDCMsg(this.enc, DCMessageType.RoundTripTime, this.rtt));
}
if (jitter > 0) {
this.dc.send(encodeDCMsg(this.enc, DCMessageType.Jitter, jitter));
Expand Down
16 changes: 10 additions & 6 deletions src/rtc_stats.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,15 @@ export function newRTCRemoteInboundStats(report: any) {
};
}

export function newRTCRemoteOutboundStats(report: any) {
return {
timestamp: report.timestamp,
kind: report.kind,
packetsSent: report.packetsSent,
bytesSent: report.bytesSent,
};
}

export function newRTCCandidatePairStats(report: any, reports: RTCStatsReport): RTCCandidatePairStats {
let local;
let remote;
Expand Down Expand Up @@ -98,12 +107,7 @@ export function parseSSRCStats(reports: RTCStatsReport): SSRCStats {
stats[report.ssrc].remote.in = newRTCRemoteInboundStats(report);
break;
case 'remote-outbound-rtp':
stats[report.ssrc].remote.out = {
timestamp: report.timestamp,
kind: report.kind,
packetsSent: report.packetsSent,
bytesSent: report.bytesSent,
};
stats[report.ssrc].remote.out = newRTCRemoteOutboundStats(report);
break;
}
});
Expand Down

0 comments on commit 9ee3933

Please sign in to comment.