From 65b22c267d4e506f8d88296982839d00a1cac9d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Fuente=20P=C3=A9rez?= Date: Wed, 25 Oct 2023 17:07:20 +0200 Subject: [PATCH 1/3] Fix TrackSubscriptionPermissionChanged docstring (#900) --- src/room/events.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/room/events.ts b/src/room/events.ts index ac7fbba0f1..652b3f7869 100644 --- a/src/room/events.ts +++ b/src/room/events.ts @@ -223,7 +223,7 @@ export enum RoomEvent { * be emitted. * * args: (pub: [[RemoteTrackPublication]], - * status: [[TrackPublication.SubscriptionStatus]], + * status: [[TrackPublication.PermissionStatus]], * participant: [[RemoteParticipant]]) */ TrackSubscriptionPermissionChanged = 'trackSubscriptionPermissionChanged', From 6bb1a9de680e05ac5cbaf37d82aa5a5a4681eb22 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Thu, 26 Oct 2023 09:46:43 +0200 Subject: [PATCH 2/3] Update protocol (#902) * update protocol * Create sixty-clocks-fail.md * fix another occurrence --- .changeset/sixty-clocks-fail.md | 5 + protocol | 2 +- src/proto/livekit_models_pb.ts | 155 ++++++++++++++++++++--- src/proto/livekit_rtc_pb.ts | 8 +- src/room/participant/LocalParticipant.ts | 4 - 5 files changed, 147 insertions(+), 27 deletions(-) create mode 100644 .changeset/sixty-clocks-fail.md diff --git a/.changeset/sixty-clocks-fail.md b/.changeset/sixty-clocks-fail.md new file mode 100644 index 0000000000..c5c310ddc5 --- /dev/null +++ b/.changeset/sixty-clocks-fail.md @@ -0,0 +1,5 @@ +--- +"livekit-client": patch +--- + +Update protocol diff --git a/protocol b/protocol index 9d0d6c9e94..07ca9d4e47 160000 --- a/protocol +++ b/protocol @@ -1 +1 @@ -Subproject commit 9d0d6c9e94279679f87056400030c2ce0e05d362 +Subproject commit 07ca9d4e47bda96cb1040ff629799d3344e8f458 diff --git a/src/proto/livekit_models_pb.ts b/src/proto/livekit_models_pb.ts index 76582bc87f..3790087c22 100644 --- a/src/proto/livekit_models_pb.ts +++ b/src/proto/livekit_models_pb.ts @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// @generated by protoc-gen-es v1.3.0 with parameter "target=ts" +// @generated by protoc-gen-es v1.3.3 with parameter "target=ts" // @generated from file livekit_models.proto (package livekit, syntax proto3) /* eslint-disable */ // @ts-nocheck @@ -84,6 +84,26 @@ proto3.util.setEnumType(VideoCodec, "livekit.VideoCodec", [ { no: 4, name: "VP8" }, ]); +/** + * @generated from enum livekit.ImageCodec + */ +export enum ImageCodec { + /** + * @generated from enum value: IC_DEFAULT = 0; + */ + IC_DEFAULT = 0, + + /** + * @generated from enum value: IC_JPEG = 1; + */ + IC_JPEG = 1, +} +// Retrieve enum metadata with: proto3.getEnumType(ImageCodec) +proto3.util.setEnumType(ImageCodec, "livekit.ImageCodec", [ + { no: 0, name: "IC_DEFAULT" }, + { no: 1, name: "IC_JPEG" }, +]); + /** * @generated from enum livekit.TrackType */ @@ -411,11 +431,6 @@ export class Room extends Message { */ activeRecording = false; - /** - * @generated from field: livekit.PlayoutDelay playout_delay = 12; - */ - playoutDelay?: PlayoutDelay; - constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -435,7 +450,6 @@ export class Room extends Message { { no: 9, name: "num_participants", kind: "scalar", T: 13 /* ScalarType.UINT32 */ }, { no: 11, name: "num_publishers", kind: "scalar", T: 13 /* ScalarType.UINT32 */ }, { no: 10, name: "active_recording", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, - { no: 12, name: "playout_delay", kind: "message", T: PlayoutDelay }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): Room { @@ -1282,6 +1296,11 @@ export class UserPacket extends Message { */ participantSid = ""; + /** + * @generated from field: string participant_identity = 5; + */ + participantIdentity = ""; + /** * user defined payload * @@ -1290,12 +1309,19 @@ export class UserPacket extends Message { payload = new Uint8Array(0); /** - * the ID of the participants who will receive the message (the message will be sent to all the people in the room if this variable is empty) + * the ID of the participants who will receive the message (sent to all by default) * * @generated from field: repeated string destination_sids = 3; */ destinationSids: string[] = []; + /** + * identities of participants who will receive the message (sent to all by default) + * + * @generated from field: repeated string destination_identities = 6; + */ + destinationIdentities: string[] = []; + /** * topic under which the message was published * @@ -1312,8 +1338,10 @@ export class UserPacket extends Message { static readonly typeName = "livekit.UserPacket"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ { no: 1, name: "participant_sid", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 5, name: "participant_identity", kind: "scalar", T: 9 /* ScalarType.STRING */ }, { no: 2, name: "payload", kind: "scalar", T: 12 /* ScalarType.BYTES */ }, { no: 3, name: "destination_sids", kind: "scalar", T: 9 /* ScalarType.STRING */, repeated: true }, + { no: 6, name: "destination_identities", kind: "scalar", T: 9 /* ScalarType.STRING */, repeated: true }, { no: 4, name: "topic", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, ]); @@ -1613,6 +1641,16 @@ export enum ClientInfo_SDK { * @generated from enum value: RUST = 8; */ RUST = 8, + + /** + * @generated from enum value: PYTHON = 9; + */ + PYTHON = 9, + + /** + * @generated from enum value: CPP = 10; + */ + CPP = 10, } // Retrieve enum metadata with: proto3.getEnumType(ClientInfo_SDK) proto3.util.setEnumType(ClientInfo_SDK, "livekit.ClientInfo.SDK", [ @@ -1625,6 +1663,8 @@ proto3.util.setEnumType(ClientInfo_SDK, "livekit.ClientInfo.SDK", [ { no: 6, name: "UNITY" }, { no: 7, name: "REACT_NATIVE" }, { no: 8, name: "RUST" }, + { no: 9, name: "PYTHON" }, + { no: 10, name: "CPP" }, ]); /** @@ -1774,6 +1814,91 @@ export class DisabledCodecs extends Message { } } +/** + * @generated from message livekit.RTPDrift + */ +export class RTPDrift extends Message { + /** + * @generated from field: google.protobuf.Timestamp start_time = 1; + */ + startTime?: Timestamp; + + /** + * @generated from field: google.protobuf.Timestamp end_time = 2; + */ + endTime?: Timestamp; + + /** + * @generated from field: double duration = 3; + */ + duration = 0; + + /** + * @generated from field: uint64 start_timestamp = 4; + */ + startTimestamp = protoInt64.zero; + + /** + * @generated from field: uint64 end_timestamp = 5; + */ + endTimestamp = protoInt64.zero; + + /** + * @generated from field: uint64 rtp_clock_ticks = 6; + */ + rtpClockTicks = protoInt64.zero; + + /** + * @generated from field: int64 drift_samples = 7; + */ + driftSamples = protoInt64.zero; + + /** + * @generated from field: double drift_ms = 8; + */ + driftMs = 0; + + /** + * @generated from field: double clock_rate = 9; + */ + clockRate = 0; + + constructor(data?: PartialMessage) { + super(); + proto3.util.initPartial(data, this); + } + + static readonly runtime: typeof proto3 = proto3; + static readonly typeName = "livekit.RTPDrift"; + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: "start_time", kind: "message", T: Timestamp }, + { no: 2, name: "end_time", kind: "message", T: Timestamp }, + { no: 3, name: "duration", kind: "scalar", T: 1 /* ScalarType.DOUBLE */ }, + { no: 4, name: "start_timestamp", kind: "scalar", T: 4 /* ScalarType.UINT64 */ }, + { no: 5, name: "end_timestamp", kind: "scalar", T: 4 /* ScalarType.UINT64 */ }, + { no: 6, name: "rtp_clock_ticks", kind: "scalar", T: 4 /* ScalarType.UINT64 */ }, + { no: 7, name: "drift_samples", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 8, name: "drift_ms", kind: "scalar", T: 1 /* ScalarType.DOUBLE */ }, + { no: 9, name: "clock_rate", kind: "scalar", T: 1 /* ScalarType.DOUBLE */ }, + ]); + + static fromBinary(bytes: Uint8Array, options?: Partial): RTPDrift { + return new RTPDrift().fromBinary(bytes, options); + } + + static fromJson(jsonValue: JsonValue, options?: Partial): RTPDrift { + return new RTPDrift().fromJson(jsonValue, options); + } + + static fromJsonString(jsonString: string, options?: Partial): RTPDrift { + return new RTPDrift().fromJsonString(jsonString, options); + } + + static equals(a: RTPDrift | PlainMessage | undefined, b: RTPDrift | PlainMessage | undefined): boolean { + return proto3.util.equals(RTPDrift, a, b); + } +} + /** * @generated from message livekit.RTPStats */ @@ -1984,16 +2109,16 @@ export class RTPStats extends Message { lastLayerLockPli?: Timestamp; /** - * @generated from field: double sample_rate = 42; + * @generated from field: livekit.RTPDrift packet_drift = 44; */ - sampleRate = 0; + packetDrift?: RTPDrift; /** - * NEXT_ID: 44 + * NEXT_ID: 46 * - * @generated from field: double drift_ms = 43; + * @generated from field: livekit.RTPDrift report_drift = 45; */ - driftMs = 0; + reportDrift?: RTPDrift; constructor(data?: PartialMessage) { super(); @@ -2044,8 +2169,8 @@ export class RTPStats extends Message { { no: 34, name: "last_key_frame", kind: "message", T: Timestamp }, { no: 35, name: "layer_lock_plis", kind: "scalar", T: 13 /* ScalarType.UINT32 */ }, { no: 36, name: "last_layer_lock_pli", kind: "message", T: Timestamp }, - { no: 42, name: "sample_rate", kind: "scalar", T: 1 /* ScalarType.DOUBLE */ }, - { no: 43, name: "drift_ms", kind: "scalar", T: 1 /* ScalarType.DOUBLE */ }, + { no: 44, name: "packet_drift", kind: "message", T: RTPDrift }, + { no: 45, name: "report_drift", kind: "message", T: RTPDrift }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): RTPStats { diff --git a/src/proto/livekit_rtc_pb.ts b/src/proto/livekit_rtc_pb.ts index 81522a4c00..b57a900b1e 100644 --- a/src/proto/livekit_rtc_pb.ts +++ b/src/proto/livekit_rtc_pb.ts @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// @generated by protoc-gen-es v1.3.0 with parameter "target=ts" +// @generated by protoc-gen-es v1.3.3 with parameter "target=ts" // @generated from file livekit_rtc.proto (package livekit, syntax proto3) /* eslint-disable */ // @ts-nocheck @@ -487,11 +487,6 @@ export class SimulcastCodec extends Message { */ cid = ""; - /** - * @generated from field: bool enable_simulcast_layers = 3; - */ - enableSimulcastLayers = false; - constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -502,7 +497,6 @@ export class SimulcastCodec extends Message { static readonly fields: FieldList = proto3.util.newFieldList(() => [ { no: 1, name: "codec", kind: "scalar", T: 9 /* ScalarType.STRING */ }, { no: 2, name: "cid", kind: "scalar", T: 9 /* ScalarType.STRING */ }, - { no: 3, name: "enable_simulcast_layers", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): SimulcastCodec { diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index 332fbf5912..c9f123b8c5 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -701,12 +701,10 @@ export default class LocalParticipant extends Participant { new SimulcastCodec({ codec: opts.videoCodec, cid: track.mediaStreamTrack.id, - enableSimulcastLayers: true, }), new SimulcastCodec({ codec: opts.backupCodec.codec, cid: '', - enableSimulcastLayers: true, }), ]; } else if (opts.videoCodec) { @@ -716,7 +714,6 @@ export default class LocalParticipant extends Participant { new SimulcastCodec({ codec: opts.videoCodec, cid: track.mediaStreamTrack.id, - enableSimulcastLayers: opts.simulcast ?? false, }), ]; } @@ -889,7 +886,6 @@ export default class LocalParticipant extends Participant { { codec: opts.videoCodec, cid: simulcastTrack.mediaStreamTrack.id, - enableSimulcastLayers: opts.simulcast, }, ], }); From 7631712a0eceaf2734c3206d0f135c4d1a095927 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Thu, 26 Oct 2023 09:47:02 +0200 Subject: [PATCH 3/3] Make peerconnection private on PCTransport (#903) * Make peerconnection private on PCTransport * revert enablesimulcast for this branch * Create moody-nails-flash.md * fix connection check --- .changeset/moody-nails-flash.md | 5 + src/connectionHelper/checks/webrtc.ts | 2 +- src/room/PCTransport.ts | 117 +++++++++++++++++++- src/room/RTCEngine.ts | 134 +++++++++-------------- src/room/Room.ts | 10 +- src/room/participant/LocalParticipant.ts | 6 +- 6 files changed, 178 insertions(+), 96 deletions(-) create mode 100644 .changeset/moody-nails-flash.md diff --git a/.changeset/moody-nails-flash.md b/.changeset/moody-nails-flash.md new file mode 100644 index 0000000000..ac429345f7 --- /dev/null +++ b/.changeset/moody-nails-flash.md @@ -0,0 +1,5 @@ +--- +"livekit-client": patch +--- + +Make peerconnection private on PCTransport diff --git a/src/connectionHelper/checks/webrtc.ts b/src/connectionHelper/checks/webrtc.ts index 6461ef06a9..9082d847c7 100644 --- a/src/connectionHelper/checks/webrtc.ts +++ b/src/connectionHelper/checks/webrtc.ts @@ -39,7 +39,7 @@ export class WebRTCCheck extends Checker { }; if (this.room.engine.subscriber) { - this.room.engine.subscriber.pc.onicecandidateerror = (ev) => { + this.room.engine.subscriber.onIceCandidateError = (ev) => { if (ev instanceof RTCPeerConnectionIceErrorEvent) { this.appendWarning( `error with ICE candidate: ${ev.errorCode} ${ev.errorText} ${ev.url}`, diff --git a/src/room/PCTransport.ts b/src/room/PCTransport.ts index 62fa4fe5ed..e6fb95ad95 100644 --- a/src/room/PCTransport.ts +++ b/src/room/PCTransport.ts @@ -32,7 +32,7 @@ export const PCEvents = { export default class PCTransport extends EventEmitter { private _pc: RTCPeerConnection | null; - public get pc() { + private get pc() { if (this._pc) return this._pc; throw new UnexpectedConnectionState('Expected peer connection to be available'); } @@ -51,12 +51,38 @@ export default class PCTransport extends EventEmitter { onOffer?: (offer: RTCSessionDescriptionInit) => void; + onIceCandidate?: (candidate: RTCIceCandidate) => void; + + onIceCandidateError?: (ev: Event) => void; + + onConnectionStateChange?: (state: RTCPeerConnectionState) => void; + + onDataChannel?: (ev: RTCDataChannelEvent) => void; + + onTrack?: (ev: RTCTrackEvent) => void; + constructor(config?: RTCConfiguration, mediaConstraints: Record = {}) { super(); this._pc = isChromiumBased() ? // @ts-expect-error chrome allows additional media constraints to be passed into the RTCPeerConnection constructor new RTCPeerConnection(config, mediaConstraints) : new RTCPeerConnection(config); + this._pc.onicecandidate = (ev) => { + if (!ev.candidate) return; + this.onIceCandidate?.(ev.candidate); + }; + this._pc.onicecandidateerror = (ev) => { + this.onIceCandidateError?.(ev); + }; + this._pc.onconnectionstatechange = () => { + this.onConnectionStateChange?.(this._pc?.connectionState ?? 'closed'); + }; + this._pc.ondatachannel = (ev) => { + this.onDataChannel?.(ev); + }; + this._pc.ontrack = (ev) => { + this.onTrack?.(ev); + }; } get isICEConnected(): boolean { @@ -270,10 +296,99 @@ export default class PCTransport extends EventEmitter { return answer; } + createDataChannel(label: string, dataChannelDict: RTCDataChannelInit) { + return this.pc.createDataChannel(label, dataChannelDict); + } + + addTransceiver(mediaStreamTrack: MediaStreamTrack, transceiverInit: RTCRtpTransceiverInit) { + return this.pc.addTransceiver(mediaStreamTrack, transceiverInit); + } + + addTrack(track: MediaStreamTrack) { + return this.pc.addTrack(track); + } + setTrackCodecBitrate(info: TrackBitrateInfo) { this.trackBitrates.push(info); } + setConfiguration(rtcConfig: RTCConfiguration) { + return this.pc.setConfiguration(rtcConfig); + } + + canRemoveTrack(): boolean { + return !!this.pc.removeTrack; + } + + removeTrack(sender: RTCRtpSender) { + return this.pc.removeTrack(sender); + } + + getConnectionState() { + return this.pc.connectionState; + } + + getICEConnectionState() { + return this.pc.iceConnectionState; + } + + getSignallingState() { + return this.pc.signalingState; + } + + getTransceivers() { + return this.pc.getTransceivers(); + } + + getSenders() { + return this.pc.getSenders(); + } + + getLocalDescription() { + return this.pc.localDescription; + } + + getRemoteDescription() { + return this.pc.remoteDescription; + } + + async getConnectedAddress(): Promise { + if (!this._pc) { + return; + } + let selectedCandidatePairId = ''; + const candidatePairs = new Map(); + // id -> candidate ip + const candidates = new Map(); + const stats: RTCStatsReport = await this._pc.getStats(); + stats.forEach((v) => { + switch (v.type) { + case 'transport': + selectedCandidatePairId = v.selectedCandidatePairId; + break; + case 'candidate-pair': + if (selectedCandidatePairId === '' && v.selected) { + selectedCandidatePairId = v.id; + } + candidatePairs.set(v.id, v); + break; + case 'remote-candidate': + candidates.set(v.id, `${v.address}:${v.port}`); + break; + default: + } + }); + + if (selectedCandidatePairId === '') { + return undefined; + } + const selectedID = candidatePairs.get(selectedCandidatePairId)?.remoteCandidateId; + if (selectedID === undefined) { + return undefined; + } + return candidates.get(selectedID); + } + close() { if (!this._pc) { return; diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts index 8203dc518c..83ccf15498 100644 --- a/src/room/RTCEngine.ts +++ b/src/room/RTCEngine.ts @@ -108,7 +108,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit private subscriberPrimary: boolean = false; - private primaryPC?: RTCPeerConnection; + private primaryTransport?: PCTransport; private pcState: PCState = PCState.New; @@ -247,12 +247,12 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } async cleanupPeerConnections() { - if (this.publisher && this.publisher.pc.signalingState !== 'closed') { - this.publisher.pc.getSenders().forEach((sender) => { + if (this.publisher && this.publisher.getSignallingState() !== 'closed') { + this.publisher.getSenders().forEach((sender) => { try { // TODO: react-native-webrtc doesn't have removeTrack yet. - if (this.publisher?.pc.removeTrack) { - this.publisher?.pc.removeTrack(sender); + if (this.publisher?.canRemoveTrack()) { + this.publisher?.removeTrack(sender); } } catch (e) { log.warn('could not removeTrack', { error: e }); @@ -268,7 +268,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit this.subscriber = undefined; } this.hasPublished = false; - this.primaryPC = undefined; + this.primaryTransport = undefined; const dcCleanup = (dc: RTCDataChannel | undefined) => { if (!dc) return; @@ -336,7 +336,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit delete this.pendingTrackResolvers[sender.track.id]; } try { - this.publisher?.pc.removeTrack(sender); + this.publisher?.removeTrack(sender); return true; } catch (e: unknown) { log.warn('failed to remove track', { error: e, method: 'removeTrack' }); @@ -353,10 +353,10 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } async getConnectedServerAddress(): Promise { - if (this.primaryPC === undefined) { + if (this.primaryTransport === undefined) { return undefined; } - return getConnectedAddress(this.primaryPC); + return this.primaryTransport.getConnectedAddress(); } /* @internal */ @@ -387,40 +387,38 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit this.emit(EngineEvent.TransportsCreated, this.publisher, this.subscriber); - this.publisher.pc.onicecandidate = (ev) => { - if (!ev.candidate) return; - log.trace('adding ICE candidate for peer', ev.candidate); - this.client.sendIceCandidate(ev.candidate, SignalTarget.PUBLISHER); + this.publisher.onIceCandidate = (candidate) => { + log.trace('adding ICE candidate for peer', candidate); + this.client.sendIceCandidate(candidate, SignalTarget.PUBLISHER); }; - this.subscriber.pc.onicecandidate = (ev) => { - if (!ev.candidate) return; - this.client.sendIceCandidate(ev.candidate, SignalTarget.SUBSCRIBER); + this.subscriber.onIceCandidate = (candidate) => { + this.client.sendIceCandidate(candidate, SignalTarget.SUBSCRIBER); }; this.publisher.onOffer = (offer) => { this.client.sendOffer(offer); }; - let primaryPC = this.publisher.pc; - let secondaryPC = this.subscriber.pc; + let primaryTransport = this.publisher; + let secondaryTransport = this.subscriber; let subscriberPrimary = joinResponse.subscriberPrimary; if (subscriberPrimary) { - primaryPC = this.subscriber.pc; - secondaryPC = this.publisher.pc; + primaryTransport = this.subscriber; + secondaryTransport = this.publisher; // in subscriber primary mode, server side opens sub data channels. - this.subscriber.pc.ondatachannel = this.handleDataChannel; + this.subscriber.onDataChannel = this.handleDataChannel; } - this.primaryPC = primaryPC; - primaryPC.onconnectionstatechange = async () => { - log.debug(`primary PC state changed ${primaryPC.connectionState}`); - if (primaryPC.connectionState === 'connected') { + this.primaryTransport = primaryTransport; + primaryTransport.onConnectionStateChange = async (connectionState) => { + log.debug(`primary PC state changed ${connectionState}`); + if (connectionState === 'connected') { const shouldEmit = this.pcState === PCState.New; this.pcState = PCState.Connected; if (shouldEmit) { this.emit(EngineEvent.Connected, joinResponse); } - } else if (primaryPC.connectionState === 'failed') { + } else if (connectionState === 'failed') { // on Safari, PeerConnection will switch to 'disconnected' during renegotiation if (this.pcState === PCState.Connected) { this.pcState = PCState.Disconnected; @@ -434,10 +432,10 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } } }; - secondaryPC.onconnectionstatechange = async () => { - log.debug(`secondary PC state changed ${secondaryPC.connectionState}`); + secondaryTransport.onConnectionStateChange = async (connectionState) => { + log.debug(`secondary PC state changed ${connectionState}`); // also reconnect if secondary peerconnection fails - if (secondaryPC.connectionState === 'failed') { + if (connectionState === 'failed') { this.handleDisconnect( 'secondary peerconnection', subscriberPrimary @@ -447,7 +445,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } }; - this.subscriber.pc.ontrack = (ev: RTCTrackEvent) => { + this.subscriber.onTrack = (ev: RTCTrackEvent) => { this.emit(EngineEvent.MediaTrackAdded, ev.track, ev.streams[0], ev.receiver); }; @@ -462,7 +460,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } log.debug('received server answer', { RTCSdpType: sd.type, - signalingState: this.publisher.pc.signalingState.toString(), + signalingState: this.publisher.getSignallingState().toString(), }); await this.publisher.setRemoteDescription(sd); }; @@ -487,7 +485,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } log.debug('received server offer', { RTCSdpType: sd.type, - signalingState: this.subscriber.pc.signalingState.toString(), + signalingState: this.subscriber.getSignallingState().toString(), }); await this.subscriber.setRemoteDescription(sd); @@ -518,7 +516,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit this.client.onLeave = (leave?: LeaveRequest) => { if (leave?.canReconnect) { this.fullReconnectOnNext = true; - this.primaryPC = undefined; + this.primaryTransport = undefined; // reconnect immediately instead of waiting for next attempt this.handleDisconnect(leaveReconnect); } else { @@ -579,12 +577,12 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } // create data channels - this.lossyDC = this.publisher.pc.createDataChannel(lossyDataChannel, { + this.lossyDC = this.publisher.createDataChannel(lossyDataChannel, { // will drop older packets that arrive ordered: true, maxRetransmits: 0, }); - this.reliableDC = this.publisher.pc.createDataChannel(reliableDataChannel, { + this.reliableDC = this.publisher.createDataChannel(reliableDataChannel, { ordered: true, }); @@ -765,7 +763,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit transceiverInit.sendEncodings = encodings; } // addTransceiver for react-native is async. web is synchronous, but await won't effect it. - const transceiver = await this.publisher.pc.addTransceiver( + const transceiver = await this.publisher.addTransceiver( track.mediaStreamTrack, transceiverInit, ); @@ -791,7 +789,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit transceiverInit.sendEncodings = encodings; } // addTransceiver for react-native is async. web is synchronous, but await won't effect it. - const transceiver = await this.publisher.pc.addTransceiver( + const transceiver = await this.publisher.addTransceiver( simulcastTrack.mediaStreamTrack, transceiverInit, ); @@ -807,7 +805,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit if (!this.publisher) { throw new UnexpectedConnectionState('publisher is closed'); } - return this.publisher.pc.addTrack(track); + return this.publisher.addTrack(track); } // websocket reconnect behavior. if websocket is interrupted, and the PeerConnection @@ -872,7 +870,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit this.clientConfiguration?.resumeConnection === ClientConfigSetting.DISABLED || // signaling state could change to closed due to hardware sleep // those connections cannot be resumed - (this.primaryPC?.signalingState ?? 'closed') === 'closed' + (this.primaryTransport?.getSignallingState() ?? 'closed') === 'closed' ) { this.fullReconnectOnNext = true; } @@ -999,8 +997,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit const res = await this.client.reconnect(this.url, this.token, this.participantSid, reason); if (res) { const rtcConfig = this.makeRTCConfiguration(res); - this.publisher.pc.setConfiguration(rtcConfig); - this.subscriber.pc.setConfiguration(rtcConfig); + this.publisher.setConfiguration(rtcConfig); + this.subscriber.setConfiguration(rtcConfig); } } catch (e) { let message = ''; @@ -1084,7 +1082,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit log.debug('waiting for peer connection to reconnect'); while (now - startTime < this.peerConnectionTimeout) { - if (this.primaryPC === undefined) { + if (this.primaryTransport === undefined) { // we can abort early, connection is hosed break; } else if ( @@ -1092,8 +1090,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit // this means we'd have to check its status manually and update address // manually now - startTime > minReconnectWait && - this.primaryPC?.connectionState === 'connected' && - (!this.hasPublished || this.publisher?.pc.connectionState === 'connected') + this.primaryTransport?.getConnectionState() === 'connected' && + (!this.hasPublished || this.publisher?.getConnectionState() === 'connected') ) { this.pcState = PCState.Connected; } @@ -1172,7 +1170,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit if ( !subscriber && !this.publisher?.isICEConnected && - this.publisher?.pc.iceConnectionState !== 'checking' + this.publisher?.getICEConnectionState() !== 'checking' ) { // start negotiation this.negotiate(); @@ -1196,7 +1194,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } throw new ConnectionError( - `could not establish ${transportName} connection, state: ${transport.pc.iceConnectionState}`, + `could not establish ${transportName} connection, state: ${transport.getICEConnectionState()}`, ); } @@ -1207,12 +1205,12 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit /* @internal */ verifyTransport(): boolean { // primary connection - if (!this.primaryPC) { + if (!this.primaryTransport) { return false; } if ( - this.primaryPC.connectionState === 'closed' || - this.primaryPC.connectionState === 'failed' + this.primaryTransport.getConnectionState() === 'closed' || + this.primaryTransport.getConnectionState() === 'failed' ) { return false; } @@ -1223,8 +1221,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit return false; } if ( - this.publisher.pc.connectionState === 'closed' || - this.publisher.pc.connectionState === 'failed' + this.publisher.getConnectionState() === 'closed' || + this.publisher.getConnectionState() === 'failed' ) { return false; } @@ -1355,40 +1353,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } } -async function getConnectedAddress(pc: RTCPeerConnection): Promise { - let selectedCandidatePairId = ''; - const candidatePairs = new Map(); - // id -> candidate ip - const candidates = new Map(); - const stats: RTCStatsReport = await pc.getStats(); - stats.forEach((v) => { - switch (v.type) { - case 'transport': - selectedCandidatePairId = v.selectedCandidatePairId; - break; - case 'candidate-pair': - if (selectedCandidatePairId === '' && v.selected) { - selectedCandidatePairId = v.id; - } - candidatePairs.set(v.id, v); - break; - case 'remote-candidate': - candidates.set(v.id, `${v.address}:${v.port}`); - break; - default: - } - }); - - if (selectedCandidatePairId === '') { - return undefined; - } - const selectedID = candidatePairs.get(selectedCandidatePairId)?.remoteCandidateId; - if (selectedID === undefined) { - return undefined; - } - return candidates.get(selectedID); -} - class SignalReconnectError extends Error {} export type EngineEventCallbacks = { diff --git a/src/room/Room.ts b/src/room/Room.ts index 0c49af49ee..b4688e51cf 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -1546,14 +1546,12 @@ class Room extends (EventEmitter as new () => TypedEmitter) } private sendSyncState() { - if ( - this.engine.subscriber === undefined || - this.engine.subscriber.pc.localDescription === null - ) { + const previousAnswer = this.engine.subscriber?.getLocalDescription(); + const previousOffer = this.engine.subscriber?.getRemoteDescription(); + + if (!previousAnswer) { return; } - const previousAnswer = this.engine.subscriber.pc.localDescription; - const previousOffer = this.engine.subscriber.pc.remoteDescription; /* 1. autosubscribe on, so subscribed tracks = all tracks - unsub tracks, in this case, we send unsub tracks, so server add all tracks to this diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index c9f123b8c5..e93d8024a9 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -796,7 +796,7 @@ export default class LocalParticipant extends Participant { fix the issue. */ let trackTransceiver: RTCRtpTransceiver | undefined = undefined; - for (const transceiver of this.engine.publisher.pc.getTransceivers()) { + for (const transceiver of this.engine.publisher.getTransceivers()) { if (transceiver.sender === track.sender) { trackTransceiver = transceiver; break; @@ -943,11 +943,11 @@ export default class LocalParticipant extends Participant { track.sender = undefined; if ( this.engine.publisher && - this.engine.publisher.pc.connectionState !== 'closed' && + this.engine.publisher.getConnectionState() !== 'closed' && trackSender ) { try { - for (const transceiver of this.engine.publisher.pc.getTransceivers()) { + for (const transceiver of this.engine.publisher.getTransceivers()) { // if sender is not currently sending (after replaceTrack(null)) // removeTrack would have no effect. // to ensure we end up successfully removing the track, manually set