Karkakol - add simulcast data to tracks

fishjam-dev · Jan 25, 2024 · d860181 · d860181
1 parent b8a8d3d
commit d860181
Show file tree

Hide file tree

Showing 9 changed files with 109 additions and 44 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,4 +8,5 @@ Debug.xcconfig
 .swiftpm
 /.build
 /Packages
-DerivedData/
+DerivedData/
+/.idea/
diff --git a/Sources/MembraneRTC/Events/Event.swift b/Sources/MembraneRTC/Events/Event.swift
@@ -305,7 +305,7 @@ struct EndpointAddedEvent: ReceivableEvent, Codable {
         let id: String
         let type: String
         let metadata: Metadata?
-        let trackIdToMetadata: [String: Metadata]?
+        let tracks: [String: TrackData]?
     }
 
     let type: ReceivableEventType
@@ -349,14 +349,14 @@ struct OfferDataEvent: ReceivableEvent, Codable {
     let data: Data
 }
 
-struct TracksAddedEvent: ReceivableEvent, Codable {
-    struct Data: Codable {
+public struct TracksAddedEvent: ReceivableEvent, Codable {
+    public struct Data: Codable {
         let endpointId: String
-        let trackIdToMetadata: [String: Metadata]
+        let tracks: [String: TrackData]
     }
 
-    let type: ReceivableEventType
-    let data: Data
+    public let type: ReceivableEventType
+    public let data: Data
 }
 
 struct TracksRemovedEvent: ReceivableEvent, Codable {

diff --git a/Sources/MembraneRTC/MembraneRTC.swift b/Sources/MembraneRTC/MembraneRTC.swift
@@ -58,7 +58,7 @@ public class MembraneRTC: MulticastDelegate<MembraneRTCDelegate>, ObservableObje
 
     private var localTracks: [LocalTrack] = []
 
-    private var localEndpoint = Endpoint(id: "", type: "webrtc", metadata: .init([:]), trackIdToMetadata: [:])
+    private var localEndpoint = Endpoint(id: "", type: "webrtc", metadata: .init([:]), tracks: [:])
 
     // mapping from peer's id to itself
     private var remoteEndpoints: [String: Endpoint] = [:]
@@ -173,7 +173,10 @@ public class MembraneRTC: MulticastDelegate<MembraneRTCDelegate>, ObservableObje
 
    - Returns: `LocalCameraVideoTrack` instance that user then can use for things such as front / back camera switch.
    */
-    public func createVideoTrack(videoParameters: VideoParameters, metadata: Metadata, captureDeviceId: String? = nil)
+    public func createVideoTrack(
+        videoParameters: VideoParameters, metadata: Metadata, captureDeviceId: String? = nil,
+        simulcastConfig: SimulcastConfig? = nil
+    )
         -> LocalVideoTrack
     {
         DispatchQueue.webRTC.sync {
@@ -191,7 +194,8 @@ public class MembraneRTC: MulticastDelegate<MembraneRTCDelegate>, ObservableObje
 
             localTracks.append(videoTrack)
 
-            localEndpoint = localEndpoint.withTrack(trackId: videoTrack.rtcTrack().trackId, metadata: metadata)
+            localEndpoint = localEndpoint.withTrack(
+                trackId: videoTrack.rtcTrack().trackId, metadata: metadata, simulcastConfig: simulcastConfig)
 
             engineCommunication.renegotiateTracks()
 
@@ -219,7 +223,8 @@ public class MembraneRTC: MulticastDelegate<MembraneRTCDelegate>, ObservableObje
 
             localTracks.append(audioTrack)
 
-            localEndpoint = localEndpoint.withTrack(trackId: audioTrack.rtcTrack().trackId, metadata: metadata)
+            localEndpoint = localEndpoint.withTrack(
+                trackId: audioTrack.rtcTrack().trackId, metadata: metadata, simulcastConfig: nil)
 
             engineCommunication.renegotiateTracks()
 
@@ -252,6 +257,7 @@ public class MembraneRTC: MulticastDelegate<MembraneRTCDelegate>, ObservableObje
                 appGroup: appGroup, videoParameters: videoParameters,
                 peerConnectionFactoryWrapper: peerConnectionFactoryWrapper)
             localTracks.append(screensharingTrack)
+            let simulcastConfig = videoParameters.simulcastConfig
 
             broadcastScreenshareReceiver = ScreenBroadcastNotificationReceiver(
                 onStart: { [weak self, weak screensharingTrack] in
@@ -260,7 +266,7 @@ public class MembraneRTC: MulticastDelegate<MembraneRTCDelegate>, ObservableObje
                     }
 
                     DispatchQueue.main.async {
-                        self?.setupScreencastTrack(track: track, metadata: metadata)
+                        self?.setupScreencastTrack(track: track, metadata: metadata, simulcastConfig: simulcastConfig)
                         onStart(track)
                     }
                 },
@@ -369,7 +375,7 @@ public class MembraneRTC: MulticastDelegate<MembraneRTCDelegate>, ObservableObje
     public func updateTrackMetadata(trackId: String, trackMetadata: Metadata) {
         DispatchQueue.webRTC.sync {
             engineCommunication.updateTrackMetadata(trackId: trackId, trackMetadata: trackMetadata)
-            localEndpoint = localEndpoint.withTrack(trackId: trackId, metadata: trackMetadata)
+            localEndpoint = localEndpoint.withTrack(trackId: trackId, metadata: trackMetadata, simulcastConfig: nil)
         }
     }
 
@@ -403,12 +409,15 @@ public class MembraneRTC: MulticastDelegate<MembraneRTCDelegate>, ObservableObje
     }
 
     /// Adds given broadcast track to the peer connection and forces track renegotiation.
-    private func setupScreencastTrack(track: LocalScreenBroadcastTrack, metadata: Metadata) {
+    private func setupScreencastTrack(
+        track: LocalScreenBroadcastTrack, metadata: Metadata, simulcastConfig: SimulcastConfig?
+    ) {
         let screencastStreamId = UUID().uuidString
 
         peerConnectionManager.addTrack(track: track, localStreamId: screencastStreamId)
 
-        localEndpoint = localEndpoint.withTrack(trackId: track.rtcTrack().trackId, metadata: metadata)
+        localEndpoint = localEndpoint.withTrack(
+            trackId: track.rtcTrack().trackId, metadata: metadata, simulcastConfig: simulcastConfig)
 
         engineCommunication.renegotiateTracks()
     }
@@ -469,8 +478,10 @@ public class MembraneRTC: MulticastDelegate<MembraneRTCDelegate>, ObservableObje
             self.remoteEndpoints[endpoint.id] = endpoint
 
             // initialize peer's track contexts
-            endpoint.trackIdToMetadata?.forEach { trackId, metadata in
-                let context = TrackContext(track: nil, enpoint: endpoint, trackId: trackId, metadata: metadata)
+            endpoint.tracks?.forEach { trackId, trackData in
+                let context = TrackContext(
+                    track: nil, enpoint: endpoint, trackId: trackId, metadata: trackData.metadata,
+                    simulcastConfig: trackData.simulcastConfig)
 
                 self.trackContexts[trackId] = context
 
@@ -512,7 +523,7 @@ public class MembraneRTC: MulticastDelegate<MembraneRTCDelegate>, ObservableObje
         remoteEndpoints.removeValue(forKey: endpoint.id)
 
         // for a leaving peer clear his track contexts
-        if let trackIds = endpoint.trackIdToMetadata?.keys {
+        if let trackIds = endpoint.tracks?.keys {
             let contexts = trackIds.compactMap { id in
                 self.trackContexts[id]
             }
@@ -560,7 +571,10 @@ public class MembraneRTC: MulticastDelegate<MembraneRTCDelegate>, ObservableObje
 
             if let sdp = sdp, let midToTrackId = midToTrackId {
                 self.engineCommunication.sdpOffer(
-                    sdp: sdp, trackIdToTrackMetadata: self.localEndpoint.trackIdToMetadata ?? [:],
+                    sdp: sdp,
+                    trackIdToTrackMetadata: self.localEndpoint.tracks?.mapValues({ trackData in
+                        trackData.metadata
+                    }) ?? [:],
                     midToTrackId: midToTrackId)
             }
 
@@ -577,7 +591,7 @@ public class MembraneRTC: MulticastDelegate<MembraneRTCDelegate>, ObservableObje
         peerConnectionManager.onRemoteCandidate(candidate: candidate)
     }
 
-    func onTracksAdded(endpointId: String, trackIdToMetadata: [String: Metadata]) {
+    func onTracksAdded(endpointId: String, tracks: [String: TrackData]) {
         // ignore local participant
         guard localEndpoint.id != endpointId else {
             return
@@ -589,12 +603,14 @@ public class MembraneRTC: MulticastDelegate<MembraneRTCDelegate>, ObservableObje
         }
 
         // update tracks of the remote peer
-        endpoint = endpoint.with(trackIdToMetadata: trackIdToMetadata)
+        endpoint = endpoint.with(tracks: tracks)
         remoteEndpoints[endpoint.id] = endpoint
 
         // for each track create a corresponding track context
-        endpoint.trackIdToMetadata?.forEach { trackId, metadata in
-            let context = TrackContext(track: nil, enpoint: endpoint, trackId: trackId, metadata: metadata)
+        endpoint.tracks?.forEach { trackId, trackData in
+            let context = TrackContext(
+                track: nil, enpoint: endpoint, trackId: trackId, metadata: trackData.metadata,
+                simulcastConfig: trackData.simulcastConfig)
 
             self.trackContexts[trackId] = context
 

diff --git a/Sources/MembraneRTC/RTCEngineCommunication.swift b/Sources/MembraneRTC/RTCEngineCommunication.swift
@@ -60,7 +60,8 @@ internal class RTCEngineCommunication {
             engineListener.onEndpointAdded(
                 endpoint: Endpoint(
                     id: endpointAdded.data.id, type: endpointAdded.data.type, metadata: endpointAdded.data.metadata,
-                    trackIdToMetadata: endpointAdded.data.trackIdToMetadata))
+                    tracks: endpointAdded.data.tracks)
+            )
         case .EndpointRemoved:
             let endpointRemoved = event as! EndpointRemovedEvent
             engineListener.onEndpointRemoved(endpointId: endpointRemoved.data.id)
@@ -80,7 +81,7 @@ internal class RTCEngineCommunication {
         case .TracksAdded:
             let tracksAdded = event as! TracksAddedEvent
             engineListener.onTracksAdded(
-                endpointId: tracksAdded.data.endpointId, trackIdToMetadata: tracksAdded.data.trackIdToMetadata)
+                endpointId: tracksAdded.data.endpointId, tracks: tracksAdded.data.tracks)
         case .TracksRemoved:
             let tracksRemoved = event as! TracksRemovedEvent
             engineListener.onTracksRemoved(

diff --git a/Sources/MembraneRTC/RTCEngineListener.swift b/Sources/MembraneRTC/RTCEngineListener.swift
@@ -8,7 +8,7 @@ internal protocol RTCEngineListener {
     func onOfferData(integratedTurnServers: [OfferDataEvent.TurnServer], tracksTypes: [String: Int])
     func onSdpAnswer(type: String, sdp: String, midToTrackId: [String: String?])
     func onRemoteCandidate(candidate: String, sdpMLineIndex: Int32, sdpMid: String?)
-    func onTracksAdded(endpointId: String, trackIdToMetadata: [String: Metadata])
+    func onTracksAdded(endpointId: String, tracks: [String: TrackData])
     func onTracksRemoved(endpointId: String, trackIds: [String])
     func onTrackUpdated(endpointId: String, trackId: String, metadata: Metadata)
     func onTrackEncodingChanged(endpointId: String, trackId: String, encoding: String, encodingReason: String)

diff --git a/Sources/MembraneRTC/Types/Endpoint.swift b/Sources/MembraneRTC/Types/Endpoint.swift
@@ -2,50 +2,51 @@ public struct Endpoint: Codable {
     public let id: String
     public let type: String
     public let metadata: Metadata
-    public let trackIdToMetadata: [String: Metadata]?
+    public let tracks: [String: TrackData]?
 
-    public init(id: String, type: String, metadata: Metadata?, trackIdToMetadata: [String: Metadata]?) {
+    public init(id: String, type: String, metadata: Metadata?, tracks: [String: TrackData]? = nil) {
         self.id = id
         self.type = type
         self.metadata = metadata ?? Metadata()
-        self.trackIdToMetadata = trackIdToMetadata
+        self.tracks = tracks
     }
 
     public func with(
-        id: String? = nil, type: String? = nil, metadata: Metadata? = nil, trackIdToMetadata: [String: Metadata]? = nil
+        id: String? = nil, type: String? = nil, metadata: Metadata? = nil, tracks: [String: TrackData]? = nil
     ) -> Self {
         return Endpoint(
             id: id ?? self.id,
             type: type ?? self.type,
             metadata: metadata ?? self.metadata,
-            trackIdToMetadata: trackIdToMetadata ?? self.trackIdToMetadata
+            tracks: tracks ?? self.tracks
         )
     }
 
-    public func withTrack(trackId: String, metadata: Metadata?) -> Self {
-        var newTrackIdToMetadata = self.trackIdToMetadata
-        newTrackIdToMetadata?[trackId] = metadata ?? Metadata()
+    public func withTrack(trackId: String, metadata: Metadata?, simulcastConfig: SimulcastConfig?) -> Self {
+        var newTracks = self.tracks
+        let oldSimulcastConfig = newTracks?[trackId]?.simulcastConfig
+        newTracks?[trackId] = TrackData(
+            metadata: metadata ?? Metadata(), simulcastConfig: simulcastConfig ?? oldSimulcastConfig)
 
-        return Endpoint(id: self.id, type: self.type, metadata: self.metadata, trackIdToMetadata: newTrackIdToMetadata)
+        return Endpoint(id: self.id, type: self.type, metadata: self.metadata, tracks: newTracks)
     }
 
     public func withoutTrack(trackId: String) -> Self {
-        var newTrackIdToMetadata = self.trackIdToMetadata
-        newTrackIdToMetadata?.removeValue(forKey: trackId)
+        var newTracks = self.tracks
+        newTracks?.removeValue(forKey: trackId)
 
-        return Endpoint(id: self.id, type: self.type, metadata: self.metadata, trackIdToMetadata: newTrackIdToMetadata)
+        return Endpoint(id: self.id, type: self.type, metadata: self.metadata, tracks: newTracks)
     }
 
     enum CodingKeys: String, CodingKey {
-        case id, type, metadata, trackIdToMetadata
+        case id, type, metadata, tracks
     }
 
     public init(from decoder: Decoder) throws {
         let container = try decoder.container(keyedBy: CodingKeys.self)
         self.id = try container.decode(String.self, forKey: .id)
         self.type = try container.decode(String.self, forKey: .type)
         self.metadata = try container.decodeIfPresent(Metadata.self, forKey: .metadata) ?? Metadata()
-        self.trackIdToMetadata = try container.decodeIfPresent(
-            [String: Metadata].self, forKey: .trackIdToMetadata)
+        self.tracks = try container.decodeIfPresent([String: TrackData].self, forKey: .tracks)
     }
 }
diff --git a/Sources/MembraneRTC/Types/SimulcastConfig.swift b/Sources/MembraneRTC/Types/SimulcastConfig.swift
@@ -2,7 +2,7 @@
 /// `"h"` - original encoding
 /// `"m"` - original encoding scaled down by 2
 /// `"l"` - original encoding scaled down by 4
-public enum TrackEncoding: Int, CustomStringConvertible {
+public enum TrackEncoding: Int, CustomStringConvertible, Codable {
     case l = 0
     case m
     case h
@@ -15,6 +15,10 @@ public enum TrackEncoding: Int, CustomStringConvertible {
         }
     }
 
+    enum TrackEncodingCodingError: Error {
+        case decoding(String)
+    }
+
     static func fromString(_ s: String) -> TrackEncoding? {
         switch s {
         case "l":
@@ -27,14 +31,36 @@ public enum TrackEncoding: Int, CustomStringConvertible {
             return nil
         }
     }
+
+    public init(from decoder: Decoder) throws {
+        let container = try decoder.singleValueContainer()
+        let encodingString = try container.decode(String.self)
+
+        switch encodingString {
+            case "l":
+                self = .l
+            case "m":
+                self = .m
+            case "h":
+                self = .h
+            default:
+                throw TrackEncodingCodingError.decoding("\(encodingString) is not a valid encoding")
+        }
+    }
+
+    public func encode(to encoder: Swift.Encoder) throws {
+        var container = encoder.singleValueContainer()
+        let encodingString = description
+        try container.encode(encodingString)
+    }
 }
 
 /// Simulcast configuration.
 ///
 /// At the moment, simulcast track is initialized in three versions - low, medium and high.
 /// High resolution is the original track resolution, while medium and low resolutions
 /// are the original track resolution scaled down by 2 and 4 respectively.
-public struct SimulcastConfig {
+public struct SimulcastConfig: Codable {
     /**
      * Whether to simulcast track or not.
      */

diff --git a/Sources/MembraneRTC/Types/TrackContext.swift b/Sources/MembraneRTC/Types/TrackContext.swift
@@ -24,6 +24,9 @@ public class TrackContext {
 
     /// The reason of currently selected encoding. Only present for remote tracks.
     public private(set) var encodingReason: EncodingReason? = nil
+
+    /// Possible encodings that can be requested. Only present for remote tracks.
+    public internal(set) var simulcastConfig: SimulcastConfig? = nil
 
     internal func setEncoding(encoding: TrackEncoding, encodingReason: EncodingReason) {
         self.encoding = encoding
@@ -33,11 +36,12 @@ public class TrackContext {
         }
     }
 
-    init(track: RemoteTrack? = nil, enpoint: Endpoint, trackId: String, metadata: Metadata) {
+    init(track: RemoteTrack? = nil, enpoint: Endpoint, trackId: String, metadata: Metadata, simulcastConfig: SimulcastConfig?) {
         self.track = track
         self.endpoint = enpoint
         self.trackId = trackId
         self.metadata = metadata
+        self.simulcastConfig = simulcastConfig
     }
 
     private var onTrackEncodingChangedListener: ((_ trackContext: TrackContext) -> Void)?

diff --git a/Sources/MembraneRTC/Types/TrackData.swift b/Sources/MembraneRTC/Types/TrackData.swift
@@ -0,0 +1,16 @@
+public struct TrackData: Codable{
+    public let metadata: Metadata
+    public let simulcastConfig: SimulcastConfig?
+
+    public init (metadata : Metadata, simulcastConfig : SimulcastConfig? = nil) {
+        self.metadata = metadata
+        self.simulcastConfig = simulcastConfig
+    }
+
+    func copyWith(metadata: Metadata? = nil, simulcastConfig: SimulcastConfig? = nil) -> TrackData {
+        return TrackData(
+            metadata: metadata ?? self.metadata,
+            simulcastConfig: simulcastConfig ?? self.simulcastConfig
+        )
+    }
+}