From ae4650231738b06337fca8c4fed015628f50562c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=C5=9Aled=C5=BA?= Date: Sat, 4 Nov 2023 10:08:06 +0100 Subject: [PATCH] Make codecs and rtp header extensions configurable --- lib/ex_webrtc/peer_connection.ex | 63 +++--- .../peer_connection/configuration.ex | 199 +++++++++++++++++- lib/ex_webrtc/rtp_codec_parameters.ex | 9 + lib/ex_webrtc/rtp_transceiver.ex | 122 +++++++++-- lib/ex_webrtc/sdp_utils.ex | 94 --------- test/configuration_test.exs | 149 +++++++++++++ test/fixtures/audio_sdp.txt | 36 ++++ test/fixtures/audio_video_sdp.txt | 157 ++++++++++++++ test/peer_connection_test.exs | 86 +------- 9 files changed, 683 insertions(+), 232 deletions(-) create mode 100644 test/configuration_test.exs create mode 100644 test/fixtures/audio_sdp.txt create mode 100644 test/fixtures/audio_video_sdp.txt diff --git a/lib/ex_webrtc/peer_connection.ex b/lib/ex_webrtc/peer_connection.ex index 7af373fe..6a6d91bf 100644 --- a/lib/ex_webrtc/peer_connection.ex +++ b/lib/ex_webrtc/peer_connection.ex @@ -95,13 +95,15 @@ defmodule ExWebRTC.PeerConnection do GenServer.call(peer_connection, :get_transceivers) end - @spec add_transceiver( - peer_connection(), - RTPTransceiver.kind() | MediaStreamTrack.t(), - transceiver_options() - ) :: {:ok, RTPTransceiver.t()} | {:error, :TODO} - def add_transceiver(peer_connection, track_or_kind, options \\ []) do - GenServer.call(peer_connection, {:add_transceiver, track_or_kind, options}) + @spec add_transceiver(peer_connection(), RTPTransceiver.kind(), transceiver_options()) :: + {:ok, RTPTransceiver.t()} | {:error, :TODO} + def add_transceiver(peer_connection, kind, options \\ []) do + GenServer.call(peer_connection, {:add_transceiver, kind, options}) + end + + @spec close(peer_connection()) :: :ok + def close(peer_connection) do + GenServer.stop(peer_connection) end #### CALLBACKS #### @@ -145,7 +147,7 @@ defmodule ExWebRTC.PeerConnection do # we support trickle ICE only |> ExSDP.add_attribute({:ice_options, "trickle"}) - config = + opts = [ ice_ufrag: ice_ufrag, ice_pwd: ice_pwd, @@ -155,10 +157,7 @@ defmodule ExWebRTC.PeerConnection do rtcp: true ] - mlines = - Enum.map(transceivers, fn transceiver -> - SDPUtils.to_offer_mline(transceiver, config) - end) + mlines = Enum.map(transceivers, &RTPTransceiver.to_offer_mline(&1, opts)) mids = Enum.map(mlines, fn mline -> @@ -200,7 +199,7 @@ defmodule ExWebRTC.PeerConnection do # we only support trickle ICE, so non-trickle offers should be rejected earlier |> ExSDP.add_attribute({:ice_options, "trickle"}) - config = + opts = [ ice_ufrag: ice_ufrag, ice_pwd: ice_pwd, @@ -214,7 +213,7 @@ defmodule ExWebRTC.PeerConnection do Enum.map(remote_offer.media, fn mline -> {:mid, mid} = ExSDP.Media.get_attribute(mline, :mid) {_ix, transceiver} = RTPTransceiver.find_by_mid(state.transceivers, mid) - SDPUtils.get_answer_mline(mline, transceiver, config) + RTPTransceiver.to_answer_mline(transceiver, mline, opts) end) mids = @@ -299,21 +298,24 @@ defmodule ExWebRTC.PeerConnection do end @impl true - def handle_call({:add_transceiver, :audio, options}, _from, state) do - # TODO: proper implementation, change the :audio above to track_or_kind + def handle_call({:add_transceiver, kind, options}, _from, state) + when kind in [:audio, :video] do direction = Keyword.get(options, :direction, :sendrcv) - # hardcoded audio codec - codecs = [ - %ExWebRTC.RTPCodecParameters{ - payload_type: 111, - mime_type: "audio/opus", - clock_rate: 48_000, - channels: 2 - } - ] - - transceiver = %RTPTransceiver{mid: nil, direction: direction, kind: :audio, codecs: codecs} + {rtp_hdr_exts, codecs} = + case kind do + :audio -> {state.config.audio_rtp_hdr_exts, state.config.audio_codecs} + :video -> {state.config.video_rtp_hdr_exts, state.config.video_codecs} + end + + transceiver = %RTPTransceiver{ + mid: nil, + direction: direction, + kind: kind, + codecs: codecs, + rtp_hdr_exts: rtp_hdr_exts + } + transceivers = List.insert_at(state.transceivers, -1, transceiver) {:reply, {:ok, transceiver}, %{state | transceivers: transceivers}} end @@ -395,7 +397,8 @@ defmodule ExWebRTC.PeerConnection do with :ok <- SDPUtils.ensure_mid(sdp), :ok <- SDPUtils.ensure_bundle(sdp), {:ok, {ice_ufrag, ice_pwd}} <- SDPUtils.get_ice_credentials(sdp), - {:ok, new_transceivers} <- update_remote_transceivers(state.transceivers, sdp) do + {:ok, new_transceivers} <- + update_remote_transceivers(state.transceivers, sdp, state.config) do :ok = ICEAgent.set_remote_credentials(state.ice_agent, ice_ufrag, ice_pwd) :ok = ICEAgent.gather_candidates(state.ice_agent) @@ -433,11 +436,11 @@ defmodule ExWebRTC.PeerConnection do end end - defp update_remote_transceivers(transceivers, sdp) do + defp update_remote_transceivers(transceivers, sdp, config) do Enum.reduce_while(sdp.media, {:ok, transceivers}, fn mline, {:ok, transceivers} -> case ExSDP.Media.get_attribute(mline, :mid) do {:mid, mid} -> - transceivers = RTPTransceiver.update_or_create(transceivers, mid, mline) + transceivers = RTPTransceiver.update_or_create(transceivers, mid, mline, config) {:cont, {:ok, transceivers}} _other -> diff --git a/lib/ex_webrtc/peer_connection/configuration.ex b/lib/ex_webrtc/peer_connection/configuration.ex index 865d8f73..5bfb92ad 100644 --- a/lib/ex_webrtc/peer_connection/configuration.ex +++ b/lib/ex_webrtc/peer_connection/configuration.ex @@ -3,6 +3,60 @@ defmodule ExWebRTC.PeerConnection.Configuration do PeerConnection configuration """ + alias ExWebRTC.RTPCodecParameters + alias ExSDP.Attribute.{Extmap, FMTP, RTCPFeedback} + + @default_audio_codecs [ + %RTPCodecParameters{ + payload_type: 111, + mime_type: "audio/opus", + clock_rate: 48_000, + channels: 2, + sdp_fmtp_line: %FMTP{pt: 111, minptime: 10, useinbandfec: true}, + rtcp_fbs: [] + } + ] + + @default_video_codecs [ + %RTPCodecParameters{ + payload_type: 98, + mime_type: "video/H264", + clock_rate: 90_000, + channels: nil, + sdp_fmtp_line: %FMTP{ + pt: 98, + level_asymmetry_allowed: true, + packetization_mode: 1, + profile_level_id: 0x42001F + }, + rtcp_fbs: [] + }, + %RTPCodecParameters{ + payload_type: 96, + mime_type: "video/VP8", + clock_rate: 90_000, + channels: nil, + sdp_fmtp_line: nil, + rtcp_fbs: [] + } + ] + + @rtp_hdr_extensions %{ + :mid => %{media_type: :all, ext: %Extmap{id: 1, uri: "urn:ietf:params:rtp-hdrext:sdes:mid"}}, + :audio_level => %{ + media_type: :audio, + ext: %Extmap{id: 2, uri: "urn:ietf:params:rtp-hdrext:ssrc-audio-level"} + } + } + + @mandatory_audio_rtp_hdr_exts Enum.map([:mid], &Map.fetch!(@rtp_hdr_extensions, &1).ext) + @mandatory_video_rtp_hdr_exts Enum.map([:mid], &Map.fetch!(@rtp_hdr_extensions, &1).ext) + + @typedoc """ + Supported RTP header extensions. + """ + @type rtp_hdr_extension() :: :audio_level + @type ice_server() :: %{ optional(:credential) => String.t(), optional(:username) => String.t(), @@ -12,7 +66,22 @@ defmodule ExWebRTC.PeerConnection.Configuration do @typedoc """ Options that can be passed to `ExWebRTC.PeerConnection.start_link/1`. - Currently, ExWebRTC always uses the following config: + * `ice_servers` - list of STUN servers to use. + TURN servers are not supported right now and will be filtered out. + * `audio_codecs` - list of audio codecs to use. + Use `default_audio_codecs/0` to get a list of default audio codecs. + This option overrides default audio codecs. + If you wish to add codecs to default ones do + `audio_codecs: Configuration.default_audio_codecs() ++ my_codecs` + * `video_codecs` - the same as `audio_codecs` but for video. + * `rtp_hdr_extensions` - list of RTP header extensions to use. + MID extension is enabled by default and cannot be turned off. + If an extension can be used both for audio and video media, it + will be added to every mline. + If an extension is audio-only, it will only be added to audio mlines. + If an extension is video-only, it will only be added to video mlines. + + Besides options listed above, ExWebRTC uses the following config: * bundle_policy - max_bundle * ice_candidate_pool_size - 0 * ice_transport_policy - all @@ -20,24 +89,134 @@ defmodule ExWebRTC.PeerConnection.Configuration do This config cannot be changed. """ - @type options() :: [ice_servers: [ice_server()]] + @type options() :: [ + ice_servers: [ice_server()], + audio_codecs: [RTPCodecParameters.t()], + video_codecs: [RTPCodecParameters.t()], + rtp_hdr_extensions: [rtp_hdr_extension()] + ] @typedoc false - @type t() :: %__MODULE__{ice_servers: [ice_server()]} + @type t() :: %__MODULE__{ + ice_servers: [ice_server()], + audio_codecs: [RTPCodecParameters.t()], + video_codecs: [RTPCodecParameters.t()], + audio_rtp_hdr_exts: [Extmap.t()], + video_rtp_hdr_exts: [Extmap.t()] + } + + defstruct ice_servers: [], + audio_codecs: @default_audio_codecs, + video_codecs: @default_video_codecs, + audio_rtp_hdr_exts: @mandatory_audio_rtp_hdr_exts, + video_rtp_hdr_exts: @mandatory_video_rtp_hdr_exts + + @doc """ + Returns a list of default audio codecs. + """ + @spec default_audio_codecs() :: [RTPCodecParameters.t()] + def default_audio_codecs(), do: @default_audio_codecs - defstruct ice_servers: [] + @doc """ + Returns a list of default video codecs. + """ + @spec default_video_codecs() :: [RTPCodecParameters.t()] + def default_video_codecs(), do: @default_video_codecs @doc false @spec from_options!(options()) :: t() def from_options!(options) do - config = struct!(__MODULE__, options) + options = + options + |> resolve_rtp_hdr_extensions() + |> add_mandatory_rtp_hdr_extensions() + # ATM, ExICE does not support relay via TURN + |> reject_turn_servers() + + struct!(__MODULE__, options) + end + + @doc false + @spec is_supported_codec(t(), RTPCodecParameters.t()) :: boolean() + def is_supported_codec(config, codec) do + # This function doesn't check if rtcp-fb is supported. + # Instead, `is_supported_rtcp_fb` has to be used to filter out + # rtcp-fb that are not supported. + Enum.find( + config.audio_codecs ++ config.video_codecs, + fn supported_codec -> + # for the time of comparision, override payload type in our codec + supported_codec = + if supported_codec.sdp_fmtp_line != nil and codec.sdp_fmtp_line != nil do + %RTPCodecParameters{ + supported_codec + | sdp_fmtp_line: %FMTP{supported_codec.sdp_fmtp_line | pt: codec.sdp_fmtp_line.pt} + } + else + supported_codec + end + + supported_codec.mime_type == codec.mime_type and + supported_codec.clock_rate == codec.clock_rate and + supported_codec.channels == codec.channels and + supported_codec.sdp_fmtp_line == codec.sdp_fmtp_line + end + ) + end + + @doc false + @spec is_supported_rtp_hdr_extension(t(), Extmap.t(), :audio | :video) :: + boolean() + def is_supported_rtp_hdr_extension(config, rtp_hdr_extension, media_type) do + supported_uris = + case media_type do + :audio -> Enum.map(config.audio_rtp_hdr_exts, & &1.uri) + :video -> Enum.map(config.video_rtp_hdr_exts, & &1.uri) + end + + rtp_hdr_extension.uri in supported_uris + end + + @doc false + @spec is_supported_rtcp_fb(t(), RTCPFeedback.t()) :: boolean() + def is_supported_rtcp_fb(_config, _rtcp_fb), do: false - # ATM, ExICE does not support relay via TURN - stun_servers = - config.ice_servers + defp add_mandatory_rtp_hdr_extensions(options) do + options + |> Keyword.update(:audio_rtp_hdr_exts, [], fn exts -> + exts ++ @mandatory_audio_rtp_hdr_exts + end) + |> Keyword.update(:video_rtp_hdr_exts, [], fn exts -> + exts ++ @mandatory_video_rtp_hdr_exts + end) + end + + defp resolve_rtp_hdr_extensions(options) do + {audio_exts, video_exts} = + Keyword.get(options, :rtp_hdr_extensions, []) + |> Enum.reduce({[], []}, fn ext, {audio_exts, video_exts} -> + resolved_ext = Map.fetch!(@rtp_hdr_extensions, ext) + + case resolved_ext.media_type do + :audio -> + {[resolved_ext.ext | audio_exts], video_exts} + + :all -> + {[resolved_ext.ext | audio_exts], [resolved_ext.ext | video_exts]} + end + end) + + options + |> Keyword.put(:audio_rtp_hdr_exts, Enum.reverse(audio_exts)) + |> Keyword.put(:video_rtp_hdr_exts, Enum.reverse(video_exts)) + |> Keyword.delete(:rtp_hdr_extensions) + end + + defp reject_turn_servers(options) do + Keyword.update(options, :ice_servers, [], fn ice_servers -> + ice_servers |> Enum.flat_map(&List.wrap(&1.urls)) |> Enum.filter(&String.starts_with?(&1, "stun:")) - - %__MODULE__{config | ice_servers: stun_servers} + end) end end diff --git a/lib/ex_webrtc/rtp_codec_parameters.ex b/lib/ex_webrtc/rtp_codec_parameters.ex index d240f6b2..367c3068 100644 --- a/lib/ex_webrtc/rtp_codec_parameters.ex +++ b/lib/ex_webrtc/rtp_codec_parameters.ex @@ -3,6 +3,15 @@ defmodule ExWebRTC.RTPCodecParameters do RTPCodecParameters """ + @type t() :: %__MODULE__{ + payload_type: non_neg_integer(), + mime_type: binary(), + clock_rate: non_neg_integer(), + channels: non_neg_integer() | nil, + sdp_fmtp_line: ExSDP.Attribute.FMTP.t() | nil, + rtcp_fbs: [ExSDP.Attribute.RTCPFeedback.t()] + } + defstruct [:payload_type, :mime_type, :clock_rate, :channels, :sdp_fmtp_line, :rtcp_fbs] def new(type, rtp_mapping, fmtp, rtcp_fbs) do diff --git a/lib/ex_webrtc/rtp_transceiver.ex b/lib/ex_webrtc/rtp_transceiver.ex index e18d5d02..76d45324 100644 --- a/lib/ex_webrtc/rtp_transceiver.ex +++ b/lib/ex_webrtc/rtp_transceiver.ex @@ -3,7 +3,7 @@ defmodule ExWebRTC.RTPTransceiver do RTPTransceiver """ - alias ExWebRTC.{RTPCodecParameters, RTPReceiver} + alias ExWebRTC.{PeerConnection.Configuration, RTPCodecParameters, RTPReceiver} @type direction() :: :sendonly | :recvonly | :sendrecv | :inactive | :stopped @type kind() :: :audio | :video @@ -12,13 +12,13 @@ defmodule ExWebRTC.RTPTransceiver do mid: String.t(), direction: direction(), kind: kind(), - hdr_exts: [], - codecs: [], + rtp_hdr_exts: [ExSDP.Attribute.Extmap.t()], + codecs: [RTPCodecParameters.t()], rtp_receiver: nil } @enforce_keys [:mid, :direction, :kind] - defstruct @enforce_keys ++ [codecs: [], hdr_exts: [], rtp_receiver: %RTPReceiver{}] + defstruct @enforce_keys ++ [codecs: [], rtp_hdr_exts: [], rtp_receiver: %RTPReceiver{}] @doc false def find_by_mid(transceivers, mid) do @@ -27,19 +27,40 @@ defmodule ExWebRTC.RTPTransceiver do |> Enum.find(fn {_idx, tr} -> tr.mid == mid end) end + @doc false + @spec to_answer_mline(RTPTransceiver.t(), ExSDP.Media.t(), Keyword.t()) :: ExSDP.Media.t() + def to_answer_mline(transceiver, mline, opts) do + if transceiver.codecs == [] do + # reject mline and skip further processing + # see RFC 8299 sec. 5.3.1 and RFC 3264 sec. 6 + %ExSDP.Media{mline | port: 0} + else + offered_direction = ExSDP.Media.get_attribute(mline, :direction) + direction = get_direction(offered_direction, transceiver.direction) + opts = Keyword.put(opts, :direction, direction) + to_mline(transceiver, opts) + end + end + + @doc false + @spec to_offer_mline(RTPTransceiver.t(), Keyword.t()) :: ExSDP.Media.t() + def to_offer_mline(transceiver, opts) do + to_mline(transceiver, opts) + end + # searches for transceiver for a given mline # if it exists, updates its configuration # if it doesn't exist, creats a new one # returns list of updated transceivers @doc false - def update_or_create(transceivers, mid, mline) do + def update_or_create(transceivers, mid, mline, config) do case find_by_mid(transceivers, mid) do {idx, %__MODULE__{} = tr} -> - List.replace_at(transceivers, idx, update(tr, mline)) + List.replace_at(transceivers, idx, update(tr, mline, config)) nil -> - codecs = get_codecs(mline) - hdr_exts = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.Extmap) + codecs = get_codecs(mline, config) + rtp_hdr_exts = get_rtp_hdr_extensions(mline, config) ssrc = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.SSRC) tr = %__MODULE__{ @@ -47,7 +68,7 @@ defmodule ExWebRTC.RTPTransceiver do direction: :recvonly, kind: mline.type, codecs: codecs, - hdr_exts: hdr_exts, + rtp_hdr_exts: rtp_hdr_exts, rtp_receiver: %RTPReceiver{ssrc: ssrc} } @@ -55,26 +76,91 @@ defmodule ExWebRTC.RTPTransceiver do end end - defp update(transceiver, mline) do - codecs = get_codecs(mline) - hdr_exts = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.Extmap) + defp to_mline(transceiver, opts) do + pt = Enum.map(transceiver.codecs, fn codec -> codec.payload_type end) + + media_formats = + Enum.flat_map(transceiver.codecs, fn codec -> + [_type, encoding] = String.split(codec.mime_type, "/") + + rtp_mapping = %ExSDP.Attribute.RTPMapping{ + clock_rate: codec.clock_rate, + encoding: encoding, + params: codec.channels, + payload_type: codec.payload_type + } + + [rtp_mapping, codec.sdp_fmtp_line, codec.rtcp_fbs] + end) + + attributes = + if(Keyword.get(opts, :rtcp, false), do: [{"rtcp", "9 IN IP4 0.0.0.0"}], else: []) ++ + [ + Keyword.get(opts, :direction, transceiver.direction), + {:mid, transceiver.mid}, + {:ice_ufrag, Keyword.fetch!(opts, :ice_ufrag)}, + {:ice_pwd, Keyword.fetch!(opts, :ice_pwd)}, + {:ice_options, Keyword.fetch!(opts, :ice_options)}, + {:fingerprint, Keyword.fetch!(opts, :fingerprint)}, + {:setup, Keyword.fetch!(opts, :setup)}, + :rtcp_mux + ] ++ transceiver.rtp_hdr_exts + + %ExSDP.Media{ + ExSDP.Media.new(transceiver.kind, 9, "UDP/TLS/RTP/SAVPF", pt) + | # mline must be followed by a cline, which must contain + # the default value "IN IP4 0.0.0.0" (as there are no candidates yet) + connection_data: [%ExSDP.ConnectionData{address: {0, 0, 0, 0}}] + } + |> ExSDP.Media.add_attributes(attributes ++ media_formats) + end + + # RFC 3264 (6.1) + RFC 8829 (5.3.1) + # AFAIK one of the cases should always match + # bc we won't assign/create an inactive transceiver to i.e. sendonly mline + # also neither of the arguments should ever be :stopped + defp get_direction(_, :inactive), do: :inactive + defp get_direction(:sendonly, t) when t in [:sendrecv, :recvonly], do: :recvonly + defp get_direction(:recvonly, t) when t in [:sendrecv, :sendonly], do: :sendonly + defp get_direction(o, other) when o in [:sendrecv, nil], do: other + defp get_direction(:inactive, _), do: :inactive + + defp update(transceiver, mline, config) do + codecs = get_codecs(mline, config) + rtp_hdr_exts = get_rtp_hdr_extensions(mline, config) ssrc = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.SSRC) rtp_receiver = %RTPReceiver{ssrc: ssrc} - %__MODULE__{transceiver | codecs: codecs, hdr_exts: hdr_exts, rtp_receiver: rtp_receiver} + + %__MODULE__{ + transceiver + | codecs: codecs, + rtp_hdr_exts: rtp_hdr_exts, + rtp_receiver: rtp_receiver + } end - defp get_codecs(mline) do + defp get_codecs(mline, config) do rtp_mappings = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.RTPMapping) fmtps = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.FMTP) all_rtcp_fbs = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.RTCPFeedback) - for rtp_mapping <- rtp_mappings do - fmtp = Enum.find(fmtps, fn fmtp -> fmtp.pt == rtp_mapping.payload_type end) + rtp_mappings + |> Stream.map(fn rtp_mapping -> + fmtp = Enum.find(fmtps, &(&1.pt == rtp_mapping.payload_type)) rtcp_fbs = - Enum.filter(all_rtcp_fbs, fn rtcp_fb -> rtcp_fb.pt == rtp_mapping.payload_type end) + all_rtcp_fbs + |> Stream.filter(&(&1.pt == rtp_mapping.payload_type)) + |> Enum.filter(&Configuration.is_supported_rtcp_fb(config, &1)) RTPCodecParameters.new(mline.type, rtp_mapping, fmtp, rtcp_fbs) - end + end) + |> Enum.filter(fn codec -> Configuration.is_supported_codec(config, codec) end) + end + + defp get_rtp_hdr_extensions(mline, config) do + mline + |> ExSDP.Media.get_attributes(ExSDP.Attribute.Extmap) + |> Enum.filter(&Configuration.is_supported_rtp_hdr_extension(config, &1, mline.type)) end end diff --git a/lib/ex_webrtc/sdp_utils.ex b/lib/ex_webrtc/sdp_utils.ex index 8f37cb50..2eb751ad 100644 --- a/lib/ex_webrtc/sdp_utils.ex +++ b/lib/ex_webrtc/sdp_utils.ex @@ -1,90 +1,6 @@ defmodule ExWebRTC.SDPUtils do @moduledoc false - alias ExWebRTC.RTPTransceiver - - @spec get_answer_mline(ExSDP.Media.t(), RTPTransceiver.t(), Keyword.t()) :: ExSDP.Media.t() - def get_answer_mline(mline, transceiver, config) do - # TODO: we need to filter the media formats according to our capabilities - media_formats = - Enum.filter(mline.attributes, fn - %ExSDP.Attribute.RTPMapping{} -> true - %ExSDP.Attribute.FMTP{} -> true - _other -> false - end) - - payload_types = - Enum.flat_map(media_formats, fn - %ExSDP.Attribute.RTPMapping{payload_type: pt} -> [pt] - _other -> [] - end) - - offered_direction = ExSDP.Media.get_attribute(mline, :direction) - direction = get_direction(offered_direction, transceiver.direction) - - attributes = - [ - direction, - {:mid, transceiver.mid}, - {:ice_ufrag, Keyword.fetch!(config, :ice_ufrag)}, - {:ice_pwd, Keyword.fetch!(config, :ice_pwd)}, - {:ice_options, Keyword.fetch!(config, :ice_options)}, - {:fingerprint, Keyword.fetch!(config, :fingerprint)}, - {:setup, Keyword.fetch!(config, :setup)}, - # TODO: probably should fail if the offer doesn't contain rtcp-mux - # as we don't support non-muxed streams - :rtcp_mux - ] - - # TODO: validation of some the stuff in remote SDP - %ExSDP.Media{ - ExSDP.Media.new(mline.type, 9, mline.protocol, payload_types) - | # mline must be followed by a cline, which must contain - # the default value "IN IP4 0.0.0.0" (as there are no candidates yet) - connection_data: [%ExSDP.ConnectionData{address: {0, 0, 0, 0}}] - } - |> ExSDP.Media.add_attributes(attributes ++ media_formats) - end - - def to_offer_mline(transceiver, config) do - pt = Enum.map(transceiver.codecs, fn codec -> codec.payload_type end) - - media_formats = - Enum.flat_map(transceiver.codecs, fn codec -> - [_type, encoding] = String.split(codec.mime_type, "/") - - rtp_mapping = %ExSDP.Attribute.RTPMapping{ - clock_rate: codec.clock_rate, - encoding: encoding, - params: codec.channels, - payload_type: codec.payload_type - } - - [rtp_mapping, codec.sdp_fmtp_line, codec.rtcp_fbs] - end) - - attributes = - if(Keyword.get(config, :rtcp, false), do: [{"rtcp", "9 IN IP4 0.0.0.0"}], else: []) ++ - [ - transceiver.direction, - {:mid, transceiver.mid}, - {:ice_ufrag, Keyword.fetch!(config, :ice_ufrag)}, - {:ice_pwd, Keyword.fetch!(config, :ice_pwd)}, - {:ice_options, Keyword.fetch!(config, :ice_options)}, - {:fingerprint, Keyword.fetch!(config, :fingerprint)}, - {:setup, Keyword.fetch!(config, :setup)}, - :rtcp_mux - ] - - %ExSDP.Media{ - ExSDP.Media.new(transceiver.kind, 9, "UDP/TLS/RTP/SAVPF", pt) - | # mline must be followed by a cline, which must contain - # the default value "IN IP4 0.0.0.0" (as there are no candidates yet) - connection_data: [%ExSDP.ConnectionData{address: {0, 0, 0, 0}}] - } - |> ExSDP.Media.add_attributes(attributes ++ media_formats) - end - @spec get_media_direction(ExSDP.Media.t()) :: :sendrecv | :sendonly | :recvonly | :inactive | nil def get_media_direction(media) do @@ -229,14 +145,4 @@ defmodule ExWebRTC.SDPUtils do _ -> {:error, :conflicting_ice_credentials} end end - - # RFC 3264 (6.1) + RFC 8829 (5.3.1) - # AFAIK one of the cases should always match - # bc we won't assign/create an inactive transceiver to i.e. sendonly mline - # also neither of the arguments should ever be :stopped - defp get_direction(_, :inactive), do: :inactive - defp get_direction(:sendonly, t) when t in [:sendrecv, :recvonly], do: :recvonly - defp get_direction(:recvonly, t) when t in [:sendrecv, :sendonly], do: :sendonly - defp get_direction(o, other) when o in [:sendrecv, nil], do: other - defp get_direction(:inactive, _), do: :inactive end diff --git a/test/configuration_test.exs b/test/configuration_test.exs new file mode 100644 index 00000000..6c9313dd --- /dev/null +++ b/test/configuration_test.exs @@ -0,0 +1,149 @@ +defmodule ExWebRTC.PeerConnection.ConfigurationTest do + use ExUnit.Case + + alias ExWebRTC.{PeerConnection, RTPCodecParameters, RTPTransceiver, SessionDescription} + + alias ExSDP.Attribute.{Extmap, FMTP} + + @tag :debug + test "codecs and rtp hdr extensions" do + audio_level_rtp_hdr_ext = %Extmap{ + id: 1, + uri: "urn:ietf:params:rtp-hdrext:ssrc-audio-level" + } + + mid_rtp_hdr_ext = %Extmap{ + id: 4, + uri: "urn:ietf:params:rtp-hdrext:sdes:mid" + } + + opus_codec = %RTPCodecParameters{ + payload_type: 111, + mime_type: "audio/opus", + clock_rate: 48_000, + channels: 2, + sdp_fmtp_line: %FMTP{ + pt: 111, + minptime: 10, + useinbandfec: true + }, + rtcp_fbs: [] + } + + h264_codec = %RTPCodecParameters{ + payload_type: 102, + mime_type: "video/H264", + clock_rate: 90_000, + sdp_fmtp_line: %FMTP{ + pt: 102, + profile_level_id: 0x42001F, + level_asymmetry_allowed: true, + packetization_mode: 1 + }, + rtcp_fbs: [] + } + + vp8_codec = %RTPCodecParameters{ + payload_type: 96, + mime_type: "video/VP8", + clock_rate: 90_000, + rtcp_fbs: [] + } + + av1_codec = %RTPCodecParameters{ + payload_type: 45, + mime_type: "video/AV1", + clock_rate: 90_000, + rtcp_fbs: [] + } + + # default audio and video codecs + # assert there are only them - no av1, g711 or others + {:ok, pc} = PeerConnection.start_link() + + offer = %SessionDescription{ + type: :offer, + sdp: File.read!("test/fixtures/audio_video_sdp.txt") + } + + assert :ok = PeerConnection.set_remote_description(pc, offer) + transceivers = PeerConnection.get_transceivers(pc) + + assert [ + %RTPTransceiver{ + mid: "0", + direction: :recvonly, + kind: :audio, + rtp_hdr_exts: [^mid_rtp_hdr_ext], + codecs: [^opus_codec] + }, + %RTPTransceiver{ + mid: "1", + direction: :recvonly, + kind: :video, + rtp_hdr_exts: [^mid_rtp_hdr_ext], + codecs: [^vp8_codec, ^h264_codec] + } + ] = transceivers + + assert :ok = PeerConnection.close(pc) + + # audio level rtp hdr ext, no audio codecs and one non-default av1 codec + # assert there are no audio, h264 and vp8 codecs, and there is audio level + # rtp hdr extension + {:ok, pc} = + PeerConnection.start_link( + audio_codecs: [], + video_codecs: [av1_codec], + rtp_hdr_extensions: [:audio_level] + ) + + offer = %SessionDescription{ + type: :offer, + sdp: File.read!("test/fixtures/audio_video_sdp.txt") + } + + assert :ok = PeerConnection.set_remote_description(pc, offer) + + assert [ + %ExWebRTC.RTPTransceiver{ + mid: "0", + direction: :recvonly, + kind: :audio, + rtp_hdr_exts: [^audio_level_rtp_hdr_ext, ^mid_rtp_hdr_ext], + codecs: [] + }, + %RTPTransceiver{ + mid: "1", + direction: :recvonly, + kind: :video, + rtp_hdr_exts: [^mid_rtp_hdr_ext], + codecs: [^av1_codec] + } + ] = PeerConnection.get_transceivers(pc) + + {:ok, answer} = PeerConnection.create_answer(pc) + sdp = ExSDP.parse!(answer.sdp) + + # assert that audio mline has been rejected + # as we didn't add any supported audio codecs + assert List.first(sdp.media).port == 0 + assert :ok = PeerConnection.close(pc) + + # additional audio level header extension + # assert it is only present in audio transceiver + {:ok, pc} = PeerConnection.start_link(rtp_hdr_extensions: [:audio_level]) + {:ok, tr} = PeerConnection.add_transceiver(pc, :audio) + + # we can't compare ids as those used in audio_video_offer are + # different than those used by us + assert [ + %Extmap{uri: "urn:ietf:params:rtp-hdrext:ssrc-audio-level"}, + %Extmap{uri: "urn:ietf:params:rtp-hdrext:sdes:mid"} + ] = tr.rtp_hdr_exts + + {:ok, tr} = PeerConnection.add_transceiver(pc, :video) + assert [%Extmap{uri: "urn:ietf:params:rtp-hdrext:sdes:mid"}] = tr.rtp_hdr_exts + :ok = PeerConnection.close(pc) + end +end diff --git a/test/fixtures/audio_sdp.txt b/test/fixtures/audio_sdp.txt new file mode 100644 index 00000000..3a9ab88d --- /dev/null +++ b/test/fixtures/audio_sdp.txt @@ -0,0 +1,36 @@ +v=0 +o=- 7169721126327183323 3 IN IP4 127.0.0.1 +s=- +t=0 0 +a=group:BUNDLE 0 +a=extmap-allow-mixed +a=msid-semantic: WMS +m=audio 9 UDP/TLS/RTP/SAVPF 111 63 9 0 8 13 110 126 +c=IN IP4 0.0.0.0 +a=rtcp:9 IN IP4 0.0.0.0 +a=ice-ufrag:nWnq +a=ice-pwd:DTdCVj00s9nGkNgWra0P30A2 +a=ice-options:trickle +a=fingerprint:sha-256 A4:67:3A:52:5C:D4:9C:65:45:9D:61:43:B2:EC:31:E8:42:4E:EC:54:C9:70:E3:89:28:94:3E:5C:3C:ED:FC:64 +a=setup:actpass +a=mid:0 +a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level +a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid +a=sendrecv +a=msid:- 06574f1e-f4bf-4b6d-b66c-3493cd7ab50f +a=rtcp-mux +a=rtpmap:111 opus/48000/2 +a=rtcp-fb:111 transport-cc +a=fmtp:111 minptime=10;useinbandfec=1 +a=rtpmap:63 red/48000/2 +a=fmtp:63 111/111 +a=rtpmap:9 G722/8000 +a=rtpmap:0 PCMU/8000 +a=rtpmap:8 PCMA/8000 +a=rtpmap:13 CN/8000 +a=rtpmap:110 telephone-event/48000 +a=rtpmap:126 telephone-event/8000 +a=ssrc:3262661846 cname:yFbnkIWTHwPXy6MO +a=ssrc:3262661846 msid:- 06574f1e-f4bf-4b6d-b66c-3493cd7ab50f diff --git a/test/fixtures/audio_video_sdp.txt b/test/fixtures/audio_video_sdp.txt new file mode 100644 index 00000000..bea11448 --- /dev/null +++ b/test/fixtures/audio_video_sdp.txt @@ -0,0 +1,157 @@ +v=0 +o=- 7169721126327183323 3 IN IP4 127.0.0.1 +s=- +t=0 0 +a=group:BUNDLE 0 1 +a=extmap-allow-mixed +a=msid-semantic: WMS +m=audio 9 UDP/TLS/RTP/SAVPF 111 63 9 0 8 13 110 126 +c=IN IP4 0.0.0.0 +a=rtcp:9 IN IP4 0.0.0.0 +a=ice-ufrag:nWnq +a=ice-pwd:DTdCVj00s9nGkNgWra0P30A2 +a=ice-options:trickle +a=fingerprint:sha-256 A4:67:3A:52:5C:D4:9C:65:45:9D:61:43:B2:EC:31:E8:42:4E:EC:54:C9:70:E3:89:28:94:3E:5C:3C:ED:FC:64 +a=setup:actpass +a=mid:0 +a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level +a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid +a=sendrecv +a=msid:- 06574f1e-f4bf-4b6d-b66c-3493cd7ab50f +a=rtcp-mux +a=rtpmap:111 opus/48000/2 +a=rtcp-fb:111 transport-cc +a=fmtp:111 minptime=10;useinbandfec=1 +a=rtpmap:63 red/48000/2 +a=fmtp:63 111/111 +a=rtpmap:9 G722/8000 +a=rtpmap:0 PCMU/8000 +a=rtpmap:8 PCMA/8000 +a=rtpmap:13 CN/8000 +a=rtpmap:110 telephone-event/48000 +a=rtpmap:126 telephone-event/8000 +a=ssrc:3262661846 cname:yFbnkIWTHwPXy6MO +a=ssrc:3262661846 msid:- 06574f1e-f4bf-4b6d-b66c-3493cd7ab50f +m=video 9 UDP/TLS/RTP/SAVPF 96 97 102 103 104 105 106 107 108 109 127 125 39 40 45 46 98 99 100 101 112 113 114 +c=IN IP4 0.0.0.0 +a=rtcp:9 IN IP4 0.0.0.0 +a=ice-ufrag:nWnq +a=ice-pwd:DTdCVj00s9nGkNgWra0P30A2 +a=ice-options:trickle +a=fingerprint:sha-256 A4:67:3A:52:5C:D4:9C:65:45:9D:61:43:B2:EC:31:E8:42:4E:EC:54:C9:70:E3:89:28:94:3E:5C:3C:ED:FC:64 +a=setup:actpass +a=mid:1 +a=extmap:14 urn:ietf:params:rtp-hdrext:toffset +a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +a=extmap:13 urn:3gpp:video-orientation +a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay +a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type +a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing +a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space +a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid +a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id +a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id +a=sendrecv +a=msid:- 0a874693-9ca3-44a4-8c95-d8f2e26a7179 +a=rtcp-mux +a=rtcp-rsize +a=rtpmap:96 VP8/90000 +a=rtcp-fb:96 goog-remb +a=rtcp-fb:96 transport-cc +a=rtcp-fb:96 ccm fir +a=rtcp-fb:96 nack +a=rtcp-fb:96 nack pli +a=rtpmap:97 rtx/90000 +a=fmtp:97 apt=96 +a=rtpmap:102 H264/90000 +a=rtcp-fb:102 goog-remb +a=rtcp-fb:102 transport-cc +a=rtcp-fb:102 ccm fir +a=rtcp-fb:102 nack +a=rtcp-fb:102 nack pli +a=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f +a=rtpmap:103 rtx/90000 +a=fmtp:103 apt=102 +a=rtpmap:104 H264/90000 +a=rtcp-fb:104 goog-remb +a=rtcp-fb:104 transport-cc +a=rtcp-fb:104 ccm fir +a=rtcp-fb:104 nack +a=rtcp-fb:104 nack pli +a=fmtp:104 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f +a=rtpmap:105 rtx/90000 +a=fmtp:105 apt=104 +a=rtpmap:106 H264/90000 +a=rtcp-fb:106 goog-remb +a=rtcp-fb:106 transport-cc +a=rtcp-fb:106 ccm fir +a=rtcp-fb:106 nack +a=rtcp-fb:106 nack pli +a=fmtp:106 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f +a=rtpmap:107 rtx/90000 +a=fmtp:107 apt=106 +a=rtpmap:108 H264/90000 +a=rtcp-fb:108 goog-remb +a=rtcp-fb:108 transport-cc +a=rtcp-fb:108 ccm fir +a=rtcp-fb:108 nack +a=rtcp-fb:108 nack pli +a=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f +a=rtpmap:109 rtx/90000 +a=fmtp:109 apt=108 +a=rtpmap:127 H264/90000 +a=rtcp-fb:127 goog-remb +a=rtcp-fb:127 transport-cc +a=rtcp-fb:127 ccm fir +a=rtcp-fb:127 nack +a=rtcp-fb:127 nack pli +a=fmtp:127 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f +a=rtpmap:125 rtx/90000 +a=fmtp:125 apt=127 +a=rtpmap:39 H264/90000 +a=rtcp-fb:39 goog-remb +a=rtcp-fb:39 transport-cc +a=rtcp-fb:39 ccm fir +a=rtcp-fb:39 nack +a=rtcp-fb:39 nack pli +a=fmtp:39 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f +a=rtpmap:40 rtx/90000 +a=fmtp:40 apt=39 +a=rtpmap:45 AV1/90000 +a=rtcp-fb:45 goog-remb +a=rtcp-fb:45 transport-cc +a=rtcp-fb:45 ccm fir +a=rtcp-fb:45 nack +a=rtcp-fb:45 nack pli +a=rtpmap:46 rtx/90000 +a=fmtp:46 apt=45 +a=rtpmap:98 VP9/90000 +a=rtcp-fb:98 goog-remb +a=rtcp-fb:98 transport-cc +a=rtcp-fb:98 ccm fir +a=rtcp-fb:98 nack +a=rtcp-fb:98 nack pli +a=fmtp:98 profile-id=0 +a=rtpmap:99 rtx/90000 +a=fmtp:99 apt=98 +a=rtpmap:100 VP9/90000 +a=rtcp-fb:100 goog-remb +a=rtcp-fb:100 transport-cc +a=rtcp-fb:100 ccm fir +a=rtcp-fb:100 nack +a=rtcp-fb:100 nack pli +a=fmtp:100 profile-id=2 +a=rtpmap:101 rtx/90000 +a=fmtp:101 apt=100 +a=rtpmap:112 red/90000 +a=rtpmap:113 rtx/90000 +a=fmtp:113 apt=112 +a=rtpmap:114 ulpfec/90000 +a=ssrc-group:FID 1862494604 1112104850 +a=ssrc:1862494604 cname:yFbnkIWTHwPXy6MO +a=ssrc:1862494604 msid:- 0a874693-9ca3-44a4-8c95-d8f2e26a7179 +a=ssrc:1112104850 cname:yFbnkIWTHwPXy6MO +a=ssrc:1112104850 msid:- 0a874693-9ca3-44a4-8c95-d8f2e26a7179 diff --git a/test/peer_connection_test.exs b/test/peer_connection_test.exs index 7727767e..dfdeb876 100644 --- a/test/peer_connection_test.exs +++ b/test/peer_connection_test.exs @@ -3,84 +3,6 @@ defmodule ExWebRTC.PeerConnectionTest do alias ExWebRTC.{MediaStreamTrack, PeerConnection, SessionDescription} - @single_audio_offer """ - v=0 - o=- 6788894006044524728 2 IN IP4 127.0.0.1 - s=- - t=0 0 - a=group:BUNDLE 0 - a=extmap-allow-mixed - a=msid-semantic: WMS - m=audio 9 UDP/TLS/RTP/SAVPF 111 - c=IN IP4 0.0.0.0 - a=rtcp:9 IN IP4 0.0.0.0 - a=ice-ufrag:cDua - a=ice-pwd:v9SCmZHxJWtgpyzn8Ts1puT6 - a=ice-options:trickle - a=fingerprint:sha-256 11:35:68:66:A4:C3:C0:AA:37:4E:0F:97:D7:9F:76:11:08:DB:56:DA:4B:83:77:50:9A:D2:71:8D:2A:A8:E3:07 - a=setup:actpass - a=mid:0 - a=sendrecv - a=msid:- 54f0751b-086f-433c-af40-79c179182423 - a=rtcp-mux - a=rtpmap:111 opus/48000/2 - a=rtcp-fb:111 transport-cc - a=fmtp:111 minptime=10;useinbandfec=1 - a=ssrc:1463342914 cname:poWwjNZ4I2ZZgzY7 - a=ssrc:1463342914 msid:- 54f0751b-086f-433c-af40-79c179182423 - """ - - @audio_video_offer """ - v=0 - o=- 3253533641493747086 5 IN IP4 127.0.0.1 - s=- - t=0 0 - a=group:BUNDLE 0 1 - a=extmap-allow-mixed - a=msid-semantic: WMS - m=audio 9 UDP/TLS/RTP/SAVPF 111 - c=IN IP4 0.0.0.0 - a=rtcp:9 IN IP4 0.0.0.0 - a=ice-ufrag:SOct - a=ice-pwd:k9PRXt7zT32ADt/juUpt4Gx3 - a=ice-options:trickle - a=fingerprint:sha-256 45:B5:2D:3A:DA:29:93:27:B6:59:F1:5B:77:62:F5:C2:CE:16:8B:12:C7:B8:34:EF:C0:12:45:17:D0:1A:E6:F4 - a=setup:actpass - a=mid:0 - a=sendrecv - a=msid:- 0970fb0b-4750-4302-902e-70d2e403ad0d - a=rtcp-mux - a=rtpmap:111 opus/48000/2 - a=rtcp-fb:111 transport-cc - a=fmtp:111 minptime=10;useinbandfec=1 - a=ssrc:560549895 cname:QQJypppcjR+gR484 - a=ssrc:560549895 msid:- 0970fb0b-4750-4302-902e-70d2e403ad0d - m=video 9 UDP/TLS/RTP/SAVPF 96 - c=IN IP4 0.0.0.0 - a=rtcp:9 IN IP4 0.0.0.0 - a=ice-ufrag:SOct - a=ice-pwd:k9PRXt7zT32ADt/juUpt4Gx3 - a=ice-options:trickle - a=fingerprint:sha-256 45:B5:2D:3A:DA:29:93:27:B6:59:F1:5B:77:62:F5:C2:CE:16:8B:12:C7:B8:34:EF:C0:12:45:17:D0:1A:E6:F4 - a=setup:actpass - a=mid:1 - a=sendrecv - a=msid:- 1259ea70-c6b7-445a-9c20-49cec7433ccb - a=rtcp-mux - a=rtcp-rsize - a=rtpmap:96 VP8/90000 - a=rtcp-fb:96 goog-remb - a=rtcp-fb:96 transport-cc - a=rtcp-fb:96 ccm fir - a=rtcp-fb:96 nack - a=rtcp-fb:96 nack pli - a=ssrc-group:FID 381060598 184440407 - a=ssrc:381060598 cname:QQJypppcjR+gR484 - a=ssrc:381060598 msid:- 1259ea70-c6b7-445a-9c20-49cec7433ccb - a=ssrc:184440407 cname:QQJypppcjR+gR484 - a=ssrc:184440407 msid:- 1259ea70-c6b7-445a-9c20-49cec7433ccb - """ - @audio_mline ExSDP.Media.new("audio", 9, "UDP/TLS/RTP/SAVPF", [108]) |> ExSDP.Media.add_attributes(mid: "0", ice_ufrag: "someufrag", ice_pwd: "somepwd") @@ -90,12 +12,16 @@ defmodule ExWebRTC.PeerConnectionTest do test "track notification" do {:ok, pc} = PeerConnection.start_link() - offer = %SessionDescription{type: :offer, sdp: @single_audio_offer} + offer = %SessionDescription{type: :offer, sdp: File.read!("test/fixtures/audio_sdp.txt")} :ok = PeerConnection.set_remote_description(pc, offer) assert_receive {:ex_webrtc, ^pc, {:track, %MediaStreamTrack{mid: "0", kind: :audio}}} - offer = %SessionDescription{type: :offer, sdp: @audio_video_offer} + offer = %SessionDescription{ + type: :offer, + sdp: File.read!("test/fixtures/audio_video_sdp.txt") + } + :ok = PeerConnection.set_remote_description(pc, offer) assert_receive {:ex_webrtc, ^pc, {:track, %MediaStreamTrack{mid: "1", kind: :video}}}