Skip to content

Commit

Permalink
Make codecs and rtp header extensions configurable
Browse files Browse the repository at this point in the history
  • Loading branch information
@mickel8
mickel8 committed Nov 8, 2023
1 parent d9aafde commit ae46502
Show file tree
Hide file tree
Showing 9 changed files with 683 additions and 232 deletions.
63 changes: 33 additions & 30 deletions lib/ex_webrtc/peer_connection.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,13 +95,15 @@ defmodule ExWebRTC.PeerConnection do
GenServer.call(peer_connection, :get_transceivers)
end

@spec add_transceiver(
peer_connection(),
RTPTransceiver.kind() | MediaStreamTrack.t(),
transceiver_options()
) :: {:ok, RTPTransceiver.t()} | {:error, :TODO}
def add_transceiver(peer_connection, track_or_kind, options \\ []) do
GenServer.call(peer_connection, {:add_transceiver, track_or_kind, options})
@spec add_transceiver(peer_connection(), RTPTransceiver.kind(), transceiver_options()) ::
{:ok, RTPTransceiver.t()} | {:error, :TODO}
def add_transceiver(peer_connection, kind, options \\ []) do
GenServer.call(peer_connection, {:add_transceiver, kind, options})
end

@spec close(peer_connection()) :: :ok
def close(peer_connection) do
GenServer.stop(peer_connection)
end

#### CALLBACKS ####
Expand Down Expand Up @@ -145,7 +147,7 @@ defmodule ExWebRTC.PeerConnection do
# we support trickle ICE only
|> ExSDP.add_attribute({:ice_options, "trickle"})

config =
opts =
[
ice_ufrag: ice_ufrag,
ice_pwd: ice_pwd,
Expand All @@ -155,10 +157,7 @@ defmodule ExWebRTC.PeerConnection do
rtcp: true
]

mlines =
Enum.map(transceivers, fn transceiver ->
SDPUtils.to_offer_mline(transceiver, config)
end)
mlines = Enum.map(transceivers, &RTPTransceiver.to_offer_mline(&1, opts))

mids =
Enum.map(mlines, fn mline ->
Expand Down Expand Up @@ -200,7 +199,7 @@ defmodule ExWebRTC.PeerConnection do
# we only support trickle ICE, so non-trickle offers should be rejected earlier
|> ExSDP.add_attribute({:ice_options, "trickle"})

config =
opts =
[
ice_ufrag: ice_ufrag,
ice_pwd: ice_pwd,
Expand All @@ -214,7 +213,7 @@ defmodule ExWebRTC.PeerConnection do
Enum.map(remote_offer.media, fn mline ->
{:mid, mid} = ExSDP.Media.get_attribute(mline, :mid)
{_ix, transceiver} = RTPTransceiver.find_by_mid(state.transceivers, mid)
SDPUtils.get_answer_mline(mline, transceiver, config)
RTPTransceiver.to_answer_mline(transceiver, mline, opts)
end)

mids =
Expand Down Expand Up @@ -299,21 +298,24 @@ defmodule ExWebRTC.PeerConnection do
end

@impl true
def handle_call({:add_transceiver, :audio, options}, _from, state) do
# TODO: proper implementation, change the :audio above to track_or_kind
def handle_call({:add_transceiver, kind, options}, _from, state)
when kind in [:audio, :video] do
direction = Keyword.get(options, :direction, :sendrcv)

# hardcoded audio codec
codecs = [
%ExWebRTC.RTPCodecParameters{
payload_type: 111,
mime_type: "audio/opus",
clock_rate: 48_000,
channels: 2
}
]

transceiver = %RTPTransceiver{mid: nil, direction: direction, kind: :audio, codecs: codecs}
{rtp_hdr_exts, codecs} =
case kind do
:audio -> {state.config.audio_rtp_hdr_exts, state.config.audio_codecs}
:video -> {state.config.video_rtp_hdr_exts, state.config.video_codecs}
end

transceiver = %RTPTransceiver{
mid: nil,
direction: direction,
kind: kind,
codecs: codecs,
rtp_hdr_exts: rtp_hdr_exts
}

transceivers = List.insert_at(state.transceivers, -1, transceiver)
{:reply, {:ok, transceiver}, %{state | transceivers: transceivers}}
end
Expand Down Expand Up @@ -395,7 +397,8 @@ defmodule ExWebRTC.PeerConnection do
with :ok <- SDPUtils.ensure_mid(sdp),
:ok <- SDPUtils.ensure_bundle(sdp),
{:ok, {ice_ufrag, ice_pwd}} <- SDPUtils.get_ice_credentials(sdp),
{:ok, new_transceivers} <- update_remote_transceivers(state.transceivers, sdp) do
{:ok, new_transceivers} <-
update_remote_transceivers(state.transceivers, sdp, state.config) do
:ok = ICEAgent.set_remote_credentials(state.ice_agent, ice_ufrag, ice_pwd)
:ok = ICEAgent.gather_candidates(state.ice_agent)

Expand Down Expand Up @@ -433,11 +436,11 @@ defmodule ExWebRTC.PeerConnection do
end
end

defp update_remote_transceivers(transceivers, sdp) do
defp update_remote_transceivers(transceivers, sdp, config) do
Enum.reduce_while(sdp.media, {:ok, transceivers}, fn mline, {:ok, transceivers} ->
case ExSDP.Media.get_attribute(mline, :mid) do
{:mid, mid} ->
transceivers = RTPTransceiver.update_or_create(transceivers, mid, mline)
transceivers = RTPTransceiver.update_or_create(transceivers, mid, mline, config)
{:cont, {:ok, transceivers}}

_other ->
Expand Down
199 changes: 189 additions & 10 deletions lib/ex_webrtc/peer_connection/configuration.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,60 @@ defmodule ExWebRTC.PeerConnection.Configuration do
PeerConnection configuration
"""

alias ExWebRTC.RTPCodecParameters
alias ExSDP.Attribute.{Extmap, FMTP, RTCPFeedback}

@default_audio_codecs [
%RTPCodecParameters{
payload_type: 111,
mime_type: "audio/opus",
clock_rate: 48_000,
channels: 2,
sdp_fmtp_line: %FMTP{pt: 111, minptime: 10, useinbandfec: true},
rtcp_fbs: []
}
]

@default_video_codecs [
%RTPCodecParameters{
payload_type: 98,
mime_type: "video/H264",
clock_rate: 90_000,
channels: nil,
sdp_fmtp_line: %FMTP{
pt: 98,
level_asymmetry_allowed: true,
packetization_mode: 1,
profile_level_id: 0x42001F
},
rtcp_fbs: []
},
%RTPCodecParameters{
payload_type: 96,
mime_type: "video/VP8",
clock_rate: 90_000,
channels: nil,
sdp_fmtp_line: nil,
rtcp_fbs: []
}
]

@rtp_hdr_extensions %{
:mid => %{media_type: :all, ext: %Extmap{id: 1, uri: "urn:ietf:params:rtp-hdrext:sdes:mid"}},
:audio_level => %{
media_type: :audio,
ext: %Extmap{id: 2, uri: "urn:ietf:params:rtp-hdrext:ssrc-audio-level"}
}
}

@mandatory_audio_rtp_hdr_exts Enum.map([:mid], &Map.fetch!(@rtp_hdr_extensions, &1).ext)
@mandatory_video_rtp_hdr_exts Enum.map([:mid], &Map.fetch!(@rtp_hdr_extensions, &1).ext)

@typedoc """
Supported RTP header extensions.
"""
@type rtp_hdr_extension() :: :audio_level

@type ice_server() :: %{
optional(:credential) => String.t(),
optional(:username) => String.t(),
Expand All @@ -12,32 +66,157 @@ defmodule ExWebRTC.PeerConnection.Configuration do
@typedoc """
Options that can be passed to `ExWebRTC.PeerConnection.start_link/1`.
Currently, ExWebRTC always uses the following config:
* `ice_servers` - list of STUN servers to use.
TURN servers are not supported right now and will be filtered out.
* `audio_codecs` - list of audio codecs to use.
Use `default_audio_codecs/0` to get a list of default audio codecs.
This option overrides default audio codecs.
If you wish to add codecs to default ones do
`audio_codecs: Configuration.default_audio_codecs() ++ my_codecs`
* `video_codecs` - the same as `audio_codecs` but for video.
* `rtp_hdr_extensions` - list of RTP header extensions to use.
MID extension is enabled by default and cannot be turned off.
If an extension can be used both for audio and video media, it
will be added to every mline.
If an extension is audio-only, it will only be added to audio mlines.
If an extension is video-only, it will only be added to video mlines.
Besides options listed above, ExWebRTC uses the following config:
* bundle_policy - max_bundle
* ice_candidate_pool_size - 0
* ice_transport_policy - all
* rtcp_mux_policy - require
This config cannot be changed.
"""
@type options() :: [ice_servers: [ice_server()]]
@type options() :: [
ice_servers: [ice_server()],
audio_codecs: [RTPCodecParameters.t()],
video_codecs: [RTPCodecParameters.t()],
rtp_hdr_extensions: [rtp_hdr_extension()]
]

@typedoc false
@type t() :: %__MODULE__{ice_servers: [ice_server()]}
@type t() :: %__MODULE__{
ice_servers: [ice_server()],
audio_codecs: [RTPCodecParameters.t()],
video_codecs: [RTPCodecParameters.t()],
audio_rtp_hdr_exts: [Extmap.t()],
video_rtp_hdr_exts: [Extmap.t()]
}

defstruct ice_servers: [],
audio_codecs: @default_audio_codecs,
video_codecs: @default_video_codecs,
audio_rtp_hdr_exts: @mandatory_audio_rtp_hdr_exts,
video_rtp_hdr_exts: @mandatory_video_rtp_hdr_exts

@doc """
Returns a list of default audio codecs.
"""
@spec default_audio_codecs() :: [RTPCodecParameters.t()]
def default_audio_codecs(), do: @default_audio_codecs

defstruct ice_servers: []
@doc """
Returns a list of default video codecs.
"""
@spec default_video_codecs() :: [RTPCodecParameters.t()]
def default_video_codecs(), do: @default_video_codecs

@doc false
@spec from_options!(options()) :: t()
def from_options!(options) do
config = struct!(__MODULE__, options)
options =
options
|> resolve_rtp_hdr_extensions()
|> add_mandatory_rtp_hdr_extensions()
# ATM, ExICE does not support relay via TURN
|> reject_turn_servers()

struct!(__MODULE__, options)
end

@doc false
@spec is_supported_codec(t(), RTPCodecParameters.t()) :: boolean()
def is_supported_codec(config, codec) do
# This function doesn't check if rtcp-fb is supported.
# Instead, `is_supported_rtcp_fb` has to be used to filter out
# rtcp-fb that are not supported.
Enum.find(
config.audio_codecs ++ config.video_codecs,
fn supported_codec ->
# for the time of comparision, override payload type in our codec
supported_codec =
if supported_codec.sdp_fmtp_line != nil and codec.sdp_fmtp_line != nil do
%RTPCodecParameters{
supported_codec
| sdp_fmtp_line: %FMTP{supported_codec.sdp_fmtp_line | pt: codec.sdp_fmtp_line.pt}
}
else
supported_codec
end

supported_codec.mime_type == codec.mime_type and
supported_codec.clock_rate == codec.clock_rate and
supported_codec.channels == codec.channels and
supported_codec.sdp_fmtp_line == codec.sdp_fmtp_line
end
)
end

@doc false
@spec is_supported_rtp_hdr_extension(t(), Extmap.t(), :audio | :video) ::
boolean()
def is_supported_rtp_hdr_extension(config, rtp_hdr_extension, media_type) do
supported_uris =
case media_type do
:audio -> Enum.map(config.audio_rtp_hdr_exts, & &1.uri)
:video -> Enum.map(config.video_rtp_hdr_exts, & &1.uri)
end

rtp_hdr_extension.uri in supported_uris
end

@doc false
@spec is_supported_rtcp_fb(t(), RTCPFeedback.t()) :: boolean()
def is_supported_rtcp_fb(_config, _rtcp_fb), do: false

# ATM, ExICE does not support relay via TURN
stun_servers =
config.ice_servers
defp add_mandatory_rtp_hdr_extensions(options) do
options
|> Keyword.update(:audio_rtp_hdr_exts, [], fn exts ->
exts ++ @mandatory_audio_rtp_hdr_exts
end)
|> Keyword.update(:video_rtp_hdr_exts, [], fn exts ->
exts ++ @mandatory_video_rtp_hdr_exts
end)
end

defp resolve_rtp_hdr_extensions(options) do
{audio_exts, video_exts} =
Keyword.get(options, :rtp_hdr_extensions, [])
|> Enum.reduce({[], []}, fn ext, {audio_exts, video_exts} ->
resolved_ext = Map.fetch!(@rtp_hdr_extensions, ext)

case resolved_ext.media_type do
:audio ->
{[resolved_ext.ext | audio_exts], video_exts}

:all ->
{[resolved_ext.ext | audio_exts], [resolved_ext.ext | video_exts]}
end
end)

options
|> Keyword.put(:audio_rtp_hdr_exts, Enum.reverse(audio_exts))
|> Keyword.put(:video_rtp_hdr_exts, Enum.reverse(video_exts))
|> Keyword.delete(:rtp_hdr_extensions)
end

defp reject_turn_servers(options) do
Keyword.update(options, :ice_servers, [], fn ice_servers ->
ice_servers
|> Enum.flat_map(&List.wrap(&1.urls))
|> Enum.filter(&String.starts_with?(&1, "stun:"))

%__MODULE__{config | ice_servers: stun_servers}
end)
end
end
9 changes: 9 additions & 0 deletions lib/ex_webrtc/rtp_codec_parameters.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,15 @@ defmodule ExWebRTC.RTPCodecParameters do
RTPCodecParameters
"""

@type t() :: %__MODULE__{
payload_type: non_neg_integer(),
mime_type: binary(),
clock_rate: non_neg_integer(),
channels: non_neg_integer() | nil,
sdp_fmtp_line: ExSDP.Attribute.FMTP.t() | nil,
rtcp_fbs: [ExSDP.Attribute.RTCPFeedback.t()]
}

defstruct [:payload_type, :mime_type, :clock_rate, :channels, :sdp_fmtp_line, :rtcp_fbs]

def new(type, rtp_mapping, fmtp, rtcp_fbs) do
Expand Down
Loading

0 comments on commit ae46502

Please sign in to comment.