Skip to content

Commit

Permalink
fix: Partially handles screen-sharing simulcast.
Browse files Browse the repository at this point in the history
This is the first PR from a (hopefully short) series of PRs that will
implement proper support for simulcast for screen-sharing.

When VP8-based screen-sharing is activated Chrome is sending 2 streams
with the same resolution but with different bitrates and different
temporal layer bitrate allocations. This is different from what the
bridge expects, which is 3 streams with _different_ resolutions. The
different resolutions imply a quality order enabling the bridge to
switch from low quality to high quality and vica-versa.

Since we can no longer rely on the streams having different resolutions,
there's no longer an implicit order that the bridge can detect
on-the-fly, so now we take this information from the signaling.
  • Loading branch information
gpolitis authored and bbaldino committed Feb 12, 2019
1 parent 8ae7961 commit 23f6323
Show file tree
Hide file tree
Showing 5 changed files with 37 additions and 113 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,21 @@ public boolean accept(@NotNull RawPacket rtpPacket)
// suspended so that it can raise the needsKeyframe flag and also allow
// it to compute a sequence number delta when the target becomes > -1.

RTPEncodingDesc encoding = getSource()
.getMediaStreamTrackReceiver().findRTPEncodingDesc(rtpPacket);

if (encoding == null)
{
logger.warn(
"Dropping an RTP packet, because the SSRC has not " +
"been signaled:" + rtpPacket.getSSRCAsLong());

return false;
}

int targetIndexCopy = targetIndex;
boolean accept = contextCopy.accept(rtpPacket, targetIndexCopy);
boolean accept = contextCopy.accept(
rtpPacket, encoding.getIndex(), targetIndexCopy);

// We check if the context needs a keyframe regardless of whether or not
// the packet was accepted.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,11 @@ public interface AdaptiveTrackProjectionContext
* Determines whether an RTP packet should be accepted or not.
*
* @param rtpPacket the RTP packet to determine whether to accept or not.
* @param incomingIndex the quality index of the incoming RTP packet.
* @param targetIndex the target quality index
* @return true if the packet should be accepted, false otherwise.
*/
boolean accept(RawPacket rtpPacket, int targetIndex);
boolean accept(RawPacket rtpPacket, int incomingIndex, int targetIndex);

/**
* @return true if this stream context needs a keyframe in order to either
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -148,12 +148,13 @@ class GenericAdaptiveTrackProjectionContext
* thread) accessing this method at a time.
*
* @param rtpPacket the RTP packet to determine whether to accept or not.
* @param incomingIndex the quality index of the
* @param targetIndex the target quality index
* @return true if the packet should be accepted, false otherwise.
*/
@Override
public synchronized boolean
accept(@NotNull RawPacket rtpPacket, int targetIndex)
accept(@NotNull RawPacket rtpPacket, int incomingIndex, int targetIndex)
{
if (targetIndex == RTPEncodingDesc.SUSPENDED_INDEX)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -191,12 +191,13 @@ public VP8AdaptiveTrackProjectionContext(
* method at a time.
*
* @param rtpPacket the VP8 packet to decide whether or not to project.
* @param targetIndex
* @param incomingIndex the quality index of the incoming RTP packet
* @param targetIndex the target quality index we want to achieve
* @return true to project the packet, otherwise false.
*/
private synchronized
VP8FrameProjection createVP8FrameProjection(
@NotNull RawPacket rtpPacket, int targetIndex)
@NotNull RawPacket rtpPacket, int incomingIndex, int targetIndex)
{
// Creating a new VP8 projection depends on reading and results in
// writing of the last VP8 frame, therefore this method needs to be
Expand Down Expand Up @@ -236,7 +237,8 @@ VP8FrameProjection createVP8FrameProjection(
// Lastly, check whether the quality of the frame is something that we
// want to forward. We don't want to be allocating new objects unless
// we're interested in the quality of this frame.
if (!vp8QualityFilter.acceptFrame(rtpPacket, targetIndex, nowMs))
if (!vp8QualityFilter.acceptFrame(
rtpPacket, incomingIndex, targetIndex, nowMs))
{
return null;
}
Expand Down Expand Up @@ -349,18 +351,21 @@ public boolean needsKeyframe()
* Determines whether a packet should be accepted or not.
*
* @param rtpPacket the RTP packet to determine whether to project or not.
* @param targetIndex the target index to achieve
* @param incomingIndex the quality index of the incoming RTP packet
* @param targetIndex the target quality index we want to achieve
* @return true if the packet should be accepted, false otherwise.
*/
@Override
public boolean accept(@NotNull RawPacket rtpPacket, int targetIndex)
public boolean accept(
@NotNull RawPacket rtpPacket, int incomingIndex, int targetIndex)
{
VP8FrameProjection vp8FrameProjection
= lookupVP8FrameProjection(rtpPacket);

if (vp8FrameProjection == null)
{
vp8FrameProjection = createVP8FrameProjection(rtpPacket, targetIndex);
vp8FrameProjection
= createVP8FrameProjection(rtpPacket, incomingIndex, targetIndex);
}

return vp8FrameProjection != null
Expand Down
112 changes: 8 additions & 104 deletions src/main/java/org/jitsi/videobridge/cc/vp8/VP8QualityFilter.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@
import org.jitsi.service.neomedia.*;
import org.jitsi.util.*;

import java.util.*;

/**
* This class is responsible for dropping VP8 simulcast/svc packets based on
* their quality, i.e. packets that correspond to qualities that are above a
Expand All @@ -44,22 +42,10 @@ class VP8QualityFilter
*/
private static final int MIN_KEY_FRAME_WAIT_MS = 300;

/**
* Constants that define the minimum heights that a VP8 stream needs to have
* in order to be considered HD, SD or LD.
*/
private static final int MIN_HD_HEIGHT = 540, MIN_SD_HEIGHT = 360;

/**
* The HD, SD, LD and suspended spatial/quality layer IDs.
*/
private static final int HD_LAYER_ID = 2, SD_LAYER_ID = 1,
LD_LAYER_ID = 0, SUSPENDED_LAYER_ID = -1;

/**
* A map that caches the spatial/quality layer ID of each simulcast SSRC.
*/
private final Map<Long, Integer> ssrcToSpatialLayerQuality = new HashMap<>();
private static final int SUSPENDED_LAYER_ID = -1;

/**
* Holds the arrival time (in millis) of the most recent keyframe group.
Expand Down Expand Up @@ -94,85 +80,6 @@ class VP8QualityFilter
*/
private int currentSpatialLayerId = SUSPENDED_LAYER_ID;

/**
* Gets the spatial/quality layer ID of a packet. Before this method is able
* to return a meaningful value (not -1) it has to have received at least
* one keyframe from the SSRC that is queried.
*
* @param firstPacketOfFrame the RTP packet whose spatial/quality layer
* index needs to be returned.
* @param isKeyframe a boolean that indicates whether or not the packet that
* is specified as an argument is a keyframe
* @return the spatial/quality index of the packet that is specified as an
* argument, or -1 in case it's impossible to resolve.
*/
private int getSpatialLayerIndex(
@NotNull RawPacket firstPacketOfFrame, boolean isKeyframe)
{
long ssrc = firstPacketOfFrame.getSSRCAsLong();
if (ssrcToSpatialLayerQuality.containsKey(ssrc))
{
return ssrcToSpatialLayerQuality.get(ssrc);
}
else if (isKeyframe)
{
int spatialLayerId
= getSpatialLayerIndexFromKeyframe(firstPacketOfFrame);
if (spatialLayerId > -1)
{
ssrcToSpatialLayerQuality.put(ssrc, spatialLayerId);
}

return spatialLayerId;
}
{
return -1;
}
}

/**
* Gets the spatial/quality layer ID of a packet. The specified packet MUST
* be a VP8 keyframe because the frame height, which allows us to map the
* different simulcast layers into spatial/quality layer IDs, are found in
* the keyframe header.
*
* @param firstPacketOfKeyframe the first packet of a keyframe to map/get
* its spatial/quality layer id.
* @return the spatial/quality layer id of the packet.
*/
private static int getSpatialLayerIndexFromKeyframe(
@NotNull RawPacket firstPacketOfKeyframe)
{
byte[] buf = firstPacketOfKeyframe.getBuffer();
int payloadOffset = firstPacketOfKeyframe.getPayloadOffset(),
payloadLen = firstPacketOfKeyframe.getPayloadLength(),
payloadHeaderLen = 3;

int payloadDescriptorLen = DePacketizer
.VP8PayloadDescriptor.getSize(buf, payloadOffset, payloadLen);

int height = DePacketizer.VP8KeyframeHeader.getHeight(
buf, payloadOffset + payloadDescriptorLen + payloadHeaderLen);

if (height >= MIN_HD_HEIGHT)
{
return HD_LAYER_ID;
}
else if (height >= MIN_SD_HEIGHT)
{
return SD_LAYER_ID;
}
else if (height > -1)
{
return LD_LAYER_ID;
}
else
{
// Some error occurred.
return -1;
}
}

/**
* @return true if a the target spatial/quality layer id has changed and a
* keyframe hasn't been received yet, false otherwise.
Expand All @@ -192,13 +99,15 @@ boolean needsKeyframe()
* method at a time.
*
* @param firstPacketOfFrame the first packet of the VP8 frame.
* @param incomingIndex the quality index of the incoming RTP packet
* @param externalTargetIndex the target quality index that the user of this
* instance wants to achieve.
* @param nowMs the current time (in millis)
* @return true to accept the VP8 frame, otherwise false.
*/
synchronized boolean acceptFrame(
@NotNull RawPacket firstPacketOfFrame,
int incomingIndex,
int externalTargetIndex, long nowMs)
{
// We make local copies of the externalTemporalLayerIdTarget and the
Expand Down Expand Up @@ -245,14 +154,15 @@ synchronized boolean acceptFrame(
temporalLayerIdOfFrame = 0;
}

int spatialLayerId = getSpatialLayerId(incomingIndex);
if (DePacketizer.isKeyFrame(buf, payloadOff, payloadLen))
{
return acceptKeyframe(firstPacketOfFrame, nowMs);
return acceptKeyframe(spatialLayerId, nowMs);
}
else if (currentSpatialLayerId > SUSPENDED_LAYER_ID)
{
if (!isInSwitchingPhase(nowMs)
&& isPossibleToSwitch(firstPacketOfFrame))
&& isPossibleToSwitch(firstPacketOfFrame, spatialLayerId))
{
needsKeyframe = true;
}
Expand Down Expand Up @@ -311,10 +221,8 @@ private synchronized boolean isInSwitchingPhase(long nowMs)
* method for specific details).
*/
private synchronized boolean isPossibleToSwitch(
@NotNull RawPacket firstPacketOfFrame)
@NotNull RawPacket firstPacketOfFrame, int spatialLayerId)
{
int spatialLayerId
= getSpatialLayerIndex(firstPacketOfFrame, false);
if (spatialLayerId == -1)
{
// We failed to resolve the spatial/quality layer of the packet.
Expand Down Expand Up @@ -347,19 +255,15 @@ else if (spatialLayerId < currentSpatialLayerId
* synchronized keyword because there's only one thread accessing this
* method at a time.
*
* @param firstPacketOfKeyframe the first packet of a VP8 keyframe.
* @param nowMs the current time (in millis)
* @return true to accept the VP8 keyframe, otherwise false.
*/
private synchronized boolean acceptKeyframe(
@NotNull RawPacket firstPacketOfKeyframe, long nowMs)
int spatialLayerIdOfKeyframe, long nowMs)
{
// This branch writes the {@link #currentSpatialLayerId} and it
// determines whether or not we should switch to another simulcast
// stream.

int spatialLayerIdOfKeyframe
= getSpatialLayerIndex(firstPacketOfKeyframe, true);
if (spatialLayerIdOfKeyframe < 0)
{
// something went terribly wrong, normally we should be able to
Expand Down

0 comments on commit 23f6323

Please sign in to comment.