Skip to content

Commit

Permalink
feat: Silence detection.
Browse files Browse the repository at this point in the history
  • Loading branch information
bgrozev committed Aug 9, 2022
1 parent 42bc1b9 commit e270225
Show file tree
Hide file tree
Showing 7 changed files with 103 additions and 32 deletions.
20 changes: 13 additions & 7 deletions jvb/src/main/java/org/jitsi/videobridge/Conference.java
Original file line number Diff line number Diff line change
Expand Up @@ -500,17 +500,20 @@ private void lastNEndpointsChanged()
* dominant speaker.
* @param recentSpeakers the list of recent speakers (including the dominant speaker at index 0).
*/
private void recentSpeakersChanged(List<AbstractEndpoint> recentSpeakers, boolean dominantSpeakerChanged)
private void recentSpeakersChanged(
List<AbstractEndpoint> recentSpeakers,
boolean dominantSpeakerChanged,
boolean silence)
{
if (!recentSpeakers.isEmpty())
{
List<String> recentSpeakersIds
= recentSpeakers.stream().map(AbstractEndpoint::getId).collect(Collectors.toList());
logger.info("Recent speakers changed: " + recentSpeakersIds + ", dominant speaker changed: "
+ dominantSpeakerChanged);
broadcastMessage(new DominantSpeakerMessage(recentSpeakersIds));
+ dominantSpeakerChanged + " silence:" + silence);
broadcastMessage(new DominantSpeakerMessage(recentSpeakersIds, silence));

if (dominantSpeakerChanged)
if (dominantSpeakerChanged && !silence)
{
getVideobridge().getStatistics().totalDominantSpeakerChanges.increment();
if (getEndpointCount() > 2)
Expand Down Expand Up @@ -1077,7 +1080,7 @@ public void endpointMessageTransportConnected(@NotNull AbstractEndpoint abstract

if (!recentSpeakers.isEmpty())
{
endpoint.sendMessage(new DominantSpeakerMessage(recentSpeakers));
endpoint.sendMessage(new DominantSpeakerMessage(recentSpeakers, speechActivity.isInSilence()));
}
}
}
Expand Down Expand Up @@ -1522,9 +1525,12 @@ public Object put(String key, Object value)
private class SpeechActivityListener implements ConferenceSpeechActivity.Listener
{
@Override
public void recentSpeakersChanged(List<AbstractEndpoint> recentSpeakers, boolean dominantSpeakerChanged)
public void recentSpeakersChanged(
List<AbstractEndpoint> recentSpeakers,
boolean dominantSpeakerChanged,
boolean silence)
{
Conference.this.recentSpeakersChanged(recentSpeakers, dominantSpeakerChanged);
Conference.this.recentSpeakersChanged(recentSpeakers, dominantSpeakerChanged, silence);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,7 @@ public class ConferenceSpeechActivity
* The <tt>DominantSpeakerIdentification</tt> instance which detects/identifies the active/dominant speaker in a
* conference.
*/
private DominantSpeakerIdentification<String> dominantSpeakerIdentification
= new DominantSpeakerIdentification<>();
private DominantSpeakerIdentification<String> dominantSpeakerIdentification;

/**
* The listener to be notified when the dominant speaker or endpoint order changes.
Expand Down Expand Up @@ -88,7 +87,13 @@ public class ConferenceSpeechActivity
*/
@NotNull
private final RecentSpeakersList<AbstractEndpoint> recentSpeakers
= new RecentSpeakersList<>(ConferenceSpeechActivityConfig.getConfig().getRecentSpeakersCount() + 1);
= new RecentSpeakersList<>(ConferenceSpeechActivityConfig.config.getRecentSpeakersCount() + 1);

/**
* Whether we're currently in a period of silence. With silence detection enabled we initialize to `true` because
* (the {@link #dominantSpeakerIdentification} will fire an initial "silence" event and we don't want to act on it.
*/
private boolean inSilence = ConferenceSpeechActivityConfig.config.getEnableSilenceDetection();

/**
* The <tt>Object</tt> used to synchronize the access to the state of this
Expand All @@ -114,6 +119,14 @@ public ConferenceSpeechActivity(@NotNull Listener listener, Logger parentLogger)
new LoggerImpl(ConferenceSpeechActivity.class.getName()) :
parentLogger.createChildLogger(ConferenceSpeechActivity.class.getName());

long silenceTimeoutMs = -1;
if (ConferenceSpeechActivityConfig.config.getEnableSilenceDetection())
{
silenceTimeoutMs = ConferenceSpeechActivityConfig.config.getSilenceDetectionTimeout().toMillis();

}
dominantSpeakerIdentification = new DominantSpeakerIdentification<>(silenceTimeoutMs);

dominantSpeakerIdentification.addActiveSpeakerChangedListener(activeSpeakerChangedListener);
int numLoudestToTrack = LoudestConfig.Companion.getRouteLoudestOnly() ?
LoudestConfig.Companion.getNumLoudest() : 0;
Expand All @@ -122,45 +135,68 @@ public ConferenceSpeechActivity(@NotNull Listener listener, Logger parentLogger)
LoudestConfig.Companion.getEnergyAlphaPct());
}

boolean isInSilence()
{
return inSilence;
}

/**
* Notifies this instance that the underlying {@code dominant speaker identification} has elected a new
* active/dominant speaker.
*
* @param id the ID of the new active/dominant speaker.
* @param id the ID of the new active/dominant speaker or null if a period of silence began.
*/
protected void activeSpeakerChanged(@NotNull String id)
protected void activeSpeakerChanged(@Nullable String id)
{
final Listener listener = this.listener;
if (listener == null)
{
return;
}

Objects.requireNonNull(id);
logger.trace(() -> "The dominant speaker is now " + id + ".");

boolean endpointListChanged;
boolean dominantSpeakerChanged;
synchronized (syncRoot)
{
AbstractEndpoint endpoint
= endpointsBySpeechActivity.stream()
.filter(e -> id.equals(e.getId()))
.findFirst().orElse(null);
// Move this endpoint to the top of our sorted list
if (!endpointsBySpeechActivity.remove(endpoint))
if (id == null)
{
logger.warn("Got active speaker notification for an unknown endpoint: " + id + ", ignoring");
return;
endpointListChanged = false;
dominantSpeakerChanged = false;
if (!inSilence)
{
inSilence = true;
}
}
endpointsBySpeechActivity.add(0, endpoint);
else
{
dominantSpeakerChanged = true;
if (inSilence)
{
inSilence = false;
}

recentSpeakers.promote(endpoint);
AbstractEndpoint endpoint
= endpointsBySpeechActivity.stream()
.filter(e -> id.equals(e.getId()))
.findFirst().orElse(null);
// Move this endpoint to the top of our sorted list
if (!endpointsBySpeechActivity.remove(endpoint))
{
logger.warn("Got active speaker notification for an unknown endpoint: " + id + ", ignoring");
return;
}
endpointsBySpeechActivity.add(0, endpoint);

recentSpeakers.promote(endpoint);

endpointListChanged = updateLastNEndpoints();
endpointListChanged = updateLastNEndpoints();
}
}

TaskPools.IO_POOL.execute(() -> {
listener.recentSpeakersChanged(recentSpeakers.getRecentSpeakers(), true);
listener.recentSpeakersChanged(recentSpeakers.getRecentSpeakers(), dominantSpeakerChanged, inSilence);
if (endpointListChanged)
{
listener.lastNEndpointsChanged();
Expand Down Expand Up @@ -321,7 +357,8 @@ public void endpointsChanged(List<AbstractEndpoint> conferenceEndpoints)
TaskPools.IO_POOL.execute(() -> {
if (finalRecentSpeakersChanged)
{
listener.recentSpeakersChanged(recentSpeakers.getRecentSpeakers(), dominantSpeakerChanged);
listener.recentSpeakersChanged(
recentSpeakers.getRecentSpeakers(), dominantSpeakerChanged, inSilence);
}
if (finalEndpointsChanged)
{
Expand Down Expand Up @@ -386,8 +423,12 @@ interface Listener
* endpoint was removed).
* @param recentSpeakers the new list of recent speakers (including the dominant speaker at index 0).
* @param dominantSpeakerChanged whether the dominant speaker changed.
* @param silence whether we're in a period of silence
*/
void recentSpeakersChanged(List<AbstractEndpoint> recentSpeakers, boolean dominantSpeakerChanged);
void recentSpeakersChanged(
List<AbstractEndpoint> recentSpeakers,
boolean dominantSpeakerChanged,
boolean silence);
void lastNEndpointsChanged();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,23 @@ package org.jitsi.videobridge

import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
import java.time.Duration

class ConferenceSpeechActivityConfig {
val recentSpeakersCount: Int by config {
"videobridge.speech-activity.recent-speakers-count".from(JitsiConfig.newConfig)
}

val enableSilenceDetection: Boolean by config {
"videobridge.speech-activity.enable-silence-detection".from(JitsiConfig.newConfig)
}

val silenceDetectionTimeout: Duration by config {
"videobridge.speech-activity.silence-detection-timeout".from(JitsiConfig.newConfig)
}

companion object {
@JvmStatic
@JvmField
val config = ConferenceSpeechActivityConfig()
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -259,13 +259,16 @@ class LastNMessage(val lastN: Int) : BridgeChannelMessage(TYPE) {
@JsonInclude(JsonInclude.Include.NON_NULL)
class DominantSpeakerMessage @JvmOverloads constructor(
val dominantSpeakerEndpoint: String,
val previousSpeakers: List<String>? = null
val previousSpeakers: List<String>? = null,
val silence: Boolean = false
) : BridgeChannelMessage(TYPE) {
/**
* Construct a message from a list of speakers with the dominant speaker on top. The list must have at least one
* element.
*/
constructor(previousSpeakers: List<String>) : this(previousSpeakers[0], previousSpeakers.drop(1))
constructor(previousSpeakers: List<String>, silence: Boolean) : this(
previousSpeakers[0], previousSpeakers.drop(1), silence
)
companion object {
const val TYPE = "DominantSpeakerEndpointChangeEvent"
}
Expand Down
8 changes: 8 additions & 0 deletions jvb/src/main/resources/reference.conf
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,14 @@ videobridge {
# The number of speakers to include in the list of recent speakers sent with dominant speaker change
# notifications.
recent-speakers-count = 10

# Whether to enable silence detection. When silence detection is enabled and there is no speech activity for a
# certain time (see silence-detection-timeout below) we fire a "dominant speaker changed" event notifying endpoints
# that we entered a period of silence.
enable-silence-detection = false

# How long to wait for lack of speech activity before a period of silence begins.
silence-detection-timeout = 3 seconds
}

loudest {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,11 @@ class SpeechActivityTest : ShouldSpec() {
private val d = mockEndpoint("d")
private val conferenceSpeechActivity = ConferenceSpeechActivity(object : ConferenceSpeechActivity.Listener {
override fun lastNEndpointsChanged() {}
override fun recentSpeakersChanged(recentSpeakers: List<AbstractEndpoint>, dominantSpeakerChanged: Boolean) {}
override fun recentSpeakersChanged(
recentSpeakers: List<AbstractEndpoint>,
dominantSpeakerChanged: Boolean,
silence: Boolean
) {}
})

init {
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
<kotest.version>5.3.0</kotest.version>
<junit.version>5.8.2</junit.version>
<jicoco.version>1.1-115-g332f4e7</jicoco.version>
<jitsi.utils.version>1.0-119-ga7b23ff</jitsi.utils.version>
<jitsi.utils.version>1.0-123-gb819a87</jitsi.utils.version>
<ktlint-maven-plugin.version>1.13.1</ktlint-maven-plugin.version>
<maven-shade-plugin.version>3.2.2</maven-shade-plugin.version>
<spotbugs.version>4.6.0</spotbugs.version>
Expand Down

0 comments on commit e270225

Please sign in to comment.