feat: Silence detection

jvb/src/main/java/org/jitsi/videobridge/Conference.java

@@ -500,17 +500,20 @@ private void lastNEndpointsChanged()
      * dominant speaker.
      * @param recentSpeakers the list of recent speakers (including the dominant speaker at index 0).
      */
-    private void recentSpeakersChanged(List<AbstractEndpoint> recentSpeakers, boolean dominantSpeakerChanged)
+    private void recentSpeakersChanged(
+            List<AbstractEndpoint> recentSpeakers,
+            boolean dominantSpeakerChanged,
+            boolean silence)
     {
         if (!recentSpeakers.isEmpty())
         {
             List<String> recentSpeakersIds
                     = recentSpeakers.stream().map(AbstractEndpoint::getId).collect(Collectors.toList());
             logger.info("Recent speakers changed: " + recentSpeakersIds + ", dominant speaker changed: "
-                    + dominantSpeakerChanged);
-            broadcastMessage(new DominantSpeakerMessage(recentSpeakersIds));
+                    + dominantSpeakerChanged + " silence:" + silence);
+            broadcastMessage(new DominantSpeakerMessage(recentSpeakersIds, silence));
 
-            if (dominantSpeakerChanged)
+            if (dominantSpeakerChanged && !silence)
             {
                 getVideobridge().getStatistics().totalDominantSpeakerChanges.increment();
                 if (getEndpointCount() > 2)
@@ -1077,7 +1080,7 @@ public void endpointMessageTransportConnected(@NotNull AbstractEndpoint abstract
 
             if (!recentSpeakers.isEmpty())
             {
-                endpoint.sendMessage(new DominantSpeakerMessage(recentSpeakers));
+                endpoint.sendMessage(new DominantSpeakerMessage(recentSpeakers, speechActivity.isInSilence()));
             }
         }
     }
@@ -1522,9 +1525,12 @@ public Object put(String key, Object value)
     private class SpeechActivityListener implements ConferenceSpeechActivity.Listener
     {
         @Override
-        public void recentSpeakersChanged(List<AbstractEndpoint> recentSpeakers, boolean dominantSpeakerChanged)
+        public void recentSpeakersChanged(
+                List<AbstractEndpoint> recentSpeakers,
+                boolean dominantSpeakerChanged,
+                boolean silence)
         {
-            Conference.this.recentSpeakersChanged(recentSpeakers, dominantSpeakerChanged);
+            Conference.this.recentSpeakersChanged(recentSpeakers, dominantSpeakerChanged, silence);
         }
 
         @Override

jvb/src/main/java/org/jitsi/videobridge/ConferenceSpeechActivity.java

@@ -54,8 +54,7 @@ public class ConferenceSpeechActivity
      * The <tt>DominantSpeakerIdentification</tt> instance which detects/identifies the active/dominant speaker in a
      * conference.
      */
-    private DominantSpeakerIdentification<String> dominantSpeakerIdentification
-            = new DominantSpeakerIdentification<>();
+    private DominantSpeakerIdentification<String> dominantSpeakerIdentification;
 
     /**
      * The listener to be notified when the dominant speaker or endpoint order changes.
@@ -88,7 +87,13 @@ public class ConferenceSpeechActivity
      */
     @NotNull
     private final RecentSpeakersList<AbstractEndpoint> recentSpeakers
-            = new RecentSpeakersList<>(ConferenceSpeechActivityConfig.getConfig().getRecentSpeakersCount() + 1);
+            = new RecentSpeakersList<>(ConferenceSpeechActivityConfig.config.getRecentSpeakersCount() + 1);
+
+    /**
+     * Whether we're currently in a period of silence. With silence detection enabled we initialize to `true` because
+     * (the {@link #dominantSpeakerIdentification} will fire an initial "silence" event and we don't want to act on it.
+     */
+    private boolean inSilence = ConferenceSpeechActivityConfig.config.getEnableSilenceDetection();
 
     /**
      * The <tt>Object</tt> used to synchronize the access to the state of this
@@ -114,6 +119,14 @@ public ConferenceSpeechActivity(@NotNull Listener listener, Logger parentLogger)
                         new LoggerImpl(ConferenceSpeechActivity.class.getName()) :
                         parentLogger.createChildLogger(ConferenceSpeechActivity.class.getName());
 
+        long silenceTimeoutMs = -1;
+        if (ConferenceSpeechActivityConfig.config.getEnableSilenceDetection())
+        {
+            silenceTimeoutMs = ConferenceSpeechActivityConfig.config.getSilenceDetectionTimeout().toMillis();
+
+        }
+        dominantSpeakerIdentification = new DominantSpeakerIdentification<>(silenceTimeoutMs);
+
         dominantSpeakerIdentification.addActiveSpeakerChangedListener(activeSpeakerChangedListener);
         int numLoudestToTrack = LoudestConfig.Companion.getRouteLoudestOnly() ?
                 LoudestConfig.Companion.getNumLoudest() : 0;
@@ -122,45 +135,68 @@ public ConferenceSpeechActivity(@NotNull Listener listener, Logger parentLogger)
                 LoudestConfig.Companion.getEnergyAlphaPct());
     }
 
+    boolean isInSilence()
+    {
+        return inSilence;
+    }
+
     /**
      * Notifies this instance that the underlying {@code dominant speaker identification} has elected a new
      * active/dominant speaker.
      *
-     * @param id the ID of the new active/dominant speaker.
+     * @param id the ID of the new active/dominant speaker or null if a period of silence began.
      */
-    protected void activeSpeakerChanged(@NotNull String id)
+    protected void activeSpeakerChanged(@Nullable String id)
     {
         final Listener listener = this.listener;
         if (listener == null)
         {
             return;
         }
 
-        Objects.requireNonNull(id);
         logger.trace(() -> "The dominant speaker is now " + id + ".");
 
         boolean endpointListChanged;
+        boolean dominantSpeakerChanged;
         synchronized (syncRoot)
         {
-            AbstractEndpoint endpoint
-                    = endpointsBySpeechActivity.stream()
-                        .filter(e -> id.equals(e.getId()))
-                        .findFirst().orElse(null);
-            // Move this endpoint to the top of our sorted list
-            if (!endpointsBySpeechActivity.remove(endpoint))
+            if (id == null)
             {
-                logger.warn("Got active speaker notification for an unknown endpoint: " + id + ", ignoring");
-                return;
+                endpointListChanged = false;
+                dominantSpeakerChanged = false;
+                if (!inSilence)
+                {
+                    inSilence = true;
+                }
             }
-            endpointsBySpeechActivity.add(0, endpoint);
+            else
+            {
+                dominantSpeakerChanged = true;
+                if (inSilence)
+                {
+                    inSilence = false;
+                }
 
-            recentSpeakers.promote(endpoint);
+                AbstractEndpoint endpoint
+                        = endpointsBySpeechActivity.stream()
+                        .filter(e -> id.equals(e.getId()))
+                        .findFirst().orElse(null);
+                // Move this endpoint to the top of our sorted list
+                if (!endpointsBySpeechActivity.remove(endpoint))
+                {
+                    logger.warn("Got active speaker notification for an unknown endpoint: " + id + ", ignoring");
+                    return;
+                }
+                endpointsBySpeechActivity.add(0, endpoint);
+
+                recentSpeakers.promote(endpoint);
 
-            endpointListChanged = updateLastNEndpoints();
+                endpointListChanged = updateLastNEndpoints();
+            }
         }
 
         TaskPools.IO_POOL.execute(() -> {
-            listener.recentSpeakersChanged(recentSpeakers.getRecentSpeakers(), true);
+            listener.recentSpeakersChanged(recentSpeakers.getRecentSpeakers(), dominantSpeakerChanged, inSilence);
             if (endpointListChanged)
             {
                 listener.lastNEndpointsChanged();
@@ -321,7 +357,8 @@ public void endpointsChanged(List<AbstractEndpoint> conferenceEndpoints)
             TaskPools.IO_POOL.execute(() -> {
                 if (finalRecentSpeakersChanged)
                 {
-                    listener.recentSpeakersChanged(recentSpeakers.getRecentSpeakers(), dominantSpeakerChanged);
+                    listener.recentSpeakersChanged(
+                            recentSpeakers.getRecentSpeakers(), dominantSpeakerChanged, inSilence);
                 }
                 if (finalEndpointsChanged)
                 {
@@ -386,8 +423,12 @@ interface Listener
          * endpoint was removed).
          * @param recentSpeakers the new list of recent speakers (including the dominant speaker at index 0).
          * @param dominantSpeakerChanged whether the dominant speaker changed.
+         * @param silence whether we're in a period of silence
          */
-        void recentSpeakersChanged(List<AbstractEndpoint> recentSpeakers, boolean dominantSpeakerChanged);
+        void recentSpeakersChanged(
+                List<AbstractEndpoint> recentSpeakers,
+                boolean dominantSpeakerChanged,
+                boolean silence);
         void lastNEndpointsChanged();
     }
 }

jvb/src/main/kotlin/org/jitsi/videobridge/ConferenceSpeechActivityConfig.kt

@@ -17,14 +17,23 @@ package org.jitsi.videobridge
 
 import org.jitsi.config.JitsiConfig
 import org.jitsi.metaconfig.config
+import java.time.Duration
 
 class ConferenceSpeechActivityConfig {
     val recentSpeakersCount: Int by config {
         "videobridge.speech-activity.recent-speakers-count".from(JitsiConfig.newConfig)
     }
 
+    val enableSilenceDetection: Boolean by config {
+        "videobridge.speech-activity.enable-silence-detection".from(JitsiConfig.newConfig)
+    }
+
+    val silenceDetectionTimeout: Duration by config {
+        "videobridge.speech-activity.silence-detection-timeout".from(JitsiConfig.newConfig)
+    }
+
     companion object {
-        @JvmStatic
+        @JvmField
         val config = ConferenceSpeechActivityConfig()
     }
 }

jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt

@@ -259,13 +259,16 @@ class LastNMessage(val lastN: Int) : BridgeChannelMessage(TYPE) {
 @JsonInclude(JsonInclude.Include.NON_NULL)
 class DominantSpeakerMessage @JvmOverloads constructor(
     val dominantSpeakerEndpoint: String,
-    val previousSpeakers: List<String>? = null
+    val previousSpeakers: List<String>? = null,
+    val silence: Boolean = false
 ) : BridgeChannelMessage(TYPE) {
     /**
      * Construct a message from a list of speakers with the dominant speaker on top. The list must have at least one
      * element.
      */
-    constructor(previousSpeakers: List<String>) : this(previousSpeakers[0], previousSpeakers.drop(1))
+    constructor(previousSpeakers: List<String>, silence: Boolean) : this(
+        previousSpeakers[0], previousSpeakers.drop(1), silence
+    )
     companion object {
         const val TYPE = "DominantSpeakerEndpointChangeEvent"
     }

jvb/src/main/resources/reference.conf

@@ -301,6 +301,14 @@ videobridge {
     # The number of speakers to include in the list of recent speakers sent with dominant speaker change
     # notifications.
     recent-speakers-count = 10
+
+    # Whether to enable silence detection. When silence detection is enabled and there is no speech activity for a
+    # certain time (see silence-detection-timeout below) we fire a "dominant speaker changed" event notifying endpoints
+    # that we entered a period of silence.
+    enable-silence-detection = false
+
+    # How long to wait for lack of speech activity before a period of silence begins.
+    silence-detection-timeout = 3 seconds
   }
 
   loudest {

jvb/src/test/kotlin/org/jitsi/videobridge/SpeechActivityTest.kt

@@ -34,7 +34,11 @@ class SpeechActivityTest : ShouldSpec() {
     private val d = mockEndpoint("d")
     private val conferenceSpeechActivity = ConferenceSpeechActivity(object : ConferenceSpeechActivity.Listener {
         override fun lastNEndpointsChanged() {}
-        override fun recentSpeakersChanged(recentSpeakers: List<AbstractEndpoint>, dominantSpeakerChanged: Boolean) {}
+        override fun recentSpeakersChanged(
+            recentSpeakers: List<AbstractEndpoint>,
+            dominantSpeakerChanged: Boolean,
+            silence: Boolean
+        ) {}
     })
 
     init {

pom.xml

@@ -28,7 +28,7 @@
         <kotest.version>5.3.0</kotest.version>
         <junit.version>5.8.2</junit.version>
         <jicoco.version>1.1-115-g332f4e7</jicoco.version>
-        <jitsi.utils.version>1.0-119-ga7b23ff</jitsi.utils.version>
+        <jitsi.utils.version>1.0-123-gb819a87</jitsi.utils.version>
         <ktlint-maven-plugin.version>1.13.1</ktlint-maven-plugin.version>
         <maven-shade-plugin.version>3.2.2</maven-shade-plugin.version>
         <spotbugs.version>4.6.0</spotbugs.version>