From dbcf46c232e7b68eaae6217f6958623d41090fa5 Mon Sep 17 00:00:00 2001 From: Maksim Sukharev Date: Fri, 28 Jul 2023 10:03:28 +0200 Subject: [PATCH 1/4] notify users who joined call if currently speaking Signed-off-by: Maksim Sukharev --- src/utils/webrtc/simplewebrtc/localmedia.js | 7 +++++++ src/utils/webrtc/webrtc.js | 10 ++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/utils/webrtc/simplewebrtc/localmedia.js b/src/utils/webrtc/simplewebrtc/localmedia.js index 916432755d3..2a287a0e77f 100644 --- a/src/utils/webrtc/simplewebrtc/localmedia.js +++ b/src/utils/webrtc/simplewebrtc/localmedia.js @@ -63,15 +63,18 @@ function LocalMedia(opts) { this._blackVideoEnforcer = new BlackVideoEnforcer() + this._speaking = undefined this._speakingMonitor = new SpeakingMonitor() this._speakingMonitor.on('speaking', () => { this.emit('speaking') + this._speaking = true }) this._speakingMonitor.on('speakingWhileMuted', () => { this.emit('speakingWhileMuted') }) this._speakingMonitor.on('stoppedSpeaking', () => { this.emit('stoppedSpeaking') + this._speaking = false }) this._speakingMonitor.on('stoppedSpeakingWhileMuted', () => { this.emit('stoppedSpeakingWhileMuted') @@ -416,6 +419,10 @@ LocalMedia.prototype._setVideoEnabled = function(bool) { this._videoTrackEnabler.setEnabled(bool) } +LocalMedia.prototype.isSpeaking = function() { + return this._speaking +} + // check if all audio streams are enabled LocalMedia.prototype.isAudioEnabled = function() { let enabled = true diff --git a/src/utils/webrtc/webrtc.js b/src/utils/webrtc/webrtc.js index 526d7b13979..71f062bbfe3 100644 --- a/src/utils/webrtc/webrtc.js +++ b/src/utils/webrtc/webrtc.js @@ -211,6 +211,12 @@ function sendCurrentMediaState() { webrtc.webrtc.emit('audioOff') } else { webrtc.webrtc.emit('audioOn') + + if (!webrtc.webrtc.isSpeaking()) { + webrtc.webrtc.emit('stoppedSpeaking') + } else { + webrtc.webrtc.emit('speaking') + } } } @@ -1601,7 +1607,7 @@ export default function initWebRtc(signaling, _callParticipantCollection, _local const name = typeof (data.payload) === 'string' ? data.payload : data.payload.name webrtc.emit('nick', { id: peer.id, name }) } else if (data.type === 'speaking' || data.type === 'stoppedSpeaking') { - // Valid known messages, but handled elsewhere + // Valid known messages, handled by CallParticipantModel.js } else { console.debug('Unknown message type %s from %s datachannel', data.type, label, data, peer.id, peer) } @@ -1633,10 +1639,10 @@ export default function initWebRtc(signaling, _callParticipantCollection, _local } }) + // Send the speaking status events via data channel webrtc.on('speaking', function() { sendDataChannelToAll('status', 'speaking') }) - webrtc.on('stoppedSpeaking', function() { sendDataChannelToAll('status', 'stoppedSpeaking') }) From 9d0bff1d0a76296cf24b0ab7aea5d8be30ead6f6 Mon Sep 17 00:00:00 2001 From: Maksim Sukharev Date: Fri, 28 Jul 2023 19:27:43 +0200 Subject: [PATCH 2/4] add util for call time formatting Signed-off-by: Maksim Sukharev --- src/components/TopBar/CallTime.vue | 33 ++++++++---------------------- src/utils/formattedTime.js | 26 +++++++++++++++++++++++ 2 files changed, 34 insertions(+), 25 deletions(-) create mode 100644 src/utils/formattedTime.js diff --git a/src/components/TopBar/CallTime.vue b/src/components/TopBar/CallTime.vue index d43fa740eed..bc07fae29cd 100644 --- a/src/components/TopBar/CallTime.vue +++ b/src/components/TopBar/CallTime.vue @@ -39,7 +39,7 @@ :size="20" fill-color="var(--color-loading-light)" /> - {{ formattedTime }} + {{ formattedTime(callTime) }} @@ -83,6 +83,7 @@ import NcPopover from '@nextcloud/vue/dist/Components/NcPopover.js' import { CALL } from '../../constants.js' import isInLobby from '../../mixins/isInLobby.js' +import { formattedTime } from '../../utils/formattedTime.js' export default { name: 'CallTime', @@ -131,30 +132,6 @@ export default { return new Date(this.start * 1000) }, - /** - * Calculates the stopwatch string given the callTime (ms) - * - * @return {string} The formatted time - */ - formattedTime() { - if (!this.callTime) { - return '-- : --' - } - let seconds = Math.floor((this.callTime / 1000) % 60) - if (seconds < 10) { - seconds = '0' + seconds - } - let minutes = Math.floor((this.callTime / (1000 * 60)) % 60) - if (minutes < 10) { - minutes = '0' + minutes - } - const hours = Math.floor((this.callTime / (1000 * 60 * 60)) % 24) - if (hours === 0) { - return minutes + ' : ' + seconds - } - return hours + ' : ' + minutes + ' : ' + seconds - }, - token() { return this.$store.getters.getToken() }, @@ -213,6 +190,12 @@ export default { }, methods: { + /** + * Calculates the stopwatch string given the callTime (ms) + * + */ + formattedTime, + stopRecording() { this.$store.dispatch('stopCallRecording', { token: this.token, diff --git a/src/utils/formattedTime.js b/src/utils/formattedTime.js new file mode 100644 index 00000000000..9a528b10ddd --- /dev/null +++ b/src/utils/formattedTime.js @@ -0,0 +1,26 @@ +/** + * Calculates the stopwatch string given the time (ms) + * + * @param {number} time the time in ms + * @param {boolean} [condensed=false] the format of string to show + * @return {string} The formatted time + */ +function formattedTime(time, condensed = false) { + if (!time) { + return condensed ? '--:--' : '-- : --' + } + + const seconds = Math.floor((time / 1000) % 60) + const minutes = Math.floor((time / (1000 * 60)) % 60) + const hours = Math.floor((time / (1000 * 60 * 60)) % 24) + + return [ + hours, + minutes.toString().padStart(2, '0'), + seconds.toString().padStart(2, '0'), + ].filter(num => !!num).join(condensed ? ':' : ' : ') +} + +export { + formattedTime, +} From 650d0dd4d4f423b7a7c4e03e0d3a7cd972ecb81e Mon Sep 17 00:00:00 2001 From: Maksim Sukharev Date: Tue, 8 Aug 2023 09:36:04 +0200 Subject: [PATCH 3/4] add handler for participants speaking status changes Signed-off-by: Maksim Sukharev --- src/store/participantsStore.js | 62 +++++++++++ src/utils/webrtc/SpeakingStatusHandler.js | 129 ++++++++++++++++++++++ src/utils/webrtc/index.js | 6 + 3 files changed, 197 insertions(+) create mode 100644 src/utils/webrtc/SpeakingStatusHandler.js diff --git a/src/store/participantsStore.js b/src/store/participantsStore.js index b63ad6d2842..e8b96ffa677 100644 --- a/src/store/participantsStore.js +++ b/src/store/participantsStore.js @@ -57,6 +57,8 @@ const state = { }, typing: { }, + speaking: { + }, } const getters = { @@ -321,6 +323,58 @@ const mutations = { } }, + /** + * Sets the speaking status of a participant in a conversation / call. + * + * Note that "updateParticipant" should not be called to add a "speaking" + * property to an existing participant, as the participant would be reset + * when the participants are purged whenever they are fetched again. + * Similarly, "addParticipant" can not be called either to add a participant + * if it was not fetched yet but the signaling reported it as being speaking, + * as the attendeeId would be unknown. + * + * @param {object} state - current store state. + * @param {object} data - the wrapping object. + * @param {string} data.token - the conversation token participant is speaking in. + * @param {string} data.sessionId - the Nextcloud session ID of the participant. + * @param {boolean} data.speaking - whether the participant is speaking or not + */ + setSpeaking(state, { token, sessionId, speaking }) { + // create a dummy object for current call + if (!state.speaking[token]) { + Vue.set(state.speaking, token, {}) + } + if (!state.speaking[token][sessionId]) { + Vue.set(state.speaking[token], sessionId, { speaking: null, lastTimestamp: 0, totalCountedTime: 0 }) + } + + const currentTimestamp = Date.now() + const currentSpeakingState = state.speaking[token][sessionId].speaking + + // when speaking has stopped, update the total talking time + if (!speaking && state.speaking[token][sessionId].lastTimestamp) { + state.speaking[token][sessionId].totalCountedTime += (currentTimestamp - state.speaking[token][sessionId].lastTimestamp) + } + + // don't change state for consecutive identical signals + if (currentSpeakingState !== speaking) { + state.speaking[token][sessionId].speaking = speaking + state.speaking[token][sessionId].lastTimestamp = currentTimestamp + } + }, + + /** + * Purge the speaking information for recent call when local participant leaves call + * (including cases when the call ends for everyone). + * + * @param {object} state - current store state. + * @param {object} data - the wrapping object. + * @param {string} data.token - the conversation token. + */ + purgeSpeakingStore(state, { token }) { + Vue.delete(state.speaking, token) + }, + /** * Purge a given conversation from the previously added participants. * @@ -750,6 +804,14 @@ const actions = { context.commit('setTyping', { token, sessionId, typing: true, expirationTimeout }) } }, + + setSpeaking(context, { token, sessionId, speaking }) { + context.commit('setSpeaking', { token, sessionId, speaking }) + }, + + purgeSpeakingStore(context, { token }) { + context.commit('purgeSpeakingStore', { token }) + }, } export default { state, mutations, getters, actions } diff --git a/src/utils/webrtc/SpeakingStatusHandler.js b/src/utils/webrtc/SpeakingStatusHandler.js new file mode 100644 index 00000000000..feb804b1775 --- /dev/null +++ b/src/utils/webrtc/SpeakingStatusHandler.js @@ -0,0 +1,129 @@ +/** + * + * @copyright Copyright (c) 2023 Maksim Sukharev + * + * @author Maksim Sukharev + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +/** + * Helper to handle speaking status changes notified by call models. + * + * The store is updated when local or remote participants change their speaking status. + * It is expected that the speaking status of participant will be + * modified only when the current conversation is joined and call is started. + */ +export default class SpeakingStatusHandler { + + // Constants, properties + #store + #localMediaModel + #callParticipantCollection + + // Methods (bound to have access to 'this') + #handleAddParticipantBound + #handleRemoveParticipantBound + #handleLocalSpeakingBound + #handleSpeakingBound + + constructor(store, localMediaModel, callParticipantCollection) { + this.#store = store + this.#localMediaModel = localMediaModel + this.#callParticipantCollection = callParticipantCollection + + this.#handleAddParticipantBound = this.#handleAddParticipant.bind(this) + this.#handleRemoveParticipantBound = this.#handleRemoveParticipant.bind(this) + this.#handleLocalSpeakingBound = this.#handleLocalSpeaking.bind(this) + this.#handleSpeakingBound = this.#handleSpeaking.bind(this) + + this.#localMediaModel.on('change:speaking', this.#handleLocalSpeakingBound) + this.#localMediaModel.on('change:stoppedSpeaking', this.#handleLocalSpeakingBound) + + this.#callParticipantCollection.on('add', this.#handleAddParticipantBound) + this.#callParticipantCollection.on('remove', this.#handleRemoveParticipantBound) + } + + /** + * Destroy a handler, remove all listeners, purge the speaking state from store + */ + destroy() { + this.#localMediaModel.off('change:speaking', this.#handleLocalSpeakingBound) + this.#localMediaModel.off('change:stoppedSpeaking', this.#handleLocalSpeakingBound) + + this.#callParticipantCollection.off('add', this.#handleAddParticipantBound) + this.#callParticipantCollection.off('remove', this.#handleRemoveParticipantBound) + + this.#callParticipantCollection.callParticipantModels.forEach(callParticipantModel => { + callParticipantModel.off('change:speaking', this.#handleSpeakingBound) + callParticipantModel.off('change:stoppedSpeaking', this.#handleSpeakingBound) + }) + + this.#store.dispatch('purgeSpeakingStore', { token: this.#store.getters.getToken() }) + } + + /** + * Add listeners for speaking status changes on added participants model + * + * @param {object} callParticipantCollection the collection of external participant models + * @param {object} callParticipantModel the added participant model + */ + #handleAddParticipant(callParticipantCollection, callParticipantModel) { + callParticipantModel.on('change:speaking', this.#handleSpeakingBound) + callParticipantModel.on('change:stoppedSpeaking', this.#handleSpeakingBound) + } + + /** + * Remove listeners for speaking status changes on removed participants model + * + * @param {object} callParticipantCollection the collection of external participant models + * @param {object} callParticipantModel the removed participant model + */ + #handleRemoveParticipant(callParticipantCollection, callParticipantModel) { + callParticipantModel.off('change:speaking', this.#handleSpeakingBound) + callParticipantModel.off('change:stoppedSpeaking', this.#handleSpeakingBound) + } + + /** + * Dispatch speaking status of local participant to the store + * + * @param {object} localMediaModel the local media model + * @param {boolean} speaking whether the participant is speaking or not + */ + #handleLocalSpeaking(localMediaModel, speaking) { + this.#store.dispatch('setSpeaking', { + token: this.#store.getters.getToken(), + sessionId: this.#store.getters.getSessionId(), + speaking, + }) + } + + /** + * Dispatch speaking status of participant to the store + * + * @param {object} callParticipantModel the participant model + * @param {boolean} speaking whether the participant is speaking or not + */ + #handleSpeaking(callParticipantModel, speaking) { + this.#store.dispatch('setSpeaking', { + token: this.#store.getters.getToken(), + sessionId: callParticipantModel.attributes.nextcloudSessionId, + speaking, + }) + } + +} diff --git a/src/utils/webrtc/index.js b/src/utils/webrtc/index.js index 31b48be1c9f..6853fba53a6 100644 --- a/src/utils/webrtc/index.js +++ b/src/utils/webrtc/index.js @@ -37,6 +37,7 @@ import LocalMediaModel from './models/LocalMediaModel.js' import SentVideoQualityThrottler from './SentVideoQualityThrottler.js' import './shims/MediaStream.js' import './shims/MediaStreamTrack.js' +import SpeakingStatusHandler from './SpeakingStatusHandler.js' import initWebRtc from './webrtc.js' let webRtc = null @@ -46,6 +47,7 @@ const localMediaModel = new LocalMediaModel() const mediaDevicesManager = new MediaDevicesManager() let callAnalyzer = null let sentVideoQualityThrottler = null +let speakingStatusHandler = null // This does not really belongs here, as it is unrelated to WebRTC, but it is // included here for the time being until signaling and WebRTC are split. @@ -213,6 +215,7 @@ async function signalingJoinCall(token, flags, silent) { setupWebRtc() sentVideoQualityThrottler = new SentVideoQualityThrottler(localMediaModel, callParticipantCollection, webRtc.webrtc._videoTrackConstrainer) + speakingStatusHandler = new SpeakingStatusHandler(store, localMediaModel, callParticipantCollection) if (signaling.hasFeature('mcu')) { callAnalyzer = new CallAnalyzer(localMediaModel, localCallParticipantModel, callParticipantCollection) @@ -416,6 +419,9 @@ async function signalingLeaveCall(token, all = false) { sentVideoQualityThrottler.destroy() sentVideoQualityThrottler = null + speakingStatusHandler.destroy() + speakingStatusHandler = null + callAnalyzer.destroy() callAnalyzer = null From 6cd5a2143e80b3dbfa4efe9f64fef96a7df3c0aa Mon Sep 17 00:00:00 2001 From: Maksim Sukharev Date: Tue, 8 Aug 2023 09:39:38 +0200 Subject: [PATCH 4/4] show speaking participants and elapsed time in the sidebar tab Signed-off-by: Maksim Sukharev --- .../AvatarWrapper/AvatarWrapper.vue | 11 ++++ .../Participant/Participant.vue | 63 ++++++++++++++++++- src/store/participantsStore.js | 18 ++++++ 3 files changed, 89 insertions(+), 3 deletions(-) diff --git a/src/components/AvatarWrapper/AvatarWrapper.vue b/src/components/AvatarWrapper/AvatarWrapper.vue index 4c625e7b93f..c9003308da8 100644 --- a/src/components/AvatarWrapper/AvatarWrapper.vue +++ b/src/components/AvatarWrapper/AvatarWrapper.vue @@ -25,6 +25,7 @@ 'avatar-wrapper--offline': offline, 'avatar-wrapper--small': small, 'avatar-wrapper--condensed': condensed, + 'avatar-wrapper--highlighted': highlighted, }" :style="{'--condensed-overlap': condensedOverlap}">
diff --git a/src/components/RightSidebar/Participants/ParticipantsList/Participant/Participant.vue b/src/components/RightSidebar/Participants/ParticipantsList/Participant/Participant.vue index 296517b38a8..6a161e3f402 100644 --- a/src/components/RightSidebar/Participants/ParticipantsList/Participant/Participant.vue +++ b/src/components/RightSidebar/Participants/ParticipantsList/Participant/Participant.vue @@ -40,6 +40,7 @@ disable-tooltip :show-user-status="showUserStatus && !isSearched" :preloaded-user-status="preloadedUserStatus" + :highlighted="isParticipantSpeaking" :offline="isOffline" /> @@ -67,6 +68,7 @@
{{ statusMessage }}
@@ -230,11 +232,11 @@ import Tooltip from '@nextcloud/vue/dist/Directives/Tooltip.js' import AvatarWrapper from '../../../../AvatarWrapper/AvatarWrapper.vue' import ParticipantPermissionsEditor from './ParticipantPermissionsEditor/ParticipantPermissionsEditor.vue' +import { useIsInCall } from '../../../../../composables/useIsInCall.js' import { CONVERSATION, PARTICIPANT, ATTENDEE } from '../../../../../constants.js' import readableNumber from '../../../../../mixins/readableNumber.js' import UserStatus from '../../../../../mixins/userStatus.js' - -// Material design icons +import { formattedTime } from '../../../../../utils/formattedTime.js' export default { name: 'Participant', @@ -298,11 +300,18 @@ export default { emits: ['click-participant'], + setup() { + const isInCall = useIsInCall() + return { isInCall } + }, + data() { return { isUserNameTooltipVisible: false, isStatusTooltipVisible: false, permissionsEditor: false, + speakingInterval: null, + timeSpeaking: null, } }, @@ -341,7 +350,13 @@ export default { }, statusMessage() { - return this.getStatusMessage(this.participant) + if (this.isInCall && this.participant.inCall && this.timeSpeaking) { + return this.isParticipantSpeaking + ? '💬 ' + t('spreed', '{time} talking …', { time: formattedTime(this.timeSpeaking, true) }) + : '💬 ' + t('spreed', '{time} talking time', { time: formattedTime(this.timeSpeaking, true) }) + } else { + return this.getStatusMessage(this.participant) + } }, statusMessageTooltip() { @@ -483,6 +498,14 @@ export default { return this.participant.sessionIds || [] }, + participantSpeakingInformation() { + return this.$store.getters.getParticipantSpeakingInformation(this.token, this.sessionIds) + }, + + isParticipantSpeaking() { + return this.participantSpeakingInformation?.speaking + }, + lastPing() { return this.participant.lastPing }, @@ -619,8 +642,25 @@ export default { return '' }, }, + watch: { + isParticipantSpeaking(speaking) { + if (speaking) { + if (!this.speakingInterval) { + this.speakingInterval = setInterval(this.computeElapsedTime, 1000) + } + } else { + if (speaking === undefined) { + this.timeSpeaking = 0 + } + clearInterval(this.speakingInterval) + this.speakingInterval = null + } + }, + }, methods: { + formattedTime, + updateUserNameNeedsTooltip() { // check if ellipsized const e = this.$refs.userName @@ -724,6 +764,20 @@ export default { showError(t('spreed', 'Could not modify permissions for {displayName}', { displayName: this.computedName })) } }, + + computeElapsedTime() { + if (!this.participantSpeakingInformation) { + return null + } + + const { speaking, lastTimestamp, totalCountedTime } = this.participantSpeakingInformation + + if (!speaking) { + this.timeSpeaking = totalCountedTime + } else { + this.timeSpeaking = Date.now() - lastTimestamp + totalCountedTime + } + }, }, } @@ -789,6 +843,9 @@ export default { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; + &--highlighted { + font-weight: bold; + } } &__icon { width: 44px; diff --git a/src/store/participantsStore.js b/src/store/participantsStore.js index e8b96ffa677..7140e1fde26 100644 --- a/src/store/participantsStore.js +++ b/src/store/participantsStore.js @@ -140,6 +140,24 @@ const getters = { }) }, + /** + * Gets the speaking information for the participant. + * + * @param {object} state - the state object. + * param {string} token - the conversation token. + * param {Array} sessionIds - session identifiers for the participant. + * @return {object|undefined} + */ + getParticipantSpeakingInformation: (state) => (token, sessionIds) => { + if (!state.speaking[token]) { + return undefined + } + + // look for existing sessionId in the store + const sessionId = sessionIds.find(sessionId => state.speaking[token][sessionId]) + return state.speaking[token][sessionId] + }, + /** * Replaces the legacy getParticipant getter. Returns a callback function in which you can * pass in the token and attendeeId as arguments to get the participant object.