Improve default screen sharing FPS, limiting capturing surface (#972)

.changeset/little-yaks-shop.md

@@ -0,0 +1,5 @@
+---
+'livekit-client': patch
+---
+
+Default screenshare capture resolution to 1080p

example/sample.ts

@@ -19,6 +19,7 @@ import {
   RoomConnectOptions,
   RoomEvent,
   RoomOptions,
+  ScreenSharePresets,
   Track,
   TrackPublication,
   VideoCaptureOptions,
@@ -95,6 +96,7 @@ const appActions = {
         dtx: true,
         red: true,
         forceStereo: false,
+        screenShareEncoding: ScreenSharePresets.h1080fps30.encoding,
       },
       videoCaptureDefaults: {
         resolution: VideoPresets.h720.resolution,

package.json

@@ -40,7 +40,7 @@
     "build:watch": "rollup --watch --config rollup.config.js",
     "build-docs": "typedoc",
     "proto": "protoc --es_out src/proto --es_opt target=ts -I./protocol ./protocol/livekit_rtc.proto ./protocol/livekit_models.proto",
-    "sample": "vite example -c vite.config.js",
+    "sample": "vite example -c vite.config.mjs",
     "lint": "eslint src",
     "test": "vitest run src",
     "deploy": "gh-pages -d example/dist",

src/room/participant/LocalParticipant.ts

@@ -31,7 +31,7 @@ import type {
   TrackPublishOptions,
   VideoCaptureOptions,
 } from '../track/options';
-import { VideoPresets, isBackupCodec } from '../track/options';
+import { ScreenSharePresets, VideoPresets, isBackupCodec } from '../track/options';
 import {
   constraintsForOptions,
   getLogContextFromTrack,
@@ -40,7 +40,16 @@ import {
   screenCaptureToDisplayMediaStreamOptions,
 } from '../track/utils';
 import type { DataPublishOptions } from '../types';
-import { Future, isFireFox, isSVCCodec, isSafari, isWeb, supportsAV1, supportsVP9 } from '../utils';
+import {
+  Future,
+  isFireFox,
+  isSVCCodec,
+  isSafari,
+  isSafari17,
+  isWeb,
+  supportsAV1,
+  supportsVP9,
+} from '../utils';
 import Participant from './Participant';
 import type { ParticipantTrackPermission } from './ParticipantTrackPermission';
 import { trackPermissionToProto } from './ParticipantTrackPermission';
@@ -457,6 +466,13 @@ export default class LocalParticipant extends Participant {
       throw new DeviceUnsupportedError('getDisplayMedia not supported');
     }
 
+    if (options.resolution === undefined && !isSafari17()) {
+      // we need to constrain the dimensions, otherwise it could lead to low bitrate
+      // due to encoding a huge video. Encoding such large surfaces is really expensive
+      // unfortunately Safari 17 has a but and cannot be constrained by default
+      options.resolution = ScreenSharePresets.h1080fps30.resolution;
+    }
+
     const constraints = screenCaptureToDisplayMediaStreamOptions(options);
     const stream: MediaStream = await navigator.mediaDevices.getDisplayMedia(constraints);
 
@@ -469,6 +485,10 @@ export default class LocalParticipant extends Participant {
       loggerContextCb: () => this.logContext,
     });
     screenVideo.source = Track.Source.ScreenShare;
+    if (options.contentHint) {
+      screenVideo.mediaStreamTrack.contentHint = options.contentHint;
+    }
+
     const localTracks: Array<LocalTrack> = [screenVideo];
     if (stream.getAudioTracks().length > 0) {
       this.emit(ParticipantEvent.AudioStreamAcquired);

src/room/participant/publishUtils.test.ts

@@ -183,7 +183,7 @@ describe('screenShareSimulcastDefaults', () => {
     );
     expect(defaultSimulcastLayers[0].width).toBe(640);
     expect(defaultSimulcastLayers[0].height).toBe(360);
-    expect(defaultSimulcastLayers[0].encoding.maxFramerate).toBe(3);
-    expect(defaultSimulcastLayers[0].encoding.maxBitrate).toBe(150_000);
+    expect(defaultSimulcastLayers[0].encoding.maxFramerate).toBe(15);
+    expect(defaultSimulcastLayers[0].encoding.maxBitrate).toBe(375000);
   });
 });

src/room/participant/publishUtils.ts

@@ -46,7 +46,7 @@ export const defaultSimulcastPresets43 = [VideoPresets43.h180, VideoPresets43.h3
 
 /* @internal */
 export const computeDefaultScreenShareSimulcastPresets = (fromPreset: VideoPreset) => {
-  const layers = [{ scaleResolutionDownBy: 2, fps: 3 }];
+  const layers = [{ scaleResolutionDownBy: 2, fps: fromPreset.encoding.maxFramerate }];
   return layers.map(
     (t) =>
       new VideoPreset(
@@ -56,7 +56,8 @@ export const computeDefaultScreenShareSimulcastPresets = (fromPreset: VideoPrese
           150_000,
           Math.floor(
             fromPreset.encoding.maxBitrate /
-              (t.scaleResolutionDownBy ** 2 * ((fromPreset.encoding.maxFramerate ?? 30) / t.fps)),
+              (t.scaleResolutionDownBy ** 2 *
+                ((fromPreset.encoding.maxFramerate ?? 30) / (t.fps ?? 30))),
           ),
         ),
         t.fps,

src/room/track/create.ts

@@ -2,17 +2,18 @@ import DeviceManager from '../DeviceManager';
 import { audioDefaults, videoDefaults } from '../defaults';
 import { DeviceUnsupportedError, TrackInvalidError } from '../errors';
 import { mediaTrackToLocalTrack } from '../participant/publishUtils';
+import { isSafari17 } from '../utils';
 import LocalAudioTrack from './LocalAudioTrack';
 import type LocalTrack from './LocalTrack';
 import LocalVideoTrack from './LocalVideoTrack';
 import { Track } from './Track';
-import { ScreenSharePresets } from './options';
 import type {
   AudioCaptureOptions,
   CreateLocalTracksOptions,
   ScreenShareCaptureOptions,
   VideoCaptureOptions,
 } from './options';
+import { ScreenSharePresets } from './options';
 import {
   constraintsForOptions,
   mergeDefaultOptions,
@@ -116,8 +117,8 @@ export async function createLocalScreenTracks(
   if (options === undefined) {
     options = {};
   }
-  if (options.resolution === undefined) {
-    options.resolution = ScreenSharePresets.h1080fps15.resolution;
+  if (options.resolution === undefined && !isSafari17()) {
+    options.resolution = ScreenSharePresets.h1080fps30.resolution;
   }
 
   if (navigator.mediaDevices.getDisplayMedia === undefined) {

src/room/track/options.ts

@@ -169,9 +169,10 @@ export interface ScreenShareCaptureOptions {
   video?: true | { displaySurface?: 'window' | 'browser' | 'monitor' };
 
   /**
-   * capture resolution, defaults to screen resolution
-   * NOTE: In Safari 17, specifying any resolution at all would lead to a low-resolution
-   * capture. https://bugs.webkit.org/show_bug.cgi?id=263015
+   * capture resolution, defaults to 1080 for all browsers other than Safari
+   * On Safari 17, default resolution is not capped, due to a bug, specifying
+   * any resolution at all would lead to a low-resolution capture.
+   * https://bugs.webkit.org/show_bug.cgi?id=263015
    */
   resolution?: VideoResolution;
 
@@ -187,6 +188,9 @@ export interface ScreenShareCaptureOptions {
   /** specifies whether the browser should include the system audio among the possible audio sources offered to the user */
   systemAudio?: 'include' | 'exclude';
 
+  /** specify the type of content, see: https://www.w3.org/TR/mst-content-hint/#video-content-hints */
+  contentHint?: 'detail' | 'text' | 'motion';
+
   /**
    * Experimental option to control whether the audio playing in a tab will continue to be played out of a user's
    * local speakers when the tab is captured.
@@ -367,9 +371,12 @@ export const VideoPresets43 = {
 
 export const ScreenSharePresets = {
   h360fps3: new VideoPreset(640, 360, 200_000, 3, 'medium'),
-  h720fps5: new VideoPreset(1280, 720, 400_000, 5, 'medium'),
+  h360fps15: new VideoPreset(640, 360, 400_000, 15, 'medium'),
+  h720fps5: new VideoPreset(1280, 720, 800_000, 5, 'medium'),
   h720fps15: new VideoPreset(1280, 720, 1_500_000, 15, 'medium'),
   h720fps30: new VideoPreset(1280, 720, 2_000_000, 30, 'medium'),
   h1080fps15: new VideoPreset(1920, 1080, 2_500_000, 15, 'medium'),
-  h1080fps30: new VideoPreset(1920, 1080, 4_000_000, 30, 'medium'),
+  h1080fps30: new VideoPreset(1920, 1080, 5_000_000, 30, 'medium'),
+  // original resolution, without resizing
+  original: new VideoPreset(0, 0, 7_000_000, 30, 'medium'),
 } as const;

src/room/track/utils.ts

@@ -155,7 +155,8 @@ export function screenCaptureToDisplayMediaStreamOptions(
   options: ScreenShareCaptureOptions,
 ): DisplayMediaStreamOptions {
   let videoConstraints: MediaTrackConstraints | boolean = options.video ?? true;
-  if (options.resolution) {
+  // treat 0 as uncapped
+  if (options.resolution && options.resolution.width > 0 && options.resolution.height > 0) {
     videoConstraints = typeof videoConstraints === 'boolean' ? {} : videoConstraints;
     if (isSafari()) {
       videoConstraints = {

src/room/utils.ts

@@ -148,6 +148,11 @@ export function isSafari(): boolean {
   return getBrowser()?.name === 'Safari';
 }
 
+export function isSafari17(): boolean {
+  const b = getBrowser();
+  return b?.name === 'Safari' && b.version.startsWith('17.');
+}
+
 export function isMobile(): boolean {
   if (!isWeb()) return false;
   return /Tablet|iPad|Mobile|Android|BlackBerry/.test(navigator.userAgent);