[ts-sdk] Api improvments + bug fixes (#732)

software-mansion · Sep 6, 2024 · 1e6d649 · 1e6d649
1 parent fcb89a8
commit 1e6d649
Show file tree

Hide file tree

Showing 14 changed files with 270 additions and 209 deletions.
diff --git a/ts/@live-compositor/core/src/compositor.ts b/ts/@live-compositor/core/src/compositor.ts
@@ -35,6 +35,7 @@ export class LiveCompositor {
     const apiRequest = intoRegisterOutput(request, output.scene());
     const result = await this.api.registerOutput(outputId, apiRequest);
     this.outputs[outputId] = output;
+    await output.ready();
     return result;
   }
 
@@ -45,13 +46,19 @@ export class LiveCompositor {
   }
 
   public async registerInput(inputId: string, request: RegisterInput): Promise<object> {
-    const result = await this.api.registerInput(inputId, intoRegisterInput(request));
-    this.store.addInput({ inputId });
-    return result;
+    return this.store.runBlocking(async updateStore => {
+      const result = await this.api.registerInput(inputId, intoRegisterInput(request));
+      updateStore({ type: 'add_input', input: { inputId } });
+      return result;
+    });
   }
 
   public async unregisterInput(inputId: string): Promise<object> {
-    return this.api.unregisterInput(inputId);
+    return this.store.runBlocking(async updateStore => {
+      const result = this.api.unregisterInput(inputId);
+      updateStore({ type: 'remove_input', inputId });
+      return result;
+    });
   }
 
   public async registerShader(shaderId: string, request: Api.ShaderSpec): Promise<object> {

diff --git a/ts/@live-compositor/core/src/event.ts b/ts/@live-compositor/core/src/event.ts
@@ -7,17 +7,35 @@ export function onCompositorEvent(store: InstanceContextStore, rawEvent: unknown
   if (!event) {
     return;
   } else if (event.type === CompositorEventType.VIDEO_INPUT_DELIVERED) {
-    store.updateInput({ inputId: event.inputId, videoState: 'ready' });
+    store.dispatchUpdate({
+      type: 'update_input',
+      input: { inputId: event.inputId, videoState: 'ready' },
+    });
   } else if (event.type === CompositorEventType.VIDEO_INPUT_PLAYING) {
-    store.updateInput({ inputId: event.inputId, videoState: 'playing' });
+    store.dispatchUpdate({
+      type: 'update_input',
+      input: { inputId: event.inputId, videoState: 'playing' },
+    });
   } else if (event.type === CompositorEventType.VIDEO_INPUT_EOS) {
-    store.updateInput({ inputId: event.inputId, videoState: 'finished' });
+    store.dispatchUpdate({
+      type: 'update_input',
+      input: { inputId: event.inputId, videoState: 'finished' },
+    });
   } else if (event.type === CompositorEventType.AUDIO_INPUT_DELIVERED) {
-    store.updateInput({ inputId: event.inputId, audioState: 'ready' });
+    store.dispatchUpdate({
+      type: 'update_input',
+      input: { inputId: event.inputId, audioState: 'ready' },
+    });
   } else if (event.type === CompositorEventType.AUDIO_INPUT_PLAYING) {
-    store.updateInput({ inputId: event.inputId, audioState: 'playing' });
+    store.dispatchUpdate({
+      type: 'update_input',
+      input: { inputId: event.inputId, audioState: 'playing' },
+    });
   } else if (event.type === CompositorEventType.AUDIO_INPUT_EOS) {
-    store.updateInput({ inputId: event.inputId, audioState: 'finished' });
+    store.dispatchUpdate({
+      type: 'update_input',
+      input: { inputId: event.inputId, audioState: 'finished' },
+    });
   }
 }
 

diff --git a/ts/@live-compositor/core/src/output.ts b/ts/@live-compositor/core/src/output.ts
@@ -13,8 +13,8 @@ class Output {
   outputId: string;
   outputCtx: OutputContext;
   outputShutdownStateStore: OutputShutdownStateStore;
-  initialized: boolean = false;
 
+  shouldUpdateWhenReady: boolean = false;
   throttledUpdate: () => void;
   videoRenderer?: Renderer;
   initialAudioConfig?: Outputs.AudioInputsConfiguration;
@@ -28,9 +28,16 @@ class Output {
     this.api = api;
     this.outputId = outputId;
     this.outputShutdownStateStore = new OutputShutdownStateStore();
-    this.initialAudioConfig = registerRequest.audio?.initial;
+    this.shouldUpdateWhenReady = false;
+    this.throttledUpdate = () => {
+      this.shouldUpdateWhenReady = true;
+    };
+
+    if (registerRequest.audio) {
+      this.initialAudioConfig = registerRequest.audio.initial ?? { inputs: [] };
+    }
 
-    const onUpdate = () => this.throttledUpdate?.();
+    const onUpdate = () => this.throttledUpdate();
     this.outputCtx = new _liveCompositorInternals.OutputContext(onUpdate, !!registerRequest.audio);
 
     if (registerRequest.video) {
@@ -47,10 +54,6 @@ class Output {
         idPrefix: `${outputId}-`,
       });
     }
-
-    this.throttledUpdate = throttle(async () => {
-      await api.updateScene(this.outputId, this.scene());
-    }, 30);
   }
 
   public scene(): { video?: Api.Video; audio?: Api.Audio } {
@@ -67,6 +70,15 @@ class Output {
     // callback before it is called
     this.outputShutdownStateStore.close();
   }
+
+  public async ready() {
+    this.throttledUpdate = throttle(async () => {
+      await this.api.updateScene(this.outputId, this.scene());
+    }, 30);
+    if (this.shouldUpdateWhenReady) {
+      this.throttledUpdate();
+    }
+  }
 }
 
 // External store to share shutdown information between React tree

diff --git a/ts/examples/src/audio.tsx b/ts/examples/src/audio.tsx
@@ -75,7 +75,6 @@ async function run() {
         type: 'opus',
         channels: 'stereo',
       },
-      initial: { inputs: [] },
     },
   });
   gstReceiveTcpStream('127.0.0.1', 8001);

diff --git a/ts/examples/src/dynamic-inputs.tsx b/ts/examples/src/dynamic-inputs.tsx
@@ -32,8 +32,10 @@ function InputTile({ inputId }: { inputId: string }) {
       <Rescaler>
         <InputStream inputId={inputId} />
       </Rescaler>
-      <View bottom={10} left={10} height={40}>
-        <Text fontSize={40}>Input ID: {inputId}</Text>
+      <View bottom={10} left={10} height={50}>
+        <Text fontSize={40} color="#FF0000" lineHeight={50} backgroundColor="#FFFFFF88">
+          Input ID: {inputId}
+        </Text>
       </View>
     </View>
   );

diff --git a/ts/examples/src/dynamic-outputs.tsx b/ts/examples/src/dynamic-outputs.tsx
@@ -0,0 +1,120 @@
+import LiveCompositor from '@live-compositor/node';
+import { Text, InputStream, Tiles, Rescaler, View, useInputStreams } from 'live-compositor';
+import { downloadAllAssets, gstReceiveTcpStream, sleep } from './utils';
+import path from 'path';
+import fs from 'fs-extra';
+
+function ExampleApp() {
+  const inputs = useInputStreams();
+  return (
+    <Tiles transition={{ durationMs: 200 }}>
+      {Object.values(inputs).map(input => (
+        <InputTile key={input.inputId} inputId={input.inputId} />
+      ))}
+    </Tiles>
+  );
+}
+
+function InputTile({ inputId }: { inputId: string }) {
+  return (
+    <View>
+      <Rescaler>
+        <InputStream inputId={inputId} />
+      </Rescaler>
+      <View bottom={10} left={10} height={50}>
+        <Text fontSize={40} color="#FF0000" lineHeight={50} backgroundColor="#FFFFFF88">
+          Input ID: {inputId}
+        </Text>
+      </View>
+    </View>
+  );
+}
+
+async function run() {
+  await fs.mkdirp(path.join(__dirname, '../.workingdir'));
+  await downloadAllAssets();
+  const compositor = await LiveCompositor.create();
+
+  const RESOLUTION = {
+    width: 1920,
+    height: 1080,
+  } as const;
+  const VIDEO_ENCODER_OPTS = {
+    type: 'ffmpeg_h264',
+    preset: 'ultrafast',
+  } as const;
+
+  await compositor.registerOutput('output_stream', {
+    type: 'rtp_stream',
+    port: 8001,
+    transportProtocol: 'tcp_server',
+    video: {
+      encoder: VIDEO_ENCODER_OPTS,
+      resolution: RESOLUTION,
+      root: <ExampleApp />,
+    },
+    audio: {
+      encoder: {
+        type: 'opus',
+        channels: 'stereo',
+      },
+    },
+  });
+  gstReceiveTcpStream('127.0.0.1', 8001);
+  await compositor.registerOutput('output_recording', {
+    type: 'mp4',
+    serverPath: path.join(__dirname, '../.workingdir/dynamic_outputs_recording.mp4'),
+    video: {
+      encoder: VIDEO_ENCODER_OPTS,
+      resolution: RESOLUTION,
+      root: <ExampleApp />,
+    },
+    audio: {
+      encoder: {
+        type: 'aac',
+        channels: 'stereo',
+      },
+    },
+  });
+
+  await compositor.registerInput('input_1', {
+    type: 'mp4',
+    serverPath: path.join(__dirname, '../.assets/BigBuckBunny.mp4'),
+  });
+  console.log(
+    'Start LiveCompositor pipeline with single input ("input_1") and two outputs (RTP "output_stream" and MP4 "output_recording").'
+  );
+  await compositor.start();
+
+  await sleep(10_000);
+  console.log('Connect new input ("input_2") and start new output to MP4 "output_recording_part2"');
+  await compositor.registerInput('input_2', {
+    type: 'mp4',
+    serverPath: path.join(__dirname, '../.assets/ElephantsDream.mp4'),
+  });
+  await compositor.registerOutput('output_recording_part2', {
+    type: 'mp4',
+    serverPath: path.join(__dirname, '../.workingdir/dynamic_outputs_recording_10s.mp4'),
+    video: {
+      encoder: VIDEO_ENCODER_OPTS,
+      resolution: RESOLUTION,
+      root: <ExampleApp />,
+    },
+    audio: {
+      encoder: {
+        type: 'aac',
+        channels: 'stereo',
+      },
+    },
+  });
+
+  await sleep(10_000);
+  console.log('Stop output "output_recording"');
+  await compositor.unregisterOutput('output_recording');
+
+  await sleep(10_000);
+  console.log('Stop all remaining outputs.');
+  await compositor.unregisterOutput('output_recording_part2');
+  await compositor.unregisterOutput('output_stream');
+}
+run();