[ts-sdk] Make react sdk run on browser

ts/@live-compositor/browser-render/src/index.ts

@@ -1,2 +1,3 @@
 export { loadWasmModule } from './wasm';
 export * from './renderer';
+export * from './api';

ts/@live-compositor/browser-render/src/renderer.ts

@@ -8,11 +8,6 @@ export type RendererOptions = {
   streamFallbackTimeoutMs: number;
 };
 
-export type Framerate = {
-  num: number;
-  den: number;
-};
-
 export type FrameSet = {
   ptsMs: number;
   frames: { [id: string]: Frame };

ts/@live-compositor/core/src/api.ts

@@ -1,5 +1,7 @@
 import { Api } from 'live-compositor';
 import { CompositorManager } from './compositorManager.js';
+import { RegisterOutputRequest } from './api/output.js';
+import { RegisterInputRequest } from './api/input.js';
 
 export { Api };
 
@@ -24,7 +26,7 @@ export class ApiClient {
     });
   }
 
-  public async registerOutput(outptuId: string, request: Api.RegisterOutput): Promise<object> {
+  public async registerOutput(outptuId: string, request: RegisterOutputRequest): Promise<object> {
     return this.serverManager.sendRequest({
       method: 'POST',
       route: `/api/output/${encodeURIComponent(outptuId)}/register`,
@@ -40,7 +42,7 @@ export class ApiClient {
     });
   }
 
-  public async registerInput(inputId: string, request: Api.RegisterInput): Promise<object> {
+  public async registerInput(inputId: string, request: RegisterInputRequest): Promise<object> {
     return this.serverManager.sendRequest({
       method: 'POST',
       route: `/api/input/${encodeURIComponent(inputId)}/register`,

ts/@live-compositor/core/src/api/input.ts

@@ -1,7 +1,13 @@
 import { Api } from '../api.js';
-import { RegisterInput, Inputs } from 'live-compositor';
+import { RegisterMp4Input, RegisterRtpInput, Inputs } from 'live-compositor';
 
-export function intoRegisterInput(input: RegisterInput): Api.RegisterInput {
+export type RegisterInputRequest = Api.RegisterInput;
+
+export type RegisterInput =
+  | ({ type: 'rtp_stream' } & RegisterRtpInput)
+  | ({ type: 'mp4' } & RegisterMp4Input);
+
+export function intoRegisterInput(input: RegisterInput): RegisterInputRequest {
   if (input.type === 'mp4') {
     return intoMp4RegisterInput(input);
   } else if (input.type === 'rtp_stream') {
@@ -11,7 +17,7 @@ export function intoRegisterInput(input: RegisterInput): Api.RegisterInput {
   }
 }
 
-function intoMp4RegisterInput(input: Inputs.RegisterMp4Input): Api.RegisterInput {
+function intoMp4RegisterInput(input: Inputs.RegisterMp4Input): RegisterInputRequest {
   return {
     type: 'mp4',
     url: input.url,
@@ -22,7 +28,7 @@ function intoMp4RegisterInput(input: Inputs.RegisterMp4Input): Api.RegisterInput
   };
 }
 
-function intoRtpRegisterInput(input: Inputs.RegisterRtpInput): Api.RegisterInput {
+function intoRtpRegisterInput(input: Inputs.RegisterRtpInput): RegisterInputRequest {
   return {
     type: 'rtp_stream',
     port: input.port,

ts/@live-compositor/core/src/api/output.ts

@@ -1,22 +1,51 @@
-import { RegisterOutput, Api, Outputs } from 'live-compositor';
+import {
+  Api,
+  Outputs,
+  RegisterRtpOutput,
+  RegisterMp4Output,
+  RegisterCanvasOutput,
+} from 'live-compositor';
+
+export type RegisterOutputRequest = Api.RegisterOutput | RegisterCanvasOutputRequest;
+
+export type RegisterCanvasOutputRequest = {
+  type: 'canvas';
+  video: OutputCanvasVideoOptions;
+};
+
+export type OutputCanvasVideoOptions = {
+  resolution: Api.Resolution;
+  /**
+   * HTMLCanvasElement
+   */
+  canvas: any;
+  initial: Api.Video;
+};
+
+export type RegisterOutput =
+  | ({ type: 'rtp_stream' } & RegisterRtpOutput)
+  | ({ type: 'mp4' } & RegisterMp4Output)
+  | ({ type: 'canvas' } & RegisterCanvasOutput);
 
 export function intoRegisterOutput(
   output: RegisterOutput,
   initial: { video?: Api.Video; audio?: Api.Audio }
-): Api.RegisterOutput {
+): RegisterOutputRequest {
   if (output.type === 'rtp_stream') {
     return intoRegisterRtpOutput(output, initial);
   } else if (output.type === 'mp4') {
     return intoRegisterMp4Output(output, initial);
+  } else if (output.type === 'canvas') {
+    return intoRegisterCanvasOutput(output, initial);
   } else {
-    throw new Error(`Unknown input type ${(output as any).type}`);
+    throw new Error(`Unknown output type ${(output as any).type}`);
   }
 }
 
 function intoRegisterRtpOutput(
   output: Outputs.RegisterRtpOutput,
   initial: { video?: Api.Video; audio?: Api.Audio }
-): Api.RegisterOutput {
+): RegisterOutputRequest {
   return {
     type: 'rtp_stream',
     port: output.port,
@@ -30,7 +59,7 @@ function intoRegisterRtpOutput(
 function intoRegisterMp4Output(
   output: Outputs.RegisterMp4Output,
   initial: { video?: Api.Video; audio?: Api.Audio }
-): Api.RegisterOutput {
+): RegisterOutputRequest {
   return {
     type: 'mp4',
     path: output.serverPath,
@@ -39,6 +68,20 @@ function intoRegisterMp4Output(
   };
 }
 
+function intoRegisterCanvasOutput(
+  output: Outputs.RegisterCanvasOutput,
+  initial: { video?: Api.Video; _audio?: Api.Audio }
+): RegisterOutputRequest {
+  return {
+    type: 'canvas',
+    video: {
+      resolution: output.video.resolution,
+      canvas: output.video.canvas,
+      initial: initial.video!,
+    },
+  };
+}
+
 function intoOutputVideoOptions(
   video: Outputs.RtpVideoOptions | Outputs.Mp4VideoOptions,
   initial: Api.Video

ts/@live-compositor/core/src/compositor.ts

@@ -1,14 +1,9 @@
-import {
-  _liveCompositorInternals,
-  RegisterInput,
-  RegisterOutput,
-  Renderers,
-} from 'live-compositor';
+import { _liveCompositorInternals, Renderers } from 'live-compositor';
 import { ApiClient } from './api.js';
 import Output from './output.js';
 import { CompositorManager } from './compositorManager.js';
-import { intoRegisterOutput } from './api/output.js';
-import { intoRegisterInput } from './api/input.js';
+import { intoRegisterOutput, RegisterOutput } from './api/output.js';
+import { intoRegisterInput, RegisterInput } from './api/input.js';
 import { onCompositorEvent } from './event.js';
 import { intoRegisterImage, intoRegisterWebRenderer } from './api/renderer.js';
 

ts/@live-compositor/core/src/index.ts

@@ -1,3 +1,5 @@
 export { ApiClient, ApiRequest } from './api.js';
 export { LiveCompositor } from './compositor.js';
 export { CompositorManager } from './compositorManager.js';
+export { RegisterInputRequest, RegisterInput } from './api/input.js';
+export { RegisterOutputRequest, RegisterOutput } from './api/output.js';

ts/@live-compositor/core/src/output.ts

@@ -1,8 +1,8 @@
-import { _liveCompositorInternals, RegisterOutput, View, Outputs } from 'live-compositor';
+import { _liveCompositorInternals, View, Outputs } from 'live-compositor';
 import React, { useSyncExternalStore } from 'react';
 import { ApiClient, Api } from './api.js';
 import Renderer from './renderer.js';
-import { intoAudioInputsConfiguration } from './api/output.js';
+import { intoAudioInputsConfiguration, RegisterOutput } from './api/output.js';
 import { throttle } from './utils.js';
 
 type OutputContext = _liveCompositorInternals.OutputContext;
@@ -33,12 +33,13 @@ class Output {
       this.shouldUpdateWhenReady = true;
     };
 
-    if (registerRequest.audio) {
-      this.initialAudioConfig = registerRequest.audio.initial ?? { inputs: [] };
+    const hasAudio = 'audio' in registerRequest && !!registerRequest.audio;
+    if (hasAudio) {
+      this.initialAudioConfig = registerRequest.audio!.initial ?? { inputs: [] };
     }
 
     const onUpdate = () => this.throttledUpdate();
-    this.outputCtx = new _liveCompositorInternals.OutputContext(onUpdate, !!registerRequest.audio);
+    this.outputCtx = new _liveCompositorInternals.OutputContext(onUpdate, hasAudio);
 
     if (registerRequest.video) {
       const rootElement = React.createElement(OutputRootComponent, {

ts/@live-compositor/web-wasm/.eslintignore

@@ -0,0 +1 @@
+dist

ts/@live-compositor/web-wasm/.eslintrc.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "../../.eslintrc.base.json"
+  ]
+}

ts/@live-compositor/web-wasm/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@live-compositor/web-wasm",
+  "version": "0.1.0-rc.0",
+  "description": "",
+  "main": "dist/index.js",
+  "scripts": {
+    "lint": "eslint .",
+    "typecheck": "tsc --noEmit",
+    "watch": "tsc --watch --preserveWatchOutput",
+    "build": "tsc",
+    "clean": "rimraf dist",
+    "prepublishOnly": "npm run clean && npm run build"
+  },
+  "author": "",
+  "license": "MIT",
+  "files": [
+    "/dist"
+  ],
+  "dependencies": {
+    "@datastructures-js/queue": "^4.2.3",
+    "@live-compositor/browser-render": "0.1.0-rc.4",
+    "@live-compositor/core": "0.1.0",
+    "live-compositor": "^0.1.0",
+    "mp4box": "^0.5.2",
+    "path-parser": "^6.1.0"
+  }
+}

ts/@live-compositor/web-wasm/src/compositor.ts

@@ -0,0 +1,86 @@
+import { Renderer } from '@live-compositor/browser-render';
+import { LiveCompositor as CoreLiveCompositor } from '@live-compositor/core';
+import WasmInstance from './manager/wasmInstance';
+import { intoRegisterOutput, RegisterOutput } from './output/registerOutput';
+import { intoRegisterInput, RegisterInput } from './input/registerInput';
+import { RegisterImage } from './renderers';
+
+export type LiveCompositorOptions = {
+  framerate?: Framerate;
+  streamFallbackTimeoutMs?: number;
+};
+
+export type Framerate = {
+  num: number;
+  den: number;
+};
+
+export default class LiveCompositor {
+  private coreCompositor?: CoreLiveCompositor;
+  private instance?: WasmInstance;
+  private renderer?: Renderer;
+  private options: LiveCompositorOptions;
+
+  public constructor(options: LiveCompositorOptions) {
+    this.options = options;
+  }
+
+  /*
+   * Initializes LiveCompositor instance. It needs to be called before any resource is registered.
+   * Outputs won't produce any results until `start()` is called.
+   */
+  public async init(): Promise<void> {
+    this.renderer = await Renderer.create({
+      streamFallbackTimeoutMs: this.options.streamFallbackTimeoutMs ?? 500,
+    });
+    this.instance = new WasmInstance({
+      renderer: this.renderer!,
+      framerate: this.options.framerate ?? { num: 30, den: 1 },
+    });
+    this.coreCompositor = new CoreLiveCompositor(this.instance!);
+
+    await this.coreCompositor!.init();
+  }
+
+  public async registerOutput(outputId: string, request: RegisterOutput): Promise<void> {
+    await this.coreCompositor!.registerOutput(outputId, intoRegisterOutput(request));
+  }
+
+  public async unregisterOutput(outputId: string): Promise<void> {
+    await this.coreCompositor!.unregisterOutput(outputId);
+  }
+
+  public async registerInput(inputId: string, request: RegisterInput): Promise<void> {
+    await this.coreCompositor!.registerInput(inputId, intoRegisterInput(request));
+  }
+
+  public async unregisterInput(inputId: string): Promise<void> {
+    await this.coreCompositor!.unregisterInput(inputId);
+  }
+
+  public async registerImage(imageId: string, request: RegisterImage): Promise<void> {
+    await this.coreCompositor!.registerImage(imageId, request);
+  }
+
+  public async unregisterImage(imageId: string): Promise<void> {
+    await this.coreCompositor!.unregisterImage(imageId);
+  }
+
+  public async registerFont(fontUrl: string): Promise<void> {
+    await this.renderer!.registerFont(fontUrl);
+  }
+
+  /**
+   * Starts processing pipeline. Any previously registered output will start producing video data.
+   */
+  public async start(): Promise<void> {
+    await this.coreCompositor!.start();
+  }
+
+  /**
+   * Stops processing pipeline.
+   */
+  public stop(): void {
+    this.instance!.stop();
+  }
+}

ts/@live-compositor/web-wasm/src/eventSender.ts

@@ -0,0 +1,62 @@
+import { CompositorEvent, CompositorEventType } from 'live-compositor';
+
+export class EventSender {
+  private eventCallback?: (event: object) => void;
+
+  public setEventCallback(eventCallback: (event: object) => void) {
+    this.eventCallback = eventCallback;
+  }
+
+  public sendEvent(event: CompositorEvent) {
+    if (!this.eventCallback) {
+      console.warn(`Failed to send event: ${event}`);
+      return;
+    }
+
+    this.eventCallback!(toWebSocketMessage(event));
+  }
+}
+
+function toWebSocketMessage(event: CompositorEvent): WebSocketMessage {
+  if (event.type == CompositorEventType.OUTPUT_DONE) {
+    return {
+      type: event.type,
+      output_id: event.outputId,
+    };
+  }
+
+  return {
+    type: event.type,
+    input_id: event.inputId,
+  };
+}
+
+export type WebSocketMessage =
+  | {
+      type: CompositorEventType.AUDIO_INPUT_DELIVERED;
+      input_id: string;
+    }
+  | {
+      type: CompositorEventType.VIDEO_INPUT_DELIVERED;
+      input_id: string;
+    }
+  | {
+      type: CompositorEventType.AUDIO_INPUT_PLAYING;
+      input_id: string;
+    }
+  | {
+      type: CompositorEventType.VIDEO_INPUT_PLAYING;
+      input_id: string;
+    }
+  | {
+      type: CompositorEventType.AUDIO_INPUT_EOS;
+      input_id: string;
+    }
+  | {
+      type: CompositorEventType.VIDEO_INPUT_EOS;
+      input_id: string;
+    }
+  | {
+      type: CompositorEventType.OUTPUT_DONE;
+      output_id: string;
+    };