From b02108b53b566655529c080e55b10f56d6bed2ef Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 10:33:04 +0900 Subject: [PATCH 01/34] =?UTF-8?q?CI=20=E3=81=A7=20rust=20=E3=81=AE=20wasm3?= =?UTF-8?q?2=20=E3=82=BF=E3=83=BC=E3=82=B2=E3=83=83=E3=83=88=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7f90ff94..96f997b0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,6 +21,7 @@ jobs: with: version: 0.12.0 - run: zig version + - run: rustup update stable && rustup target add wasm32-unknown-unknown - uses: actions/setup-node@v4 with: node-version: ${{ matrix.node }} From 440d0965917914f75cb1b6200ed39aeb84dd0af6 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 10:33:37 +0900 Subject: [PATCH 02/34] Update .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 75583a57..c4f556f8 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,6 @@ yarn.lock # Ignore typedoc output apidoc/ + +# Rust +/target/ From cd980daecc90ebc4ccd753ac11ba59f562c8306d Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 10:34:04 +0900 Subject: [PATCH 03/34] Add Cargo.toml --- Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Cargo.toml diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000..b40c2f84 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,3 @@ +[workspace] +members = ["packages/mp4-media-stream/wasm/"] +resolver = "2" From 987f5341b6ea46a703dc2cc17bd9940edd4ac77c Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 10:39:51 +0900 Subject: [PATCH 04/34] Add Cargo.lock --- Cargo.lock | 230 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 Cargo.lock diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 00000000..082f1655 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,230 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "mp4_media_stream" +version = "0.0.0" +dependencies = [ + "futures", + "orfail", + "serde", + "serde_json", + "shiguredo_mp4", +] + +[[package]] +name = "orfail" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aba2212a0fddd290c8209fb9b0393382435ede29c24a71f245d7fa92f0f3442e" +dependencies = [ + "serde", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "proc-macro2" +version = "1.0.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "serde" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "shiguredo_mp4" +version = "2024.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298dae998ee0ea59ac50b3fb6ada19da86672fd14751a7a2d7f9aeeff7e49fa4" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "syn" +version = "2.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" From 547b9c4a4ad5fa00217455decb569c47c8a87fc1 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 10:39:57 +0900 Subject: [PATCH 05/34] Add mp4-media-stream packages --- packages/mp4-media-stream/package.json | 30 ++ packages/mp4-media-stream/rollup.config.mjs | 58 +++ .../mp4-media-stream/src/mp4_media_stream.ts | 394 ++++++++++++++++++ packages/mp4-media-stream/tsconfig.json | 22 + packages/mp4-media-stream/typedoc.json | 12 + packages/mp4-media-stream/wasm/Cargo.toml | 17 + packages/mp4-media-stream/wasm/src/engine.rs | 83 ++++ packages/mp4-media-stream/wasm/src/lib.rs | 4 + packages/mp4-media-stream/wasm/src/mp4.rs | 175 ++++++++ packages/mp4-media-stream/wasm/src/player.rs | 168 ++++++++ packages/mp4-media-stream/wasm/src/wasm.rs | 227 ++++++++++ 11 files changed, 1190 insertions(+) create mode 100644 packages/mp4-media-stream/package.json create mode 100644 packages/mp4-media-stream/rollup.config.mjs create mode 100644 packages/mp4-media-stream/src/mp4_media_stream.ts create mode 100644 packages/mp4-media-stream/tsconfig.json create mode 100644 packages/mp4-media-stream/typedoc.json create mode 100644 packages/mp4-media-stream/wasm/Cargo.toml create mode 100644 packages/mp4-media-stream/wasm/src/engine.rs create mode 100644 packages/mp4-media-stream/wasm/src/lib.rs create mode 100644 packages/mp4-media-stream/wasm/src/mp4.rs create mode 100644 packages/mp4-media-stream/wasm/src/player.rs create mode 100644 packages/mp4-media-stream/wasm/src/wasm.rs diff --git a/packages/mp4-media-stream/package.json b/packages/mp4-media-stream/package.json new file mode 100644 index 00000000..a8135e14 --- /dev/null +++ b/packages/mp4-media-stream/package.json @@ -0,0 +1,30 @@ +{ + "name": "@shiguredo/mp4-media-stream", + "version": "2024.1.0", + "description": "Generate MediaStream from MP4 file", + "author": "Shiguredo Inc.", + "license": "Apache-2.0", + "main": "dist/mp4_media_stream.js", + "module": "dist/mp4_media_stream.mjs", + "types": "dist/mp4_media_stream.d.ts", + "scripts": { + "build": "cargo build --release --target wasm32-unknown-unknown -p mp4_media_stream && rollup -c ./rollup.config.mjs --bundleConfigAsCjs && tsc --emitDeclarationOnly", + "lint": "biome lint ./src", + "fmt": "biome format --write src", + "doc": "typedoc" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/shiguredo/media-processors.git" + }, + "keywords": [ + "mp4" + ], + "bugs": { + "url": "https://discord.gg/shiguredo" + }, + "homepage": "https://github.com/shiguredo/media-processors#readme", + "files": [ + "dist" + ] +} diff --git a/packages/mp4-media-stream/rollup.config.mjs b/packages/mp4-media-stream/rollup.config.mjs new file mode 100644 index 00000000..e0dd3d9f --- /dev/null +++ b/packages/mp4-media-stream/rollup.config.mjs @@ -0,0 +1,58 @@ +import * as fs from 'fs'; +import typescript from '@rollup/plugin-typescript'; +import pkg from './package.json'; +import commonjs from '@rollup/plugin-commonjs'; +import resolve from '@rollup/plugin-node-resolve'; +import replace from '@rollup/plugin-replace'; + +const banner = `/** + * ${pkg.name} + * ${pkg.description} + * @version: ${pkg.version} + * @author: ${pkg.author} + * @license: ${pkg.license} + **/ +`; + +export default [ + { + input: 'src/mp4_media_stream.ts', + plugins: [ + replace({ + __WASM__: () => fs.readFileSync("../../target/wasm32-unknown-unknown/release/mp4_media_stream.wasm", "base64"), + preventAssignment: true + }), + typescript({module: "esnext"}), + commonjs(), + resolve() + ], + output: { + sourcemap: false, + file: './dist/mp4_media_stream.js', + format: 'umd', + name: 'Shiguredo', + extend: true, + banner: banner, + } + }, + { + input: 'src/mp4_media_stream.ts', + plugins: [ + replace({ + __WASM__: () => fs.readFileSync("../../target/wasm32-unknown-unknown/release/mp4_media_stream.wasm", "base64"), + preventAssignment: true + }), + typescript({module: "esnext"}), + commonjs(), + resolve() + ], + output: { + sourcemap: false, + file: './dist/mp4_media_stream.mjs', + format: 'module', + name: 'Shiguredo', + extend: true, + banner: banner, + } + }, +]; diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts new file mode 100644 index 00000000..be4e3acf --- /dev/null +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -0,0 +1,394 @@ +const WASM_BASE64 = '__WASM__' + +interface Mp4MediaStreamPlayOptions { + repeat?: boolean +} + +class Mp4MediaStream { + private engine: Engine + + constructor(wasm: WebAssembly.Instance) { + this.engine = new Engine(wasm, wasm.exports.memory as WebAssembly.Memory) + } + + static isSupported(): boolean { + return !(typeof MediaStreamTrackGenerator === 'undefined') + } + + static async loadMp4(mp4: Blob): Promise { + const engineRef: { value?: Engine } = { value: undefined } + const importObject = { + env: { + now() { + return performance.now() + }, + consoleLog(messageWasmJson: number) { + if (engineRef.value) { + engineRef.value.consoleLog(messageWasmJson) + } + }, + sleep(resultTx: number, duration: number) { + if (engineRef.value) { + engineRef.value.sleep(resultTx, duration) + } + }, + createVideoDecoder(playerId: number, configWasmJson: number) { + if (engineRef.value) { + engineRef.value.createVideoDecoder(playerId, configWasmJson) + } + }, + createAudioDecoder(playerId: number, configWasmJson: number) { + if (engineRef.value) { + engineRef.value.createAudioDecoder(playerId, configWasmJson) + } + }, + closeDecoder(playerId: number, decoderId: number) { + if (engineRef.value) { + engineRef.value.closeDecoder(playerId, decoderId) + } + }, + decode( + playerId: number, + decoderId: number, + metadataWasmJson: number, + dataOffset: number, + dataLen: number, + ) { + if (engineRef.value) { + engineRef.value.decode(playerId, decoderId, metadataWasmJson, dataOffset, dataLen) + } + }, + notifyEos(playerId: number) { + if (engineRef.value) { + engineRef.value.notifyEos(playerId) + } + }, + }, + } + const wasmResults = await WebAssembly.instantiateStreaming( + fetch(`data:application/wasm;base64,${WASM_BASE64}`), + importObject, + ) + + const stream = new Mp4MediaStream(wasmResults.instance) + engineRef.value = stream.engine + + const mp4Bytes = new Uint8Array(await mp4.arrayBuffer()) + await stream.engine.loadMp4(mp4Bytes) + + return stream + } + + play(options: Mp4MediaStreamPlayOptions = {}): MediaStream { + return this.engine.play(options) + } +} + +type Mp4Info = { + audioConfigs: [AudioDecoderConfig] + videoConfigs: [VideoDecoderConfig] +} + +class Engine { + private wasm: WebAssembly.Instance + private memory: WebAssembly.Memory + private engine: number + private info?: Mp4Info + private players: Map = new Map() + private nextPlayerId = 0 + + constructor(wasm: WebAssembly.Instance, memory: WebAssembly.Memory) { + this.wasm = wasm + this.memory = memory + this.engine = (this.wasm.exports.newEngine as CallableFunction)() + } + + // TODO: load() でいいかも + async loadMp4(mp4Bytes: Uint8Array): Promise<{ audio: boolean; video: boolean }> { + const mp4WasmBytes = this.toWasmBytes(mp4Bytes) + const resultWasmJson = (this.wasm.exports.loadMp4 as CallableFunction)( + this.engine, + mp4WasmBytes, + ) + + // MP4 内に含まれる映像・音声を WebCodecs のデコーダー扱えるかどうかをチェックする + const info = this.wasmResultToValue(resultWasmJson) as { + audioConfigs: [AudioDecoderConfig] + videoConfigs: [VideoDecoderConfig] + } + for (const config of info.audioConfigs) { + if (!(await AudioDecoder.isConfigSupported(config)).supported) { + // TODO: ちゃんとする + throw 'unsupported audio codec' + } + } + for (const config of info.videoConfigs) { + if (config.description !== undefined) { + // @ts-ignore TS2488: TODO + config.description = new Uint8Array(config.description) + } + if (!(await VideoDecoder.isConfigSupported(config)).supported) { + // TODO: ちゃんとする + throw 'unsupported audio codec' + } + this.info = info + } + + // TODO: remove + console.log(JSON.stringify(info)) + + return { audio: info.audioConfigs.length > 0, video: info.videoConfigs.length > 0 } + } + + play(options: Mp4MediaStreamPlayOptions = {}): MediaStream { + if (this.info === undefined) { + throw 'bug' + } + + const playerId = this.nextPlayerId + this.nextPlayerId += 1 + + const player = new Player(this.info.audioConfigs.length > 0, this.info.videoConfigs.length > 0) + this.players.set(playerId, player) + ;(this.wasm.exports.play as CallableFunction)( + this.engine, + playerId, + this.valueToWasmJson(options), + ) + console.log('player created') + + return player.createMediaStream() + } + + stop(playerId: number) { + console.log('stop player') + const player = this.players.get(playerId) + if (player === undefined) { + return + } + ;(this.wasm.exports.stop as CallableFunction)(this.engine, playerId) + this.closeDecoder(playerId, 0) + this.closeDecoder(playerId, 1) + this.players.delete(playerId) + } + + consoleLog(messageWasmJson: number) { + const message = this.wasmJsonToValue(messageWasmJson) + console.log(message) + } + + async sleep(resultTx: number, duration: number) { + setTimeout(() => { + ;(this.wasm.exports.awake as CallableFunction)(this.engine, resultTx) + }, duration) + } + + createVideoDecoder(playerId: number, configWasmJson: number) { + console.log('createVideoDecoder') + const player = this.players.get(playerId) + if (player === undefined) { + throw 'TODO-1' + } + if (player.videoDecoder !== undefined) { + throw 'TODO-2' + } + + const config = this.wasmJsonToValue(configWasmJson) as VideoDecoderConfig + // @ts-ignore TS2488: TODO + config.description = new Uint8Array(config.description) + + const init = { + output: async (frame: VideoFrame) => { + if (player.videoWriter === undefined) { + throw 'bug' + } + + // TODO: error handling (stop engine) + try { + await player.videoWriter.write(frame) + } catch (e) { + this.stop(playerId) + throw e // TODO + } + }, + error: (error: DOMException) => { + // TODO: ちゃんとしたエラーハンドリング + throw error + }, + } + + player.videoDecoder = new VideoDecoder(init) + player.videoDecoder.configure(config) + console.log('video decoder created') + } + + createAudioDecoder(playerId: number, configWasmJson: number) { + console.log('createAudioDecoder') + const player = this.players.get(playerId) + if (player === undefined) { + throw 'TODO-3' + } + if (player.audioDecoder !== undefined) { + throw 'TODO-4' + } + + const config = this.wasmJsonToValue(configWasmJson) as AudioDecoderConfig + const init = { + output: async (data: AudioData) => { + if (player.audioWriter === undefined) { + throw 'bug' + } + + try { + await player.audioWriter.write(data) + } catch (e) { + this.stop(playerId) + throw e // TODO: エラーの種類によって処理を分ける（closed なら正常系） + } + }, + error: (error: DOMException) => { + // TODO: ちゃんとしたエラーハンドリング + throw error + }, + } + + player.audioDecoder = new AudioDecoder(init) + player.audioDecoder.configure(config) + console.log('audio decoder created') + } + + async closeDecoder(playerId: number, decoderId: number) { + console.log('close decoder') + const player = this.players.get(playerId) + if (player === undefined) { + return + } + + if ( + decoderId === 0 && + player.videoDecoder !== undefined && + player.videoWriter !== undefined && + player.videoDecoder.state !== 'closed' + ) { + await player.videoDecoder.flush() + player.videoDecoder.close() + player.videoDecoder = undefined + player.videoWriter.close() + player.videoWriter = undefined + } else if ( + player.audioDecoder !== undefined && + player.audioWriter !== undefined && + player.audioDecoder.state !== 'closed' + ) { + await player.audioDecoder.flush() + player.audioDecoder.close() + player.audioDecoder = undefined + player.audioWriter.close() + player.audioWriter = undefined + } + } + + async notifyEos(playerId: number) { + this.stop(playerId) + } + + decode( + playerId: number, + decoderId: number, + metadataWasmJson: number, + dataOffset: number, + dataLen: number, + ) { + const player = this.players.get(playerId) + if (player === undefined) { + throw 'TODO-6' + } + + const chunkParams = this.wasmJsonToValue(metadataWasmJson) as + | EncodedAudioChunkInit + | EncodedVideoChunkInit + const data = new Uint8Array(this.memory.buffer, dataOffset, dataLen).slice() + + chunkParams.data = data + // @ts-ignore TS2488: TODO + chunkParams.transfer = [data.buffer] + if (decoderId === 0) { + if (player.videoDecoder === undefined) { + throw 'TODO-7' + } + const chunk = new EncodedVideoChunk(chunkParams) + player.videoDecoder.decode(chunk) + } else { + if (player.audioDecoder === undefined) { + throw 'TODO-8' + } + const chunk = new EncodedAudioChunk(chunkParams) + player.audioDecoder.decode(chunk) + } + } + + private wasmJsonToValue(wasmJson: number): object { + const offset = (this.wasm.exports.vecOffset as CallableFunction)(wasmJson) + const len = (this.wasm.exports.vecLen as CallableFunction)(wasmJson) + const buffer = new Uint8Array(this.memory.buffer, offset, len) + const value = JSON.parse(new TextDecoder('utf-8').decode(buffer)) + + // Wasm 側で所有権は放棄されているので、解放するのは呼び出し側の責務 + ;(this.wasm.exports.freeVec as CallableFunction)(wasmJson) + + return value + } + + private wasmResultToValue(wasmResult: number): object { + const result = this.wasmJsonToValue(wasmResult) as { Ok?: object; Err?: object } + if (result.Err !== undefined) { + // TODO: error handling + throw JSON.stringify(result.Err) + } + return result.Ok as object + } + + private valueToWasmJson(value: object): number { + const jsonBytes = new TextEncoder().encode(JSON.stringify(value)) + return this.toWasmBytes(jsonBytes) + } + + private toWasmBytes(bytes: Uint8Array): number { + // ここで割り当てられたメモリ領域を解放するのは Wasm 側の責務 + const wasmBytes = (this.wasm.exports.allocateVec as CallableFunction)(bytes.length) + const wasmBytesOffset = (this.wasm.exports.vecOffset as CallableFunction)(wasmBytes) + new Uint8Array(this.memory.buffer, wasmBytesOffset, bytes.length).set(bytes) + return wasmBytes + } +} + +class Player { + audio: boolean + video: boolean + audioDecoder?: AudioDecoder + videoDecoder?: VideoDecoder + audioWriter?: WritableStreamDefaultWriter + videoWriter?: WritableStreamDefaultWriter + + constructor(audio: boolean, video: boolean) { + this.audio = audio + this.video = video + } + + createMediaStream(): MediaStream { + const tracks = [] + if (this.audio) { + const generator = new MediaStreamTrackGenerator({ kind: 'audio' }) + tracks.push(generator) + this.audioWriter = generator.writable.getWriter() + } + if (this.video) { + const generator = new MediaStreamTrackGenerator({ kind: 'video' }) + tracks.push(generator) + this.videoWriter = generator.writable.getWriter() + } + return new MediaStream(tracks) + } +} + +export { Mp4MediaStream, type Mp4MediaStreamPlayOptions } diff --git a/packages/mp4-media-stream/tsconfig.json b/packages/mp4-media-stream/tsconfig.json new file mode 100644 index 00000000..f93be613 --- /dev/null +++ b/packages/mp4-media-stream/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "lib": [ + "es2020", + "dom" + ], + "target": "esnext", + "module": "commonjs", + "outDir": "dist", + "strict": true, + "declaration": true, + "strictNullChecks": true, + "importHelpers": true, + "experimentalDecorators": true, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "newLine": "LF" + }, + "include": [ + "src" + ] +} diff --git a/packages/mp4-media-stream/typedoc.json b/packages/mp4-media-stream/typedoc.json new file mode 100644 index 00000000..7793858b --- /dev/null +++ b/packages/mp4-media-stream/typedoc.json @@ -0,0 +1,12 @@ +{ + "entryPoints": [ + "src/mp4_media_stream.ts" + ], + "tsconfig": "./tsconfig.json", + "disableSources": true, + "excludePrivate": true, + "excludeProtected": true, + "excludeInternal": true, + "readme": "./README.md", + "out": "apidoc" +} diff --git a/packages/mp4-media-stream/wasm/Cargo.toml b/packages/mp4-media-stream/wasm/Cargo.toml new file mode 100644 index 00000000..3a554db8 --- /dev/null +++ b/packages/mp4-media-stream/wasm/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "mp4_media_stream" +edition = "2021" +publish = false + +# media-processors の内部でのみ使われる crate なのでバージョンは固定 +version = "0.0.0" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +futures = "0.3.31" +orfail = { version = "1.1.0", features = ["serde"] } +serde = { version = "1.0.210", features = ["derive"] } +serde_json = "1.0.128" +shiguredo_mp4 = "2024.3.0" diff --git a/packages/mp4-media-stream/wasm/src/engine.rs b/packages/mp4-media-stream/wasm/src/engine.rs new file mode 100644 index 00000000..987a7753 --- /dev/null +++ b/packages/mp4-media-stream/wasm/src/engine.rs @@ -0,0 +1,83 @@ +use std::{collections::HashMap, rc::Rc, time::Duration}; + +use futures::{executor::LocalPool, future::RemoteHandle, task::LocalSpawnExt}; +use orfail::OrFail; + +use crate::{ + mp4::{Mp4, Mp4Info, Track}, + player::{PlayOptions, Player, PlayerId, TrackPlayer}, + wasm::WasmApi, +}; + +#[derive(Debug)] +pub struct Engine { + mp4_bytes: Rc>, + tracks: Vec, + executor: LocalPool, + executing: bool, + players: HashMap>, +} + +impl Engine { + #[expect(clippy::new_without_default)] + pub fn new() -> Self { + Self { + mp4_bytes: Rc::new(Vec::new()), + tracks: Vec::new(), + executor: LocalPool::new(), + executing: false, + players: HashMap::new(), + } + } + + pub fn load_mp4(&mut self, mp4_bytes: Vec) -> orfail::Result { + (self.tracks.is_empty()).or_fail()?; + + let mp4 = Mp4::load(&mp4_bytes).or_fail()?; + self.mp4_bytes = Rc::new(mp4_bytes); + self.tracks = mp4.tracks; + + Ok(mp4.info) + } + + pub fn play(&mut self, player_id: PlayerId, options: PlayOptions) { + // MP4 はロード済みであるのが前提 + assert!(!self.tracks.is_empty()); + + let player = Player { + player_id, + start_time: WasmApi::now(), + mp4_bytes: Rc::clone(&self.mp4_bytes), + repeat: options.repeat, + timestamp_offset: Duration::ZERO, + tracks: self + .tracks + .iter() + .map(|t| TrackPlayer::new(player_id, t)) + .collect::>() + .expect("unreachable"), + }; + self.players.insert( + player_id, + self.executor + .spawner() + .spawn_local_with_handle(player.run()) + .expect("unreachable"), + ); + self.poll(); + } + + pub fn stop(&mut self, player_id: PlayerId) { + let _ = self.players.remove(&player_id); + self.poll(); + } + + pub fn poll(&mut self) { + if self.executing { + return; + } + self.executing = true; + self.executor.run_until_stalled(); + self.executing = false; + } +} diff --git a/packages/mp4-media-stream/wasm/src/lib.rs b/packages/mp4-media-stream/wasm/src/lib.rs new file mode 100644 index 00000000..d2ee3ee8 --- /dev/null +++ b/packages/mp4-media-stream/wasm/src/lib.rs @@ -0,0 +1,4 @@ +pub mod engine; +pub mod mp4; +pub mod player; +pub mod wasm; diff --git a/packages/mp4-media-stream/wasm/src/mp4.rs b/packages/mp4-media-stream/wasm/src/mp4.rs new file mode 100644 index 00000000..2577555c --- /dev/null +++ b/packages/mp4-media-stream/wasm/src/mp4.rs @@ -0,0 +1,175 @@ +use std::{collections::HashSet, num::NonZeroU32, rc::Rc}; + +use orfail::{Failure, OrFail}; +use serde::Serialize; +use shiguredo_mp4::{ + aux::SampleTableAccessor, + boxes::{ + Avc1Box, FtypBox, HdlrBox, IgnoredBox, MoovBox, OpusBox, SampleEntry, StblBox, TrakBox, + }, + BaseBox, Decode, Either, Encode, +}; + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct VideoDecoderConfig { + pub codec: String, + pub description: Vec, + pub coded_width: u16, + pub coded_height: u16, +} + +impl VideoDecoderConfig { + pub fn from_avc1_box(b: &Avc1Box) -> Self { + let mut description = Vec::new(); + b.avcc_box.encode(&mut description).expect("unreachable"); + description.drain(..8); // ボックスヘッダ部分を取り除く + + Self { + codec: format!( + "avc1.{:02x}{:02x}{:02x}", + b.avcc_box.avc_profile_indication, + b.avcc_box.profile_compatibility, + b.avcc_box.avc_level_indication + ), + description, + coded_width: b.visual.width, + coded_height: b.visual.height, + } + } +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AudioDecoderConfig { + pub codec: String, + pub sample_rate: u16, + pub number_of_channels: u8, +} + +impl AudioDecoderConfig { + pub fn from_opus_box(b: &OpusBox) -> Self { + Self { + codec: "opus".to_owned(), + sample_rate: b.audio.samplerate.integer, + number_of_channels: b.audio.channelcount as u8, + } + } +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Mp4Info { + pub audio_configs: Vec, + pub video_configs: Vec, +} + +#[derive(Debug, Clone)] +pub struct Track { + pub sample_table: Rc>, + pub timescale: NonZeroU32, +} + +impl Track { + pub fn new(trak_box: TrakBox) -> orfail::Result { + let timescale = trak_box.mdia_box.mdhd_box.timescale; + let sample_table = + SampleTableAccessor::new(trak_box.mdia_box.minf_box.stbl_box).or_fail()?; + (sample_table.sample_count() > 0).or_fail()?; + + match sample_table.stbl_box().stsd_box.entries.first() { + Some(SampleEntry::Avc1(_)) => (), + Some(SampleEntry::Opus(_)) => (), + Some(b) => return Err(Failure::new(format!("Unsupported codec: {}", b.box_type()))), + None => { + // MP4 デコード時にチェック済みなので、ここには来ない + unreachable!() + } + }; + Ok(Self { + sample_table: Rc::new(sample_table), + timescale, + }) + } +} + +#[derive(Debug)] +pub struct Mp4 { + pub info: Mp4Info, + pub tracks: Vec, +} + +impl Mp4 { + pub fn load(mp4_bytes: &[u8]) -> orfail::Result { + let moov_box = Self::load_moov_box(mp4_bytes).or_fail()?; + + let mut audio_track_count = 0; + let mut video_track_count = 0; + let mut tracks = Vec::new(); + for trak_box in moov_box.trak_boxes { + if trak_box.mdia_box.hdlr_box.handler_type == HdlrBox::HANDLER_TYPE_SOUN { + audio_track_count += 1; + } else if trak_box.mdia_box.hdlr_box.handler_type == HdlrBox::HANDLER_TYPE_VIDE { + video_track_count += 1; + } + + tracks.push(Track::new(trak_box).or_fail()?); + } + (!tracks.is_empty()).or_fail_with(|()| "No video or audio tracks found".to_owned())?; + (audio_track_count <= 1) + .or_fail_with(|()| "Unsupported: multiple audio tracks".to_owned())?; + (video_track_count <= 1) + .or_fail_with(|()| "Unsupported: multiple video tracks".to_owned())?; + + Ok(Self { + info: Self::get_mp4_info(&tracks), + tracks, + }) + } + + fn load_moov_box(mut reader: &[u8]) -> orfail::Result { + FtypBox::decode(&mut reader).or_fail()?; + loop { + if reader.is_empty() { + return Err(Failure::new("No 'moov' box found")); + } + if let Either::A(moov_box) = + IgnoredBox::decode_or_ignore(&mut reader, |ty| ty == MoovBox::TYPE).or_fail()? + { + return Ok(moov_box); + } + } + } + + fn get_mp4_info(tracks: &[Track]) -> Mp4Info { + let mut audio_configs = Vec::new(); + let mut video_configs = Vec::new(); + let mut known_sample_entries = HashSet::new(); + for track in tracks { + for chunk in track.sample_table.chunks() { + if known_sample_entries.contains(chunk.sample_entry()) { + continue; + } + known_sample_entries.insert(chunk.sample_entry().clone()); + + match chunk.sample_entry() { + SampleEntry::Avc1(b) => { + video_configs.push(VideoDecoderConfig::from_avc1_box(b)); + } + SampleEntry::Opus(b) => { + audio_configs.push(AudioDecoderConfig::from_opus_box(b)); + } + _ => { + // `Track` 作成時にチェックしているのでここには来ない + unreachable!() + } + } + } + } + + Mp4Info { + audio_configs, + video_configs, + } + } +} diff --git a/packages/mp4-media-stream/wasm/src/player.rs b/packages/mp4-media-stream/wasm/src/player.rs new file mode 100644 index 00000000..3c5ebed6 --- /dev/null +++ b/packages/mp4-media-stream/wasm/src/player.rs @@ -0,0 +1,168 @@ +use std::{num::NonZeroU32, rc::Rc, time::Duration}; + +use serde::Deserialize; +use shiguredo_mp4::{ + aux::{SampleAccessor, SampleTableAccessor}, + boxes::{SampleEntry, StblBox}, +}; + +use crate::{ + mp4::{AudioDecoderConfig, Track, VideoDecoderConfig}, + wasm::{DecoderId, WasmApi}, +}; + +pub type PlayerId = u32; + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PlayOptions { + #[serde(default)] + pub repeat: bool, +} + +#[derive(Debug)] +pub struct Player { + pub player_id: PlayerId, + pub start_time: Duration, + pub mp4_bytes: Rc>, + pub tracks: Vec, + pub repeat: bool, + pub timestamp_offset: Duration, +} + +impl Player { + fn now(&self) -> Duration { + WasmApi::now().saturating_sub(self.start_time) + } + + fn next_timestamp(&self) -> Duration { + // TODO: repeat を考慮する + self.tracks + .iter() + .map(|t| t.current_position()) + .min() + .expect("unreachable") + } + + pub async fn run(mut self) { + loop { + // TODO: 数フレーム分は先読みしてデコードしておく + while self.run_one() { + // TODO: sleep until next sample timestamp + + // 次のサンプルのタイムスタンプまで待つ + // TODO: 遅れている場合にはフレームスキップをするかも + let now = self.now(); + let wait_duration = self.next_timestamp().saturating_sub(now); + WasmApi::sleep(wait_duration).await; + } + if !self.repeat { + break; + } + + self.timestamp_offset += self.next_timestamp(); // TODO: duration を足すべき？ + self.start_time = WasmApi::now(); + for track in &mut self.tracks { + track.current_sample_index = NonZeroU32::MIN; + } + } + + WasmApi::notify_eos(self.player_id); + } + + fn run_one(&mut self) -> bool { + // TODO: sample entry が変わったらデコーダを作り直す or configure() を呼び直す + let now = self.now(); + + for track in &mut self.tracks { + if track.decoder.is_none() { + let decoder = match track.sample_table.stbl_box().stsd_box.entries.first() { + Some(SampleEntry::Avc1(b)) => { + let config = VideoDecoderConfig::from_avc1_box(b); + WasmApi::create_video_decoder(self.player_id, config) + } + Some(SampleEntry::Opus(b)) => { + let config = AudioDecoderConfig::from_opus_box(b); + WasmApi::create_audio_decoder(self.player_id, config) + } + _ => unreachable!(), + }; + track.decoder = Some(decoder); + }; + } + + for track in &self.tracks { + if now < track.current_position() { + continue; + } + self.render_sample(track); + } + + for track in &mut self.tracks { + if now < track.current_position() { + continue; + } + + track.current_sample_index = track.current_sample_index.saturating_add(1); + if track.current_sample_index.get() + 1 == track.sample_table.sample_count() { + // TODO: 尺が長い方に合わせる + return false; + } + } + + true + } + + fn render_sample(&self, track: &TrackPlayer) { + let sample = track.current_sample(); + let data = &self.mp4_bytes[sample.data_offset() as usize..][..sample.data_size() as usize]; + WasmApi::render( + self.player_id, + track.decoder.expect("unreachable"), + track.timescale, + self.timestamp_offset, + sample, + data, + ); + } +} + +#[derive(Debug, Clone)] +pub struct TrackPlayer { + player_id: PlayerId, + sample_table: Rc>, + decoder: Option, + timescale: NonZeroU32, + current_sample_index: NonZeroU32, +} + +impl TrackPlayer { + pub fn new(player_id: PlayerId, track: &Track) -> orfail::Result { + Ok(TrackPlayer { + player_id, + sample_table: track.sample_table.clone(), + decoder: None, + timescale: track.timescale, + current_sample_index: NonZeroU32::MIN, + }) + } + + fn current_sample(&self) -> SampleAccessor { + self.sample_table + .get_sample(self.current_sample_index) + .expect("unreachable") + } + + fn current_position(&self) -> Duration { + let sample = self.current_sample(); + Duration::from_secs(sample.timestamp()) / self.timescale.get() + } +} + +impl Drop for TrackPlayer { + fn drop(&mut self) { + if let Some(decoder) = self.decoder { + WasmApi::close_decoder(self.player_id, decoder); + } + } +} diff --git a/packages/mp4-media-stream/wasm/src/wasm.rs b/packages/mp4-media-stream/wasm/src/wasm.rs new file mode 100644 index 00000000..93a065db --- /dev/null +++ b/packages/mp4-media-stream/wasm/src/wasm.rs @@ -0,0 +1,227 @@ +use std::{marker::PhantomData, num::NonZeroU32, time::Duration}; + +use futures::{channel::oneshot, TryFutureExt}; +use orfail::OrFail; +use serde::{Deserialize, Serialize}; +use shiguredo_mp4::{aux::SampleAccessor, boxes::StblBox}; + +use crate::{ + engine::Engine, + mp4::{AudioDecoderConfig, Mp4Info, VideoDecoderConfig}, + player::{PlayOptions, PlayerId}, +}; + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct EncodedChunkMetadata { + #[serde(rename = "type")] + pub ty: &'static str, // key | delta + pub timestamp: u32, // millis + pub duration: u32, // millis +} + +#[derive(Debug)] +pub struct WasmApi; + +impl WasmApi { + pub fn now() -> Duration { + Duration::from_secs_f64(unsafe { now() / 1000.0 }) + } + + pub async fn sleep(duration: Duration) { + let (tx, rx) = oneshot::channel::<()>(); + unsafe { sleep(Box::into_raw(Box::new(tx)), duration.as_millis() as u32) }; + rx.unwrap_or_else(|_| ()).await; + } + + pub fn create_video_decoder(player_id: PlayerId, config: VideoDecoderConfig) -> DecoderId { + unsafe { + createVideoDecoder(player_id, JsonVec::new(config)); + } + + // TODO: 結果を返さなくする + 0 + } + + pub fn create_audio_decoder(player_id: PlayerId, config: AudioDecoderConfig) -> DecoderId { + unsafe { + createAudioDecoder(player_id, JsonVec::new(config)); + } + + // TODO: 結果を返さなくする + 1 + } + + pub fn render( + player_id: PlayerId, + decoder: DecoderId, + timescale: NonZeroU32, + timestamp_offset: Duration, + sample: SampleAccessor<'_, StblBox>, + sample_data: &[u8], + ) { + let metadata = EncodedChunkMetadata { + ty: if sample.is_sync_sample() { + "key" + } else { + "delta" + }, + timestamp: (Duration::from_secs(sample.timestamp()) / timescale.get() + + timestamp_offset) + .as_millis() as u32, + duration: (Duration::from_secs(sample.duration() as u64) / timescale.get()).as_millis() + as u32, + }; + unsafe { + decode( + player_id, + decoder, + JsonVec::new(metadata), + sample_data.as_ptr(), + sample_data.len() as u32, + ); + } + } + + pub fn close_decoder(player_id: PlayerId, decoder: DecoderId) { + unsafe { closeDecoder(player_id, decoder) } + } + + pub fn notify_eos(player_id: PlayerId) { + unsafe { notifyEos(player_id) } + } +} + +pub type DecoderId = u32; + +extern "C" { + #[expect(improper_ctypes)] + pub fn consoleLog(messageWasmJson: JsonVec); + + pub fn now() -> f64; + + #[expect(improper_ctypes)] + pub fn sleep(result_tx: *mut oneshot::Sender<()>, duration: u32); + + #[expect(improper_ctypes)] + pub fn decode( + player_id: PlayerId, + decoder: DecoderId, + metadata: JsonVec, + data_ptr: *const u8, + data_len: u32, + ); + + #[expect(improper_ctypes)] + pub fn createVideoDecoder(player_id: PlayerId, config: JsonVec); + + #[expect(improper_ctypes)] + pub fn createAudioDecoder(player_id: PlayerId, config: JsonVec); + + pub fn closeDecoder(player_id: PlayerId, decoder: DecoderId); + + pub fn notifyEos(player_id: PlayerId); +} + +#[no_mangle] +#[expect(clippy::not_unsafe_ptr_arg_deref)] +pub fn awake(engine: *mut Engine, result_tx: *mut oneshot::Sender<()>) { + let is_ok = unsafe { Box::from_raw(result_tx) }.send(()).is_ok(); + if !is_ok { + return; + } + unsafe { &mut *engine }.poll(); +} + +#[no_mangle] +#[expect(non_snake_case)] +pub fn newEngine() -> *mut Engine { + std::panic::set_hook(Box::new(|info| { + let msg = info.to_string(); + unsafe { + consoleLog(JsonVec::new(msg)); + } + })); + + Box::into_raw(Box::new(Engine::new())) +} + +#[no_mangle] +#[expect(non_snake_case, clippy::not_unsafe_ptr_arg_deref)] +pub fn freeEngine(engine: *mut Engine) { + let _ = unsafe { Box::from_raw(engine) }; +} + +#[no_mangle] +#[expect(non_snake_case, clippy::not_unsafe_ptr_arg_deref)] +pub fn loadMp4(engine: *mut Engine, mp4_bytes: *mut Vec) -> JsonVec> { + let engine = unsafe { &mut *engine }; + let result = engine + .load_mp4(*unsafe { Box::from_raw(mp4_bytes) }) + .or_fail(); + JsonVec::new(result) +} + +#[no_mangle] +#[expect(clippy::not_unsafe_ptr_arg_deref)] +pub fn play(engine: *mut Engine, player_id: PlayerId, options: JsonVec) { + let engine = unsafe { &mut *engine }; + let options = unsafe { options.into_value() }; + engine.play(player_id, options); +} + +#[no_mangle] +#[expect(clippy::not_unsafe_ptr_arg_deref)] +pub fn stop(engine: *mut Engine, player_id: PlayerId) { + let engine = unsafe { &mut *engine }; + engine.stop(player_id) +} + +#[no_mangle] +#[expect(non_snake_case, clippy::not_unsafe_ptr_arg_deref)] +pub fn vecOffset(v: *mut Vec) -> *mut u8 { + unsafe { &mut *v }.as_mut_ptr() +} + +#[no_mangle] +#[expect(non_snake_case, clippy::not_unsafe_ptr_arg_deref)] +pub fn vecLen(v: *mut Vec) -> i32 { + unsafe { &*v }.len() as i32 +} + +#[no_mangle] +#[expect(non_snake_case)] +pub fn allocateVec(len: i32) -> *mut Vec { + Box::into_raw(Box::new(vec![0; len as usize])) +} + +#[no_mangle] +#[expect(non_snake_case, clippy::not_unsafe_ptr_arg_deref)] +pub fn freeVec(v: *mut Vec) { + let _ = unsafe { Box::from_raw(v) }; +} + +#[repr(transparent)] +pub struct JsonVec { + bytes: *mut Vec, + _ty: PhantomData, +} + +impl JsonVec { + fn new(value: T) -> Self { + let bytes = Box::into_raw(Box::new(serde_json::to_vec(&value).expect("unreachable"))); + Self { + bytes, + _ty: PhantomData, + } + } +} + +impl Deserialize<'de>> JsonVec { + #[track_caller] + unsafe fn into_value(self) -> T { + let bytes = Box::from_raw(self.bytes); + let value: T = serde_json::from_slice(&bytes).expect("Invalid JSON"); + value + } +} From de41b0cde15e8223cd650cd612893215b049c8ff Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 10:43:07 +0900 Subject: [PATCH 06/34] Add README.md --- packages/mp4-media-stream/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 packages/mp4-media-stream/README.md diff --git a/packages/mp4-media-stream/README.md b/packages/mp4-media-stream/README.md new file mode 100644 index 00000000..8981f705 --- /dev/null +++ b/packages/mp4-media-stream/README.md @@ -0,0 +1,11 @@ +# @shiguredo/mp4-media-stream + +未対応: +- H.264 / Opus 以外のコーデック +- 再生位置指定（シーク） +- 再生の一時中断 +- 再生速度変更 +- ファイルのインクリメンタルな読み込み（メモリ消費量削減） +- edts ボックスのハンドリング +- B フレーム +- Chrome / Edge 以外のブラウザ From 9b40730812e314bba225f74ce51a1be499eb8a91 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 11:07:19 +0900 Subject: [PATCH 07/34] =?UTF-8?q?vite=20=E7=89=88=E3=81=AE=E3=82=B5?= =?UTF-8?q?=E3=83=B3=E3=83=97=E3=83=AB=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/index.html | 3 ++- examples/mp4-media-stream/index.html | 21 +++++++++++++++ examples/mp4-media-stream/main.mts | 39 ++++++++++++++++++++++++++++ examples/package.json | 5 ++-- examples/vite.config.mjs | 3 +++ 5 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 examples/mp4-media-stream/index.html create mode 100644 examples/mp4-media-stream/main.mts diff --git a/examples/index.html b/examples/index.html index 91c5253e..fcffa487 100644 --- a/examples/index.html +++ b/examples/index.html @@ -12,8 +12,9 @@
  • ライト調整サンプル
  • ライト調整GPUサンプル
  • ノイズ抑制サンプル
    • +
  • MP4 メディアストリームサンプル
    • - \ No newline at end of file + diff --git a/examples/mp4-media-stream/index.html b/examples/mp4-media-stream/index.html new file mode 100644 index 00000000..3e6328d1 --- /dev/null +++ b/examples/mp4-media-stream/index.html @@ -0,0 +1,21 @@ + + + + Media Processors: MP4 メディアストリームサンプル + + +

    Media Processors: MP4 メディアストリームサンプル

    + +

    入力 MP4 ファイル

    + + +

    + 繰り返し再生 +
    + +

    出力映像

    + + + + + diff --git a/examples/mp4-media-stream/main.mts b/examples/mp4-media-stream/main.mts new file mode 100644 index 00000000..ac69abcb --- /dev/null +++ b/examples/mp4-media-stream/main.mts @@ -0,0 +1,39 @@ +import { Mp4MediaStream } from '@shiguredo/mp4-media-stream' + +document.addEventListener('DOMContentLoaded', async () => { + let mp4MediaStream + + if (!Mp4MediaStream.isSupported()) { + alert('Unsupported platform') + throw Error('Unsupported platform') + } + + async function play() { + let mp4File = getInputFile() + + if (mp4MediaStream !== undefined) { + mp4MediaStream.stop() + } + mp4MediaStream = await Mp4MediaStream.loadMp4(mp4File) + + const options = { + repeat: document.getElementById('repeat').checked, + } + const stream = mp4MediaStream.play(options) + + let output = document.getElementById('output') + output.srcObject = stream + // TODO: stream.getTracks()[0].stop() + } + + function getInputFile() { + const input = document.getElementById('input') + const files = input.files + if (files === null || files.length === 0) { + return + } + return files[0] + } + + document.getElementById('input').addEventListener('change', play) +}) diff --git a/examples/package.json b/examples/package.json index 4203680d..b3d6fa75 100644 --- a/examples/package.json +++ b/examples/package.json @@ -12,9 +12,10 @@ "@shiguredo/light-adjustment": "workspace:*", "@shiguredo/light-adjustment-gpu": "workspace:*", "@shiguredo/noise-suppression": "workspace:*", - "@shiguredo/virtual-background": "workspace:*" + "@shiguredo/virtual-background": "workspace:*", + "@shiguredo/mp4-media-stream": "workspace:*" }, "devDependencies": { "vite-plugin-static-copy": "1.0.6" } -} \ No newline at end of file +} diff --git a/examples/vite.config.mjs b/examples/vite.config.mjs index 98fd8f88..8193ba6b 100644 --- a/examples/vite.config.mjs +++ b/examples/vite.config.mjs @@ -24,6 +24,7 @@ export default defineConfig({ __dirname, '../packages/light-adjustment-gpu/dist/light_adjustment_gpu.mjs', ), + '@shiguredo/mp4-media-stream': resolve(__dirname, '../packages/mp4-media-stream/dist/mp4_media_stream.mjs'), }, }, optimizeDeps: { @@ -32,6 +33,7 @@ export default defineConfig({ '@shiguredo/noise-suppression', '@shiguredo/light-adjustment', '@shiguredo/light-adjustment-gpu', + '@shiguredo/mp4-media-stream', ], }, build: { @@ -44,6 +46,7 @@ export default defineConfig({ noiseSuppression: resolve(__dirname, 'noise-suppression/index.html'), videoMultiProcessors: resolve(__dirname, 'video-multi-processors/index.html'), videoMultiProcessorsGpu: resolve(__dirname, 'video-multi-processors-gpu/index.html'), + mp4MediaStream: resolve(__dirname, 'mp4-media-stream/index.html'), }, }, }, From 4cae955e0a26201f3306238bb0ab0a9fd4ae342b Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 11:08:27 +0900 Subject: [PATCH 08/34] Update pnpm-lock.yaml --- pnpm-lock.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 492dda75..56c27e6a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -54,6 +54,9 @@ importers: '@shiguredo/light-adjustment-gpu': specifier: workspace:* version: link:../packages/light-adjustment-gpu + '@shiguredo/mp4-media-stream': + specifier: workspace:* + version: link:../packages/mp4-media-stream '@shiguredo/noise-suppression': specifier: workspace:* version: link:../packages/noise-suppression @@ -107,6 +110,8 @@ importers: specifier: workspace:* version: link:../image-to-image-video-processor + packages/mp4-media-stream: {} + packages/noise-suppression: devDependencies: '@shiguredo/rnnoise-wasm': From 7ffc6c86b6bc2ac98c764c0fc72d9d11edd253b1 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 11:46:47 +0900 Subject: [PATCH 09/34] =?UTF-8?q?wasm=20=E3=81=A8=20player=20=E4=BB=A5?= =?UTF-8?q?=E5=A4=96=E3=82=92=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/wasm/src/engine.rs | 19 ++--------- packages/mp4-media-stream/wasm/src/player.rs | 36 +++++++++++++++----- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/packages/mp4-media-stream/wasm/src/engine.rs b/packages/mp4-media-stream/wasm/src/engine.rs index 987a7753..af6b74cc 100644 --- a/packages/mp4-media-stream/wasm/src/engine.rs +++ b/packages/mp4-media-stream/wasm/src/engine.rs @@ -1,12 +1,11 @@ -use std::{collections::HashMap, rc::Rc, time::Duration}; +use std::{collections::HashMap, rc::Rc}; use futures::{executor::LocalPool, future::RemoteHandle, task::LocalSpawnExt}; use orfail::OrFail; use crate::{ mp4::{Mp4, Mp4Info, Track}, - player::{PlayOptions, Player, PlayerId, TrackPlayer}, - wasm::WasmApi, + player::{PlayOptions, Player, PlayerId}, }; #[derive(Debug)] @@ -44,19 +43,7 @@ impl Engine { // MP4 はロード済みであるのが前提 assert!(!self.tracks.is_empty()); - let player = Player { - player_id, - start_time: WasmApi::now(), - mp4_bytes: Rc::clone(&self.mp4_bytes), - repeat: options.repeat, - timestamp_offset: Duration::ZERO, - tracks: self - .tracks - .iter() - .map(|t| TrackPlayer::new(player_id, t)) - .collect::>() - .expect("unreachable"), - }; + let player = Player::new(player_id, options, self.mp4_bytes.clone(), &self.tracks); self.players.insert( player_id, self.executor diff --git a/packages/mp4-media-stream/wasm/src/player.rs b/packages/mp4-media-stream/wasm/src/player.rs index 3c5ebed6..5d64adeb 100644 --- a/packages/mp4-media-stream/wasm/src/player.rs +++ b/packages/mp4-media-stream/wasm/src/player.rs @@ -22,15 +22,35 @@ pub struct PlayOptions { #[derive(Debug)] pub struct Player { - pub player_id: PlayerId, - pub start_time: Duration, - pub mp4_bytes: Rc>, - pub tracks: Vec, - pub repeat: bool, - pub timestamp_offset: Duration, + player_id: PlayerId, + start_time: Duration, + mp4_bytes: Rc>, + tracks: Vec, + repeat: bool, + timestamp_offset: Duration, } impl Player { + pub fn new( + player_id: PlayerId, + options: PlayOptions, + mp4_bytes: Rc>, + tracks: &[Track], + ) -> Self { + Self { + player_id, + start_time: WasmApi::now(), + mp4_bytes, + repeat: options.repeat, + timestamp_offset: Duration::ZERO, + tracks: tracks + .iter() + .map(|t| TrackPlayer::new(player_id, t)) + .collect::>() + .expect("unreachable"), + } + } + fn now(&self) -> Duration { WasmApi::now().saturating_sub(self.start_time) } @@ -128,7 +148,7 @@ impl Player { } #[derive(Debug, Clone)] -pub struct TrackPlayer { +struct TrackPlayer { player_id: PlayerId, sample_table: Rc>, decoder: Option, @@ -137,7 +157,7 @@ pub struct TrackPlayer { } impl TrackPlayer { - pub fn new(player_id: PlayerId, track: &Track) -> orfail::Result { + fn new(player_id: PlayerId, track: &Track) -> orfail::Result { Ok(TrackPlayer { player_id, sample_table: track.sample_table.clone(), From 1626beeb0513cf854ada4877dade4dda0d208ea8 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 11:55:51 +0900 Subject: [PATCH 10/34] =?UTF-8?q?wasm=20=E3=83=A2=E3=82=B8=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E3=83=AB=E3=82=92=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mp4-media-stream/src/mp4_media_stream.ts | 6 ++--- packages/mp4-media-stream/wasm/src/player.rs | 2 +- packages/mp4-media-stream/wasm/src/wasm.rs | 26 +++++++++++-------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index be4e3acf..7670fe56 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -58,9 +58,9 @@ class Mp4MediaStream { engineRef.value.decode(playerId, decoderId, metadataWasmJson, dataOffset, dataLen) } }, - notifyEos(playerId: number) { + onEos(playerId: number) { if (engineRef.value) { - engineRef.value.notifyEos(playerId) + engineRef.value.onEos(playerId) } }, }, @@ -288,7 +288,7 @@ class Engine { } } - async notifyEos(playerId: number) { + async onEos(playerId: number) { this.stop(playerId) } diff --git a/packages/mp4-media-stream/wasm/src/player.rs b/packages/mp4-media-stream/wasm/src/player.rs index 5d64adeb..f5a0260c 100644 --- a/packages/mp4-media-stream/wasm/src/player.rs +++ b/packages/mp4-media-stream/wasm/src/player.rs @@ -136,7 +136,7 @@ impl Player { fn render_sample(&self, track: &TrackPlayer) { let sample = track.current_sample(); let data = &self.mp4_bytes[sample.data_offset() as usize..][..sample.data_size() as usize]; - WasmApi::render( + WasmApi::decode( self.player_id, track.decoder.expect("unreachable"), track.timescale, diff --git a/packages/mp4-media-stream/wasm/src/wasm.rs b/packages/mp4-media-stream/wasm/src/wasm.rs index 93a065db..13735053 100644 --- a/packages/mp4-media-stream/wasm/src/wasm.rs +++ b/packages/mp4-media-stream/wasm/src/wasm.rs @@ -11,6 +11,12 @@ use crate::{ player::{PlayOptions, PlayerId}, }; +pub type DecoderId = u32; + +// 複数の映像・音声トラックに対応するまでは ID はハードコーディングしてしまう +const AUDIO_DECODER_ID: DecoderId = 0; +const VIDEO_DECODER_ID: DecoderId = 1; + #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct EncodedChunkMetadata { @@ -34,25 +40,23 @@ impl WasmApi { rx.unwrap_or_else(|_| ()).await; } + // 事前に TypeScript 側で decoder configuration のチェックを行っているので、これは常に成功する pub fn create_video_decoder(player_id: PlayerId, config: VideoDecoderConfig) -> DecoderId { unsafe { createVideoDecoder(player_id, JsonVec::new(config)); } - - // TODO: 結果を返さなくする - 0 + AUDIO_DECODER_ID } + // 事前に TypeScript 側で decoder configuration のチェックを行っているので、これは常に成功する pub fn create_audio_decoder(player_id: PlayerId, config: AudioDecoderConfig) -> DecoderId { unsafe { createAudioDecoder(player_id, JsonVec::new(config)); } - - // TODO: 結果を返さなくする - 1 + VIDEO_DECODER_ID } - pub fn render( + pub fn decode( player_id: PlayerId, decoder: DecoderId, timescale: NonZeroU32, @@ -87,13 +91,13 @@ impl WasmApi { unsafe { closeDecoder(player_id, decoder) } } + // MP4 の終端に達したことを TypeScript 側に伝える + // (repeat=true の場合にはこれが呼ばれることはない） pub fn notify_eos(player_id: PlayerId) { - unsafe { notifyEos(player_id) } + unsafe { onEos(player_id) } } } -pub type DecoderId = u32; - extern "C" { #[expect(improper_ctypes)] pub fn consoleLog(messageWasmJson: JsonVec); @@ -120,7 +124,7 @@ extern "C" { pub fn closeDecoder(player_id: PlayerId, decoder: DecoderId); - pub fn notifyEos(player_id: PlayerId); + pub fn onEos(player_id: PlayerId); } #[no_mangle] From 50fbf5bd67735182ba9226e5c123bd5c7ff5a804 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 12:07:16 +0900 Subject: [PATCH 11/34] =?UTF-8?q?=E3=81=93=E3=81=BE=E3=81=94=E3=81=BE?= =?UTF-8?q?=E3=81=A8=E3=81=97=E3=81=9F=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/wasm/src/mp4.rs | 13 ++++++++++++- packages/mp4-media-stream/wasm/src/player.rs | 19 ++++++++++--------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/packages/mp4-media-stream/wasm/src/mp4.rs b/packages/mp4-media-stream/wasm/src/mp4.rs index 2577555c..0b2431ab 100644 --- a/packages/mp4-media-stream/wasm/src/mp4.rs +++ b/packages/mp4-media-stream/wasm/src/mp4.rs @@ -80,7 +80,18 @@ impl Track { match sample_table.stbl_box().stsd_box.entries.first() { Some(SampleEntry::Avc1(_)) => (), Some(SampleEntry::Opus(_)) => (), - Some(b) => return Err(Failure::new(format!("Unsupported codec: {}", b.box_type()))), + Some(b) => { + let kind = match trak_box.mdia_box.hdlr_box.handler_type { + HdlrBox::HANDLER_TYPE_SOUN => "audio ", + HdlrBox::HANDLER_TYPE_VIDE => "video ", + _ => "", + }; + return Err(Failure::new(format!( + "Unsupported {}codec: {}", + kind, + b.box_type() + ))); + } None => { // MP4 デコード時にチェック済みなので、ここには来ない unreachable!() diff --git a/packages/mp4-media-stream/wasm/src/player.rs b/packages/mp4-media-stream/wasm/src/player.rs index f5a0260c..2bdc2129 100644 --- a/packages/mp4-media-stream/wasm/src/player.rs +++ b/packages/mp4-media-stream/wasm/src/player.rs @@ -45,13 +45,14 @@ impl Player { timestamp_offset: Duration::ZERO, tracks: tracks .iter() - .map(|t| TrackPlayer::new(player_id, t)) + .map(|track| TrackPlayer::new(player_id, track)) .collect::>() .expect("unreachable"), } } - fn now(&self) -> Duration { + // 再生開始からの経過時間を返す + fn elapsed(&self) -> Duration { WasmApi::now().saturating_sub(self.start_time) } @@ -59,7 +60,7 @@ impl Player { // TODO: repeat を考慮する self.tracks .iter() - .map(|t| t.current_position()) + .map(|t| t.current_timestamp()) .min() .expect("unreachable") } @@ -72,8 +73,8 @@ impl Player { // 次のサンプルのタイムスタンプまで待つ // TODO: 遅れている場合にはフレームスキップをするかも - let now = self.now(); - let wait_duration = self.next_timestamp().saturating_sub(now); + let elapsed = self.elapsed(); + let wait_duration = self.next_timestamp().saturating_sub(elapsed); WasmApi::sleep(wait_duration).await; } if !self.repeat { @@ -92,7 +93,7 @@ impl Player { fn run_one(&mut self) -> bool { // TODO: sample entry が変わったらデコーダを作り直す or configure() を呼び直す - let now = self.now(); + let now = self.elapsed(); for track in &mut self.tracks { if track.decoder.is_none() { @@ -112,14 +113,14 @@ impl Player { } for track in &self.tracks { - if now < track.current_position() { + if now < track.current_timestamp() { continue; } self.render_sample(track); } for track in &mut self.tracks { - if now < track.current_position() { + if now < track.current_timestamp() { continue; } @@ -173,7 +174,7 @@ impl TrackPlayer { .expect("unreachable") } - fn current_position(&self) -> Duration { + fn current_timestamp(&self) -> Duration { let sample = self.current_sample(); Duration::from_secs(sample.timestamp()) / self.timescale.get() } From 37bb326d900beb3c075704ae8883194d81857541 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 12:32:12 +0900 Subject: [PATCH 12/34] =?UTF-8?q?=E3=82=B5=E3=83=B3=E3=83=97=E3=83=AB?= =?UTF-8?q?=E3=82=A8=E3=83=B3=E3=83=88=E3=83=AA=E3=83=BC=E3=81=8C=E9=80=94?= =?UTF-8?q?=E4=B8=AD=E3=81=A7=E5=A4=89=E3=82=8F=E3=81=A3=E3=81=9F=E5=A0=B4?= =?UTF-8?q?=E5=90=88=E3=81=AB=E3=81=AF=E3=83=87=E3=82=B3=E3=83=BC=E3=83=80?= =?UTF-8?q?=E3=82=82=E4=BD=9C=E3=82=8A=E7=9B=B4=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/wasm/src/player.rs | 64 ++++++++++++++------ 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/packages/mp4-media-stream/wasm/src/player.rs b/packages/mp4-media-stream/wasm/src/player.rs index 2bdc2129..98f42bb7 100644 --- a/packages/mp4-media-stream/wasm/src/player.rs +++ b/packages/mp4-media-stream/wasm/src/player.rs @@ -73,8 +73,8 @@ impl Player { // 次のサンプルのタイムスタンプまで待つ // TODO: 遅れている場合にはフレームスキップをするかも - let elapsed = self.elapsed(); - let wait_duration = self.next_timestamp().saturating_sub(elapsed); + let now = self.elapsed(); + let wait_duration = self.next_timestamp().saturating_sub(now); WasmApi::sleep(wait_duration).await; } if !self.repeat { @@ -92,24 +92,10 @@ impl Player { } fn run_one(&mut self) -> bool { - // TODO: sample entry が変わったらデコーダを作り直す or configure() を呼び直す let now = self.elapsed(); for track in &mut self.tracks { - if track.decoder.is_none() { - let decoder = match track.sample_table.stbl_box().stsd_box.entries.first() { - Some(SampleEntry::Avc1(b)) => { - let config = VideoDecoderConfig::from_avc1_box(b); - WasmApi::create_video_decoder(self.player_id, config) - } - Some(SampleEntry::Opus(b)) => { - let config = AudioDecoderConfig::from_opus_box(b); - WasmApi::create_audio_decoder(self.player_id, config) - } - _ => unreachable!(), - }; - track.decoder = Some(decoder); - }; + track.maybe_setup_decoder(); } for track in &self.tracks { @@ -137,6 +123,8 @@ impl Player { fn render_sample(&self, track: &TrackPlayer) { let sample = track.current_sample(); let data = &self.mp4_bytes[sample.data_offset() as usize..][..sample.data_size() as usize]; + + // Rust 側で関与するのはデコードまでで、その先の処理は TypeScript 側で行われる WasmApi::decode( self.player_id, track.decoder.expect("unreachable"), @@ -168,6 +156,48 @@ impl TrackPlayer { }) } + fn maybe_setup_decoder(&mut self) { + if self.decoder.is_some() && !self.is_sample_entry_changed() { + return; + } + + if let Some(decoder) = self.decoder { + WasmApi::close_decoder(self.player_id, decoder); + } + + let decoder = match self.current_sample().chunk().sample_entry() { + SampleEntry::Avc1(b) => { + let config = VideoDecoderConfig::from_avc1_box(b); + WasmApi::create_video_decoder(self.player_id, config) + } + SampleEntry::Opus(b) => { + let config = AudioDecoderConfig::from_opus_box(b); + WasmApi::create_audio_decoder(self.player_id, config) + } + _ => { + // MP4::load() の中で非対応コーデックのチェックは行っているので、ここに来ることはない + unreachable!() + } + }; + self.decoder = Some(decoder); + } + + fn is_sample_entry_changed(&self) -> bool { + // `Mp4::load()` の中で空サンプルがないことは確認済みなので、以降の処理が失敗することはない + let prev_sample_index = NonZeroU32::new(self.current_sample_index.get() - 1) + .or_else(|| self.sample_table.samples().last().map(|s| s.index())) + .expect("unreachable"); + let prev_sample = self + .sample_table + .get_sample(prev_sample_index) + .expect("unreachable"); + let current_sample = self + .sample_table + .get_sample(self.current_sample_index) + .expect("unreachable"); + prev_sample.chunk().sample_entry() != current_sample.chunk().sample_entry() + } + fn current_sample(&self) -> SampleAccessor { self.sample_table .get_sample(self.current_sample_index) From 7ba49240f8343d193e734d617a7bc9fe0f0c4512 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 12:50:49 +0900 Subject: [PATCH 13/34] =?UTF-8?q?=E3=83=87=E3=82=B3=E3=83=BC=E3=83=80?= =?UTF-8?q?=E3=83=BC=E3=81=AE=E9=9A=9B=E4=BD=9C=E6=88=90=E6=99=82=E3=81=AB?= =?UTF-8?q?=E3=81=AF=E3=80=81=E4=B8=80=E3=81=A4=E5=89=8D=E3=81=AE=E3=83=87?= =?UTF-8?q?=E3=82=B3=E3=83=BC=E3=83=80=E3=83=BC=E3=81=AE=E7=B5=82=E4=BA=86?= =?UTF-8?q?=E3=81=8C=E5=AE=8C=E4=BA=86=E3=81=99=E3=82=8B=E3=81=AE=E3=82=92?= =?UTF-8?q?=E5=BE=85=E6=A9=9F=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mp4-media-stream/src/mp4_media_stream.ts | 19 ++++++++++++------- packages/mp4-media-stream/wasm/src/wasm.rs | 8 ++++---- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index 7670fe56..5002a06e 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -4,6 +4,9 @@ interface Mp4MediaStreamPlayOptions { repeat?: boolean } +const AUDIO_DECODER_ID: number = 0 +const VIDEO_DECODER_ID: number = 1 + class Mp4MediaStream { private engine: Engine @@ -183,14 +186,15 @@ class Engine { }, duration) } - createVideoDecoder(playerId: number, configWasmJson: number) { + async createVideoDecoder(playerId: number, configWasmJson: number) { console.log('createVideoDecoder') const player = this.players.get(playerId) if (player === undefined) { throw 'TODO-1' } - if (player.videoDecoder !== undefined) { - throw 'TODO-2' + if (player.videoWriter !== undefined) { + // 一つ前のデコーダーの終了処理が進行中の場合には、それを待機する + await player.videoWriter.closed } const config = this.wasmJsonToValue(configWasmJson) as VideoDecoderConfig @@ -222,14 +226,15 @@ class Engine { console.log('video decoder created') } - createAudioDecoder(playerId: number, configWasmJson: number) { + async createAudioDecoder(playerId: number, configWasmJson: number) { console.log('createAudioDecoder') const player = this.players.get(playerId) if (player === undefined) { throw 'TODO-3' } - if (player.audioDecoder !== undefined) { - throw 'TODO-4' + if (player.audioWriter !== undefined) { + // 一つ前のデコーダーの終了処理が進行中の場合には、それを待機する + await player.audioWriter.closed } const config = this.wasmJsonToValue(configWasmJson) as AudioDecoderConfig @@ -265,7 +270,7 @@ class Engine { } if ( - decoderId === 0 && + decoderId === VIDEO_DECODER_ID && player.videoDecoder !== undefined && player.videoWriter !== undefined && player.videoDecoder.state !== 'closed' diff --git a/packages/mp4-media-stream/wasm/src/wasm.rs b/packages/mp4-media-stream/wasm/src/wasm.rs index 13735053..7ec82965 100644 --- a/packages/mp4-media-stream/wasm/src/wasm.rs +++ b/packages/mp4-media-stream/wasm/src/wasm.rs @@ -41,17 +41,17 @@ impl WasmApi { } // 事前に TypeScript 側で decoder configuration のチェックを行っているので、これは常に成功する - pub fn create_video_decoder(player_id: PlayerId, config: VideoDecoderConfig) -> DecoderId { + pub fn create_audio_decoder(player_id: PlayerId, config: AudioDecoderConfig) -> DecoderId { unsafe { - createVideoDecoder(player_id, JsonVec::new(config)); + createAudioDecoder(player_id, JsonVec::new(config)); } AUDIO_DECODER_ID } // 事前に TypeScript 側で decoder configuration のチェックを行っているので、これは常に成功する - pub fn create_audio_decoder(player_id: PlayerId, config: AudioDecoderConfig) -> DecoderId { + pub fn create_video_decoder(player_id: PlayerId, config: VideoDecoderConfig) -> DecoderId { unsafe { - createAudioDecoder(player_id, JsonVec::new(config)); + createVideoDecoder(player_id, JsonVec::new(config)); } VIDEO_DECODER_ID } From afcadb6d73a56f0bf933b07f707ba12887fc9d0d Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 13:00:46 +0900 Subject: [PATCH 14/34] =?UTF-8?q?=E5=AE=9A=E6=95=B0=E3=82=92=E4=BD=BF?= =?UTF-8?q?=E3=81=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/src/mp4_media_stream.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index 5002a06e..02d9d3d9 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -170,8 +170,8 @@ class Engine { return } ;(this.wasm.exports.stop as CallableFunction)(this.engine, playerId) - this.closeDecoder(playerId, 0) - this.closeDecoder(playerId, 1) + this.closeDecoder(playerId, AUDIO_DECODER_ID) + this.closeDecoder(playerId, VIDEO_DECODER_ID) this.players.delete(playerId) } @@ -304,6 +304,7 @@ class Engine { dataOffset: number, dataLen: number, ) { + console.log(`decode: ${playerId}, ${decoderId}`) const player = this.players.get(playerId) if (player === undefined) { throw 'TODO-6' From 0872cb27908278bdf5a0576be206f20e7f52cbef Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 13:12:54 +0900 Subject: [PATCH 15/34] =?UTF-8?q?=E3=83=87=E3=82=B0=E3=83=AC=E3=81=97?= =?UTF-8?q?=E3=81=A6=E3=81=84=E3=81=9F=E3=81=AE=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/src/mp4_media_stream.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index 02d9d3d9..aebac874 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -304,7 +304,6 @@ class Engine { dataOffset: number, dataLen: number, ) { - console.log(`decode: ${playerId}, ${decoderId}`) const player = this.players.get(playerId) if (player === undefined) { throw 'TODO-6' @@ -318,7 +317,7 @@ class Engine { chunkParams.data = data // @ts-ignore TS2488: TODO chunkParams.transfer = [data.buffer] - if (decoderId === 0) { + if (decoderId === VIDEO_DECODER_ID) { if (player.videoDecoder === undefined) { throw 'TODO-7' } From d99fb4beda7f1bd156b5ddee4994a2d0c64a6562 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 14:22:50 +0900 Subject: [PATCH 16/34] =?UTF-8?q?=E3=83=87=E3=82=B3=E3=83=BC=E3=83=80?= =?UTF-8?q?=E3=83=BC=E3=81=AE=E4=BD=9C=E6=88=90=E5=AE=8C=E4=BA=86=E3=82=92?= =?UTF-8?q?=20rust=20=E5=81=B4=E3=81=A7=E5=BE=85=E3=81=A1=E5=90=88?= =?UTF-8?q?=E3=82=8F=E3=81=9B=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=E3=81=99?= =?UTF-8?q?=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mp4-media-stream/src/mp4_media_stream.ts | 119 +++++++++++------- packages/mp4-media-stream/wasm/src/player.rs | 12 +- packages/mp4-media-stream/wasm/src/wasm.rs | 50 ++++++-- 3 files changed, 120 insertions(+), 61 deletions(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index aebac874..bbcdcff4 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -35,14 +35,14 @@ class Mp4MediaStream { engineRef.value.sleep(resultTx, duration) } }, - createVideoDecoder(playerId: number, configWasmJson: number) { + createVideoDecoder(resultTx: number, playerId: number, configWasmJson: number) { if (engineRef.value) { - engineRef.value.createVideoDecoder(playerId, configWasmJson) + engineRef.value.createVideoDecoder(resultTx, playerId, configWasmJson) } }, - createAudioDecoder(playerId: number, configWasmJson: number) { + createAudioDecoder(resultTx: number, playerId: number, configWasmJson: number) { if (engineRef.value) { - engineRef.value.createAudioDecoder(playerId, configWasmJson) + engineRef.value.createAudioDecoder(resultTx, playerId, configWasmJson) } }, closeDecoder(playerId: number, decoderId: number) { @@ -163,15 +163,15 @@ class Engine { return player.createMediaStream() } - stop(playerId: number) { + async stop(playerId: number) { console.log('stop player') const player = this.players.get(playerId) if (player === undefined) { return } ;(this.wasm.exports.stop as CallableFunction)(this.engine, playerId) - this.closeDecoder(playerId, AUDIO_DECODER_ID) - this.closeDecoder(playerId, VIDEO_DECODER_ID) + await player.stop() + this.players.delete(playerId) } @@ -186,16 +186,15 @@ class Engine { }, duration) } - async createVideoDecoder(playerId: number, configWasmJson: number) { + async createVideoDecoder(resultTx: number, playerId: number, configWasmJson: number) { console.log('createVideoDecoder') const player = this.players.get(playerId) if (player === undefined) { throw 'TODO-1' } - if (player.videoWriter !== undefined) { - // 一つ前のデコーダーの終了処理が進行中の場合には、それを待機する - await player.videoWriter.closed - } + + // 一つ前のデコーダーの終了処理が進行中の場合に備えて、ここでも close を呼び出して終了を待機する + await player.closeVideoDecoder() const config = this.wasmJsonToValue(configWasmJson) as VideoDecoderConfig // @ts-ignore TS2488: TODO @@ -211,7 +210,7 @@ class Engine { try { await player.videoWriter.write(frame) } catch (e) { - this.stop(playerId) + await this.stop(playerId) throw e // TODO } }, @@ -223,19 +222,23 @@ class Engine { player.videoDecoder = new VideoDecoder(init) player.videoDecoder.configure(config) + ;(this.wasm.exports.notifyDecoderId as CallableFunction)( + this.engine, + resultTx, + VIDEO_DECODER_ID, + ) console.log('video decoder created') } - async createAudioDecoder(playerId: number, configWasmJson: number) { + async createAudioDecoder(resultTx: number, playerId: number, configWasmJson: number) { console.log('createAudioDecoder') const player = this.players.get(playerId) if (player === undefined) { throw 'TODO-3' } - if (player.audioWriter !== undefined) { - // 一つ前のデコーダーの終了処理が進行中の場合には、それを待機する - await player.audioWriter.closed - } + + // 一つ前のデコーダーの終了処理が進行中の場合に備えて、ここでも close を呼び出して終了を待機する + await player.closeAudioDecoder() const config = this.wasmJsonToValue(configWasmJson) as AudioDecoderConfig const init = { @@ -247,7 +250,7 @@ class Engine { try { await player.audioWriter.write(data) } catch (e) { - this.stop(playerId) + await this.stop(playerId) throw e // TODO: エラーの種類によって処理を分ける（closed なら正常系） } }, @@ -259,6 +262,11 @@ class Engine { player.audioDecoder = new AudioDecoder(init) player.audioDecoder.configure(config) + ;(this.wasm.exports.notifyDecoderId as CallableFunction)( + this.engine, + resultTx, + AUDIO_DECODER_ID, + ) console.log('audio decoder created') } @@ -269,32 +277,16 @@ class Engine { return } - if ( - decoderId === VIDEO_DECODER_ID && - player.videoDecoder !== undefined && - player.videoWriter !== undefined && - player.videoDecoder.state !== 'closed' - ) { - await player.videoDecoder.flush() - player.videoDecoder.close() - player.videoDecoder = undefined - player.videoWriter.close() - player.videoWriter = undefined - } else if ( - player.audioDecoder !== undefined && - player.audioWriter !== undefined && - player.audioDecoder.state !== 'closed' - ) { - await player.audioDecoder.flush() - player.audioDecoder.close() - player.audioDecoder = undefined - player.audioWriter.close() - player.audioWriter = undefined + if (decoderId === AUDIO_DECODER_ID) { + await player.closeAudioDecoder() + } else { + await player.closeVideoDecoder() } } + // MP4 の終端に達した場合に呼ばれるコールバック async onEos(playerId: number) { - this.stop(playerId) + await this.stop(playerId) } decode( @@ -368,8 +360,8 @@ class Engine { } class Player { - audio: boolean - video: boolean + private audio: boolean + private video: boolean audioDecoder?: AudioDecoder videoDecoder?: VideoDecoder audioWriter?: WritableStreamDefaultWriter @@ -394,6 +386,47 @@ class Player { } return new MediaStream(tracks) } + + async closeAudioDecoder() { + if (this.audioDecoder !== undefined && this.audioDecoder.state !== 'closed') { + console.log('close audio decoder') + await this.audioDecoder.flush() + + // await 前後で状態が変わっている可能性があるのでもう一度チェックする + if (this.audioDecoder !== undefined) { + this.audioDecoder.close() + this.audioDecoder = undefined + } + } + } + + async closeVideoDecoder() { + if (this.videoDecoder !== undefined && this.videoDecoder.state !== 'closed') { + console.log('close video decoder') + + await this.videoDecoder.flush() + + // await 前後で状態が変わっている可能性があるのでもう一度チェックする + if (this.videoDecoder !== undefined) { + this.videoDecoder.close() + this.videoDecoder = undefined + } + } + } + + async stop() { + await this.closeAudioDecoder() + await this.closeVideoDecoder() + + if (this.audioWriter !== undefined) { + await this.audioWriter.close() + this.audioWriter = undefined + } + if (this.videoWriter !== undefined) { + await this.videoWriter.close() + this.videoWriter = undefined + } + } } export { Mp4MediaStream, type Mp4MediaStreamPlayOptions } diff --git a/packages/mp4-media-stream/wasm/src/player.rs b/packages/mp4-media-stream/wasm/src/player.rs index 98f42bb7..74a1f00c 100644 --- a/packages/mp4-media-stream/wasm/src/player.rs +++ b/packages/mp4-media-stream/wasm/src/player.rs @@ -68,7 +68,7 @@ impl Player { pub async fn run(mut self) { loop { // TODO: 数フレーム分は先読みしてデコードしておく - while self.run_one() { + while self.run_one().await { // TODO: sleep until next sample timestamp // 次のサンプルのタイムスタンプまで待つ @@ -91,11 +91,11 @@ impl Player { WasmApi::notify_eos(self.player_id); } - fn run_one(&mut self) -> bool { + async fn run_one(&mut self) -> bool { let now = self.elapsed(); for track in &mut self.tracks { - track.maybe_setup_decoder(); + track.maybe_setup_decoder().await; } for track in &self.tracks { @@ -156,7 +156,7 @@ impl TrackPlayer { }) } - fn maybe_setup_decoder(&mut self) { + async fn maybe_setup_decoder(&mut self) { if self.decoder.is_some() && !self.is_sample_entry_changed() { return; } @@ -168,11 +168,11 @@ impl TrackPlayer { let decoder = match self.current_sample().chunk().sample_entry() { SampleEntry::Avc1(b) => { let config = VideoDecoderConfig::from_avc1_box(b); - WasmApi::create_video_decoder(self.player_id, config) + WasmApi::create_video_decoder(self.player_id, config).await } SampleEntry::Opus(b) => { let config = AudioDecoderConfig::from_opus_box(b); - WasmApi::create_audio_decoder(self.player_id, config) + WasmApi::create_audio_decoder(self.player_id, config).await } _ => { // MP4::load() の中で非対応コーデックのチェックは行っているので、ここに来ることはない diff --git a/packages/mp4-media-stream/wasm/src/wasm.rs b/packages/mp4-media-stream/wasm/src/wasm.rs index 7ec82965..6fb8e355 100644 --- a/packages/mp4-media-stream/wasm/src/wasm.rs +++ b/packages/mp4-media-stream/wasm/src/wasm.rs @@ -13,10 +13,6 @@ use crate::{ pub type DecoderId = u32; -// 複数の映像・音声トラックに対応するまでは ID はハードコーディングしてしまう -const AUDIO_DECODER_ID: DecoderId = 0; -const VIDEO_DECODER_ID: DecoderId = 1; - #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct EncodedChunkMetadata { @@ -41,19 +37,27 @@ impl WasmApi { } // 事前に TypeScript 側で decoder configuration のチェックを行っているので、これは常に成功する - pub fn create_audio_decoder(player_id: PlayerId, config: AudioDecoderConfig) -> DecoderId { + pub async fn create_audio_decoder( + player_id: PlayerId, + config: AudioDecoderConfig, + ) -> DecoderId { + let (tx, rx) = oneshot::channel::(); unsafe { - createAudioDecoder(player_id, JsonVec::new(config)); + createAudioDecoder(Box::into_raw(Box::new(tx)), player_id, JsonVec::new(config)); } - AUDIO_DECODER_ID + rx.unwrap_or_else(|_| unreachable!()).await } // 事前に TypeScript 側で decoder configuration のチェックを行っているので、これは常に成功する - pub fn create_video_decoder(player_id: PlayerId, config: VideoDecoderConfig) -> DecoderId { + pub async fn create_video_decoder( + player_id: PlayerId, + config: VideoDecoderConfig, + ) -> DecoderId { + let (tx, rx) = oneshot::channel::(); unsafe { - createVideoDecoder(player_id, JsonVec::new(config)); + createVideoDecoder(Box::into_raw(Box::new(tx)), player_id, JsonVec::new(config)); } - VIDEO_DECODER_ID + rx.unwrap_or_else(|_| unreachable!()).await } pub fn decode( @@ -117,10 +121,18 @@ extern "C" { ); #[expect(improper_ctypes)] - pub fn createVideoDecoder(player_id: PlayerId, config: JsonVec); + pub fn createVideoDecoder( + result_tx: *mut oneshot::Sender, + player_id: PlayerId, + config: JsonVec, + ); #[expect(improper_ctypes)] - pub fn createAudioDecoder(player_id: PlayerId, config: JsonVec); + pub fn createAudioDecoder( + result_tx: *mut oneshot::Sender, + player_id: PlayerId, + config: JsonVec, + ); pub fn closeDecoder(player_id: PlayerId, decoder: DecoderId); @@ -137,6 +149,20 @@ pub fn awake(engine: *mut Engine, result_tx: *mut oneshot::Sender<()>) { unsafe { &mut *engine }.poll(); } +#[no_mangle] +#[expect(non_snake_case, clippy::not_unsafe_ptr_arg_deref)] +pub fn notifyDecoderId( + engine: *mut Engine, + result_tx: *mut oneshot::Sender, + decoder_id: DecoderId, +) { + let is_ok = unsafe { Box::from_raw(result_tx) }.send(decoder_id).is_ok(); + if !is_ok { + return; + } + unsafe { &mut *engine }.poll(); +} + #[no_mangle] #[expect(non_snake_case)] pub fn newEngine() -> *mut Engine { From 046f6dd7833637f86614ff6350bd2b4bde594be9 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 14:50:43 +0900 Subject: [PATCH 17/34] =?UTF-8?q?=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E7=B5=82=E7=AB=AF=E3=81=AB=E9=81=94=E3=81=97=E3=81=9F=E5=A0=B4?= =?UTF-8?q?=E5=90=88=E3=81=AB=E4=BD=99=E8=A8=88=E3=81=AA=E4=BE=8B=E5=A4=96?= =?UTF-8?q?=E3=81=8C=E5=87=BA=E3=81=AA=E3=81=84=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mp4-media-stream/src/mp4_media_stream.ts | 59 +++++++++++++------ 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index bbcdcff4..36093ade 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -388,30 +388,55 @@ class Player { } async closeAudioDecoder() { - if (this.audioDecoder !== undefined && this.audioDecoder.state !== 'closed') { - console.log('close audio decoder') - await this.audioDecoder.flush() - - // await 前後で状態が変わっている可能性があるのでもう一度チェックする - if (this.audioDecoder !== undefined) { - this.audioDecoder.close() - this.audioDecoder = undefined + const decoder = this.audioDecoder + if (decoder === undefined) { + return + } + console.log('close audio decoder') + + if (decoder.state !== 'closed') { + try { + await decoder.flush() + } catch (e) { + if (e instanceof DOMException && e.name === 'AbortError') { + // デコーダーのクローズ処理と競合した場合にはここに来る（単に無視すればいい） + } else { + throw e + } } } + + // await 前後で状態が変わっている可能性があるのでもう一度チェックする + if (decoder === this.audioDecoder && decoder.state !== 'closed') { + decoder.close() + this.audioDecoder = undefined + } } async closeVideoDecoder() { - if (this.videoDecoder !== undefined && this.videoDecoder.state !== 'closed') { - console.log('close video decoder') - - await this.videoDecoder.flush() - - // await 前後で状態が変わっている可能性があるのでもう一度チェックする - if (this.videoDecoder !== undefined) { - this.videoDecoder.close() - this.videoDecoder = undefined + const decoder = this.videoDecoder + if (decoder === undefined) { + return + } + console.log('close video decoder') + + if (decoder.state !== 'closed') { + try { + await decoder.flush() + } catch (e) { + if (e instanceof DOMException && e.name === 'AbortError') { + // デコーダーのクローズ処理と競合した場合にはここに来る（単に無視すればいい） + } else { + throw e + } } } + + // await 前後で状態が変わっている可能性があるのでもう一度チェックする + if (decoder === this.videoDecoder && decoder.state !== 'closed') { + decoder.close() + this.videoDecoder = undefined + } } async stop() { From 4ef61793163937a5ba8e00d3e0624b509f182b2a Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 15:19:01 +0900 Subject: [PATCH 18/34] =?UTF-8?q?=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E7=B5=82=E7=AB=AF=E4=BB=98=E8=BF=91=E3=81=AE=E6=89=B1=E3=81=84?= =?UTF-8?q?=E3=82=92=E3=81=A1=E3=82=83=E3=82=93=E3=81=A8=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/wasm/src/mp4.rs | 27 +++++--- packages/mp4-media-stream/wasm/src/player.rs | 65 +++++++++++++------- 2 files changed, 59 insertions(+), 33 deletions(-) diff --git a/packages/mp4-media-stream/wasm/src/mp4.rs b/packages/mp4-media-stream/wasm/src/mp4.rs index 0b2431ab..937c350d 100644 --- a/packages/mp4-media-stream/wasm/src/mp4.rs +++ b/packages/mp4-media-stream/wasm/src/mp4.rs @@ -71,24 +71,24 @@ pub struct Track { } impl Track { - pub fn new(trak_box: TrakBox) -> orfail::Result { + pub fn new(trak_box: TrakBox, mp4_size: usize) -> orfail::Result { + let kind = match trak_box.mdia_box.hdlr_box.handler_type { + HdlrBox::HANDLER_TYPE_SOUN => "audio ", + HdlrBox::HANDLER_TYPE_VIDE => "video ", + _ => "", + }; + let timescale = trak_box.mdia_box.mdhd_box.timescale; let sample_table = SampleTableAccessor::new(trak_box.mdia_box.minf_box.stbl_box).or_fail()?; - (sample_table.sample_count() > 0).or_fail()?; + (sample_table.sample_count() > 0).or_fail_with(|()| format!("Empty {kind}track"))?; match sample_table.stbl_box().stsd_box.entries.first() { Some(SampleEntry::Avc1(_)) => (), Some(SampleEntry::Opus(_)) => (), Some(b) => { - let kind = match trak_box.mdia_box.hdlr_box.handler_type { - HdlrBox::HANDLER_TYPE_SOUN => "audio ", - HdlrBox::HANDLER_TYPE_VIDE => "video ", - _ => "", - }; return Err(Failure::new(format!( - "Unsupported {}codec: {}", - kind, + "Unsupported {kind}codec: {}", b.box_type() ))); } @@ -97,6 +97,13 @@ impl Track { unreachable!() } }; + + if let Some(last) = sample_table.samples().last() { + let expected_min_mp4_size = last.data_offset() + last.data_size() as u64; + (expected_min_mp4_size <= mp4_size as u64) + .or_fail_with(|()| format!("Last {kind}sample's data is out of range"))?; + } + Ok(Self { sample_table: Rc::new(sample_table), timescale, @@ -124,7 +131,7 @@ impl Mp4 { video_track_count += 1; } - tracks.push(Track::new(trak_box).or_fail()?); + tracks.push(Track::new(trak_box, mp4_bytes.len()).or_fail()?); } (!tracks.is_empty()).or_fail_with(|()| "No video or audio tracks found".to_owned())?; (audio_track_count <= 1) diff --git a/packages/mp4-media-stream/wasm/src/player.rs b/packages/mp4-media-stream/wasm/src/player.rs index 74a1f00c..054c17cb 100644 --- a/packages/mp4-media-stream/wasm/src/player.rs +++ b/packages/mp4-media-stream/wasm/src/player.rs @@ -56,32 +56,43 @@ impl Player { WasmApi::now().saturating_sub(self.start_time) } - fn next_timestamp(&self) -> Duration { - // TODO: repeat を考慮する + // 再生時刻が一番近いサンプルのタイムスタンプを返す + // 全てのトラックが終端に達している場合には None が返される + fn next_timestamp(&self) -> Option { self.tracks .iter() + .filter(|t| !t.eos()) .map(|t| t.current_timestamp()) .min() + } + + // ファイル全体の尺を返す + fn file_duration(&self) -> Duration { + self.tracks + .iter() + .map(|t| t.duration()) + .max() .expect("unreachable") } pub async fn run(mut self) { loop { - // TODO: 数フレーム分は先読みしてデコードしておく - while self.run_one().await { - // TODO: sleep until next sample timestamp - + while let Some(wait) = self + .next_timestamp() + .map(|t| t.saturating_sub(self.elapsed())) + { // 次のサンプルのタイムスタンプまで待つ - // TODO: 遅れている場合にはフレームスキップをするかも - let now = self.elapsed(); - let wait_duration = self.next_timestamp().saturating_sub(now); - WasmApi::sleep(wait_duration).await; + WasmApi::sleep(wait).await; + + self.run_one().await; } + if !self.repeat { break; } - self.timestamp_offset += self.next_timestamp(); // TODO: duration を足すべき？ + // 繰り返し再生を行う場合には、開始時刻を調整する + self.timestamp_offset += self.file_duration(); self.start_time = WasmApi::now(); for track in &mut self.tracks { track.current_sample_index = NonZeroU32::MIN; @@ -91,7 +102,7 @@ impl Player { WasmApi::notify_eos(self.player_id); } - async fn run_one(&mut self) -> bool { + async fn run_one(&mut self) { let now = self.elapsed(); for track in &mut self.tracks { @@ -99,32 +110,28 @@ impl Player { } for track in &self.tracks { - if now < track.current_timestamp() { + if track.eos() || now < track.current_timestamp() { continue; } - self.render_sample(track); + self.decode_sample(track); + break; } for track in &mut self.tracks { - if now < track.current_timestamp() { + if track.eos() || now < track.current_timestamp() { continue; } track.current_sample_index = track.current_sample_index.saturating_add(1); - if track.current_sample_index.get() + 1 == track.sample_table.sample_count() { - // TODO: 尺が長い方に合わせる - return false; - } + return; } - - true } - fn render_sample(&self, track: &TrackPlayer) { + fn decode_sample(&self, track: &TrackPlayer) { let sample = track.current_sample(); let data = &self.mp4_bytes[sample.data_offset() as usize..][..sample.data_size() as usize]; - // Rust 側で関与するのはデコードまでで、その先の処理は TypeScript 側で行われる + // Rust 側で関与するのはデコードのトリガーを引くところまでで、その先の処理は TypeScript 側で行われる WasmApi::decode( self.player_id, track.decoder.expect("unreachable"), @@ -157,6 +164,9 @@ impl TrackPlayer { } async fn maybe_setup_decoder(&mut self) { + if self.eos() { + return; + } if self.decoder.is_some() && !self.is_sample_entry_changed() { return; } @@ -208,6 +218,15 @@ impl TrackPlayer { let sample = self.current_sample(); Duration::from_secs(sample.timestamp()) / self.timescale.get() } + + fn duration(&self) -> Duration { + let last = self.sample_table.samples().last().expect("unreachable"); + Duration::from_secs(last.timestamp() + last.duration() as u64) / self.timescale.get() + } + + fn eos(&self) -> bool { + self.current_sample_index.get() + 1 == self.sample_table.sample_count() + } } impl Drop for TrackPlayer { From 74a1c97c17b14cc2f4b050a23ee4c309b29f82d3 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 15:48:53 +0900 Subject: [PATCH 19/34] Add TODO comment --- packages/mp4-media-stream/src/mp4_media_stream.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index 36093ade..0754031c 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -85,6 +85,8 @@ class Mp4MediaStream { play(options: Mp4MediaStreamPlayOptions = {}): MediaStream { return this.engine.play(options) } + + // TODO: stop() } type Mp4Info = { From 23c50624a369fb68d19073a034b3f275e69f1814 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Tue, 15 Oct 2024 16:08:48 +0900 Subject: [PATCH 20/34] =?UTF-8?q?=E3=83=87=E3=82=B3=E3=83=BC=E3=83=80?= =?UTF-8?q?=E3=83=BC=E3=81=AB=E6=B8=A1=E3=81=99=E3=82=BF=E3=82=A4=E3=83=A0?= =?UTF-8?q?=E3=82=B9=E3=82=BF=E3=83=B3=E3=83=97=E3=81=AF=E3=83=9E=E3=82=A4?= =?UTF-8?q?=E3=82=AF=E3=83=AD=E7=A7=92=E5=8D=98=E4=BD=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/wasm/src/wasm.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/mp4-media-stream/wasm/src/wasm.rs b/packages/mp4-media-stream/wasm/src/wasm.rs index 6fb8e355..f2564ac3 100644 --- a/packages/mp4-media-stream/wasm/src/wasm.rs +++ b/packages/mp4-media-stream/wasm/src/wasm.rs @@ -18,8 +18,8 @@ pub type DecoderId = u32; pub struct EncodedChunkMetadata { #[serde(rename = "type")] pub ty: &'static str, // key | delta - pub timestamp: u32, // millis - pub duration: u32, // millis + pub timestamp: u64, // micros + pub duration: u32, // micros } #[derive(Debug)] @@ -76,8 +76,8 @@ impl WasmApi { }, timestamp: (Duration::from_secs(sample.timestamp()) / timescale.get() + timestamp_offset) - .as_millis() as u32, - duration: (Duration::from_secs(sample.duration() as u64) / timescale.get()).as_millis() + .as_micros() as u64, + duration: (Duration::from_secs(sample.duration() as u64) / timescale.get()).as_micros() as u32, }; unsafe { From 16de0d140c524ac7e87957e0d8966bdb6db47b5c Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Wed, 16 Oct 2024 09:13:09 +0900 Subject: [PATCH 21/34] =?UTF-8?q?AudioDecoder=20/=20VideoDecoder=20?= =?UTF-8?q?=E3=81=8C=E5=88=A9=E7=94=A8=E5=8F=AF=E8=83=BD=E3=81=8B=E3=81=A9?= =?UTF-8?q?=E3=81=86=E3=81=8B=E3=82=82=E3=83=81=E3=82=A7=E3=83=83=E3=82=AF?= =?UTF-8?q?=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/src/mp4_media_stream.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index 0754031c..6e3416e2 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -15,7 +15,11 @@ class Mp4MediaStream { } static isSupported(): boolean { - return !(typeof MediaStreamTrackGenerator === 'undefined') + return !( + typeof MediaStreamTrackGenerator === 'undefined' || + typeof AudioDecoder === 'undefined' || + typeof VideoDecoder === 'undefined' + ) } static async loadMp4(mp4: Blob): Promise { @@ -86,7 +90,7 @@ class Mp4MediaStream { return this.engine.play(options) } - // TODO: stop() + // TODO: stop() } type Mp4Info = { From 8009fc5f098f67cbdbbc59871ee7d83945c85fc6 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Wed, 16 Oct 2024 09:14:42 +0900 Subject: [PATCH 22/34] =?UTF-8?q?=E3=83=87=E3=83=90=E3=83=83=E3=82=B0?= =?UTF-8?q?=E3=83=AD=E3=82=B0=E3=82=92=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/src/mp4_media_stream.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index 6e3416e2..005158ed 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -143,9 +143,6 @@ class Engine { this.info = info } - // TODO: remove - console.log(JSON.stringify(info)) - return { audio: info.audioConfigs.length > 0, video: info.videoConfigs.length > 0 } } @@ -164,13 +161,11 @@ class Engine { playerId, this.valueToWasmJson(options), ) - console.log('player created') return player.createMediaStream() } async stop(playerId: number) { - console.log('stop player') const player = this.players.get(playerId) if (player === undefined) { return @@ -193,7 +188,6 @@ class Engine { } async createVideoDecoder(resultTx: number, playerId: number, configWasmJson: number) { - console.log('createVideoDecoder') const player = this.players.get(playerId) if (player === undefined) { throw 'TODO-1' @@ -233,11 +227,9 @@ class Engine { resultTx, VIDEO_DECODER_ID, ) - console.log('video decoder created') } async createAudioDecoder(resultTx: number, playerId: number, configWasmJson: number) { - console.log('createAudioDecoder') const player = this.players.get(playerId) if (player === undefined) { throw 'TODO-3' @@ -273,11 +265,9 @@ class Engine { resultTx, AUDIO_DECODER_ID, ) - console.log('audio decoder created') } async closeDecoder(playerId: number, decoderId: number) { - console.log('close decoder') const player = this.players.get(playerId) if (player === undefined) { return @@ -398,7 +388,6 @@ class Player { if (decoder === undefined) { return } - console.log('close audio decoder') if (decoder.state !== 'closed') { try { @@ -424,7 +413,6 @@ class Player { if (decoder === undefined) { return } - console.log('close video decoder') if (decoder.state !== 'closed') { try { From db1803952514e0ea4c2d5697c173012bfd90aa05 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Wed, 16 Oct 2024 09:36:32 +0900 Subject: [PATCH 23/34] Add stop() method --- packages/mp4-media-stream/src/mp4_media_stream.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index 005158ed..295b2ce4 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -90,7 +90,12 @@ class Mp4MediaStream { return this.engine.play(options) } - // TODO: stop() + /** + * 再生中の全てのメディアストリームを停止する + */ + async stop() { + await this.engine.stopAll() + } } type Mp4Info = { @@ -176,6 +181,12 @@ class Engine { this.players.delete(playerId) } + async stopAll() { + for (const playerId of this.players.keys()) { + await this.stop(playerId) + } + } + consoleLog(messageWasmJson: number) { const message = this.wasmJsonToValue(messageWasmJson) console.log(message) From 4b592955a6c25d7368add5a0c540da4b883c4f75 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Wed, 16 Oct 2024 09:48:49 +0900 Subject: [PATCH 24/34] s/loadMp4/load/ --- examples/mp4-media-stream/main.mts | 2 +- packages/mp4-media-stream/src/mp4_media_stream.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/mp4-media-stream/main.mts b/examples/mp4-media-stream/main.mts index ac69abcb..b16916a2 100644 --- a/examples/mp4-media-stream/main.mts +++ b/examples/mp4-media-stream/main.mts @@ -14,7 +14,7 @@ document.addEventListener('DOMContentLoaded', async () => { if (mp4MediaStream !== undefined) { mp4MediaStream.stop() } - mp4MediaStream = await Mp4MediaStream.loadMp4(mp4File) + mp4MediaStream = await Mp4MediaStream.load(mp4File) const options = { repeat: document.getElementById('repeat').checked, diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index 295b2ce4..c0adb76f 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -22,7 +22,7 @@ class Mp4MediaStream { ) } - static async loadMp4(mp4: Blob): Promise { + static async load(mp4: Blob): Promise { const engineRef: { value?: Engine } = { value: undefined } const importObject = { env: { @@ -117,7 +117,6 @@ class Engine { this.engine = (this.wasm.exports.newEngine as CallableFunction)() } - // TODO: load() でいいかも async loadMp4(mp4Bytes: Uint8Array): Promise<{ audio: boolean; video: boolean }> { const mp4WasmBytes = this.toWasmBytes(mp4Bytes) const resultWasmJson = (this.wasm.exports.loadMp4 as CallableFunction)( From cc6d80871752465c6308221ef63d3ddda3bab530 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Wed, 16 Oct 2024 09:50:18 +0900 Subject: [PATCH 25/34] s/Mp4MediaStreamPlayOptions/PlayOptions/ --- packages/mp4-media-stream/src/mp4_media_stream.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index c0adb76f..faccf0f3 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -1,6 +1,6 @@ const WASM_BASE64 = '__WASM__' -interface Mp4MediaStreamPlayOptions { +interface PlayOptions { repeat?: boolean } @@ -86,7 +86,7 @@ class Mp4MediaStream { return stream } - play(options: Mp4MediaStreamPlayOptions = {}): MediaStream { + play(options: PlayOptions = {}): MediaStream { return this.engine.play(options) } @@ -150,7 +150,7 @@ class Engine { return { audio: info.audioConfigs.length > 0, video: info.videoConfigs.length > 0 } } - play(options: Mp4MediaStreamPlayOptions = {}): MediaStream { + play(options: PlayOptions = {}): MediaStream { if (this.info === undefined) { throw 'bug' } @@ -458,4 +458,4 @@ class Player { } } -export { Mp4MediaStream, type Mp4MediaStreamPlayOptions } +export { Mp4MediaStream, type PlayOptions } From 678ec130707620f72b2c65b4c40946049dedd5d6 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Wed, 16 Oct 2024 10:13:37 +0900 Subject: [PATCH 26/34] =?UTF-8?q?=E3=82=A8=E3=83=A9=E3=83=BC=E3=83=8F?= =?UTF-8?q?=E3=83=B3=E3=83=89=E3=83=AA=E3=83=B3=E3=82=B0=E3=82=92=E6=94=B9?= =?UTF-8?q?=E5=96=84=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mp4-media-stream/src/mp4_media_stream.ts | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index faccf0f3..599d47de 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -125,24 +125,20 @@ class Engine { ) // MP4 内に含まれる映像・音声を WebCodecs のデコーダー扱えるかどうかをチェックする - const info = this.wasmResultToValue(resultWasmJson) as { - audioConfigs: [AudioDecoderConfig] - videoConfigs: [VideoDecoderConfig] - } + const info = this.wasmResultToValue(resultWasmJson) as Mp4Info for (const config of info.audioConfigs) { if (!(await AudioDecoder.isConfigSupported(config)).supported) { - // TODO: ちゃんとする - throw 'unsupported audio codec' + throw new Error(`Unsupported audio decoder configuration: ${JSON.stringify(config)}`) } } for (const config of info.videoConfigs) { if (config.description !== undefined) { - // @ts-ignore TS2488: TODO - config.description = new Uint8Array(config.description) + // JSON.parse() の結果では config.description の型は number[] となって期待とは異なるので + // ここで適切な型に変換している + config.description = new Uint8Array(config.description as object as number[]) } if (!(await VideoDecoder.isConfigSupported(config)).supported) { - // TODO: ちゃんとする - throw 'unsupported audio codec' + throw new Error(`Unsupported video decoder configuration: ${JSON.stringify(config)}`) } this.info = info } @@ -152,7 +148,8 @@ class Engine { play(options: PlayOptions = {}): MediaStream { if (this.info === undefined) { - throw 'bug' + // ここには来ないはず + throw new Error('bug') } const playerId = this.nextPlayerId @@ -200,7 +197,9 @@ class Engine { async createVideoDecoder(resultTx: number, playerId: number, configWasmJson: number) { const player = this.players.get(playerId) if (player === undefined) { - throw 'TODO-1' + // stop() と競合したらここに来る可能性がある + // すでに停止済みであり、新規でデコーダーを作成する必要はないので、ここで return する + return } // 一つ前のデコーダーの終了処理が進行中の場合に備えて、ここでも close を呼び出して終了を待機する @@ -242,7 +241,9 @@ class Engine { async createAudioDecoder(resultTx: number, playerId: number, configWasmJson: number) { const player = this.players.get(playerId) if (player === undefined) { - throw 'TODO-3' + // stop() と競合したらここに来る可能性がある + // すでに停止済みであり、新規でデコーダーを作成する必要はないので、ここで return する + return } // 一つ前のデコーダーの終了処理が進行中の場合に備えて、ここでも close を呼び出して終了を待機する From 802b6407768bbeb50a83c12d1f7814c14eac89e4 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Wed, 16 Oct 2024 10:47:32 +0900 Subject: [PATCH 27/34] =?UTF-8?q?=E3=83=87=E3=82=B3=E3=83=BC=E3=83=89?= =?UTF-8?q?=E7=B5=90=E6=9E=9C=E5=87=A6=E7=90=86=E9=83=A8=E5=88=86=E3=81=AE?= =?UTF-8?q?=E3=82=A8=E3=83=A9=E3=83=BC=E3=83=8F=E3=83=B3=E3=83=89=E3=83=AA?= =?UTF-8?q?=E3=83=B3=E3=82=B0=E3=82=92=E3=81=A1=E3=82=83=E3=82=93=E3=81=A8?= =?UTF-8?q?=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mp4-media-stream/src/mp4_media_stream.ts | 78 ++++++++++++++----- 1 file changed, 59 insertions(+), 19 deletions(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index 599d47de..0c6a666e 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -132,11 +132,10 @@ class Engine { } } for (const config of info.videoConfigs) { - if (config.description !== undefined) { - // JSON.parse() の結果では config.description の型は number[] となって期待とは異なるので - // ここで適切な型に変換している - config.description = new Uint8Array(config.description as object as number[]) - } + // JSON.parse() の結果では config.description の型は number[] となって期待とは異なるので + // ここで適切な型に変換している + config.description = new Uint8Array(config.description as object as number[]) + if (!(await VideoDecoder.isConfigSupported(config)).supported) { throw new Error(`Unsupported video decoder configuration: ${JSON.stringify(config)}`) } @@ -206,25 +205,40 @@ class Engine { await player.closeVideoDecoder() const config = this.wasmJsonToValue(configWasmJson) as VideoDecoderConfig - // @ts-ignore TS2488: TODO - config.description = new Uint8Array(config.description) + + // JSON.parse() の結果では config.description の型は number[] となって期待とは異なるので + // ここで適切な型に変換している + config.description = new Uint8Array(config.description as object as number[]) const init = { output: async (frame: VideoFrame) => { if (player.videoWriter === undefined) { - throw 'bug' + // writer の出力先がすでに閉じられている場合などにここに来る可能性がある + return } - // TODO: error handling (stop engine) try { await player.videoWriter.write(frame) - } catch (e) { + } catch (error) { + // 書き込みエラーが発生した場合には再生を停止する + + if (error instanceof DOMException && error.name === 'InvalidStateError') { + // 出力先の MediaStreamTrack が停止済み、などの理由で write() が失敗した場合にここに来る。 + // このケースは普通に発生し得るので正常系の一部。 + // writer はすでに閉じているので、重複 close() による警告ログ出力を避けるために undefined に設定する。 + player.videoWriter = undefined + await this.stop(playerId) + return + } + + // 想定外のエラーの場合は再送する await this.stop(playerId) - throw e // TODO + throw error } }, - error: (error: DOMException) => { - // TODO: ちゃんとしたエラーハンドリング + error: async (error: DOMException) => { + // デコードエラーが発生した場合には再生を停止する + await this.stop(playerId) throw error }, } @@ -253,18 +267,32 @@ class Engine { const init = { output: async (data: AudioData) => { if (player.audioWriter === undefined) { - throw 'bug' + // writer の出力先がすでに閉じられている場合などにここに来る可能性がある + return } try { await player.audioWriter.write(data) } catch (e) { + // 書き込みエラーが発生した場合には再生を停止する + + if (e instanceof DOMException && e.name === 'InvalidStateError') { + // 出力先の MediaStreamTrack が停止済み、などの理由で write() が失敗した場合にここに来る。 + // このケースは普通に発生し得るので正常系の一部。 + // writer はすでに閉じているので、重複 close() による警告ログ出力を避けるために undefined に設定する。 + player.audioWriter = undefined + await this.stop(playerId) + return + } + + // 想定外のエラーの場合は再送する await this.stop(playerId) - throw e // TODO: エラーの種類によって処理を分ける（closed なら正常系） + throw e } }, - error: (error: DOMException) => { - // TODO: ちゃんとしたエラーハンドリング + error: async (error: DOMException) => { + // デコードエラーが発生した場合には再生を停止する + await this.stop(playerId) throw error }, } @@ -449,11 +477,23 @@ class Player { await this.closeVideoDecoder() if (this.audioWriter !== undefined) { - await this.audioWriter.close() + try { + await this.audioWriter.close() + } catch (e) { + // writer がエラー状態になっている場合などには close() に失敗する模様 + // 特に対処法も実害もなさそうなので、ログだけ出して無視しておく + console.log(`[WARNING] ${e}`) + } this.audioWriter = undefined } if (this.videoWriter !== undefined) { - await this.videoWriter.close() + try { + await this.videoWriter.close() + } catch (e) { + // writer がエラー状態になっている場合などには close() に失敗する模様 + // 特に対処法も実害もなさそうなので、ログだけ出して無視しておく + console.log(`[WARNING] ${e}`) + } this.videoWriter = undefined } } From 0545acd790786a526349cf4d8077a0c970ba52d9 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Wed, 16 Oct 2024 11:00:28 +0900 Subject: [PATCH 28/34] =?UTF-8?q?=E3=82=A8=E3=83=A9=E3=83=BC=E3=83=8F?= =?UTF-8?q?=E3=83=B3=E3=83=89=E3=83=AA=E3=83=B3=E3=82=B0=E5=91=A8=E3=82=8A?= =?UTF-8?q?=E3=82=92=E4=B8=80=E9=80=9A=E3=82=8A=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mp4-media-stream/src/mp4_media_stream.ts | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index 0c6a666e..2241d33e 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -309,6 +309,7 @@ class Engine { async closeDecoder(playerId: number, decoderId: number) { const player = this.players.get(playerId) if (player === undefined) { + // すでに停止済みなので、何もする必要はない return } @@ -333,26 +334,27 @@ class Engine { ) { const player = this.players.get(playerId) if (player === undefined) { - throw 'TODO-6' + // stop() 呼び出しと競合した場合にここに来る可能性がある + // すでに停止済みなので、これ以上デコードを行う必要はない + return } const chunkParams = this.wasmJsonToValue(metadataWasmJson) as | EncodedAudioChunkInit | EncodedVideoChunkInit - const data = new Uint8Array(this.memory.buffer, dataOffset, dataLen).slice() + chunkParams.data = new Uint8Array(this.memory.buffer, dataOffset, dataLen) - chunkParams.data = data - // @ts-ignore TS2488: TODO - chunkParams.transfer = [data.buffer] if (decoderId === VIDEO_DECODER_ID) { if (player.videoDecoder === undefined) { - throw 'TODO-7' + // ここには来ないはず + throw new Error('bug') } const chunk = new EncodedVideoChunk(chunkParams) player.videoDecoder.decode(chunk) } else { if (player.audioDecoder === undefined) { - throw 'TODO-8' + // ここには来ないはず + throw new Error('bug') } const chunk = new EncodedAudioChunk(chunkParams) player.audioDecoder.decode(chunk) @@ -372,10 +374,9 @@ class Engine { } private wasmResultToValue(wasmResult: number): object { - const result = this.wasmJsonToValue(wasmResult) as { Ok?: object; Err?: object } + const result = this.wasmJsonToValue(wasmResult) as { Ok?: object; Err?: { message: string } } if (result.Err !== undefined) { - // TODO: error handling - throw JSON.stringify(result.Err) + throw new Error(result.Err.message) } return result.Ok as object } From 4172b01330c829aa055a4d8afdeffcd29ea8dd2f Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Wed, 16 Oct 2024 11:15:52 +0900 Subject: [PATCH 29/34] Remove `Engine` class --- .../mp4-media-stream/src/mp4_media_stream.ts | 168 ++++++++---------- packages/mp4-media-stream/wasm/src/wasm.rs | 6 - 2 files changed, 77 insertions(+), 97 deletions(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index 2241d33e..fde4beb6 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -8,10 +8,17 @@ const AUDIO_DECODER_ID: number = 0 const VIDEO_DECODER_ID: number = 1 class Mp4MediaStream { - private engine: Engine + private wasm: WebAssembly.Instance + private memory: WebAssembly.Memory + private engine: number + private info?: Mp4Info + private players: Map = new Map() + private nextPlayerId = 0 - constructor(wasm: WebAssembly.Instance) { - this.engine = new Engine(wasm, wasm.exports.memory as WebAssembly.Memory) + private constructor(wasm: WebAssembly.Instance) { + this.wasm = wasm + this.memory = wasm.exports.memory as WebAssembly.Memory + this.engine = (this.wasm.exports.newEngine as CallableFunction)() } static isSupported(): boolean { @@ -23,35 +30,37 @@ class Mp4MediaStream { } static async load(mp4: Blob): Promise { - const engineRef: { value?: Engine } = { value: undefined } + // インポート関数の中で this を参照したいけど、この時点ではまだ作成されていないので + // 間接的に参照するようにする + const ref: { stream?: Mp4MediaStream } = { stream: undefined } const importObject = { env: { now() { return performance.now() }, consoleLog(messageWasmJson: number) { - if (engineRef.value) { - engineRef.value.consoleLog(messageWasmJson) + if (ref.stream) { + ref.stream.consoleLog(messageWasmJson) } }, sleep(resultTx: number, duration: number) { - if (engineRef.value) { - engineRef.value.sleep(resultTx, duration) + if (ref.stream) { + ref.stream.sleep(resultTx, duration) } }, createVideoDecoder(resultTx: number, playerId: number, configWasmJson: number) { - if (engineRef.value) { - engineRef.value.createVideoDecoder(resultTx, playerId, configWasmJson) + if (ref.stream) { + ref.stream.createVideoDecoder(resultTx, playerId, configWasmJson) } }, createAudioDecoder(resultTx: number, playerId: number, configWasmJson: number) { - if (engineRef.value) { - engineRef.value.createAudioDecoder(resultTx, playerId, configWasmJson) + if (ref.stream) { + ref.stream.createAudioDecoder(resultTx, playerId, configWasmJson) } }, closeDecoder(playerId: number, decoderId: number) { - if (engineRef.value) { - engineRef.value.closeDecoder(playerId, decoderId) + if (ref.stream) { + ref.stream.closeDecoder(playerId, decoderId) } }, decode( @@ -61,13 +70,13 @@ class Mp4MediaStream { dataOffset: number, dataLen: number, ) { - if (engineRef.value) { - engineRef.value.decode(playerId, decoderId, metadataWasmJson, dataOffset, dataLen) + if (ref.stream) { + ref.stream.decode(playerId, decoderId, metadataWasmJson, dataOffset, dataLen) } }, onEos(playerId: number) { - if (engineRef.value) { - engineRef.value.onEos(playerId) + if (ref.stream) { + ref.stream.onEos(playerId) } }, }, @@ -78,43 +87,52 @@ class Mp4MediaStream { ) const stream = new Mp4MediaStream(wasmResults.instance) - engineRef.value = stream.engine + ref.stream = stream const mp4Bytes = new Uint8Array(await mp4.arrayBuffer()) - await stream.engine.loadMp4(mp4Bytes) + await stream.loadMp4(mp4Bytes) return stream } play(options: PlayOptions = {}): MediaStream { - return this.engine.play(options) + if (this.info === undefined) { + // ここには来ないはず + throw new Error('bug') + } + + const playerId = this.nextPlayerId + this.nextPlayerId += 1 + + const player = new Player(this.info.audioConfigs.length > 0, this.info.videoConfigs.length > 0) + this.players.set(playerId, player) + ;(this.wasm.exports.play as CallableFunction)( + this.engine, + playerId, + this.valueToWasmJson(options), + ) + + return player.createMediaStream() } /** * 再生中の全てのメディアストリームを停止する */ async stop() { - await this.engine.stopAll() + for (const playerId of this.players.keys()) { + await this.stopPlayer(playerId) + } } -} - -type Mp4Info = { - audioConfigs: [AudioDecoderConfig] - videoConfigs: [VideoDecoderConfig] -} -class Engine { - private wasm: WebAssembly.Instance - private memory: WebAssembly.Memory - private engine: number - private info?: Mp4Info - private players: Map = new Map() - private nextPlayerId = 0 + private async stopPlayer(playerId: number) { + const player = this.players.get(playerId) + if (player === undefined) { + return + } + ;(this.wasm.exports.stop as CallableFunction)(this.engine, playerId) + await player.stop() - constructor(wasm: WebAssembly.Instance, memory: WebAssembly.Memory) { - this.wasm = wasm - this.memory = memory - this.engine = (this.wasm.exports.newEngine as CallableFunction)() + this.players.delete(playerId) } async loadMp4(mp4Bytes: Uint8Array): Promise<{ audio: boolean; video: boolean }> { @@ -145,55 +163,18 @@ class Engine { return { audio: info.audioConfigs.length > 0, video: info.videoConfigs.length > 0 } } - play(options: PlayOptions = {}): MediaStream { - if (this.info === undefined) { - // ここには来ないはず - throw new Error('bug') - } - - const playerId = this.nextPlayerId - this.nextPlayerId += 1 - - const player = new Player(this.info.audioConfigs.length > 0, this.info.videoConfigs.length > 0) - this.players.set(playerId, player) - ;(this.wasm.exports.play as CallableFunction)( - this.engine, - playerId, - this.valueToWasmJson(options), - ) - - return player.createMediaStream() - } - - async stop(playerId: number) { - const player = this.players.get(playerId) - if (player === undefined) { - return - } - ;(this.wasm.exports.stop as CallableFunction)(this.engine, playerId) - await player.stop() - - this.players.delete(playerId) - } - - async stopAll() { - for (const playerId of this.players.keys()) { - await this.stop(playerId) - } - } - - consoleLog(messageWasmJson: number) { + private consoleLog(messageWasmJson: number) { const message = this.wasmJsonToValue(messageWasmJson) console.log(message) } - async sleep(resultTx: number, duration: number) { + private async sleep(resultTx: number, duration: number) { setTimeout(() => { ;(this.wasm.exports.awake as CallableFunction)(this.engine, resultTx) }, duration) } - async createVideoDecoder(resultTx: number, playerId: number, configWasmJson: number) { + private async createVideoDecoder(resultTx: number, playerId: number, configWasmJson: number) { const player = this.players.get(playerId) if (player === undefined) { // stop() と競合したらここに来る可能性がある @@ -227,18 +208,18 @@ class Engine { // このケースは普通に発生し得るので正常系の一部。 // writer はすでに閉じているので、重複 close() による警告ログ出力を避けるために undefined に設定する。 player.videoWriter = undefined - await this.stop(playerId) + await this.stopPlayer(playerId) return } // 想定外のエラーの場合は再送する - await this.stop(playerId) + await this.stopPlayer(playerId) throw error } }, error: async (error: DOMException) => { // デコードエラーが発生した場合には再生を停止する - await this.stop(playerId) + await this.stopPlayer(playerId) throw error }, } @@ -252,7 +233,7 @@ class Engine { ) } - async createAudioDecoder(resultTx: number, playerId: number, configWasmJson: number) { + private async createAudioDecoder(resultTx: number, playerId: number, configWasmJson: number) { const player = this.players.get(playerId) if (player === undefined) { // stop() と競合したらここに来る可能性がある @@ -281,18 +262,18 @@ class Engine { // このケースは普通に発生し得るので正常系の一部。 // writer はすでに閉じているので、重複 close() による警告ログ出力を避けるために undefined に設定する。 player.audioWriter = undefined - await this.stop(playerId) + await this.stopPlayer(playerId) return } // 想定外のエラーの場合は再送する - await this.stop(playerId) + await this.stopPlayer(playerId) throw e } }, error: async (error: DOMException) => { // デコードエラーが発生した場合には再生を停止する - await this.stop(playerId) + await this.stopPlayer(playerId) throw error }, } @@ -306,7 +287,7 @@ class Engine { ) } - async closeDecoder(playerId: number, decoderId: number) { + private async closeDecoder(playerId: number, decoderId: number) { const player = this.players.get(playerId) if (player === undefined) { // すでに停止済みなので、何もする必要はない @@ -321,11 +302,11 @@ class Engine { } // MP4 の終端に達した場合に呼ばれるコールバック - async onEos(playerId: number) { - await this.stop(playerId) + private async onEos(playerId: number) { + await this.stopPlayer(playerId) } - decode( + private decode( playerId: number, decoderId: number, metadataWasmJson: number, @@ -376,7 +357,7 @@ class Engine { private wasmResultToValue(wasmResult: number): object { const result = this.wasmJsonToValue(wasmResult) as { Ok?: object; Err?: { message: string } } if (result.Err !== undefined) { - throw new Error(result.Err.message) + throw new Error(result.Err.message) } return result.Ok as object } @@ -395,6 +376,11 @@ class Engine { } } +type Mp4Info = { + audioConfigs: [AudioDecoderConfig] + videoConfigs: [VideoDecoderConfig] +} + class Player { private audio: boolean private video: boolean diff --git a/packages/mp4-media-stream/wasm/src/wasm.rs b/packages/mp4-media-stream/wasm/src/wasm.rs index f2564ac3..e67b7b5a 100644 --- a/packages/mp4-media-stream/wasm/src/wasm.rs +++ b/packages/mp4-media-stream/wasm/src/wasm.rs @@ -176,12 +176,6 @@ pub fn newEngine() -> *mut Engine { Box::into_raw(Box::new(Engine::new())) } -#[no_mangle] -#[expect(non_snake_case, clippy::not_unsafe_ptr_arg_deref)] -pub fn freeEngine(engine: *mut Engine) { - let _ = unsafe { Box::from_raw(engine) }; -} - #[no_mangle] #[expect(non_snake_case, clippy::not_unsafe_ptr_arg_deref)] pub fn loadMp4(engine: *mut Engine, mp4_bytes: *mut Vec) -> JsonVec> { From a840d87122c07055d88fa53e952903fbe481ee54 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Wed, 16 Oct 2024 12:03:53 +0900 Subject: [PATCH 30/34] Add doc comment --- .../mp4-media-stream/src/mp4_media_stream.ts | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index fde4beb6..fb80b541 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -1,12 +1,23 @@ const WASM_BASE64 = '__WASM__' +/** + * {@link Mp4MediaStream.play} に指定可能なオプション + */ interface PlayOptions { + /** + * true が指定されると、MP4 ファイルの終端に達した場合に、先頭に戻って再生が繰り返されます + * + * デフォルト値は false + */ repeat?: boolean } const AUDIO_DECODER_ID: number = 0 const VIDEO_DECODER_ID: number = 1 +/** + * MP4 を入力にとって、それを再生する MediaStream を生成するクラス + */ class Mp4MediaStream { private wasm: WebAssembly.Instance private memory: WebAssembly.Memory @@ -21,6 +32,16 @@ class Mp4MediaStream { this.engine = (this.wasm.exports.newEngine as CallableFunction)() } + /** + * 実行環境が必要な機能をサポートしているかどうかを判定します + * + * 以下のクラスが利用可能である必要があります: + * - MediaStreamTrackGenerator + * - AudioDecoder + * - VideoDecoder + * + * @returns サポートされているかどうか + */ static isSupported(): boolean { return !( typeof MediaStreamTrackGenerator === 'undefined' || @@ -29,6 +50,16 @@ class Mp4MediaStream { ) } + /** + * 指定された MP4 をロードします + * + * @param mp4 対象の MP4 データ + * + * @returns ロード結果の Mp4MediaStream インスタンス + * + * @throws + * 指定された MP4 が不正であったり、非対応コーデックを含んでいる場合には例外が送出されます + */ static async load(mp4: Blob): Promise { // インポート関数の中で this を参照したいけど、この時点ではまだ作成されていないので // 間接的に参照するようにする @@ -95,6 +126,21 @@ class Mp4MediaStream { return stream } + /** + * ロード済み MP4 の再生するための MediaStream インスタンスを生成します + * + * @param options 再生オプション + * + * @returns MP4 再生用の MediaStream インスタンス + * + * 複数回呼び出された場合には、それぞれが別々の MediaStream インスタンスを返します + * + * [注意] + * 返り値の MediaStream は RTCPeerConnection に渡して WebRTC のソースとすることも可能です。 + * ただし、その場合には「同時に VideoElement.srcObject に指定する」などのように別に経路でも参照される + * ようにしないと、WebRTC の受信側で映像のフレームレートが極端に下がったり、止まったりする現象が確認されています。 + * なお、VideoElement はミュートかつ hidden visibility でも問題ありません。 + */ play(options: PlayOptions = {}): MediaStream { if (this.info === undefined) { // ここには来ないはず @@ -116,7 +162,10 @@ class Mp4MediaStream { } /** - * 再生中の全てのメディアストリームを停止する + * 再生中の全ての MediaStream を停止します + * + * 個別の MediaStream を停止したい場合には、単にそのインスタンスを破棄するか、 + * MedisStreamTrack.stop() を呼び出してください */ async stop() { for (const playerId of this.players.keys()) { From 82b8339a4fb94675112a86ef0c99a34c20ccdc7d Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Wed, 16 Oct 2024 12:04:24 +0900 Subject: [PATCH 31/34] =?UTF-8?q?private=20=E6=8C=87=E5=AE=9A=E3=82=82?= =?UTF-8?q?=E3=82=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/src/mp4_media_stream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index fb80b541..3e3a1a95 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -184,7 +184,7 @@ class Mp4MediaStream { this.players.delete(playerId) } - async loadMp4(mp4Bytes: Uint8Array): Promise<{ audio: boolean; video: boolean }> { + private async loadMp4(mp4Bytes: Uint8Array): Promise<{ audio: boolean; video: boolean }> { const mp4WasmBytes = this.toWasmBytes(mp4Bytes) const resultWasmJson = (this.wasm.exports.loadMp4 as CallableFunction)( this.engine, From 027abedadbb082c2c3a39bb505c56c5eedff38f2 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Wed, 16 Oct 2024 12:24:29 +0900 Subject: [PATCH 32/34] =?UTF-8?q?=E3=82=B5=E3=83=B3=E3=83=97=E3=83=AB?= =?UTF-8?q?=E3=82=B3=E3=83=BC=E3=83=89=E3=82=92=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/mp4-media-stream/index.html | 9 +++++-- examples/mp4-media-stream/main.mts | 40 ++++++++++++++++++++-------- examples/vite.config.mjs | 5 +++- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/examples/mp4-media-stream/index.html b/examples/mp4-media-stream/index.html index 3e6328d1..16df1a79 100644 --- a/examples/mp4-media-stream/index.html +++ b/examples/mp4-media-stream/index.html @@ -7,10 +7,15 @@

    Media Processors: MP4 メディアストリームサンプル

    入力 MP4 ファイル

    - + ファイル選択後に「再生開始」ボタンを押してください。 +

    + +

    + +

    - 繰り返し再生 + リピート再生を有効にする

    出力映像

    diff --git a/examples/mp4-media-stream/main.mts b/examples/mp4-media-stream/main.mts index b16916a2..2a311037 100644 --- a/examples/mp4-media-stream/main.mts +++ b/examples/mp4-media-stream/main.mts @@ -8,13 +8,31 @@ document.addEventListener('DOMContentLoaded', async () => { throw Error('Unsupported platform') } - async function play() { - let mp4File = getInputFile() + async function load() { + const input = document.getElementById('input') + const files = input.files + if (files === null || files.length === 0) { + return + } + const file = files[0] if (mp4MediaStream !== undefined) { - mp4MediaStream.stop() + await mp4MediaStream.stop() + } + + try { + mp4MediaStream = await Mp4MediaStream.load(file) + } catch (e) { + alert(e.message) + throw e + } + } + + function play() { + if (mp4MediaStream === undefined) { + alert('MP4 ファイルが未選択です') + return } - mp4MediaStream = await Mp4MediaStream.load(mp4File) const options = { repeat: document.getElementById('repeat').checked, @@ -23,17 +41,17 @@ document.addEventListener('DOMContentLoaded', async () => { let output = document.getElementById('output') output.srcObject = stream - // TODO: stream.getTracks()[0].stop() } - function getInputFile() { - const input = document.getElementById('input') - const files = input.files - if (files === null || files.length === 0) { + async function stop() { + if (mp4MediaStream === undefined) { return } - return files[0] + + await mp4MediaStream.stop() } - document.getElementById('input').addEventListener('change', play) + document.getElementById('input').addEventListener('change', load) + document.getElementById('play').addEventListener('click', play) + document.getElementById('stop').addEventListener('click', stop) }) diff --git a/examples/vite.config.mjs b/examples/vite.config.mjs index 8193ba6b..b7687971 100644 --- a/examples/vite.config.mjs +++ b/examples/vite.config.mjs @@ -24,7 +24,10 @@ export default defineConfig({ __dirname, '../packages/light-adjustment-gpu/dist/light_adjustment_gpu.mjs', ), - '@shiguredo/mp4-media-stream': resolve(__dirname, '../packages/mp4-media-stream/dist/mp4_media_stream.mjs'), + '@shiguredo/mp4-media-stream': resolve( + __dirname, + '../packages/mp4-media-stream/dist/mp4_media_stream.mjs', + ), }, }, optimizeDeps: { From bcdda7c9bc8ee9128bfe09fd6e90fa449d17ca77 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Wed, 16 Oct 2024 12:46:49 +0900 Subject: [PATCH 33/34] Update README.md --- examples/mp4-media-stream/main.mts | 2 +- packages/mp4-media-stream/README.md | 67 +++++++++++++++++++++++++---- 2 files changed, 59 insertions(+), 10 deletions(-) diff --git a/examples/mp4-media-stream/main.mts b/examples/mp4-media-stream/main.mts index 2a311037..6c5f88da 100644 --- a/examples/mp4-media-stream/main.mts +++ b/examples/mp4-media-stream/main.mts @@ -39,7 +39,7 @@ document.addEventListener('DOMContentLoaded', async () => { } const stream = mp4MediaStream.play(options) - let output = document.getElementById('output') + const output = document.getElementById('output') output.srcObject = stream } diff --git a/packages/mp4-media-stream/README.md b/packages/mp4-media-stream/README.md index 8981f705..a39641a6 100644 --- a/packages/mp4-media-stream/README.md +++ b/packages/mp4-media-stream/README.md @@ -1,11 +1,60 @@ # @shiguredo/mp4-media-stream -未対応: -- H.264 / Opus 以外のコーデック -- 再生位置指定（シーク） -- 再生の一時中断 -- 再生速度変更 -- ファイルのインクリメンタルな読み込み（メモリ消費量削減） -- edts ボックスのハンドリング -- B フレーム -- Chrome / Edge 以外のブラウザ +[![npm version](https://badge.fury.io/js/@shiguredo%2Fmp4-media-stream.svg)](https://badge.fury.io/js/@shiguredo%2Fmp4-media-stream) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +ファイルや URL から取得した MP4 データから MediaStream を作成するためのライブラリです。 + +## 使い方 + +```typescript +import { Mp4MediaStream } from '@shiguredo/mp4-media-stream' + +// ファイルなどから取得した MP4 データを読み込む +const mp4MediaStream = await Mp4MediaStream.load(mp4FileBlob) + +// 指定の MP4 を再生するための MediaStream を作成する +const stream = mp4MediaStream.play(options) + +// Video 要素の入力に作成された MediaStream を設定する +const video = document.getElementById('video') +video.srcObject = stream +``` + +実際の動作は[デモページ](https://shiguredo.github.io/media-processors/examples/mp4-media-stream/)（ +[ソースコード](https://github.com/shiguredo/media-processors/blob/develop/examples/mp4-media-stream/main.mts)）で確認できます。 + +## サポートブラウザ + +本ライブラリは Chrome や Edge 等の Chromium ベースのブラウザで動作します。 + +## 未対応機能 + +以下の機能には現時点では対応していません: +- H.264 / Opus 以外のコーデックを含んだ MP4 の再生 +- 再生開始位置の指定（シーク） +- 再生の一時停止・再開 +- 数 GB を超える MP4 ファイルの再生 +- MP4 の edts ボックスのハンドリング（edts を使ってリップシンクが調整されている場合には再生時にズレる可能性がある） +- B フレームが有効になっている H.264 ストリーム（映像の再生順序が正しくならない可能性がある） + +## ライセンス + +[Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) + +``` +Copyright 2024-2024, Takeru Ohta (Original Author) +Copyright 2024-2024, Shiguredo Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` From f09464a37c3759f9bb7e348a3cd5208a6476a961 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Wed, 16 Oct 2024 14:23:17 +0900 Subject: [PATCH 34/34] Update package.json --- packages/mp4-media-stream/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mp4-media-stream/package.json b/packages/mp4-media-stream/package.json index a8135e14..6d457587 100644 --- a/packages/mp4-media-stream/package.json +++ b/packages/mp4-media-stream/package.json @@ -1,7 +1,7 @@ { "name": "@shiguredo/mp4-media-stream", "version": "2024.1.0", - "description": "Generate MediaStream from MP4 file", + "description": "Library to generate MediaStream from MP4 file", "author": "Shiguredo Inc.", "license": "Apache-2.0", "main": "dist/mp4_media_stream.js",