From 140e857d940986a116d1d476aa8ed8655823e229 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Fri, 6 Dec 2024 12:51:28 +0900 Subject: [PATCH 1/7] =?UTF-8?q?VP8=20=E5=AF=BE=E5=BF=9C=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/wasm/src/mp4.rs | 15 ++++++++++++++- packages/mp4-media-stream/wasm/src/player.rs | 4 ++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/mp4-media-stream/wasm/src/mp4.rs b/packages/mp4-media-stream/wasm/src/mp4.rs index 1312247e..6a6be341 100644 --- a/packages/mp4-media-stream/wasm/src/mp4.rs +++ b/packages/mp4-media-stream/wasm/src/mp4.rs @@ -6,7 +6,7 @@ use shiguredo_mp4::{ aux::SampleTableAccessor, boxes::{ Avc1Box, FtypBox, HdlrBox, IgnoredBox, MoovBox, Mp4aBox, OpusBox, SampleEntry, StblBox, - TrakBox, + TrakBox, Vp08Box, }, BaseBox, Decode, Either, Encode, }; @@ -38,6 +38,15 @@ impl VideoDecoderConfig { coded_height: b.visual.height, } } + + pub fn from_vp08_box(b: &Vp08Box) -> Self { + Self { + codec: "vp08".to_owned(), + description: Vec::new(), + coded_width: b.visual.width, + coded_height: b.visual.height, + } + } } #[derive(Debug, Serialize)] @@ -111,6 +120,7 @@ impl Track { match sample_table.stbl_box().stsd_box.entries.first() { Some(SampleEntry::Avc1(_)) => (), + Some(SampleEntry::Vp08(_)) => (), Some(SampleEntry::Opus(_)) => (), Some(SampleEntry::Mp4a(_)) => (), Some(b) => { @@ -201,6 +211,9 @@ impl Mp4 { SampleEntry::Avc1(b) => { video_configs.push(VideoDecoderConfig::from_avc1_box(b)); } + SampleEntry::Vp08(b) => { + video_configs.push(VideoDecoderConfig::from_vp08_box(b)); + } SampleEntry::Opus(b) => { audio_configs.push(AudioDecoderConfig::from_opus_box(b)); } diff --git a/packages/mp4-media-stream/wasm/src/player.rs b/packages/mp4-media-stream/wasm/src/player.rs index 7569a6c1..af09f4f4 100644 --- a/packages/mp4-media-stream/wasm/src/player.rs +++ b/packages/mp4-media-stream/wasm/src/player.rs @@ -180,6 +180,10 @@ impl TrackPlayer { let config = VideoDecoderConfig::from_avc1_box(b); WasmApi::create_video_decoder(self.player_id, config).await } + SampleEntry::Vp08(b) => { + let config = VideoDecoderConfig::from_vp08_box(b); + WasmApi::create_video_decoder(self.player_id, config).await + } SampleEntry::Opus(b) => { let config = AudioDecoderConfig::from_opus_box(b); WasmApi::create_audio_decoder(self.player_id, config).await From e1f1f1de06181bc8b18c664f41e2b47f0a0e8537 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Fri, 6 Dec 2024 13:00:51 +0900 Subject: [PATCH 2/7] =?UTF-8?q?=E3=82=B3=E3=83=BC=E3=83=87=E3=83=83?= =?UTF-8?q?=E3=82=AF=E5=90=8D=E3=81=AF=20vp8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/wasm/src/mp4.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mp4-media-stream/wasm/src/mp4.rs b/packages/mp4-media-stream/wasm/src/mp4.rs index 6a6be341..a6c25f5a 100644 --- a/packages/mp4-media-stream/wasm/src/mp4.rs +++ b/packages/mp4-media-stream/wasm/src/mp4.rs @@ -41,7 +41,7 @@ impl VideoDecoderConfig { pub fn from_vp08_box(b: &Vp08Box) -> Self { Self { - codec: "vp08".to_owned(), + codec: "vp8".to_owned(), description: Vec::new(), coded_width: b.visual.width, coded_height: b.visual.height, From 154d23a42bf54a945a1470009e34663549927965 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Fri, 6 Dec 2024 13:01:33 +0900 Subject: [PATCH 3/7] =?UTF-8?q?description=20=E3=81=8C=E7=A9=BA=E3=81=AE?= =?UTF-8?q?=E5=A0=B4=E5=90=88=E3=81=AE=E5=AF=BE=E5=BF=9C=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/src/mp4_media_stream.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index 130e05eb..69b2a920 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -201,6 +201,10 @@ class Mp4MediaStream { // JSON.parse() の結果では config.description の型は number[] となって期待とは異なるので // ここで適切な型に変換している config.description = new Uint8Array(config.description as object as number[]) + if (config.description.byteLength == 0) { + // コーデックによっては description が存在しないので空なら削除する + delete config.description + } if (!(await VideoDecoder.isConfigSupported(config)).supported) { throw new Error(`Unsupported video decoder configuration: ${JSON.stringify(config)}`) @@ -238,6 +242,10 @@ class Mp4MediaStream { // JSON.parse() の結果では config.description の型は number[] となって期待とは異なるので // ここで適切な型に変換している config.description = new Uint8Array(config.description as object as number[]) + if (config.description.byteLength == 0) { + // コーデックによっては description が存在しないので空なら削除する + delete config.description + } const init = { output: async (frame: VideoFrame) => { From 588f2acd879ab8de6f6165c1e877250580e1ac23 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Fri, 6 Dec 2024 13:04:27 +0900 Subject: [PATCH 4/7] =?UTF-8?q?VideoFrame=20/=20AudioData=20=E3=81=AE=20cl?= =?UTF-8?q?ose()=20=E3=81=AF=20finally=20=E3=81=AE=E4=B8=AD=E3=81=A7?= =?UTF-8?q?=E5=91=BC=E3=81=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mp4-media-stream/src/mp4_media_stream.ts | 54 ++++++++++--------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index 69b2a920..23833334 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -249,19 +249,22 @@ class Mp4MediaStream { const init = { output: async (frame: VideoFrame) => { - if (player.canvas === undefined || player.canvasCtx === undefined) { - return - } - try { - player.canvas.width = frame.displayWidth - player.canvas.height = frame.displayHeight - player.canvasCtx.drawImage(frame, 0, 0) + if (player.canvas === undefined || player.canvasCtx === undefined) { + return + } + + try { + player.canvas.width = frame.displayWidth + player.canvas.height = frame.displayHeight + player.canvasCtx.drawImage(frame, 0, 0) + } catch (error) { + // エラーが発生した場合には再生を停止する + await this.stopPlayer(playerId) + throw error + } + } finally { frame.close() - } catch (error) { - // エラーが発生した場合には再生を停止する - await this.stopPlayer(playerId) - throw error } }, error: async (error: DOMException) => { @@ -294,21 +297,24 @@ class Mp4MediaStream { const config = this.wasmJsonToValue(configWasmJson) as AudioDecoderConfig const init = { output: async (data: AudioData) => { - if (player.audioInputNode === undefined) { - return - } - try { - const samples = new Float32Array(data.numberOfFrames * data.numberOfChannels) - data.copyTo(samples, { planeIndex: 0, format: 'f32' }) - data.close() + if (player.audioInputNode === undefined) { + return + } - const timestamp = data.timestamp - player.audioInputNode.port.postMessage({ timestamp, samples }, [samples.buffer]) - } catch (e) { - // エラーが発生した場合には再生を停止する - await this.stopPlayer(playerId) - throw e + try { + const samples = new Float32Array(data.numberOfFrames * data.numberOfChannels) + data.copyTo(samples, { planeIndex: 0, format: 'f32' }) + + const timestamp = data.timestamp + player.audioInputNode.port.postMessage({ timestamp, samples }, [samples.buffer]) + } catch (e) { + // エラーが発生した場合には再生を停止する + await this.stopPlayer(playerId) + throw e + } + } finally { + data.close() } }, error: async (error: DOMException) => { From aa9f30f54c65e694721ee24e1794c20e4ea163ca Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Fri, 6 Dec 2024 13:08:55 +0900 Subject: [PATCH 5/7] =?UTF-8?q?lint=20=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/src/mp4_media_stream.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index 23833334..d3ad83d0 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -201,9 +201,9 @@ class Mp4MediaStream { // JSON.parse() の結果では config.description の型は number[] となって期待とは異なるので // ここで適切な型に変換している config.description = new Uint8Array(config.description as object as number[]) - if (config.description.byteLength == 0) { + if (config.description.byteLength === 0) { // コーデックによっては description が存在しないので空なら削除する - delete config.description + config.description = undefined } if (!(await VideoDecoder.isConfigSupported(config)).supported) { @@ -242,9 +242,9 @@ class Mp4MediaStream { // JSON.parse() の結果では config.description の型は number[] となって期待とは異なるので // ここで適切な型に変換している config.description = new Uint8Array(config.description as object as number[]) - if (config.description.byteLength == 0) { + if (config.description.byteLength === 0) { // コーデックによっては description が存在しないので空なら削除する - delete config.description + config.description = undefined } const init = { From fd5619b6cee96773e3d38398cafd7f59fadef28d Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Fri, 6 Dec 2024 13:46:15 +0900 Subject: [PATCH 6/7] =?UTF-8?q?=E3=80=8C=E5=AF=BE=E5=BF=9C=E3=82=B3?= =?UTF-8?q?=E3=83=BC=E3=83=87=E3=83=83=E3=82=AF=E3=80=8D=E3=82=BB=E3=82=AF?= =?UTF-8?q?=E3=82=B7=E3=83=A7=E3=83=B3=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/mp4-media-stream/README.md b/packages/mp4-media-stream/README.md index 51cf85fe..66df35b0 100644 --- a/packages/mp4-media-stream/README.md +++ b/packages/mp4-media-stream/README.md @@ -28,10 +28,18 @@ video.srcObject = stream 本ライブラリは Chrome や Edge 等の Chromium ベースのブラウザで動作します。 +## 対応コーデック + +- 映像: + - H.264 + - VP8 +- 音声: + - AAC + - Opus + ## 未対応機能 以下の機能には現時点では対応していません: -- H.264 / Opus / AAC 以外のコーデックを含んだ MP4 の再生 - 再生開始位置の指定（シーク） - 再生の一時停止・再開 - 数 GB を超える MP4 ファイルの再生 From 6a1ee92357669cf59c67f37efef931c4b2712307 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Fri, 6 Dec 2024 13:46:53 +0900 Subject: [PATCH 7/7] Update CHANGES.md --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index fe2af3c4..eb229120 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,8 @@ ## develop +- [ADD] `Mp4MediaStream` の対応コーデックに VP8 を追加する + - @sile - [ADD] `Mp4MediaStream` の対応コーデックに AAC を追加する - @sile