From 22ea2e1a1a8d1f630e557e90b42aa88a8cc6a85e Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Thu, 5 Dec 2024 16:32:36 +0900 Subject: [PATCH 1/7] =?UTF-8?q?shiguredo=5Fmp4=20=E3=81=AE=E3=83=90?= =?UTF-8?q?=E3=83=BC=E3=82=B8=E3=83=A7=E3=83=B3=E3=82=92=202024.4.0=20?= =?UTF-8?q?=E3=81=AB=E4=B8=8A=E3=81=92=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 38 +++++++++++------------ packages/mp4-media-stream/wasm/Cargo.toml | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 082f1655..29a59f47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "autocfg" @@ -99,9 +99,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "memchr" @@ -131,9 +131,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -143,9 +143,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "proc-macro2" -version = "1.0.87" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -167,18 +167,18 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", @@ -187,9 +187,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "itoa", "memchr", @@ -199,9 +199,9 @@ dependencies = [ [[package]] name = "shiguredo_mp4" -version = "2024.3.0" +version = "2024.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298dae998ee0ea59ac50b3fb6ada19da86672fd14751a7a2d7f9aeeff7e49fa4" +checksum = "dcf4f0930981f4cbd245d45fe7f3159ee71a66fe3027fc0f4615f6f7b55d9fcd" [[package]] name = "slab" @@ -214,9 +214,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.79" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -225,6 +225,6 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" diff --git a/packages/mp4-media-stream/wasm/Cargo.toml b/packages/mp4-media-stream/wasm/Cargo.toml index 3a554db8..e21e331f 100644 --- a/packages/mp4-media-stream/wasm/Cargo.toml +++ b/packages/mp4-media-stream/wasm/Cargo.toml @@ -14,4 +14,4 @@ futures = "0.3.31" orfail = { version = "1.1.0", features = ["serde"] } serde = { version = "1.0.210", features = ["derive"] } serde_json = "1.0.128" -shiguredo_mp4 = "2024.3.0" +shiguredo_mp4 = "2024.4.0" From 89c29957d7ac22fd8594d0faa0f1b0264b8ac0bc Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Fri, 6 Dec 2024 10:37:40 +0900 Subject: [PATCH 2/7] =?UTF-8?q?=E3=81=84=E3=81=A3=E3=81=9F=E3=82=93?= =?UTF-8?q?=E3=82=B3=E3=83=BC=E3=83=87=E3=83=83=E3=82=AF=E6=AF=8E=E3=81=AF?= =?UTF-8?q?=20mp4a.40.2=20=E3=81=AB=E6=B1=BA=E3=82=81=E6=89=93=E3=81=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/wasm/src/mp4.rs | 15 ++++++++++++++- packages/mp4-media-stream/wasm/src/player.rs | 4 ++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/mp4-media-stream/wasm/src/mp4.rs b/packages/mp4-media-stream/wasm/src/mp4.rs index 937c350d..6bd22251 100644 --- a/packages/mp4-media-stream/wasm/src/mp4.rs +++ b/packages/mp4-media-stream/wasm/src/mp4.rs @@ -5,7 +5,8 @@ use serde::Serialize; use shiguredo_mp4::{ aux::SampleTableAccessor, boxes::{ - Avc1Box, FtypBox, HdlrBox, IgnoredBox, MoovBox, OpusBox, SampleEntry, StblBox, TrakBox, + Avc1Box, FtypBox, HdlrBox, IgnoredBox, MoovBox, Mp4aBox, OpusBox, SampleEntry, StblBox, + TrakBox, }, BaseBox, Decode, Either, Encode, }; @@ -55,6 +56,14 @@ impl AudioDecoderConfig { number_of_channels: b.audio.channelcount as u8, } } + + pub fn from_mp4a_box(b: &Mp4aBox) -> Self { + Self { + codec: "mp4a.40.2".to_owned(), + sample_rate: b.audio.samplerate.integer, + number_of_channels: b.audio.channelcount as u8, + } + } } #[derive(Debug, Serialize)] @@ -86,6 +95,7 @@ impl Track { match sample_table.stbl_box().stsd_box.entries.first() { Some(SampleEntry::Avc1(_)) => (), Some(SampleEntry::Opus(_)) => (), + Some(SampleEntry::Mp4a(_)) => (), Some(b) => { return Err(Failure::new(format!( "Unsupported {kind}codec: {}", @@ -177,6 +187,9 @@ impl Mp4 { SampleEntry::Opus(b) => { audio_configs.push(AudioDecoderConfig::from_opus_box(b)); } + SampleEntry::Mp4a(b) => { + audio_configs.push(AudioDecoderConfig::from_mp4a_box(b)); + } _ => { // `Track` 作成時にチェックしているのでここには来ない unreachable!() diff --git a/packages/mp4-media-stream/wasm/src/player.rs b/packages/mp4-media-stream/wasm/src/player.rs index 054c17cb..7569a6c1 100644 --- a/packages/mp4-media-stream/wasm/src/player.rs +++ b/packages/mp4-media-stream/wasm/src/player.rs @@ -184,6 +184,10 @@ impl TrackPlayer { let config = AudioDecoderConfig::from_opus_box(b); WasmApi::create_audio_decoder(self.player_id, config).await } + SampleEntry::Mp4a(b) => { + let config = AudioDecoderConfig::from_mp4a_box(b); + WasmApi::create_audio_decoder(self.player_id, config).await + } _ => { // MP4::load() の中で非対応コーデックのチェックは行っているので、ここに来ることはない unreachable!() From c653ab61c612ccee5c2c69b6087cae1d42e84c0f Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Fri, 6 Dec 2024 10:44:31 +0900 Subject: [PATCH 3/7] =?UTF-8?q?f32=20=E4=BB=A5=E5=A4=96=E3=81=AE=20AudioDa?= =?UTF-8?q?ta.format=20=E3=81=AB=E5=AF=BE=E5=BF=9C=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/src/mp4_media_stream.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/packages/mp4-media-stream/src/mp4_media_stream.ts b/packages/mp4-media-stream/src/mp4_media_stream.ts index 4e18618c..130e05eb 100644 --- a/packages/mp4-media-stream/src/mp4_media_stream.ts +++ b/packages/mp4-media-stream/src/mp4_media_stream.ts @@ -291,14 +291,8 @@ class Mp4MediaStream { } try { - if (data.format !== 'f32') { - // フォーマットは f32 だけが来る想定。 - // もし他のフォーマットが来ることがあれば、その都度対応すること。 - throw Error(`Unsupported audio data format: ${data.format}"`) - } - const samples = new Float32Array(data.numberOfFrames * data.numberOfChannels) - data.copyTo(samples, { planeIndex: 0 }) + data.copyTo(samples, { planeIndex: 0, format: 'f32' }) data.close() const timestamp = data.timestamp From d2af2b6a3583b93ae9edadcb54fcd342e4740acd Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Fri, 6 Dec 2024 11:34:48 +0900 Subject: [PATCH 4/7] =?UTF-8?q?[WIP]=20=E3=82=B3=E3=83=BC=E3=83=87?= =?UTF-8?q?=E3=83=83=E3=82=AF=E5=90=8D=E3=82=92=E3=81=A1=E3=82=83=E3=82=93?= =?UTF-8?q?=E3=81=A8=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/wasm/src/mp4.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/mp4-media-stream/wasm/src/mp4.rs b/packages/mp4-media-stream/wasm/src/mp4.rs index 6bd22251..4db0e11d 100644 --- a/packages/mp4-media-stream/wasm/src/mp4.rs +++ b/packages/mp4-media-stream/wasm/src/mp4.rs @@ -58,8 +58,16 @@ impl AudioDecoderConfig { } pub fn from_mp4a_box(b: &Mp4aBox) -> Self { + let mut codec = format!( + "mp4a.{:02X}", + b.esds_box.es.dec_config_descr.object_type_indication + ); + if b.esds_box.es.dec_config_descr.object_type_indication == 0x40 { + // TODO: profile を見る + codec.push_str(".2"); + }; Self { - codec: "mp4a.40.2".to_owned(), + codec, sample_rate: b.audio.samplerate.integer, number_of_channels: b.audio.channelcount as u8, } From ddd2ab2dc3c9c5d37d367b0b149f2ffeaaeab644 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Fri, 6 Dec 2024 12:36:17 +0900 Subject: [PATCH 5/7] =?UTF-8?q?=E3=82=B3=E3=83=BC=E3=83=87=E3=83=83?= =?UTF-8?q?=E3=82=AF=E5=90=8D=E3=82=92=E3=81=A1=E3=82=83=E3=82=93=E3=81=A8?= =?UTF-8?q?=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mp4-media-stream/wasm/src/mp4.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/mp4-media-stream/wasm/src/mp4.rs b/packages/mp4-media-stream/wasm/src/mp4.rs index 4db0e11d..1312247e 100644 --- a/packages/mp4-media-stream/wasm/src/mp4.rs +++ b/packages/mp4-media-stream/wasm/src/mp4.rs @@ -63,8 +63,17 @@ impl AudioDecoderConfig { b.esds_box.es.dec_config_descr.object_type_indication ); if b.esds_box.es.dec_config_descr.object_type_indication == 0x40 { - // TODO: profile を見る - codec.push_str(".2"); + if let Some(b) = b + .esds_box + .es + .dec_config_descr + .dec_specific_info + .payload + .get(0) + { + let audio_object_type = b >> 3; + codec.push_str(&format!(".{audio_object_type}")); + } }; Self { codec, From 369852fb1508588cf86ed8d0d55dee5d3301e0f2 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Fri, 6 Dec 2024 12:39:25 +0900 Subject: [PATCH 6/7] Update README.md --- packages/mp4-media-stream/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mp4-media-stream/README.md b/packages/mp4-media-stream/README.md index 4b999c15..51cf85fe 100644 --- a/packages/mp4-media-stream/README.md +++ b/packages/mp4-media-stream/README.md @@ -31,7 +31,7 @@ video.srcObject = stream ## 未対応機能 以下の機能には現時点では対応していません: -- H.264 / Opus 以外のコーデックを含んだ MP4 の再生 +- H.264 / Opus / AAC 以外のコーデックを含んだ MP4 の再生 - 再生開始位置の指定（シーク） - 再生の一時停止・再開 - 数 GB を超える MP4 ファイルの再生 From bde1e5636bfc63a2922421598a48f7465396832d Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Fri, 6 Dec 2024 12:40:15 +0900 Subject: [PATCH 7/7] Update CHANGES.md --- CHANGES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 28e8afde..edf4aebe 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,9 @@ ## develop +- [ADD] `Mp4MediaStream` の対応コーデックに AAC を追加する + - @sile + ## mp4-media-stream-2024.2.0 - [CHANGE] `Mp4MediaStream.play()` を非同期にする