From 86a85292aecf29f22f495500921abd95f1440e40 Mon Sep 17 00:00:00 2001 From: Steven Vandevelde Date: Sun, 10 Mar 2024 15:32:54 +0100 Subject: [PATCH] chore: add seeking to audio example --- examples/audio/src/index.ts | 169 ++++++++++++++++++++++++++++-------- 1 file changed, 133 insertions(+), 36 deletions(-) diff --git a/examples/audio/src/index.ts b/examples/audio/src/index.ts index 2137dff..06fbf7b 100644 --- a/examples/audio/src/index.ts +++ b/examples/audio/src/index.ts @@ -34,6 +34,7 @@ if (fi) const reader = new FileReader() note('Reading file') + console.log('File selected', file) reader.onload = (event: any) => { const data: ArrayBuffer = event.target.result @@ -54,11 +55,16 @@ async function init(fileName: string, fileData: ArrayBuffer) { note('Adding file to WNFS') const path = Path.file('private', fileName) - const { dataRoot } = await fs.write(path, 'bytes', new Uint8Array(fileData)) - FS.savePointer(dataRoot) + if ((await fs.exists(path)) === false) { + const { dataRoot } = await fs.write(path, 'bytes', new Uint8Array(fileData)) + FS.savePointer(dataRoot) + } + const fileSize = await fs.size(path) + console.log('File size (WNFS)', fileSize) + // Audio metadata note('Looking up audio metadata') @@ -81,33 +87,61 @@ async function init(fileName: string, fileData: ArrayBuffer) { console.log('Audio duration', audioDuration) console.log('Audio metadata', mediainfo.media.track) + // Audio frames + if (!mediainfo.media.track[1]?.FrameCount) + throw new Error('Failed to determine audio frame count') + if (!mediainfo.media.track[1]?.StreamSize) + throw new Error('Failed to determine audio stream size') + const audioFrameCount = mediainfo.media.track[1]?.FrameCount + const audioStreamSize = mediainfo.media.track[1]?.StreamSize + const audioFrameSize = Math.ceil(audioStreamSize / audioFrameCount) + + console.log('Audio frame count', audioFrameCount) + console.log('Audio stream size', audioStreamSize) + console.log('Audio frame size', audioFrameSize) + // Buffering const bufferSize = 512 * 1024 // 512 KB - const metadataSize = mediainfo?.media?.track[0]?.StreamSize + const metadataSize = mediainfo?.media?.track[0]?.StreamSize || 0 + const amountOfFramesToLoad = bufferSize / audioFrameSize + + let loading = false + let seeking = false - let start = 0 - let end = 0 let sourceBuffer: SourceBuffer + let buffered: { start: number; end: number } = { + start: 0, + end: 0, + } async function loadNext() { - if (src.readyState === 'closed' || sourceBuffer.updating) return - - if (end >= fileSize) { - note('Loaded all audio data') - if (src.readyState === 'open') src.endOfStream() + if ( + src.readyState !== 'open' || + sourceBuffer.updating || + seeking || + loading + ) return + loading = true + + // const buffered = { + // from: sourceBuffer.buffered.length ? sourceBuffer.buffered.start(0) : 0, + // to: sourceBuffer.buffered.length ? sourceBuffer.buffered.end(0) : 0, + // } + + let start = buffered.end + let end = start + bufferSize + let reachedEnd = false + + if (end > fileSize) { + end = fileSize + reachedEnd = true } - start = end - end = - start === 0 - ? metadataSize === undefined - ? bufferSize - : metadataSize - : start + bufferSize - if (end >= fileSize) end = fileSize + buffered.end = end note(`Loading bytes, offset: ${start} - length: ${end - start}`) + console.log(`Loading bytes from ${start} to ${end}`) const buffer = await fs.read(path, 'bytes', { offset: start, @@ -115,8 +149,18 @@ async function init(fileName: string, fileData: ArrayBuffer) { }) sourceBuffer.appendBuffer(buffer) + + loading = false + + if (reachedEnd) { + sourceBuffer.addEventListener('updateend', () => src.endOfStream(), { + once: true, + }) + } } + globalThis.loadNext = loadNext + // Media source note('Setting up media source') @@ -124,42 +168,73 @@ async function init(fileName: string, fileData: ArrayBuffer) { src.addEventListener('sourceopen', () => { if (src.sourceBuffers.length > 0) return - console.log('src.readyState', src.readyState) + src.duration = audioDuration - if (src.readyState == 'open') { - src.duration = audioDuration + sourceBuffer = src.addSourceBuffer(mimeType) + sourceBuffer.mode = 'sequence' - sourceBuffer = src.addSourceBuffer(mimeType) - sourceBuffer.addEventListener('updateend', () => loadNext(), { - once: true, - }) + // sourceBuffer.addEventListener('updateend', () => { + // console.log('updateend') + // }) - note('Loading initial audio buffer') - loadNext() - } + // Load initial frames + loadNext() }) // Create audio const audio = new Audio() audio.src = URL.createObjectURL(src) audio.controls = true - audio.volume = 0.5 - // audio.preload = 'metadata' + audio.volume = 0.1 audio.addEventListener('seeking', () => { + if (seeking) return + seeking = true + if (src.readyState === 'open') { - // Abort current segment append. + // Abort current segment append sourceBuffer.abort() } - // TODO: - // How do we determine what byte offset to load from based on the time. - // start = n + const time = audio.currentTime - loadNext() + sourceBuffer.remove(0, Infinity) + sourceBuffer.addEventListener( + 'updateend', + async (): Promise => { + sourceBuffer.timestampOffset = time + + const frame = Math.floor((time / audio.duration) * audioFrameCount) + + const buffer = await fs.read(path, 'bytes', { + offset: metadataSize + frame * audioFrameSize, + length: bufferSize, + }) + + const headerStart = getHeaderStart(buffer) + console.log('Header start', headerStart) + + buffered.start = metadataSize + frame * audioFrameSize + headerStart + buffered.end = buffered.start + + seeking = false + loadNext() + }, + { once: true } + ) + + console.log(`Seeking to ${Math.round((time / audio.duration) * 100)}%`) }) - audio.addEventListener('progress', () => loadNext()) + audio.addEventListener('timeupdate', () => { + if (seeking) return + if (audio.currentTime + 60 > sourceBuffer.timestampOffset) loadNext() + }) + + audio.addEventListener('waiting', () => { + console.log('Audio element is waiting for data') + loadNext() + }) document.body.appendChild(audio) } @@ -172,9 +247,31 @@ async function mediaInfoClient(covers: boolean) { return await MediaInfoFactory({ coverData: covers, + full: true, locateFile: () => { return new URL('mediainfo.js/MediaInfoModule.wasm', import.meta.url) .pathname }, }) } + +function getHeaderStart(buffer: Uint8Array) { + let headerStart = 0 + const SyncByte1 = 0xff + const SyncByte2 = 0xfb + const SyncByte3 = 0x90 // 224 + const SyncByte4 = 0x64 // 64 + + for (let i = 0; i + 1 < buffer.length; i++) { + if ( + buffer[i] === SyncByte1 && + buffer[i + 1] === SyncByte2 && + buffer[i + 2] === SyncByte3 && + buffer[i + 3] === SyncByte4 + ) { + return i + } + } + + return headerStart +}