diff --git a/android/src/main/java/com/oney/WebRTCModule/AudioSamplesInterceptor.java b/android/src/main/java/com/oney/WebRTCModule/AudioSamplesInterceptor.java new file mode 100644 index 000000000..cea5c0b18 --- /dev/null +++ b/android/src/main/java/com/oney/WebRTCModule/AudioSamplesInterceptor.java @@ -0,0 +1,37 @@ +package com.oney.WebRTCModule; + +import android.annotation.SuppressLint; + +import org.webrtc.audio.JavaAudioDeviceModule.SamplesReadyCallback; +import org.webrtc.audio.JavaAudioDeviceModule.AudioSamples; + +import java.util.HashMap; + +// Note: This is a happy path implementation and heavily based off of Flutters webrtc implementation +// https://github.com/flutter-webrtc/flutter-webrtc/blob/main/android/src/main/java/com/cloudwebrtc/webrtc/record/AudioSamplesInterceptor.java + +/** JavaAudioDeviceModule allows attaching samples callback only on building + * We don't want to instantiate VideoFileRenderer and codecs at this step + * It's simple dummy class, it does nothing until samples are necessary */ +@SuppressWarnings("WeakerAccess") +public class AudioSamplesInterceptor implements SamplesReadyCallback { + + @SuppressLint("UseSparseArrays") + protected final HashMap callbacks = new HashMap<>(); + + @Override + public void onWebRtcAudioRecordSamplesReady(AudioSamples audioSamples) { + for (SamplesReadyCallback callback : callbacks.values()) { + callback.onWebRtcAudioRecordSamplesReady(audioSamples); + } + } + + public void attachCallback(String id, SamplesReadyCallback callback) throws Exception { + callbacks.put(id, callback); + } + + public void detachCallback(String id) { + callbacks.remove(id); + } + +} \ No newline at end of file diff --git a/android/src/main/java/com/oney/WebRTCModule/MediaRecorderImpl.java b/android/src/main/java/com/oney/WebRTCModule/MediaRecorderImpl.java index 43623df98..b3a580f35 100644 --- a/android/src/main/java/com/oney/WebRTCModule/MediaRecorderImpl.java +++ b/android/src/main/java/com/oney/WebRTCModule/MediaRecorderImpl.java @@ -12,6 +12,7 @@ public class MediaRecorderImpl { private final VideoTrack videoTrack; private final HashMap videoTrackInfo; private File file; + private AudioSamplesInterceptor audioSamplesInterceptor; /** * VideoAudioFileRenderer is heavily influenced by Flutter's webRTC implementation @@ -24,12 +25,19 @@ public class MediaRecorderImpl { * * @param id Id * @param videoTrack Video track + * @param videoTrackInfo Necessary video track info, like frame rate * @param interceptor Eventually we'll add this param as an AudioSamplesInterceptor to pipe audio */ - protected MediaRecorderImpl(String id, VideoTrack videoTrack, HashMap videoTrackInfo) { + protected MediaRecorderImpl( + String id, + VideoTrack videoTrack, + HashMap videoTrackInfo, + AudioSamplesInterceptor interceptor + ) { this.id = id; this.videoTrack = videoTrack; this.videoTrackInfo = videoTrackInfo; + this.audioSamplesInterceptor = interceptor; } protected void start(File file) throws Exception { @@ -50,15 +58,19 @@ protected void start(File file) throws Exception { videoTrackInfo ); videoTrack.addSink(videoAudioFileRenderer); + audioSamplesInterceptor.attachCallback(id, videoAudioFileRenderer); Log.i(TAG, "Started media recorder! " + videoAudioFileRenderer.toString()); } } protected File stop() { - // Remove sink from videoTrack (https://chromium.googlesource.com/external/webrtc/+/HEAD/sdk/android/api/org/webrtc/VideoTrack.java#49) - // Release videoAudioFileRenderer, and set to null + videoTrack.removeSink(videoAudioFileRenderer); + videoAudioFileRenderer.release(); + videoAudioFileRenderer = null; - return file; + audioSamplesInterceptor.detachCallback(id); + + return file; } } \ No newline at end of file diff --git a/android/src/main/java/com/oney/WebRTCModule/VideoAudioFileRenderer.java b/android/src/main/java/com/oney/WebRTCModule/VideoAudioFileRenderer.java index d097f3f18..6633a0798 100644 --- a/android/src/main/java/com/oney/WebRTCModule/VideoAudioFileRenderer.java +++ b/android/src/main/java/com/oney/WebRTCModule/VideoAudioFileRenderer.java @@ -9,6 +9,8 @@ import android.util.Log; import android.view.Surface; +import org.webrtc.audio.JavaAudioDeviceModule; +import org.webrtc.audio.JavaAudioDeviceModule.SamplesReadyCallback; import org.webrtc.EglBase; import org.webrtc.GlRectDrawer; import org.webrtc.VideoFrame; @@ -22,7 +24,7 @@ // Note: This is a happy path implementation and heavily based off of Flutters webrtc implementation // https://github.com/flutter-webrtc/flutter-webrtc/blob/main/android/src/main/java/com/cloudwebrtc/webrtc/record/VideoFileRenderer.java -public class VideoAudioFileRenderer implements VideoSink { +public class VideoAudioFileRenderer implements VideoSink, SamplesReadyCallback { static final String TAG = VideoAudioFileRenderer.class.getCanonicalName(); // Following are used to render the frames as they come in @@ -36,14 +38,18 @@ public class VideoAudioFileRenderer implements VideoSink { // Following are used to encode the video and audio and mux them together private MediaMuxer mediaMuxer; private MediaCodec videoEncoder = null; + private MediaCodec audioEncoder = null; // Buffer info for the separate tracks private MediaCodec.BufferInfo videoBufferInfo; + private MediaCodec.BufferInfo audioBufferInfo; // Threads and handlers to do the actual work on so we don't lock up // the main thread private final HandlerThread videoRenderThread; private final Handler videoRenderThreadHandler; + private final HandlerThread audioRenderThread; + private final Handler audioRenderThreadHandler; // Used to determine if it's the first frame and get the proper incoming width and height // These are then used when rendering the frame out with EGL @@ -70,15 +76,21 @@ public class VideoAudioFileRenderer implements VideoSink { // Keep track of the currently selected audio and video track index. This is needed as the // proper index is required when writing out the sample data private int videoTrackIndex = -1; + private int audioTrackIndex = -1; public VideoAudioFileRenderer(String outputFilePath, final EglBase.Context eglContext, final HashMap videoTrackInfo) throws IOException { videoBufferInfo = new MediaCodec.BufferInfo(); + audioBufferInfo = new MediaCodec.BufferInfo(); videoRenderThread = new HandlerThread(TAG + "VideoRenderThread"); videoRenderThread.start(); videoRenderThreadHandler = new Handler(videoRenderThread.getLooper()); + audioRenderThread = new HandlerThread(TAG + "AudioRenderThread"); + audioRenderThread.start(); + audioRenderThreadHandler = new Handler(audioRenderThread.getLooper()); + this.eglContext = eglContext; this.frameRate = videoTrackInfo.get("frameRate"); @@ -149,76 +161,206 @@ private void muxVideo() { // Keep dequeueing until nothing left boolean shouldMux = true; while(shouldMux) { - int encoderStatus = videoEncoder.dequeueOutputBuffer(videoBufferInfo, 10000); - - switch(encoderStatus) { - case MediaCodec.INFO_TRY_AGAIN_LATER: - shouldMux = false; - break; - case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: { - MediaFormat newFormat = videoEncoder.getOutputFormat(); - Log.w(TAG, "video encoder format changed: " + newFormat); - - // In theory we should handle if this changes although was running into some crashing - // if I saw any. So for now since we lock a lot of params this is likely fine but - // may need to find some time to figure out why it was crashing and address it - if (videoTrackIndex == -1) { - videoTrackIndex = mediaMuxer.addTrack(newFormat); - Log.i(TAG, "ADDING video track to muxer " + videoTrackIndex); - mediaMuxer.start(); - muxerStarted = true; - } - - if (!muxerStarted) { - shouldMux = false; - } - break; - } - default: { - // Some valid statuses are less than zero but if we hit this default case and - // we still hit a status we didn't expect to handle we shouldn't try to encode - // any data and just skip handling the status - if (encoderStatus < 0) { - Log.e(TAG, "Unexpected video encoderStatus: " + encoderStatus); - break; - } - - // No edge case then do the work - ByteBuffer encodedData = videoEncoder.getOutputBuffer(encoderStatus); - if (encodedData == null) { - Log.e(TAG, "encoded video data " + encoderStatus + "null for some reason"); - shouldMux = false; - break; - } - - encodedData.position(videoBufferInfo.offset); - encodedData.limit(videoBufferInfo.offset + videoBufferInfo.size); - - if (videoFrameStart == 0 && videoBufferInfo.presentationTimeUs != 0) { - videoFrameStart = videoBufferInfo.presentationTimeUs; - } - - videoBufferInfo.presentationTimeUs -= videoFrameStart; - if (muxerStarted) { - Log.i(TAG, "WRITING VIDEO DATA"); - mediaMuxer.writeSampleData(videoTrackIndex, encodedData, videoBufferInfo); - } - - isRunning = isRunning && (videoBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == 0; - - videoEncoder.releaseOutputBuffer(encoderStatus, false); - if ((videoBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { - shouldMux = false; - break; - } - } - } + int encoderStatus = videoEncoder.dequeueOutputBuffer(videoBufferInfo, 10000); + + switch(encoderStatus) { + case MediaCodec.INFO_TRY_AGAIN_LATER: + shouldMux = false; + break; + case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: { + MediaFormat newFormat = videoEncoder.getOutputFormat(); + Log.w(TAG, "video encoder format changed: " + newFormat); + + // In theory we should handle if this changes although was running into some crashing + // if I saw any. So for now since we lock a lot of params this is likely fine but + // may need to find some time to figure out why it was crashing and address it + if (videoTrackIndex == -1) { + videoTrackIndex = mediaMuxer.addTrack(newFormat); + Log.i(TAG, "ADDING video track to muxer " + videoTrackIndex); + } + + if (audioTrackIndex != -1 && !muxerStarted) { + Log.i(TAG, "muxVideo STARTING MUXER"); + mediaMuxer.start(); + muxerStarted = true; + } + + if (!muxerStarted) { + shouldMux = false; + } + break; + } + default: { + // Some valid statuses are less than zero but if we hit this default case and + // we still hit a status we didn't expect to handle we shouldn't try to encode + // any data and just skip handling the status + if (encoderStatus < 0) { + Log.e(TAG, "Unexpected video encoderStatus: " + encoderStatus); + break; + } + + // No edge case then do the work + ByteBuffer encodedData = videoEncoder.getOutputBuffer(encoderStatus); + if (encodedData == null) { + Log.e(TAG, "encoded video data " + encoderStatus + "null for some reason"); + shouldMux = false; + break; + } + + encodedData.position(videoBufferInfo.offset); + encodedData.limit(videoBufferInfo.offset + videoBufferInfo.size); + + if (videoFrameStart == 0 && videoBufferInfo.presentationTimeUs != 0) { + videoFrameStart = videoBufferInfo.presentationTimeUs; + } + + videoBufferInfo.presentationTimeUs -= videoFrameStart; + if (muxerStarted) { + Log.i(TAG, "WRITING VIDEO DATA"); + mediaMuxer.writeSampleData(videoTrackIndex, encodedData, videoBufferInfo); + } + + isRunning = isRunning && (videoBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == 0; + + videoEncoder.releaseOutputBuffer(encoderStatus, false); + if ((videoBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + shouldMux = false; + break; + } + } + } + } + } + + // This is the event that receives the audio data from the device + @Override + public void onWebRtcAudioRecordSamplesReady(JavaAudioDeviceModule.AudioSamples audioSamples) { + if (!isRunning) { + return; + } + + // TODO: Might need to validate that we are good to do the following work... + audioRenderThreadHandler.post(() -> { + if (audioEncoder == null) { + try { + audioEncoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC); + MediaFormat audioFormat = MediaFormat.createAudioFormat( + MediaFormat.MIMETYPE_AUDIO_AAC, + audioSamples.getSampleRate(), + audioSamples.getChannelCount() + ); + + // If these aren't set it will cause an exception. I can't remember if it was when + // you configure or when you tried to dequeue + audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, 64 * 1024); + audioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); + + audioEncoder.configure(audioFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + audioEncoder.start(); + } catch (IOException exception) { + Log.wtf(TAG, exception); + } + } + + int index = audioEncoder.dequeueInputBuffer(0); + if(index >= 0) { + ByteBuffer buffer = audioEncoder.getInputBuffer(index); + + byte[] audioData = audioSamples.getData(); + buffer.put(audioData); + + audioEncoder.queueInputBuffer(index, 0, audioData.length, presentationTime, 0); + // Comment from the flutter webrtc says -- // 1000000 microseconds / 48000hz / 2 bytes + // Likely this just uses the length and converts it over to the right unit over the span of + // the entire sample and we increment our stored presentationTime by that amount + presentationTime += audioData.length * 125 / 12; // TODO do better + } + muxAudio(); + }); + } + + private void muxAudio() { + // Keep dequeueing until nothing left + boolean shouldMux = true; + while (shouldMux) { + int encoderStatus = audioEncoder.dequeueOutputBuffer(audioBufferInfo, 0); + + switch(encoderStatus) { + case MediaCodec.INFO_TRY_AGAIN_LATER: + shouldMux = false; + break; + case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: { + MediaFormat newFormat = audioEncoder.getOutputFormat(); + Log.w(TAG, "audio encoder format changed: " + newFormat); + + if (audioTrackIndex == -1) { + audioTrackIndex = mediaMuxer.addTrack(newFormat); + Log.i(TAG, "ADDING audio track to muxer " + audioTrackIndex); + } + + if (videoTrackIndex != -1 && !muxerStarted) { + Log.i(TAG, "muxAudio STARTING MUXER"); + mediaMuxer.start(); + muxerStarted = true; + } + + if (!muxerStarted) { + shouldMux = false; + } + break; + } + default: { + // Some valid statuses are less than zero but if we hit this default case and + // we still hit a status we didn't expect to handle we shouldn't try to encode + // any data and just skip handling the status + if (encoderStatus < 0) { + Log.e(TAG, "Unexpected video encoderStatus: " + encoderStatus); + break; + } + + // No edge case then do the work + try { + ByteBuffer encodedData = audioEncoder.getOutputBuffer(encoderStatus); + if (encodedData == null) { + Log.e(TAG, "encoded audio data " + encoderStatus + "null for some reason"); + shouldMux = false; + break; + } + + encodedData.position(audioBufferInfo.offset); + encodedData.limit(audioBufferInfo.offset + audioBufferInfo.size); + + if (muxerStarted) { + Log.i(TAG, "WRITING AUDIO DATA"); + mediaMuxer.writeSampleData(audioTrackIndex, encodedData, audioBufferInfo); + } + + isRunning = isRunning && (audioBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == 0; + + audioEncoder.releaseOutputBuffer(encoderStatus, false); + if ((audioBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + shouldMux = false; + break; + } + } catch (Exception e) { + Log.wtf(TAG, e); + shouldMux = false; + } + break; + } + } } } public void release() { isRunning = false; + audioRenderThreadHandler.post(() -> { + audioEncoder.stop(); + audioEncoder.release(); + audioRenderThread.quit(); + }); + videoRenderThreadHandler.post(() -> { videoEncoder.stop(); videoEncoder.release(); diff --git a/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java b/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java index f75c36318..f4affcd41 100644 --- a/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java +++ b/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java @@ -41,6 +41,8 @@ public class WebRTCModule extends ReactContextBaseJavaModule { static final String TAG = WebRTCModule.class.getCanonicalName(); + private AudioDeviceModule audioDeviceModule; + PeerConnectionFactory mFactory; VideoEncoderFactory mVideoEncoderFactory; VideoDecoderFactory mVideoDecoderFactory; @@ -61,7 +63,7 @@ public WebRTCModule(ReactApplicationContext reactContext) { WebRTCModuleOptions options = WebRTCModuleOptions.getInstance(); - AudioDeviceModule adm = options.audioDeviceModule; + audioDeviceModule = options.audioDeviceModule; VideoEncoderFactory encoderFactory = options.videoEncoderFactory; VideoDecoderFactory decoderFactory = options.videoDecoderFactory; Loggable injectableLogger = options.injectableLogger; @@ -91,15 +93,15 @@ public WebRTCModule(ReactApplicationContext reactContext) { } } - if (adm == null) { - adm = JavaAudioDeviceModule.builder(reactContext).setEnableVolumeLogger(false).createAudioDeviceModule(); + if (audioDeviceModule == null) { + audioDeviceModule = JavaAudioDeviceModule.builder(reactContext).setEnableVolumeLogger(false).createAudioDeviceModule(); } Log.d(TAG, "Using video encoder factory: " + encoderFactory.getClass().getCanonicalName()); Log.d(TAG, "Using video decoder factory: " + decoderFactory.getClass().getCanonicalName()); mFactory = PeerConnectionFactory.builder() - .setAudioDeviceModule(adm) + .setAudioDeviceModule(audioDeviceModule) .setVideoEncoderFactory(encoderFactory) .setVideoDecoderFactory(decoderFactory) .createPeerConnectionFactory(); @@ -1349,8 +1351,11 @@ public void mediaRecorderCreate(String recorderId, String streamId) { // Get track info HashMap videoTrackInfo = getUserMediaImpl.getVideoTrackInfo(videoTrack); + // Create new audio interceptor + AudioSamplesInterceptor audioSamplesInterceptor = new AudioSamplesInterceptor(); + // Create new media recorder - MediaRecorderImpl recorder = new MediaRecorderImpl(recorderId, videoTrack, videoTrackInfo); + MediaRecorderImpl recorder = new MediaRecorderImpl(recorderId, videoTrack, videoTrackInfo, audioSamplesInterceptor); mediaRecorders.put(recorderId, recorder); }); }