Support WebAudio

content/shell/BUILD.gn

@@ -446,6 +446,7 @@ static_library("content_shell_lib") {
   if (is_cobalt && use_starboard_media) {
     deps += [
       "//starboard($starboard_toolchain)",
+      "//media/starboard:webaudio($starboard_toolchain)",
     ]
   }
 }

content/shell/renderer/shell_content_renderer_client.cc

@@ -191,7 +191,13 @@ void CreateRendererTestService(
 
 }  // namespace
 
-ShellContentRendererClient::ShellContentRendererClient() {}
+ShellContentRendererClient::ShellContentRendererClient() {
+#if BUILDFLAG(USE_STARBOARD_MEDIA)
+  // Registers a custom content::AudioDeviceFactory
+  starboard_audio_device_factory_ =
+      std::make_unique<media::StarboardAudioDeviceFactory>();
+#endif  // BUILDFLAG(USE_STARBOARD_MEDIA)
+}
 
 ShellContentRendererClient::~ShellContentRendererClient() {
 }

content/shell/renderer/shell_content_renderer_client.h

@@ -16,6 +16,10 @@
 // For BUILDFLAG(USE_STARBOARD_MEDIA)
 #include "build/build_config.h"
 
+#if BUILDFLAG(USE_STARBOARD_MEDIA)
+#include "media/starboard/starboard_audio_device_factory.h"
+#endif  // BUILDFLAG(USE_STARBOARD_MEDIA)
+
 namespace blink {
 class URLLoaderThrottleProvider;
 enum class URLLoaderThrottleProviderType;
@@ -65,6 +69,7 @@ class ShellContentRendererClient : public ContentRendererClient {
 #if BUILDFLAG(USE_STARBOARD_MEDIA)
   bool IsSupportedAudioType(const media::AudioType& type) override;
   bool IsSupportedVideoType(const media::VideoType& type) override;
+  std::unique_ptr<media::StarboardAudioDeviceFactory> starboard_audio_device_factory_;
 #endif  // BUILDFLAG(USE_STARBOARD_MEDIA)
 
   std::unique_ptr<blink::WebPrescientNetworking> CreatePrescientNetworking(

media/starboard/BUILD.gn

@@ -126,6 +126,19 @@ source_set("starboard") {
   configs += [ "//media:subcomponent_config" ]
 }
 
+source_set("webaudio") {
+  sources = [
+    "starboard_audio_device_factory.cc",
+    "starboard_audio_device_factory.h",
+    "starboard_audio_renderer_sink.cc",
+    "starboard_audio_renderer_sink.h",
+  ]
+  deps = [
+    "//starboard:starboard_headers_only",
+    "//third_party/blink/public:blink_headers",
+  ]
+}
+
 source_set("unit_tests") {
   testonly = true
   sources = []

media/starboard/starboard_audio_device_factory.cc

@@ -0,0 +1,47 @@
+// Copyright 2024 The Cobalt Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "media/starboard/starboard_audio_device_factory.h"
+#include "media/starboard/starboard_audio_renderer_sink.h"
+
+#include "base/logging.h"
+
+namespace media {
+
+StarboardAudioDeviceFactory::StarboardAudioDeviceFactory() {
+  LOG(INFO) << "Register StarboardAudioDeviceFactory";
+}
+
+StarboardAudioDeviceFactory::~StarboardAudioDeviceFactory() {
+  LOG(INFO) << "Unregister StarboardAudioDeviceFactory";
+}
+
+scoped_refptr<media::AudioRendererSink>
+StarboardAudioDeviceFactory::NewAudioRendererSink(
+    blink::WebAudioDeviceSourceType source_type,
+    const blink::LocalFrameToken& frame_token,
+    const media::AudioSinkParameters& params) {
+  return base::MakeRefCounted<media::StarboardAudioRendererSink>();
+}
+
+OutputDeviceInfo StarboardAudioDeviceFactory::GetOutputDeviceInfo(
+    const blink::LocalFrameToken& frame_token,
+    const std::string& device_id) {
+  LOG(ERROR) << "GetOutputDeviceInfo - NOT IMPL";
+  return OutputDeviceInfo(
+      std::string(), OUTPUT_DEVICE_STATUS_OK,
+      AudioParameters(AudioParameters::AUDIO_PCM_LOW_LATENCY,
+                      ChannelLayoutConfig::Stereo(), 48000, 480));
+}
+}  // namespace media

media/starboard/starboard_audio_device_factory.h

@@ -0,0 +1,42 @@
+// Copyright 2024 The Cobalt Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIA_STARBOARD_STARBOARD_AUDIO_DEVICE_FACTORY_H
+#define MEDIA_STARBOARD_STARBOARD_AUDIO_DEVICE_FACTORY_H
+
+#include "media/base/media_export.h"
+#include "third_party/blink/public/web/modules/media/audio/audio_device_factory.h"
+
+namespace media {
+
+class MEDIA_EXPORT StarboardAudioDeviceFactory final
+    : public blink::AudioDeviceFactory {
+ public:
+  StarboardAudioDeviceFactory();
+  ~StarboardAudioDeviceFactory() override;
+
+  // blink::AudioDeviceFactory overrides.
+  scoped_refptr<media::AudioRendererSink> NewAudioRendererSink(
+      blink::WebAudioDeviceSourceType source_type,
+      const blink::LocalFrameToken& frame_token,
+      const media::AudioSinkParameters& params) override;
+
+  OutputDeviceInfo GetOutputDeviceInfo(
+      const blink::LocalFrameToken& frame_token,
+      const std::string& device_id) override;
+};
+
+}  // namespace media
+
+#endif  // MEDIA_STARBOARD_STARBOARD_AUDIO_DEVICE_FACTORY_H

media/starboard/starboard_audio_renderer_sink.cc

@@ -0,0 +1,188 @@
+// Copyright 2024 The Cobalt Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "media/starboard/starboard_audio_renderer_sink.h"
+
+#include "base/logging.h"
+
+#include "media/base/audio_glitch_info.h"
+
+namespace media {
+
+namespace {
+int AlignUp(int value, int alignment) {
+  int decremented_value = value - 1;
+  return decremented_value + alignment - (decremented_value % alignment);
+}
+}  // namespace
+
+void StarboardAudioRendererSink::Initialize(const AudioParameters& params,
+                                            RenderCallback* callback) {
+  LOG(INFO) << "StarboardAudioRendererSink::Initialize - called with following "
+               "parameters:"
+            << params.AsHumanReadableString();
+  params_ = params;
+
+  DCHECK(callback);
+  {
+    base::AutoLock auto_lock(callback_lock_);
+    DCHECK(!callback_);
+    callback_ = callback;
+  }
+}
+
+void StarboardAudioRendererSink::Start() {
+  CreateStreamSink();
+}
+
+void StarboardAudioRendererSink::Stop() {
+  is_eos_reached_ = true;
+  SbAudioSinkDestroy(audio_sink_);
+  {
+    base::AutoLock auto_lock(callback_lock_);
+    callback_ = nullptr;
+  }
+}
+
+void StarboardAudioRendererSink::Play() {
+  is_playing_ = true;
+}
+
+void StarboardAudioRendererSink::Pause() {
+  is_playing_ = false;
+}
+void StarboardAudioRendererSink::Flush() {
+  LOG(INFO) << "Flush - NOT IMPL";
+}
+bool StarboardAudioRendererSink::SetVolume(double volume) {
+  LOG(INFO) << "SetVolume - NOT IMPL";
+  return true;
+}
+OutputDeviceInfo StarboardAudioRendererSink::GetOutputDeviceInfo() {
+  LOG(INFO) << "GetOutputDeviceInfo - NOT IMPL";
+  return media::OutputDeviceInfo(
+      std::string(), media::OUTPUT_DEVICE_STATUS_OK,
+      media::AudioParameters(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
+                             media::ChannelLayoutConfig::Stereo(), 48000, 480));
+}
+void StarboardAudioRendererSink::GetOutputDeviceInfoAsync(
+    OutputDeviceInfoCB info_cb) {
+  LOG(INFO) << "GetOutputDeviceInfoAsync - NOT IMPL";
+}
+bool StarboardAudioRendererSink::IsOptimizedForHardwareParameters() {
+  LOG(INFO) << "IsOptimizedForHardwareParameters - NOT IMPL";
+  return true;
+}
+bool StarboardAudioRendererSink::CurrentThreadIsRenderingThread() {
+  LOG(INFO) << "CurrentThreadIsRenderingThread - NOT IMPL";
+  return true;
+}
+
+// static
+void StarboardAudioRendererSink::UpdateSourceStatusFunc(int* frames_in_buffer,
+                                                        int* offset_in_frames,
+                                                        bool* is_playing,
+                                                        bool* is_eos_reached,
+                                                        void* context) {
+  StarboardAudioRendererSink* sink =
+      static_cast<StarboardAudioRendererSink*>(context);
+  DCHECK(sink);
+  DCHECK(frames_in_buffer);
+  DCHECK(offset_in_frames);
+  DCHECK(is_playing);
+  DCHECK(is_eos_reached);
+  sink->UpdateSourceStatus(frames_in_buffer, offset_in_frames, is_playing,
+                           is_eos_reached);
+}
+
+// static
+void StarboardAudioRendererSink::ConsumeFramesFunc(int frames_consumed,
+                                                   void* context) {
+  StarboardAudioRendererSink* sink =
+      static_cast<StarboardAudioRendererSink*>(context);
+  DCHECK(sink);
+  sink->ConsumeFrames(frames_consumed);
+}
+
+void StarboardAudioRendererSink::UpdateSourceStatus(int* frames_in_buffer,
+                                                    int* offset_in_frames,
+                                                    bool* is_playing,
+                                                    bool* is_eos_reached) {
+  *is_playing = is_playing_;
+  *is_eos_reached = is_eos_reached_;
+
+  // Assert that we never consume more than we've rendered.
+  DCHECK_GE(frames_rendered_, frames_consumed_);
+  *frames_in_buffer = static_cast<int>(frames_rendered_ - frames_consumed_);
+
+  DCHECK(input_audio_bus_);
+  base::AutoLock lock(callback_lock_);
+  if (!callback_) {
+    return;
+  }
+
+  while ((frames_per_channel_ - *frames_in_buffer) >=
+         params_.frames_per_buffer()) {
+    int frames_filled =
+        callback_->Render(base::TimeDelta(), base::TimeTicks(),
+                          /*glitch_info=*/{}, input_audio_bus_.get());
+    FillOutputAudioBuffer(frames_filled);
+
+    frames_rendered_ += frames_filled;
+    *frames_in_buffer += frames_filled;
+  }
+  *offset_in_frames = frames_consumed_ % frames_per_channel_;
+}
+
+void StarboardAudioRendererSink::ConsumeFrames(int frames_consumed) {
+  frames_consumed_ += frames_consumed;
+}
+
+void StarboardAudioRendererSink::CreateStreamSink() {
+  int frames_per_render_buffer = params_.frames_per_buffer();
+  frames_per_channel_ =
+      std::max(AlignUp(SbAudioSinkGetMinBufferSizeInFrames(
+                           params_.channels(), kSbMediaAudioSampleTypeFloat32,
+                           params_.sample_rate()) +
+                           frames_per_render_buffer * 2,
+                       frames_per_render_buffer),
+               frames_per_render_buffer * 8);
+
+  input_audio_bus_ = AudioBus::Create(params_);
+  output_frame_buffer_.resize(frames_per_channel_ * params_.channels() *
+                              sizeof(float));
+
+  output_frame_buffers_.resize(1);
+  output_frame_buffers_[0] = &output_frame_buffer_[0];
+
+  audio_sink_ = SbAudioSinkCreate(
+      params_.channels(),
+      SbAudioSinkGetNearestSupportedSampleFrequency(params_.sample_rate()),
+      kSbMediaAudioSampleTypeFloat32, kSbMediaAudioFrameStorageTypeInterleaved,
+      &output_frame_buffers_[0], frames_per_channel_,
+      &StarboardAudioRendererSink::UpdateSourceStatusFunc,
+      &StarboardAudioRendererSink::ConsumeFramesFunc,
+      reinterpret_cast<void*>(this));
+  DCHECK(SbAudioSinkIsValid(audio_sink_));
+}
+
+void StarboardAudioRendererSink::FillOutputAudioBuffer(int num_frames) {
+  uint64_t channel_offset = frames_rendered_ % frames_per_channel_;
+  float* output_buffer = reinterpret_cast<float*>(output_frame_buffer_.data());
+  output_buffer += channel_offset * params_.channels();
+  input_audio_bus_->ToInterleaved<media::Float32SampleTypeTraitsNoClip>(
+      num_frames, output_buffer);
+}
+
+}  // namespace media