diff --git a/cobalt/demos/content/video-background-demo/dash-audio.mp4 b/cobalt/demos/content/video-background-demo/dash-audio.mp4 new file mode 100644 index 000000000000..b26721c13d6b Binary files /dev/null and b/cobalt/demos/content/video-background-demo/dash-audio.mp4 differ diff --git a/cobalt/demos/content/video-background-demo/sddefault.jpg b/cobalt/demos/content/video-background-demo/sddefault.jpg new file mode 100644 index 000000000000..ef2dae52a88e Binary files /dev/null and b/cobalt/demos/content/video-background-demo/sddefault.jpg differ diff --git a/cobalt/demos/content/video-background-demo/video-background-demo.html b/cobalt/demos/content/video-background-demo/video-background-demo.html new file mode 100644 index 000000000000..c68d4dadb345 --- /dev/null +++ b/cobalt/demos/content/video-background-demo/video-background-demo.html @@ -0,0 +1,77 @@ + + + + + + Video Elements With Background + + + +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ + + diff --git a/cobalt/demos/content/video-background-demo/video-background-demo.js b/cobalt/demos/content/video-background-demo/video-background-demo.js new file mode 100644 index 000000000000..1c50180702f6 --- /dev/null +++ b/cobalt/demos/content/video-background-demo/video-background-demo.js @@ -0,0 +1,106 @@ +// Copyright 2023 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +var nextVideoElementIndex = 0; +var audioData; +var videoData; + +function downloadMediaData(downloadedCallback) { + var xhr = new XMLHttpRequest; + + xhr.onload = function() { + audioData = xhr.response; + console.log("Downloaded " + audioData.byteLength + " of audio data."); + + xhr.onload = function() { + videoData = xhr.response; + console.log("Downloaded " + videoData.byteLength + " of video data."); + downloadedCallback(); + } + + xhr.open("GET", "vp9-720p.webm", true); + xhr.send(); + } + + xhr.open("GET", "dash-audio.mp4", true); + xhr.responseType = "arraybuffer"; + xhr.send(); +} + +function playVideoOn(videoElement) { + var ms = new MediaSource; + ms.addEventListener('sourceopen', function() { + console.log("Creating SourceBuffer objects."); + var audioBuffer = ms.addSourceBuffer('audio/mp4; codecs="mp4a.40.2"'); + var videoBuffer = ms.addSourceBuffer('video/webm; codecs="vp9"; decode-to-texture=true'); + audioBuffer.addEventListener("updateend", function() { + audioBuffer.abort(); + videoBuffer.addEventListener("updateend", function() { + setTimeout(function() { + videoBuffer.addEventListener("updateend", function() { + videoBuffer.abort(); + ms.endOfStream(); + videoElement.ontimeupdate = function() { + if (videoElement.currentTime > 10) { + console.log("Stop playback."); + videoElement.src = ''; + videoElement.load(); + videoElement.ontimeupdate = null; + } + } + console.log("Start playback."); + videoElement.play(); + }); + videoBuffer.appendBuffer(videoData.slice(1024)); + }, 5000); + }); + videoBuffer.appendBuffer(videoData.slice(0, 1024)); + }); + audioBuffer.appendBuffer(audioData); + }); + + console.log("Attaching MediaSource to video element."); + videoElement.src = URL.createObjectURL(ms); +} + +function setupKeyHandler() { + document.onkeydown = function() { + videoElements = document.getElementsByTagName('video'); + for(let i = 0; i < videoElements.length; i++) { + if (videoElements[i].playing) { + console.log("Ignore key press as a video is still playing."); + return; + } + } + + nextVideoElementIndex = nextVideoElementIndex % videoElements.length; + + console.log("Trying to play next video at index " + nextVideoElementIndex); + + var currentVideoElement = videoElements[nextVideoElementIndex]; + if (currentVideoElement.setMaxVideoCapabilities) { + if (nextVideoElementIndex < videoElements.length / 2) { + currentVideoElement.setMaxVideoCapabilities(""); + } else { + currentVideoElement.setMaxVideoCapabilities("width=1920; height=1080"); + } + } + + nextVideoElementIndex++; + + playVideoOn(currentVideoElement); + }; +} + +downloadMediaData(setupKeyHandler); diff --git a/cobalt/demos/content/video-background-demo/vp9-720p.webm b/cobalt/demos/content/video-background-demo/vp9-720p.webm new file mode 100644 index 000000000000..08a670e72367 Binary files /dev/null and b/cobalt/demos/content/video-background-demo/vp9-720p.webm differ diff --git a/cobalt/dom/html_video_element.cc b/cobalt/dom/html_video_element.cc index a973f3e724e9..75b3f9f25762 100644 --- a/cobalt/dom/html_video_element.cc +++ b/cobalt/dom/html_video_element.cc @@ -18,6 +18,7 @@ #include "base/strings/string_number_conversions.h" #include "base/trace_event/trace_event.h" #include "cobalt/dom/dom_settings.h" +#include "cobalt/dom/media_settings.h" #include "cobalt/dom/performance.h" #include "cobalt/dom/window.h" #include "cobalt/math/size_f.h" @@ -30,6 +31,15 @@ using media::WebMediaPlayer; const char HTMLVideoElement::kTagName[] = "video"; +const MediaSettings& GetMediaSettings(web::EnvironmentSettings* settings) { + DCHECK(settings); + DCHECK(settings->context()); + DCHECK(settings->context()->web_settings()); + + const auto& web_settings = settings->context()->web_settings(); + return web_settings->media_settings(); +} + HTMLVideoElement::HTMLVideoElement(Document* document) : HTMLMediaElement(document, base::Token(kTagName)) {} @@ -98,9 +108,15 @@ scoped_refptr HTMLVideoElement::GetVideoPlaybackQuality( } } -scoped_refptr -HTMLVideoElement::GetDecodeTargetProvider() { +scoped_refptr HTMLVideoElement::GetDecodeTargetProvider( + bool* paint_to_black) { DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + DCHECK(paint_to_black); + + *paint_to_black = GetMediaSettings(environment_settings()) + .IsPaintingVideoBackgroundToBlack() + .value_or(false); + return player() ? player()->GetDecodeTargetProvider() : NULL; } diff --git a/cobalt/dom/html_video_element.h b/cobalt/dom/html_video_element.h index b98b6f6cbe75..914a69e9f611 100644 --- a/cobalt/dom/html_video_element.h +++ b/cobalt/dom/html_video_element.h @@ -52,7 +52,10 @@ class HTMLVideoElement : public HTMLMediaElement { // From HTMLElement scoped_refptr AsHTMLVideoElement() override { return this; } - scoped_refptr GetDecodeTargetProvider(); + // When the return value is nullptr, and |paint_to_black| is set to true, the + // caller is expected to paint the area covered by the video to black. + scoped_refptr GetDecodeTargetProvider( + bool* paint_to_black); WebMediaPlayer::SetBoundsCB GetSetBoundsCB(); diff --git a/cobalt/dom/media_settings.cc b/cobalt/dom/media_settings.cc index 2915df3815e1..ed39dbee25ed 100644 --- a/cobalt/dom/media_settings.cc +++ b/cobalt/dom/media_settings.cc @@ -77,6 +77,12 @@ bool MediaSettingsImpl::Set(const std::string& name, int value) { LOG(INFO) << name << ": set to " << value; return true; } + } else if (name == "MediaElement.PaintingVideoBackgroundToBlack") { + if (value == 0 || value == 1) { + is_painting_video_background_to_black_ = value != 0; + LOG(INFO) << name << ": set to " << value; + return true; + } } else { LOG(WARNING) << "Ignore unknown setting with name \"" << name << "\""; return false; diff --git a/cobalt/dom/media_settings.h b/cobalt/dom/media_settings.h index eace0224dd7f..9f1a6f48798f 100644 --- a/cobalt/dom/media_settings.h +++ b/cobalt/dom/media_settings.h @@ -41,6 +41,7 @@ class MediaSettings { virtual base::Optional GetMediaElementTimeupdateEventIntervalInMilliseconds() const = 0; + virtual base::Optional IsPaintingVideoBackgroundToBlack() const = 0; protected: MediaSettings() = default; @@ -89,6 +90,9 @@ class MediaSettingsImpl : public MediaSettings { const override { return media_element_timeupdate_event_interval_in_milliseconds_; } + base::Optional IsPaintingVideoBackgroundToBlack() const override { + return is_painting_video_background_to_black_; + } // Returns true when the setting associated with `name` is set to `value`. // Returns false when `name` is not associated with any settings, or if @@ -106,6 +110,8 @@ class MediaSettingsImpl : public MediaSettings { base::Optional max_source_buffer_append_size_in_bytes_; base::Optional media_element_timeupdate_event_interval_in_milliseconds_; + + base::Optional is_painting_video_background_to_black_; }; } // namespace dom diff --git a/cobalt/dom/media_settings_test.cc b/cobalt/dom/media_settings_test.cc index 8850f7264942..e4addfe40c23 100644 --- a/cobalt/dom/media_settings_test.cc +++ b/cobalt/dom/media_settings_test.cc @@ -31,6 +31,7 @@ TEST(MediaSettingsImplTest, Empty) { EXPECT_FALSE(impl.GetMaxSizeForImmediateJob()); EXPECT_FALSE(impl.GetMaxSourceBufferAppendSizeInBytes()); EXPECT_FALSE(impl.GetMediaElementTimeupdateEventIntervalInMilliseconds()); + EXPECT_FALSE(impl.IsPaintingVideoBackgroundToBlack()); } TEST(MediaSettingsImplTest, SunnyDay) { @@ -46,6 +47,7 @@ TEST(MediaSettingsImplTest, SunnyDay) { ASSERT_TRUE(impl.Set("MediaSource.MaxSourceBufferAppendSizeInBytes", 100000)); ASSERT_TRUE( impl.Set("MediaElement.TimeupdateEventIntervalInMilliseconds", 100001)); + ASSERT_TRUE(impl.Set("MediaElement.PaintingVideoBackgroundToBlack", 1)); EXPECT_EQ(impl.GetSourceBufferEvictExtraInBytes().value(), 100); EXPECT_EQ(impl.GetMinimumProcessorCountToOffloadAlgorithm().value(), 101); @@ -56,6 +58,7 @@ TEST(MediaSettingsImplTest, SunnyDay) { EXPECT_EQ(impl.GetMaxSourceBufferAppendSizeInBytes().value(), 100000); EXPECT_EQ(impl.GetMediaElementTimeupdateEventIntervalInMilliseconds().value(), 100001); + EXPECT_TRUE(impl.IsPaintingVideoBackgroundToBlack().value()); } TEST(MediaSettingsImplTest, RainyDay) { @@ -71,6 +74,7 @@ TEST(MediaSettingsImplTest, RainyDay) { ASSERT_FALSE(impl.Set("MediaSource.MaxSourceBufferAppendSizeInBytes", 0)); ASSERT_FALSE( impl.Set("MediaElement.TimeupdateEventIntervalInMilliseconds", 0)); + ASSERT_FALSE(impl.Set("MediaElement.PaintingVideoBackgroundToBlack", 2)); EXPECT_FALSE(impl.GetSourceBufferEvictExtraInBytes()); EXPECT_FALSE(impl.GetMinimumProcessorCountToOffloadAlgorithm()); @@ -80,6 +84,7 @@ TEST(MediaSettingsImplTest, RainyDay) { EXPECT_FALSE(impl.GetMaxSizeForImmediateJob()); EXPECT_FALSE(impl.GetMaxSourceBufferAppendSizeInBytes()); EXPECT_FALSE(impl.GetMediaElementTimeupdateEventIntervalInMilliseconds()); + EXPECT_FALSE(impl.IsPaintingVideoBackgroundToBlack()); } TEST(MediaSettingsImplTest, ZeroValuesWork) { @@ -95,6 +100,7 @@ TEST(MediaSettingsImplTest, ZeroValuesWork) { // O is an invalid value for "MediaSource.MaxSourceBufferAppendSizeInBytes". // O is an invalid value for // "MediaElement.TimeupdateEventIntervalInMilliseconds". + ASSERT_TRUE(impl.Set("MediaElement.PaintingVideoBackgroundToBlack", 0)); EXPECT_EQ(impl.GetSourceBufferEvictExtraInBytes().value(), 0); EXPECT_EQ(impl.GetMinimumProcessorCountToOffloadAlgorithm().value(), 0); @@ -102,6 +108,7 @@ TEST(MediaSettingsImplTest, ZeroValuesWork) { EXPECT_FALSE(impl.IsAvoidCopyingArrayBufferEnabled().value()); EXPECT_FALSE(impl.IsCallingEndedWhenClosedEnabled().value()); EXPECT_EQ(impl.GetMaxSizeForImmediateJob().value(), 0); + EXPECT_FALSE(impl.IsPaintingVideoBackgroundToBlack().value()); } TEST(MediaSettingsImplTest, Updatable) { @@ -117,6 +124,7 @@ TEST(MediaSettingsImplTest, Updatable) { ASSERT_TRUE(impl.Set("MediaSource.MaxSourceBufferAppendSizeInBytes", 1)); ASSERT_TRUE( impl.Set("MediaElement.TimeupdateEventIntervalInMilliseconds", 1)); + ASSERT_TRUE(impl.Set("MediaElement.PaintingVideoBackgroundToBlack", 0)); ASSERT_TRUE(impl.Set("MediaSource.SourceBufferEvictExtraInBytes", 1)); ASSERT_TRUE( @@ -128,6 +136,7 @@ TEST(MediaSettingsImplTest, Updatable) { ASSERT_TRUE(impl.Set("MediaSource.MaxSourceBufferAppendSizeInBytes", 2)); ASSERT_TRUE( impl.Set("MediaElement.TimeupdateEventIntervalInMilliseconds", 2)); + ASSERT_TRUE(impl.Set("MediaElement.PaintingVideoBackgroundToBlack", 1)); EXPECT_EQ(impl.GetSourceBufferEvictExtraInBytes().value(), 1); EXPECT_EQ(impl.GetMinimumProcessorCountToOffloadAlgorithm().value(), 1); @@ -138,6 +147,7 @@ TEST(MediaSettingsImplTest, Updatable) { EXPECT_EQ(impl.GetMaxSourceBufferAppendSizeInBytes().value(), 2); EXPECT_EQ(impl.GetMediaElementTimeupdateEventIntervalInMilliseconds().value(), 2); + EXPECT_TRUE(impl.IsPaintingVideoBackgroundToBlack().value()); } TEST(MediaSettingsImplTest, InvalidSettingNames) { diff --git a/cobalt/layout/box_generator.cc b/cobalt/layout/box_generator.cc index 6eee421dfc50..b279f0c7b369 100644 --- a/cobalt/layout/box_generator.cc +++ b/cobalt/layout/box_generator.cc @@ -339,22 +339,31 @@ void BoxGenerator::VisitVideoElement(dom::HTMLVideoElement* video_element) { // If the optional is disengaged, then we don't know if punch out is enabled // or not. base::Optional replaced_box_mode; - if (video_element->GetDecodeTargetProvider()) { + bool paint_to_black = false; + auto decode_target_provider = + video_element->GetDecodeTargetProvider(&paint_to_black); + if (decode_target_provider) { DecodeTargetProvider::OutputMode output_mode = - video_element->GetDecodeTargetProvider()->GetOutputMode(); - if (output_mode != DecodeTargetProvider::kOutputModeInvalid) { + decode_target_provider->GetOutputMode(); + if (output_mode == DecodeTargetProvider::kOutputModeInvalid) { + if (paint_to_black) { + replaced_box_mode = ReplacedBox::ReplacedBoxMode::kPaintToBlack; + } + } else { replaced_box_mode = (output_mode == DecodeTargetProvider::kOutputModePunchOut) ? ReplacedBox::ReplacedBoxMode::kPunchOutVideo - : ReplacedBox::ReplacedBoxMode::kVideo; + : ReplacedBox::ReplacedBoxMode::kDecodeToTextureVideo; } + } else { + // ReplacedBox won't paint anything when |decode_target_provider| is + // nullptr, as |replace_image_cb_| is also null in this case. } ReplacedBoxGenerator replaced_box_generator( video_element->css_computed_style_declaration(), - video_element->GetDecodeTargetProvider() - ? base::Bind(GetVideoFrame, video_element->GetDecodeTargetProvider(), - resource_provider) + decode_target_provider + ? base::Bind(GetVideoFrame, decode_target_provider, resource_provider) : ReplacedBox::ReplaceImageCB(), video_element->GetSetBoundsCB(), *paragraph_, text_position, base::nullopt, base::nullopt, base::nullopt, context_, replaced_box_mode, diff --git a/cobalt/layout/replaced_box.cc b/cobalt/layout/replaced_box.cc index 21eba9ac7fa8..a31407e15035 100644 --- a/cobalt/layout/replaced_box.cc +++ b/cobalt/layout/replaced_box.cc @@ -316,9 +316,14 @@ void ReplacedBox::RenderAndAnimateContent( return; } - if (replaced_box_mode_ == base::nullopt) { - // If we don't have a data stream associated with this video [yet], then - // we don't yet know if it is punched out or not, and so render black. + if (!replaced_box_mode_.has_value()) { + // Don't render anything, so any background color or image will be visible. + return; + } + + if (replaced_box_mode_ == ReplacedBoxMode::kPaintToBlack) { + // Explicitly render black if we don't have a data stream associated with + // this video [yet], this is the same as the previous behavior. border_node_builder->AddChild(new RectNode( math::RectF(content_box_size()), std::unique_ptr(new render_tree::SolidColorBrush( @@ -327,7 +332,7 @@ void ReplacedBox::RenderAndAnimateContent( return; } - if (*replaced_box_mode_ == ReplacedBox::ReplacedBoxMode::kLottie) { + if (*replaced_box_mode_ == ReplacedBoxMode::kLottie) { AnimateNode::Builder animate_node_builder; scoped_refptr lottie_node = new LottieNode(nullptr, math::RectF()); @@ -347,7 +352,7 @@ void ReplacedBox::RenderAndAnimateContent( // Map-to-mesh is only supported with decode-to-texture videos. const bool supports_mtm = replaced_box_mode_ && - *replaced_box_mode_ == ReplacedBox::ReplacedBoxMode::kVideo; + *replaced_box_mode_ == ReplacedBoxMode::kDecodeToTextureVideo; if (supports_mtm && mtm_filter_function && mtm_filter_function->mesh_spec().mesh_type() != @@ -759,7 +764,7 @@ void ReplacedBox::RenderAndAnimateContentWithLetterboxing( scoped_refptr composition_node = new CompositionNode(composition_node_builder); - if (*replaced_box_mode_ == ReplacedBox::ReplacedBoxMode::kPunchOutVideo) { + if (*replaced_box_mode_ == ReplacedBoxMode::kPunchOutVideo) { LetterboxDimensions letterbox_dims = GetLetterboxDimensions(content_size_, content_box_size()); AddLetterboxedPunchThroughVideoNodeToRenderTree( diff --git a/cobalt/layout/replaced_box.h b/cobalt/layout/replaced_box.h index f4384e562af3..3837b7c191e8 100644 --- a/cobalt/layout/replaced_box.h +++ b/cobalt/layout/replaced_box.h @@ -43,7 +43,12 @@ class ReplacedBox : public Box { typedef base::Callback()> ReplaceImageCB; typedef render_tree::PunchThroughVideoNode::SetBoundsCB SetBoundsCB; - enum class ReplacedBoxMode { kVideo, kPunchOutVideo, kLottie }; + enum class ReplacedBoxMode { + kPaintToBlack, // Paint a black rectangle + kDecodeToTextureVideo, + kPunchOutVideo, + kLottie + }; ReplacedBox(const scoped_refptr& css_computed_style_declaration,