diff --git a/cobalt/dom/BUILD.gn b/cobalt/dom/BUILD.gn index 202821b3a25..73ef77ef0bf 100644 --- a/cobalt/dom/BUILD.gn +++ b/cobalt/dom/BUILD.gn @@ -197,6 +197,7 @@ static_library("dom") { "media_source.cc", "media_source.h", "media_source_attachment.h", + "media_source_attachment_supplement.cc", "media_source_attachment_supplement.h", "memory_info.cc", "memory_info.h", diff --git a/cobalt/dom/cross_thread_media_source_attachment.cc b/cobalt/dom/cross_thread_media_source_attachment.cc index 1f798cd9e9f..299a14cface 100644 --- a/cobalt/dom/cross_thread_media_source_attachment.cc +++ b/cobalt/dom/cross_thread_media_source_attachment.cc @@ -18,6 +18,9 @@ #include "cobalt/dom/cross_thread_media_source_attachment.h" +#include + +#include "base/synchronization/lock.h" #include "base/task/sequenced_task_runner.h" #include "cobalt/dom/media_source.h" #include "cobalt/dom/media_source_attachment.h" @@ -34,75 +37,191 @@ namespace dom { CrossThreadMediaSourceAttachment::CrossThreadMediaSourceAttachment( scoped_refptr media_source) : media_source_(media_source), - task_runner_(base::SequencedTaskRunner::GetCurrentDefault()), + // worker_runner_(base::SingleThreadTaskRunner::GetCurrentDefault()), + worker_runner_(base::SequencedTaskRunner::GetCurrentDefault()), recent_element_time_(0.0), - element_has_error_(false) {} + element_has_error_(false), + element_has_max_video_capabilities_(false), + have_ever_attached_(false), + have_ever_started_closing_(false) {} void CrossThreadMediaSourceAttachment::TraceMembers(script::Tracer* tracer) { - DCHECK_NE(task_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + // TODO(at-ninja): How does this need to work for cross-thread? Tracing + // should be called from both threads. Can this be forced for testing? + // Should this only trace the objects on the current thread, or should this + // go cross-thread to trace everything? + + base::AutoLock auto_lock(attachment_state_lock_); + // DCHECK(worker_runner_->BelongsToCurrentThread()); + DCHECK_EQ(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); - tracer->Trace(attached_element_); - tracer->Trace(media_source_); + tracer->Trace(attached_element_); // main thread. + tracer->Trace(media_source_); // worker thread. } bool CrossThreadMediaSourceAttachment::StartAttachingToMediaElement( HTMLMediaElement* media_element) { - DCHECK_NE(task_runner_, base::SequencedTaskRunner::GetCurrentDefault()); - NOTIMPLEMENTED(); - return false; + base::AutoLock auto_lock(attachment_state_lock_); + // Called from main thread. At this point, we can only check if + // this is not being called from the worker thread. + // DCHECK(!worker_runner_->BelongsToCurrentThread()); + DCHECK_NE(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + if (have_ever_attached_) { + return false; + } + + DCHECK(!have_ever_started_closing_); + DCHECK(!attached_element_); + + // Attach on the main thread. + bool success = media_source_->StartAttachingToMediaElement(this); + if (!success) { + return false; + } + + attached_element_ = base::AsWeakPtr(media_element); + // main_runner_ = base::SingleThreadTaskRunner::GetCurrentDefault(); + main_runner_ = base::SequencedTaskRunner::GetCurrentDefault(); + // DCHECK(main_runner_->BelongsToCurrentThread()); + DCHECK_EQ(main_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + // Grab initial state from the HTMLMediaElement while on the main thread. + recent_element_time_ = media_element->current_time(NULL); + element_has_error_ = static_cast(media_element->error()); + element_has_max_video_capabilities_ = + media_element->HasMaxVideoCapabilities(); + + DCHECK(!element_has_error_); + + have_ever_attached_ = true; + return true; } void CrossThreadMediaSourceAttachment::CompleteAttachingToMediaElement( ::media::ChunkDemuxer* chunk_demuxer) { - DCHECK_NE(task_runner_, base::SequencedTaskRunner::GetCurrentDefault()); - NOTIMPLEMENTED(); + base::AutoLock auto_lock(attachment_state_lock_); + DCHECK(have_ever_attached_); + // DCHECK(main_runner_->BelongsToCurrentThread()); + DCHECK_EQ(main_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + // DCHECK(!worker_runner_->BelongsToCurrentThread()); + DCHECK_NE(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + worker_runner_->PostTask( + FROM_HERE, + base::BindOnce(&CrossThreadMediaSourceAttachment:: + CompleteAttachingToMediaElementOnWorkerThread, + this, chunk_demuxer)); +} + +void CrossThreadMediaSourceAttachment:: + CompleteAttachingToMediaElementOnWorkerThread( + ::media::ChunkDemuxer* chunk_demuxer) { + base::AutoLock auto_lock(attachment_state_lock_); + // DCHECK(worker_runner_->BelongsToCurrentThread()); + DCHECK_EQ(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + // DCHECK(!main_runner_->BelongsToCurrentThread()); + DCHECK_NE(main_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + if (have_ever_started_closing_) { + return; + } + + media_source_->CompleteAttachingToMediaElement(chunk_demuxer); } void CrossThreadMediaSourceAttachment::Close() { - DCHECK_NE(task_runner_, base::SequencedTaskRunner::GetCurrentDefault()); - NOTIMPLEMENTED(); + base::AutoLock auto_lock(attachment_state_lock_); + DCHECK(have_ever_attached_); + DCHECK(!have_ever_started_closing_); + // DCHECK(main_runner_->BelongsToCurrentThread()); + DCHECK_EQ(main_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + // DCHECK(!worker_runner_->BelongsToCurrentThread()); + DCHECK_NE(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + have_ever_started_closing_ = true; + + worker_runner_->PostTask( + FROM_HERE, + base::BindOnce(&CrossThreadMediaSourceAttachment::CloseOnWorkerThread, + this)); +} + +void CrossThreadMediaSourceAttachment::CloseOnWorkerThread() { + base::AutoLock auto_lock(attachment_state_lock_); + DCHECK(have_ever_started_closing_); + DCHECK(attached_element_); + // DCHECK(worker_runner_->BelongsToCurrentThread()); + DCHECK_EQ(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + // DCHECK(!main_runner_->BelongsToCurrentThread()); + DCHECK_NE(main_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + media_source_->Close(); } scoped_refptr CrossThreadMediaSourceAttachment::GetBufferedRange() const { - DCHECK_NE(task_runner_, base::SequencedTaskRunner::GetCurrentDefault()); - NOTIMPLEMENTED(); - return nullptr; + base::AutoLock auto_lock(attachment_state_lock_); + DCHECK(attached_element_); + DCHECK(!have_ever_started_closing_); + DCHECK(have_ever_attached_); + // DCHECK(main_runner_->BelongsToCurrentThread()); + DCHECK_EQ(main_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + // DCHECK(!worker_runner_->BelongsToCurrentThread()); + DCHECK_NE(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + return media_source_->GetBufferedRange(); } MediaSourceReadyState CrossThreadMediaSourceAttachment::GetReadyState() const { - DCHECK_NE(task_runner_, base::SequencedTaskRunner::GetCurrentDefault()); - NOTIMPLEMENTED(); - return kMediaSourceReadyStateClosed; + base::AutoLock auto_lock(attachment_state_lock_); + DCHECK(attached_element_); + DCHECK(!have_ever_started_closing_); + DCHECK(have_ever_attached_); + // DCHECK(main_runner_->BelongsToCurrentThread()); + DCHECK_EQ(main_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + // DCHECK(!worker_runner_->BelongsToCurrentThread()); + DCHECK_NE(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + return media_source_->ready_state(); } -void CrossThreadMediaSourceAttachment::NotifyDurationChanged(double duration) { - DCHECK_NE(task_runner_, base::SequencedTaskRunner::GetCurrentDefault()); - NOTIMPLEMENTED(); +void CrossThreadMediaSourceAttachment::NotifyDurationChanged( + double /*duration*/) { + attachment_state_lock_.AssertAcquired(); + // DCHECK(worker_runner_->BelongsToCurrentThread()); + DCHECK_EQ(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + // No-op for cross thread MSA. } bool CrossThreadMediaSourceAttachment::HasMaxVideoCapabilities() const { - DCHECK_NE(task_runner_, base::SequencedTaskRunner::GetCurrentDefault()); - NOTIMPLEMENTED(); - return false; + attachment_state_lock_.AssertAcquired(); + // DCHECK(worker_runner_->BelongsToCurrentThread()); + DCHECK_EQ(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + return element_has_max_video_capabilities_; } double CrossThreadMediaSourceAttachment::GetRecentMediaTime() { - DCHECK_NE(task_runner_, base::SequencedTaskRunner::GetCurrentDefault()); - NOTIMPLEMENTED(); - return 0; + attachment_state_lock_.AssertAcquired(); + // DCHECK(worker_runner_->BelongsToCurrentThread()); + DCHECK_EQ(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + return recent_element_time_; } bool CrossThreadMediaSourceAttachment::GetElementError() { - DCHECK_NE(task_runner_, base::SequencedTaskRunner::GetCurrentDefault()); - NOTIMPLEMENTED(); - return false; + attachment_state_lock_.AssertAcquired(); + // DCHECK(worker_runner_->BelongsToCurrentThread()); + DCHECK_EQ(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + return element_has_error_; } scoped_refptr CrossThreadMediaSourceAttachment::CreateAudioTrackList( script::EnvironmentSettings* settings) { - DCHECK_NE(task_runner_, base::SequencedTaskRunner::GetCurrentDefault()); NOTIMPLEMENTED(); return nullptr; } @@ -110,22 +229,120 @@ CrossThreadMediaSourceAttachment::CreateAudioTrackList( scoped_refptr CrossThreadMediaSourceAttachment::CreateVideoTrackList( script::EnvironmentSettings* settings) { - DCHECK_NE(task_runner_, base::SequencedTaskRunner::GetCurrentDefault()); NOTIMPLEMENTED(); return nullptr; } void CrossThreadMediaSourceAttachment::OnElementTimeUpdate(double time) { - NOTIMPLEMENTED(); + base::AutoLock auto_lock(attachment_state_lock_); + // DCHECK(main_runner_->BelongsToCurrentThread()); + DCHECK_EQ(main_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + // DCHECK(!worker_runner_->BelongsToCurrentThread()); + DCHECK_NE(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + worker_runner_->PostTask( + FROM_HERE, + base::BindOnce( + &CrossThreadMediaSourceAttachment::UpdateWorkerThreadTimeCache, this, + time)); +} + +void CrossThreadMediaSourceAttachment::UpdateWorkerThreadTimeCache( + double time) { + base::AutoLock auto_lock(attachment_state_lock_); + // DCHECK(worker_runner_->BelongsToCurrentThread()); + DCHECK_EQ(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + // DCHECK(!main_runner_->BelongsToCurrentThread()); + DCHECK_NE(main_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + recent_element_time_ = time; } void CrossThreadMediaSourceAttachment::OnElementError() { + base::AutoLock auto_lock(attachment_state_lock_); + // DCHECK(main_runner_->BelongsToCurrentThread()); + DCHECK_EQ(main_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + // DCHECK(!worker_runner_->BelongsToCurrentThread()); + DCHECK_NE(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + worker_runner_->PostTask( + FROM_HERE, + base::BindOnce( + &CrossThreadMediaSourceAttachment::HandleElementErrorOnWorkerThread, + this)); +} + +void CrossThreadMediaSourceAttachment::HandleElementErrorOnWorkerThread() { + base::AutoLock auto_lock(attachment_state_lock_); DCHECK(!element_has_error_) << "At most one transition to element error per attachment is expected"; - NOTIMPLEMENTED(); + // DCHECK(worker_runner_->BelongsToCurrentThread()); + DCHECK_EQ(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + // DCHECK(main_runner_->BelongsToCurrentThread()); + DCHECK_EQ(main_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + element_has_error_ = true; } +void CrossThreadMediaSourceAttachment::OnElementHasMaxVideoCapabilitiesUpdate( + bool has_max_video_capabilities) { + base::AutoLock auto_lock(attachment_state_lock_); + // DCHECK(main_runner_->BelongsToCurrentThread()); + DCHECK_EQ(main_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + // DCHECK(!worker_runner_->BelongsToCurrentThread()); + DCHECK_NE(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + worker_runner_->PostTask( + FROM_HERE, + base::BindOnce(&CrossThreadMediaSourceAttachment:: + UpdateElementMaxVideoCapabilitiesOnWorkerThread, + this, has_max_video_capabilities)); +} + +void CrossThreadMediaSourceAttachment:: + UpdateElementMaxVideoCapabilitiesOnWorkerThread( + bool has_max_video_capabilities) { + base::AutoLock auto_lock(attachment_state_lock_); + // DCHECK(worker_runner_->BelongsToCurrentThread()); + DCHECK_EQ(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + // DCHECK(main_runner_->BelongsToCurrentThread()); + DCHECK_EQ(main_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + element_has_max_video_capabilities_ = has_max_video_capabilities; +} + +bool CrossThreadMediaSourceAttachment::FullyAttachedOrSameThread() const { + attachment_state_lock_.AssertAcquired(); + DCHECK(have_ever_attached_); + // DCHECK(worker_runner_->BelongsToCurrentThread()); + DCHECK_EQ(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + // DCHECK(!main_runner_->BelongsToCurrentThread()); + DCHECK_NE(main_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + return attached_element_ && !have_ever_started_closing_; +} + +bool CrossThreadMediaSourceAttachment::RunExclusively( + bool abort_if_not_fully_attached, RunExclusivelyCB cb) { + base::AutoLock auto_lock(attachment_state_lock_); + + DCHECK(have_ever_attached_); + // DCHECK(worker_runner_->BelongsToCurrentThread()); + DCHECK_EQ(worker_runner_, base::SequencedTaskRunner::GetCurrentDefault()); + + if (abort_if_not_fully_attached && + (!attached_element_ || have_ever_started_closing_)) { + return false; + } + + std::move(cb).Run(); + return true; +} + +void CrossThreadMediaSourceAttachment:: + AssertCrossThreadMutexIsAcquiredForDebugging() { + attachment_state_lock_.AssertAcquired(); +} + } // namespace dom } // namespace cobalt diff --git a/cobalt/dom/cross_thread_media_source_attachment.h b/cobalt/dom/cross_thread_media_source_attachment.h index 134b0a5550a..00200d76d85 100644 --- a/cobalt/dom/cross_thread_media_source_attachment.h +++ b/cobalt/dom/cross_thread_media_source_attachment.h @@ -20,7 +20,9 @@ #define COBALT_DOM_CROSS_THREAD_MEDIA_SOURCE_ATTACHMENT_H_ #include "base/memory/weak_ptr.h" +#include "base/synchronization/lock.h" #include "base/task/sequenced_task_runner.h" +#include "base/task/single_thread_task_runner.h" #include "cobalt/dom/audio_track_list.h" #include "cobalt/dom/media_source.h" #include "cobalt/dom/media_source_attachment_supplement.h" @@ -47,47 +49,99 @@ class CrossThreadMediaSourceAttachment void TraceMembers(script::Tracer* tracer) override; // MediaSourceAttachment - bool StartAttachingToMediaElement(HTMLMediaElement* media_element) override; - void CompleteAttachingToMediaElement( - ::media::ChunkDemuxer* chunk_demuxer) override; - void Close() override; - scoped_refptr GetBufferedRange() const override; - MediaSourceReadyState GetReadyState() const override; - - void OnElementTimeUpdate(double time) override; - void OnElementError() override; + bool StartAttachingToMediaElement(HTMLMediaElement* media_element) override + LOCKS_EXCLUDED(attachment_state_lock_); + void CompleteAttachingToMediaElement(::media::ChunkDemuxer* chunk_demuxer) + override LOCKS_EXCLUDED(attachment_state_lock_); + void Close() override LOCKS_EXCLUDED(attachment_state_lock_); + scoped_refptr GetBufferedRange() const override + LOCKS_EXCLUDED(attachment_state_lock_); + MediaSourceReadyState GetReadyState() const override + LOCKS_EXCLUDED(attachment_state_lock_); + + void OnElementTimeUpdate(double time) override + LOCKS_EXCLUDED(attachment_state_lock_); + void OnElementError() override LOCKS_EXCLUDED(attachment_state_lock_); + void OnElementHasMaxVideoCapabilitiesUpdate(bool has_max_video_capabilities) + override LOCKS_EXCLUDED(attachment_state_lock_); // MediaSourceAttachmentSupplement - void NotifyDurationChanged(double duration) override; - bool HasMaxVideoCapabilities() const override; - double GetRecentMediaTime() override; - bool GetElementError() override; + void NotifyDurationChanged(double duration) override + EXCLUSIVE_LOCKS_REQUIRED(attachment_state_lock_); + bool HasMaxVideoCapabilities() const override + EXCLUSIVE_LOCKS_REQUIRED(attachment_state_lock_); + double GetRecentMediaTime() override + EXCLUSIVE_LOCKS_REQUIRED(attachment_state_lock_); + bool GetElementError() override + EXCLUSIVE_LOCKS_REQUIRED(attachment_state_lock_); scoped_refptr CreateAudioTrackList( script::EnvironmentSettings* settings) override; scoped_refptr CreateVideoTrackList( script::EnvironmentSettings* settings) override; + bool RunExclusively(bool abort_if_not_fully_attached, + RunExclusivelyCB cb) override + LOCKS_EXCLUDED(attachment_state_lock_); + void AssertCrossThreadMutexIsAcquiredForDebugging() override + EXCLUSIVE_LOCKS_REQUIRED(attachment_state_lock_); + bool FullyAttachedOrSameThread() const override + EXCLUSIVE_LOCKS_REQUIRED(attachment_state_lock_); // TODO(338425449): Remove methods after feature rollout. - scoped_refptr media_source() const override { return nullptr; } + scoped_refptr media_source() const override { + // Intentionally not implemented. Needed for the migration to + // MediaSourceAttachment methods. + NOTIMPLEMENTED(); + return nullptr; + } base::WeakPtr media_element() const override { + // Intentionally not implemented. Needed for the migration to + // MediaSourceAttachment methods. + NOTIMPLEMENTED(); return nullptr; } private: ~CrossThreadMediaSourceAttachment() = default; + void CompleteAttachingToMediaElementOnWorkerThread( + ::media::ChunkDemuxer* chunk_demuxer) + LOCKS_EXCLUDED(attachment_state_lock_); + void CloseOnWorkerThread() LOCKS_EXCLUDED(attachment_state_lock_); + void UpdateWorkerThreadTimeCache(double time) + LOCKS_EXCLUDED(attachment_state_lock_); + void HandleElementErrorOnWorkerThread() + LOCKS_EXCLUDED(attachment_state_lock_); + void UpdateElementMaxVideoCapabilitiesOnWorkerThread( + bool has_max_video_capabilities) LOCKS_EXCLUDED(attachment_state_lock_); + + mutable base::Lock attachment_state_lock_; + + // TODO(at-ninja): Remove one set of these. + // scoped_refptr worker_runner_ + // GUARDED_BY(attachment_state_lock_); + // scoped_refptr main_runner_ + // GUARDED_BY(attachment_state_lock_); + scoped_refptr worker_runner_ + GUARDED_BY(attachment_state_lock_); + scoped_refptr main_runner_ + GUARDED_BY(attachment_state_lock_); + // Reference to the registered MediaSource. - scoped_refptr media_source_; + scoped_refptr media_source_ GUARDED_BY(attachment_state_lock_); // Reference to the HTMLMediaElement the associated MediaSource is attached // to. Only set after StartAttachingToMediaElement is called. - base::WeakPtr attached_element_ = nullptr; + base::WeakPtr attached_element_ + GUARDED_BY(attachment_state_lock_) = nullptr; - // Used to ensure all calls are made on the thread that created this object. - base::SequencedTaskRunner* task_runner_; + double recent_element_time_ + GUARDED_BY(attachment_state_lock_); // See OnElementTimeUpdate(). + bool element_has_error_ + GUARDED_BY(attachment_state_lock_); // See OnElementError(). + bool element_has_max_video_capabilities_ GUARDED_BY(attachment_state_lock_); - double recent_element_time_; // See OnElementTimeUpdate(). - bool element_has_error_; // See OnElementError(). + bool have_ever_attached_ GUARDED_BY(attachment_state_lock_); + bool have_ever_started_closing_ GUARDED_BY(attachment_state_lock_); DISALLOW_COPY_AND_ASSIGN(CrossThreadMediaSourceAttachment); }; diff --git a/cobalt/dom/html_media_element.cc b/cobalt/dom/html_media_element.cc index 2be5f86f59b..af80c4dce03 100644 --- a/cobalt/dom/html_media_element.cc +++ b/cobalt/dom/html_media_element.cc @@ -676,6 +676,9 @@ void HTMLMediaElement::TraceMembers(script::Tracer* tracer) { tracer->Trace(event_queue_); tracer->Trace(played_time_ranges_); if (is_using_media_source_attachment_methods_) { + // TODO(at-ninja): How will this interact with + // CrossThreadMediaSourceAttachment? Should this be called if the attachment + // is cross-thread? Should this no-op? tracer->Trace(media_source_attachment_); } else { tracer->Trace(media_source_); diff --git a/cobalt/dom/media_source.cc b/cobalt/dom/media_source.cc index 0785b4fa211..2a9ec49cb3b 100644 --- a/cobalt/dom/media_source.cc +++ b/cobalt/dom/media_source.cc @@ -47,6 +47,7 @@ #include #include #include +#include #include #include "base/compiler_specific.h" @@ -206,7 +207,36 @@ scoped_refptr MediaSource::active_source_buffers() const { MediaSourceReadyState MediaSource::ready_state() const { return ready_state_; } -double MediaSource::duration(script::ExceptionState* exception_state) const { +double MediaSource::duration(script::ExceptionState* exception_state) { + double duration_result = std::numeric_limits::quiet_NaN(); + + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + if (IsClosed()) { + return duration_result; + } + + if (!RunUnlessElementGoneOrClosingUs( + base::Bind(&MediaSource::GetDurationInternal_Locked, this, + &duration_result))) { + DCHECK_EQ(duration_result, std::numeric_limits::quiet_NaN()); + } + + return duration_result; + } else { + return GetDuration_Locked(); + } +} + +void MediaSource::GetDurationInternal_Locked(double* result) { + AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + + *result = GetDuration_Locked(); +} + +double MediaSource::GetDuration_Locked() const { + AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + if (ready_state_ == kMediaSourceReadyStateClosed) { return std::numeric_limits::quiet_NaN(); } @@ -228,8 +258,35 @@ void MediaSource::set_duration(double duration, } // Run the duration change algorithm - if (duration == this->duration(NULL)) { - return; + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + if (!RunUnlessElementGoneOrClosingUs( + base::Bind(&MediaSource::DurationChangeAlgorithm, this, duration, + exception_state))) { + web::DOMException::Raise(web::DOMException::kInvalidStateErr, + exception_state); + return; + } + } else { + DurationChangeAlgorithm(duration, exception_state); + } +} + +void MediaSource::DurationChangeAlgorithm( + double duration, script::ExceptionState* exception_state) { + AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + + double old_duration; + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + old_duration = GetDuration_Locked(); + if (duration == old_duration) { + return; + } + } else { + if (duration == this->duration(NULL)) { + return; + } } double highest_buffered_presentation_timestamp = 0; @@ -246,7 +303,10 @@ void MediaSource::set_duration(double duration, } // 3. Set old duration to the current value of duration. - double old_duration = this->duration(NULL); + if (!(is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings()))) { + old_duration = this->duration(NULL); + } DCHECK_LE(highest_buffered_presentation_timestamp, std::isnan(old_duration) ? 0 : old_duration); @@ -267,6 +327,7 @@ void MediaSource::set_duration(double duration, // 6. Update the media controller duration to new duration and run the // HTMLMediaElement duration change algorithm. if (is_using_media_source_attachment_methods_) { + base::AutoLock auto_lock(attachment_link_lock_); media_source_attachment_->NotifyDurationChanged(duration); } else { attached_element_->DurationChanged(duration, request_seek); @@ -298,12 +359,34 @@ scoped_refptr MediaSource::AddSourceBuffer( return NULL; } + SourceBuffer* source_buffer = nullptr; + + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + if (!RunUnlessElementGoneOrClosingUs( + base::Bind(&MediaSource::AddSourceBuffer_Locked, this, settings, + type, exception_state, &source_buffer))) { + web::DOMException::Raise(web::DOMException::kInvalidStateErr, + exception_state); + return NULL; + } + } else { + AddSourceBuffer_Locked(settings, type, exception_state, &source_buffer); + } + + return base::WrapRefCounted(source_buffer); +} + +void MediaSource::AddSourceBuffer_Locked( + script::EnvironmentSettings* settings, const std::string& type, + script::ExceptionState* exception_state, SourceBuffer** created_buffer) { + AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + std::string guid = base::GenerateGUID(); - scoped_refptr source_buffer; ChunkDemuxer::Status status = chunk_demuxer_->AddId(guid, type); switch (status) { case ChunkDemuxer::kOk: - source_buffer = + *created_buffer = new SourceBuffer(settings, guid, this, chunk_demuxer_, &event_queue_, is_using_media_source_attachment_methods_); break; @@ -317,12 +400,11 @@ scoped_refptr MediaSource::AddSourceBuffer( return NULL; } - DCHECK(source_buffer); - source_buffers_->Add(source_buffer); - LOG(INFO) << "added SourceBuffer (0x" << source_buffer.get() + DCHECK(*created_buffer); + source_buffers_->Add(base::WrapRefCounted(*created_buffer)); + LOG(INFO) << "added SourceBuffer (0x" << *created_buffer << ") to MediaSource (0x" << this << ") with type " << type << " id = " << guid; - return source_buffer; } void MediaSource::RemoveSourceBuffer( @@ -341,6 +423,23 @@ void MediaSource::RemoveSourceBuffer( return; } + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + if (!RunUnlessElementGoneOrClosingUs(base::Bind( + &MediaSource::RemoveSourceBuffer_Locked, this, source_buffer))) { + web::DOMException::Raise(web::DOMException::kInvalidStateErr, + exception_state); + return; + } + } else { + RemoveSourceBuffer_Locked(source_buffer); + } +} + +void MediaSource::RemoveSourceBuffer_Locked( + const scoped_refptr& source_buffer) { + AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + source_buffer->OnRemovedFromMediaSource(); active_source_buffers_->Remove(source_buffer); @@ -348,6 +447,8 @@ void MediaSource::RemoveSourceBuffer( } void MediaSource::EndOfStreamAlgorithm(MediaSourceEndOfStreamError error) { + AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + if (IsClosed()) { if (IsCallingEndedWhenClosedEnabled(environment_settings())) { LOG(INFO) << "Setting state to ended when MediaSource object is closed"; @@ -390,7 +491,18 @@ void MediaSource::EndOfStream(MediaSourceEndOfStreamError error, exception_state); return; } - EndOfStreamAlgorithm(error); + + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + if (!RunUnlessElementGoneOrClosingUs( + base::Bind(&MediaSource::EndOfStreamAlgorithm, this, error))) { + web::DOMException::Raise(web::DOMException::kInvalidStateErr, + exception_state); + return; + } + } else { + EndOfStreamAlgorithm(error); + } } void MediaSource::SetLiveSeekableRange( @@ -408,7 +520,10 @@ void MediaSource::SetLiveSeekableRange( return; } - live_seekable_range_ = new TimeRanges(start, end); + { + base::AutoLock auto_lock(attachment_link_lock_); + live_seekable_range_ = new TimeRanges(start, end); + } } void MediaSource::ClearLiveSeekableRange( @@ -420,8 +535,11 @@ void MediaSource::ClearLiveSeekableRange( return; } - if (live_seekable_range_->length() != 0) { - live_seekable_range_ = new TimeRanges; + { + base::AutoLock auto_lock(attachment_link_lock_); + if (live_seekable_range_->length() != 0) { + live_seekable_range_ = new TimeRanges; + } } } @@ -530,6 +648,8 @@ bool MediaSource::StartAttachingToMediaElement( bool MediaSource::StartAttachingToMediaElement( MediaSourceAttachmentSupplement* media_source_attachment) { + base::AutoLock auto_lock(attachment_link_lock_); + is_using_media_source_attachment_methods_ = true; if (media_source_attachment_) { @@ -540,8 +660,10 @@ bool MediaSource::StartAttachingToMediaElement( DCHECK(!algorithm_process_thread_); media_source_attachment_ = base::AsWeakPtr(media_source_attachment); - has_max_video_capabilities_ = - media_source_attachment->HasMaxVideoCapabilities(); + if (!IsMseInWorkersEnabled(environment_settings())) { + has_max_video_capabilities_ = + media_source_attachment->HasMaxVideoCapabilities(); + } if (algorithm_offload_enabled_) { algorithm_process_thread_.reset(new base::Thread("MSEAlgorithm")); @@ -565,9 +687,11 @@ bool MediaSource::StartAttachingToMediaElement( } void MediaSource::CompleteAttachingToMediaElement(ChunkDemuxer* chunk_demuxer) { + AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); DCHECK(chunk_demuxer); DCHECK(!chunk_demuxer_); if (is_using_media_source_attachment_methods_) { + base::AutoLock auto_lock(attachment_link_lock_); DCHECK(media_source_attachment_); } else { DCHECK(attached_element_); @@ -576,13 +700,18 @@ void MediaSource::CompleteAttachingToMediaElement(ChunkDemuxer* chunk_demuxer) { SetReadyState(kMediaSourceReadyStateOpen); } -void MediaSource::Close() { SetReadyState(kMediaSourceReadyStateClosed); } +void MediaSource::Close() { + AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + SetReadyState(kMediaSourceReadyStateClosed); +} bool MediaSource::IsClosed() const { return ready_state_ == kMediaSourceReadyStateClosed; } scoped_refptr MediaSource::GetBufferedRange() const { + AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + std::vector > ranges( active_source_buffers_->length()); for (uint32 i = 0; i < active_source_buffers_->length(); ++i) @@ -626,8 +755,10 @@ scoped_refptr MediaSource::GetBufferedRange() const { } scoped_refptr MediaSource::GetSeekable() const { + AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + // Implements MediaSource algorithm for HTMLMediaElement.seekable. - double source_duration = duration(NULL); + double source_duration = GetDuration_Locked(); if (std::isnan(source_duration)) { return new TimeRanges; @@ -640,23 +771,28 @@ scoped_refptr MediaSource::GetSeekable() const { buffered = GetBufferedRange(); } else { if (is_using_media_source_attachment_methods_) { + base::AutoLock auto_lock(attachment_link_lock_); buffered = media_source_attachment_->media_element()->buffered(); } else { buffered = attached_element_->buffered(); } } - if (live_seekable_range_->length() != 0) { - if (buffered->length() == 0) { - return new TimeRanges(live_seekable_range_->Start(0, NULL), - live_seekable_range_->End(0, NULL)); - } + { + base::AutoLock auto_lock(attachment_link_lock_); + + if (live_seekable_range_->length() != 0) { + if (buffered->length() == 0) { + return new TimeRanges(live_seekable_range_->Start(0, NULL), + live_seekable_range_->End(0, NULL)); + } - return new TimeRanges( - std::min(live_seekable_range_->Start(0, NULL), - buffered->Start(0, NULL)), - std::max(live_seekable_range_->End(0, NULL), - buffered->End(buffered->length() - 1, NULL))); + return new TimeRanges( + std::min(live_seekable_range_->Start(0, NULL), + buffered->Start(0, NULL)), + std::max(live_seekable_range_->End(0, NULL), + buffered->End(buffered->length() - 1, NULL))); + } } if (buffered->length() == 0) { @@ -708,6 +844,9 @@ void MediaSource::OpenIfInEndedState() { return; } + DCHECK(!IsClosed()); + AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + SetReadyState(kMediaSourceReadyStateOpen); chunk_demuxer_->UnmarkEndOfStream(); } @@ -750,17 +889,25 @@ HTMLMediaElement* MediaSource::GetMediaElement() const { } MediaSourceAttachmentSupplement* MediaSource::GetMediaSourceAttachment() const { + base::AutoLock auto_lock(attachment_link_lock_); DCHECK(is_using_media_source_attachment_methods_); return media_source_attachment_; } -bool MediaSource::MediaElementHasMaxVideoCapabilities() const { +bool MediaSource::MediaElementHasMaxVideoCapabilities() { if (is_using_media_source_attachment_methods_) { - SB_DCHECK(media_source_attachment_); + if (IsMseInWorkersEnabled(environment_settings())) { + base::AutoLock auto_lock(attachment_link_lock_); + return media_source_attachment_->HasMaxVideoCapabilities(); + } else { + base::AutoLock auto_lock(attachment_link_lock_); + SB_DCHECK(media_source_attachment_); + return has_max_video_capabilities_; + } } else { SB_DCHECK(attached_element_); + return has_max_video_capabilities_; } - return has_max_video_capabilities_; } SerializedAlgorithmRunner* @@ -791,14 +938,18 @@ void MediaSource::TraceMembers(script::Tracer* tracer) { web::EventTarget::TraceMembers(tracer); tracer->Trace(event_queue_); - if (is_using_media_source_attachment_methods_) { - tracer->Trace(media_source_attachment_); - } else { - tracer->Trace(attached_element_); - } tracer->Trace(source_buffers_); tracer->Trace(active_source_buffers_); - tracer->Trace(live_seekable_range_); + { + // TODO(at-ninja): remove the lock from Trace to avoid deadlocks. + base::AutoLock auto_lock(attachment_link_lock_); + if (is_using_media_source_attachment_methods_) { + tracer->Trace(media_source_attachment_); + } else { + tracer->Trace(attached_element_); + } + tracer->Trace(live_seekable_range_); + } } void MediaSource::SetReadyState(MediaSourceReadyState ready_state) { @@ -842,6 +993,7 @@ void MediaSource::SetReadyState(MediaSourceReadyState ready_state) { source_buffers_->Clear(); if (is_using_media_source_attachment_methods_) { + base::AutoLock auto_lock(attachment_link_lock_); media_source_attachment_.reset(); } else { attached_element_.reset(); @@ -874,5 +1026,26 @@ void MediaSource::ScheduleEvent(base_token::Token event_name) { event_queue_.Enqueue(event); } +bool MediaSource::RunUnlessElementGoneOrClosingUs( + MediaSourceAttachmentSupplement::RunExclusivelyCB cb) { + auto attachment = GetMediaSourceAttachment(); + CHECK(attachment) << "Attempt to run operation requiring attachment, but " + "without having one."; + + return attachment->RunExclusively( + /*abort_if_not_fully_attached=*/true, std::move(cb)); +} + +void MediaSource::AssertAttachmentsMutexHeldIfCrossThreadForDebugging() const { +#if DCHECK_IS_ON() + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + base::AutoLock auto_lock(attachment_link_lock_); + DCHECK(media_source_attachment_); + media_source_attachment_->AssertCrossThreadMutexIsAcquiredForDebugging(); + } +#endif // DCHECK_IS_ON() +} + } // namespace dom } // namespace cobalt diff --git a/cobalt/dom/media_source.h b/cobalt/dom/media_source.h index 44087ec640f..a6088660ab8 100644 --- a/cobalt/dom/media_source.h +++ b/cobalt/dom/media_source.h @@ -50,6 +50,7 @@ #include "base/memory/ref_counted.h" #include "base/memory/weak_ptr.h" +#include "base/synchronization/lock.h" #include "base/threading/thread.h" #include "base/threading/thread_task_runner_handle.h" #include "cobalt/base/token.h" @@ -90,21 +91,30 @@ class MediaSource : public web::EventTarget { scoped_refptr source_buffers() const; scoped_refptr active_source_buffers() const; MediaSourceReadyState ready_state() const; - double duration(script::ExceptionState* exception_state) const; - void set_duration(double duration, script::ExceptionState* exception_state); + double duration(script::ExceptionState* exception_state) + LOCKS_EXCLUDED(attachment_link_lock_); + void set_duration(double duration, script::ExceptionState* exception_state) + LOCKS_EXCLUDED(attachment_link_lock_); scoped_refptr AddSourceBuffer( script::EnvironmentSettings* settings, const std::string& type, - script::ExceptionState* exception_state); + script::ExceptionState* exception_state) + LOCKS_EXCLUDED(attachment_link_lock_); void RemoveSourceBuffer(const scoped_refptr& source_buffer, - script::ExceptionState* exception_state); + script::ExceptionState* exception_state) + LOCKS_EXCLUDED(attachment_link_lock_); - void EndOfStreamAlgorithm(MediaSourceEndOfStreamError error); - void EndOfStream(script::ExceptionState* exception_state); + void EndOfStreamAlgorithm(MediaSourceEndOfStreamError error) + LOCKS_EXCLUDED(attachment_link_lock_); + void EndOfStream(script::ExceptionState* exception_state) + LOCKS_EXCLUDED(attachment_link_lock_); void EndOfStream(MediaSourceEndOfStreamError error, - script::ExceptionState* exception_state); + script::ExceptionState* exception_state) + LOCKS_EXCLUDED(attachment_link_lock_); void SetLiveSeekableRange(double start, double end, - script::ExceptionState* exception_state); - void ClearLiveSeekableRange(script::ExceptionState* exception_state); + script::ExceptionState* exception_state) + LOCKS_EXCLUDED(attachment_link_lock_); + void ClearLiveSeekableRange(script::ExceptionState* exception_state) + LOCKS_EXCLUDED(attachment_link_lock_); static bool IsTypeSupported(script::EnvironmentSettings* settings, const std::string& type); @@ -118,33 +128,64 @@ class MediaSource : public web::EventTarget { // TODO(b/338425449): Remove direct references to HTMLMediaElement. bool StartAttachingToMediaElement(HTMLMediaElement* media_element); bool StartAttachingToMediaElement( - MediaSourceAttachmentSupplement* media_source_attachment); - void CompleteAttachingToMediaElement(ChunkDemuxer* chunk_demuxer); + MediaSourceAttachmentSupplement* media_source_attachment) + LOCKS_EXCLUDED(attachment_link_lock_); + void CompleteAttachingToMediaElement(ChunkDemuxer* chunk_demuxer) + LOCKS_EXCLUDED(attachment_link_lock_); void Close(); bool IsClosed() const; - scoped_refptr GetBufferedRange() const; - scoped_refptr GetSeekable() const; + scoped_refptr GetBufferedRange() const + LOCKS_EXCLUDED(attachment_link_lock_); + scoped_refptr GetSeekable() const + LOCKS_EXCLUDED(attachment_link_lock_); void OnAudioTrackChanged(AudioTrack* audio_track); void OnVideoTrackChanged(VideoTrack* video_track); // Used by SourceBuffer. - void OpenIfInEndedState(); + void OpenIfInEndedState() LOCKS_EXCLUDED(attachment_link_lock_); bool IsOpen() const; - void SetSourceBufferActive(SourceBuffer* source_buffer, bool is_active); + void SetSourceBufferActive(SourceBuffer* source_buffer, bool is_active) + LOCKS_EXCLUDED(attachment_link_lock_); // TODO(b/338425449): Remove direct references to HTMLMediaElement. HTMLMediaElement* GetMediaElement() const; - MediaSourceAttachmentSupplement* GetMediaSourceAttachment() const; - bool MediaElementHasMaxVideoCapabilities() const; + MediaSourceAttachmentSupplement* GetMediaSourceAttachment() const + LOCKS_EXCLUDED(attachment_link_lock_); + // TODO(at-ninja): Add comment for this and _Locked methods. + bool MediaElementHasMaxVideoCapabilities() + LOCKS_EXCLUDED(attachment_link_lock_); + double GetDuration_Locked() const LOCKS_EXCLUDED(attachment_link_lock_); SerializedAlgorithmRunner* GetAlgorithmRunner( int job_size); DEFINE_WRAPPABLE_TYPE(MediaSource); void TraceMembers(script::Tracer* tracer) override; + bool RunUnlessElementGoneOrClosingUs( + MediaSourceAttachmentSupplement::RunExclusivelyCB cb) + LOCKS_EXCLUDED(attachment_link_lock_); + void AssertAttachmentsMutexHeldIfCrossThreadForDebugging() const + LOCKS_EXCLUDED(attachment_link_lock_); + private: - void SetReadyState(MediaSourceReadyState ready_state); + void SetReadyState(MediaSourceReadyState ready_state) + LOCKS_EXCLUDED(attachment_link_lock_); bool IsUpdating() const; void ScheduleEvent(base_token::Token event_name); + void DurationChangeAlgorithm(double new_duration, + script::ExceptionState* exception_state) + LOCKS_EXCLUDED(attachment_link_lock_); + + // Helpers that must be run bound with RunUnlessElementGoneOrClosingUs. + void AddSourceBuffer_Locked(script::EnvironmentSettings* settings, + const std::string& type, + script::ExceptionState* exception_state, + SourceBuffer** created_buffer) + LOCKS_EXCLUDED(attachment_link_lock_); + void RemoveSourceBuffer_Locked( + const scoped_refptr& source_buffer) + LOCKS_EXCLUDED(attachment_link_lock_); + void GetDurationInternal_Locked(double* result) + LOCKS_EXCLUDED(attachment_link_lock_); // Set to true to offload SourceBuffer buffer append and removal algorithms to // a non-web thread. @@ -175,14 +216,18 @@ class MediaSource : public web::EventTarget { // TODO(b/338425449): Remove direct references to HTMLMediaElement. bool is_using_media_source_attachment_methods_ = false; base::WeakPtr attached_element_; - base::WeakPtr media_source_attachment_; + base::WeakPtr media_source_attachment_ + GUARDED_BY(attachment_link_lock_); scoped_refptr source_buffers_; scoped_refptr active_source_buffers_; - scoped_refptr live_seekable_range_; + scoped_refptr live_seekable_range_ + GUARDED_BY(attachment_link_lock_); bool has_max_video_capabilities_ = false; + + mutable base::Lock attachment_link_lock_; }; } // namespace dom diff --git a/cobalt/dom/media_source_attachment.h b/cobalt/dom/media_source_attachment.h index 870e1e97dca..f09c74487b8 100644 --- a/cobalt/dom/media_source_attachment.h +++ b/cobalt/dom/media_source_attachment.h @@ -87,6 +87,11 @@ class MediaSourceAttachment // having an error. virtual void OnElementError() = 0; + // Provide state updates to the MediaSource. Needed for cross-thread + // notifications for MSE in DedicatedWorkers. + virtual void OnElementHasMaxVideoCapabilitiesUpdate( + bool has_max_video_capabilities) = 0; + private: friend class base::RefCountedThreadSafe; diff --git a/cobalt/dom/media_source_attachment_supplement.cc b/cobalt/dom/media_source_attachment_supplement.cc new file mode 100644 index 00000000000..d26f821d0df --- /dev/null +++ b/cobalt/dom/media_source_attachment_supplement.cc @@ -0,0 +1,33 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2020 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "cobalt/dom/media_source_attachment_supplement.h" + +#include + +namespace cobalt { +namespace dom { + +bool MediaSourceAttachmentSupplement::RunExclusively( + bool /*abort_if_not_fully_attached*/, RunExclusivelyCB cb) { + std::move(cb).Run(); + return true; +} + +} // namespace dom +} // namespace cobalt diff --git a/cobalt/dom/media_source_attachment_supplement.h b/cobalt/dom/media_source_attachment_supplement.h index acce3e03913..1f7b48db44e 100644 --- a/cobalt/dom/media_source_attachment_supplement.h +++ b/cobalt/dom/media_source_attachment_supplement.h @@ -19,7 +19,9 @@ #ifndef COBALT_DOM_MEDIA_SOURCE_ATTACHMENT_SUPPLEMENT_H_ #define COBALT_DOM_MEDIA_SOURCE_ATTACHMENT_SUPPLEMENT_H_ +#include "base/functional/callback.h" #include "base/memory/weak_ptr.h" +#include "base/types/pass_key.h" #include "cobalt/dom/audio_track_list.h" #include "cobalt/dom/media_source_attachment.h" #include "cobalt/dom/video_track_list.h" @@ -35,8 +37,7 @@ class MediaSourceAttachmentSupplement : public MediaSourceAttachment, public base::SupportsWeakPtr { public: - MediaSourceAttachmentSupplement() = default; - ~MediaSourceAttachmentSupplement() = default; + using RunExclusivelyCB = base::OnceCallback; // Communicates a change in the media resource duration to the attached media // element. In a same-thread attachment, communicates this information @@ -71,12 +72,26 @@ class MediaSourceAttachmentSupplement virtual scoped_refptr CreateVideoTrackList( script::EnvironmentSettings* settings) = 0; + virtual bool RunExclusively(bool abort_if_not_fully_attached, + RunExclusivelyCB cb); + + // Default implementation is a no-op. CrossThreadMediaSourceAttachment + // overrides this. Used by MediaSource and SourceBuffer for verifying + // that the correct mutex locks are held when doing cross-thread work. + virtual void AssertCrossThreadMutexIsAcquiredForDebugging() {} + + // Should only be called from SourceBuffer::RemovedFromMediaSource(). + virtual bool FullyAttachedOrSameThread() const { return true; } + // TODO(b/338425449): Remove media_element method after rollout. // References to underlying objects are exposed for when the H5VCC // flag MediaElement.EnableUsingMediaSourceAttachmentMethods is disabled. virtual base::WeakPtr media_element() const = 0; - private: + protected: + MediaSourceAttachmentSupplement() = default; + ~MediaSourceAttachmentSupplement() = default; + DISALLOW_COPY_AND_ASSIGN(MediaSourceAttachmentSupplement); }; diff --git a/cobalt/dom/same_thread_media_source_attachment.cc b/cobalt/dom/same_thread_media_source_attachment.cc index 8f85a02d26e..19dc3746b12 100644 --- a/cobalt/dom/same_thread_media_source_attachment.cc +++ b/cobalt/dom/same_thread_media_source_attachment.cc @@ -103,7 +103,11 @@ bool SameThreadMediaSourceAttachment::HasMaxVideoCapabilities() const { DCHECK_EQ(task_runner_, base::SequencedTaskRunner::GetCurrentDefault()); DCHECK(attached_element_); - return attached_element_->HasMaxVideoCapabilities(); + bool result = attached_element_->HasMaxVideoCapabilities(); + + DCHECK_EQ(result, element_has_max_video_capabilities_); + + return result; } double SameThreadMediaSourceAttachment::GetRecentMediaTime() { @@ -158,5 +162,10 @@ void SameThreadMediaSourceAttachment::OnElementError() { element_has_error_ = true; } +void SameThreadMediaSourceAttachment::OnElementHasMaxVideoCapabilitiesUpdate( + bool has_max_video_capabilities) { + element_has_max_video_capabilities_ = has_max_video_capabilities; +} + } // namespace dom } // namespace cobalt diff --git a/cobalt/dom/same_thread_media_source_attachment.h b/cobalt/dom/same_thread_media_source_attachment.h index abaf41c1f13..67db63cfb79 100644 --- a/cobalt/dom/same_thread_media_source_attachment.h +++ b/cobalt/dom/same_thread_media_source_attachment.h @@ -53,6 +53,8 @@ class SameThreadMediaSourceAttachment : public MediaSourceAttachmentSupplement { void OnElementTimeUpdate(double time) override; void OnElementError() override; + void OnElementHasMaxVideoCapabilitiesUpdate( + bool has_max_video_capabilities) override; // MediaSourceAttachmentSupplement void NotifyDurationChanged(double duration) override; @@ -89,6 +91,7 @@ class SameThreadMediaSourceAttachment : public MediaSourceAttachmentSupplement { double recent_element_time_; // See OnElementTimeUpdate(). bool element_has_error_; // See OnElementError(). + bool element_has_max_video_capabilities_; DISALLOW_COPY_AND_ASSIGN(SameThreadMediaSourceAttachment); }; diff --git a/cobalt/dom/source_buffer.cc b/cobalt/dom/source_buffer.cc index 2865b74062e..aba58a776d6 100644 --- a/cobalt/dom/source_buffer.cc +++ b/cobalt/dom/source_buffer.cc @@ -62,6 +62,7 @@ #include "cobalt/web/context.h" #include "cobalt/web/dom_exception.h" #include "cobalt/web/web_settings.h" +#include "cobalt/web/window_or_worker_global_scope.h" #include "media/base/ranges.h" #include "media/base/timestamp_constants.h" @@ -129,6 +130,17 @@ bool IsAvoidCopyingArrayBufferEnabled(web::EnvironmentSettings* settings) { return media_settings.IsAvoidCopyingArrayBufferEnabled().value_or(false); } +// If this function returns true, experimental support for creating MediaSource +// objects in Dedicated Workers will be enabled. This also allows MSE handles +// to be transferred from Dedicated Workers back to the main thread. +// Requires MediaElement.EnableUsingMediaSourceBufferedRange and +// MediaElement.EnableUsingMediaSourceAttachmentMethods as prerequisites for +// this feature. +// The default value is false. +bool IsMseInWorkersEnabled(web::EnvironmentSettings* settings) { + return GetMediaSettings(settings).IsMseInWorkersEnabled().value_or(false); +} + } // namespace SourceBuffer::OnInitSegmentReceivedHelper::OnInitSegmentReceivedHelper( @@ -172,24 +184,48 @@ SourceBuffer::SourceBuffer(script::EnvironmentSettings* settings, event_queue_(event_queue), is_using_media_source_attachment_methods_( is_using_media_source_attachment_methods), - audio_tracks_( - is_using_media_source_attachment_methods_ - ? media_source->GetMediaSourceAttachment()->CreateAudioTrackList( - settings) - : base::MakeRefCounted( - settings, media_source->GetMediaElement())), - video_tracks_( - is_using_media_source_attachment_methods_ - ? media_source->GetMediaSourceAttachment()->CreateVideoTrackList( - settings) - : base::MakeRefCounted( - settings, media_source->GetMediaElement())), metrics_(!media_source_->MediaElementHasMaxVideoCapabilities()) { DCHECK(!id_.empty()); DCHECK(media_source_); DCHECK(chunk_demuxer); DCHECK(event_queue); + web::EnvironmentSettings* web_settings = + base::polymorphic_downcast(settings); + DCHECK(web_settings); + if (is_using_media_source_attachment_methods && + IsMseInWorkersEnabled(web_settings)) { + DCHECK(web_settings->context()); + web::WindowOrWorkerGlobalScope* global_scope = + web_settings->context()->GetWindowOrWorkerGlobalScope(); + + if (global_scope->IsWindow()) { + audio_tracks_ = + media_source->GetMediaSourceAttachment()->CreateAudioTrackList( + settings); + video_tracks_ = + media_source->GetMediaSourceAttachment()->CreateVideoTrackList( + settings); + } else { + // Media tracks are not supported on worker threads. + DCHECK(!audio_tracks_); + DCHECK(!video_tracks_); + } + } else { + audio_tracks_ = + is_using_media_source_attachment_methods_ + ? media_source->GetMediaSourceAttachment()->CreateAudioTrackList( + settings) + : base::MakeRefCounted( + settings, media_source->GetMediaElement()); + video_tracks_ = + is_using_media_source_attachment_methods_ + ? media_source->GetMediaSourceAttachment()->CreateVideoTrackList( + settings) + : base::MakeRefCounted( + settings, media_source->GetMediaElement()); + } + LOG(INFO) << "Evict extra in bytes is set to " << evict_extra_in_bytes_; LOG(INFO) << "Max append size in bytes is set to " << max_append_buffer_size_; @@ -217,6 +253,28 @@ void SourceBuffer::set_mode(SourceBufferAppendMode mode, return; } + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + if (!media_source_->RunUnlessElementGoneOrClosingUs(base::Bind( + &SourceBuffer::SetMode_Locked, this, mode, exception_state))) { + web::DOMException::Raise(web::DOMException::kInvalidStateErr, + exception_state); + return; + } + } else { + SetMode_Locked(mode, exception_state); + return; + } +} + +void SourceBuffer::SetMode_Locked(SourceBufferAppendMode mode, + script::ExceptionState* exception_state) { + DCHECK(media_source_); + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + media_source_->AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + } + media_source_->OpenIfInEndedState(); if (chunk_demuxer_->IsParsingMediaSegment(id_)) { @@ -239,12 +297,32 @@ scoped_refptr SourceBuffer::buffered( } scoped_refptr time_ranges = new TimeRanges; + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + if (!media_source_->RunUnlessElementGoneOrClosingUs( + base::Bind(&SourceBuffer::GetBuffered_Locked, this, time_ranges))) { + web::DOMException::Raise(web::DOMException::kInvalidStateErr, + exception_state); + return NULL; + } + } else { + GetBuffered_Locked(time_ranges); + } + return time_ranges; +} + +void SourceBuffer::GetBuffered_Locked(scoped_refptr out) const { + DCHECK(out); + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + media_source_->AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + } + ::media::Ranges ranges = chunk_demuxer_->GetBufferedRanges(id_); for (size_t i = 0; i < ranges.size(); i++) { - time_ranges->Add(ranges.start(i).InSecondsF(), ranges.end(i).InSecondsF()); + out->Add(ranges.start(i).InSecondsF(), ranges.end(i).InSecondsF()); } - return time_ranges; } double SourceBuffer::timestamp_offset( @@ -266,6 +344,28 @@ void SourceBuffer::set_timestamp_offset( return; } + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + if (!media_source_->RunUnlessElementGoneOrClosingUs( + base::Bind(&SourceBuffer::SetTimestampOffset_Locked, this, offset, + exception_state))) { + web::DOMException::Raise(web::DOMException::kInvalidStateErr, + exception_state); + return; + } + } else { + SetTimestampOffset_Locked(offset, exception_state); + } +} + +void SourceBuffer::SetTimestampOffset_Locked( + double offset, script::ExceptionState* exception_state) { + DCHECK(media_source_); + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + media_source_->AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + } + media_source_->OpenIfInEndedState(); if (chunk_demuxer_->IsParsingMediaSegment(id_)) { @@ -301,6 +401,26 @@ void SourceBuffer::set_append_window_start( return; } + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + if (!media_source_->RunUnlessElementGoneOrClosingUs(base::Bind( + &SourceBuffer::SetAppendWindowStart_Locked, this, start))) { + web::DOMException::Raise(web::DOMException::kInvalidStateErr, + exception_state); + return; + } + } else { + SetAppendWindowStart_Locked(start); + } +} + +void SourceBuffer::SetAppendWindowStart_Locked(double start) { + DCHECK(media_source_); + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + media_source_->AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + } + append_window_start_ = start; } @@ -328,6 +448,26 @@ void SourceBuffer::set_append_window_end( return; } + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + if (!media_source_->RunUnlessElementGoneOrClosingUs( + base::Bind(&SourceBuffer::SetAppendWindowEnd_Locked, this, end))) { + web::DOMException::Raise(web::DOMException::kInvalidStateErr, + exception_state); + return; + } + } else { + SetAppendWindowEnd_Locked(end); + } +} + +void SourceBuffer::SetAppendWindowEnd_Locked(double end) { + DCHECK(media_source_); + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + media_source_->AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + } + append_window_end_ = end; } @@ -391,6 +531,26 @@ void SourceBuffer::Abort(script::ExceptionState* exception_state) { active_algorithm_handle_ = nullptr; } + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + if (!media_source_->RunUnlessElementGoneOrClosingUs( + base::Bind(&SourceBuffer::Abort_Locked, this, exception_state))) { + web::DOMException::Raise(web::DOMException::kInvalidStateErr, + exception_state); + return; + } + } else { + Abort_Locked(exception_state); + } +} + +void SourceBuffer::Abort_Locked(script::ExceptionState* exception_state) { + DCHECK(media_source_); + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + media_source_->AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + } + base::TimeDelta timestamp_offset = DoubleToTimeDelta(timestamp_offset_); chunk_demuxer_->ResetParserState(id_, DoubleToTimeDelta(append_window_start_), DoubleToTimeDelta(append_window_end_), @@ -417,8 +577,35 @@ void SourceBuffer::Remove(double start, double end, return; } - if (start < 0 || std::isnan(media_source_->duration(NULL)) || - start > media_source_->duration(NULL)) { + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + if (!media_source_->RunUnlessElementGoneOrClosingUs(base::Bind( + &SourceBuffer::Remove_Locked, this, start, end, exception_state))) { + web::DOMException::Raise(web::DOMException::kInvalidStateErr, + exception_state); + return; + } + } else { + Remove_Locked(start, end, exception_state); + } +} + +void SourceBuffer::Remove_Locked(double start, double end, + script::ExceptionState* exception_state) { + DCHECK(media_source_); + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + media_source_->AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + } + + double duration; + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + duration = media_source_->GetDuration_Locked(); + } else { + duration = media_source_->duration(NULL); + } + if (start < 0 || std::isnan(duration) || start > duration) { exception_state->SetSimpleException(script::kSimpleTypeError); return; } @@ -480,6 +667,7 @@ void SourceBuffer::OnRemovedFromMediaSource() { if (media_source_ == NULL) { return; } + media_source_->AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); DCHECK(on_init_segment_received_helper_); on_init_segment_received_helper_->Detach(); @@ -502,9 +690,22 @@ void SourceBuffer::OnRemovedFromMediaSource() { // track, and print the steam type along with the metrics. metrics_.PrintCurrentMetricsAndUpdateAccumulatedMetrics(); - chunk_demuxer_->RemoveId(id_); - if (chunk_demuxer_->GetAllStreams().empty()) { - metrics_.PrintAccumulatedMetrics(); + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + MediaSourceAttachmentSupplement* attachment = + media_source_->GetMediaSourceAttachment(); + DCHECK(attachment); + if (attachment->FullyAttachedOrSameThread()) { + chunk_demuxer_->RemoveId(id_); + if (chunk_demuxer_->GetAllStreams().empty()) { + metrics_.PrintAccumulatedMetrics(); + } + } + } else { + chunk_demuxer_->RemoveId(id_); + if (chunk_demuxer_->GetAllStreams().empty()) { + metrics_.PrintAccumulatedMetrics(); + } } chunk_demuxer_ = NULL; @@ -519,6 +720,7 @@ void SourceBuffer::OnRemovedFromMediaSource() { double SourceBuffer::GetHighestPresentationTimestamp() const { DCHECK(media_source_ != NULL); + media_source_->AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); return chunk_demuxer_->GetHighestPresentationTimestamp(id_).InSecondsF(); } @@ -641,6 +843,44 @@ void SourceBuffer::AppendBufferInternal( script::ExceptionState* exception_state) { TRACE_EVENT1("cobalt::dom", "SourceBuffer::AppendBufferInternal()", "size", size); + + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + if (media_source_ == NULL) { + web::DOMException::Raise(web::DOMException::kInvalidStateErr, + exception_state); + return false; + } + if (updating()) { + web::DOMException::Raise(web::DOMException::kInvalidStateErr, + exception_state); + return false; + } + } + + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + if (!media_source_->RunUnlessElementGoneOrClosingUs( + base::Bind(&SourceBuffer::AppendBufferInternal_Locked, this, data, + size, exception_state))) { + web::DOMException::Raise(web::DOMException::kInvalidStateErr, + exception_state); + return; + } + } else { + AppendBufferInternal_Locked(data, size, exception_state); + } +} + +void SourceBuffer::AppendBufferInternal_Locked( + const unsigned char* data, size_t size, + script::ExceptionState* exception_state) { + DCHECK(media_source_); + if (is_using_media_source_attachment_methods_ && + IsMseInWorkersEnabled(environment_settings())) { + media_source_->AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); + } + if (!PrepareAppend(size, exception_state)) { return; } diff --git a/cobalt/dom/source_buffer.h b/cobalt/dom/source_buffer.h index ae9efef8bda..2da7da7ec21 100644 --- a/cobalt/dom/source_buffer.h +++ b/cobalt/dom/source_buffer.h @@ -149,6 +149,11 @@ class SourceBuffer : public web::EventTarget { size_t memory_limit(script::ExceptionState* exception_state) const; void set_memory_limit(size_t limit, script::ExceptionState* exception_state); + // TODO(at-ninja): Add comment about _Locked methods. + void SetMode_Locked(SourceBufferAppendMode mode, + script::ExceptionState* exception_state); + void GetBuffered_Locked(scoped_refptr out) const; + private: typedef ::media::MediaTracks MediaTracks; typedef script::ArrayBuffer ArrayBuffer; @@ -195,6 +200,17 @@ class SourceBuffer : public web::EventTarget { const std::string& track_type, const std::string& byte_stream_track_id) const; + // TODO(at-ninja): Add comment about _Locked methods. + void SetTimestampOffset_Locked(double offset, + script::ExceptionState* exception_state); + void SetAppendWindowStart_Locked(double start); + void SetAppendWindowEnd_Locked(double end); + void Abort_Locked(script::ExceptionState* exception_state); + void Remove_Locked(double start, double end, + script::ExceptionState* exception_state); + void AppendBufferInternal_Locked(const unsigned char* data, size_t size, + script::ExceptionState* exception_state); + static const size_t kDefaultMaxAppendBufferSize = 128 * 1024; scoped_refptr on_init_segment_received_helper_;