From 10bea3bb9dfaea4ea2f0fb2d641ef526f417731a Mon Sep 17 00:00:00 2001 From: Matthew Lai Date: Mon, 23 Dec 2024 20:23:38 +0800 Subject: [PATCH 1/2] Fixed HWAccel so we don't share contexts between streams --- av/codec/hwaccel.pyx | 27 ++++++++++++++++----------- av/container/input.pyx | 7 +++++++ av/video/codeccontext.pyx | 2 +- examples/basics/hw_decode.py | 9 ++++++--- tests/test_decode.py | 30 ++++++++++++++++++++---------- 5 files changed, 50 insertions(+), 25 deletions(-) diff --git a/av/codec/hwaccel.pyx b/av/codec/hwaccel.pyx index 1c96d02e8..90140f59e 100644 --- a/av/codec/hwaccel.pyx +++ b/av/codec/hwaccel.pyx @@ -1,3 +1,4 @@ +import copy import weakref from enum import IntEnum @@ -7,6 +8,7 @@ from av.codec.codec cimport Codec from av.dictionary cimport _Dictionary from av.error cimport err_check from av.video.format cimport get_video_format + from av.dictionary import Dictionary @@ -94,11 +96,13 @@ cpdef hwdevices_available(): cdef class HWAccel: - def __init__(self, device_type, device=None, codec=None, allow_software_fallback=True, options=None): + def __init__(self, device_type, device=None, allow_software_fallback=True, options=None): if isinstance(device_type, HWDeviceType): self._device_type = device_type elif isinstance(device_type, str): self._device_type = int(lib.av_hwdevice_find_type_by_name(device_type)) + elif isinstance(device_type, int): + self._device_type = device_type else: raise ValueError("Unknown type for device_type") @@ -106,22 +110,18 @@ cdef class HWAccel: self.allow_software_fallback = allow_software_fallback self.options = {} if not options else dict(options) self.ptr = NULL - self.codec = codec self.config = None - if codec: - self._initialize_hw_context() - - def _initialize_hw_context(self): + def _initialize_hw_context(self, Codec codec not None): cdef HWConfig config - for config in self.codec.hardware_configs: + for config in codec.hardware_configs: if not (config.ptr.methods & lib.AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX): continue if self._device_type and config.device_type != self._device_type: continue break else: - raise NotImplementedError(f"No supported hardware config for {self.codec}") + raise NotImplementedError(f"No supported hardware config for {codec}") self.config = config @@ -142,9 +142,14 @@ cdef class HWAccel: if self.ptr: raise RuntimeError("Hardware context already initialized") - self.codec = codec - self._initialize_hw_context() - return self + ret = HWAccel( + device_type=self._device_type, + device=self._device, + allow_software_fallback=self.allow_software_fallback, + options=self.options + ) + ret._initialize_hw_context(codec) + return ret def __dealloc__(self): if self.ptr: diff --git a/av/container/input.pyx b/av/container/input.pyx index aa9940452..1ba4750d7 100644 --- a/av/container/input.pyx +++ b/av/container/input.pyx @@ -68,6 +68,8 @@ cdef class InputContainer(Container): lib.av_dict_free(&c_options[i]) free(c_options) + at_least_one_accelerated_context = False + self.streams = StreamContainer() for i in range(self.ptr.nb_streams): stream = self.ptr.streams[i] @@ -78,11 +80,16 @@ cdef class InputContainer(Container): err_check(lib.avcodec_parameters_to_context(codec_context, stream.codecpar)) codec_context.pkt_timebase = stream.time_base py_codec_context = wrap_codec_context(codec_context, codec, self.hwaccel) + if py_codec_context.is_hwaccel: + at_least_one_accelerated_context = True else: # no decoder is available py_codec_context = None self.streams.add_stream(wrap_stream(self, stream, py_codec_context)) + if self.hwaccel and not self.hwaccel.allow_software_fallback and not at_least_one_accelerated_context: + raise RuntimeError("Hardware accelerated decode requested but no stream is compatible") + self.metadata = avdict_to_dict(self.ptr.metadata, self.metadata_encoding, self.metadata_errors) def __dealloc__(self): diff --git a/av/video/codeccontext.pyx b/av/video/codeccontext.pyx index 92470c159..c9d8eb4c0 100644 --- a/av/video/codeccontext.pyx +++ b/av/video/codeccontext.pyx @@ -54,7 +54,7 @@ cdef class VideoCodecContext(CodecContext): # stream with it, so we shouldn't abort even if we find a stream that can't # be HW decoded. # If the user wants to make sure hwaccel is actually used, they can check with the - # is_hardware_accelerated() function on each stream's codec context. + # is_hwaccel() function on each stream's codec context. self.hwaccel_ctx = None self._build_format() diff --git a/examples/basics/hw_decode.py b/examples/basics/hw_decode.py index 1ce7a11af..eb2ac7157 100644 --- a/examples/basics/hw_decode.py +++ b/examples/basics/hw_decode.py @@ -30,11 +30,14 @@ ) if HW_DEVICE is None: - av.codec.hwaccel.dump_hwdevices() - print("Please set HW_DEVICE.") + print( + f"Please set HW_DEVICE. Options are: {str(av.codec.hwaccel.hwdevices_available())}" + ) exit() -assert HW_DEVICE in av.codec.hwaccel.hwdevices_available, f"{HW_DEVICE} not available." +assert ( + HW_DEVICE in av.codec.hwaccel.hwdevices_available() +), f"{HW_DEVICE} not available." print("Decoding in software (auto threading)...") diff --git a/tests/test_decode.py b/tests/test_decode.py index fc293d201..91119d324 100644 --- a/tests/test_decode.py +++ b/tests/test_decode.py @@ -8,12 +8,16 @@ import av +# This import is needed to make the test_decoded_time_base test pass when we run only this test file. +# Not sure why. +from av.subtitles import subtitle + from .common import TestCase, fate_suite @functools.cache def make_h264_test_video(path: str) -> None: - """Generates a black H264 test video for testing hardware decoding.""" + """Generates a black H264 test video with two streams for testing hardware decoding.""" # We generate a file here that's designed to be as compatible as possible with hardware # encoders. Hardware encoders are sometimes very picky and the errors we get are often @@ -23,21 +27,27 @@ def make_h264_test_video(path: str) -> None: # 8-bit yuv420p. pathlib.Path(path).parent.mkdir(parents=True, exist_ok=True) output_container = av.open(path, "w") - stream = output_container.add_stream("libx264", rate=24) - assert isinstance(stream, av.VideoStream) - stream.width = 1280 - stream.height = 720 - stream.pix_fmt = "yuv420p" + + streams = [] + for _ in range(2): + stream = output_container.add_stream("libx264", rate=24) + assert isinstance(stream, av.VideoStream) + stream.width = 1280 + stream.height = 720 + stream.pix_fmt = "yuv420p" + streams.append(stream) for _ in range(24): frame = av.VideoFrame.from_ndarray( np.zeros((720, 1280, 3), dtype=np.uint8), format="rgb24" ) - for packet in stream.encode(frame): - output_container.mux(packet) + for stream in streams: + for packet in stream.encode(frame): + output_container.mux(packet) - for packet in stream.encode(): - output_container.mux(packet) + for stream in streams: + for packet in stream.encode(): + output_container.mux(packet) output_container.close() From be9d4daa430c383b8e2eb2d807534738cbbce4af Mon Sep 17 00:00:00 2001 From: WyattBlue Date: Tue, 24 Dec 2024 00:43:24 -0500 Subject: [PATCH 2/2] Fix --- av/codec/hwaccel.pyx | 1 - examples/basics/hw_decode.py | 18 ++++++++---------- tests/test_decode.py | 4 ---- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/av/codec/hwaccel.pyx b/av/codec/hwaccel.pyx index 90140f59e..b80c194af 100644 --- a/av/codec/hwaccel.pyx +++ b/av/codec/hwaccel.pyx @@ -1,4 +1,3 @@ -import copy import weakref from enum import IntEnum diff --git a/examples/basics/hw_decode.py b/examples/basics/hw_decode.py index eb2ac7157..605ee1841 100644 --- a/examples/basics/hw_decode.py +++ b/examples/basics/hw_decode.py @@ -3,6 +3,7 @@ import av import av.datasets +from av.codec.hwaccel import HWAccel, hwdevices_available # What accelerator to use. # Recommendations: @@ -30,14 +31,10 @@ ) if HW_DEVICE is None: - print( - f"Please set HW_DEVICE. Options are: {str(av.codec.hwaccel.hwdevices_available())}" - ) + print(f"Please set HW_DEVICE. Options are: {hwdevices_available()}") exit() -assert ( - HW_DEVICE in av.codec.hwaccel.hwdevices_available() -), f"{HW_DEVICE} not available." +assert HW_DEVICE in hwdevices_available(), f"{HW_DEVICE} not available." print("Decoding in software (auto threading)...") @@ -56,11 +53,12 @@ assert frame_count == container.streams.video[0].frames container.close() -print(f"Decoded with software in {sw_time:.2f}s ({sw_fps:.2f} fps).") - -print(f"Decoding with {HW_DEVICE}") +print( + f"Decoded with software in {sw_time:.2f}s ({sw_fps:.2f} fps).\n" + f"Decoding with {HW_DEVICE}" +) -hwaccel = av.codec.hwaccel.HWAccel(device_type=HW_DEVICE, allow_software_fallback=False) +hwaccel = HWAccel(device_type=HW_DEVICE, allow_software_fallback=False) # Note the additional argument here. container = av.open(test_file_path, hwaccel=hwaccel) diff --git a/tests/test_decode.py b/tests/test_decode.py index 91119d324..c1846af69 100644 --- a/tests/test_decode.py +++ b/tests/test_decode.py @@ -8,10 +8,6 @@ import av -# This import is needed to make the test_decoded_time_base test pass when we run only this test file. -# Not sure why. -from av.subtitles import subtitle - from .common import TestCase, fate_suite