diff --git a/lib/membrane_h265_plugin/parser.ex b/lib/membrane_h265_plugin/parser.ex index 26200bf..f0bce44 100644 --- a/lib/membrane_h265_plugin/parser.ex +++ b/lib/membrane_h265_plugin/parser.ex @@ -66,8 +66,7 @@ defmodule Membrane.H265.Parser do def_output_pad :output, demand_mode: :auto, - accepted_format: - %H265{alignment: alignment, nalu_in_metadata?: true} when alignment in [:au, :nalu] + accepted_format: %H265{nalu_in_metadata?: true} def_options vpss: [ spec: [binary()], @@ -114,6 +113,18 @@ defmodule Membrane.H265.Parser do Defaults to true. """ ], + repeat_parameter_sets: [ + spec: boolean(), + default: false, + description: """ + Repeat all parameter sets (`vps`, `sps` and `pps`) on each IRAP picture. + + Parameter sets may be retrieved from: + * The stream + * `Parser` options. + * `Decoder Configuration Record`, sent in `:hcv1` and `:hev1` stream types + """ + ], output_alignment: [ spec: :au | :nalu, default: :au, @@ -161,6 +172,7 @@ defmodule Membrane.H265.Parser do output_alignment: opts.output_alignment, framerate: opts.framerate, skip_until_keyframe: opts.skip_until_keyframe, + repeat_parameter_sets: opts.repeat_parameter_sets, frame_prefix: <<>>, cached_vpss: %{}, cached_spss: %{}, @@ -443,7 +455,9 @@ defmodule Membrane.H265.Parser do remove_parameter_sets(au) _stream_structure -> - delete_duplicate_parameter_sets(au) + au + |> maybe_add_parameter_sets(state) + |> delete_duplicate_parameter_sets() end {au, stream_format_actions, state} @@ -507,6 +521,17 @@ defmodule Membrane.H265.Parser do end) end + @spec maybe_add_parameter_sets(AUSplitter.access_unit(), state()) :: AUSplitter.access_unit() + defp maybe_add_parameter_sets(au, %{repeat_parameter_sets: false}), do: au + + defp maybe_add_parameter_sets(au, state) do + if irap_au?(au), + do: + Map.values(state.cached_vpss) ++ + Map.values(state.cached_spss) ++ Map.values(state.cached_ppss) ++ au, + else: au + end + @spec remove_parameter_sets(AUSplitter.access_unit()) :: AUSplitter.access_unit() defp remove_parameter_sets(au) do Enum.reject(au, &(&1.type in [:vps, :sps, :pps])) @@ -514,11 +539,11 @@ defmodule Membrane.H265.Parser do @spec delete_duplicate_parameter_sets(AUSplitter.access_unit()) :: AUSplitter.access_unit() defp delete_duplicate_parameter_sets(au) do - if idr_au?(au), do: Enum.uniq(au), else: au + if irap_au?(au), do: Enum.uniq(au), else: au end - @spec idr_au?(AUSplitter.access_unit()) :: boolean() - defp idr_au?(au), do: Enum.any?(au, &(&1.type in NALuTypes.irap_nalus())) + @spec irap_au?(AUSplitter.access_unit()) :: boolean() + defp irap_au?(au), do: Enum.any?(au, &(&1.type in NALuTypes.irap_nalus())) @spec wrap_into_buffer( AUSplitter.access_unit(), diff --git a/lib/membrane_h265_plugin/parser/nalu_parser.ex b/lib/membrane_h265_plugin/parser/nalu_parser.ex index a67ba6d..c426c43 100644 --- a/lib/membrane_h265_plugin/parser/nalu_parser.ex +++ b/lib/membrane_h265_plugin/parser/nalu_parser.ex @@ -71,30 +71,25 @@ defmodule Membrane.H265.Parser.NALuParser do type = NALuTypes.get_type(parsed_fields.nal_unit_type) - {nalu, scheme_parser_state} = - try do - {parsed_fields, scheme_parser_state} = - parse_proper_nalu_type(nalu_body, scheme_parser_state, type) - - {%NALu{ - parsed_fields: parsed_fields, - type: type, - status: :valid, - stripped_prefix: prefix, - payload: unprefixed_nalu_payload, - timestamps: timestamps - }, scheme_parser_state} - catch - "Cannot load information from SPS" -> - {%NALu{ - parsed_fields: parsed_fields, - type: type, - status: :error, - stripped_prefix: prefix, - payload: unprefixed_nalu_payload, - timestamps: timestamps - }, scheme_parser_state} - end + {parsed_fields, scheme_parser_state} = + parse_proper_nalu_type(nalu_body, scheme_parser_state, type) + + # Mark nalu as invalid if there's no parameter sets + nalu_status = + if type in NALuTypes.vcl_nalu_types() and + not Map.has_key?(parsed_fields, :separate_colour_plane_flag), + do: :error, + else: :valid + + nalu = + %NALu{ + parsed_fields: parsed_fields, + type: type, + status: nalu_status, + stripped_prefix: prefix, + payload: unprefixed_nalu_payload, + timestamps: timestamps + } state = %{state | scheme_parser_state: scheme_parser_state} diff --git a/lib/membrane_h265_plugin/parser/nalu_parser/schemes/slice.ex b/lib/membrane_h265_plugin/parser/nalu_parser/schemes/slice.ex index 5ad0bff..6fb18c0 100644 --- a/lib/membrane_h265_plugin/parser/nalu_parser/schemes/slice.ex +++ b/lib/membrane_h265_plugin/parser/nalu_parser/schemes/slice.ex @@ -37,7 +37,7 @@ defmodule Membrane.H265.Parser.NALuParser.Schemes.Slice do {payload, state} else - _error -> throw("Cannot load information from SPS") + _error -> {payload, state} end end end diff --git a/test/fixtures/input-60-640x480-no-parameter-sets.h265 b/test/fixtures/input-60-640x480-no-parameter-sets.h265 index 8646d44..4dce315 100644 Binary files a/test/fixtures/input-60-640x480-no-parameter-sets.h265 and b/test/fixtures/input-60-640x480-no-parameter-sets.h265 differ diff --git a/test/fixtures/input-60-640x480-variable-parameters.h265 b/test/fixtures/input-60-640x480-variable-parameters.h265 new file mode 100644 index 0000000..e255f60 Binary files /dev/null and b/test/fixtures/input-60-640x480-variable-parameters.h265 differ diff --git a/test/fixtures/reference-60-640x480-variable-parameters.h265 b/test/fixtures/reference-60-640x480-variable-parameters.h265 new file mode 100644 index 0000000..93e4a0e Binary files /dev/null and b/test/fixtures/reference-60-640x480-variable-parameters.h265 differ diff --git a/test/fixtures/reference-60-640x480-with-parameter-sets.h265 b/test/fixtures/reference-60-640x480-with-parameter-sets.h265 new file mode 100644 index 0000000..c9a4be7 Binary files /dev/null and b/test/fixtures/reference-60-640x480-with-parameter-sets.h265 differ diff --git a/test/parser/au_splitter_test.exs b/test/parser/au_splitter_test.exs index a7a072e..3e6b20e 100644 --- a/test/parser/au_splitter_test.exs +++ b/test/parser/au_splitter_test.exs @@ -7,7 +7,7 @@ defmodule AUSplitterTest do # These values were obtained with the use of FFmpeg @au_lengths_ffmpeg %{ - "10-1920x1080" => [133_766, 1517, 474, 311, 298, 200, 3257, 547, 209, 233], + "10-1920x1080" => [10_117, 493, 406, 447, 428, 320, 285, 297, 306, 296], "10-480x320-mainstillpicture" => [ 35_114, 8824, diff --git a/test/parser/repeat_parameter_sets_test.exs b/test/parser/repeat_parameter_sets_test.exs new file mode 100644 index 0000000..6dfff31 --- /dev/null +++ b/test/parser/repeat_parameter_sets_test.exs @@ -0,0 +1,131 @@ +defmodule Membrane.H265.RepeatParameterSetsTest do + @moduledoc false + + use ExUnit.Case, async: true + + import Membrane.ChildrenSpec + import Membrane.Testing.Assertions + import Membrane.H265.Support.Common + + alias Membrane.H265 + alias Membrane.H265.Parser.NALuSplitter + alias Membrane.Testing.{Pipeline, Sink} + + @in_path "../fixtures/input-60-640x480-no-parameter-sets.h265" |> Path.expand(__DIR__) + @ref_path "../fixtures/reference-60-640x480-with-parameter-sets.h265" |> Path.expand(__DIR__) + + @vps <<64, 1, 12, 1, 255, 255, 33, 96, 0, 0, 3, 0, 144, 0, 0, 3, 0, 0, 3, 0, 153, 149, 152, 9>> + @sps <<66, 1, 1, 33, 96, 0, 0, 3, 0, 144, 0, 0, 3, 0, 0, 3, 0, 153, 160, 5, 2, 1, 225, 101, 149, + 154, 73, 50, 188, 57, 160, 32, 0, 0, 3, 0, 32, 0, 0, 3, 3, 193>> + @pps <<68, 1, 193, 114, 180, 98, 64>> + @dcr <<1, 33, 96, 0, 0, 0, 144, 0, 0, 0, 0, 0, 153, 240, 0, 252, 253, 248, 248, 0, 0, 15, 3, + 160, 0, 1, 0, 24, 64, 1, 12, 1, 255, 255, 33, 96, 0, 0, 3, 0, 144, 0, 0, 3, 0, 0, 3, 0, + 153, 149, 152, 9, 161, 0, 1, 0, 42, 66, 1, 1, 33, 96, 0, 0, 3, 0, 144, 0, 0, 3, 0, 0, 3, + 0, 153, 160, 5, 2, 1, 225, 101, 149, 154, 73, 50, 188, 57, 160, 32, 0, 0, 3, 0, 32, 0, 0, + 3, 3, 193, 162, 0, 1, 0, 7, 68, 1, 193, 114, 180, 98, 64>> + + defp make_pipeline( + source, + vpss \\ [], + spss \\ [], + ppss \\ [], + output_stream_structure \\ :annexb + ) do + structure = + child(:source, source) + |> child(:parser, %H265.Parser{ + vpss: vpss, + spss: spss, + ppss: ppss, + repeat_parameter_sets: true, + output_stream_structure: output_stream_structure + }) + |> child(:sink, Sink) + + Pipeline.start_link_supervised!(structure: structure) + end + + defp perform_test( + pipeline_pid, + data, + mode \\ :bytestream, + parser_input_stream_structure \\ :annexb, + parser_output_stream_structure \\ :annexb + ) do + buffers = prepare_buffers(data, mode, parser_input_stream_structure) + + assert_pipeline_play(pipeline_pid) + actions = for buffer <- buffers, do: {:buffer, {:output, buffer}} + Pipeline.message_child(pipeline_pid, :source, actions ++ [end_of_stream: :output]) + + output_buffers = + prepare_buffers( + File.read!(@ref_path), + :au_aligned, + parser_output_stream_structure + ) + + Enum.each(output_buffers, fn output_buffer -> + assert_sink_buffer(pipeline_pid, :sink, buffer) + assert buffer.payload == output_buffer.payload + end) + + assert_end_of_stream(pipeline_pid, :sink, :input, 3_000) + Pipeline.terminate(pipeline_pid, blocking?: true) + end + + defp split_access_unit(access_unit) do + {nalus, _splitter} = NALuSplitter.split(access_unit, true, NALuSplitter.new()) + Enum.sort(nalus) + end + + describe "Parameter sets should be reapeated on each IRAP access unit" do + test "when provided by parser options" do + source = %H265.Support.TestSource{mode: :bytestream} + pid = make_pipeline(source, [@vps], [@sps], [@pps]) + perform_test(pid, File.read!(@in_path)) + end + + test "when retrieved from the bytestream" do + source = %H265.Support.TestSource{mode: :bytestream} + pid = make_pipeline(source) + + data = Enum.join([<<>>, @vps, @sps, @pps], <<0, 0, 0, 1>>) <> File.read!(@in_path) + perform_test(pid, data) + end + + test "when provided via DCR" do + source = %H265.Support.TestSource{ + mode: :au_aligned, + output_raw_stream_structure: {:hev1, @dcr} + } + + pid = make_pipeline(source, [], [], [], {:hev1, 4}) + perform_test(pid, File.read!(@in_path), :au_aligned, {:hev1, 4}, {:hev1, 4}) + end + + test "when bytestream has variable parameter sets" do + in_path = "./test/fixtures/input-60-640x480-variable-parameters.h265" + ref_path = "./test/fixtures/reference-60-640x480-variable-parameters.h265" + + source = %H265.Support.TestSource{mode: :bytestream} + pid = make_pipeline(source) + + buffers = prepare_buffers(File.read!(in_path), :bytestream) + + assert_pipeline_play(pid) + actions = for buffer <- buffers, do: {:buffer, {:output, buffer}} + Pipeline.message_child(pid, :source, actions ++ [end_of_stream: :output]) + + File.read!(ref_path) + |> prepare_buffers(:au_aligned) + |> Enum.each(fn output_buffer -> + assert_sink_buffer(pid, :sink, buffer) + assert split_access_unit(output_buffer.payload) == split_access_unit(buffer.payload) + end) + + assert_end_of_stream(pid, :sink, :input, 3_000) + Pipeline.terminate(pid, blocking?: true) + end + end +end