Repeat parameter sets on each IRAP picture (#6)

lib/membrane_h265_plugin/parser.ex

@@ -66,8 +66,7 @@ defmodule Membrane.H265.Parser do
 
   def_output_pad :output,
     demand_mode: :auto,
-    accepted_format:
-      %H265{alignment: alignment, nalu_in_metadata?: true} when alignment in [:au, :nalu]
+    accepted_format: %H265{nalu_in_metadata?: true}
 
   def_options vpss: [
                 spec: [binary()],
@@ -114,6 +113,18 @@ defmodule Membrane.H265.Parser do
                 Defaults to true.
                 """
               ],
+              repeat_parameter_sets: [
+                spec: boolean(),
+                default: false,
+                description: """
+                Repeat all parameter sets (`vps`, `sps` and `pps`) on each IRAP picture.
+
+                Parameter sets may be retrieved from:
+                  * The stream
+                  * `Parser` options.
+                  * `Decoder Configuration Record`, sent in `:hcv1` and `:hev1` stream types
+                """
+              ],
               output_alignment: [
                 spec: :au | :nalu,
                 default: :au,
@@ -161,6 +172,7 @@ defmodule Membrane.H265.Parser do
       output_alignment: opts.output_alignment,
       framerate: opts.framerate,
       skip_until_keyframe: opts.skip_until_keyframe,
+      repeat_parameter_sets: opts.repeat_parameter_sets,
       frame_prefix: <<>>,
       cached_vpss: %{},
       cached_spss: %{},
@@ -443,7 +455,9 @@ defmodule Membrane.H265.Parser do
           remove_parameter_sets(au)
 
         _stream_structure ->
-          delete_duplicate_parameter_sets(au)
+          au
+          |> maybe_add_parameter_sets(state)
+          |> delete_duplicate_parameter_sets()
       end
 
     {au, stream_format_actions, state}
@@ -507,18 +521,29 @@ defmodule Membrane.H265.Parser do
     end)
   end
 
+  @spec maybe_add_parameter_sets(AUSplitter.access_unit(), state()) :: AUSplitter.access_unit()
+  defp maybe_add_parameter_sets(au, %{repeat_parameter_sets: false}), do: au
+
+  defp maybe_add_parameter_sets(au, state) do
+    if irap_au?(au),
+      do:
+        Map.values(state.cached_vpss) ++
+          Map.values(state.cached_spss) ++ Map.values(state.cached_ppss) ++ au,
+      else: au
+  end
+
   @spec remove_parameter_sets(AUSplitter.access_unit()) :: AUSplitter.access_unit()
   defp remove_parameter_sets(au) do
     Enum.reject(au, &(&1.type in [:vps, :sps, :pps]))
   end
 
   @spec delete_duplicate_parameter_sets(AUSplitter.access_unit()) :: AUSplitter.access_unit()
   defp delete_duplicate_parameter_sets(au) do
-    if idr_au?(au), do: Enum.uniq(au), else: au
+    if irap_au?(au), do: Enum.uniq(au), else: au
   end
 
-  @spec idr_au?(AUSplitter.access_unit()) :: boolean()
-  defp idr_au?(au), do: Enum.any?(au, &(&1.type in NALuTypes.irap_nalus()))
+  @spec irap_au?(AUSplitter.access_unit()) :: boolean()
+  defp irap_au?(au), do: Enum.any?(au, &(&1.type in NALuTypes.irap_nalus()))
 
   @spec wrap_into_buffer(
           AUSplitter.access_unit(),

lib/membrane_h265_plugin/parser/nalu_parser.ex

@@ -71,30 +71,25 @@ defmodule Membrane.H265.Parser.NALuParser do
 
     type = NALuTypes.get_type(parsed_fields.nal_unit_type)
 
-    {nalu, scheme_parser_state} =
-      try do
-        {parsed_fields, scheme_parser_state} =
-          parse_proper_nalu_type(nalu_body, scheme_parser_state, type)
-
-        {%NALu{
-           parsed_fields: parsed_fields,
-           type: type,
-           status: :valid,
-           stripped_prefix: prefix,
-           payload: unprefixed_nalu_payload,
-           timestamps: timestamps
-         }, scheme_parser_state}
-      catch
-        "Cannot load information from SPS" ->
-          {%NALu{
-             parsed_fields: parsed_fields,
-             type: type,
-             status: :error,
-             stripped_prefix: prefix,
-             payload: unprefixed_nalu_payload,
-             timestamps: timestamps
-           }, scheme_parser_state}
-      end
+    {parsed_fields, scheme_parser_state} =
+      parse_proper_nalu_type(nalu_body, scheme_parser_state, type)
+
+    # Mark nalu as invalid if there's no parameter sets
+    nalu_status =
+      if type in NALuTypes.vcl_nalu_types() and
+           not Map.has_key?(parsed_fields, :separate_colour_plane_flag),
+         do: :error,
+         else: :valid
+
+    nalu =
+      %NALu{
+        parsed_fields: parsed_fields,
+        type: type,
+        status: nalu_status,
+        stripped_prefix: prefix,
+        payload: unprefixed_nalu_payload,
+        timestamps: timestamps
+      }
 
     state = %{state | scheme_parser_state: scheme_parser_state}
 

lib/membrane_h265_plugin/parser/nalu_parser/schemes/slice.ex

@@ -37,7 +37,7 @@ defmodule Membrane.H265.Parser.NALuParser.Schemes.Slice do
 
       {payload, state}
     else
-      _error -> throw("Cannot load information from SPS")
+      _error -> {payload, state}
     end
   end
 end

test/parser/au_splitter_test.exs

@@ -7,7 +7,7 @@ defmodule AUSplitterTest do
 
   # These values were obtained with the use of FFmpeg
   @au_lengths_ffmpeg %{
-    "10-1920x1080" => [133_766, 1517, 474, 311, 298, 200, 3257, 547, 209, 233],
+    "10-1920x1080" => [10_117, 493, 406, 447, 428, 320, 285, 297, 306, 296],
     "10-480x320-mainstillpicture" => [
       35_114,
       8824,

test/parser/repeat_parameter_sets_test.exs

@@ -0,0 +1,131 @@
+defmodule Membrane.H265.RepeatParameterSetsTest do
+  @moduledoc false
+
+  use ExUnit.Case, async: true
+
+  import Membrane.ChildrenSpec
+  import Membrane.Testing.Assertions
+  import Membrane.H265.Support.Common
+
+  alias Membrane.H265
+  alias Membrane.H265.Parser.NALuSplitter
+  alias Membrane.Testing.{Pipeline, Sink}
+
+  @in_path "../fixtures/input-60-640x480-no-parameter-sets.h265" |> Path.expand(__DIR__)
+  @ref_path "../fixtures/reference-60-640x480-with-parameter-sets.h265" |> Path.expand(__DIR__)
+
+  @vps <<64, 1, 12, 1, 255, 255, 33, 96, 0, 0, 3, 0, 144, 0, 0, 3, 0, 0, 3, 0, 153, 149, 152, 9>>
+  @sps <<66, 1, 1, 33, 96, 0, 0, 3, 0, 144, 0, 0, 3, 0, 0, 3, 0, 153, 160, 5, 2, 1, 225, 101, 149,
+         154, 73, 50, 188, 57, 160, 32, 0, 0, 3, 0, 32, 0, 0, 3, 3, 193>>
+  @pps <<68, 1, 193, 114, 180, 98, 64>>
+  @dcr <<1, 33, 96, 0, 0, 0, 144, 0, 0, 0, 0, 0, 153, 240, 0, 252, 253, 248, 248, 0, 0, 15, 3,
+         160, 0, 1, 0, 24, 64, 1, 12, 1, 255, 255, 33, 96, 0, 0, 3, 0, 144, 0, 0, 3, 0, 0, 3, 0,
+         153, 149, 152, 9, 161, 0, 1, 0, 42, 66, 1, 1, 33, 96, 0, 0, 3, 0, 144, 0, 0, 3, 0, 0, 3,
+         0, 153, 160, 5, 2, 1, 225, 101, 149, 154, 73, 50, 188, 57, 160, 32, 0, 0, 3, 0, 32, 0, 0,
+         3, 3, 193, 162, 0, 1, 0, 7, 68, 1, 193, 114, 180, 98, 64>>
+
+  defp make_pipeline(
+         source,
+         vpss \\ [],
+         spss \\ [],
+         ppss \\ [],
+         output_stream_structure \\ :annexb
+       ) do
+    structure =
+      child(:source, source)
+      |> child(:parser, %H265.Parser{
+        vpss: vpss,
+        spss: spss,
+        ppss: ppss,
+        repeat_parameter_sets: true,
+        output_stream_structure: output_stream_structure
+      })
+      |> child(:sink, Sink)
+
+    Pipeline.start_link_supervised!(structure: structure)
+  end
+
+  defp perform_test(
+         pipeline_pid,
+         data,
+         mode \\ :bytestream,
+         parser_input_stream_structure \\ :annexb,
+         parser_output_stream_structure \\ :annexb
+       ) do
+    buffers = prepare_buffers(data, mode, parser_input_stream_structure)
+
+    assert_pipeline_play(pipeline_pid)
+    actions = for buffer <- buffers, do: {:buffer, {:output, buffer}}
+    Pipeline.message_child(pipeline_pid, :source, actions ++ [end_of_stream: :output])
+
+    output_buffers =
+      prepare_buffers(
+        File.read!(@ref_path),
+        :au_aligned,
+        parser_output_stream_structure
+      )
+
+    Enum.each(output_buffers, fn output_buffer ->
+      assert_sink_buffer(pipeline_pid, :sink, buffer)
+      assert buffer.payload == output_buffer.payload
+    end)
+
+    assert_end_of_stream(pipeline_pid, :sink, :input, 3_000)
+    Pipeline.terminate(pipeline_pid, blocking?: true)
+  end
+
+  defp split_access_unit(access_unit) do
+    {nalus, _splitter} = NALuSplitter.split(access_unit, true, NALuSplitter.new())
+    Enum.sort(nalus)
+  end
+
+  describe "Parameter sets should be reapeated on each IRAP access unit" do
+    test "when provided by parser options" do
+      source = %H265.Support.TestSource{mode: :bytestream}
+      pid = make_pipeline(source, [@vps], [@sps], [@pps])
+      perform_test(pid, File.read!(@in_path))
+    end
+
+    test "when retrieved from the bytestream" do
+      source = %H265.Support.TestSource{mode: :bytestream}
+      pid = make_pipeline(source)
+
+      data = Enum.join([<<>>, @vps, @sps, @pps], <<0, 0, 0, 1>>) <> File.read!(@in_path)
+      perform_test(pid, data)
+    end
+
+    test "when provided via DCR" do
+      source = %H265.Support.TestSource{
+        mode: :au_aligned,
+        output_raw_stream_structure: {:hev1, @dcr}
+      }
+
+      pid = make_pipeline(source, [], [], [], {:hev1, 4})
+      perform_test(pid, File.read!(@in_path), :au_aligned, {:hev1, 4}, {:hev1, 4})
+    end
+
+    test "when bytestream has variable parameter sets" do
+      in_path = "./test/fixtures/input-60-640x480-variable-parameters.h265"
+      ref_path = "./test/fixtures/reference-60-640x480-variable-parameters.h265"
+
+      source = %H265.Support.TestSource{mode: :bytestream}
+      pid = make_pipeline(source)
+
+      buffers = prepare_buffers(File.read!(in_path), :bytestream)
+
+      assert_pipeline_play(pid)
+      actions = for buffer <- buffers, do: {:buffer, {:output, buffer}}
+      Pipeline.message_child(pid, :source, actions ++ [end_of_stream: :output])
+
+      File.read!(ref_path)
+      |> prepare_buffers(:au_aligned)
+      |> Enum.each(fn output_buffer ->
+        assert_sink_buffer(pid, :sink, buffer)
+        assert split_access_unit(output_buffer.payload) == split_access_unit(buffer.payload)
+      end)
+
+      assert_end_of_stream(pid, :sink, :input, 3_000)
+      Pipeline.terminate(pid, blocking?: true)
+    end
+  end
+end