h.264 parser: prepend new SPS/PPS NALUs found mid-stream to key frames, too

This was already done when SPS or PPS NALUs were found with the same
ID as active SPS or PPS but with different content. The missing case
were SPS or PPS NALUs whose IDs hadn't been seen before.

After such an encounter all following key frames must be prepended
with the active SPS & PPS, too. Otherwise seeking would not work as
the player only used the SPS & PPS from CodecPrivate but didn't know
about the new ones from mid-stream.

Part of the fix of #2250.

NEWS.md

@@ -5,11 +5,15 @@
 * mkvmerge: AVC/h.264 parser: the order of the NALUs before each key frame was
   sometimes wrong: mkvmerge wrote SPS & PPS after SEI NALUs. Now SPS & PPS
   NALUs are always written before the other NALUs. Patch by Torsten
-  Hauska. Part of the implementation of #2709.
+  Hauska. Part of the implementation of #2709 and part of the fix of #2250.
 * MKVToolNix GUI: multiplexer: dragging & dropping XML files with chapters or
   tags to the GUI's window was broken in v42. Instead of adding the file names
   to the appropriate input boxes the GUI was running mkvmerge for file
   identification purposes which then failed. Fixes #2718.
+* mkvmerge: AVC/h.264 parser: when additional SPS or PPS NALUs (with IDs that
+  haven't been seen so far) are found mid-stream, mkvmerge will prepend all
+  following key frames with all current valid SPS & PPS NALUs (just like when
+  SPS & PPS NALUs are overwritten mid-stream). Part of the fix of #2250.
 
 
 # Version 42.0.0 "Overtime" 2020-01-02

src/common/avc_es_parser.cpp

@@ -344,7 +344,7 @@ es_parser_c::handle_slice_nalu(memory_cptr const &nalu,
     if (!si.field_pic_flag || !m_current_key_frame_bottom_field || (*m_current_key_frame_bottom_field == si.bottom_field_flag)) {
       cleanup();
 
-      if (m_sps_or_sps_overwritten)
+      if (m_avcc_changed)
         add_sps_and_pps_to_extra_data();
     }
 
@@ -385,17 +385,20 @@ es_parser_c::handle_sps_nalu(memory_cptr const &nalu) {
   if (m_sps_info_list.size() == i) {
     m_sps_list.push_back(parsed_nalu);
     m_sps_info_list.push_back(sps_info);
-    m_avcc_changed = true;
+
+    if (m_avcc_ready)
+      m_avcc_changed = true;
 
   } else if (m_sps_info_list[i].checksum != sps_info.checksum) {
     mxdebug_if(m_debug_sps_pps_changes, fmt::format("mpeg4::p10: SPS ID {0:04x} changed; checksum old {1:04x} new {2:04x}\n", sps_info.id, m_sps_info_list[i].checksum, sps_info.checksum));
 
     cleanup();
 
-    m_sps_info_list[i]       = sps_info;
+    m_sps_info_list[i] = sps_info;
-    m_sps_list[i]            = parsed_nalu;
+    m_sps_list[i]      = parsed_nalu;
-    m_avcc_changed           = true;
+
-    m_sps_or_sps_overwritten = true;
+    if (m_avcc_ready)
+      m_avcc_changed = true;
 
   } else
     use_sps_info = false;
@@ -437,7 +440,9 @@ es_parser_c::handle_pps_nalu(memory_cptr const &nalu) {
   if (m_pps_info_list.size() == i) {
     m_pps_list.push_back(nalu);
     m_pps_info_list.push_back(pps_info);
-    m_avcc_changed = true;
+
+    if (m_avcc_ready)
+      m_avcc_changed = true;
 
   } else if (m_pps_info_list[i].checksum != pps_info.checksum) {
     mxdebug_if(m_debug_sps_pps_changes, fmt::format("mpeg4::p10: PPS ID {0:04x} changed; checksum old {1:04x} new {2:04x}\n", pps_info.id, m_pps_info_list[i].checksum, pps_info.checksum));
@@ -445,10 +450,11 @@ es_parser_c::handle_pps_nalu(memory_cptr const &nalu) {
     if (m_pps_info_list[i].sps_id != pps_info.sps_id)
       cleanup();
 
-    m_pps_info_list[i]       = pps_info;
+    m_pps_info_list[i] = pps_info;
-    m_pps_list[i]            = nalu;
+    m_pps_list[i]      = nalu;
-    m_avcc_changed           = true;
+
-    m_sps_or_sps_overwritten = true;
+    if (m_avcc_ready)
+      m_avcc_changed = true;
   }
 
   m_extra_data.push_back(create_nalu_with_size(nalu));

src/common/avc_es_parser.h

@@ -77,7 +77,7 @@ protected:
   int m_nalu_size_length;
 
   bool m_keep_ar_info, m_fix_bitstream_frame_rate;
-  bool m_avcc_ready, m_avcc_changed, m_sps_or_sps_overwritten{};
+  bool m_avcc_ready, m_avcc_changed;
 
   int64_t m_stream_default_duration, m_forced_default_duration, m_container_default_duration;
   int m_frame_number, m_num_skipped_frames;

tests/results.txt

@@ -149,7 +149,7 @@ T_300ts_dts_duplicate_timestamps:4b57b0976c7658478afe86198b7833c5:passed:2011091
 T_301ts_pgssub:bd9e003c32c1d7138252953f535e781d:passed:20110918-154732:1.701870051
 T_302pat_pmt_only_once:659aa04d44404941846a381f8daf1bda:passed:20110927-222121:0.300545156
 T_303mpeg_ts_eac3_pmt_descriptor_tag_0x7a:0_video_MPEG_4p10_AVC_H.264_220_1920x1080/1_audio_E_AC_3_qaa_231_6_48000/2_audio_E_AC_3_fre_230_2_48000/3_subtitles_DVBSUB_fre_240:passed:20111008-150823:0.033890898
-T_304eac3_pes_private_but_no_pmt_descriptor_tag:d3cb1cec039661f4c7469c1b84a07eb7:passed:20111009-113137:2.713279639
+T_304eac3_pes_private_but_no_pmt_descriptor_tag:08334303b95cb926e9f57fa483fb2e3f:passed:20111009-113137:2.713279639
 T_305ui_locale_en_US:23026ac2ed9767541e89f2261bcf8b60-0136434e2e22ebcc806ee944cdd2b853:passed:20111016-192531:0.067743615
 T_306ui_locale_de_DE:5c2281373b0f5d9d7145cca6b2391db7-fe29d5dd8da942a9deb4aac92b2f0514:passed:20111016-192531:0.06032807
 T_307ui_locale_es_ES:3f230a5da62a4650a8874bde6cb3c0e7-343c6efff52830d0484900df270904fb:passed:20111016-192531:0.061216592
@@ -450,7 +450,7 @@ T_601mp4_mpeg2_via_esds:e0c34382eebed4b260d992ec550a67e9:passed:20170618-150319:
 T_602vob_with_garbage_at_start:8af88c4ad2e460c323e5e58207b55cf8:passed:20170619-185256:0.236132323
 T_603mpeg_ps_ac3_not_enough_data_in_first_packet:1+189+128+AC_3-2+189+129+AC_3-3+189+130+AC_3-4+189+131+AC_3-5+189+132+AC_3-6+189+133+AC_3-7+189+134+AC_3:passed:20170624-092201:0.028142136
 T_604append_only_one_video_track_with_codec_private:0de389022964c738d7d5b44ae0963844:passed:20170624-105837:0.144927484
-T_605h264_changing_sps_pps_wrong_frame_order_and_timestamps:59090953698c6a80a1ddc06c5af0a29a:passed:20170704-211727:0.912982996
+T_605h264_changing_sps_pps_wrong_frame_order_and_timestamps:ad034028fe25e5b06ccde66150078f6a:passed:20170704-211727:0.912982996
 T_606aac_960_samples_per_frame:492a1baf607138ab3bfd82d2773da4b6-2975773d480040ecb0662b42f3aad56b:passed:20170720-215449:0.065007779
 T_607wave64:e3d6fd4ca2bd4ac1de44759f4a6ea79a-c930ad78a30eff8e8c04ed5f27c6230c:passed:20170721-221321:0.021307011
 T_608ui_locale_ro_RO:3a6386df8e8d7aefb9258f070615b27d-0136434e2e22ebcc806ee944cdd2b853:passed:20170722-160005:0.021529144
@@ -532,3 +532,4 @@ T_683prores_in_matroska_with_icpf_atom:e9b7885ea5f48aedd9b40b2780ba0ad0:passed:2
 T_684deterministic:a2144f0a4d4830798c54a23cf6cf7019+16296316534582144190+6467223910221523705+15003652824273206442+17408887814429376554-a2144f0a4d4830798c54a23cf6cf7019+16296316534582144190+6467223910221523705+15003652824273206442+17408887814429376554-f5282b59ef01ea57fd93e8b64bb1147b+11796936434107936534+2062519427281921607+10692493273940629375+13482037488641761990-f5282b59ef01ea57fd93e8b64bb1147b+11796936434107936534+2062519427281921607+10692493273940629375+13482037488641761990:passed:20191228-125752:0.070415141
 T_685propedit_track_uid_changes:4711+815+4711+815+4711+815:passed:20191229-143919:0.039698154
 T_686container_wrong_num_channels:true:passed:20200101-235713:0.009057097
+T_687h264_additional_sps_pps_in_middle:3c9e0781b9b5e75be5dc3da6d59b8795:passed:20200105-163253:0.029150634

tests/test-687h264_additional_sps_pps_in_middle.rb

@@ -0,0 +1,5 @@
+#!/usr/bin/ruby -w
+
+# T_687h264_additional_sps_pps_in_middle
+describe "mkvmerge / h.264 additional SPS & PPS with so-far unused IDs in the middle"
+test_merge "data/h264/additional_sps_pps_in_middle.h264", :args => "--default-duration 0:25fps"