MPEG TS: parse all PMTs in streams with multiple programs properly

Earlier versions of mkvmerge used to detect all tracks in MPEG
transport streams with multiple programs, even though the code wasn't
really implemented & tested for that. However, some tracks (usually
those from the second or a later program) were broken: they might not
contain any data, or only invalid data.

On top of that mkvmerge v12.0.0 contains a fix for #1980 where a track
isn't part of a PMT at all. An unintentional consequence of that fix
was that mkvmerge no longer detected all of the tracks in
multi-program streams. The reason is that in order to detect tracks
not mentioned in a PMT mkvmerge has to do detection by content in the
PES packets. That's only implemented for AAC at the moment. All other
tracks will be blacklisted as soon as they're found.

This wouldn't be a problem if all PMTs of all programs were always
located right at the start of the file with nothing in
between. Unfortunately many files contain track content between
PMTs. So that workflow was:

• mkvmerge finds first PMT, determines types for tracks listed in it
• mkvmerge now considers the PMT to be found
• Continuing scanning the file mkvmerge encounters content for tracks
  not listed in the first PMT, attempting type detection by content,
  failing for most and blacklisting their PIDs
• Next a second PMT is found, however, the PIDs listed in that PMT may
  have already been found and blacklisted before — therefor they won't
  be considered anymore

With this fix mkvmerge actively looks for the PMTs for all
programs. Detection by content is only attempted once all PMTs have
been located. That way all tracks will be detected again.

A side effect of either this patch or one of the other ones before is
that the track content is now OK. I don't know exactly why or which
commit actually fixed it.

Fixes parts of #1990.

NEWS.md

@@ -1,3 +1,14 @@
+# Version ?
+
+## Bug fixes
+
+* mkvmerge: MPEG TS reader: fixed mkvmerge not detecting all tracks in MPEG
+  transport streams containing multiple programs. Fixes one part of #1990.
+* mkvmerge: MPEG TS reader: fixed track content being broken for some tracks
+  read from MPEG transport streams containing multiple programs. Fixes another
+  part of #1990.
+
+
 # Version 12.0.0 "Trust / Lust" 2017-05-20
 
 ## New features and enhancements

src/input/r_mpeg_ts.cpp

@@ -928,7 +928,8 @@ track_c::reset_processing_state() {
 file_t::file_t(mm_io_cptr const &in)
   : m_in{in}
   , m_pat_found{}
-  , m_pmt_found{}
+  , m_num_pmts_found{}
+  , m_num_pmts_to_find{}
   , m_es_to_process{}
   , m_stream_timestamp{timestamp_c::ns(0)}
   , m_timestamp_mpls_sync{timestamp_c::ns(0)}
@@ -963,6 +964,12 @@ file_t::reset_processing_state(processing_state_e new_state) {
   m_last_non_subtitle_dts.reset();
 }
 
+bool
+file_t::all_pmts_found()
+  const {
+  return (0 != m_num_pmts_to_find) && (m_num_pmts_found >= m_num_pmts_to_find);
+}
+
 // ------------------------------------------------------------
 
 bool
@@ -1085,7 +1092,7 @@ reader_c::read_headers_for_file(std::size_t file_num) {
       parse_packet(buf);
 
       if (   f.m_pat_found
-          && f.m_pmt_found
+          && f.all_pmts_found()
           && (0 == f.m_es_to_process)
           && (f.m_in->getFilePointer() >= min_size_to_probe))
         break;
@@ -1099,13 +1106,13 @@ reader_c::read_headers_for_file(std::size_t file_num) {
       // we should read from the start again, this time ignoring the
       // errors for the specific type.
       mxdebug_if(m_debug_headers,
-                 boost::format("read_headers: EOF during detection. #tracks %1% #PAT CRC errors %2% #PMT CRC errors %3% PAT found %4% PMT found %5%\n")
-                 % m_tracks.size() % f.m_num_pat_crc_errors % f.m_num_pmt_crc_errors % f.m_pat_found % f.m_pmt_found);
+                 boost::format("read_headers: EOF during detection. #tracks %1% #PAT CRC errors %2% #PMT CRC errors %3% PAT found %4% PMT found %5%/%6%\n")
+                 % m_tracks.size() % f.m_num_pat_crc_errors % f.m_num_pmt_crc_errors % f.m_pat_found % f.m_num_pmts_found % f.m_num_pmts_to_find);
 
       if (!f.m_pat_found && f.m_validate_pat_crc)
         f.m_validate_pat_crc = false;
 
-      else if (f.m_pat_found && !f.m_pmt_found && f.m_validate_pmt_crc) {
+      else if (f.m_pat_found && !f.all_pmts_found() && f.m_validate_pmt_crc) {
         f.m_validate_pmt_crc = false;
         f.m_pmt_pid_seen.clear();
 
@@ -1398,12 +1405,15 @@ reader_c::parse_pat(track_c &track) {
     f.m_ignored_pids[tmp_pid] = true;
     f.m_pmt_pid_seen[tmp_pid] = true;
     f.m_pat_found             = true;
+    ++f.m_num_pmts_to_find;
 
     pmt->set_pid(tmp_pid);
 
     m_tracks.push_back(pmt);
   }
 
+  mxdebug_if(m_debug_pat_pmt, boost::format("parse_pat: number of PMTs to find: %1%\n") % f.m_num_pmts_to_find);
+
   return true;
 }
 
@@ -1481,7 +1491,6 @@ reader_c::parse_pmt_pid_info(mm_mem_io_c &mem,
   }
 
   if (track->type != pid_type_e::unknown) {
-    f.m_pmt_found    = true;
     track->processed = false;
     m_tracks.push_back(track);
     ++f.m_es_to_process;
@@ -1577,6 +1586,15 @@ reader_c::parse_pmt(track_c &track) {
 
   f.m_ignored_pids[track.pid] = true;
 
+  ++f.m_num_pmts_found;
+
+  mxdebug_if(m_debug_pat_pmt,
+             boost::format("parse_pmt: %1% num PMTs found %2% vs. to find %3%\n")
+             % (  f.m_num_pmts_found  < f.m_num_pmts_to_find ? "find_ongoing"
+                : f.m_num_pmts_found == f.m_num_pmts_to_find ? "find_done"
+                :                                              "find_error")
+             % f.m_num_pmts_found % f.m_num_pmts_to_find);
+
   return true;
 }
 
@@ -1703,7 +1721,7 @@ reader_c::handle_packet_for_pid_not_listed_in_pmt(uint16_t pid) {
   if (   (f.m_state != processing_state_e::probing)
       || f.m_ignored_pids[pid]
       || !f.m_pat_found
-      || !f.m_pmt_found)
+      || !f.all_pmts_found())
     return {};
 
   mxdebug_if(m_debug_pat_pmt, boost::format("found packet for track PID %1% not listed in PMT, attempting type detection by content\n") % pid);

src/input/r_mpeg_ts.h

@@ -383,7 +383,8 @@ struct file_t {
   std::unordered_map<uint16_t, bool> m_ignored_pids, m_pmt_pid_seen;
   std::vector<generic_packetizer_c *> m_packetizers;
 
-  bool m_pat_found, m_pmt_found;
+  bool m_pat_found;
+  unsigned int m_num_pmts_found, m_num_pmts_to_find;
   int m_es_to_process;
   timestamp_c m_global_timestamp_offset, m_stream_timestamp, m_timestamp_restriction_min, m_timestamp_restriction_max, m_timestamp_mpls_sync, m_last_non_subtitle_pts, m_last_non_subtitle_dts;
 
@@ -399,6 +400,7 @@ struct file_t {
 
   int64_t get_queued_bytes() const;
   void reset_processing_state(processing_state_e new_state);
+  bool all_pmts_found() const;
 };
 using file_cptr = std::shared_ptr<file_t>;
 

tests/results.txt

@@ -445,3 +445,4 @@ T_596mpeg_ts_aac_loas_latm_misdetected_as_adts:5ae490a6b1e59d5382aa887355b7d96e:
 T_597mpeg_ts_starting_with_avc_start_code:88846632aaba92f49f072ab0423e5bdc:passed:20170510-145746:0.077795359
 T_598aac_track_not_listed_in_pmt:444929dd4db38e68b59a3ebf833e5128-AAC:passed:20170511-221910:0.0849745
 T_599mp4_nclx_colour_type_in_colr_atom:3639a6fdf7a0e46d158188fdd932bd2b:passed:20170514-203828:0.018287634
+T_600mpeg_ts_multiple_programs:890b456227714da673b137a941bf45b2-116bf940510e195d9ec3b99eb4a0625d-d78702c82db3e49891717626ad0fb9fb-abbe7e3bb753fb6153e122e771b02d0c:passed:20170522-193901:1.342170107

tests/test-600mpeg_ts_multiple_programs.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/ruby -w
+
+files = (1..2).map { |n| "data/ts/multiple_programs_#{n}.ts" }
+
+# T_600mpeg_ts_multiple_programs
+describe "mkvmerge / MPEG transport streams with multiple programs"
+
+files.each do |file|
+  test_merge file
+  test_identify file
+end