MPEG TS: workaround for subtitle timestamps differing from audio/video timestamps widely

There are MPEG TS files where subtitle packets are multiplexed with
audio and video packets properly, meaning that packets that are supposed
to be shown/played together are also stored next to each other. However,
the timestamps in the PES streams have huge differences. For example,
the first timestamps for audio and video packets are around 00:11:08.418
whereas the timestamps for corresponding subtitle packets start at
09:19:25.912.

This workaround attempts to detect such situations. In that case
mkvmerge will discard subtitle timestamps and use the most recent audio
or video timestamp instead.

Implements #1841.

NEWS.md

@@ -2,6 +2,9 @@
 
 ## New features and enhancements
 
+* mkvmerge: MPEG TS: added a workaround for files where the subtitle packets
+  are mulitplexed properly, but where their timestamps are way off from the
+  audio and video timestamps. Implements #1841.
 * mkvmerge: added support for Digital Video Broadcasting (DVB) subtitles
   (CodecID `S_DVBSUB`). They can be read from MPEG transport streams and from
   Matroska files. Implements #1843.

src/input/r_mpeg_ts.cpp

@@ -134,16 +134,6 @@ track_c::send_to_packetizer() {
     timestamp_to_use     -= std::max(f.m_global_timestamp_offset, min.valid() ? min : timestamp_c::ns(0));
   }
 
-  if (timestamp_to_use.valid()) {
-    if (mtx::included_in(type, pid_type_e::video, pid_type_e::audio))
-      f.m_last_non_subtitle_timestamp = timestamp_to_use;
-
-    else if (   (type == pid_type_e::subtitles)
-             && f.m_last_non_subtitle_timestamp.valid()
-             && ((timestamp_to_use - f.m_last_non_subtitle_timestamp).abs() >= timestamp_c::s(5)))
-      timestamp_to_use = f.m_last_non_subtitle_timestamp;
-  }
-
   mxdebug_if(m_debug_delivery, boost::format("send_to_packetizer: PID %1% expected %2% actual %3% timestamp_to_use %4% m_previous_timestamp %5%\n")
              % pid % pes_payload_size_to_read % pes_payload_read->get_size() % timestamp_to_use % m_previous_timestamp);
 
@@ -588,6 +578,41 @@ track_c::handle_timestamp_wrap(timestamp_c &pts,
   // mxinfo(boost::format("pid %5% PTS before %1% now %2% wrapped %3% reset prevvalid %4% diff %6%\n") % before % pts % m_timestamps_wrapped % m_previous_valid_timestamp % pid % (pts - m_previous_valid_timestamp));
 }
 
+drop_decision_e
+track_c::handle_bogus_subtitle_timestamps(timestamp_c &pts,
+                                          timestamp_c &dts) {
+  auto &f = reader.file();
+
+  if (processing_state_e::muxing != f.m_state)
+    return drop_decision_e::keep;
+
+  // Sometimes the subtitle timestamps aren't valid, they can be off
+  // from the audio/video timestamps by hours. In such a case replace
+  // the subtitle's timestamps with last valid one from audio/video
+  // packets.
+  if (   mtx::included_in(type, pid_type_e::audio, pid_type_e::video)
+      && pts.valid()) {
+    f.m_last_non_subtitle_pts = pts;
+    f.m_last_non_subtitle_dts = dts;
+    return drop_decision_e::keep;
+
+  }
+
+  if (pid_type_e::subtitles != type)
+    return drop_decision_e::keep;
+
+  if (!f.m_last_non_subtitle_pts.valid())
+    return f.m_has_audio_or_video_track ? drop_decision_e::drop : drop_decision_e::keep;
+
+  if (   !pts.valid()
+      || ((pts - f.m_last_non_subtitle_pts).abs() >= timestamp_c::s(5))) {
+    pts = f.m_last_non_subtitle_pts;
+    dts = f.m_last_non_subtitle_dts;
+  }
+
+  return drop_decision_e::keep;
+}
+
 bool
 track_c::parse_ac3_pmt_descriptor(pmt_descriptor_t const &,
                                   pmt_pid_info_t const &pmt_pid_info) {
@@ -782,6 +807,18 @@ track_c::derive_hdmv_textst_pts_from_content() {
   return timestamp;
 }
 
+void
+track_c::reset_processing_state() {
+  m_timestamp.reset();
+  m_previous_timestamp.reset();
+  m_previous_valid_timestamp.reset();
+
+  clear_pes_payload();
+  processed            = false;
+  m_timestamps_wrapped = false;
+  m_timestamp_wrap_add = timestamp_c::ns(0);
+}
+
 // ------------------------------------------------------------
 
 file_t::file_t(mm_io_cptr const &in)
@@ -789,7 +826,6 @@ file_t::file_t(mm_io_cptr const &in)
   , m_pat_found{}
   , m_pmt_found{}
   , m_es_to_process{}
-  , m_global_timestamp_offset{}
   , m_stream_timestamp{timestamp_c::ns(0)}
   , m_timestamp_mpls_sync{timestamp_c::ns(0)}
   , m_state{processing_state_e::probing}
@@ -801,6 +837,7 @@ file_t::file_t(mm_io_cptr const &in)
   , m_num_pmt_crc_errors{}
   , m_validate_pat_crc{true}
   , m_validate_pmt_crc{true}
+  , m_has_audio_or_video_track{}
 {
 }
 
@@ -815,6 +852,13 @@ file_t::get_queued_bytes()
   return bytes;
 }
 
+void
+file_t::reset_processing_state(processing_state_e new_state) {
+  m_state = new_state;
+  m_last_non_subtitle_pts.reset();
+  m_last_non_subtitle_dts.reset();
+}
+
 // ------------------------------------------------------------
 
 bool
@@ -1013,10 +1057,20 @@ reader_c::read_headers() {
   show_demuxer_info();
 }
 
+void
+reader_c::reset_processing_state(processing_state_e new_state) {
+  for (auto const &file : m_files)
+    file->reset_processing_state(new_state);
+
+  for (auto const &track : m_tracks)
+    track->reset_processing_state();
+}
+
 void
 reader_c::determine_global_timestamp_offset() {
-  auto &f   = file();
-  f.m_state = processing_state_e::determining_timestamp_offset;
+  reset_processing_state(processing_state_e::determining_timestamp_offset);
+
+  auto &f = file();
 
   f.m_in->setFilePointer(0);
   f.m_in->clear_eof();
@@ -1047,12 +1101,7 @@ reader_c::determine_global_timestamp_offset() {
   f.m_in->setFilePointer(0);
   f.m_in->clear_eof();
 
-  for (auto const &track : m_tracks) {
-    track->clear_pes_payload();
-    track->processed            = false;
-    track->m_timestamps_wrapped = false;
-    track->m_timestamp_wrap_add = timestamp_c::ns(0);
-  }
+  reset_processing_state(processing_state_e::muxing);
 }
 
 void
@@ -1522,6 +1571,10 @@ reader_c::parse_pes(track_c &track) {
   auto orig_pts = pts;
   auto orig_dts = dts;
 
+  auto result = track.handle_bogus_subtitle_timestamps(pts, dts);
+
+  if (drop_decision_e::drop == result)
+    return;
 
   track.handle_timestamp_wrap(pts, dts);
 
@@ -1744,6 +1797,10 @@ reader_c::probe_packet_complete(track_c &track) {
     track.processed = true;
     track.probed_ok = true;
     --f.m_es_to_process;
+
+    if (mtx::included_in(track.type, pid_type_e::audio, pid_type_e::video))
+      f.m_has_audio_or_video_track = true;
+
     mxdebug_if(m_debug_headers, boost::format("probe_packet_complete: ES to process: %1%\n") % f.m_es_to_process);
   }
 }
@@ -1913,9 +1970,6 @@ void
 reader_c::create_packetizers() {
   determine_global_timestamp_offset();
 
-  for (auto const &file : m_files)
-    file->m_state = processing_state_e::muxing;
-
   mxdebug_if(m_debug_headers, boost::format("create_packetizers: create packetizers...\n"));
   for (std::size_t i = 0u, end = m_tracks.size(); i < end; ++i)
     create_packetizer(i);
@@ -2121,13 +2175,21 @@ reader_c::find_track_for_pid(uint16_t pid)
   const {
   auto &f = *m_files[m_current_file];
 
-  for (auto const &track : m_tracks)
-    if (   (track->m_file_num == m_current_file)
-        && (track->pid        == pid)
-        && (   mtx::included_in(f.m_state, processing_state_e::probing, processing_state_e::determining_timestamp_offset)
-            || track->has_packetizer()))
+  for (auto const &track : m_tracks) {
+    if (   (track->m_file_num != m_current_file)
+        || (track->pid        != pid))
+      continue;
+
+    if (track->has_packetizer() || mtx::included_in(f.m_state, processing_state_e::probing, processing_state_e::determining_timestamp_offset))
       return track;
 
+    for (auto const &coupled_track : track->m_coupled_tracks)
+      if (coupled_track->has_packetizer())
+        return coupled_track;
+
+    return track;
+  }
+
   return {};
 }
 

src/input/r_mpeg_ts.h

@@ -82,6 +82,11 @@ enum class stream_type_e : unsigned char {
   stream_subtitles_hdmv_textst = 0x92, // HDMV TextST subtitles
 };
 
+enum class drop_decision_e {
+  keep,
+  drop,
+};
+
 #if defined(COMP_MSC)
 #pragma pack(push,1)
 #endif
@@ -352,6 +357,7 @@ public:
 
   void set_pid(uint16_t new_pid);
 
+  drop_decision_e handle_bogus_subtitle_timestamps(timestamp_c &pts, timestamp_c &dts);
   void handle_timestamp_wrap(timestamp_c &pts, timestamp_c &dts);
   bool detect_timestamp_wrap(timestamp_c const &timestamp) const;
   void adjust_timestamp_for_wrap(timestamp_c &timestamp);
@@ -362,6 +368,8 @@ public:
   void process(packet_cptr const &packet);
 
   void parse_iso639_language_from(void const *buffer);
+
+  void reset_processing_state();
 };
 
 struct file_t {
@@ -372,7 +380,7 @@ struct file_t {
 
   bool m_pat_found, m_pmt_found;
   int m_es_to_process;
-  timestamp_c m_global_timestamp_offset, m_stream_timestamp, m_last_non_subtitle_timestamp, m_timestamp_restriction_min, m_timestamp_restriction_max, m_timestamp_mpls_sync;
+  timestamp_c m_global_timestamp_offset, m_stream_timestamp, m_timestamp_restriction_min, m_timestamp_restriction_max, m_timestamp_mpls_sync, m_last_non_subtitle_pts, m_last_non_subtitle_dts;
 
   processing_state_e m_state;
   uint64_t m_probe_range;
@@ -380,10 +388,12 @@ struct file_t {
   bool m_file_done, m_packet_sent_to_packetizer;
 
   unsigned int m_detected_packet_size, m_num_pat_crc_errors, m_num_pmt_crc_errors;
-  bool m_validate_pat_crc, m_validate_pmt_crc;
+  bool m_validate_pat_crc, m_validate_pmt_crc, m_has_audio_or_video_track;
 
   file_t(mm_io_cptr const &in);
+
   int64_t get_queued_bytes() const;
+  void reset_processing_state(processing_state_e new_state);
 };
 using file_cptr = std::shared_ptr<file_t>;
 
@@ -456,6 +466,7 @@ private:
   void create_srt_subtitles_packetizer(track_ptr const &track);
   void create_dvbsub_subtitles_packetizer(track_ptr const &track);
 
+  void reset_processing_state(processing_state_e new_state);
   void determine_global_timestamp_offset();
 
   bfs::path find_file(bfs::path const &source_file, std::string const &sub_directory, std::string const &extension) const;

tests/results.txt

@@ -242,7 +242,7 @@ T_393aac_audiospecificconfig_0channels:c948233b0562fa997aa426e39b5772d2:passed:2
 T_394flv_negative_cts_offset:c33944c32b31498ddbf25c1354688271:passed:20130414-115331:0.123013461
 T_395remove_bitstream_ar_info:12d0f2056e9b79cafec12352c94ac846-3c2ba23992cba3441396e25b2740cf9f-4c97e5c442155fe54cb90eb10b440211-49f413b658608e0ea01623801a717dc7:passed:20130427-171243:0.334408574
 T_396X_pcm_mono_16bit:49b80add61d8a11514d747c9616f7e80:passed:20130624-200214:0.055106028
-T_397mpeg_ts_broken_pes_track_detection:59282e6da966f4ca1bbe14a0ccc2c587:passed:20130624-220549:1.12831027
+T_397mpeg_ts_broken_pes_track_detection:3d6512e4d604a85c1ad4c519279bdeeb:passed:20130624-220549:1.12831027
 T_398flv1_no_pixel_dimensions:870014b911f1fcc626a7921525b4c185:passed:20130624-225648:0.103188723
 T_399h264_append_and_default_duration:d18de651ba99423741592b203011bcd3:passed:20130627-195946:5.558958754
 T_400opus_experimental:d74c651a826ccde9053ab088032d5870:passed:20130703-213929:0.055921025
@@ -420,3 +420,4 @@ T_571bluray_pcm_odd_number_of_channels:77cf4c999faf85d0e37c790ee7c75541:passed:2
 T_572hdmv_textst:0c7491296aca98abf20af85926468bac-0c7491296aca98abf20af85926468bac-0f8722f155c1c32f4b58ef8661fea6a1+2d080bee45d9cf8fdd1f235f56550863:passed:20161128-164101:0.072007646
 T_573teletext_subs_data_unit_id_0x02_and_multiple_magazines:03b961cb80532c45829bd91345f327d0:passed:20161208-202941:0.037765249
 T_574tta_with_apev2_tags:342b123b71e0a40a0688f4647c38b1c5-4870070dccb00f5aaeda933c301050f5:passed:20161223-210853:0.019411679
+T_575mpeg_ts_subtitle_timestamps_way_off_from_video_timestamps:78a0ab3566507be5684cde3fa601bb06:passed:20170106-200606:0.508113744

tests/test-575mpeg_ts_subtitle_timestamps_way_off_from_video_timestamps.rb

@@ -0,0 +1,5 @@
+#!/usr/bin/ruby -w
+
+# T_575mpeg_ts_subtitle_timestamps_way_off_from_video_timestamps
+describe "mkvmerge / MPEG TS: subtitles multiplexed properly, but timestamps way off from audio & video timestamps"
+test_merge "data/ts/teletext_subs_timestamps_way_off_from_video_timestamps.ts"