From 17fd1fd53c89f4f60108297f17b849daa77ddd18 Mon Sep 17 00:00:00 2001 From: Moritz Bunkus Date: Fri, 6 Jan 2017 20:06:22 +0100 Subject: [PATCH] MPEG TS: workaround for subtitle timestamps differing from audio/video timestamps widely There are MPEG TS files where subtitle packets are multiplexed with audio and video packets properly, meaning that packets that are supposed to be shown/played together are also stored next to each other. However, the timestamps in the PES streams have huge differences. For example, the first timestamps for audio and video packets are around 00:11:08.418 whereas the timestamps for corresponding subtitle packets start at 09:19:25.912. This workaround attempts to detect such situations. In that case mkvmerge will discard subtitle timestamps and use the most recent audio or video timestamp instead. Implements #1841. --- NEWS.md | 3 + src/input/r_mpeg_ts.cpp | 116 ++++++++++++++---- src/input/r_mpeg_ts.h | 15 ++- tests/results.txt | 3 +- ...imestamps_way_off_from_video_timestamps.rb | 5 + 5 files changed, 112 insertions(+), 30 deletions(-) create mode 100755 tests/test-575mpeg_ts_subtitle_timestamps_way_off_from_video_timestamps.rb diff --git a/NEWS.md b/NEWS.md index 39d4aea6a..7f6a4494d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,9 @@ ## New features and enhancements +* mkvmerge: MPEG TS: added a workaround for files where the subtitle packets + are mulitplexed properly, but where their timestamps are way off from the + audio and video timestamps. Implements #1841. * mkvmerge: added support for Digital Video Broadcasting (DVB) subtitles (CodecID `S_DVBSUB`). They can be read from MPEG transport streams and from Matroska files. Implements #1843. diff --git a/src/input/r_mpeg_ts.cpp b/src/input/r_mpeg_ts.cpp index 3dc5f32c0..acfce5fb9 100644 --- a/src/input/r_mpeg_ts.cpp +++ b/src/input/r_mpeg_ts.cpp @@ -134,16 +134,6 @@ track_c::send_to_packetizer() { timestamp_to_use -= std::max(f.m_global_timestamp_offset, min.valid() ? min : timestamp_c::ns(0)); } - if (timestamp_to_use.valid()) { - if (mtx::included_in(type, pid_type_e::video, pid_type_e::audio)) - f.m_last_non_subtitle_timestamp = timestamp_to_use; - - else if ( (type == pid_type_e::subtitles) - && f.m_last_non_subtitle_timestamp.valid() - && ((timestamp_to_use - f.m_last_non_subtitle_timestamp).abs() >= timestamp_c::s(5))) - timestamp_to_use = f.m_last_non_subtitle_timestamp; - } - mxdebug_if(m_debug_delivery, boost::format("send_to_packetizer: PID %1% expected %2% actual %3% timestamp_to_use %4% m_previous_timestamp %5%\n") % pid % pes_payload_size_to_read % pes_payload_read->get_size() % timestamp_to_use % m_previous_timestamp); @@ -588,6 +578,41 @@ track_c::handle_timestamp_wrap(timestamp_c &pts, // mxinfo(boost::format("pid %5% PTS before %1% now %2% wrapped %3% reset prevvalid %4% diff %6%\n") % before % pts % m_timestamps_wrapped % m_previous_valid_timestamp % pid % (pts - m_previous_valid_timestamp)); } +drop_decision_e +track_c::handle_bogus_subtitle_timestamps(timestamp_c &pts, + timestamp_c &dts) { + auto &f = reader.file(); + + if (processing_state_e::muxing != f.m_state) + return drop_decision_e::keep; + + // Sometimes the subtitle timestamps aren't valid, they can be off + // from the audio/video timestamps by hours. In such a case replace + // the subtitle's timestamps with last valid one from audio/video + // packets. + if ( mtx::included_in(type, pid_type_e::audio, pid_type_e::video) + && pts.valid()) { + f.m_last_non_subtitle_pts = pts; + f.m_last_non_subtitle_dts = dts; + return drop_decision_e::keep; + + } + + if (pid_type_e::subtitles != type) + return drop_decision_e::keep; + + if (!f.m_last_non_subtitle_pts.valid()) + return f.m_has_audio_or_video_track ? drop_decision_e::drop : drop_decision_e::keep; + + if ( !pts.valid() + || ((pts - f.m_last_non_subtitle_pts).abs() >= timestamp_c::s(5))) { + pts = f.m_last_non_subtitle_pts; + dts = f.m_last_non_subtitle_dts; + } + + return drop_decision_e::keep; +} + bool track_c::parse_ac3_pmt_descriptor(pmt_descriptor_t const &, pmt_pid_info_t const &pmt_pid_info) { @@ -782,6 +807,18 @@ track_c::derive_hdmv_textst_pts_from_content() { return timestamp; } +void +track_c::reset_processing_state() { + m_timestamp.reset(); + m_previous_timestamp.reset(); + m_previous_valid_timestamp.reset(); + + clear_pes_payload(); + processed = false; + m_timestamps_wrapped = false; + m_timestamp_wrap_add = timestamp_c::ns(0); +} + // ------------------------------------------------------------ file_t::file_t(mm_io_cptr const &in) @@ -789,7 +826,6 @@ file_t::file_t(mm_io_cptr const &in) , m_pat_found{} , m_pmt_found{} , m_es_to_process{} - , m_global_timestamp_offset{} , m_stream_timestamp{timestamp_c::ns(0)} , m_timestamp_mpls_sync{timestamp_c::ns(0)} , m_state{processing_state_e::probing} @@ -801,6 +837,7 @@ file_t::file_t(mm_io_cptr const &in) , m_num_pmt_crc_errors{} , m_validate_pat_crc{true} , m_validate_pmt_crc{true} + , m_has_audio_or_video_track{} { } @@ -815,6 +852,13 @@ file_t::get_queued_bytes() return bytes; } +void +file_t::reset_processing_state(processing_state_e new_state) { + m_state = new_state; + m_last_non_subtitle_pts.reset(); + m_last_non_subtitle_dts.reset(); +} + // ------------------------------------------------------------ bool @@ -1013,10 +1057,20 @@ reader_c::read_headers() { show_demuxer_info(); } +void +reader_c::reset_processing_state(processing_state_e new_state) { + for (auto const &file : m_files) + file->reset_processing_state(new_state); + + for (auto const &track : m_tracks) + track->reset_processing_state(); +} + void reader_c::determine_global_timestamp_offset() { - auto &f = file(); - f.m_state = processing_state_e::determining_timestamp_offset; + reset_processing_state(processing_state_e::determining_timestamp_offset); + + auto &f = file(); f.m_in->setFilePointer(0); f.m_in->clear_eof(); @@ -1047,12 +1101,7 @@ reader_c::determine_global_timestamp_offset() { f.m_in->setFilePointer(0); f.m_in->clear_eof(); - for (auto const &track : m_tracks) { - track->clear_pes_payload(); - track->processed = false; - track->m_timestamps_wrapped = false; - track->m_timestamp_wrap_add = timestamp_c::ns(0); - } + reset_processing_state(processing_state_e::muxing); } void @@ -1522,6 +1571,10 @@ reader_c::parse_pes(track_c &track) { auto orig_pts = pts; auto orig_dts = dts; + auto result = track.handle_bogus_subtitle_timestamps(pts, dts); + + if (drop_decision_e::drop == result) + return; track.handle_timestamp_wrap(pts, dts); @@ -1744,6 +1797,10 @@ reader_c::probe_packet_complete(track_c &track) { track.processed = true; track.probed_ok = true; --f.m_es_to_process; + + if (mtx::included_in(track.type, pid_type_e::audio, pid_type_e::video)) + f.m_has_audio_or_video_track = true; + mxdebug_if(m_debug_headers, boost::format("probe_packet_complete: ES to process: %1%\n") % f.m_es_to_process); } } @@ -1913,9 +1970,6 @@ void reader_c::create_packetizers() { determine_global_timestamp_offset(); - for (auto const &file : m_files) - file->m_state = processing_state_e::muxing; - mxdebug_if(m_debug_headers, boost::format("create_packetizers: create packetizers...\n")); for (std::size_t i = 0u, end = m_tracks.size(); i < end; ++i) create_packetizer(i); @@ -2121,13 +2175,21 @@ reader_c::find_track_for_pid(uint16_t pid) const { auto &f = *m_files[m_current_file]; - for (auto const &track : m_tracks) - if ( (track->m_file_num == m_current_file) - && (track->pid == pid) - && ( mtx::included_in(f.m_state, processing_state_e::probing, processing_state_e::determining_timestamp_offset) - || track->has_packetizer())) + for (auto const &track : m_tracks) { + if ( (track->m_file_num != m_current_file) + || (track->pid != pid)) + continue; + + if (track->has_packetizer() || mtx::included_in(f.m_state, processing_state_e::probing, processing_state_e::determining_timestamp_offset)) return track; + for (auto const &coupled_track : track->m_coupled_tracks) + if (coupled_track->has_packetizer()) + return coupled_track; + + return track; + } + return {}; } diff --git a/src/input/r_mpeg_ts.h b/src/input/r_mpeg_ts.h index bfaeae842..2fdad800c 100644 --- a/src/input/r_mpeg_ts.h +++ b/src/input/r_mpeg_ts.h @@ -82,6 +82,11 @@ enum class stream_type_e : unsigned char { stream_subtitles_hdmv_textst = 0x92, // HDMV TextST subtitles }; +enum class drop_decision_e { + keep, + drop, +}; + #if defined(COMP_MSC) #pragma pack(push,1) #endif @@ -352,6 +357,7 @@ public: void set_pid(uint16_t new_pid); + drop_decision_e handle_bogus_subtitle_timestamps(timestamp_c &pts, timestamp_c &dts); void handle_timestamp_wrap(timestamp_c &pts, timestamp_c &dts); bool detect_timestamp_wrap(timestamp_c const ×tamp) const; void adjust_timestamp_for_wrap(timestamp_c ×tamp); @@ -362,6 +368,8 @@ public: void process(packet_cptr const &packet); void parse_iso639_language_from(void const *buffer); + + void reset_processing_state(); }; struct file_t { @@ -372,7 +380,7 @@ struct file_t { bool m_pat_found, m_pmt_found; int m_es_to_process; - timestamp_c m_global_timestamp_offset, m_stream_timestamp, m_last_non_subtitle_timestamp, m_timestamp_restriction_min, m_timestamp_restriction_max, m_timestamp_mpls_sync; + timestamp_c m_global_timestamp_offset, m_stream_timestamp, m_timestamp_restriction_min, m_timestamp_restriction_max, m_timestamp_mpls_sync, m_last_non_subtitle_pts, m_last_non_subtitle_dts; processing_state_e m_state; uint64_t m_probe_range; @@ -380,10 +388,12 @@ struct file_t { bool m_file_done, m_packet_sent_to_packetizer; unsigned int m_detected_packet_size, m_num_pat_crc_errors, m_num_pmt_crc_errors; - bool m_validate_pat_crc, m_validate_pmt_crc; + bool m_validate_pat_crc, m_validate_pmt_crc, m_has_audio_or_video_track; file_t(mm_io_cptr const &in); + int64_t get_queued_bytes() const; + void reset_processing_state(processing_state_e new_state); }; using file_cptr = std::shared_ptr; @@ -456,6 +466,7 @@ private: void create_srt_subtitles_packetizer(track_ptr const &track); void create_dvbsub_subtitles_packetizer(track_ptr const &track); + void reset_processing_state(processing_state_e new_state); void determine_global_timestamp_offset(); bfs::path find_file(bfs::path const &source_file, std::string const &sub_directory, std::string const &extension) const; diff --git a/tests/results.txt b/tests/results.txt index 90a484b90..8ada574a7 100644 --- a/tests/results.txt +++ b/tests/results.txt @@ -242,7 +242,7 @@ T_393aac_audiospecificconfig_0channels:c948233b0562fa997aa426e39b5772d2:passed:2 T_394flv_negative_cts_offset:c33944c32b31498ddbf25c1354688271:passed:20130414-115331:0.123013461 T_395remove_bitstream_ar_info:12d0f2056e9b79cafec12352c94ac846-3c2ba23992cba3441396e25b2740cf9f-4c97e5c442155fe54cb90eb10b440211-49f413b658608e0ea01623801a717dc7:passed:20130427-171243:0.334408574 T_396X_pcm_mono_16bit:49b80add61d8a11514d747c9616f7e80:passed:20130624-200214:0.055106028 -T_397mpeg_ts_broken_pes_track_detection:59282e6da966f4ca1bbe14a0ccc2c587:passed:20130624-220549:1.12831027 +T_397mpeg_ts_broken_pes_track_detection:3d6512e4d604a85c1ad4c519279bdeeb:passed:20130624-220549:1.12831027 T_398flv1_no_pixel_dimensions:870014b911f1fcc626a7921525b4c185:passed:20130624-225648:0.103188723 T_399h264_append_and_default_duration:d18de651ba99423741592b203011bcd3:passed:20130627-195946:5.558958754 T_400opus_experimental:d74c651a826ccde9053ab088032d5870:passed:20130703-213929:0.055921025 @@ -420,3 +420,4 @@ T_571bluray_pcm_odd_number_of_channels:77cf4c999faf85d0e37c790ee7c75541:passed:2 T_572hdmv_textst:0c7491296aca98abf20af85926468bac-0c7491296aca98abf20af85926468bac-0f8722f155c1c32f4b58ef8661fea6a1+2d080bee45d9cf8fdd1f235f56550863:passed:20161128-164101:0.072007646 T_573teletext_subs_data_unit_id_0x02_and_multiple_magazines:03b961cb80532c45829bd91345f327d0:passed:20161208-202941:0.037765249 T_574tta_with_apev2_tags:342b123b71e0a40a0688f4647c38b1c5-4870070dccb00f5aaeda933c301050f5:passed:20161223-210853:0.019411679 +T_575mpeg_ts_subtitle_timestamps_way_off_from_video_timestamps:78a0ab3566507be5684cde3fa601bb06:passed:20170106-200606:0.508113744 diff --git a/tests/test-575mpeg_ts_subtitle_timestamps_way_off_from_video_timestamps.rb b/tests/test-575mpeg_ts_subtitle_timestamps_way_off_from_video_timestamps.rb new file mode 100755 index 000000000..2c9653f8a --- /dev/null +++ b/tests/test-575mpeg_ts_subtitle_timestamps_way_off_from_video_timestamps.rb @@ -0,0 +1,5 @@ +#!/usr/bin/ruby -w + +# T_575mpeg_ts_subtitle_timestamps_way_off_from_video_timestamps +describe "mkvmerge / MPEG TS: subtitles multiplexed properly, but timestamps way off from audio & video timestamps" +test_merge "data/ts/teletext_subs_timestamps_way_off_from_video_timestamps.ts"