From b7d46e9ee09e804555ce6ea4691b1fa6a9e2caa4 Mon Sep 17 00:00:00 2001 From: Moritz Bunkus Date: Mon, 3 Feb 2014 20:36:15 +0100 Subject: [PATCH 1/7] MPEG TS reader: honor in/out time from MPLS Fixes #985. --- ChangeLog | 7 +++++++ src/common/mm_mpls_multi_file_io.cpp | 8 ++------ src/common/mm_mpls_multi_file_io.h | 4 ++++ src/input/r_mpeg_ts.cpp | 20 +++++++++++++++++--- src/input/r_mpeg_ts.h | 2 +- src/merge/mkvmerge.cpp | 13 +++++++++++-- src/merge/output_control.cpp | 6 ++++++ src/merge/output_control.h | 2 ++ src/merge/pr_generic.cpp | 19 +++++++++++++++++++ src/merge/pr_generic.h | 9 ++++++++- 10 files changed, 77 insertions(+), 13 deletions(-) diff --git a/ChangeLog b/ChangeLog index 242789c02..cf0584d08 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,10 @@ +2014-02-03 Moritz Bunkus + + * mkvmerge: bug fix: When reading M2TS files belonging to an MPLS + playlist mkvmerge will now only copy packets whose timestamps lie + between the »in time« and »out time« restrictions from the + playlist's entry corresponding to that M2TS file. Fixes #985. + 2014-01-22 Moritz Bunkus * all: Windows 64bit: fixed return value checks for opening diff --git a/src/common/mm_mpls_multi_file_io.cpp b/src/common/mm_mpls_multi_file_io.cpp index c7907315b..1afc93026 100644 --- a/src/common/mm_mpls_multi_file_io.cpp +++ b/src/common/mm_mpls_multi_file_io.cpp @@ -77,17 +77,13 @@ mm_mpls_multi_file_io_c::open_multi(mm_io_c *in) { }; std::vector file_names; - std::unordered_map file_names_seen; for (auto const &item : mpls_parser->get_playlist().items) { auto file = find_file(item); mxdebug_if(ms_debug, boost::format("Item clip ID: %1% codec ID: %2%: have file? %3% file: %4%\n") % item.clip_id % item.codec_id % !file.empty() % file.string()); - if (file.empty() || file_names_seen[file.string()]) - continue; - - file_names.push_back(file); - file_names_seen[file.string()] = true; + if (!file.empty()) + file_names.push_back(file); } mxdebug_if(ms_debug, boost::format("Number of files left: %1%\n") % file_names.size()); diff --git a/src/common/mm_mpls_multi_file_io.h b/src/common/mm_mpls_multi_file_io.h index 2977494f6..4d4ffe525 100644 --- a/src/common/mm_mpls_multi_file_io.h +++ b/src/common/mm_mpls_multi_file_io.h @@ -41,6 +41,10 @@ public: return m_files; } + mtx::mpls::parser_c const &get_mpls_parser() const { + return *m_mpls_parser; + } + std::vector const &get_chapters() const; virtual void create_verbose_identification_info(std::vector &verbose_info); diff --git a/src/input/r_mpeg_ts.cpp b/src/input/r_mpeg_ts.cpp index 50cd92dc1..8214645b1 100644 --- a/src/input/r_mpeg_ts.cpp +++ b/src/input/r_mpeg_ts.cpp @@ -54,13 +54,26 @@ mpeg_ts_track_c::send_to_packetizer() { auto timecode_to_use = !m_timecode.valid() ? timecode_c{} : reader.m_dont_use_audio_pts && (ES_AUDIO_TYPE == type) ? timecode_c{} : m_apply_dts_timecode_fix && (m_previous_timecode == m_timecode) ? timecode_c{} - : (m_timecode < reader.m_global_timecode_offset) ? timecode_c::ns(0) - : m_timecode - reader.m_global_timecode_offset; + : std::max(m_timecode, reader.m_global_timecode_offset); + + auto timecode_to_check = timecode_to_use.valid() ? timecode_to_use : reader.m_stream_timecode; + auto const &min = reader.get_timecode_restriction_min(); + auto const &max = reader.get_timecode_restriction_max(); + auto use_packet = ptzr != -1; + + if ( (min.valid() && (timecode_to_check < min)) + || (max.valid() && (timecode_to_check > max))) + use_packet = false; + + if (timecode_to_use.valid()) { + reader.m_stream_timecode = timecode_to_use; + timecode_to_use -= std::max(reader.m_global_timecode_offset, min.valid() ? min : timecode_c::ns(0)); + } mxdebug_if(m_debug_delivery, boost::format("send_to_packetizer() PID %1% expected %2% actual %3% timecode_to_use %4% m_previous_timecode %5%\n") % pid % pes_payload_size % pes_payload->get_size() % timecode_to_use % m_previous_timecode); - if (ptzr != -1) + if (use_packet) reader.m_reader_packetizers[ptzr]->process(new packet_t(memory_c::clone(pes_payload->get_buffer(), pes_payload->get_size()), timecode_to_use.to_ns(-1))); pes_payload->remove(pes_payload->get_size()); @@ -387,6 +400,7 @@ mpeg_ts_reader_c::mpeg_ts_reader_c(const track_info_c &ti, , PMT_pid(-1) , es_to_process{} , m_global_timecode_offset{} + , m_stream_timecode{timecode_c::ns(0)} , input_status(INPUT_PROBE) , track_buffer_ready(-1) , file_done{} diff --git a/src/input/r_mpeg_ts.h b/src/input/r_mpeg_ts.h index 8ec729b62..07d162d83 100644 --- a/src/input/r_mpeg_ts.h +++ b/src/input/r_mpeg_ts.h @@ -362,7 +362,7 @@ protected: bool PAT_found, PMT_found; int16_t PMT_pid; int es_to_process; - timecode_c m_global_timecode_offset; + timecode_c m_global_timecode_offset, m_stream_timecode; mpeg_ts_input_type_e input_status; // can be INPUT_PROBE, INPUT_READ int track_buffer_ready; diff --git a/src/merge/mkvmerge.cpp b/src/merge/mkvmerge.cpp index 540924b75..8e09c667e 100644 --- a/src/merge/mkvmerge.cpp +++ b/src/merge/mkvmerge.cpp @@ -2391,10 +2391,19 @@ add_filelists_for_playlists() { auto &file_names = filelist->playlist_mpls_in->get_file_names(); auto previous_filelist_id = filelist->id; + auto const &play_items = filelist->playlist_mpls_in->get_mpls_parser().get_playlist().items; + + assert(file_names.size() == play_items.size()); + + filelist->restricted_timecode_min = play_items[0].in_time; + filelist->restricted_timecode_max = play_items[0].out_time; for (size_t idx = 1, idx_end = file_names.size(); idx < idx_end; ++idx) { - auto current_filelist_id = g_files.size() + new_filelists.size(); - auto new_filelist = create_filelist_for_playlist(file_names[idx], previous_filelist_id, current_filelist_id, idx, *filelist->ti); + auto current_filelist_id = g_files.size() + new_filelists.size(); + auto new_filelist = create_filelist_for_playlist(file_names[idx], previous_filelist_id, current_filelist_id, idx, *filelist->ti); + new_filelist.restricted_timecode_min = play_items[idx].in_time; + new_filelist.restricted_timecode_max = play_items[idx].out_time; + new_filelists.push_back(new_filelist); previous_filelist_id = new_filelist.id; diff --git a/src/merge/output_control.cpp b/src/merge/output_control.cpp index 79404308a..1a3f923f2 100644 --- a/src/merge/output_control.cpp +++ b/src/merge/output_control.cpp @@ -1301,6 +1301,8 @@ calc_attachment_sizes() { */ void create_readers() { + static auto s_debug_timecode_restrictions = debugging_option_c{"timecode_restrictions"}; + for (auto &file : g_files) { try { mm_io_cptr input_file = file.playlist_mpls_in ? std::static_pointer_cast(file.playlist_mpls_in) : open_input_file(file); @@ -1401,11 +1403,15 @@ create_readers() { } file.reader->read_headers(); + file.reader->set_timecode_restrictions(file.restricted_timecode_min, file.restricted_timecode_max); // Re-calculate file size because the reader might switch to a // multi I/O reader in read_headers(). file.size = file.reader->get_file_size(); + mxdebug_if(s_debug_timecode_restrictions, + boost::format("Timecode restrictions for %3%: min %1% max %2%\n") % file.restricted_timecode_min % file.restricted_timecode_max % file.ti->m_fname); + } catch (mtx::mm_io::open_x &error) { mxerror(boost::format(Y("The demultiplexer for the file '%1%' failed to initialize:\n%2%\n")) % file.ti->m_fname % Y("The file could not be opened for reading, or there was not enough data to parse its headers.")); diff --git a/src/merge/output_control.h b/src/merge/output_control.h index 6451775ca..247a60389 100644 --- a/src/merge/output_control.h +++ b/src/merge/output_control.h @@ -119,6 +119,8 @@ struct filelist_t { size_t playlist_index, playlist_previous_filelist_id; mm_mpls_multi_file_io_cptr playlist_mpls_in; + timecode_c restricted_timecode_min, restricted_timecode_max; + filelist_t() : size{} , id{} diff --git a/src/merge/pr_generic.cpp b/src/merge/pr_generic.cpp index d3a4265a1..27bc91d70 100644 --- a/src/merge/pr_generic.cpp +++ b/src/merge/pr_generic.cpp @@ -1255,6 +1255,25 @@ generic_reader_c::~generic_reader_c() { delete m_reader_packetizers[i]; } +void +generic_reader_c::set_timecode_restrictions(timecode_c const &min, + timecode_c const &max) { + m_restricted_timecodes_min = min; + m_restricted_timecodes_max = max; +} + +timecode_c const & +generic_reader_c::get_timecode_restriction_min() + const { + return m_restricted_timecodes_min; +} + +timecode_c const & +generic_reader_c::get_timecode_restriction_max() + const { + return m_restricted_timecodes_max; +} + void generic_reader_c::read_all() { for (auto &packetizer : m_reader_packetizers) diff --git a/src/merge/pr_generic.h b/src/merge/pr_generic.h index 4fc0f7a98..5f537d984 100644 --- a/src/merge/pr_generic.h +++ b/src/merge/pr_generic.h @@ -32,6 +32,7 @@ #include "common/stereo_mode.h" #include "common/strings/editing.h" #include "common/tags/tags.h" +#include "common/timecode.h" #include "common/translation.h" #include "merge/item_selector.h" #include "merge/packet.h" @@ -353,10 +354,12 @@ public: int64_t m_reference_timecode_tolerance; -private: +protected: id_result_t m_id_results_container; std::vector m_id_results_tracks, m_id_results_attachments, m_id_results_chapters, m_id_results_tags; + timecode_c m_restricted_timecodes_min, m_restricted_timecodes_max; + public: generic_reader_c(const track_info_c &ti, const mm_io_cptr &in); virtual ~generic_reader_c(); @@ -366,6 +369,10 @@ public: return true; } + virtual void set_timecode_restrictions(timecode_c const &min, timecode_c const &max); + virtual timecode_c const &get_timecode_restriction_min() const; + virtual timecode_c const &get_timecode_restriction_max() const; + virtual void read_headers() = 0; virtual file_status_e read(generic_packetizer_c *ptzr, bool force = false) = 0; virtual void read_all(); From 8f6db59305bd1319463b9c3ce110721e3eb9e171 Mon Sep 17 00:00:00 2001 From: Moritz Bunkus Date: Mon, 3 Feb 2014 21:13:03 +0100 Subject: [PATCH 2/7] fix compilation on Solaris: ldexp requires the std:: prefix for int64_t Fixes #977. --- src/common/math.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/math.cpp b/src/common/math.cpp index 6a0806a16..4dbf42774 100644 --- a/src/common/math.cpp +++ b/src/common/math.cpp @@ -45,5 +45,5 @@ double int_to_double(int64_t value) { if (static_cast(value + value) > (0xffeull << 52)) return NAN; - return ldexp(((value & ((1ll << 52) - 1)) + (1ll << 52)) * (value >> 63 | 1), (value >> 52 & 0x7ff) - 1075); + return std::ldexp(((value & ((1ll << 52) - 1)) + (1ll << 52)) * (value >> 63 | 1), (value >> 52 & 0x7ff) - 1075); } From d3cf4f575644850688e40bbce5c093a97bc30bff Mon Sep 17 00:00:00 2001 From: Moritz Bunkus Date: Mon, 3 Feb 2014 21:23:54 +0100 Subject: [PATCH 3/7] fix compilation on Solaris: ioctl requires more headers --- ac/initialization.m4 | 2 +- src/common/terminal.cpp | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ac/initialization.m4 b/ac/initialization.m4 index f7258f9a2..189c495b1 100644 --- a/ac/initialization.m4 +++ b/ac/initialization.m4 @@ -17,5 +17,5 @@ PKG_PROG_PKG_CONFIG dnl Check for headers AC_HEADER_STDC() -AC_CHECK_HEADERS([inttypes.h stdint.h sys/types.h sys/syscall.h]) +AC_CHECK_HEADERS([inttypes.h stdint.h sys/types.h sys/syscall.h stropts.h]) AC_CHECK_FUNCS([vsscanf syscall],,) diff --git a/src/common/terminal.cpp b/src/common/terminal.cpp index 6c616f806..b3186298c 100644 --- a/src/common/terminal.cpp +++ b/src/common/terminal.cpp @@ -18,6 +18,12 @@ # if defined(HAVE_SYS_IOCTL_H) || defined(GWINSZ_IN_SYS_IOCTL) # include # endif // HAVE_SYS_IOCTL_H || GWINSZ_IN_SYS_IOCTL +# if defined(HAVE_UNISTD_H) +# include +# endif // HAVE_UNISTD_H +# if defined(HAVE_STROPTS_H) +# include +# endif // HAVE_STROPTS_H #endif // HAVE_TIOCGWINSZ #include "common/terminal.h" @@ -37,4 +43,3 @@ get_terminal_columns() { return DEFAULT_TERMINAL_COLUMNS; #endif // HAVE_TIOCGWINSZ } - From 8546756f7a5bf8bc0a9ac1478b6fe913196ac9cf Mon Sep 17 00:00:00 2001 From: Moritz Bunkus Date: Tue, 4 Feb 2014 20:14:07 +0100 Subject: [PATCH 4/7] mpls_dump: a new tool for dumping MPLS file structures --- .gitignore | 1 + Rakefile | 12 +++++- src/tools/mpls_dump.cpp | 86 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 src/tools/mpls_dump.cpp diff --git a/.gitignore b/.gitignore index 822f6e57d..d32a11a2a 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,4 @@ BROWSE Makefile TAGS lib*.a +/src/tools/mpls_dump diff --git a/Rakefile b/Rakefile index df2756fc5..67295c7ae 100755 --- a/Rakefile +++ b/Rakefile @@ -43,7 +43,7 @@ def setup_globals $programs = %w{mkvmerge mkvinfo mkvextract mkvpropedit} $programs << "mmg" if c?(:USE_WXWIDGETS) $programs << "mkvtoolnix-gui" if $build_mkvtoolnix_gui - $tools = %w{ac3parser base64tool diracparser ebml_validator vc1parser} + $tools = %w{ac3parser base64tool diracparser ebml_validator mpls_dump vc1parser} $mmg_bin = c(:MMG_BIN) $mmg_bin = "mmg" if $mmg_bin.empty? @@ -814,6 +814,16 @@ if $build_tools libraries($common_libs). create + # + # tools: mpls_dump + # + Application.new("src/tools/mpls_dump"). + description("Build the mpls_dump executable"). + aliases("tools:mpls_dump"). + sources("src/tools/mpls_dump.cpp"). + libraries($common_libs). + create + # # tools: vc1parser # diff --git a/src/tools/mpls_dump.cpp b/src/tools/mpls_dump.cpp new file mode 100644 index 000000000..92073b3c4 --- /dev/null +++ b/src/tools/mpls_dump.cpp @@ -0,0 +1,86 @@ +/* + mpls_dump - A tool dumping MPLS structures + + Distributed under the GPL + see the file COPYING for details + or visit http://www.gnu.org/copyleft/gpl.html + + Written by Moritz Bunkus . +*/ + +#include "common/common_pch.h" + +#include "common/command_line.h" +#include "common/mpls.h" + +static void +show_help() { + mxinfo("mpls_dump [options] input_file_name\n" + "\n" + "General options:\n" + "\n" + " -h, --help This help text\n" + " -V, --version Print version information\n"); + mxexit(0); +} + +static void +show_version() { + mxinfo("mpls_dump v" VERSION "\n"); + mxexit(0); +} + +static std::string +parse_args(std::vector &args) { + std::string file_name; + + for (auto & arg: args) { + if ((arg == "-h") || (arg == "--help")) + show_help(); + + else if ((arg == "-V") || (arg == "--version")) + show_version(); + + else if (!file_name.empty()) + mxerror(Y("More than one input file given\n")); + + else + file_name = arg; + } + + if (file_name.empty()) + mxerror(Y("No file name given\n")); + + return file_name; +} + +static void +parse_file(const std::string &file_name) { + auto in = mm_file_io_c{file_name}; + auto parser = mtx::mpls::parser_c{}; + + if (!parser.parse(&in)) + mxerror("MPLS file could not be parsed.\n"); + + parser.dump(); +} + +int +main(int argc, + char **argv) { + mtx_common_init("mpls_dump"); + + auto args = command_line_utf8(argc, argv); + while (handle_common_cli_args(args, "-r")) + ; + + auto file_name = parse_args(args); + + try { + parse_file(file_name); + } catch (mtx::mm_io::exception &) { + mxerror(Y("File not found\n")); + } + + return 0; +} From 4b802144c116ea873d186a74c6d2e79e5558b0fc Mon Sep 17 00:00:00 2001 From: Moritz Bunkus Date: Wed, 5 Feb 2014 22:38:14 +0100 Subject: [PATCH 5/7] AVI reader: output key frames with video index dump --- src/input/r_avi.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/input/r_avi.cpp b/src/input/r_avi.cpp index 7e5d3db59..ee8177c4c 100644 --- a/src/input/r_avi.cpp +++ b/src/input/r_avi.cpp @@ -934,6 +934,11 @@ avi_reader_c::debug_dump_video_index() { int num_video_frames = AVI_video_frames(m_avi), i; mxinfo(boost::format("AVI video index dump: %1% entries; frame rate: %2%\n") % num_video_frames % m_fps); - for (i = 0; num_video_frames > i; ++i) - mxinfo(boost::format(" %1%: %2% bytes\n") % i % AVI_frame_size(m_avi, i)); + for (i = 0; num_video_frames > i; ++i) { + int key = 0; + AVI_read_frame(m_avi, nullptr, &key); + mxinfo(boost::format(" %1%: %2% bytes; key: %3%\n") % i % AVI_frame_size(m_avi, i) % key); + } + + AVI_set_video_position(m_avi, 0); } From 363769ac4dbee63d87d910a6786d41ad453815d0 Mon Sep 17 00:00:00 2001 From: Moritz Bunkus Date: Wed, 5 Feb 2014 22:43:22 +0100 Subject: [PATCH 6/7] AVI reader: skip audio chunks bigger than 10 MB --- ChangeLog | 5 +++++ src/input/r_avi.cpp | 15 ++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index cf0584d08..2acfbcad6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,8 @@ +2014-02-05 Moritz Bunkus + + * mkvmerge: enhancements: AVI reader: audio chunks with obvious + wrong size information (bigger than 10 MB) will be skipped. + 2014-02-03 Moritz Bunkus * mkvmerge: bug fix: When reading M2TS files belonging to an MPLS diff --git a/src/input/r_avi.cpp b/src/input/r_avi.cpp index ee8177c4c..21166f1bc 100644 --- a/src/input/r_avi.cpp +++ b/src/input/r_avi.cpp @@ -49,6 +49,8 @@ #include "output/p_vorbis.h" #include "output/p_vpx.h" +#define AVI_MAX_AUDIO_CHUNK_SIZE (10 * 1024 * 1024) + #define GAB2_TAG FOURCC('G', 'A', 'B', '2') #define GAB2_ID_LANGUAGE 0x0000 #define GAB2_ID_LANGUAGE_UNICODE 0x0002 @@ -509,8 +511,11 @@ avi_reader_c::add_audio_demuxer(int aid) { m_audio_demuxers.push_back(demuxer); int i, maxchunks = AVI_audio_chunks(m_avi); - for (i = 0; i < maxchunks; i++) - m_bytes_to_process += AVI_audio_size(m_avi, i); + for (i = 0; i < maxchunks; i++) { + auto size = AVI_audio_size(m_avi, i); + if (size < AVI_MAX_AUDIO_CHUNK_SIZE) + m_bytes_to_process += size; + } } generic_packetizer_c * @@ -762,10 +767,14 @@ avi_reader_c::read_audio(avi_demuxer_t &demuxer) { while (true) { int size = AVI_audio_size(m_avi, AVI_get_audio_position_index(m_avi)); + // -1 indicates the last chunk. if (-1 == size) return flush_packetizer(demuxer.m_ptzr); - if (!size) { + // Sanity check. Ignore chunks with obvious wrong size information + // (> 10 MB). Also skip 0-sized blocks. Those are officially + // skipped. + if (!size || (size > AVI_MAX_AUDIO_CHUNK_SIZE)) { AVI_set_audio_position_index(m_avi, AVI_get_audio_position_index(m_avi) + 1); continue; } From e0d697c3e4101a243595e8b706e0ed5c28a82001 Mon Sep 17 00:00:00 2001 From: Moritz Bunkus Date: Fri, 7 Feb 2014 22:30:29 +0100 Subject: [PATCH 7/7] MPEG4 p10 handler: debug option for forcefully using the simple picture ordering --- src/common/mpeg4_p10.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/common/mpeg4_p10.cpp b/src/common/mpeg4_p10.cpp index 6b4a4a9a6..34a26130d 100644 --- a/src/common/mpeg4_p10.cpp +++ b/src/common/mpeg4_p10.cpp @@ -1559,6 +1559,8 @@ mpeg4::p10::avc_es_parser_c::get_most_often_used_duration() void mpeg4::p10::avc_es_parser_c::cleanup() { + auto s_debug_force_simple_picture_order = debugging_option_c{"avc_parser_force_simple_picture_order"}; + if (m_frames.empty()) return; @@ -1632,7 +1634,7 @@ mpeg4::p10::avc_es_parser_c::cleanup() { } } - if (!simple_picture_order) + if (!simple_picture_order && !s_debug_force_simple_picture_order) brng::sort(m_frames, [](const avc_frame_t &f1, const avc_frame_t &f2) { return f1.m_presentation_order < f2.m_presentation_order; }); brng::sort(m_provided_timecodes);