diff --git a/src/common/mpeg4_p10.cpp b/src/common/mpeg4_p10.cpp
index e1c61729a..7a3b5761b 100644
--- a/src/common/mpeg4_p10.cpp
+++ b/src/common/mpeg4_p10.cpp
@@ -993,6 +993,7 @@ mpeg4::p10::avc_es_parser_c::avc_es_parser_c()
   , m_ignore_nalu_size_length_errors(false)
   , m_discard_actual_frames(false)
   , m_simple_picture_order{}
+  , m_first_cleanup{true}
   , m_debug_keyframe_detection{"avc_parser|avc_keyframe_detection"}
   , m_debug_nalu_types{        "avc_parser|avc_nalu_types"}
   , m_debug_timecodes{         "avc_parser|avc_timecodes"}
@@ -1715,7 +1716,10 @@ mpeg4::p10::avc_es_parser_c::cleanup() {
 
   // This may be wrong but is needed for mkvmerge to work correctly
   // (cluster_helper etc).
-  frames_begin->m_keyframe = true;
+  if (m_first_cleanup) {
+    frames_begin->m_keyframe = true;
+    m_first_cleanup          = false;
+  }
 
   for (frame_itr = frames_begin; frames_end != frame_itr; ++frame_itr) {
     if (frames_begin != frame_itr)
diff --git a/src/common/mpeg4_p10.h b/src/common/mpeg4_p10.h
index 06a865e7f..61b7b79f8 100644
--- a/src/common/mpeg4_p10.h
+++ b/src/common/mpeg4_p10.h
@@ -263,7 +263,7 @@ protected:
   bool m_have_incomplete_frame;
   std::deque<memory_cptr> m_unhandled_nalus;
 
-  bool m_ignore_nalu_size_length_errors, m_discard_actual_frames, m_simple_picture_order;
+  bool m_ignore_nalu_size_length_errors, m_discard_actual_frames, m_simple_picture_order, m_first_cleanup;
 
   debugging_option_c m_debug_keyframe_detection, m_debug_nalu_types, m_debug_timecodes, m_debug_sps_info, m_debug_trailing_zero_byte_removal;
   std::map<int, std::string> m_nalu_names_by_type;