From 97a8decce111208d5606f2e390e02b870bd6a310 Mon Sep 17 00:00:00 2001 From: Moritz Bunkus Date: Thu, 27 Feb 2003 19:51:53 +0000 Subject: [PATCH] New cluster caching system. --- mkvmerge.cpp | 60 ++------ p_video.cpp | 18 +-- p_video.h | 6 +- pr_generic.cpp | 378 ++++++++++++++++++++++++++++++++++++++++++------- pr_generic.h | 55 ++++--- queue.cpp | 10 +- queue.h | 10 +- 7 files changed, 398 insertions(+), 139 deletions(-) diff --git a/mkvmerge.cpp b/mkvmerge.cpp index 069a4ac02..f4f3988ac 100644 --- a/mkvmerge.cpp +++ b/mkvmerge.cpp @@ -13,7 +13,7 @@ /*! \file - \version \$Id: mkvmerge.cpp,v 1.17 2003/02/27 09:35:55 mosu Exp $ + \version \$Id: mkvmerge.cpp,v 1.18 2003/02/27 19:51:53 mosu Exp $ \brief command line parameter parsing, looping, output handling \author Moritz Bunkus */ @@ -89,9 +89,6 @@ int max_ms_per_cluster = 1000; float video_fps = -1.0; -packet_t **packet_queue = NULL; -int num_packets_in_packetq = 0; - cluster_helper_c *cluster_helper = NULL; KaxSegment kax_segment; KaxTracks *kax_tracks; @@ -735,54 +732,18 @@ static void parse_args(int argc, char **argv) { }*/ } -static void write_packetq() { - KaxCues dummy_cues; - KaxBlockGroup *last_group = NULL; - KaxCluster *cluster; - int i, num_packets; - u_int64_t cluster_timecode; - - cluster = cluster_helper->get_cluster(); - cluster_timecode = cluster_helper->get_timecode(); - num_packets = cluster_helper->get_packet_count(); - - for (i = 0; i < num_packets; i++) { - packet_t *pack; - - pack = cluster_helper->get_packet(i); - - if (last_group == NULL) - pack->group = &GetChild(*cluster); - else - pack->group = &GetNextChild(*cluster, *last_group); - last_group = pack->group; - pack->block = &GetChild(*pack->group); - pack->data_buffer = new DataBuffer((binary *)pack->data, pack->length); - KaxTrackEntry &track_entry = - static_cast(*pack->source->track_entry); - - pack->block->AddFrame(track_entry, pack->timestamp - cluster_timecode, - *pack->data_buffer); - pack->source->added_packet_to_cluster(pack, cluster_helper); - } - - cluster->Render(static_cast(*out), dummy_cues); - - cluster_helper->release(); -} - static int write_packet(packet_t *pack) { u_int64_t timecode; - if (cluster_helper == NULL) - cluster_helper = new cluster_helper_c(); - cluster_helper->add_packet(pack); timecode = cluster_helper->get_timecode(); if (((pack->timestamp - timecode) > max_ms_per_cluster) || - (num_packets_in_packetq > max_blocks_per_cluster)) - write_packetq(); + (cluster_helper->get_packet_count() > max_blocks_per_cluster) || + (cluster_helper->get_cluster_content_size() > 1500000)) { + cluster_helper->render(out); + cluster_helper->add_cluster(new KaxCluster()); + } return 1; } @@ -793,6 +754,9 @@ int main(int argc, char **argv) { nice(2); + cluster_helper = new cluster_helper_c(); + cluster_helper->add_cluster(new KaxCluster()); + parse_args(argc, argv); /* let her rip! */ @@ -845,8 +809,10 @@ int main(int argc, char **argv) { display_progress(0); } - if (num_packets_in_packetq > 0) - write_packetq(); + if ((cluster_helper != NULL) && (cluster_helper->get_packet_count() > 0)) + cluster_helper->render(out); + + delete cluster_helper; if (verbose == 1) { display_progress(1); diff --git a/p_video.cpp b/p_video.cpp index 991cb37ec..20085544c 100644 --- a/p_video.cpp +++ b/p_video.cpp @@ -13,7 +13,7 @@ /*! \file - \version \$Id: p_video.cpp,v 1.4 2003/02/27 09:35:55 mosu Exp $ + \version \$Id: p_video.cpp,v 1.5 2003/02/27 19:51:53 mosu Exp $ \brief video output module \author Moritz Bunkus */ @@ -126,9 +126,11 @@ int video_packetizer_c::process(char *buf, int size, int num_frames, if ((packetno >= range.start) && ((range.end == 0) || (packetno < range.end))) { if (key) + // Add a key frame and save its ID so that we can reference it later. last_id = add_packet(buf, size, (u_int64_t)(1000.0 * frames_output / fps)); else + // This is a P frame - let's reference the last key frame (I frame). add_packet(buf, size, (u_int64_t)(1000.0 * frames_output / fps), last_id); frames_output += num_frames; @@ -143,13 +145,13 @@ video_packetizer_c::~video_packetizer_c() { free(tempbuf); } -void video_packetizer_c::added_packet_to_cluster(packet_t *packet, - cluster_helper_c *helper) { - if (packet->ref == 0) { // this is a keyframe - if (last_helper) - last_helper->release(); - last_helper = helper; +void video_packetizer_c::added_packet_to_cluster(packet_t *packet) { + if (packet->bref == 0) { // this is a keyframe + // Free the last key frame and all others that (indirectly) reference it. + if (last_keyframe != NULL) + cluster_helper->free_ref(last_keyframe->id); + // Save this key frame so that we can later free all references to it + // (and make references to it in the first place). last_keyframe = packet; - last_helper->add_ref(); } } diff --git a/p_video.h b/p_video.h index 15386b8ff..c8b66159c 100644 --- a/p_video.h +++ b/p_video.h @@ -13,7 +13,7 @@ /*! \file - \version \$Id: p_video.h,v 1.5 2003/02/27 09:52:37 mosu Exp $ + \version \$Id: p_video.h,v 1.6 2003/02/27 19:51:53 mosu Exp $ \brief class definition for the video output module \author Moritz Bunkus */ @@ -35,7 +35,6 @@ private: range_t range; u_int64_t last_id; packet_t *last_keyframe; - cluster_helper_c *last_helper; public: video_packetizer_c(void *, int, char *, double, int, int, int, int, @@ -45,8 +44,7 @@ public: virtual int process(char *buf, int size, int num_frames, int key, int last_frame); virtual void set_header(); - virtual void added_packet_to_cluster(packet_t *packet, - cluster_helper_c *helper); + virtual void added_packet_to_cluster(packet_t *packet); }; #endif // __P_VIDEO_H diff --git a/pr_generic.cpp b/pr_generic.cpp index fa277dded..4eaad014d 100644 --- a/pr_generic.cpp +++ b/pr_generic.cpp @@ -13,13 +13,16 @@ /*! \file - \version \$Id: pr_generic.cpp,v 1.3 2003/02/27 09:35:55 mosu Exp $ + \version \$Id: pr_generic.cpp,v 1.4 2003/02/27 19:51:53 mosu Exp $ \brief functions common for all readers/packetizers \author Moritz Bunkus */ +#include #include +#include "IOCallback.h" +#include "StdIOCallback.h" #include "KaxCluster.h" #include "KaxClusterData.h" @@ -47,8 +50,8 @@ void generic_packetizer_c::set_private_data(void *data, int size) { private_data_size = size; } -void generic_packetizer_c::added_packet_to_cluster(packet_t *, - cluster_helper_c *) { +void generic_packetizer_c::added_packet_to_cluster(packet_t *packet) { + packet->superseeded = 1; } //-------------------------------------------------------------------- @@ -61,80 +64,351 @@ generic_reader_c::~generic_reader_c() { //-------------------------------------------------------------------- -cluster_helper_c::cluster_helper_c(KaxCluster *ncluster) { - num_packets = 0; - packet_q = NULL; - refcount = 1; - if (ncluster == NULL) - cluster = new KaxCluster(); - else - cluster = ncluster; +//#define walk_clusters() check_clusters(__LINE__) +#define walk_clusters() + +cluster_helper_c::cluster_helper_c() { + num_clusters = 0; + clusters = NULL; + cluster_content_size = 0; } cluster_helper_c::~cluster_helper_c() { int i; - if (packet_q) { - for (i = 0; i < num_packets; i++) { - free(packet_q[i]->data); - delete packet_q[i]->data_buffer; - free(packet_q[i]); - } - free(packet_q); + for (i = 0; i < num_clusters; i++) + free_contents(clusters[i]); + + if (clusters != NULL) + free(clusters); +} + +void cluster_helper_c::free_contents(ch_contents *clstr) { + packet_t *p; + int i; + + assert(clstr != NULL); + assert(clstr->cluster != NULL); + delete clstr->cluster; + + assert(!((clstr->num_packets != 0) && (clstr->packets == NULL))); + for (i = 0; i < clstr->num_packets; i++) { + p = clstr->packets[i]; + if (p->data != NULL) + free(p->data); + if (p->data_buffer != NULL) + delete p->data_buffer; +// fprintf(stdout, "* deleted %llu\n", p->id); + free(p); } - if (cluster) - delete cluster; + if (clstr->packets != NULL) + free(clstr->packets); + free(clstr); } KaxCluster *cluster_helper_c::get_cluster() { - return cluster; -} - -KaxCluster &cluster_helper_c::operator *() { - return *cluster; -} - -int cluster_helper_c::add_ref() { - refcount++; - return refcount; -} - -int cluster_helper_c::release() { - int ref; - refcount--; - ref = refcount; - if (refcount == 0) - delete this; - return ref; + if (clusters != NULL) + return clusters[num_clusters - 1]->cluster; + return NULL; } void cluster_helper_c::add_packet(packet_t *packet) { - packet_q = (packet_t **)realloc(packet_q, sizeof(packet_t *) * - (num_packets + 1)); - if (packet_q == NULL) + ch_contents *c; + + if (clusters == NULL) + return; + + c = clusters[num_clusters - 1]; + c->packets = (packet_t **)realloc(c->packets, sizeof(packet_t *) * + (c->num_packets + 1)); + if (c->packets == NULL) die("realloc"); - packet_q[num_packets] = packet; - if (num_packets == 0) { - KaxClusterTimecode &timecode = GetChild(*cluster); + + c->packets[c->num_packets] = packet; + if (c->num_packets == 0) { + KaxClusterTimecode &timecode = GetChild(*c->cluster); *(static_cast(&timecode)) = packet->timestamp; } - num_packets++; + + c->num_packets++; + + cluster_content_size += packet->length; + +// fprintf(stdout, "& new %llu\n", packet->id); + + walk_clusters(); } u_int64_t cluster_helper_c::get_timecode() { - if (packet_q == NULL) + if (clusters == NULL) return 0; - return packet_q[0]->timestamp; + if (clusters[num_clusters - 1]->packets == NULL) + return 0; + return clusters[num_clusters - 1]->packets[0]->timestamp; } packet_t *cluster_helper_c::get_packet(int num) { - if (packet_q == NULL) + ch_contents *c; + if (clusters == NULL) return NULL; - if ((num < 0) || (num > num_packets)) + c = clusters[num_clusters - 1]; + if (c->packets == NULL) return NULL; - return packet_q[num]; + if ((num < 0) || (num > c->num_packets)) + return NULL; + return c->packets[num]; } int cluster_helper_c::get_packet_count() { - return num_packets; + if (clusters == NULL) + return -1; + return clusters[num_clusters - 1]->num_packets; +} + +int cluster_helper_c::find_cluster(KaxCluster *cluster) { + int i; + + if (clusters == NULL) + return -1; + for (i = 0; i < num_clusters; i++) + if (clusters[i]->cluster == cluster) + return i; + return -1; +} + +void cluster_helper_c::add_cluster(KaxCluster *cluster) { + ch_contents *c; + + if (find_cluster(cluster) != -1) + return; + c = (ch_contents *)malloc(sizeof(ch_contents)); + if (c == NULL) + die("malloc"); + clusters = (ch_contents **)realloc(clusters, sizeof(ch_contents *) * + (num_clusters + 1)); + if (clusters == NULL) + die("realloc"); + memset(c, 0, sizeof(ch_contents)); + clusters[num_clusters] = c; + num_clusters++; + c->cluster = cluster; + cluster_content_size = 0; +} + +int cluster_helper_c::get_cluster_content_size() { + return cluster_content_size; +} + +int cluster_helper_c::render(IOCallback *out) { + KaxCues dummy_cues; + KaxBlockGroup *last_group = NULL; + int i; + u_int64_t cluster_timecode; + ch_contents *clstr; + packet_t *pack; + + if ((clusters == NULL) || (num_clusters == 0)) + return 0; + + walk_clusters(); + clstr = clusters[num_clusters - 1]; + cluster_timecode = get_timecode(); + + for (i = 0; i < clstr->num_packets; i++) { + pack = clstr->packets[i]; + + if (last_group == NULL) + pack->group = &GetChild(*clstr->cluster); + else + pack->group = &GetNextChild(*clstr->cluster, *last_group); + last_group = pack->group; + pack->block = &GetChild(*pack->group); + pack->data_buffer = new DataBuffer((binary *)pack->data, pack->length); + KaxTrackEntry &track_entry = + static_cast(*pack->source->track_entry); + + pack->block->AddFrame(track_entry, pack->timestamp - cluster_timecode, + *pack->data_buffer); + pack->source->added_packet_to_cluster(pack); + } + + clstr->cluster->Render(static_cast(*out), dummy_cues); + + for (i = 0; i < clstr->num_packets; i++) { + pack = clstr->packets[i]; + free(pack->data); + pack->data = NULL; + } + + clstr->rendered = 1; + + return 1; +} + +ch_contents *cluster_helper_c::find_packet_cluster(u_int64_t pid) { + int i, k; + + if (clusters == NULL) + return NULL; + + for (i = 0; i < num_clusters; i++) + for (k = 0; k < clusters[i]->num_packets; k++) + if (clusters[i]->packets[k]->id == pid) + return clusters[i]; + + return NULL; +} + +packet_t *cluster_helper_c::find_packet(u_int64_t pid) { + int i, k; + + if (clusters == NULL) + return NULL; + + for (i = 0; i < num_clusters; i++) + for (k = 0; k < clusters[i]->num_packets; k++) + if (clusters[i]->packets[k]->id == pid) + return clusters[i]->packets[k]; + + return NULL; +} + +void cluster_helper_c::check_clusters(int num) { + int i, k; + packet_t *p; + ch_contents *clstr; + + for (i = 0; i < num_clusters; i++) { + for (k = 0; k < clusters[i]->num_packets; k++) { + p = clusters[i]->packets[k]; + if (p->bref == 0) + continue; + clstr = find_packet_cluster(p->bref); + if (clstr == NULL) { + fprintf(stderr, "Error: backward refenrece could not be resolved " + "(%llu). Called from %d.\n", p->bref, num); + die("internal error"); + } + clstr->is_referenced = 1; + } + } +} + +int cluster_helper_c::free_clusters() { + int i, k, idx; + packet_t *p; + ch_contents *clstr, **new_clusters; + + if (clusters == NULL) + return 0; + + for (i = 0; i < num_clusters; i++) + clusters[i]->is_referenced = 0; + + // Part 2 - Mark all clusters that are still referenced. + for (i = 0; i < num_clusters; i++) { + for (k = 0; k < clusters[i]->num_packets; k++) { + p = clusters[i]->packets[k]; + if (!p->superseeded) + clusters[i]->is_referenced = 1; + if (p->bref == 0) + continue; + clstr = find_packet_cluster(p->bref); + if (clstr == NULL) { + fprintf(stderr, "Error: backward refenrece could not be resolved " + "(%llu).\n", p->bref); + die("internal error"); + } + clstr->is_referenced = 1; + } + } + + // Part 3 - remove all clusters and the data belonging to them that + // are not referenced anymore and that have already been rendered. + // Also count the number of clusters that are still referenced. + k = 0; + for (i = 0; i < num_clusters; i++) { + if (!clusters[i]->rendered) { + k++; + continue; + } + + if (!clusters[i]->is_referenced) { + free_contents(clusters[i]); + clusters[i] = NULL; + } else + k++; + } + + // Part 4 - prune the cluster list and remove all the entries freed in + // part 3. + if (k == 0) { + free(clusters); + num_clusters = 0; + add_cluster(new KaxCluster()); + } else if (k != num_clusters) { + new_clusters = (ch_contents **)malloc(sizeof(ch_contents *) * k); + if (new_clusters == NULL) + die("malloc"); + + idx = 0; + for (i = 0; i < num_clusters; i++) + if (clusters[i] != NULL) { + new_clusters[idx] = clusters[i]; + idx++; + } + + free(clusters); + clusters = new_clusters; + num_clusters = k; + } + + return 1; +} + +int cluster_helper_c::free_ref(u_int64_t pid) { + packet_t *p; + int l; + + p = find_packet(pid); +// fprintf(stdout, "> asked to prune %llu\n", pid); + assert(p != NULL); + free_ref_recursive(pid, p->source); + + l = num_clusters; + + walk_clusters(); + free_clusters(); + walk_clusters(); + + fprintf(stdout, "freed %d of %d, new: %d\n", l - num_clusters, l, + num_clusters); + + return 1; +} + +int cluster_helper_c::free_ref_recursive(u_int64_t pid, void *source) { + int i, k; + packet_t *p; + + if (clusters == NULL) + return 0; + + for (i = 0; i < num_clusters; i++) { + if (!clusters[i]->rendered) + continue; + for (k = 0; k < clusters[i]->num_packets; k++) { + p = clusters[i]->packets[k]; + if (p->bref == 0) + continue; + if ((source == p->source) && (p->bref <= pid)) { +// fprintf(stdout, ". cleaning %llu\n", p->id); + p->superseeded = 1; + p->bref = 0; + free_ref_recursive(p->id, source); + } + } + } + + return 1; } diff --git a/pr_generic.h b/pr_generic.h index 682cfdbcf..fd1324d41 100644 --- a/pr_generic.h +++ b/pr_generic.h @@ -13,7 +13,7 @@ /*! \file - \version \$Id: pr_generic.h,v 1.6 2003/02/27 09:52:37 mosu Exp $ + \version \$Id: pr_generic.h,v 1.7 2003/02/27 19:51:53 mosu Exp $ \brief class definition for the generic reader and packetizer \author Moritz Bunkus */ @@ -23,11 +23,13 @@ #include -#include "common.h" +#include "IOCallback.h" #include "KaxBlock.h" #include "KaxCluster.h" #include "KaxTracks.h" +#include "common.h" + using namespace LIBMATROSKA_NAMESPACE; extern KaxTracks *kax_tracks; @@ -36,26 +38,42 @@ extern int track_number; struct packet_t; +typedef struct { + KaxCluster *cluster; + packet_t **packets; + int num_packets, is_referenced, rendered; +} ch_contents; + class cluster_helper_c { private: - int refcount; - KaxCluster *cluster; - packet_t **packet_q; - int num_packets; + ch_contents **clusters; + int num_clusters, cluster_content_size; public: - cluster_helper_c(KaxCluster *ncluster = NULL); + cluster_helper_c(); virtual ~cluster_helper_c(); - KaxCluster *get_cluster(); - int add_ref(); - void add_packet(packet_t *packet); - u_int64_t get_timecode(); - packet_t *get_packet(int num); - int get_packet_count(); - int release(); - KaxCluster &operator *(); + void add_cluster(KaxCluster *cluster); + KaxCluster *get_cluster(); + void add_packet(packet_t *packet); + u_int64_t get_timecode(); + packet_t *get_packet(int num); + int get_packet_count(); + int render(IOCallback *out); + int free_ref(u_int64_t pid); + int free_clusters(); + int get_cluster_content_size(); + +private: + int find_cluster(KaxCluster *cluster); + ch_contents *find_packet_cluster(u_int64_t pid); + packet_t *find_packet(u_int64_t pid); + void free_contents(ch_contents *clstr); + void check_clusters(int num); + int free_ref_recursive(u_int64_t pid, void *source); }; +extern cluster_helper_c *cluster_helper; + class generic_packetizer_c { protected: int serialno; @@ -71,8 +89,7 @@ public: virtual void set_header() = 0; virtual stamp_t get_smallest_timestamp() = 0; virtual void set_private_data(void *data, int size); - virtual void added_packet_to_cluster(packet_t *packet, - cluster_helper_c *helper); + virtual void added_packet_to_cluster(packet_t *packet); }; class generic_reader_c { @@ -91,8 +108,8 @@ typedef struct packet_t { KaxBlock *block; KaxCluster *cluster; char *data; - int length; - u_int64_t timestamp, id, ref; + int length, superseeded; + u_int64_t timestamp, id, bref, fref; generic_packetizer_c *source; } packet_t; diff --git a/queue.cpp b/queue.cpp index 95162ca1b..bc1800643 100644 --- a/queue.cpp +++ b/queue.cpp @@ -13,7 +13,7 @@ /*! \file - \version \$Id: queue.cpp,v 1.5 2003/02/27 09:35:55 mosu Exp $ + \version \$Id: queue.cpp,v 1.6 2003/02/27 19:51:53 mosu Exp $ \brief packet queueing class used by every packetizer \author Moritz Bunkus */ @@ -31,10 +31,11 @@ #include #endif +u_int64_t q_c::id = 1; + q_c::q_c() throw (error_c) : generic_packetizer_c() { first = NULL; current = NULL; - id = 1; } q_c::~q_c() { @@ -54,7 +55,7 @@ q_c::~q_c() { } u_int64_t q_c::add_packet(char *data, int length, u_int64_t timestamp, - u_int64_t ref) { + u_int64_t bref, u_int64_t fref) { q_page_t *qpage; if (data == NULL) @@ -71,7 +72,8 @@ u_int64_t q_c::add_packet(char *data, int length, u_int64_t timestamp, memcpy(qpage->pack->data, data, length); qpage->pack->length = length; qpage->pack->timestamp = timestamp; - qpage->pack->ref = ref; + qpage->pack->bref = bref; + qpage->pack->fref = fref; qpage->pack->source = this; qpage->pack->id = id; id++; diff --git a/queue.h b/queue.h index c48f8f75f..1162b06fd 100644 --- a/queue.h +++ b/queue.h @@ -13,7 +13,7 @@ /*! \file - \version \$Id: queue.h,v 1.5 2003/02/27 09:52:37 mosu Exp $ + \version \$Id: queue.h,v 1.6 2003/02/27 19:51:53 mosu Exp $ \brief class definition for the queueing class \author Moritz Bunkus */ @@ -32,16 +32,16 @@ typedef struct q_page { class q_c: public generic_packetizer_c { private: - u_int64_t id; - struct q_page *first; - struct q_page *current; + static u_int64_t id; + struct q_page *first, *current; public: q_c() throw (error_c); virtual ~q_c(); virtual u_int64_t add_packet(char *data, int lenth, - u_int64_t timestamp, u_int64_t ref = 0); + u_int64_t timestamp, u_int64_t bref = 0, + u_int64_t fref = 0); virtual packet_t *get_packet(); virtual int packet_available(); virtual stamp_t get_smallest_timestamp();