New cluster caching system.

This commit is contained in:
Moritz Bunkus 2003-02-27 19:51:53 +00:00
parent b9982b375f
commit 97a8decce1
7 changed files with 398 additions and 139 deletions

View File

@ -13,7 +13,7 @@
/*!
\file
\version \$Id: mkvmerge.cpp,v 1.17 2003/02/27 09:35:55 mosu Exp $
\version \$Id: mkvmerge.cpp,v 1.18 2003/02/27 19:51:53 mosu Exp $
\brief command line parameter parsing, looping, output handling
\author Moritz Bunkus <moritz @ bunkus.org>
*/
@ -89,9 +89,6 @@ int max_ms_per_cluster = 1000;
float video_fps = -1.0;
packet_t **packet_queue = NULL;
int num_packets_in_packetq = 0;
cluster_helper_c *cluster_helper = NULL;
KaxSegment kax_segment;
KaxTracks *kax_tracks;
@ -735,54 +732,18 @@ static void parse_args(int argc, char **argv) {
}*/
}
static void write_packetq() {
KaxCues dummy_cues;
KaxBlockGroup *last_group = NULL;
KaxCluster *cluster;
int i, num_packets;
u_int64_t cluster_timecode;
cluster = cluster_helper->get_cluster();
cluster_timecode = cluster_helper->get_timecode();
num_packets = cluster_helper->get_packet_count();
for (i = 0; i < num_packets; i++) {
packet_t *pack;
pack = cluster_helper->get_packet(i);
if (last_group == NULL)
pack->group = &GetChild<KaxBlockGroup>(*cluster);
else
pack->group = &GetNextChild<KaxBlockGroup>(*cluster, *last_group);
last_group = pack->group;
pack->block = &GetChild<KaxBlock>(*pack->group);
pack->data_buffer = new DataBuffer((binary *)pack->data, pack->length);
KaxTrackEntry &track_entry =
static_cast<KaxTrackEntry &>(*pack->source->track_entry);
pack->block->AddFrame(track_entry, pack->timestamp - cluster_timecode,
*pack->data_buffer);
pack->source->added_packet_to_cluster(pack, cluster_helper);
}
cluster->Render(static_cast<StdIOCallback &>(*out), dummy_cues);
cluster_helper->release();
}
static int write_packet(packet_t *pack) {
u_int64_t timecode;
if (cluster_helper == NULL)
cluster_helper = new cluster_helper_c();
cluster_helper->add_packet(pack);
timecode = cluster_helper->get_timecode();
if (((pack->timestamp - timecode) > max_ms_per_cluster) ||
(num_packets_in_packetq > max_blocks_per_cluster))
write_packetq();
(cluster_helper->get_packet_count() > max_blocks_per_cluster) ||
(cluster_helper->get_cluster_content_size() > 1500000)) {
cluster_helper->render(out);
cluster_helper->add_cluster(new KaxCluster());
}
return 1;
}
@ -793,6 +754,9 @@ int main(int argc, char **argv) {
nice(2);
cluster_helper = new cluster_helper_c();
cluster_helper->add_cluster(new KaxCluster());
parse_args(argc, argv);
/* let her rip! */
@ -845,8 +809,10 @@ int main(int argc, char **argv) {
display_progress(0);
}
if (num_packets_in_packetq > 0)
write_packetq();
if ((cluster_helper != NULL) && (cluster_helper->get_packet_count() > 0))
cluster_helper->render(out);
delete cluster_helper;
if (verbose == 1) {
display_progress(1);

View File

@ -13,7 +13,7 @@
/*!
\file
\version \$Id: p_video.cpp,v 1.4 2003/02/27 09:35:55 mosu Exp $
\version \$Id: p_video.cpp,v 1.5 2003/02/27 19:51:53 mosu Exp $
\brief video output module
\author Moritz Bunkus <moritz @ bunkus.org>
*/
@ -126,9 +126,11 @@ int video_packetizer_c::process(char *buf, int size, int num_frames,
if ((packetno >= range.start) &&
((range.end == 0) || (packetno < range.end))) {
if (key)
// Add a key frame and save its ID so that we can reference it later.
last_id = add_packet(buf, size,
(u_int64_t)(1000.0 * frames_output / fps));
else
// This is a P frame - let's reference the last key frame (I frame).
add_packet(buf, size, (u_int64_t)(1000.0 * frames_output / fps),
last_id);
frames_output += num_frames;
@ -143,13 +145,13 @@ video_packetizer_c::~video_packetizer_c() {
free(tempbuf);
}
void video_packetizer_c::added_packet_to_cluster(packet_t *packet,
cluster_helper_c *helper) {
if (packet->ref == 0) { // this is a keyframe
if (last_helper)
last_helper->release();
last_helper = helper;
void video_packetizer_c::added_packet_to_cluster(packet_t *packet) {
if (packet->bref == 0) { // this is a keyframe
// Free the last key frame and all others that (indirectly) reference it.
if (last_keyframe != NULL)
cluster_helper->free_ref(last_keyframe->id);
// Save this key frame so that we can later free all references to it
// (and make references to it in the first place).
last_keyframe = packet;
last_helper->add_ref();
}
}

View File

@ -13,7 +13,7 @@
/*!
\file
\version \$Id: p_video.h,v 1.5 2003/02/27 09:52:37 mosu Exp $
\version \$Id: p_video.h,v 1.6 2003/02/27 19:51:53 mosu Exp $
\brief class definition for the video output module
\author Moritz Bunkus <moritz @ bunkus.org>
*/
@ -35,7 +35,6 @@ private:
range_t range;
u_int64_t last_id;
packet_t *last_keyframe;
cluster_helper_c *last_helper;
public:
video_packetizer_c(void *, int, char *, double, int, int, int, int,
@ -45,8 +44,7 @@ public:
virtual int process(char *buf, int size, int num_frames, int key,
int last_frame);
virtual void set_header();
virtual void added_packet_to_cluster(packet_t *packet,
cluster_helper_c *helper);
virtual void added_packet_to_cluster(packet_t *packet);
};
#endif // __P_VIDEO_H

View File

@ -13,13 +13,16 @@
/*!
\file
\version \$Id: pr_generic.cpp,v 1.3 2003/02/27 09:35:55 mosu Exp $
\version \$Id: pr_generic.cpp,v 1.4 2003/02/27 19:51:53 mosu Exp $
\brief functions common for all readers/packetizers
\author Moritz Bunkus <moritz @ bunkus.org>
*/
#include <assert.h>
#include <malloc.h>
#include "IOCallback.h"
#include "StdIOCallback.h"
#include "KaxCluster.h"
#include "KaxClusterData.h"
@ -47,8 +50,8 @@ void generic_packetizer_c::set_private_data(void *data, int size) {
private_data_size = size;
}
void generic_packetizer_c::added_packet_to_cluster(packet_t *,
cluster_helper_c *) {
void generic_packetizer_c::added_packet_to_cluster(packet_t *packet) {
packet->superseeded = 1;
}
//--------------------------------------------------------------------
@ -61,80 +64,351 @@ generic_reader_c::~generic_reader_c() {
//--------------------------------------------------------------------
cluster_helper_c::cluster_helper_c(KaxCluster *ncluster) {
num_packets = 0;
packet_q = NULL;
refcount = 1;
if (ncluster == NULL)
cluster = new KaxCluster();
else
cluster = ncluster;
//#define walk_clusters() check_clusters(__LINE__)
#define walk_clusters()
cluster_helper_c::cluster_helper_c() {
num_clusters = 0;
clusters = NULL;
cluster_content_size = 0;
}
cluster_helper_c::~cluster_helper_c() {
int i;
if (packet_q) {
for (i = 0; i < num_packets; i++) {
free(packet_q[i]->data);
delete packet_q[i]->data_buffer;
free(packet_q[i]);
}
free(packet_q);
for (i = 0; i < num_clusters; i++)
free_contents(clusters[i]);
if (clusters != NULL)
free(clusters);
}
void cluster_helper_c::free_contents(ch_contents *clstr) {
packet_t *p;
int i;
assert(clstr != NULL);
assert(clstr->cluster != NULL);
delete clstr->cluster;
assert(!((clstr->num_packets != 0) && (clstr->packets == NULL)));
for (i = 0; i < clstr->num_packets; i++) {
p = clstr->packets[i];
if (p->data != NULL)
free(p->data);
if (p->data_buffer != NULL)
delete p->data_buffer;
// fprintf(stdout, "* deleted %llu\n", p->id);
free(p);
}
if (cluster)
delete cluster;
if (clstr->packets != NULL)
free(clstr->packets);
free(clstr);
}
KaxCluster *cluster_helper_c::get_cluster() {
return cluster;
}
KaxCluster &cluster_helper_c::operator *() {
return *cluster;
}
int cluster_helper_c::add_ref() {
refcount++;
return refcount;
}
int cluster_helper_c::release() {
int ref;
refcount--;
ref = refcount;
if (refcount == 0)
delete this;
return ref;
if (clusters != NULL)
return clusters[num_clusters - 1]->cluster;
return NULL;
}
void cluster_helper_c::add_packet(packet_t *packet) {
packet_q = (packet_t **)realloc(packet_q, sizeof(packet_t *) *
(num_packets + 1));
if (packet_q == NULL)
ch_contents *c;
if (clusters == NULL)
return;
c = clusters[num_clusters - 1];
c->packets = (packet_t **)realloc(c->packets, sizeof(packet_t *) *
(c->num_packets + 1));
if (c->packets == NULL)
die("realloc");
packet_q[num_packets] = packet;
if (num_packets == 0) {
KaxClusterTimecode &timecode = GetChild<KaxClusterTimecode>(*cluster);
c->packets[c->num_packets] = packet;
if (c->num_packets == 0) {
KaxClusterTimecode &timecode = GetChild<KaxClusterTimecode>(*c->cluster);
*(static_cast<EbmlUInteger *>(&timecode)) = packet->timestamp;
}
num_packets++;
c->num_packets++;
cluster_content_size += packet->length;
// fprintf(stdout, "& new %llu\n", packet->id);
walk_clusters();
}
u_int64_t cluster_helper_c::get_timecode() {
if (packet_q == NULL)
if (clusters == NULL)
return 0;
return packet_q[0]->timestamp;
if (clusters[num_clusters - 1]->packets == NULL)
return 0;
return clusters[num_clusters - 1]->packets[0]->timestamp;
}
packet_t *cluster_helper_c::get_packet(int num) {
if (packet_q == NULL)
ch_contents *c;
if (clusters == NULL)
return NULL;
if ((num < 0) || (num > num_packets))
c = clusters[num_clusters - 1];
if (c->packets == NULL)
return NULL;
return packet_q[num];
if ((num < 0) || (num > c->num_packets))
return NULL;
return c->packets[num];
}
int cluster_helper_c::get_packet_count() {
return num_packets;
if (clusters == NULL)
return -1;
return clusters[num_clusters - 1]->num_packets;
}
int cluster_helper_c::find_cluster(KaxCluster *cluster) {
int i;
if (clusters == NULL)
return -1;
for (i = 0; i < num_clusters; i++)
if (clusters[i]->cluster == cluster)
return i;
return -1;
}
void cluster_helper_c::add_cluster(KaxCluster *cluster) {
ch_contents *c;
if (find_cluster(cluster) != -1)
return;
c = (ch_contents *)malloc(sizeof(ch_contents));
if (c == NULL)
die("malloc");
clusters = (ch_contents **)realloc(clusters, sizeof(ch_contents *) *
(num_clusters + 1));
if (clusters == NULL)
die("realloc");
memset(c, 0, sizeof(ch_contents));
clusters[num_clusters] = c;
num_clusters++;
c->cluster = cluster;
cluster_content_size = 0;
}
int cluster_helper_c::get_cluster_content_size() {
return cluster_content_size;
}
int cluster_helper_c::render(IOCallback *out) {
KaxCues dummy_cues;
KaxBlockGroup *last_group = NULL;
int i;
u_int64_t cluster_timecode;
ch_contents *clstr;
packet_t *pack;
if ((clusters == NULL) || (num_clusters == 0))
return 0;
walk_clusters();
clstr = clusters[num_clusters - 1];
cluster_timecode = get_timecode();
for (i = 0; i < clstr->num_packets; i++) {
pack = clstr->packets[i];
if (last_group == NULL)
pack->group = &GetChild<KaxBlockGroup>(*clstr->cluster);
else
pack->group = &GetNextChild<KaxBlockGroup>(*clstr->cluster, *last_group);
last_group = pack->group;
pack->block = &GetChild<KaxBlock>(*pack->group);
pack->data_buffer = new DataBuffer((binary *)pack->data, pack->length);
KaxTrackEntry &track_entry =
static_cast<KaxTrackEntry &>(*pack->source->track_entry);
pack->block->AddFrame(track_entry, pack->timestamp - cluster_timecode,
*pack->data_buffer);
pack->source->added_packet_to_cluster(pack);
}
clstr->cluster->Render(static_cast<StdIOCallback &>(*out), dummy_cues);
for (i = 0; i < clstr->num_packets; i++) {
pack = clstr->packets[i];
free(pack->data);
pack->data = NULL;
}
clstr->rendered = 1;
return 1;
}
ch_contents *cluster_helper_c::find_packet_cluster(u_int64_t pid) {
int i, k;
if (clusters == NULL)
return NULL;
for (i = 0; i < num_clusters; i++)
for (k = 0; k < clusters[i]->num_packets; k++)
if (clusters[i]->packets[k]->id == pid)
return clusters[i];
return NULL;
}
packet_t *cluster_helper_c::find_packet(u_int64_t pid) {
int i, k;
if (clusters == NULL)
return NULL;
for (i = 0; i < num_clusters; i++)
for (k = 0; k < clusters[i]->num_packets; k++)
if (clusters[i]->packets[k]->id == pid)
return clusters[i]->packets[k];
return NULL;
}
void cluster_helper_c::check_clusters(int num) {
int i, k;
packet_t *p;
ch_contents *clstr;
for (i = 0; i < num_clusters; i++) {
for (k = 0; k < clusters[i]->num_packets; k++) {
p = clusters[i]->packets[k];
if (p->bref == 0)
continue;
clstr = find_packet_cluster(p->bref);
if (clstr == NULL) {
fprintf(stderr, "Error: backward refenrece could not be resolved "
"(%llu). Called from %d.\n", p->bref, num);
die("internal error");
}
clstr->is_referenced = 1;
}
}
}
int cluster_helper_c::free_clusters() {
int i, k, idx;
packet_t *p;
ch_contents *clstr, **new_clusters;
if (clusters == NULL)
return 0;
for (i = 0; i < num_clusters; i++)
clusters[i]->is_referenced = 0;
// Part 2 - Mark all clusters that are still referenced.
for (i = 0; i < num_clusters; i++) {
for (k = 0; k < clusters[i]->num_packets; k++) {
p = clusters[i]->packets[k];
if (!p->superseeded)
clusters[i]->is_referenced = 1;
if (p->bref == 0)
continue;
clstr = find_packet_cluster(p->bref);
if (clstr == NULL) {
fprintf(stderr, "Error: backward refenrece could not be resolved "
"(%llu).\n", p->bref);
die("internal error");
}
clstr->is_referenced = 1;
}
}
// Part 3 - remove all clusters and the data belonging to them that
// are not referenced anymore and that have already been rendered.
// Also count the number of clusters that are still referenced.
k = 0;
for (i = 0; i < num_clusters; i++) {
if (!clusters[i]->rendered) {
k++;
continue;
}
if (!clusters[i]->is_referenced) {
free_contents(clusters[i]);
clusters[i] = NULL;
} else
k++;
}
// Part 4 - prune the cluster list and remove all the entries freed in
// part 3.
if (k == 0) {
free(clusters);
num_clusters = 0;
add_cluster(new KaxCluster());
} else if (k != num_clusters) {
new_clusters = (ch_contents **)malloc(sizeof(ch_contents *) * k);
if (new_clusters == NULL)
die("malloc");
idx = 0;
for (i = 0; i < num_clusters; i++)
if (clusters[i] != NULL) {
new_clusters[idx] = clusters[i];
idx++;
}
free(clusters);
clusters = new_clusters;
num_clusters = k;
}
return 1;
}
int cluster_helper_c::free_ref(u_int64_t pid) {
packet_t *p;
int l;
p = find_packet(pid);
// fprintf(stdout, "> asked to prune %llu\n", pid);
assert(p != NULL);
free_ref_recursive(pid, p->source);
l = num_clusters;
walk_clusters();
free_clusters();
walk_clusters();
fprintf(stdout, "freed %d of %d, new: %d\n", l - num_clusters, l,
num_clusters);
return 1;
}
int cluster_helper_c::free_ref_recursive(u_int64_t pid, void *source) {
int i, k;
packet_t *p;
if (clusters == NULL)
return 0;
for (i = 0; i < num_clusters; i++) {
if (!clusters[i]->rendered)
continue;
for (k = 0; k < clusters[i]->num_packets; k++) {
p = clusters[i]->packets[k];
if (p->bref == 0)
continue;
if ((source == p->source) && (p->bref <= pid)) {
// fprintf(stdout, ". cleaning %llu\n", p->id);
p->superseeded = 1;
p->bref = 0;
free_ref_recursive(p->id, source);
}
}
}
return 1;
}

View File

@ -13,7 +13,7 @@
/*!
\file
\version \$Id: pr_generic.h,v 1.6 2003/02/27 09:52:37 mosu Exp $
\version \$Id: pr_generic.h,v 1.7 2003/02/27 19:51:53 mosu Exp $
\brief class definition for the generic reader and packetizer
\author Moritz Bunkus <moritz @ bunkus.org>
*/
@ -23,11 +23,13 @@
#include <stdio.h>
#include "common.h"
#include "IOCallback.h"
#include "KaxBlock.h"
#include "KaxCluster.h"
#include "KaxTracks.h"
#include "common.h"
using namespace LIBMATROSKA_NAMESPACE;
extern KaxTracks *kax_tracks;
@ -36,26 +38,42 @@ extern int track_number;
struct packet_t;
typedef struct {
KaxCluster *cluster;
packet_t **packets;
int num_packets, is_referenced, rendered;
} ch_contents;
class cluster_helper_c {
private:
int refcount;
KaxCluster *cluster;
packet_t **packet_q;
int num_packets;
ch_contents **clusters;
int num_clusters, cluster_content_size;
public:
cluster_helper_c(KaxCluster *ncluster = NULL);
cluster_helper_c();
virtual ~cluster_helper_c();
KaxCluster *get_cluster();
int add_ref();
void add_packet(packet_t *packet);
u_int64_t get_timecode();
packet_t *get_packet(int num);
int get_packet_count();
int release();
KaxCluster &operator *();
void add_cluster(KaxCluster *cluster);
KaxCluster *get_cluster();
void add_packet(packet_t *packet);
u_int64_t get_timecode();
packet_t *get_packet(int num);
int get_packet_count();
int render(IOCallback *out);
int free_ref(u_int64_t pid);
int free_clusters();
int get_cluster_content_size();
private:
int find_cluster(KaxCluster *cluster);
ch_contents *find_packet_cluster(u_int64_t pid);
packet_t *find_packet(u_int64_t pid);
void free_contents(ch_contents *clstr);
void check_clusters(int num);
int free_ref_recursive(u_int64_t pid, void *source);
};
extern cluster_helper_c *cluster_helper;
class generic_packetizer_c {
protected:
int serialno;
@ -71,8 +89,7 @@ public:
virtual void set_header() = 0;
virtual stamp_t get_smallest_timestamp() = 0;
virtual void set_private_data(void *data, int size);
virtual void added_packet_to_cluster(packet_t *packet,
cluster_helper_c *helper);
virtual void added_packet_to_cluster(packet_t *packet);
};
class generic_reader_c {
@ -91,8 +108,8 @@ typedef struct packet_t {
KaxBlock *block;
KaxCluster *cluster;
char *data;
int length;
u_int64_t timestamp, id, ref;
int length, superseeded;
u_int64_t timestamp, id, bref, fref;
generic_packetizer_c *source;
} packet_t;

View File

@ -13,7 +13,7 @@
/*!
\file
\version \$Id: queue.cpp,v 1.5 2003/02/27 09:35:55 mosu Exp $
\version \$Id: queue.cpp,v 1.6 2003/02/27 19:51:53 mosu Exp $
\brief packet queueing class used by every packetizer
\author Moritz Bunkus <moritz @ bunkus.org>
*/
@ -31,10 +31,11 @@
#include <dmalloc.h>
#endif
u_int64_t q_c::id = 1;
q_c::q_c() throw (error_c) : generic_packetizer_c() {
first = NULL;
current = NULL;
id = 1;
}
q_c::~q_c() {
@ -54,7 +55,7 @@ q_c::~q_c() {
}
u_int64_t q_c::add_packet(char *data, int length, u_int64_t timestamp,
u_int64_t ref) {
u_int64_t bref, u_int64_t fref) {
q_page_t *qpage;
if (data == NULL)
@ -71,7 +72,8 @@ u_int64_t q_c::add_packet(char *data, int length, u_int64_t timestamp,
memcpy(qpage->pack->data, data, length);
qpage->pack->length = length;
qpage->pack->timestamp = timestamp;
qpage->pack->ref = ref;
qpage->pack->bref = bref;
qpage->pack->fref = fref;
qpage->pack->source = this;
qpage->pack->id = id;
id++;

10
queue.h
View File

@ -13,7 +13,7 @@
/*!
\file
\version \$Id: queue.h,v 1.5 2003/02/27 09:52:37 mosu Exp $
\version \$Id: queue.h,v 1.6 2003/02/27 19:51:53 mosu Exp $
\brief class definition for the queueing class
\author Moritz Bunkus <moritz @ bunkus.org>
*/
@ -32,16 +32,16 @@ typedef struct q_page {
class q_c: public generic_packetizer_c {
private:
u_int64_t id;
struct q_page *first;
struct q_page *current;
static u_int64_t id;
struct q_page *first, *current;
public:
q_c() throw (error_c);
virtual ~q_c();
virtual u_int64_t add_packet(char *data, int lenth,
u_int64_t timestamp, u_int64_t ref = 0);
u_int64_t timestamp, u_int64_t bref = 0,
u_int64_t fref = 0);
virtual packet_t *get_packet();
virtual int packet_available();
virtual stamp_t get_smallest_timestamp();