Download-Tools/ccextractor
1
0
Fork 0
mirror of https://github.com/CCExtractor/ccextractor.git synced 2024-12-25 04:11:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixes #425 - the 708 decoder needs access the encoder. Reference was missing for .bin.

Browse Source
This commit is contained in:
Carlos Fernandez 2016-09-20 16:04:33 -07:00
parent 9833090318
commit 4101fe3880
14 changed files with 16875 additions and 523 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/lib_ccx/ccx_common_option.h
View File

@ -46,6 +46,7 @@ struct encoder_cfg
int trim_subs; // " Remove spaces at sides? "
int sentence_cap ; // FIX CASE? = Fix case?
int splitbysentence; // Split text into complete sentences and prorate time?
int with_semaphore; // Write a .sem file on file open and delete it on close?
/* Credit stuff */
char *start_credits_text;

2
src/lib_ccx/ccx_decoders_xds.c
View File

@ -734,8 +734,8 @@ int xds_do_current_and_future (struct cc_subtitle *sub, struct ccx_decoders_xds_
ccx_common_logging.debug_ftn(CCX_DMT_DECODER_XDS, "\rXDS Notice: Aspect ratio info, start line=%u, end line=%u\n", ar_start, ar_end);
ccx_common_logging.debug_ftn(CCX_DMT_DECODER_XDS, "\rXDS Notice: Aspect ratio info, active picture height=%u, ratio=%f\n", active_picture_height, aspect_ratio);
}
}
break;
case XDS_TYPE_PROGRAM_DESC_1:
case XDS_TYPE_PROGRAM_DESC_2:
case XDS_TYPE_PROGRAM_DESC_3:

693
src/lib_ccx/ccx_encoders_common.c
View File

@ -535,130 +535,6 @@ int write_cc_subtitle_as_simplexml(struct cc_subtitle *sub, struct encoder_ctx *
return ret;
}
int write_cc_subtitle_as_transcript(struct cc_subtitle *sub, struct encoder_ctx *context)
{
int length;
int ret = 0;
LLONG start_time = -1;
LLONG end_time = -1;
char *str;
char *save_str;
struct cc_subtitle *osub = sub;
struct cc_subtitle *lsub = sub;
while(sub)
{
if(sub->type == CC_TEXT)
{
start_time = sub->start_time;
end_time = sub->end_time;
}
if (context->sentence_cap)
{
//TODO capitalize (context, line_number,data);
//TODO correct_case_with_dictionary(line_number, data);
}
if (start_time == -1)
{
// CFS: Means that the line has characters but we don't have a timestamp for the first one. Since the timestamp
// is set for example by the write_char function, it possible that we don't have one in empty lines (unclear)
// For now, let's not consider this a bug as before and just return.
// fatal (EXIT_BUG_BUG, "Bug in timedtranscript (ts_start_of_current_line==-1). Please report.");
return 0;
}
str = sub->data;
str = strtok_r(str, "\r\n", &save_str);
do
{
length = get_str_basic(context->subline, (unsigned char*)str, context->trim_subs, sub->enc_type, context->encoding, strlen(str));
if (length <= 0)
{
continue;
}
if (context->transcript_settings->showStartTime)
{
char buf[80];
if (context->transcript_settings->relativeTimestamp)
{
millis_to_date(start_time + context->subs_delay, buf, context->date_format, context->millis_separator);
fdprintf(context->out->fh, "%s|", buf);
}
else
{
time_t start_time_int = (start_time + context->subs_delay) / 1000;
int start_time_dec = (start_time + context->subs_delay) % 1000;
struct tm *start_time_struct = gmtime(&start_time_int);
strftime(buf, sizeof(buf), "%Y%m%d%H%M%S", start_time_struct);
fdprintf(context->out->fh, "%s%c%03d|", buf, context->millis_separator, start_time_dec);
}
}
if (context->transcript_settings->showEndTime)
{
char buf[80];
if (context->transcript_settings->relativeTimestamp)
{
millis_to_date(end_time + context->subs_delay, buf, context->date_format, context->millis_separator);
fdprintf(context->out->fh, "%s|", buf);
}
else
{
time_t end_time_int = (end_time + context->subs_delay) / 1000;
int end_time_dec = (end_time + context->subs_delay) % 1000;
struct tm *end_time_struct = gmtime(&end_time_int);
strftime(buf, sizeof(buf), "%Y%m%d%H%M%S", end_time_struct);
fdprintf(context->out->fh, "%s%c%03d|", buf, context->millis_separator, end_time_dec);
}
}
if (context->transcript_settings->showCC)
{
if(context->in_fileformat == 1)
//TODO, data->my_field == 1 ? data->channel : data->channel + 2); // Data from field 2 is CC3 or 4
fdprintf(context->out->fh, "CC?|");
else if (!context->ucla || !strcmp(sub->mode,"TLT"))
fdprintf(context->out->fh, sub->info);
}
if (context->transcript_settings->showMode)
{
if(context->ucla && strcmp(sub->mode,"TLT") == 0)
fdprintf(context->out->fh, "|");
else
fdprintf(context->out->fh, "%s|", sub->mode);
}
ret = write(context->out->fh, context->subline, length);
if(ret < length)
{
mprint("Warning:Loss of data\n");
}
ret = write(context->out->fh, context->encoded_crlf, context->encoded_crlf_length);
if(ret < context->encoded_crlf_length)
{
mprint("Warning:Loss of data\n");
}
} while ( (str = strtok_r(NULL, "\r\n", &save_str)) );
freep(&sub->data);
lsub = sub;
sub = sub->next;
}
while(lsub != osub)
{
sub = lsub->prev;
freep(&lsub);
lsub = sub;
}
return ret;
}
void write_cc_line_as_simplexml(struct eia608_screen *data, struct encoder_ctx *context, int line_number)
{
int ret;
@ -683,112 +559,7 @@ void write_cc_line_as_simplexml(struct eia608_screen *data, struct encoder_ctx *
ret = write(context->out->fh, context->encoded_crlf, context->encoded_crlf_length);
}
//TODO Convert CC line to TEXT format and remove this function
void write_cc_line_as_transcript2(struct eia608_screen *data, struct encoder_ctx *context, int line_number)
{
int ret = 0;
LLONG start_time = data->start_time;
LLONG end_time = data->end_time;
if (context->sentence_cap)
{
if (clever_capitalize (context, line_number, data))
correct_case_with_dictionary(line_number, data);
}
int length = get_str_basic (context->subline, data->characters[line_number],
context->trim_subs, CCX_ENC_ASCII, context->encoding, CCX_DECODER_608_SCREEN_WIDTH);
if (context->encoding!=CCX_ENC_UNICODE)
{
dbg_print(CCX_DMT_DECODER_608, "\r");
dbg_print(CCX_DMT_DECODER_608, "%s\n",context->subline);
}
if (length>0)
{
if (data->start_time == -1)
{
// CFS: Means that the line has characters but we don't have a timestamp for the first one. Since the timestamp
// is set for example by the write_char function, it possible that we don't have one in empty lines (unclear)
// For now, let's not consider this a bug as before and just return.
// fatal (EXIT_BUG_BUG, "Bug in timedtranscript (ts_start_of_current_line==-1). Please report.");
return;
}
if (context->transcript_settings->showStartTime){
char buf1[80];
if (context->transcript_settings->relativeTimestamp){
millis_to_date(start_time + context->subs_delay, buf1, context->date_format, context->millis_separator);
fdprintf(context->out->fh, "%s|", buf1);
}
else {
time_t start_time_int = (start_time + context->subs_delay) / 1000;
int start_time_dec = (start_time + context->subs_delay) % 1000;
struct tm *start_time_struct = gmtime(&start_time_int);
strftime(buf1, sizeof(buf1), "%Y%m%d%H%M%S", start_time_struct);
fdprintf(context->out->fh, "%s%c%03d|", buf1,context->millis_separator,start_time_dec);
}
}
if (context->transcript_settings->showEndTime){
char buf2[80];
if (context->transcript_settings->relativeTimestamp){
millis_to_date(end_time, buf2, context->date_format, context->millis_separator);
fdprintf(context->out->fh, "%s|", buf2);
}
else {
time_t end_time_int = end_time / 1000;
int end_time_dec = end_time % 1000;
struct tm *end_time_struct = gmtime(&end_time_int);
strftime(buf2, sizeof(buf2), "%Y%m%d%H%M%S", end_time_struct);
fdprintf(context->out->fh, "%s%c%03d|", buf2,context->millis_separator,end_time_dec);
}
}
if (context->transcript_settings->showCC){
fdprintf(context->out->fh, "CC%d|", data->my_field == 1 ? data->channel : data->channel + 2); // Data from field 2 is CC3 or 4
}
if (context->transcript_settings->showMode){
const char *mode = "???";
switch (data->mode)
{
case MODE_POPON:
mode = "POP";
break;
case MODE_FAKE_ROLLUP_1:
mode = "RU1";
break;
case MODE_ROLLUP_2:
mode = "RU2";
break;
case MODE_ROLLUP_3:
mode = "RU3";
break;
case MODE_ROLLUP_4:
mode = "RU4";
break;
case MODE_TEXT:
mode = "TXT";
break;
case MODE_PAINTON:
mode = "PAI";
break;
}
fdprintf(context->out->fh, "%s|", mode);
}
ret = write(context->out->fh, context->subline, length);
if(ret < length)
{
mprint("Warning:Loss of data\n");
}
ret = write(context->out->fh, context->encoded_crlf, context->encoded_crlf_length);
if(ret < context->encoded_crlf_length)
{
mprint("Warning:Loss of data\n");
}
}
// fprintf (wb->fh,encoded_crlf);
}
int write_cc_buffer_as_simplexml(struct eia608_screen *data, struct encoder_ctx *context)
{
@ -805,22 +576,6 @@ int write_cc_buffer_as_simplexml(struct eia608_screen *data, struct encoder_ctx
return wrote_something;
}
int write_cc_buffer_as_transcript2(struct eia608_screen *data, struct encoder_ctx *context)
{
int wrote_something = 0;
dbg_print(CCX_DMT_DECODER_608, "\n- - - TRANSCRIPT caption - - -\n");
for (int i=0;i<15;i++)
{
if (data->row_used[i])
{
write_cc_line_as_transcript2 (data, context, i);
}
wrote_something=1;
}
dbg_print(CCX_DMT_DECODER_608, "- - - - - - - - - - - -\r\n");
return wrote_something;
}
//Dummy Function for support DVB in simple xml
int write_cc_bitmap_as_simplexml(struct cc_subtitle *sub, struct encoder_ctx *context)
@ -832,118 +587,6 @@ int write_cc_bitmap_as_simplexml(struct cc_subtitle *sub, struct encoder_ctx *co
return ret;
}
int write_cc_bitmap_as_transcript(struct cc_subtitle *sub, struct encoder_ctx *context)
{
int ret = 0;
#ifdef ENABLE_OCR
struct cc_bitmap* rect;
unsigned h1,m1,s1,ms1;
unsigned h2,m2,s2,ms2;
LLONG start_time, end_time;
if (context->prev_start != -1 && (sub->flags & SUB_EOD_MARKER))
{
start_time = context->prev_start + context->subs_delay;
end_time = sub->start_time - 1;
}
else if ( !(sub->flags & SUB_EOD_MARKER))
{
start_time = sub->start_time + context->subs_delay;
end_time = sub->end_time - 1;
}
if(sub->nb_data == 0 )
return ret;
rect = sub->data;
if ( sub->flags & SUB_EOD_MARKER )
context->prev_start = sub->start_time;
if (rect[0].ocr_text && *(rect[0].ocr_text))
{
if (context->prev_start != -1 || !(sub->flags & SUB_EOD_MARKER))
{
char *token = NULL;
token = paraof_ocrtext(sub, context->encoded_crlf, context->encoded_crlf_length);
if (context->transcript_settings->showStartTime)
{
char buf1[80];
if (context->transcript_settings->relativeTimestamp)
{
millis_to_date(start_time + context->subs_delay, buf1, context->date_format, context->millis_separator);
fdprintf(context->out->fh, "%s|", buf1);
}
else
{
mstotime(start_time + context->subs_delay, &h1, &m1, &s1, &ms1);
time_t start_time_int = (start_time + context->subs_delay) / 1000;
int start_time_dec = (start_time + context->subs_delay) % 1000;
struct tm *start_time_struct = gmtime(&start_time_int);
strftime(buf1, sizeof(buf1), "%Y%m%d%H%M%S", start_time_struct);
fdprintf(context->out->fh, "%s%c%03d|", buf1,context->millis_separator,start_time_dec);
}
}
if (context->transcript_settings->showEndTime)
{
char buf2[80];
if (context->transcript_settings->relativeTimestamp)
{
millis_to_date(end_time, buf2, context->date_format, context->millis_separator);
fdprintf(context->out->fh, "%s|", buf2);
}
else
{
time_t end_time_int = end_time / 1000;
int end_time_dec = end_time % 1000;
struct tm *end_time_struct = gmtime(&end_time_int);
strftime(buf2, sizeof(buf2), "%Y%m%d%H%M%S", end_time_struct);
fdprintf(context->out->fh, "%s%c%03d|", buf2,context->millis_separator,end_time_dec);
}
}
if (context->transcript_settings->showCC)
{
fdprintf(context->out->fh,"%s|",language[sub->lang_index]);
}
if (context->transcript_settings->showMode)
{
fdprintf(context->out->fh,"DVB|");
}
while(token)
{
char *newline_pos = strstr(token, context->encoded_crlf);
if(!newline_pos)
{
fdprintf(context->out->fh,"%s",token);
break;
}
else
{
while(token!=newline_pos)
{
fdprintf(context->out->fh,"%c", *token);
token++;
}
token+=context->encoded_crlf_length;
fdprintf(context->out->fh,"%c", ' ');
}
}
write(context->out->fh, context->encoded_crlf, context->encoded_crlf_length);
}
}
#endif
sub->nb_data = 0;
freep(&sub->data);
return ret;
}
/**
* @brief Function to add credits at end of subtitles File
@ -1263,8 +906,7 @@ struct encoder_ctx *init_encoder(struct encoder_cfg *opt)
ctx->write_format = opt->write_format;
ctx->transcript_settings = &opt->transcript_settings;
ctx->no_bom = opt->no_bom;
ctx->sentence_cap = opt->sentence_cap;
ctx->splitbysentence = opt->splitbysentence;
ctx->sentence_cap = opt->sentence_cap;
ctx->trim_subs = opt->trim_subs;
ctx->autodash = opt->autodash;
ctx->no_font_color = opt->no_font_color;
@ -1274,6 +916,15 @@ struct encoder_ctx *init_encoder(struct encoder_cfg *opt)
ctx->keep_output_closed = opt->keep_output_closed;
ctx->force_flush = opt->force_flush;
ctx->ucla = opt->ucla;
ctx->splitbysentence = opt->splitbysentence;
ctx->sbs_newblock_start_time = -1;
ctx->sbs_newblock_end_time = -1;
ctx->sbs_newblock = NULL;
ctx->sbs_newblock_capacity = 0;
ctx->sbs_newblock_size = 0;
ctx->sbs_buffer = NULL;
ctx->sbs_buffer_capacity = 0;
ctx->sbs_buffer_size = 0;
ctx->subline = (unsigned char *) malloc (SUBLINESIZE);
if(!ctx->subline)
@ -1349,177 +1000,187 @@ int encode_sub(struct encoder_ctx *context, struct cc_subtitle *sub)
ccx_share_send(sub);
#endif //ENABLE_SHARING
if (sub->type == CC_608)
if (context->splitbysentence)
{
struct eia608_screen *data = NULL;
struct ccx_s_write *out;
for(data = sub->data; sub->nb_data ; sub->nb_data--,data++)
// Write to a buffer that is later s+plit to generate split
// in sentences
if (sub->type == CC_BITMAP)
wrote_something = write_cc_bitmap_to_sentence_buffer(sub, context);
}
else
{
// Write subtitles as they come
if (sub->type == CC_608)
{
// Determine context based on channel. This replaces the code that was above, as this was incomplete (for cases where -12 was used for example)
out = get_output_ctx(context, data->my_field);
if(data->format == SFORMAT_XDS)
struct eia608_screen *data = NULL;
struct ccx_s_write *out;
for (data = sub->data; sub->nb_data; sub->nb_data--, data++)
{
data->end_time = data->end_time + context->subs_delay;
xds_write_transcript_line_prefix (context, out, data->start_time, data->end_time, data->cur_xds_packet_class);
if(data->xds_len > 0)
{
ret = write (out->fh, data->xds_str, data->xds_len);
if (ret < data->xds_len)
{
mprint("WARNING:Loss of data\n");
}
}
freep (&data->xds_str);
write_newline(context, 0);
continue;
}
// Determine context based on channel. This replaces the code that was above, as this was incomplete (for cases where -12 was used for example)
out = get_output_ctx(context, data->my_field);
data->end_time = data->end_time + context->subs_delay;
if (data->format == SFORMAT_XDS)
{
data->end_time = data->end_time + context->subs_delay;
xds_write_transcript_line_prefix(context, out, data->start_time, data->end_time, data->cur_xds_packet_class);
if (data->xds_len > 0)
{
ret = write(out->fh, data->xds_str, data->xds_len);
if (ret < data->xds_len)
{
mprint("WARNING:Loss of data\n");
}
}
freep(&data->xds_str);
write_newline(context, 0);
continue;
}
data->end_time = data->end_time + context->subs_delay;
switch (context->write_format)
{
case CCX_OF_SRT:
if (!context->startcredits_displayed && context->start_credits_text != NULL)
try_to_add_start_credits(context, data->start_time);
wrote_something = write_cc_buffer_as_srt(data, context);
break;
case CCX_OF_G608:
wrote_something = write_cc_buffer_as_g608(data, context);
break;
case CCX_OF_WEBVTT:
if (!context->startcredits_displayed && context->start_credits_text != NULL)
try_to_add_start_credits(context, data->start_time);
wrote_something = write_cc_buffer_as_webvtt(data, context);
break;
case CCX_OF_SAMI:
if (!context->startcredits_displayed && context->start_credits_text != NULL)
try_to_add_start_credits(context, data->start_time);
wrote_something = write_cc_buffer_as_sami(data, context);
break;
case CCX_OF_SMPTETT:
if (!context->startcredits_displayed && context->start_credits_text != NULL)
try_to_add_start_credits(context, data->start_time);
wrote_something = write_cc_buffer_as_smptett(data, context);
break;
case CCX_OF_TRANSCRIPT:
wrote_something = write_cc_buffer_as_transcript2(data, context);
break;
case CCX_OF_SPUPNG:
wrote_something = write_cc_buffer_as_spupng(data, context);
break;
case CCX_OF_SIMPLE_XML:
if (ccx_options.keep_output_closed && context->out->temporarily_closed)
{
temporarily_open_output(context->out);
write_subtitle_file_header(context, context->out);
}
wrote_something = write_cc_buffer_as_simplexml(data, context);
if (ccx_options.keep_output_closed)
{
write_subtitle_file_footer(context, context->out);
temporarily_close_output(context->out);
}
break;
default:
break;
}
if (wrote_something)
context->last_displayed_subs_ms = data->end_time;
if (context->gui_mode_reports)
write_cc_buffer_to_gui(sub->data, context);
}
freep(&sub->data);
}
if (sub->type == CC_BITMAP)
{
switch (context->write_format)
{
case CCX_OF_SRT:
if (!context->startcredits_displayed && context->start_credits_text!=NULL)
try_to_add_start_credits(context, data->start_time);
wrote_something = write_cc_buffer_as_srt(data, context);
break;
case CCX_OF_G608:
wrote_something = write_cc_buffer_as_g608(data, context);
break;
case CCX_OF_WEBVTT:
if (!context->startcredits_displayed && context->start_credits_text != NULL)
try_to_add_start_credits(context, data->start_time);
wrote_something = write_cc_buffer_as_webvtt(data, context);
break;
case CCX_OF_SAMI:
if (!context->startcredits_displayed && context->start_credits_text!=NULL)
try_to_add_start_credits(context, data->start_time);
wrote_something = write_cc_buffer_as_sami(data, context);
break;
case CCX_OF_SMPTETT:
if (!context->startcredits_displayed && context->start_credits_text!=NULL)
try_to_add_start_credits(context, data->start_time);
wrote_something = write_cc_buffer_as_smptett(data, context);
break;
case CCX_OF_TRANSCRIPT:
wrote_something = write_cc_buffer_as_transcript2(data, context);
break;
case CCX_OF_SPUPNG:
wrote_something = write_cc_buffer_as_spupng(data, context);
break;
case CCX_OF_SIMPLE_XML:
if (ccx_options.keep_output_closed && context->out->temporarily_closed)
{
temporarily_open_output(context->out);
write_subtitle_file_header(context, context->out);
}
wrote_something = write_cc_buffer_as_simplexml(data, context);
if (ccx_options.keep_output_closed)
{
write_subtitle_file_footer(context, context->out);
temporarily_close_output(context->out);
}
break;
default:
break;
case CCX_OF_SRT:
if (!context->startcredits_displayed && context->start_credits_text != NULL)
try_to_add_start_credits(context, sub->start_time);
wrote_something = write_cc_bitmap_as_srt(sub, context);
break;
case CCX_OF_WEBVTT:
if (!context->startcredits_displayed && context->start_credits_text != NULL)
try_to_add_start_credits(context, sub->start_time);
wrote_something = write_cc_bitmap_as_webvtt(sub, context);
break;
case CCX_OF_SAMI:
if (!context->startcredits_displayed && context->start_credits_text != NULL)
try_to_add_start_credits(context, sub->start_time);
wrote_something = write_cc_bitmap_as_sami(sub, context);
break;
case CCX_OF_SMPTETT:
if (!context->startcredits_displayed && context->start_credits_text != NULL)
try_to_add_start_credits(context, sub->start_time);
wrote_something = write_cc_bitmap_as_smptett(sub, context);
break;
case CCX_OF_TRANSCRIPT:
wrote_something = write_cc_bitmap_as_transcript(sub, context);
break;
case CCX_OF_SPUPNG:
wrote_something = write_cc_bitmap_as_spupng(sub, context);
break;
case CCX_OF_SIMPLE_XML:
wrote_something = write_cc_bitmap_as_simplexml(sub, context);
break;
default:
break;
}
if (wrote_something)
context->last_displayed_subs_ms = data->end_time;
if (context->gui_mode_reports)
write_cc_buffer_to_gui(sub->data, context);
}
freep(&sub->data);
}
if(sub->type == CC_BITMAP)
{
switch (context->write_format)
if (sub->type == CC_RAW)
{
case CCX_OF_SRT:
if (!context->startcredits_displayed && context->start_credits_text!=NULL)
try_to_add_start_credits(context, sub->start_time);
wrote_something = write_cc_bitmap_as_srt(sub, context);
break;
case CCX_OF_WEBVTT:
if (!context->startcredits_displayed && context->start_credits_text != NULL)
try_to_add_start_credits(context, sub->start_time);
wrote_something = write_cc_bitmap_as_webvtt(sub, context);
break;
case CCX_OF_SAMI:
if (!context->startcredits_displayed && context->start_credits_text!=NULL)
try_to_add_start_credits(context, sub->start_time);
wrote_something = write_cc_bitmap_as_sami(sub, context);
break;
case CCX_OF_SMPTETT:
if (!context->startcredits_displayed && context->start_credits_text!=NULL)
try_to_add_start_credits(context, sub->start_time);
wrote_something = write_cc_bitmap_as_smptett(sub, context);
break;
case CCX_OF_TRANSCRIPT:
wrote_something = write_cc_bitmap_as_transcript(sub, context);
break;
case CCX_OF_SPUPNG:
wrote_something = write_cc_bitmap_as_spupng(sub, context);
break;
case CCX_OF_SIMPLE_XML:
wrote_something = write_cc_bitmap_as_simplexml(sub, context);
break;
default:
break;
}
}
if (sub->type == CC_RAW)
{
if (context->send_to_srv)
net_send_header(sub->data, sub->nb_data);
else
{
ret = write(context->out->fh, sub->data, sub->nb_data);
if ( ret < sub->nb_data) {
mprint("WARNING: Loss of data\n");
if (context->send_to_srv)
net_send_header(sub->data, sub->nb_data);
else
{
ret = write(context->out->fh, sub->data, sub->nb_data);
if (ret < sub->nb_data) {
mprint("WARNING: Loss of data\n");
}
}
sub->nb_data = 0;
}
sub->nb_data = 0;
}
if(sub->type == CC_TEXT)
{
switch (context->write_format)
if (sub->type == CC_TEXT)
{
case CCX_OF_SRT:
if (!context->startcredits_displayed && context->start_credits_text!=NULL)
try_to_add_start_credits(context, sub->start_time);
wrote_something = write_cc_subtitle_as_srt(sub, context);
break;
case CCX_OF_WEBVTT:
if (!context->startcredits_displayed && context->start_credits_text != NULL)
try_to_add_start_credits(context, sub->start_time);
wrote_something = write_cc_subtitle_as_webvtt(sub, context);
break;
case CCX_OF_SAMI:
if (!context->startcredits_displayed && context->start_credits_text!=NULL)
try_to_add_start_credits(context, sub->start_time);
wrote_something = write_cc_subtitle_as_sami(sub, context);
break;
case CCX_OF_SMPTETT:
if (!context->startcredits_displayed && context->start_credits_text!=NULL)
try_to_add_start_credits(context, sub->start_time);
wrote_something = write_cc_subtitle_as_smptett(sub, context);
break;
case CCX_OF_TRANSCRIPT:
wrote_something = write_cc_subtitle_as_transcript(sub, context);
break;
case CCX_OF_SPUPNG:
wrote_something = write_cc_subtitle_as_spupng(sub, context);
break;
case CCX_OF_SIMPLE_XML:
wrote_something = write_cc_subtitle_as_simplexml(sub, context);
break;
default:
break;
switch (context->write_format)
{
case CCX_OF_SRT:
if (!context->startcredits_displayed && context->start_credits_text != NULL)
try_to_add_start_credits(context, sub->start_time);
wrote_something = write_cc_subtitle_as_srt(sub, context);
break;
case CCX_OF_WEBVTT:
if (!context->startcredits_displayed && context->start_credits_text != NULL)
try_to_add_start_credits(context, sub->start_time);
wrote_something = write_cc_subtitle_as_webvtt(sub, context);
break;
case CCX_OF_SAMI:
if (!context->startcredits_displayed && context->start_credits_text != NULL)
try_to_add_start_credits(context, sub->start_time);
wrote_something = write_cc_subtitle_as_sami(sub, context);
break;
case CCX_OF_SMPTETT:
if (!context->startcredits_displayed && context->start_credits_text != NULL)
try_to_add_start_credits(context, sub->start_time);
wrote_something = write_cc_subtitle_as_smptett(sub, context);
break;
case CCX_OF_TRANSCRIPT:
wrote_something = write_cc_subtitle_as_transcript(sub, context);
break;
case CCX_OF_SPUPNG:
wrote_something = write_cc_subtitle_as_spupng(sub, context);
break;
case CCX_OF_SIMPLE_XML:
wrote_something = write_cc_subtitle_as_simplexml(sub, context);
break;
default:
break;
}
sub->nb_data = 0;
}
sub->nb_data = 0;
}
if (!sub->nb_data)
freep(&sub->data);

21
src/lib_ccx/ccx_encoders_common.h
View File

@ -24,6 +24,14 @@ typedef struct ccx_dtvcc_writer_ctx
iconv_t cd;
} ccx_dtvcc_writer_ctx;
typedef struct ccx_sbs_utf8_character
{
int32_t ch;
LLONG ts;
char encoded[4];
int enc_len;
} ccx_sbs_utf8_character;
/**
* Context of encoder, This structure gives single interface
* to all encoder
@ -68,7 +76,7 @@ struct encoder_ctx
struct ccx_encoders_transcript_format *transcript_settings; // Keeps the settings for generating transcript output files.
int no_bom;
int sentence_cap ; // FIX CASE? = Fix case?
int splitbysentence;
int trim_subs; // " Remove spaces at sides? "
int autodash; // Add dashes (-) before each speaker automatically?
int no_font_color;
@ -107,7 +115,16 @@ struct encoder_ctx
int program_number;
struct list_head list;
/* Buffers for the line-by-line implementation. */
/* split-by-sentence stuff */
int splitbysentence;
LLONG sbs_newblock_start_time; // Used by the split-by-sentence code to know when the current block starts...
LLONG sbs_newblock_end_time; // ... and ends
ccx_sbs_utf8_character *sbs_newblock;
int sbs_newblock_capacity;
int sbs_newblock_size;
ccx_sbs_utf8_character *sbs_buffer;
int sbs_buffer_capacity;
int sbs_buffer_size;
};

135
src/lib_ccx/ccx_encoders_splitbysentence.c Normal file
View File

@ -0,0 +1,135 @@
#include "ccx_decoders_common.h"
#include "ccx_encoders_common.h"
#include "spupng_encoder.h"
#include "ccx_encoders_spupng.h"
#include "utility.h"
#include "ocr.h"
#include "ccx_decoders_608.h"
#include "ccx_decoders_708.h"
#include "ccx_decoders_708_output.h"
#include "ccx_encoders_xds.h"
#include "ccx_encoders_helpers.h"
#include "utf8proc.h"
#ifdef ENABLE_SHARING
#include "ccx_share.h"
#endif //ENABLE_SHARING
void lbl_start_block(LLONG start_time, struct encoder_ctx *context)
{
context->sbs_newblock_start_time = start_time;
}
void lbl_add_character(struct encoder_ctx *context, ccx_sbs_utf8_character ch)
{
if (context->sbs_newblock_capacity == context->sbs_newblock_size)
{
int newcapacity = (context->sbs_newblock_capacity < 512) ? 1024 : context->sbs_newblock_capacity * 2;
context->sbs_newblock = (ccx_sbs_utf8_character *)realloc(context->sbs_newblock, newcapacity*sizeof(ccx_sbs_utf8_character));
if (!context->sbs_newblock)
fatal(EXIT_NOT_ENOUGH_MEMORY, "Not enough memory in lbl_add_character");
context->sbs_newblock_capacity = newcapacity;
}
memcpy(&context->sbs_newblock[context->sbs_newblock_size++], &ch, sizeof ch);
}
void lbl_end_block(LLONG end_time, struct encoder_ctx *context)
{
context->sbs_newblock_end_time = end_time;
}
int write_cc_bitmap_to_sentence_buffer(struct cc_subtitle *sub, struct encoder_ctx *context)
{
int ret = 0;
#ifdef ENABLE_OCR
struct cc_bitmap* rect;
LLONG ms_start, ms_end;
if (context->prev_start != -1 && (sub->flags & SUB_EOD_MARKER))
{
ms_start = context->prev_start;
ms_end = sub->start_time;
}
else if (!(sub->flags & SUB_EOD_MARKER))
{
ms_start = sub->start_time;
ms_end = sub->end_time;
}
else if (context->prev_start == -1 && (sub->flags & SUB_EOD_MARKER))
{
ms_start = 1;
ms_end = sub->start_time;
}
if (sub->nb_data == 0)
return ret;
rect = sub->data;
if (sub->flags & SUB_EOD_MARKER)
context->prev_start = sub->start_time;
if (rect[0].ocr_text && *(rect[0].ocr_text))
{
lbl_start_block(ms_start, context);
if (context->prev_start != -1 || !(sub->flags & SUB_EOD_MARKER))
{
char *token = NULL;
token = paraof_ocrtext(sub, " ", 1); // Get text with spaces instead of newlines
uint32_t offset=0;
utf8proc_ssize_t ls; // Last size
char *s = token;
int32_t uc;
while ((ls=utf8proc_iterate(s, -1, &uc)))
{
ccx_sbs_utf8_character sbsc;
// Note: We don't care about uc here, since we will be writing the encoded bytes, not the code points in binary.
//TODO: Deal with ls < 0
if (!uc) // End of string
break;
printf("%3d | %08X | %c %c %c %c\n", ls, uc, ((uc & 0xFF000000) >> 24), ((uc & 0xFF0000) >> 16),
((uc & 0xFF00) >> 8), ( uc & 0xFF));
sbsc.ch = uc;
sbsc.encoded[0] = 0; sbsc.encoded[1] = 0; sbsc.encoded[2] = 0; sbsc.encoded[3] = 0;
memcpy(sbsc.encoded, s, ls);
sbsc.enc_len = ls;
sbsc.ts = 0; // We don't know yet
lbl_add_character(context, sbsc);
s += ls;
// TO-DO: Add each of these characters to the buffer, splitting the timestamps. Remember to add character length to the array
}
printf("-------\n");
/*
while (token)
{
char *newline_pos = strstr(token, context->encoded_crlf);
if (!newline_pos)
{
fdprintf(context->out->fh, "%s", token);
break;
}
else
{
while (token != newline_pos)
{
fdprintf(context->out->fh, "%c", *token);
token++;
}
token += context->encoded_crlf_length;
fdprintf(context->out->fh, "%c", ' ');
}
}*/
}
lbl_end_block(ms_end, context);
}
#endif
sub->nb_data = 0;
freep(&sub->data);
return ret;
}

379
src/lib_ccx/ccx_encoders_transcript.c Normal file
View File

@ -0,0 +1,379 @@
#include "ccx_decoders_common.h"
#include "ccx_encoders_common.h"
#include "spupng_encoder.h"
#include "ccx_encoders_spupng.h"
#include "utility.h"
#include "ocr.h"
#include "ccx_decoders_608.h"
#include "ccx_decoders_708.h"
#include "ccx_decoders_708_output.h"
#include "ccx_encoders_xds.h"
#include "ccx_encoders_helpers.h"
#ifdef ENABLE_SHARING
#include "ccx_share.h"
#endif //ENABLE_SHARING
int write_cc_bitmap_as_transcript(struct cc_subtitle *sub, struct encoder_ctx *context)
{
int ret = 0;
#ifdef ENABLE_OCR
struct cc_bitmap* rect;
unsigned h1, m1, s1, ms1;
unsigned h2, m2, s2, ms2;
LLONG start_time, end_time;
if (context->prev_start != -1 && (sub->flags & SUB_EOD_MARKER))
{
start_time = context->prev_start + context->subs_delay;
end_time = sub->start_time - 1;
}
else if (!(sub->flags & SUB_EOD_MARKER))
{
start_time = sub->start_time + context->subs_delay;
end_time = sub->end_time - 1;
}
if (sub->nb_data == 0)
return ret;
rect = sub->data;
if (sub->flags & SUB_EOD_MARKER)
context->prev_start = sub->start_time;
if (rect[0].ocr_text && *(rect[0].ocr_text))
{
if (context->prev_start != -1 || !(sub->flags & SUB_EOD_MARKER))
{
char *token = NULL;
token = paraof_ocrtext(sub, context->encoded_crlf, context->encoded_crlf_length);
if (context->transcript_settings->showStartTime)
{
char buf1[80];
if (context->transcript_settings->relativeTimestamp)
{
millis_to_date(start_time + context->subs_delay, buf1, context->date_format, context->millis_separator);
fdprintf(context->out->fh, "%s|", buf1);
}
else
{
mstotime(start_time + context->subs_delay, &h1, &m1, &s1, &ms1);
time_t start_time_int = (start_time + context->subs_delay) / 1000;
int start_time_dec = (start_time + context->subs_delay) % 1000;
struct tm *start_time_struct = gmtime(&start_time_int);
strftime(buf1, sizeof(buf1), "%Y%m%d%H%M%S", start_time_struct);
fdprintf(context->out->fh, "%s%c%03d|", buf1, context->millis_separator, start_time_dec);
}
}
if (context->transcript_settings->showEndTime)
{
char buf2[80];
if (context->transcript_settings->relativeTimestamp)
{
millis_to_date(end_time, buf2, context->date_format, context->millis_separator);
fdprintf(context->out->fh, "%s|", buf2);
}
else
{
time_t end_time_int = end_time / 1000;
int end_time_dec = end_time % 1000;
struct tm *end_time_struct = gmtime(&end_time_int);
strftime(buf2, sizeof(buf2), "%Y%m%d%H%M%S", end_time_struct);
fdprintf(context->out->fh, "%s%c%03d|", buf2, context->millis_separator, end_time_dec);
}
}
if (context->transcript_settings->showCC)
{
fdprintf(context->out->fh, "%s|", language[sub->lang_index]);
}
if (context->transcript_settings->showMode)
{
fdprintf(context->out->fh, "DVB|");
}
while (token)
{
char *newline_pos = strstr(token, context->encoded_crlf);
if (!newline_pos)
{
fdprintf(context->out->fh, "%s", token);
break;
}
else
{
while (token != newline_pos)
{
fdprintf(context->out->fh, "%c", *token);
token++;
}
token += context->encoded_crlf_length;
fdprintf(context->out->fh, "%c", ' ');
}
}
write(context->out->fh, context->encoded_crlf, context->encoded_crlf_length);
}
}
#endif
sub->nb_data = 0;
freep(&sub->data);
return ret;
}
int write_cc_subtitle_as_transcript(struct cc_subtitle *sub, struct encoder_ctx *context)
{
int length;
int ret = 0;
LLONG start_time = -1;
LLONG end_time = -1;
char *str;
char *save_str;
struct cc_subtitle *osub = sub;
struct cc_subtitle *lsub = sub;
while (sub)
{
if (sub->type == CC_TEXT)
{
start_time = sub->start_time;
end_time = sub->end_time;
}
if (context->sentence_cap)
{
//TODO capitalize (context, line_number,data);
//TODO correct_case_with_dictionary(line_number, data);
}
if (start_time == -1)
{
// CFS: Means that the line has characters but we don't have a timestamp for the first one. Since the timestamp
// is set for example by the write_char function, it possible that we don't have one in empty lines (unclear)
// For now, let's not consider this a bug as before and just return.
// fatal (EXIT_BUG_BUG, "Bug in timedtranscript (ts_start_of_current_line==-1). Please report.");
return 0;
}
str = sub->data;
str = strtok_r(str, "\r\n", &save_str);
do
{
length = get_str_basic(context->subline, (unsigned char*)str, context->trim_subs, sub->enc_type, context->encoding, strlen(str));
if (length <= 0)
{
continue;
}
if (context->transcript_settings->showStartTime)
{
char buf[80];
if (context->transcript_settings->relativeTimestamp)
{
millis_to_date(start_time + context->subs_delay, buf, context->date_format, context->millis_separator);
fdprintf(context->out->fh, "%s|", buf);
}
else
{
time_t start_time_int = (start_time + context->subs_delay) / 1000;
int start_time_dec = (start_time + context->subs_delay) % 1000;
struct tm *start_time_struct = gmtime(&start_time_int);
strftime(buf, sizeof(buf), "%Y%m%d%H%M%S", start_time_struct);
fdprintf(context->out->fh, "%s%c%03d|", buf, context->millis_separator, start_time_dec);
}
}
if (context->transcript_settings->showEndTime)
{
char buf[80];
if (context->transcript_settings->relativeTimestamp)
{
millis_to_date(end_time + context->subs_delay, buf, context->date_format, context->millis_separator);
fdprintf(context->out->fh, "%s|", buf);
}
else
{
time_t end_time_int = (end_time + context->subs_delay) / 1000;
int end_time_dec = (end_time + context->subs_delay) % 1000;
struct tm *end_time_struct = gmtime(&end_time_int);
strftime(buf, sizeof(buf), "%Y%m%d%H%M%S", end_time_struct);
fdprintf(context->out->fh, "%s%c%03d|", buf, context->millis_separator, end_time_dec);
}
}
if (context->transcript_settings->showCC)
{
if (context->in_fileformat == 1)
//TODO, data->my_field == 1 ? data->channel : data->channel + 2); // Data from field 2 is CC3 or 4
fdprintf(context->out->fh, "CC?|");
else if (!context->ucla || !strcmp(sub->mode, "TLT"))
fdprintf(context->out->fh, sub->info);
}
if (context->transcript_settings->showMode)
{
if (context->ucla && strcmp(sub->mode, "TLT") == 0)
fdprintf(context->out->fh, "|");
else
fdprintf(context->out->fh, "%s|", sub->mode);
}
ret = write(context->out->fh, context->subline, length);
if (ret < length)
{
mprint("Warning:Loss of data\n");
}
ret = write(context->out->fh, context->encoded_crlf, context->encoded_crlf_length);
if (ret < context->encoded_crlf_length)
{
mprint("Warning:Loss of data\n");
}
} while ((str = strtok_r(NULL, "\r\n", &save_str)));
freep(&sub->data);
lsub = sub;
sub = sub->next;
}
while (lsub != osub)
{
sub = lsub->prev;
freep(&lsub);
lsub = sub;
}
return ret;
}
//TODO Convert CC line to TEXT format and remove this function
void write_cc_line_as_transcript2(struct eia608_screen *data, struct encoder_ctx *context, int line_number)
{
int ret = 0;
LLONG start_time = data->start_time;
LLONG end_time = data->end_time;
if (context->sentence_cap)
{
if (clever_capitalize(context, line_number, data))
correct_case_with_dictionary(line_number, data);
}
int length = get_str_basic(context->subline, data->characters[line_number],
context->trim_subs, CCX_ENC_ASCII, context->encoding, CCX_DECODER_608_SCREEN_WIDTH);
if (context->encoding != CCX_ENC_UNICODE)
{
dbg_print(CCX_DMT_DECODER_608, "\r");
dbg_print(CCX_DMT_DECODER_608, "%s\n", context->subline);
}
if (length>0)
{
if (data->start_time == -1)
{
// CFS: Means that the line has characters but we don't have a timestamp for the first one. Since the timestamp
// is set for example by the write_char function, it possible that we don't have one in empty lines (unclear)
// For now, let's not consider this a bug as before and just return.
// fatal (EXIT_BUG_BUG, "Bug in timedtranscript (ts_start_of_current_line==-1). Please report.");
return;
}
if (context->transcript_settings->showStartTime){
char buf1[80];
if (context->transcript_settings->relativeTimestamp){
millis_to_date(start_time + context->subs_delay, buf1, context->date_format, context->millis_separator);
fdprintf(context->out->fh, "%s|", buf1);
}
else {
time_t start_time_int = (start_time + context->subs_delay) / 1000;
int start_time_dec = (start_time + context->subs_delay) % 1000;
struct tm *start_time_struct = gmtime(&start_time_int);
strftime(buf1, sizeof(buf1), "%Y%m%d%H%M%S", start_time_struct);
fdprintf(context->out->fh, "%s%c%03d|", buf1, context->millis_separator, start_time_dec);
}
}
if (context->transcript_settings->showEndTime){
char buf2[80];
if (context->transcript_settings->relativeTimestamp){
millis_to_date(end_time, buf2, context->date_format, context->millis_separator);
fdprintf(context->out->fh, "%s|", buf2);
}
else {
time_t end_time_int = end_time / 1000;
int end_time_dec = end_time % 1000;
struct tm *end_time_struct = gmtime(&end_time_int);
strftime(buf2, sizeof(buf2), "%Y%m%d%H%M%S", end_time_struct);
fdprintf(context->out->fh, "%s%c%03d|", buf2, context->millis_separator, end_time_dec);
}
}
if (context->transcript_settings->showCC){
fdprintf(context->out->fh, "CC%d|", data->my_field == 1 ? data->channel : data->channel + 2); // Data from field 2 is CC3 or 4
}
if (context->transcript_settings->showMode){
const char *mode = "???";
switch (data->mode)
{
case MODE_POPON:
mode = "POP";
break;
case MODE_FAKE_ROLLUP_1:
mode = "RU1";
break;
case MODE_ROLLUP_2:
mode = "RU2";
break;
case MODE_ROLLUP_3:
mode = "RU3";
break;
case MODE_ROLLUP_4:
mode = "RU4";
break;
case MODE_TEXT:
mode = "TXT";
break;
case MODE_PAINTON:
mode = "PAI";
break;
}
fdprintf(context->out->fh, "%s|", mode);
}
ret = write(context->out->fh, context->subline, length);
if (ret < length)
{
mprint("Warning:Loss of data\n");
}
ret = write(context->out->fh, context->encoded_crlf, context->encoded_crlf_length);
if (ret < context->encoded_crlf_length)
{
mprint("Warning:Loss of data\n");
}
}
// fprintf (wb->fh,encoded_crlf);
}
int write_cc_buffer_as_transcript2(struct eia608_screen *data, struct encoder_ctx *context)
{
int wrote_something = 0;
dbg_print(CCX_DMT_DECODER_608, "\n- - - TRANSCRIPT caption - - -\n");
for (int i = 0; i<15; i++)
{
if (data->row_used[i])
{
write_cc_line_as_transcript2(data, context, i);
}
wrote_something = 1;
}
dbg_print(CCX_DMT_DECODER_608, "- - - - - - - - - - - -\r\n");
return wrote_something;
}

6
src/lib_ccx/general_loop.c
View File

@ -1002,8 +1002,7 @@ void rcwt_loop(struct lib_ccx_ctx *ctx)
int bread = 0; // Bytes read
LLONG result;
struct encoder_ctx *enc_ctx = update_encoder_list(ctx);
// As BUFSIZE is a macro this is just a reminder
if (BUFSIZE < (3*0xFFFF + 10))
fatal (CCX_COMMON_EXIT_BUG_BUG, "BUFSIZE too small for RCWT caption block.\n");
@ -1037,6 +1036,7 @@ void rcwt_loop(struct lib_ccx_ctx *ctx)
}
dec_ctx = update_decoder_list(ctx);
dec_ctx->dtvcc->encoder = (void *)enc_ctx; //WARN: otherwise cea-708 will not work
if (parsebuf[6] == 0 && parsebuf[7] == 2)
{
dec_ctx->codec = CCX_CODEC_TELETEXT;
@ -1110,7 +1110,7 @@ void rcwt_loop(struct lib_ccx_ctx *ctx)
set_fts(dec_ctx->timing); // Now set the FTS related variables
for (int j=0; j<cbcount*3; j=j+3)
{
{
do_cb(dec_ctx, parsebuf+j, dec_sub);
}
}

282
src/microutf8/microutf8.c Normal file
View File

@ -0,0 +1,282 @@
/*
Copyright (C) 2011 by Tomasz Konojacki
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#include "microutf8.h"
#include <string.h>
int32_t utf8_strlen(uint8_t *str)
{
/* This function doesn't properly check validity of UTF-8 character
sequence, it is supposed to use only with valid UTF-8 strings. */
int32_t character_count = 0;
int32_t i = 0; /* Counter used to iterate over string. */
uint8_t maybe_bom[4];
/* If there is UTF-8 BOM ignore it. */
if (strlen(str) > 2)
{
strncpy(maybe_bom, str, 3);
maybe_bom[3] = 0;
if (strcmp(maybe_bom, (uint8_t*)UTF8_BOM) == 0)
i += 3;
}
while(str[i])
{
/* If bit pattern begins with 0 we have ascii character. */
if (str[i] >> 7 == 0)
++character_count;
/* If bit pattern begins with 11 it is beginning of UTF-8 byte
sequence. */
else if (str[i] >> 6 == 3)
++character_count;
/* If bit pattern begins with 10 it is middle of utf-8 byte sequence. */
else if (str[i] >> 6 == 2)
;
/* In any other case this is not valid UTF-8. */
else
return UTF8_INVALID_SEQUENCE;
++i;
}
return character_count;
}
int8_t utf8_get_nth_char(uint8_t *str, uint8_t *target, uint32_t *offset, uint8_t *is_bom_present, uint32_t n, uint8_t max_length)
{
/* This function doesn't properly check validity of UTF-8 character
sequence, it is supposed to use only with valid UTF-8 strings. */
uint8_t character_length = 0;
int32_t character_count = 0;
int32_t i = 0; /* Counter used to iterate over string. */
int32_t _i = 0; /* Second counter, used to calculate size of UTF-8 character
and to iterate over target character array */
uint8_t maybe_bom[4];
/* If there is UTF-8 BOM ignore it. */
if (strlen(str) > 2)
{
strncpy(maybe_bom, str, 3);
maybe_bom[3] = 0;
if (strcmp(maybe_bom, (uint8_t*)UTF8_BOM) == 0)
{
/* There is BOM, set variable which is pointed by is_bom_present
to 1 */
if (is_bom_present != NULL)
*is_bom_present = 1;
i += 3;
}
else
if (is_bom_present != NULL)
*is_bom_present = 0;
}
else
if (is_bom_present != NULL)
*is_bom_present = 0;
while(str[i])
{
/* If bit pattern begins with 0 we have ascii character. */
if (str[i] >> 7 == 0)
if (++character_count == n)
{
/* That's the character that we were after. */
character_length = 1;
if (target != NULL)
{
if (max_length < 2)
return UTF8_NOT_ENOUGH_SPACE;
target[0] = str[i];
target[1] = 0;
}
if (offset != NULL)
*offset = i;
return 1;
}
/* If bit pattern begins with 11 it is beginning of UTF-8 byte
sequence. */
else if (str[i] >> 6 == 3)
if (++character_count == n)
{
/* That's the character that we were after. */
character_length = 1;
_i = i;
/* Check size (in bytes) of character. */
while (str[++_i] >> 6 == 2)
++character_length;
/* Check if there is enough space for character. */
if (target != NULL)
if (character_length + 1 > max_length)
return UTF8_NOT_ENOUGH_SPACE;
/* This sequence must be longer than one byte. */
if (character_length == 1)
return UTF8_INVALID_SEQUENCE;
/* Write offset of character. */
if (offset != NULL)
{
*offset = i;
}
/* Write character to target array. */
if (target != NULL)
{
for (_i = 0; _i < character_length; ++_i)
target[_i] = str[i++];
/* End string which contains character with 0. */
target[_i++] = 0;
}
return 1;
}
/* If bit pattern begins with 10 it is middle of UTF-8 byte sequence. */
else if (str[i] >> 6 == 2)
;
/* In any other case this is not valid UTF-8. */
else
return UTF8_INVALID_SEQUENCE;
++i;
}
/* It should not happen. */
return 0;
}
int8_t utf8_strip_bom(uint8_t *str, uint8_t *target, uint32_t max_length)
{
int i = 0; /* Counter used to iterate over input string. */
int _i = 0; /* Counter used to iterate over output string. */
uint8_t maybe_bom[4];
/* Skip BOM if present. */
if (strlen(str) > 2)
{
strncpy(maybe_bom, str, 3);
maybe_bom[3] = 0;
if (strcmp(maybe_bom, (uint8_t*)UTF8_BOM) == 0)
{
/* Check if there is enough space. */
if (strlen(str) - 2 > max_length)
return UTF8_NOT_ENOUGH_SPACE;
i += 3;
}
else
/* Check if there is enough space. */
if (strlen(str) + 1 > max_length)
return UTF8_NOT_ENOUGH_SPACE;
}
/* Write new string. */
while (str[i])
{
target[_i] = str[i];
++i;
++_i;
}
/* Insert trailing 0. */
target[_i] = 0;
/* Probably success, return 1. */
return 1;
}
int8_t utf8_check_validity(uint8_t *str)
{
int i = 0; /* Counter used to iterate over input string. */
int _i = 0; /* Counter used to iterate over character. */
int charlen = 0; /* Current character length */
uint8_t maybe_bom[4];
/* Check BOM. */
if (strlen(str) > 2)
{
strncpy(maybe_bom, str, 3);
maybe_bom[3] = 0;
if (strcmp(maybe_bom, (uint8_t*)UTF8_BOM) == 0)
i += 3;
}
while (str[i])
{
/* If bit pattern begins with 0 we have ascii character. */
if ((str[i] >> 7 == 0) && (_i == 0) && (charlen == 0))
++i;
else if ((str[i] >> 6 == 3) && _i == 0)
{
if (str[i] >> 5 == 6)
/* Character sequence length equals 2. */
charlen = 2;
else if (str[i] >> 4 == 14)
/* Character sequence length equals 3. */
charlen = 3;
else if (str[i] >> 3 == 30)
/* Character sequence length equals 4. */
charlen = 4;
else if (str[i] >> 2 == 62)
/* Character sequence length equals 5. */
charlen = 5;
else if (str[i] >> 1 == 126)
/* Character sequence length equals 6. */
charlen = 6;
else
/* This is not valid UTF-8 string. */
return UTF8_INVALID_SEQUENCE;
++i;
++_i;
}
else if ((str[i] >> 6 == 2) && (charlen > 1))
{
if (charlen - 1 == _i)
{
/* This is last byte in character sequence, reset character
counters. */
charlen = 0;
_i = 0;
++i;
}
else
{
/* Continue iterating over string. */
++_i;
++i;
}
}
else
/* This is invalid UTF-8 string. */
return UTF8_INVALID_SEQUENCE;
}
/* It seems that this is valid UTF-8 string. */
return UTF8_VALID_SEQUENCE;
}
int8_t utf8_is_bom_present(uint8_t *str)
{
uint8_t maybe_bom[4];
if (strlen(str) > 2)
{
strncpy(maybe_bom, str, 3);
maybe_bom[3] = 0;
if (strcmp(maybe_bom, (uint8_t*)UTF8_BOM) == 0)
{
return 1;
}
else
return 0;
}
}

112
src/microutf8/microutf8.h Normal file
View File

@ -0,0 +1,112 @@
/*
Copyright (C) 2011 by Tomasz Konojacki
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef __MICROUTF8_H__
#define __MICROUTF8_H__
#pragma once
#include <stdint.h>
/******************************************************************************
* NOTE !!! *
* *
* ALL FUNCTIONS (EXCEPT utf8_check_validity()) EXPECT VALID UTF-8 STRING! *
******************************************************************************/
/* Constants. */
#define UTF8_MICROUTF8_VERSION "1.2.0"
#define UTF8_VALID_SEQUENCE 1
#define UTF8_INVALID_SEQUENCE -1
#define NOPE_CHUCK_TESTA -1 /* Is it valid UTF-8 string? */
#define UTF8_NOT_ENOUGH_SPACE -2
#define UTF8_BOM "\xEF\xBB\xBF" /* note that it need to be casted to uint8_t* */
/* Functions. */
/*
utf8_strlen() takes null-terminated UTF-8 string as argument and returns number
of UTF-8 characters contained in string. Returns UTF8_INVALID_SEQUENCE if UTF-8
sequence is invalid. utf8_strlen() ignores BOM if present.
*/
int32_t utf8_strlen(uint8_t *str);
/*
utf8_get_nth_char() gives you requested n-th UTF-8 character from UTF-8 string.
utf8_get_nth_char() takes six arguments:
str - null-terminated UTF-8 string
target - string where requested character will be written (can be set to NULL)
offset - pointer to integer where offset of character may be stored
(can be set to NULL)
is_bom_present - pointer to integer (can be set to NULL) which will be set to
1 if BOM is present in string or to 0 if there's no BOM
n - number (from the left) of requested character
max_length - maximal length (including trailing \0) of character in bytes.
It can be set to 0 if target == NULL.
Return value:
1 - success
UTF8_INVALID_SEQUENCE - supplied UTF-8 sequence was invalid
UTF8_NOT_ENOUGH_SPACE - max_length is too small
*/
int8_t utf8_get_nth_char(uint8_t *str, uint8_t *target, uint32_t *offset, uint8_t *is_bom_present, uint32_t n, uint8_t max_length);
/*
utf8_strip_bom() copies requested null-terminated UTF-8 string to another
string without BOM, if there is no BOM resulting string will be exactly the
same.
utf8_strip_bom() takes three arguments:
str - null-terminated UTF-8 string
target - target string, if there is BOM (you can check it with
utf8_get_nth_char()) needed size will be strlen(str) - 2. If there
is no BOM needed size will be strlen(str) + 1.
max_length - max length of new string.
Return value:
1 - success
UTF8_NOT_ENOUGH_SPACE - max_length is too small
*/
int8_t utf8_strip_bom(uint8_t *str, uint8_t *target, uint32_t max_length);
/*
utf8_check_validity() checks if supplied string is valid UTF-8 string.
Return value:
UTF8_VALID_SEQUENCE - supplied UTF-8 sequence was valid
UTF8_INVALID_SEQUENCE - supplied UTF-8 sequence was invalid
*/
int8_t utf8_check_validity(uint8_t *str);
/*
utf8_is_bom_present() checks if supplied string contains UTF-8 BOM.
Return value:
1 - supplied UTF-8 string has UTF-8 BOM
0 - there is no BOM
*/
int8_t utf8_is_bom_present(uint8_t *str);
#endif

733
src/utf8proc/utf8proc.c Normal file
View File

@ -0,0 +1,733 @@
/* -*- mode: c; c-basic-offset: 2; tab-width: 2; indent-tabs-mode: nil -*- */
/*
* Copyright (c) 2015 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
* Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
/*
* This library contains derived data from a modified version of the
* Unicode data files.
*
* The original data files are available at
* http://www.unicode.org/Public/UNIDATA/
*
* Please notice the copyright statement in the file "utf8proc_data.c".
*/
/*
* File name: utf8proc.c
*
* Description:
* Implementation of libutf8proc.
*/
#include "utf8proc.h"
#include "utf8proc_data.c"
const utf8proc_int8_t utf8proc_utf8class[256] = {
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0 };
#define UTF8PROC_HANGUL_SBASE 0xAC00
#define UTF8PROC_HANGUL_LBASE 0x1100
#define UTF8PROC_HANGUL_VBASE 0x1161
#define UTF8PROC_HANGUL_TBASE 0x11A7
#define UTF8PROC_HANGUL_LCOUNT 19
#define UTF8PROC_HANGUL_VCOUNT 21
#define UTF8PROC_HANGUL_TCOUNT 28
#define UTF8PROC_HANGUL_NCOUNT 588
#define UTF8PROC_HANGUL_SCOUNT 11172
/* END is exclusive */
#define UTF8PROC_HANGUL_L_START 0x1100
#define UTF8PROC_HANGUL_L_END 0x115A
#define UTF8PROC_HANGUL_L_FILLER 0x115F
#define UTF8PROC_HANGUL_V_START 0x1160
#define UTF8PROC_HANGUL_V_END 0x11A3
#define UTF8PROC_HANGUL_T_START 0x11A8
#define UTF8PROC_HANGUL_T_END 0x11FA
#define UTF8PROC_HANGUL_S_START 0xAC00
#define UTF8PROC_HANGUL_S_END 0xD7A4
/* Should follow semantic-versioning rules (semver.org) based on API
compatibility. (Note that the shared-library version number will
be different, being based on ABI compatibility.): */
#define STRINGIZEx(x) #x
#define STRINGIZE(x) STRINGIZEx(x)
const char *utf8proc_version(void) {
return STRINGIZE(UTF8PROC_VERSION_MAJOR) "." STRINGIZE(UTF8PROC_VERSION_MINOR) "." STRINGIZE(UTF8PROC_VERSION_PATCH) "";
}
const char *utf8proc_errmsg(utf8proc_ssize_t errcode) {
switch (errcode) {
case UTF8PROC_ERROR_NOMEM:
return "Memory for processing UTF-8 data could not be allocated.";
case UTF8PROC_ERROR_OVERFLOW:
return "UTF-8 string is too long to be processed.";
case UTF8PROC_ERROR_INVALIDUTF8:
return "Invalid UTF-8 string";
case UTF8PROC_ERROR_NOTASSIGNED:
return "Unassigned Unicode code point found in UTF-8 string.";
case UTF8PROC_ERROR_INVALIDOPTS:
return "Invalid options for UTF-8 processing chosen.";
default:
return "An unknown error occurred while processing UTF-8 data.";
}
}
#define utf_cont(ch) (((ch) & 0xc0) == 0x80)
utf8proc_ssize_t utf8proc_iterate(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *dst
) {
utf8proc_uint32_t uc;
const utf8proc_uint8_t *end;
*dst = -1;
if (!strlen) return 0;
end = str + ((strlen < 0) ? 4 : strlen);
uc = *str++;
if (uc < 0x80) {
*dst = uc;
return 1;
}
// Must be between 0xc2 and 0xf4 inclusive to be valid
if ((uc - 0xc2) > (0xf4-0xc2)) return UTF8PROC_ERROR_INVALIDUTF8;
if (uc < 0xe0) { // 2-byte sequence
// Must have valid continuation character
if (str >= end || !utf_cont(*str)) return UTF8PROC_ERROR_INVALIDUTF8;
*dst = ((uc & 0x1f)<<6) | (*str & 0x3f);
return 2;
}
if (uc < 0xf0) { // 3-byte sequence
if ((str + 1 >= end) || !utf_cont(*str) || !utf_cont(str[1]))
return UTF8PROC_ERROR_INVALIDUTF8;
// Check for surrogate chars
if (uc == 0xed && *str > 0x9f)
return UTF8PROC_ERROR_INVALIDUTF8;
uc = ((uc & 0xf)<<12) | ((*str & 0x3f)<<6) | (str[1] & 0x3f);
if (uc < 0x800)
return UTF8PROC_ERROR_INVALIDUTF8;
*dst = uc;
return 3;
}
// 4-byte sequence
// Must have 3 valid continuation characters
if ((str + 2 >= end) || !utf_cont(*str) || !utf_cont(str[1]) || !utf_cont(str[2]))
return UTF8PROC_ERROR_INVALIDUTF8;
// Make sure in correct range (0x10000 - 0x10ffff)
if (uc == 0xf0) {
if (*str < 0x90) return UTF8PROC_ERROR_INVALIDUTF8;
} else if (uc == 0xf4) {
if (*str > 0x8f) return UTF8PROC_ERROR_INVALIDUTF8;
}
*dst = ((uc & 7)<<18) | ((*str & 0x3f)<<12) | ((str[1] & 0x3f)<<6) | (str[2] & 0x3f);
return 4;
}
utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t uc) {
return (((utf8proc_uint32_t)uc)-0xd800 > 0x07ff) && ((utf8proc_uint32_t)uc < 0x110000);
}
utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
if (uc < 0x00) {
return 0;
} else if (uc < 0x80) {
dst[0] = uc;
return 1;
} else if (uc < 0x800) {
dst[0] = 0xC0 + (uc >> 6);
dst[1] = 0x80 + (uc & 0x3F);
return 2;
// Note: we allow encoding 0xd800-0xdfff here, so as not to change
// the API, however, these are actually invalid in UTF-8
} else if (uc < 0x10000) {
dst[0] = 0xE0 + (uc >> 12);
dst[1] = 0x80 + ((uc >> 6) & 0x3F);
dst[2] = 0x80 + (uc & 0x3F);
return 3;
} else if (uc < 0x110000) {
dst[0] = 0xF0 + (uc >> 18);
dst[1] = 0x80 + ((uc >> 12) & 0x3F);
dst[2] = 0x80 + ((uc >> 6) & 0x3F);
dst[3] = 0x80 + (uc & 0x3F);
return 4;
} else return 0;
}
/* internal "unsafe" version that does not check whether uc is in range */
static utf8proc_ssize_t unsafe_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
if (uc < 0x00) {
return 0;
} else if (uc < 0x80) {
dst[0] = uc;
return 1;
} else if (uc < 0x800) {
dst[0] = 0xC0 + (uc >> 6);
dst[1] = 0x80 + (uc & 0x3F);
return 2;
} else if (uc == 0xFFFF) {
dst[0] = 0xFF;
return 1;
} else if (uc == 0xFFFE) {
dst[0] = 0xFE;
return 1;
} else if (uc < 0x10000) {
dst[0] = 0xE0 + (uc >> 12);
dst[1] = 0x80 + ((uc >> 6) & 0x3F);
dst[2] = 0x80 + (uc & 0x3F);
return 3;
} else if (uc < 0x110000) {
dst[0] = 0xF0 + (uc >> 18);
dst[1] = 0x80 + ((uc >> 12) & 0x3F);
dst[2] = 0x80 + ((uc >> 6) & 0x3F);
dst[3] = 0x80 + (uc & 0x3F);
return 4;
} else return 0;
}
/* internal "unsafe" version that does not check whether uc is in range */
static const utf8proc_property_t *unsafe_get_property(utf8proc_int32_t uc) {
/* ASSERT: uc >= 0 && uc < 0x110000 */
return utf8proc_properties + (
utf8proc_stage2table[
utf8proc_stage1table[uc >> 8] + (uc & 0xFF)
]
);
}
const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t uc) {
return uc < 0 || uc >= 0x110000 ? utf8proc_properties : unsafe_get_property(uc);
}
/* return whether there is a grapheme break between boundclasses lbc and tbc
(according to the definition of extended grapheme clusters)
Rule numbering refers to TR29 Version 29 (Unicode 9.0.0):
http://www.unicode.org/reports/tr29/tr29-29.html
CAVEATS:
Please note that evaluation of GB10 (grapheme breaks between emoji zwj sequences)
and GB 12/13 (regional indicator code points) require knowledge of previous characters
and are thus not handled by this function. This may result in an incorrect break before
an E_Modifier class codepoint and an incorrectly missing break between two
REGIONAL_INDICATOR class code points if such support does not exist in the caller.
See the special support in grapheme_break_extended, for required bookkeeping by the caller.
*/
static utf8proc_bool grapheme_break_simple(int lbc, int tbc) {
return
(lbc == UTF8PROC_BOUNDCLASS_START) ? true : // GB1
(lbc == UTF8PROC_BOUNDCLASS_CR && // GB3
tbc == UTF8PROC_BOUNDCLASS_LF) ? false : // ---
(lbc >= UTF8PROC_BOUNDCLASS_CR && lbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true : // GB4
(tbc >= UTF8PROC_BOUNDCLASS_CR && tbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true : // GB5
(lbc == UTF8PROC_BOUNDCLASS_L && // GB6
(tbc == UTF8PROC_BOUNDCLASS_L || // ---
tbc == UTF8PROC_BOUNDCLASS_V || // ---
tbc == UTF8PROC_BOUNDCLASS_LV || // ---
tbc == UTF8PROC_BOUNDCLASS_LVT)) ? false : // ---
((lbc == UTF8PROC_BOUNDCLASS_LV || // GB7
lbc == UTF8PROC_BOUNDCLASS_V) && // ---
(tbc == UTF8PROC_BOUNDCLASS_V || // ---
tbc == UTF8PROC_BOUNDCLASS_T)) ? false : // ---
((lbc == UTF8PROC_BOUNDCLASS_LVT || // GB8
lbc == UTF8PROC_BOUNDCLASS_T) && // ---
tbc == UTF8PROC_BOUNDCLASS_T) ? false : // ---
(tbc == UTF8PROC_BOUNDCLASS_EXTEND || // GB9
tbc == UTF8PROC_BOUNDCLASS_ZWJ || // ---
tbc == UTF8PROC_BOUNDCLASS_SPACINGMARK || // GB9a
lbc == UTF8PROC_BOUNDCLASS_PREPEND) ? false : // GB9b
((lbc == UTF8PROC_BOUNDCLASS_E_BASE || // GB10 (requires additional handling below)
lbc == UTF8PROC_BOUNDCLASS_E_BASE_GAZ) && // ----
tbc == UTF8PROC_BOUNDCLASS_E_MODIFIER) ? false : // ----
(lbc == UTF8PROC_BOUNDCLASS_ZWJ && // GB11
(tbc == UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ || // ----
tbc == UTF8PROC_BOUNDCLASS_E_BASE_GAZ)) ? false : // ----
(lbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR && // GB12/13 (requires additional handling below)
tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR) ? false : // ----
true; // GB999
}
static utf8proc_bool grapheme_break_extended(int lbc, int tbc, utf8proc_int32_t *state)
{
int lbc_override = lbc;
if (state && *state != UTF8PROC_BOUNDCLASS_START)
lbc_override = *state;
utf8proc_bool break_permitted = grapheme_break_simple(lbc_override, tbc);
if (state) {
// Special support for GB 12/13 made possible by GB999. After two RI
// class codepoints we want to force a break. Do this by resetting the
// second RI's bound class to UTF8PROC_BOUNDCLASS_OTHER, to force a break
// after that character according to GB999 (unless of course such a break is
// forbidden by a different rule such as GB9).
if (*state == tbc && tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR)
*state = UTF8PROC_BOUNDCLASS_OTHER;
// Special support for GB10. Fold any EXTEND codepoints into the previous
// boundclass if we're dealing with an emoji base boundclass.
else if ((*state == UTF8PROC_BOUNDCLASS_E_BASE ||
*state == UTF8PROC_BOUNDCLASS_E_BASE_GAZ) &&
tbc == UTF8PROC_BOUNDCLASS_EXTEND)
*state = UTF8PROC_BOUNDCLASS_E_BASE;
else
*state = tbc;
}
return break_permitted;
}
utf8proc_bool utf8proc_grapheme_break_stateful(
utf8proc_int32_t c1, utf8proc_int32_t c2, utf8proc_int32_t *state) {
return grapheme_break_extended(utf8proc_get_property(c1)->boundclass,
utf8proc_get_property(c2)->boundclass,
state);
}
utf8proc_bool utf8proc_grapheme_break(
utf8proc_int32_t c1, utf8proc_int32_t c2) {
return utf8proc_grapheme_break_stateful(c1, c2, NULL);
}
static utf8proc_int32_t seqindex_decode_entry(const utf8proc_uint16_t **entry)
{
utf8proc_int32_t entry_cp = **entry;
if ((entry_cp & 0xF800) == 0xD800) {
*entry = *entry + 1;
entry_cp = ((entry_cp & 0x03FF) << 10) | (**entry & 0x03FF);
entry_cp += 0x10000;
}
return entry_cp;
}
static utf8proc_int32_t seqindex_decode_index(const utf8proc_uint32_t seqindex)
{
const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex];
return seqindex_decode_entry(&entry);
}
static utf8proc_ssize_t seqindex_write_char_decomposed(utf8proc_uint16_t seqindex, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
utf8proc_ssize_t written = 0;
const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex & 0x1FFF];
int len = seqindex >> 13;
if (len >= 7) {
len = *entry;
entry++;
}
for (; len >= 0; entry++, len--) {
utf8proc_int32_t entry_cp = seqindex_decode_entry(&entry);
written += utf8proc_decompose_char(entry_cp, dst+written,
(bufsize > written) ? (bufsize - written) : 0, options,
last_boundclass);
if (written < 0) return UTF8PROC_ERROR_OVERFLOW;
}
return written;
}
utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c)
{
utf8proc_int32_t cl = utf8proc_get_property(c)->lowercase_seqindex;
return cl != UINT16_MAX ? seqindex_decode_index(cl) : c;
}
utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c)
{
utf8proc_int32_t cu = utf8proc_get_property(c)->uppercase_seqindex;
return cu != UINT16_MAX ? seqindex_decode_index(cu) : c;
}
utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c)
{
utf8proc_int32_t cu = utf8proc_get_property(c)->titlecase_seqindex;
return cu != UINT16_MAX ? seqindex_decode_index(cu) : c;
}
/* return a character width analogous to wcwidth (except portable and
hopefully less buggy than most system wcwidth functions). */
int utf8proc_charwidth(utf8proc_int32_t c) {
return utf8proc_get_property(c)->charwidth;
}
utf8proc_category_t utf8proc_category(utf8proc_int32_t c) {
return utf8proc_get_property(c)->category;
}
const char *utf8proc_category_string(utf8proc_int32_t c) {
static const char s[][3] = {"Cn","Lu","Ll","Lt","Lm","Lo","Mn","Mc","Me","Nd","Nl","No","Pc","Pd","Ps","Pe","Pi","Pf","Po","Sm","Sc","Sk","So","Zs","Zl","Zp","Cc","Cf","Cs","Co"};
return s[utf8proc_category(c)];
}
#define utf8proc_decompose_lump(replacement_uc) \
return utf8proc_decompose_char((replacement_uc), dst, bufsize, \
options & ~UTF8PROC_LUMP, last_boundclass)
utf8proc_ssize_t utf8proc_decompose_char(utf8proc_int32_t uc, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
const utf8proc_property_t *property;
utf8proc_propval_t category;
utf8proc_int32_t hangul_sindex;
if (uc < 0 || uc >= 0x110000) return UTF8PROC_ERROR_NOTASSIGNED;
property = unsafe_get_property(uc);
category = property->category;
hangul_sindex = uc - UTF8PROC_HANGUL_SBASE;
if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT) {
utf8proc_int32_t hangul_tindex;
if (bufsize >= 1) {
dst[0] = UTF8PROC_HANGUL_LBASE +
hangul_sindex / UTF8PROC_HANGUL_NCOUNT;
if (bufsize >= 2) dst[1] = UTF8PROC_HANGUL_VBASE +
(hangul_sindex % UTF8PROC_HANGUL_NCOUNT) / UTF8PROC_HANGUL_TCOUNT;
}
hangul_tindex = hangul_sindex % UTF8PROC_HANGUL_TCOUNT;
if (!hangul_tindex) return 2;
if (bufsize >= 3) dst[2] = UTF8PROC_HANGUL_TBASE + hangul_tindex;
return 3;
}
}
if (options & UTF8PROC_REJECTNA) {
if (!category) return UTF8PROC_ERROR_NOTASSIGNED;
}
if (options & UTF8PROC_IGNORE) {
if (property->ignorable) return 0;
}
if (options & UTF8PROC_LUMP) {
if (category == UTF8PROC_CATEGORY_ZS) utf8proc_decompose_lump(0x0020);
if (uc == 0x2018 || uc == 0x2019 || uc == 0x02BC || uc == 0x02C8)
utf8proc_decompose_lump(0x0027);
if (category == UTF8PROC_CATEGORY_PD || uc == 0x2212)
utf8proc_decompose_lump(0x002D);
if (uc == 0x2044 || uc == 0x2215) utf8proc_decompose_lump(0x002F);
if (uc == 0x2236) utf8proc_decompose_lump(0x003A);
if (uc == 0x2039 || uc == 0x2329 || uc == 0x3008)
utf8proc_decompose_lump(0x003C);
if (uc == 0x203A || uc == 0x232A || uc == 0x3009)
utf8proc_decompose_lump(0x003E);
if (uc == 0x2216) utf8proc_decompose_lump(0x005C);
if (uc == 0x02C4 || uc == 0x02C6 || uc == 0x2038 || uc == 0x2303)
utf8proc_decompose_lump(0x005E);
if (category == UTF8PROC_CATEGORY_PC || uc == 0x02CD)
utf8proc_decompose_lump(0x005F);
if (uc == 0x02CB) utf8proc_decompose_lump(0x0060);
if (uc == 0x2223) utf8proc_decompose_lump(0x007C);
if (uc == 0x223C) utf8proc_decompose_lump(0x007E);
if ((options & UTF8PROC_NLF2LS) && (options & UTF8PROC_NLF2PS)) {
if (category == UTF8PROC_CATEGORY_ZL ||
category == UTF8PROC_CATEGORY_ZP)
utf8proc_decompose_lump(0x000A);
}
}
if (options & UTF8PROC_STRIPMARK) {
if (category == UTF8PROC_CATEGORY_MN ||
category == UTF8PROC_CATEGORY_MC ||
category == UTF8PROC_CATEGORY_ME) return 0;
}
if (options & UTF8PROC_CASEFOLD) {
if (property->casefold_seqindex != UINT16_MAX) {
return seqindex_write_char_decomposed(property->casefold_seqindex, dst, bufsize, options, last_boundclass);
}
}
if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
if (property->decomp_seqindex != UINT16_MAX &&
(!property->decomp_type || (options & UTF8PROC_COMPAT))) {
return seqindex_write_char_decomposed(property->decomp_seqindex, dst, bufsize, options, last_boundclass);
}
}
if (options & UTF8PROC_CHARBOUND) {
utf8proc_bool boundary;
int tbc = property->boundclass;
boundary = grapheme_break_extended(*last_boundclass, tbc, last_boundclass);
if (boundary) {
if (bufsize >= 1) dst[0] = 0xFFFF;
if (bufsize >= 2) dst[1] = uc;
return 2;
}
}
if (bufsize >= 1) *dst = uc;
return 1;
}
utf8proc_ssize_t utf8proc_decompose(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
) {
/* strlen will be ignored, if UTF8PROC_NULLTERM is set in options */
utf8proc_ssize_t wpos = 0;
if ((options & UTF8PROC_COMPOSE) && (options & UTF8PROC_DECOMPOSE))
return UTF8PROC_ERROR_INVALIDOPTS;
if ((options & UTF8PROC_STRIPMARK) &&
!(options & UTF8PROC_COMPOSE) && !(options & UTF8PROC_DECOMPOSE))
return UTF8PROC_ERROR_INVALIDOPTS;
{
utf8proc_int32_t uc;
utf8proc_ssize_t rpos = 0;
utf8proc_ssize_t decomp_result;
int boundclass = UTF8PROC_BOUNDCLASS_START;
while (1) {
if (options & UTF8PROC_NULLTERM) {
rpos += utf8proc_iterate(str + rpos, -1, &uc);
/* checking of return value is not necessary,
as 'uc' is < 0 in case of error */
if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
if (rpos < 0) return UTF8PROC_ERROR_OVERFLOW;
if (uc == 0) break;
} else {
if (rpos >= strlen) break;
rpos += utf8proc_iterate(str + rpos, strlen - rpos, &uc);
if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
}
decomp_result = utf8proc_decompose_char(
uc, buffer + wpos, (bufsize > wpos) ? (bufsize - wpos) : 0, options,
&boundclass
);
if (decomp_result < 0) return decomp_result;
wpos += decomp_result;
/* prohibiting integer overflows due to too long strings: */
if (wpos < 0 ||
wpos > (utf8proc_ssize_t)(SSIZE_MAX/sizeof(utf8proc_int32_t)/2))
return UTF8PROC_ERROR_OVERFLOW;
}
}
if ((options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) && bufsize >= wpos) {
utf8proc_ssize_t pos = 0;
while (pos < wpos-1) {
utf8proc_int32_t uc1, uc2;
const utf8proc_property_t *property1, *property2;
uc1 = buffer[pos];
uc2 = buffer[pos+1];
property1 = unsafe_get_property(uc1);
property2 = unsafe_get_property(uc2);
if (property1->combining_class > property2->combining_class &&
property2->combining_class > 0) {
buffer[pos] = uc2;
buffer[pos+1] = uc1;
if (pos > 0) pos--; else pos++;
} else {
pos++;
}
}
}
return wpos;
}
utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options) {
/* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored
ASSERT: 'buffer' has one spare byte of free space at the end! */
if (options & (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS | UTF8PROC_STRIPCC)) {
utf8proc_ssize_t rpos;
utf8proc_ssize_t wpos = 0;
utf8proc_int32_t uc;
for (rpos = 0; rpos < length; rpos++) {
uc = buffer[rpos];
if (uc == 0x000D && rpos < length-1 && buffer[rpos+1] == 0x000A) rpos++;
if (uc == 0x000A || uc == 0x000D || uc == 0x0085 ||
((options & UTF8PROC_STRIPCC) && (uc == 0x000B || uc == 0x000C))) {
if (options & UTF8PROC_NLF2LS) {
if (options & UTF8PROC_NLF2PS) {
buffer[wpos++] = 0x000A;
} else {
buffer[wpos++] = 0x2028;
}
} else {
if (options & UTF8PROC_NLF2PS) {
buffer[wpos++] = 0x2029;
} else {
buffer[wpos++] = 0x0020;
}
}
} else if ((options & UTF8PROC_STRIPCC) &&
(uc < 0x0020 || (uc >= 0x007F && uc < 0x00A0))) {
if (uc == 0x0009) buffer[wpos++] = 0x0020;
} else {
buffer[wpos++] = uc;
}
}
length = wpos;
}
if (options & UTF8PROC_COMPOSE) {
utf8proc_int32_t *starter = NULL;
utf8proc_int32_t current_char;
const utf8proc_property_t *starter_property = NULL, *current_property;
utf8proc_propval_t max_combining_class = -1;
utf8proc_ssize_t rpos;
utf8proc_ssize_t wpos = 0;
utf8proc_int32_t composition;
for (rpos = 0; rpos < length; rpos++) {
current_char = buffer[rpos];
current_property = unsafe_get_property(current_char);
if (starter && current_property->combining_class > max_combining_class) {
/* combination perhaps possible */
utf8proc_int32_t hangul_lindex;
utf8proc_int32_t hangul_sindex;
hangul_lindex = *starter - UTF8PROC_HANGUL_LBASE;
if (hangul_lindex >= 0 && hangul_lindex < UTF8PROC_HANGUL_LCOUNT) {
utf8proc_int32_t hangul_vindex;
hangul_vindex = current_char - UTF8PROC_HANGUL_VBASE;
if (hangul_vindex >= 0 && hangul_vindex < UTF8PROC_HANGUL_VCOUNT) {
*starter = UTF8PROC_HANGUL_SBASE +
(hangul_lindex * UTF8PROC_HANGUL_VCOUNT + hangul_vindex) *
UTF8PROC_HANGUL_TCOUNT;
starter_property = NULL;
continue;
}
}
hangul_sindex = *starter - UTF8PROC_HANGUL_SBASE;
if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT &&
(hangul_sindex % UTF8PROC_HANGUL_TCOUNT) == 0) {
utf8proc_int32_t hangul_tindex;
hangul_tindex = current_char - UTF8PROC_HANGUL_TBASE;
if (hangul_tindex >= 0 && hangul_tindex < UTF8PROC_HANGUL_TCOUNT) {
*starter += hangul_tindex;
starter_property = NULL;
continue;
}
}
if (!starter_property) {
starter_property = unsafe_get_property(*starter);
}
if (starter_property->comb_index < 0x8000 &&
current_property->comb_index != UINT16_MAX &&
current_property->comb_index >= 0x8000) {
int sidx = starter_property->comb_index;
int idx = (current_property->comb_index & 0x3FFF) - utf8proc_combinations[sidx];
if (idx >= 0 && idx <= utf8proc_combinations[sidx + 1] ) {
idx += sidx + 2;
if (current_property->comb_index & 0x4000) {
composition = (utf8proc_combinations[idx] << 16) | utf8proc_combinations[idx+1];
} else
composition = utf8proc_combinations[idx];
if (composition > 0 && (!(options & UTF8PROC_STABLE) ||
!(unsafe_get_property(composition)->comp_exclusion))) {
*starter = composition;
starter_property = NULL;
continue;
}
}
}
}
buffer[wpos] = current_char;
if (current_property->combining_class) {
if (current_property->combining_class > max_combining_class) {
max_combining_class = current_property->combining_class;
}
} else {
starter = buffer + wpos;
starter_property = NULL;
max_combining_class = -1;
}
wpos++;
}
length = wpos;
}
{
utf8proc_ssize_t rpos, wpos = 0;
utf8proc_int32_t uc;
if (options & UTF8PROC_CHARBOUND) {
for (rpos = 0; rpos < length; rpos++) {
uc = buffer[rpos];
wpos += unsafe_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
}
} else {
for (rpos = 0; rpos < length; rpos++) {
uc = buffer[rpos];
wpos += utf8proc_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
}
}
((utf8proc_uint8_t *)buffer)[wpos] = 0;
return wpos;
}
}
utf8proc_ssize_t utf8proc_map(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
) {
utf8proc_int32_t *buffer;
utf8proc_ssize_t result;
*dstptr = NULL;
result = utf8proc_decompose(str, strlen, NULL, 0, options);
if (result < 0) return result;
buffer = (utf8proc_int32_t *) malloc(result * sizeof(utf8proc_int32_t) + 1);
if (!buffer) return UTF8PROC_ERROR_NOMEM;
result = utf8proc_decompose(str, strlen, buffer, result, options);
if (result < 0) {
free(buffer);
return result;
}
result = utf8proc_reencode(buffer, result, options);
if (result < 0) {
free(buffer);
return result;
}
{
utf8proc_int32_t *newptr;
newptr = (utf8proc_int32_t *) realloc(buffer, (size_t)result+1);
if (newptr) buffer = newptr;
}
*dstptr = (utf8proc_uint8_t *)buffer;
return result;
}
utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_DECOMPOSE);
return retval;
}
utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_COMPOSE);
return retval;
}
utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
return retval;
}
utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
return retval;
}

618
src/utf8proc/utf8proc.h Normal file
View File

@ -0,0 +1,618 @@
/*
* Copyright (c) 2015 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
* Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
/**
* @mainpage
*
* utf8proc is a free/open-source (MIT/expat licensed) C library
* providing Unicode normalization, case-folding, and other operations
* for strings in the UTF-8 encoding, supporting Unicode version
* 8.0.0. See the utf8proc home page (http://julialang.org/utf8proc/)
* for downloads and other information, or the source code on github
* (https://github.com/JuliaLang/utf8proc).
*
* For the utf8proc API documentation, see: @ref utf8proc.h
*
* The features of utf8proc include:
*
* - Transformation of strings (@ref utf8proc_map) to:
* - decompose (@ref UTF8PROC_DECOMPOSE) or compose (@ref UTF8PROC_COMPOSE) Unicode combining characters (http://en.wikipedia.org/wiki/Combining_character)
* - canonicalize Unicode compatibility characters (@ref UTF8PROC_COMPAT)
* - strip "ignorable" (@ref UTF8PROC_IGNORE) characters, control characters (@ref UTF8PROC_STRIPCC), or combining characters such as accents (@ref UTF8PROC_STRIPMARK)
* - case-folding (@ref UTF8PROC_CASEFOLD)
* - Unicode normalization: @ref utf8proc_NFD, @ref utf8proc_NFC, @ref utf8proc_NFKD, @ref utf8proc_NFKC
* - Detecting grapheme boundaries (@ref utf8proc_grapheme_break and @ref UTF8PROC_CHARBOUND)
* - Character-width computation: @ref utf8proc_charwidth
* - Classification of characters by Unicode category: @ref utf8proc_category and @ref utf8proc_category_string
* - Encode (@ref utf8proc_encode_char) and decode (@ref utf8proc_iterate) Unicode codepoints to/from UTF-8.
*/
/** @file */
#ifndef UTF8PROC_H
#define UTF8PROC_H
/** @name API version
*
* The utf8proc API version MAJOR.MINOR.PATCH, following
* semantic-versioning rules (http://semver.org) based on API
* compatibility.
*
* This is also returned at runtime by @ref utf8proc_version; however, the
* runtime version may append a string like "-dev" to the version number
* for prerelease versions.
*
* @note The shared-library version number in the Makefile
* (and CMakeLists.txt, and MANIFEST) may be different,
* being based on ABI compatibility rather than API compatibility.
*/
/** @{ */
/** The MAJOR version number (increased when backwards API compatibility is broken). */
#define UTF8PROC_VERSION_MAJOR 2
/** The MINOR version number (increased when new functionality is added in a backwards-compatible manner). */
#define UTF8PROC_VERSION_MINOR 0
/** The PATCH version (increased for fixes that do not change the API). */
#define UTF8PROC_VERSION_PATCH 2
/** @} */
#include <stdlib.h>
#include <sys/types.h>
#ifdef _MSC_VER
typedef signed char utf8proc_int8_t;
typedef unsigned char utf8proc_uint8_t;
typedef short utf8proc_int16_t;
typedef unsigned short utf8proc_uint16_t;
typedef int utf8proc_int32_t;
typedef unsigned int utf8proc_uint32_t;
# ifdef _WIN64
typedef __int64 utf8proc_ssize_t;
typedef unsigned __int64 utf8proc_size_t;
# else
typedef int utf8proc_ssize_t;
typedef unsigned int utf8proc_size_t;
# endif
# ifndef __cplusplus
typedef unsigned char utf8proc_bool;
//enum {false, true};
# else
typedef bool utf8proc_bool;
# endif
#else
# include <stdbool.h>
# include <inttypes.h>
typedef int8_t utf8proc_int8_t;
typedef uint8_t utf8proc_uint8_t;
typedef int16_t utf8proc_int16_t;
typedef uint16_t utf8proc_uint16_t;
typedef int32_t utf8proc_int32_t;
typedef uint32_t utf8proc_uint32_t;
typedef size_t utf8proc_size_t;
typedef ssize_t utf8proc_ssize_t;
typedef bool utf8proc_bool;
#endif
#include <limits.h>
#ifndef SSIZE_MAX
#define SSIZE_MAX ((size_t)SIZE_MAX/2)
#endif
#ifndef UINT16_MAX
# define UINT16_MAX ~(utf8proc_uint16_t)0
#endif
/**
* Option flags used by several functions in the library.
*/
typedef enum {
/** The given UTF-8 input is NULL terminated. */
UTF8PROC_NULLTERM = (1<<0),
/** Unicode Versioning Stability has to be respected. */
UTF8PROC_STABLE = (1<<1),
/** Compatibility decomposition (i.e. formatting information is lost). */
UTF8PROC_COMPAT = (1<<2),
/** Return a result with decomposed characters. */
UTF8PROC_COMPOSE = (1<<3),
/** Return a result with decomposed characters. */
UTF8PROC_DECOMPOSE = (1<<4),
/** Strip "default ignorable characters" such as SOFT-HYPHEN or ZERO-WIDTH-SPACE. */
UTF8PROC_IGNORE = (1<<5),
/** Return an error, if the input contains unassigned codepoints. */
UTF8PROC_REJECTNA = (1<<6),
/**
* Indicating that NLF-sequences (LF, CRLF, CR, NEL) are representing a
* line break, and should be converted to the codepoint for line
* separation (LS).
*/
UTF8PROC_NLF2LS = (1<<7),
/**
* Indicating that NLF-sequences are representing a paragraph break, and
* should be converted to the codepoint for paragraph separation
* (PS).
*/
UTF8PROC_NLF2PS = (1<<8),
/** Indicating that the meaning of NLF-sequences is unknown. */
UTF8PROC_NLF2LF = (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS),
/** Strips and/or convers control characters.
*
* NLF-sequences are transformed into space, except if one of the
* NLF2LS/PS/LF options is given. HorizontalTab (HT) and FormFeed (FF)
* are treated as a NLF-sequence in this case. All other control
* characters are simply removed.
*/
UTF8PROC_STRIPCC = (1<<9),
/**
* Performs unicode case folding, to be able to do a case-insensitive
* string comparison.
*/
UTF8PROC_CASEFOLD = (1<<10),
/**
* Inserts 0xFF bytes at the beginning of each sequence which is
* representing a single grapheme cluster (see UAX#29).
*/
UTF8PROC_CHARBOUND = (1<<11),
/** Lumps certain characters together.
*
* E.g. HYPHEN U+2010 and MINUS U+2212 to ASCII "-". See lump.md for details.
*
* If NLF2LF is set, this includes a transformation of paragraph and
* line separators to ASCII line-feed (LF).
*/
UTF8PROC_LUMP = (1<<12),
/** Strips all character markings.
*
* This includes non-spacing, spacing and enclosing (i.e. accents).
* @note This option works only with @ref UTF8PROC_COMPOSE or
* @ref UTF8PROC_DECOMPOSE
*/
UTF8PROC_STRIPMARK = (1<<13),
} utf8proc_option_t;
/** @name Error codes
* Error codes being returned by almost all functions.
*/
/** @{ */
/** Memory could not be allocated. */
#define UTF8PROC_ERROR_NOMEM -1
/** The given string is too long to be processed. */
#define UTF8PROC_ERROR_OVERFLOW -2
/** The given string is not a legal UTF-8 string. */
#define UTF8PROC_ERROR_INVALIDUTF8 -3
/** The @ref UTF8PROC_REJECTNA flag was set and an unassigned codepoint was found. */
#define UTF8PROC_ERROR_NOTASSIGNED -4
/** Invalid options have been used. */
#define UTF8PROC_ERROR_INVALIDOPTS -5
/** @} */
/* @name Types */
/** Holds the value of a property. */
typedef utf8proc_int16_t utf8proc_propval_t;
/** Struct containing information about a codepoint. */
typedef struct utf8proc_property_struct {
/**
* Unicode category.
* @see utf8proc_category_t.
*/
utf8proc_propval_t category;
utf8proc_propval_t combining_class;
/**
* Bidirectional class.
* @see utf8proc_bidi_class_t.
*/
utf8proc_propval_t bidi_class;
/**
* @anchor Decomposition type.
* @see utf8proc_decomp_type_t.
*/
utf8proc_propval_t decomp_type;
utf8proc_uint16_t decomp_seqindex;
utf8proc_uint16_t casefold_seqindex;
utf8proc_uint16_t uppercase_seqindex;
utf8proc_uint16_t lowercase_seqindex;
utf8proc_uint16_t titlecase_seqindex;
utf8proc_uint16_t comb_index;
unsigned bidi_mirrored:1;
unsigned comp_exclusion:1;
/**
* Can this codepoint be ignored?
*
* Used by @ref utf8proc_decompose_char when @ref UTF8PROC_IGNORE is
* passed as an option.
*/
unsigned ignorable:1;
unsigned control_boundary:1;
/** The width of the codepoint. */
unsigned charwidth:2;
unsigned pad:2;
/**
* Boundclass.
* @see utf8proc_boundclass_t.
*/
unsigned boundclass:8;
} utf8proc_property_t;
/** Unicode categories. */
typedef enum {
UTF8PROC_CATEGORY_CN = 0, /**< Other, not assigned */
UTF8PROC_CATEGORY_LU = 1, /**< Letter, uppercase */
UTF8PROC_CATEGORY_LL = 2, /**< Letter, lowercase */
UTF8PROC_CATEGORY_LT = 3, /**< Letter, titlecase */
UTF8PROC_CATEGORY_LM = 4, /**< Letter, modifier */
UTF8PROC_CATEGORY_LO = 5, /**< Letter, other */
UTF8PROC_CATEGORY_MN = 6, /**< Mark, nonspacing */
UTF8PROC_CATEGORY_MC = 7, /**< Mark, spacing combining */
UTF8PROC_CATEGORY_ME = 8, /**< Mark, enclosing */
UTF8PROC_CATEGORY_ND = 9, /**< Number, decimal digit */
UTF8PROC_CATEGORY_NL = 10, /**< Number, letter */
UTF8PROC_CATEGORY_NO = 11, /**< Number, other */
UTF8PROC_CATEGORY_PC = 12, /**< Punctuation, connector */
UTF8PROC_CATEGORY_PD = 13, /**< Punctuation, dash */
UTF8PROC_CATEGORY_PS = 14, /**< Punctuation, open */
UTF8PROC_CATEGORY_PE = 15, /**< Punctuation, close */
UTF8PROC_CATEGORY_PI = 16, /**< Punctuation, initial quote */
UTF8PROC_CATEGORY_PF = 17, /**< Punctuation, final quote */
UTF8PROC_CATEGORY_PO = 18, /**< Punctuation, other */
UTF8PROC_CATEGORY_SM = 19, /**< Symbol, math */
UTF8PROC_CATEGORY_SC = 20, /**< Symbol, currency */
UTF8PROC_CATEGORY_SK = 21, /**< Symbol, modifier */
UTF8PROC_CATEGORY_SO = 22, /**< Symbol, other */
UTF8PROC_CATEGORY_ZS = 23, /**< Separator, space */
UTF8PROC_CATEGORY_ZL = 24, /**< Separator, line */
UTF8PROC_CATEGORY_ZP = 25, /**< Separator, paragraph */
UTF8PROC_CATEGORY_CC = 26, /**< Other, control */
UTF8PROC_CATEGORY_CF = 27, /**< Other, format */
UTF8PROC_CATEGORY_CS = 28, /**< Other, surrogate */
UTF8PROC_CATEGORY_CO = 29, /**< Other, private use */
} utf8proc_category_t;
/** Bidirectional character classes. */
typedef enum {
UTF8PROC_BIDI_CLASS_L = 1, /**< Left-to-Right */
UTF8PROC_BIDI_CLASS_LRE = 2, /**< Left-to-Right Embedding */
UTF8PROC_BIDI_CLASS_LRO = 3, /**< Left-to-Right Override */
UTF8PROC_BIDI_CLASS_R = 4, /**< Right-to-Left */
UTF8PROC_BIDI_CLASS_AL = 5, /**< Right-to-Left Arabic */
UTF8PROC_BIDI_CLASS_RLE = 6, /**< Right-to-Left Embedding */
UTF8PROC_BIDI_CLASS_RLO = 7, /**< Right-to-Left Override */
UTF8PROC_BIDI_CLASS_PDF = 8, /**< Pop Directional Format */
UTF8PROC_BIDI_CLASS_EN = 9, /**< European Number */
UTF8PROC_BIDI_CLASS_ES = 10, /**< European Separator */
UTF8PROC_BIDI_CLASS_ET = 11, /**< European Number Terminator */
UTF8PROC_BIDI_CLASS_AN = 12, /**< Arabic Number */
UTF8PROC_BIDI_CLASS_CS = 13, /**< Common Number Separator */
UTF8PROC_BIDI_CLASS_NSM = 14, /**< Nonspacing Mark */
UTF8PROC_BIDI_CLASS_BN = 15, /**< Boundary Neutral */
UTF8PROC_BIDI_CLASS_B = 16, /**< Paragraph Separator */
UTF8PROC_BIDI_CLASS_S = 17, /**< Segment Separator */
UTF8PROC_BIDI_CLASS_WS = 18, /**< Whitespace */
UTF8PROC_BIDI_CLASS_ON = 19, /**< Other Neutrals */
UTF8PROC_BIDI_CLASS_LRI = 20, /**< Left-to-Right Isolate */
UTF8PROC_BIDI_CLASS_RLI = 21, /**< Right-to-Left Isolate */
UTF8PROC_BIDI_CLASS_FSI = 22, /**< First Strong Isolate */
UTF8PROC_BIDI_CLASS_PDI = 23, /**< Pop Directional Isolate */
} utf8proc_bidi_class_t;
/** Decomposition type. */
typedef enum {
UTF8PROC_DECOMP_TYPE_FONT = 1, /**< Font */
UTF8PROC_DECOMP_TYPE_NOBREAK = 2, /**< Nobreak */
UTF8PROC_DECOMP_TYPE_INITIAL = 3, /**< Initial */
UTF8PROC_DECOMP_TYPE_MEDIAL = 4, /**< Medial */
UTF8PROC_DECOMP_TYPE_FINAL = 5, /**< Final */
UTF8PROC_DECOMP_TYPE_ISOLATED = 6, /**< Isolated */
UTF8PROC_DECOMP_TYPE_CIRCLE = 7, /**< Circle */
UTF8PROC_DECOMP_TYPE_SUPER = 8, /**< Super */
UTF8PROC_DECOMP_TYPE_SUB = 9, /**< Sub */
UTF8PROC_DECOMP_TYPE_VERTICAL = 10, /**< Vertical */
UTF8PROC_DECOMP_TYPE_WIDE = 11, /**< Wide */
UTF8PROC_DECOMP_TYPE_NARROW = 12, /**< Narrow */
UTF8PROC_DECOMP_TYPE_SMALL = 13, /**< Small */
UTF8PROC_DECOMP_TYPE_SQUARE = 14, /**< Square */
UTF8PROC_DECOMP_TYPE_FRACTION = 15, /**< Fraction */
UTF8PROC_DECOMP_TYPE_COMPAT = 16, /**< Compat */
} utf8proc_decomp_type_t;
/** Boundclass property. (TR29) */
typedef enum {
UTF8PROC_BOUNDCLASS_START = 0, /**< Start */
UTF8PROC_BOUNDCLASS_OTHER = 1, /**< Other */
UTF8PROC_BOUNDCLASS_CR = 2, /**< Cr */
UTF8PROC_BOUNDCLASS_LF = 3, /**< Lf */
UTF8PROC_BOUNDCLASS_CONTROL = 4, /**< Control */
UTF8PROC_BOUNDCLASS_EXTEND = 5, /**< Extend */
UTF8PROC_BOUNDCLASS_L = 6, /**< L */
UTF8PROC_BOUNDCLASS_V = 7, /**< V */
UTF8PROC_BOUNDCLASS_T = 8, /**< T */
UTF8PROC_BOUNDCLASS_LV = 9, /**< Lv */
UTF8PROC_BOUNDCLASS_LVT = 10, /**< Lvt */
UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR = 11, /**< Regional indicator */
UTF8PROC_BOUNDCLASS_SPACINGMARK = 12, /**< Spacingmark */
UTF8PROC_BOUNDCLASS_PREPEND = 13, /**< Prepend */
UTF8PROC_BOUNDCLASS_ZWJ = 14, /**< Zero Width Joiner */
UTF8PROC_BOUNDCLASS_E_BASE = 15, /**< Emoji Base */
UTF8PROC_BOUNDCLASS_E_MODIFIER = 16, /**< Emoji Modifier */
UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ = 17, /**< Glue_After_ZWJ */
UTF8PROC_BOUNDCLASS_E_BASE_GAZ = 18, /**< E_BASE + GLUE_AFTER_ZJW */
} utf8proc_boundclass_t;
/**
* Array containing the byte lengths of a UTF-8 encoded codepoint based
* on the first byte.
*/
extern const utf8proc_int8_t utf8proc_utf8class[256];
/**
* Returns the utf8proc API version as a string MAJOR.MINOR.PATCH
* (http://semver.org format), possibly with a "-dev" suffix for
* development versions.
*/
const char *utf8proc_version(void);
/**
* Returns an informative error string for the given utf8proc error code
* (e.g. the error codes returned by @ref utf8proc_map).
*/
const char *utf8proc_errmsg(utf8proc_ssize_t errcode);
/**
* Reads a single codepoint from the UTF-8 sequence being pointed to by `str`.
* The maximum number of bytes read is `strlen`, unless `strlen` is
* negative (in which case up to 4 bytes are read).
*
* If a valid codepoint could be read, it is stored in the variable
* pointed to by `codepoint_ref`, otherwise that variable will be set to -1.
* In case of success, the number of bytes read is returned; otherwise, a
* negative error code is returned.
*/
utf8proc_ssize_t utf8proc_iterate(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *codepoint_ref);
/**
* Check if a codepoint is valid (regardless of whether it has been
* assigned a value by the current Unicode standard).
*
* @return 1 if the given `codepoint` is valid and otherwise return 0.
*/
utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t codepoint);
/**
* Encodes the codepoint as an UTF-8 string in the byte array pointed
* to by `dst`. This array must be at least 4 bytes long.
*
* In case of success the number of bytes written is returned, and
* otherwise 0 is returned.
*
* This function does not check whether `codepoint` is valid Unicode.
*/
utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t codepoint, utf8proc_uint8_t *dst);
/**
* Look up the properties for a given codepoint.
*
* @param codepoint The Unicode codepoint.
*
* @returns
* A pointer to a (constant) struct containing information about
* the codepoint.
* @par
* If the codepoint is unassigned or invalid, a pointer to a special struct is
* returned in which `category` is 0 (@ref UTF8PROC_CATEGORY_CN).
*/
const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t codepoint);
/** Decompose a codepoint into an array of codepoints.
*
* @param codepoint the codepoint.
* @param dst the destination buffer.
* @param bufsize the size of the destination buffer.
* @param options one or more of the following flags:
* - @ref UTF8PROC_REJECTNA - return an error `codepoint` is unassigned
* - @ref UTF8PROC_IGNORE - strip "default ignorable" codepoints
* - @ref UTF8PROC_CASEFOLD - apply Unicode casefolding
* - @ref UTF8PROC_COMPAT - replace certain codepoints with their
* compatibility decomposition
* - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster
* - @ref UTF8PROC_LUMP - lump certain different codepoints together
* - @ref UTF8PROC_STRIPMARK - remove all character marks
* @param last_boundclass
* Pointer to an integer variable containing
* the previous codepoint's boundary class if the @ref UTF8PROC_CHARBOUND
* option is used. Otherwise, this parameter is ignored.
*
* @return
* In case of success, the number of codepoints written is returned; in case
* of an error, a negative error code is returned (@ref utf8proc_errmsg).
* @par
* If the number of written codepoints would be bigger than `bufsize`, the
* required buffer size is returned, while the buffer will be overwritten with
* undefined data.
*/
utf8proc_ssize_t utf8proc_decompose_char(
utf8proc_int32_t codepoint, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize,
utf8proc_option_t options, int *last_boundclass
);
/**
* The same as @ref utf8proc_decompose_char, but acts on a whole UTF-8
* string and orders the decomposed sequences correctly.
*
* If the @ref UTF8PROC_NULLTERM flag in `options` is set, processing
* will be stopped, when a NULL byte is encounted, otherwise `strlen`
* bytes are processed. The result (in the form of 32-bit unicode
* codepoints) is written into the buffer being pointed to by
* `buffer` (which must contain at least `bufsize` entries). In case of
* success, the number of codepoints written is returned; in case of an
* error, a negative error code is returned (@ref utf8proc_errmsg).
*
* If the number of written codepoints would be bigger than `bufsize`, the
* required buffer size is returned, while the buffer will be overwritten with
* undefined data.
*/
utf8proc_ssize_t utf8proc_decompose(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
);
/**
* Reencodes the sequence of `length` codepoints pointed to by `buffer`
* UTF-8 data in-place (i.e., the result is also stored in `buffer`).
*
* @param buffer the (native-endian UTF-32) unicode codepoints to re-encode.
* @param length the length (in codepoints) of the buffer.
* @param options a bitwise or (`|`) of one or more of the following flags:
* - @ref UTF8PROC_NLF2LS - convert LF, CRLF, CR and NEL into LS
* - @ref UTF8PROC_NLF2PS - convert LF, CRLF, CR and NEL into PS
* - @ref UTF8PROC_NLF2LF - convert LF, CRLF, CR and NEL into LF
* - @ref UTF8PROC_STRIPCC - strip or convert all non-affected control characters
* - @ref UTF8PROC_COMPOSE - try to combine decomposed codepoints into composite
* codepoints
* - @ref UTF8PROC_STABLE - prohibit combining characters that would violate
* the unicode versioning stability
*
* @return
* In case of success, the length (in bytes) of the resulting UTF-8 string is
* returned; otherwise, a negative error code is returned (@ref utf8proc_errmsg).
*
* @warning The amount of free space pointed to by `buffer` must
* exceed the amount of the input data by one byte, and the
* entries of the array pointed to by `str` have to be in the
* range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
*/
utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options);
/**
* Given a pair of consecutive codepoints, return whether a grapheme break is
* permitted between them (as defined by the extended grapheme clusters in UAX#29).
*
* @param state Beginning with Version 29 (Unicode 9.0.0), this algorithm requires
* state to break graphemes. This state can be passed in as a pointer
* in the `state` argument and should initially be set to 0. If the
* state is not passed in (i.e. a null pointer is passed), UAX#29 rules
* GB10/12/13 which require this state will not be applied, essentially
* matching the rules in Unicode 8.0.0.
*
* @warning If the state parameter is used, `utf8proc_grapheme_break_stateful` must
* be called IN ORDER on ALL potential breaks in a string.
*/
utf8proc_bool utf8proc_grapheme_break_stateful(
utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2, utf8proc_int32_t *state);
/**
* Same as @ref utf8proc_grapheme_break_stateful, except without support for the
* Unicode 9 additions to the algorithm. Supported for legacy reasons.
*/
utf8proc_bool utf8proc_grapheme_break(
utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2);
/**
* Given a codepoint `c`, return the codepoint of the corresponding
* lower-case character, if any; otherwise (if there is no lower-case
* variant, or if `c` is not a valid codepoint) return `c`.
*/
utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c);
/**
* Given a codepoint `c`, return the codepoint of the corresponding
* upper-case character, if any; otherwise (if there is no upper-case
* variant, or if `c` is not a valid codepoint) return `c`.
*/
utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c);
/**
* Given a codepoint `c`, return the codepoint of the corresponding
* title-case character, if any; otherwise (if there is no title-case
* variant, or if `c` is not a valid codepoint) return `c`.
*/
utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c);
/**
* Given a codepoint, return a character width analogous to `wcwidth(codepoint)`,
* except that a width of 0 is returned for non-printable codepoints
* instead of -1 as in `wcwidth`.
*
* @note
* If you want to check for particular types of non-printable characters,
* (analogous to `isprint` or `iscntrl`), use @ref utf8proc_category. */
int utf8proc_charwidth(utf8proc_int32_t codepoint);
/**
* Return the Unicode category for the codepoint (one of the
* @ref utf8proc_category_t constants.)
*/
utf8proc_category_t utf8proc_category(utf8proc_int32_t codepoint);
/**
* Return the two-letter (nul-terminated) Unicode category string for
* the codepoint (e.g. `"Lu"` or `"Co"`).
*/
const char *utf8proc_category_string(utf8proc_int32_t codepoint);
/**
* Maps the given UTF-8 string pointed to by `str` to a new UTF-8
* string, allocated dynamically by `malloc` and returned via `dstptr`.
*
* If the @ref UTF8PROC_NULLTERM flag in the `options` field is set,
* the length is determined by a NULL terminator, otherwise the
* parameter `strlen` is evaluated to determine the string length, but
* in any case the result will be NULL terminated (though it might
* contain NULL characters with the string if `str` contained NULL
* characters). Other flags in the `options` field are passed to the
* functions defined above, and regarded as described.
*
* In case of success the length of the new string is returned,
* otherwise a negative error code is returned.
*
* @note The memory of the new UTF-8 string will have been allocated
* with `malloc`, and should therefore be deallocated with `free`.
*/
utf8proc_ssize_t utf8proc_map(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
);
/** @name Unicode normalization
*
* Returns a pointer to newly allocated memory of a NFD, NFC, NFKD or NFKC
* normalized version of the null-terminated string `str`. These
* are shortcuts to calling @ref utf8proc_map with @ref UTF8PROC_NULLTERM
* combined with @ref UTF8PROC_STABLE and flags indicating the normalization.
*/
/** @{ */
/** NFD normalization (@ref UTF8PROC_DECOMPOSE). */
utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str);
/** NFC normalization (@ref UTF8PROC_COMPOSE). */
utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str);
/** NFD normalization (@ref UTF8PROC_DECOMPOSE and @ref UTF8PROC_COMPAT). */
utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str);
/** NFD normalization (@ref UTF8PROC_COMPOSE and @ref UTF8PROC_COMPAT). */
utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str);
/** @} */
#ifdef __cplusplus
}
#endif
#endif

14388
src/utf8proc/utf8proc_data.c Normal file
View File

File diff suppressed because it is too large Load Diff

7
windows/ccextractor.vcxproj
View File

@ -79,7 +79,9 @@
<ClInclude Include="..\src\lib_ccx\teletext.h" />
<ClInclude Include="..\src\lib_ccx\utility.h" />
<ClInclude Include="..\src\lib_hash\sha2.h" />
<ClInclude Include="..\src\microutf8\microutf8.h" />
<ClInclude Include="..\src\protobuf-c\protobuf-c.h" />
<ClInclude Include="..\src\utf8proc\utf8proc.h" />
<ClInclude Include="..\src\win_iconv\iconv.h" />
<ClInclude Include="..\src\win_spec_incld\dirent.h" />
<ClInclude Include="..\src\zlib\crc32.h" />
@ -186,8 +188,10 @@
<ClCompile Include="..\src\lib_ccx\ccx_encoders_helpers.c" />
<ClCompile Include="..\src\lib_ccx\ccx_encoders_sami.c" />
<ClCompile Include="..\src\lib_ccx\ccx_encoders_smptett.c" />
<ClCompile Include="..\src\lib_ccx\ccx_encoders_splitbysentence.c" />
<ClCompile Include="..\src\lib_ccx\ccx_encoders_spupng.c" />
<ClCompile Include="..\src\lib_ccx\ccx_encoders_srt.c" />
<ClCompile Include="..\src\lib_ccx\ccx_encoders_transcript.c" />
<ClCompile Include="..\src\lib_ccx\ccx_encoders_webvtt.c" />
<ClCompile Include="..\src\lib_ccx\ccx_encoders_xds.c" />
<ClCompile Include="..\src\lib_ccx\ccx_gxf.c" />
@ -220,6 +224,7 @@
<ClCompile Include="..\src\lib_ccx\wtv_functions.c" />
<ClCompile Include="..\src\lib_hash\sha2.c" />
<ClCompile Include="..\src\protobuf-c\protobuf-c.c" />
<ClCompile Include="..\src\utf8proc\utf8proc.c" />
<ClCompile Include="..\src\win_iconv\win_iconv.c" />
<ClCompile Include="..\src\zlib\adler32.c" />
<ClCompile Include="..\src\zlib\crc32.c" />
@ -329,7 +334,7 @@
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug-OCR|Win32'">
<ClCompile>
<Optimization>Disabled</Optimization>
<AdditionalIncludeDirectories>../src/win_spec_incld;../src/gpacmp4;../src/libpng;../src/zlib;../src;../src/lib_ccx;../src/zvbi;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>../src/utf8proc;../src/win_spec_incld;../src/gpacmp4;../src/libpng;../src/zlib;../src;../src/lib_ccx;../src/zvbi;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<PreprocessorDefinitions>ENABLE_OCR;WIN32;_DEBUG;_CONSOLE;_FILE_OFFSET_BITS=64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<MinimalRebuild>true</MinimalRebuild>
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>

21
windows/ccextractor.vcxproj.filters
View File

@ -82,6 +82,12 @@
<Filter Include="Header Files\protobuf-c">
<UniqueIdentifier>{fce551b0-7304-4677-8e66-61b475fdb924}</UniqueIdentifier>
</Filter>
<Filter Include="Source Files\utf8proc">
<UniqueIdentifier>{6e23dd8b-166b-4a80-92e2-194e7b6e6c86}</UniqueIdentifier>
</Filter>
<Filter Include="Header Files\utf8proc">
<UniqueIdentifier>{fcf59c1c-8b84-484a-8fb4-a9a301e23610}</UniqueIdentifier>
</Filter>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\src\gpacmp4\gpac\avparse.h">
@ -336,6 +342,12 @@
<ClInclude Include="..\src\protobuf-c\protobuf-c.h">
<Filter>Header Files\protobuf-c</Filter>
</ClInclude>
<ClInclude Include="..\src\microutf8\microutf8.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="..\src\utf8proc\utf8proc.h">
<Filter>Header Files\utf8proc</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\src\ccextractor.c">
@ -734,6 +746,15 @@
<ClCompile Include="..\src\lib_ccx\dvd_subtitle_decoder.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\src\lib_ccx\ccx_encoders_transcript.c">
<Filter>Source Files\ccx_encoders</Filter>
</ClCompile>
<ClCompile Include="..\src\lib_ccx\ccx_encoders_splitbysentence.c">
<Filter>Source Files\ccx_encoders</Filter>
</ClCompile>
<ClCompile Include="..\src\utf8proc\utf8proc.c">
<Filter>Source Files\utf8proc</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<None Include="..\src\lib_ccx\ccx_sub_entry_message.proto">