mirror of
https://gitlab.com/mbunkus/mkvtoolnix.git
synced 2024-10-22 07:21:21 +00:00
39529c226b
The prior method was to generate one line of `g_container.emplace_back(…)` per entry in the list & letting the compiler chew on that. Each string argument in that call was done was `u8"Some Name"s`, meaning as a std::string instance. Drawbacks: • takes the compiler ages to compile, even forcing me to drop all optimizations for the ISO-639 language list file • even smaller files such as the IANA language subtag registry lists take more than 30s to compile • due to no optimizations initialization is actually not as fast as could be The new method uses a plain C-style array of structs with `char const *` entries for the initial list. The initialization method then copies the entries from that list to the actual container, again using `std::emplace_back(…)`. This yields sub-1s compilation times even with the longest file, the ISO-639 language list, and the runtime initialization is actually faster.
197 lines
5.6 KiB
Ruby
197 lines
5.6 KiB
Ruby
def create_iso639_language_list_file
|
|
name_overrides = {
|
|
"grc" => "Greek (ancient, -1453)",
|
|
"gre" => "Greek (modern, 1453-)",
|
|
}
|
|
|
|
content = Mtx::OnlineFile.download("https://www.loc.gov/standards/iso639-2/php/code_list.php", "iso-639-2.html")
|
|
|
|
entries_by_alpha_3 = {}
|
|
|
|
parse_html_extract_table_data(content, %r{^.*?<table[^>]+>.*?<table[^>]+>}im).
|
|
drop(1).
|
|
each do |row|
|
|
if %r{^([a-z]{3}) *\(([bt])\)<.*?>([a-z]{3})}.match(row[0].downcase)
|
|
alpha_3_b = $2 == 'b' ? $1 : $3
|
|
alpha_3_t = $2 == 'b' ? $3 : $1
|
|
else
|
|
alpha_3_b = row[0]
|
|
alpha_3_t = row[0]
|
|
end
|
|
|
|
entries_by_alpha_3[alpha_3_b] = {
|
|
"name" => row[2],
|
|
"bibliographic" => alpha_3_b == alpha_3_t ? nil : alpha_3_b,
|
|
"alpha_2" => row[1],
|
|
"alpha_3" => alpha_3_t,
|
|
"alpha_3_to_use" => alpha_3_b,
|
|
"has_639_2" => true,
|
|
}
|
|
end
|
|
|
|
lines = Mtx::OnlineFile.download("https://iso639-3.sil.org/sites/iso639-3/files/downloads/iso-639-3.tab").
|
|
split(%r{\n+}).
|
|
map(&:chomp)
|
|
|
|
headers = Hash[ *
|
|
lines.
|
|
shift.
|
|
split(%r{\t}).
|
|
map(&:downcase).
|
|
each_with_index.
|
|
map { |name, index| [ index, name ] }.
|
|
flatten
|
|
]
|
|
|
|
lines.
|
|
map do |line|
|
|
parts = line.split(%r{\t})
|
|
entry = Hash[ *
|
|
(0..parts.size).
|
|
map { |idx| [ headers[idx], !parts[idx] || parts[idx].empty? ? nil : parts[idx] ] }.
|
|
flatten
|
|
]
|
|
|
|
entry
|
|
end.
|
|
each do |entry|
|
|
alpha_3_to_use = entry["part2b"] || entry["id"]
|
|
|
|
entry_639_2 = entries_by_alpha_3[alpha_3_to_use]
|
|
entries_by_alpha_3[alpha_3_to_use] = {
|
|
"name" => entry["ref_name"],
|
|
"bibliographic" => entry["part2b"] && (entry["part2b"] != entry["part2t"]) ? entry["part2b"] : nil,
|
|
"alpha_2" => entry["part1"],
|
|
"alpha_3" => entry["part2t"] || entry["id"],
|
|
"alpha_3_to_use" => alpha_3_to_use,
|
|
"has_639_2" => !!entry_639_2,
|
|
}
|
|
end
|
|
|
|
content = Mtx::OnlineFile.download("https://en.wikipedia.org/wiki/List_of_ISO_639-5_codes")
|
|
|
|
parse_html_extract_table_data(content, %r{^.*?<table[^>]+>}im).
|
|
drop(1).
|
|
each do |row|
|
|
alpha_3 = row[1]
|
|
|
|
entries_by_alpha_3[alpha_3] ||= {
|
|
"name" => row[3].gsub(%r{<[^>]+>}, ''),
|
|
"bibliographic" => nil,
|
|
"alpha_2" => nil,
|
|
"alpha_3" => alpha_3,
|
|
"alpha_3_to_use" => alpha_3,
|
|
"has_639_2" => false,
|
|
}
|
|
end
|
|
|
|
("a".."t").each do |letter1|
|
|
("a".."z").each do |letter2|
|
|
alpha_3 = "q#{letter1}#{letter2}"
|
|
entries_by_alpha_3[alpha_3] = {
|
|
"name" => "Reserved for local use: #{alpha_3}",
|
|
"bibliographic" => nil,
|
|
"alpha_2" => nil,
|
|
"alpha_3" => alpha_3,
|
|
"alpha_3_to_use" => alpha_3,
|
|
"has_639_2" => true,
|
|
}
|
|
end
|
|
end
|
|
|
|
entries_by_alpha_2 = Hash[ *
|
|
entries_by_alpha_3.
|
|
values.
|
|
map { |entry| [ entry["alpha_2"], entry ] }.
|
|
flatten
|
|
]
|
|
|
|
Mtx::IANALanguageSubtagRegistry.
|
|
fetch_registry["language"].
|
|
reject { |entry| %r{\.\.}.match(entry[:subtag]) }.
|
|
each do |entry|
|
|
|
|
is_alpha_2 = entry[:subtag].length == 2
|
|
entries_map = is_alpha_2 ? entries_by_alpha_2 : entries_by_alpha_3
|
|
|
|
if !entries_map.key?(entry[:subtag])
|
|
entries_map[entry[:subtag]] = {
|
|
"name" => entry[:description],
|
|
"bibliographic" => nil,
|
|
"alpha_2" => is_alpha_2 ? entry[:subtag] : nil,
|
|
"alpha_3" => is_alpha_2 ? nil : entry[:subtag],
|
|
"alpha_3_to_use" => entry[:subtag],
|
|
"has_639_2" => is_alpha_2,
|
|
}
|
|
end
|
|
|
|
entries_map[entry[:subtag]]["deprecated"] = entry.key?(:deprecated)
|
|
end
|
|
|
|
rows = entries_by_alpha_3.
|
|
values.
|
|
map do |entry|
|
|
name = name_overrides[ entry["alpha_3_to_use"] ] || entry["name"]
|
|
[ name.to_u8_c_string,
|
|
entry["alpha_3_to_use"].to_c_string,
|
|
(entry["alpha_2"] || '').to_c_string,
|
|
entry["bibliographic"] ? entry["alpha_3"].to_c_string : '""',
|
|
entry["has_639_2"].to_s,
|
|
(entry["deprecated"] || false).to_s,
|
|
]
|
|
end
|
|
|
|
header = <<EOT
|
|
/*
|
|
mkvmerge -- utility for splicing together matroska files
|
|
from component media subtypes
|
|
|
|
Distributed under the GPL v2
|
|
see the file COPYING for details
|
|
or visit https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
|
|
|
|
ISO 639 language definitions, lookup functions
|
|
|
|
Written by Moritz Bunkus <moritz@bunkus.org>.
|
|
*/
|
|
|
|
// -----------------------------------------------------------------------
|
|
// NOTE: this file is auto-generated by the "dev:iso639_list" rake target.
|
|
// -----------------------------------------------------------------------
|
|
|
|
#include "common/iso639_types.h"
|
|
|
|
using namespace std::string_literals;
|
|
|
|
namespace mtx::iso639 {
|
|
|
|
std::vector<language_t> g_languages;
|
|
|
|
struct language_init_t {
|
|
char const *english_name, *alpha_3_code, *alpha_2_code, *terminology_abbrev;
|
|
bool is_part_of_iso639_2, is_deprecated;
|
|
};
|
|
|
|
static language_init_t const s_languages_init[] = {
|
|
EOT
|
|
|
|
footer = <<EOT
|
|
};
|
|
|
|
void
|
|
init() {
|
|
g_languages.reserve(#{rows.size});
|
|
|
|
for (language_init_t const *lang = s_languages_init, *end = lang + #{rows.size}; lang < end; ++lang)
|
|
g_languages.emplace_back(lang->english_name, lang->alpha_3_code, lang->alpha_2_code, lang->terminology_abbrev, lang->is_part_of_iso639_2, lang->is_deprecated);
|
|
}
|
|
|
|
} // namespace mtx::iso639
|
|
EOT
|
|
|
|
content = header + format_table(rows.sort, :column_suffix => ',', :row_prefix => " { ", :row_suffix => " },").join("\n") + "\n" + footer
|
|
cpp_file_name = "src/common/iso639_language_list.cpp"
|
|
|
|
runq("write", cpp_file_name) { IO.write("#{$source_dir}/#{cpp_file_name}", content); 0 }
|
|
end
|