build system: implement fmt::format & Qt verification in translation files

Rakefile

@@ -343,6 +343,12 @@ if $building_for[:windows]
 end
 
 rule '.mo' => '.po' do |t|
+  if !%r{doc/man}.match(t.sources[0])
+    runq_code "VERIFY-PO-FMT", :target => t.sources[0] do
+      FormatStringVerifier.new.verify t.sources[0]
+    end
+  end
+
   runq "msgfmt", t.source, "#{c(:MSGFMT)} -c -o #{t.name} #{t.sources.join(" ")}"
 end
 

rake.d/format_string_verifier.rb

@@ -1,45 +1,44 @@
 #!/usr/bin/env ruby
 
+require "digest"
+
 class FormatStringVerifier
   def verify file_name
+    language  = file_name.gsub(%r{.*/|\.po$}, '')
+    to_ignore = {
+      'eu' => [ '74f8165349457fb0d89a559ffa83c4b3b3c1993b', 'aa8f04e33094d67c75256a298364323f38967abf' ],
+      'tr' => [ '62fc63d5ae795e24fcdbdfd5ec300c7bf5e46b76' ],
+    }
+
     entries = read_po(file_name).
       reject { |e| e[:obsolete] }.
       select { |e| e[:msgid] && e[:msgstr] && e[:flags] }.
       reject { |e| e[:msgid].empty? || e[:msgstr].empty? }.
       reject { |e| e[:msgstr].all?(&:empty?) }.
-      reject { |e| e[:flags].include?("fuzzy") }.
-      select { |e| e[:flags].include?("c-format") || e[:flags].include?("boost-format") }
+      reject { |e| e[:flags].include?("fuzzy") }
 
     errors = []
 
     matchers = {
-      :boost => /%(?:
-                     %
-                   | \|[0-9$a-zA-Z\.\-]+\|
-                   | [0-9]+%?
-                   )/ix,
-
-      :c      => /%(?:
-                      %
-                    | -?\.?\s?[0-9]*l?l?[a-zA-Z]
-                    )/ix,
+      :fmt => /( \{ \d+ (?: : [^\}]+ )? \} )/ix,
+      :qt  => /( (?<! %) % \d+ (?! [a-z]) )/ix,
     }
 
     format_types = matchers.keys
 
     entries.each do |e|
-      formats = {
-        :c     => { :id => [], :str => [] },
-        :boost => { :id => [], :str => [] },
-      }
-
-      has_errors = false
+      formats       = Hash[ *matchers.keys.map { |key| [ key, { :id => [], :str => [] } ] }.flatten ]
+      error_digests = []
 
       e[:msgstr].each_with_index do |msgstr, idx|
         ( (e[:msgid] || []) + (e[:msgid_plural] || []) ).
           reject(&:nil?).
           each do |msgid|
-            format_types.select { |type| e[:flags].include?("#{type}-format") }.each do |type|
+            format_types.each do |type|
+              sha1 = Digest::SHA1.hexdigest "#{type}:#{msgid}:#{msgstr}"
+
+              next if to_ignore.key?(language) && to_ignore[language].include?(sha1)
+
               store = formats[type]
 
               store[:id]  << msgid .scan(matchers[type]).uniq.sort
@@ -48,18 +47,18 @@ class FormatStringVerifier
               missing = store[:id]  - store[:str]
               added   = store[:str] - store[:id]
 
-              has_errors = true if !missing.empty? || !added.empty?
+              error_digests << sha1 if !missing.empty? || !added.empty?
             end
         end
       end
 
-      errors << e[:line] if has_errors
+      errors << { :line => e[:line], :digests => error_digests } if !error_digests.empty?
     end
 
     # pp errors
 
     errors.each do |error|
-      puts "#{file_name}:#{error}: error: format string differences"
+      puts "#{file_name}:#{error[:line]}: error: format string differences (IDs: #{error[:digests].sort.join(' ')})"
     end
 
     return errors.empty?

rake.d/helpers.rb

@@ -5,7 +5,7 @@ require "fileutils"
 
 $git_mutex     = Mutex.new
 $message_mutex = Mutex.new
-$action_width  = 12
+$action_width  = 14
 
 def puts(message)
   $message_mutex.synchronize {
@@ -109,6 +109,21 @@ def runq_git(msg, cmdline, options = {})
   runq "git #{cmdline.split(/\s+/)[0]}", "#{msg}", "git #{cmdline}", :mutex => $git_mutex
 end
 
+def runq_code(action, options = {})
+  start = Time.now
+
+  puts_action action, :target => options[:target], :prefix => $run_show_start_stop ? "[start          ]" : nil
+
+  ok       = yield
+  duration = Time.now - start
+
+  if $run_show_start_stop
+    puts_action action, :target => options[:target], :prefix => sprintf("[%s  %02d:%02d.%03d]", ok ? "done" : "fail", (duration / 60).to_i, duration.to_i % 60, (duration * 1000).to_i % 1000)
+  end
+
+  exit 1 if !ok
+end
+
 def ensure_dir dir
   File.unlink(dir) if FileTest.exist?(dir) && !FileTest.directory?(dir)
   FileUtils.mkdir_p(dir)

rake.d/po.rb

@@ -152,8 +152,9 @@ def write_po file_name, items
 end
 
 def normalize_po file
-  puts_action "NORMALIZE-PO", :target => file
-  write_po file, read_po(file)
+  runq_code "NORMALIZE-PO", :target => file do
+    write_po file, read_po(file)
+  end
 end
 
 def replace_po_meta_info orig_metas, transifex_meta, key
@@ -219,13 +220,13 @@ def transifex_pull_and_merge resource, language
 
   runq "tx pull", po_file, "tx pull -f -r mkvtoolnix.#{resource} -l #{language} > /dev/null"
 
-  puts_qaction "merge", :target => po_file
+  runq_code "merge", :target => po_file do
+    transifex_items = read_po(po_file)
+    merged_items    = merge_po orig_items, transifex_items
+    fixed_items     = fix_po_msgstr_plurals merged_items
 
-  transifex_items = read_po(po_file)
-  merged_items    = merge_po orig_items, transifex_items
-  fixed_items     = fix_po_msgstr_plurals merged_items
-
-  write_po po_file, fixed_items
+    write_po po_file, fixed_items
+  end
 end
 
 def transifex_remove_fuzzy_and_push resource, language