diff --git a/lib/releaseme/l10n.rb b/lib/releaseme/l10n.rb index 10d7e99..62230d2 100644 --- a/lib/releaseme/l10n.rb +++ b/lib/releaseme/l10n.rb @@ -1,314 +1,314 @@ #-- # Copyright (C) 2007-2015 Harald Sitter # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License or (at your option) version 3 or any later version # accepted by the membership of KDE e.V. (or its successor approved # by the membership of KDE e.V.), which shall act as a proxy # defined in Section 14 of version 3 of the license. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #++ require 'English' require 'fileutils' require 'thwait' require 'tmpdir' require_relative 'cmakeeditor' require_relative 'logable' require_relative 'source' require_relative 'svn' require_relative 'translationunit' module ReleaseMe # https://techbase.kde.org/Localization/Concepts/Transcript # Downloads scripted l10n helpers. class L10nScriptDownloader attr_reader :artifacts attr_reader :lang attr_reader :tmpdir # Caches available scripts for template (i.e. po file). # For every template in every language we'd have to do vcs.get the cache # does a vcs.list for each language exactly once. It records the directories # available so that we later can do a fast lookup and skip vcs.get # altogether. With each svn request taking ~1 second that is a huge # time saver. class TemplateCache def initialize(l10n) @data = {} @l10n = l10n queue = l10n.languages_queue threads = each_thread do loop_queue(queue) end ThreadsWait.all_waits(threads) end def [](*args) @data[*args] end private attr_reader :l10n def loop_queue(queue) loop do lang = begin queue.pop(true) rescue break # loop empty if an exception was raised end @data[lang] = list(lang) # GIL secures this. end end def each_thread threads = [] l10n.class::THREAD_COUNT.times do threads << Thread.new do Thread.current.abort_on_exception = true yield end end threads end def list(lang) list = l10n.vcs.list(script_file_dir(lang, l10n.i18n_path)) list.split($INPUT_RECORD_SEPARATOR).collect do |x| x.delete('/') end end def script_file_dir(lang, i18n_path) "#{lang}/scripts/#{i18n_path}" end end def initialize(lang, tmpdir, cache, l10n) @lang = lang @tmpdir = tmpdir @scripts_dir = "#{tmpdir}/scripts" @l10n = l10n @artifacts = [] @cache = cache end def download templates.each do |template| name = File.basename(template, '.po') next unless @cache[lang].include?(name) target_dir = "#{@scripts_dir}/#{name}" @l10n.vcs.get(target_dir, "#{script_file_dir}/#{name}") unless Dir.glob("#{target_dir}/*").select { |f| File.file?(f) }.empty? @artifacts = [@scripts_dir] end end @artifacts end private def templates @l10n.templates end def script_file_dir "#{lang}/scripts/#{@l10n.i18n_path}" end end # FIXME: doesn't write master cmake right now... class L10n < TranslationUnit prepend Logable RELEASEME_TEST_DIR = File.absolute_path("#{__dir__}/../../test").freeze def verify_pot(potname) return unless potname.include?('$') raise "l10n pot appears to be a variable. cannot resolve #{potname}" end - def find_templates(directory, pos = []) + def find_templates(directory, pos = [], skip_dir: RELEASEME_TEST_DIR) Dir.glob("#{directory}/**/**/Messages.sh").each do |file| - next if File.absolute_path(file).start_with?(RELEASEME_TEST_DIR) + next if skip_dir && File.absolute_path(file).start_with?(skip_dir) File.readlines(file).each do |line| line.match(%r{[^/\s=]+\.pot}).to_a.each do |match| verify_pot(match) pos << match.sub('.pot', '.po') end end end # Templates must be unique as multiple lines can contribute to the same # template, as such it can happen that a.pot appears twice which can # have unintended consequences by an outside user of the Array. pos.uniq end # FIXME: this has no test backing right now def strip_comments(file) # Strip #~ lines, which once were sensible translations, but then the # strings got removed, so they now stick around in case the strings # return, poor souls, waiting for a comeback, reminds me of Sunset Blvd :( # Problem is that msgfmt adds those to the binary! file = File.new(file, File::RDWR) str = file.read file.rewind file.truncate(0) # Sometimes a fuzzy marker can precede an obsolete translation block, so # first remove any fuzzy obsoletion in the file and then remove any # additional obsoleted lines. # This prevents the fuzzy markers from getting left over. str.gsub!(/^#, fuzzy\n#~.*/, '') str.gsub!(/^#~.*/, '') str = str.strip file << str file.close end def po_file_dir(lang) "#{lang}/messages/#{@i18n_path}" end def get_single(lang, tmpdir) # TODO: maybe class this po_file_name = templates[0] vcs_file_path = "#{po_file_dir(lang)}/#{po_file_name}" po_file_path = "#{tmpdir}/#{po_file_name}" vcs.export(po_file_path, vcs_file_path) files = [] if File.exist?(po_file_path) files << po_file_path strip_comments(po_file_path) end files.uniq end def get_multiple(lang, tmpdir) vcs_path = po_file_dir(lang) return [] if @vcs.list(vcs_path).empty? @vcs.get(tmpdir, vcs_path) files = [] templates.each do |po| po_file_path = tmpdir.dup.concat("/#{po}") next unless File.exist?(po_file_path) files << po_file_path strip_comments(po_file_path) end files.uniq end def get(srcdir, target = File.expand_path("#{srcdir}/po"), edit_cmake: true) Dir.mkdir(target) @templates = find_templates(srcdir) log_info "Downloading translations for #{srcdir}" languages_without_translation = [] has_translation = false # FIXME: due to threading we do explicit pathing, so this probably can go Dir.chdir(srcdir) do queue = languages_queue threads = [] script_cache = L10nScriptDownloader::TemplateCache.new(self) THREAD_COUNT.times do threads << Thread.new do Thread.current.abort_on_exception = true until queue.empty? begin lang = queue.pop(true) rescue # When pop runs into an empty queue with non_block=true it raises # an exception. We'll simply continue with it as our loop should # naturally end anyway. continue end Dir.mktmpdir(self.class.to_s) do |tmpdir| log_debug "#{srcdir} - downloading #{lang}" if templates.count > 1 files = get_multiple(lang, tmpdir) elsif templates.count == 1 files = get_single(lang, tmpdir) else # FIXME: needs testcase # TODO: this previously aborted entirely, not sure that makes # sense with threading next # No translations need fetching end files += L10nScriptDownloader.new(lang, tmpdir, script_cache, self).download # No files obtained :( if files.empty? # FIXME: not thread safe without GIL languages_without_translation << lang next end # FIXME: not thread safe without GIL has_translation = true # TODO: path confusing with target destination = "#{target}/#{lang}" Dir.mkdir(destination) FileUtils.mv(files, destination) end # FIXME: this is not thread safe without a GIL @languages += [lang] end end end ThreadsWait.all_waits(threads) if completion_requirement = ENV.fetch('RELEASEME_L10N_REQUIREMENT', nil).to_i require_relative 'l10nstatistics' stats = L10nStatistics.new.tap { |l| l.gather!(target) }.stats drop = stats.delete_if { |_, s| s[:percentage] >= completion_requirement } drop.each { |lang, _| FileUtils.rm_r("#{target}/#{lang}", verbose: true) } has_translation = false if Dir.glob("#{target}/*").empty? end if has_translation && edit_cmake # Update CMakeLists.txt CMakeEditor.append_po_install_instructions!(Dir.pwd, 'po') elsif !has_translation # Remove the empty translations directory Dir.delete('po') end end return if languages_without_translation.empty? print_missing_languages(languages_without_translation) end def print_missing_languages(missing) if (languages - missing).empty? path = po_file_dir('$lang') log_warn "!!! No translations found at SVN path #{path} !!!" log_warn "Looked for templates: #{@templates}" else log_info "No translations for: #{missing.join(', ')}" end end end end diff --git a/test/test_l10n.rb b/test/test_l10n.rb index b97e18a..92d7472 100644 --- a/test/test_l10n.rb +++ b/test/test_l10n.rb @@ -1,276 +1,276 @@ #-- # Copyright (C) 2015-2017 Harald Sitter # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License or (at your option) version 3 or any later version # accepted by the membership of KDE e.V. (or its successor approved # by the membership of KDE e.V.), which shall act as a proxy # defined in Section 14 of version 3 of the license. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #++ require 'fileutils' require_relative 'lib/testme' require_relative '../lib/releaseme/l10n.rb' require_relative '../lib/releaseme/l10nstatistics.rb' require_relative '../lib/releaseme/documentation.rb' # FIXME: test stable class TestL10n < Testme def setup @repo_data_dir = data('l10nrepo/') @i18n_path = 'extragear-multimedia' @trunk_url = 'trunk/l10n-kf5/' @stable_url = 'branches/stable/l10n-kf5' @dir = 'tmp_l10n' @svn_template_dir = 'tmp_l10n_repo' @svn_checkout_dir = 'tmp_l10n_check' `svnadmin create #{@svn_template_dir}` assert(File.exist?(@svn_template_dir)) `svn co file://#{Dir.pwd}/#{@svn_template_dir} #{@svn_checkout_dir}` FileUtils.cp_r("#{@repo_data_dir}/trunk", @svn_checkout_dir) FileUtils.cp_r("#{@repo_data_dir}/branches", @svn_checkout_dir) Dir.chdir(@svn_checkout_dir) do `svn add *` `svn ci -m 'yolo'` end end def teardown FileUtils.rm_rf(@svn_template_dir) FileUtils.rm_rf(@svn_checkout_dir) FileUtils.rm_rf(@dir) end def create_l10n l = ReleaseMe::L10n.new(ReleaseMe::L10n::TRUNK, 'amarok', @i18n_path) l.target = "#{@dir}/l10n" l end def test_find_templates l = create_l10n - templates = l.find_templates(data('multi-pot')) + templates = l.find_templates(data('multi-pot'), skip_dir: nil) assert_equal(templates.count, 2) - templates = l.find_templates(data('single-pot')) + templates = l.find_templates(data('single-pot'), skip_dir: nil) assert_equal(templates.count, 1) end def test_get_po # For visual string consinstency we actually interpolate pointlessly below. # rubocop:disable Style/UnneededInterpolation l = create_l10n l.init_repo_url("file://#{Dir.pwd}/#{@svn_template_dir}") FileUtils.rm_rf(@dir) FileUtils.cp_r(data('single-pot'), @dir) l.get(@dir) assert(File.exist?("#{@dir}")) assert(File.exist?("#{@dir}/CMakeLists.txt")) assert(!File.exist?("#{@dir}/l10n")) # temp dir must not be there assert(File.exist?("#{@dir}/po")) assert(File.exist?("#{@dir}/po/de/amarok.po")) FileUtils.rm_rf(@dir) FileUtils.cp_r(data('multi-pot'), @dir) l.get(@dir) assert(File.exist?("#{@dir}")) assert(File.exist?("#{@dir}/CMakeLists.txt")) assert(!File.exist?("#{@dir}/l10n")) # temp dir must not be there assert(File.exist?("#{@dir}/po")) assert(File.exist?("#{@dir}/po/de/amarok.po")) assert(File.exist?("#{@dir}/po/de/amarokcollectionscanner_qt.po")) end def test_get_po_elsewhere l = create_l10n l.init_repo_url("file://#{Dir.pwd}/#{@svn_template_dir}") @elsewhere = "#{Dir.pwd}/elsewhere_tmp_l10n" FileUtils.rm_rf(@dir) FileUtils.cp_r(data('single-pot'), @dir) l.get(@dir, @elsewhere, edit_cmake: false) assert_path_exist("#{@elsewhere}/de/amarok.po") end def test_get_po_absolute_srcdir # Make sure we can pass an absolute dir as srcdir param. l = create_l10n l.init_repo_url("file://#{Dir.pwd}/#{@svn_template_dir}") FileUtils.rm_rf(@dir) FileUtils.cp_r(data('single-pot'), @dir) l.get(File.absolute_path(@dir)) assert_path_exist("#{@dir}/po/de/amarok.po") end def test_get_po_edit_cmake l = create_l10n l.init_repo_url("file://#{Dir.pwd}/#{@svn_template_dir}") FileUtils.rm_rf(@dir) FileUtils.cp_r(data('single-pot'), @dir) l.get(@dir, edit_cmake: true) assert(File.exist?("#{@dir}/CMakeLists.txt")) assert_include(File.read("#{@dir}/CMakeLists.txt"), 'ki18n_install(po)') end def test_get_po_no_edit_cmake l = create_l10n l.init_repo_url("file://#{Dir.pwd}/#{@svn_template_dir}") FileUtils.rm_rf(@dir) FileUtils.cp_r(data('single-pot'), @dir) l.get(@dir, edit_cmake: false) assert(File.exist?("#{@dir}/CMakeLists.txt")) assert_not_include(File.read("#{@dir}/CMakeLists.txt"), 'ki18n_install(po)') end def test_statistics l = create_l10n l.init_repo_url("file://#{Dir.pwd}/#{@svn_template_dir}") FileUtils.rm_rf(@dir) FileUtils.cp_r(data('multi-pot'), @dir) l.get(@dir) statistics = ReleaseMe::L10nStatistics.new statistics.gather!("#{@dir}/po") assert_equal({"de"=>{:all=>4, :shown=>3, :notshown=>1, :percentage=>75.0}, "fr"=>{:all=>4, :shown=>4, :notshown=>0, :percentage=>100.0}}, statistics.stats) -end + end def test_variable_potname l = create_l10n l.init_repo_url("file://#{Dir.pwd}/#{@svn_template_dir}") FileUtils.rm_rf(@dir) FileUtils.cp_r(data('variable-pot'), @dir) assert_raises RuntimeError do l.get(@dir) end end def test_space_and_declared_multi_pot l = create_l10n l.init_repo_url("file://#{Dir.pwd}/#{@svn_template_dir}") FileUtils.rm_rf(@dir) FileUtils.cp_r(data('space-and-declared-multi-pot'), @dir) l.get(@dir) assert_path_exist("#{@dir}/po/de/amarok.po") assert_path_exist("#{@dir}/po/de/amarokcollectionscanner_qt.po") end def test_find_templates_bogus l = create_l10n - templates = l.find_templates(data('bogus-pot')) + templates = l.find_templates(data('bogus-pot'), skip_dir: nil) assert_equal(templates, []) end def test_diff_output_some_not_found_all_not_found # When no translations were found we expect different output versus when # only some were not found. l = create_l10n l.init_repo_url("file://#{Dir.pwd}/#{@svn_template_dir}") FileUtils.rm_rf(@dir) FileUtils.cp_r(data('multi-pot'), @dir) l.get(@dir) ENV.delete('RELEASEME_SHUTUP') # Reset by testme setup some_missing_stdout = StringIO.open do |io| $stdout = io l.instance_variable_set(:@__logger, nil) # Reset l.print_missing_languages([l.languages.pop]) io.string.strip end all_missing_stdout = StringIO.open do |io| $stdout = io l.instance_variable_set(:@__logger, nil) # Reset l.print_missing_languages(l.languages) io.string.strip end $stdout = STDOUT assert_not_empty(some_missing_stdout) assert_not_empty(all_missing_stdout) assert_not_equal(some_missing_stdout, all_missing_stdout) ensure $stdout = STDOUT end def test_script # https://techbase.kde.org/Localization/Concepts/Transcript l = ReleaseMe::L10n.new(ReleaseMe::L10n::TRUNK, 'ki18n', 'frameworks') l.target = "#{@dir}/l10n" l.init_repo_url("file://#{Dir.pwd}/#{@svn_template_dir}") FileUtils.rm_rf(@dir) FileUtils.cp_r(data('multi-pot-script'), @dir) l.get(@dir) assert_path_exist("#{@dir}/po/sr/scripts") assert_path_exist("#{@dir}/po/sr/scripts/ki18n5") assert_path_exist("#{@dir}/po/sr/scripts/libplasma5") assert_path_not_exist("#{@dir}/po/sr/scripts/proto") assert_path_exist("#{@dir}/po/sr/scripts/ki18n5/ki18n5.js") assert_path_exist("#{@dir}/po/sr/scripts/ki18n5/trapnakron.pmap") assert_path_exist("#{@dir}/po/sr/scripts/libplasma5/libplasma5.js") assert_path_exist("#{@dir}/po/sr/scripts/libplasma5/plasmoid.js") end def test_pot_detection_without_releaseme # Do not find templates in the releaseme directory itself. # If releaseme was cloned into a source directory (or submoduled') # we'd otherwise trip over test fixtures. # One such fixture is: assert_path_exist("#{__dir__}/data/variable-pot/Messages.sh") l = ReleaseMe::L10n.new(ReleaseMe::L10n::TRUNK, 'ki18n', 'frameworks') # Make sure this doesn't raise anything. pos = l.find_templates(__dir__) assert_empty(pos) end def test_releaseme_dir # This is a bit of a silly test. It is meant as an additional safeguard # against breaking relative path resolution. RELEASEME_DIR is meant # to be resolved to the main releaseme directory. The idea here is that # it's less likely both the test AND the lib get moved, so we'd get a # failing teset if any of the two move. # If you have come here because you moved the lib and get a failure: # Make sure RELEASEME_DIR still resolves to the main releasme dir! # If you have come here because you moved the test and get a failure: # Simply adjust the assertion to match reality. assert_equal(File.absolute_path(__dir__), - ReleaseMe::L10n.RELEASEME_TEST_DIR) + ReleaseMe::L10n::RELEASEME_TEST_DIR) end end