diff --git a/fetchpo.rb b/fetchpo.rb new file mode 100755 index 0000000..a6e021c --- /dev/null +++ b/fetchpo.rb @@ -0,0 +1,68 @@ +#!/usr/bin/env ruby +#-- +# Copyright (C) 2017 Aleix Pol Gonzalez +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License or (at your option) version 3 or any later version +# accepted by the membership of KDE e.V. (or its successor approved +# by the membership of KDE e.V.), which shall act as a proxy +# defined in Section 14 of version 3 of the license. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#++ + +require_relative 'lib/releaseme' +require 'ostruct' +require 'optparse' + +options = OpenStruct.new +OptionParser.new do |opts| + opts.banner = 'Usage: fetchpo.rb --origin ORIGIN SOURCE_DIR OUTPUT_PO_DIR' + + opts.separator '' + + opts.on('--origin ORIGIN', ReleaseMe::Origin::ALL, + "Origin (#{ReleaseMe::Origin::ALL.join(' | ')}).", + ' Used to deduce release branch and localization branches.') do |v| + options[:origin] = v + end + + opts.on('--project NAME', 'ProjectName.', + ' Repository name in git.kde.org') do |v| + options[:project] = v + end +end.parse! + +unless options.origin && options.project && ARGV.count==2 + warn 'error, you need to set an origin' + exit 1 +end + +output_dir = File.expand_path(ARGV.pop) +source_dir = File.expand_path(ARGV.pop) + +elements = ReleaseMe::Project.from_repo_url("git://anongit.kde.org/#{options.project}") +unless elements.count == 1 + warn "Found #{elements.count} elements for #{options.project}" + exit 2 +end + +if File.exist?(output_dir) + warn "#{output_dir} should be created by the script, please remove first" + exit 3 +end + +# ./fetchpo.rb --origin stable --project kalgebra ~/devel/frameworks/kalgebra/ /tmp/foo/po + +project_information = elements[0] + +l10n = ReleaseMe::L10n.new(options.origin, options.project, project_information.i18n_path) +l10n.get(source_dir, output_dir, false) diff --git a/lib/releaseme/l10n.rb b/lib/releaseme/l10n.rb index 50ce649..ec727ca 100644 --- a/lib/releaseme/l10n.rb +++ b/lib/releaseme/l10n.rb @@ -1,313 +1,311 @@ #-- # Copyright (C) 2007-2015 Harald Sitter # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License or (at your option) version 3 or any later version # accepted by the membership of KDE e.V. (or its successor approved # by the membership of KDE e.V.), which shall act as a proxy # defined in Section 14 of version 3 of the license. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #++ require 'English' require 'fileutils' require 'thwait' require 'tmpdir' require_relative 'cmakeeditor' require_relative 'logable' require_relative 'source' require_relative 'svn' require_relative 'translationunit' module ReleaseMe # https://techbase.kde.org/Localization/Concepts/Transcript # Downloads scripted l10n helpers. class L10nScriptDownloader attr_reader :artifacts attr_reader :lang attr_reader :tmpdir # Caches available scripts for template (i.e. po file). # For every template in every language we'd have to do vcs.get the cache # does a vcs.list for each language exactly once. It records the directories # available so that we later can do a fast lookup and skip vcs.get # altogether. With each svn request taking ~1 second that is a huge # time saver. class TemplateCache def initialize(l10n) @data = {} @l10n = l10n queue = l10n.languages_queue threads = each_thread do loop_queue(queue) end ThreadsWait.all_waits(threads) end def [](*args) @data[*args] end private attr_reader :l10n def loop_queue(queue) loop do lang = begin queue.pop(true) rescue break # loop empty if an exception was raised end @data[lang] = list(lang) # GIL secures this. end end def each_thread threads = [] l10n.class::THREAD_COUNT.times do threads << Thread.new do Thread.current.abort_on_exception = true yield end end threads end def list(lang) list = l10n.vcs.list(script_file_dir(lang, l10n.i18n_path)) list.split($INPUT_RECORD_SEPARATOR).collect do |x| x.delete('/') end end def script_file_dir(lang, i18n_path) "#{lang}/scripts/#{i18n_path}" end end def initialize(lang, tmpdir, cache, l10n) @lang = lang @tmpdir = tmpdir @scripts_dir = "#{tmpdir}/scripts" @l10n = l10n @artifacts = [] @cache = cache end def download templates.each do |template| name = File.basename(template, '.po') next unless @cache[lang].include?(name) target_dir = "#{@scripts_dir}/#{name}" @l10n.vcs.get(target_dir, "#{script_file_dir}/#{name}") unless Dir.glob("#{target_dir}/*").select { |f| File.file?(f) }.empty? @artifacts = [@scripts_dir] end end @artifacts end private def templates @l10n.templates end def script_file_dir "#{lang}/scripts/#{@l10n.i18n_path}" end end # FIXME: doesn't write master cmake right now... class L10n < TranslationUnit prepend Logable def verify_pot(potname) return unless potname.include?('$') raise "l10n pot appears to be a variable. cannot resolve #{potname}" end def find_templates(directory, pos = []) Dir.glob("#{directory}/**/**/Messages.sh").each do |file| File.readlines(file).each do |line| line.match(%r{[^/\s=]+\.pot}).to_a.each do |match| verify_pot(match) pos << match.sub('.pot', '.po') end end end # Templates must be unique as multiple lines can contribute to the same # template, as such it can happen that a.pot appears twice which can # have unintended consequences by an outside user of the Array. pos.uniq end # FIXME: this has no test backing right now def strip_comments(file) # Strip #~ lines, which once were sensible translations, but then the # strings got removed, so they now stick around in case the strings # return, poor souls, waiting for a comeback, reminds me of Sunset Blvd :( # Problem is that msgfmt adds those to the binary! file = File.new(file, File::RDWR) str = file.read file.rewind file.truncate(0) # Sometimes a fuzzy marker can precede an obsolete translation block, so # first remove any fuzzy obsoletion in the file and then remove any # additional obsoleted lines. # This prevents the fuzzy markers from getting left over. str.gsub!(/^#, fuzzy\n#~.*/, '') str.gsub!(/^#~.*/, '') str = str.strip file << str file.close end def po_file_dir(lang) "#{lang}/messages/#{@i18n_path}" end def get_single(lang, tmpdir) # TODO: maybe class this po_file_name = templates[0] vcs_file_path = "#{po_file_dir(lang)}/#{po_file_name}" po_file_path = "#{tmpdir}/#{po_file_name}" vcs.export(po_file_path, vcs_file_path) files = [] if File.exist?(po_file_path) files << po_file_path strip_comments(po_file_path) end files.uniq end def get_multiple(lang, tmpdir) vcs_path = po_file_dir(lang) return [] if @vcs.list(vcs_path).empty? @vcs.get(tmpdir, vcs_path) files = [] templates.each do |po| po_file_path = tmpdir.dup.concat("/#{po}") next unless File.exist?(po_file_path) files << po_file_path strip_comments(po_file_path) end files.uniq end - def get(srcdir) - # FIXME: this is later used as destination for the weirdest of reasons... - target = "#{srcdir}/po/" + def get(srcdir, target = File.expand_path("#{Dir.getwd}/#{srcdir}/po"), allow_edit = true) Dir.mkdir(target) @templates = find_templates(srcdir) log_info "Downloading translations for #{srcdir}" languages_without_translation = [] has_translation = false # FIXME: due to threading we do explicit pathing, so this probably can go Dir.chdir(srcdir) do queue = languages_queue threads = [] script_cache = L10nScriptDownloader::TemplateCache.new(self) THREAD_COUNT.times do threads << Thread.new do Thread.current.abort_on_exception = true until queue.empty? begin lang = queue.pop(true) rescue # When pop runs into an empty queue with non_block=true it raises # an exception. We'll simply continue with it as our loop should # naturally end anyway. continue end Dir.mktmpdir(self.class.to_s) do |tmpdir| log_debug "#{srcdir} - downloading #{lang}" if templates.count > 1 files = get_multiple(lang, tmpdir) elsif templates.count == 1 files = get_single(lang, tmpdir) else # FIXME: needs testcase # TODO: this previously aborted entirely, not sure that makes # sense with threading next # No translations need fetching end files += L10nScriptDownloader.new(lang, tmpdir, script_cache, self).download # No files obtained :( if files.empty? # FIXME: not thread safe without GIL languages_without_translation << lang next end # FIXME: not thread safe without GIL has_translation = true # TODO: path confusing with target - destination = "po/#{lang}" + destination = "#{target}/#{lang}" Dir.mkdir(destination) FileUtils.mv(files, destination) end # FIXME: this is not thread safe without a GIL @languages += [lang] end end end ThreadsWait.all_waits(threads) if completion_requirement = ENV.fetch('RELEASEME_L10N_REQUIREMENT', nil).to_i require_relative 'l10nstatistics' - stats = L10nStatistics.new.tap { |l| l.gather!(Dir.pwd) }.stats + stats = L10nStatistics.new.tap { |l| l.gather!(target) }.stats drop = stats.delete_if { |_, s| s[:percentage] >= completion_requirement } - drop.each { |lang, _| FileUtils.rm_r("po/#{lang}", verbose: true) } - has_translation = false if Dir.glob('po/*').empty? + drop.each { |lang, _| FileUtils.rm_r("#{target}/#{lang}", verbose: true) } + has_translation = false if Dir.glob("#{target}/*").empty? end if has_translation # Update CMakeLists.txt - CMakeEditor.append_po_install_instructions!(Dir.pwd, 'po') + CMakeEditor.append_po_install_instructions!(Dir.pwd, 'po') if allow_edit else # Remove the empty translations directory Dir.delete('po') end end return if languages_without_translation.empty? print_missing_languages(languages_without_translation) end def print_missing_languages(missing) if (languages - missing).empty? path = po_file_dir('$lang') log_warn "!!! No translations found at SVN path #{path} !!!" log_warn "Looked for templates: #{@templates}" else log_info "No translations for: #{missing.join(', ')}" end end end end diff --git a/lib/releaseme/l10nstatistics.rb b/lib/releaseme/l10nstatistics.rb index c090e9d..65c7ee8 100644 --- a/lib/releaseme/l10nstatistics.rb +++ b/lib/releaseme/l10nstatistics.rb @@ -1,75 +1,74 @@ #-- # Copyright (C) 2014 Harald Sitter # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License or (at your option) version 3 or any later version # accepted by the membership of KDE e.V. (or its successor approved # by the membership of KDE e.V.), which shall act as a proxy # defined in Section 14 of version 3 of the license. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #++ module ReleaseMe class L10nStatistics attr_reader :stats def initialize # (project) # project = project @stats = {} end - def gather!(srcdir) - podir = "#{srcdir}/po/" + def gather!(podir) Dir.chdir(podir) do languages = Dir.glob('*') languages.each do |language| next unless File.directory?(language) Dir.chdir(language) do translated = 0 fuzzy = 0 untranslated = 0 Dir.glob('*.po').each do |file| data = `LC_ALL=C LANG=C msgfmt --statistics #{file} -o /dev/null > /dev/stdout 2>&1` # tear the data apart and create some variables data.split(',').each do |x| if x.include?('untranslated') untranslated += x.scan(/[\d]+/)[0].to_i elsif x.include?('fuzzy') fuzzy += x.scan(/[\d]+/)[0].to_i elsif x.include?('translated') translated += x.scan(/[\d]+/)[0].to_i end end end all = translated + fuzzy + untranslated notshown = fuzzy + untranslated shown = all - notshown percentage = ((100.0 * shown.to_f) / all.to_f) @stats[language] = { all: all, shown: shown, notshown: notshown, percentage: percentage } end end end end def write(_html_file_path) end end end diff --git a/test/test_l10n.rb b/test/test_l10n.rb index ca8aed3..8124c5c 100644 --- a/test/test_l10n.rb +++ b/test/test_l10n.rb @@ -1,202 +1,216 @@ #-- # Copyright (C) 2015 Harald Sitter # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License or (at your option) version 3 or any later version # accepted by the membership of KDE e.V. (or its successor approved # by the membership of KDE e.V.), which shall act as a proxy # defined in Section 14 of version 3 of the license. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #++ require 'fileutils' require_relative 'lib/testme' require_relative '../lib/releaseme/l10n.rb' require_relative '../lib/releaseme/l10nstatistics.rb' require_relative '../lib/releaseme/documentation.rb' # FIXME: test stable class TestL10n < Testme def setup @repoDataDir = data("l10nrepo/") @i18n_path = "extragear-multimedia" @trunkUrl = "trunk/l10n-kf5/" @stableUrl = "branches/stable/l10n-kf5" @dir = "tmp_l10n_" + (0...16).map{ ('a'..'z').to_a[rand(26)] }.join @svnTemplateDir = "tmp_l10n_repo_" + (0...16).map{ ('a'..'z').to_a[rand(26)] }.join @svnCheckoutDir = "tmp_l10n_check_" + (0...16).map{ ('a'..'z').to_a[rand(26)] }.join `svnadmin create #{@svnTemplateDir}` assert(File.exist?(@svnTemplateDir)) `svn co file://#{Dir.pwd}/#{@svnTemplateDir} #{@svnCheckoutDir}` FileUtils.cp_r("#{@repoDataDir}/trunk", @svnCheckoutDir) FileUtils.cp_r("#{@repoDataDir}/branches", @svnCheckoutDir) Dir.chdir(@svnCheckoutDir) do `svn add *` `svn ci -m 'yolo'` end end def teardown FileUtils.rm_rf(@svnTemplateDir) FileUtils.rm_rf(@svnCheckoutDir) FileUtils.rm_rf(@dir) end def create_l10n l = ReleaseMe::L10n.new(ReleaseMe::L10n::TRUNK, 'amarok', @i18n_path) l.target = "#{@dir}/l10n" l end def test_find_templates l = create_l10n templates = l.find_templates(data("multi-pot")) assert_equal(templates.count, 2) templates = l.find_templates(data("single-pot")) assert_equal(templates.count, 1) end def test_get_po l = create_l10n l.init_repo_url("file://#{Dir.pwd}/#{@svnTemplateDir}") FileUtils.rm_rf(@dir) FileUtils.cp_r(data("single-pot"), @dir) l.get(@dir) assert(File.exist?("#{@dir}")) assert(File.exist?("#{@dir}/CMakeLists.txt")) assert(!File.exist?("#{@dir}/l10n")) # temp dir must not be there assert(File.exist?("#{@dir}/po")) assert(File.exist?("#{@dir}/po/de/amarok.po")) FileUtils.rm_rf(@dir) FileUtils.cp_r(data("multi-pot"), @dir) l.get(@dir) assert(File.exist?("#{@dir}")) assert(File.exist?("#{@dir}/CMakeLists.txt")) assert(!File.exist?("#{@dir}/l10n")) # temp dir must not be there assert(File.exist?("#{@dir}/po")) assert(File.exist?("#{@dir}/po/de/amarok.po")) assert(File.exist?("#{@dir}/po/de/amarokcollectionscanner_qt.po")) end + def test_get_po_elsewhere + l = create_l10n + l.init_repo_url("file://#{Dir.pwd}/#{@svnTemplateDir}") + + @elsewhere = "#{Dir.pwd}/elsewhere_tmp_l10n_" + (0...16).map{ ('a'..'z').to_a[rand(26)] }.join + + FileUtils.rm_rf(@dir) + FileUtils.cp_r(data("single-pot"), @dir) + l.get(@dir, @elsewhere, false) + assert(File.exist?("#{@elsewhere}/de/amarok.po")) + + FileUtils.rm_rf(@elsewhere) + end + def test_statistics l = create_l10n l.init_repo_url("file://#{Dir.pwd}/#{@svnTemplateDir}") FileUtils.rm_rf(@dir) FileUtils.cp_r(data("multi-pot"), @dir) l.get(@dir) statistics = ReleaseMe::L10nStatistics.new - statistics.gather!(@dir) - assert_equal(statistics.stats, {"de"=>{:all=>4, - :shown=>3, - :notshown=>1, - :percentage=>75.0}, - "fr"=>{:all=>4, - :shown=>4, - :notshown=>0, - :percentage=>100.0}}) - end + statistics.gather!("#{@dir}/po") + assert_equal({"de"=>{:all=>4, + :shown=>3, + :notshown=>1, + :percentage=>75.0}, + "fr"=>{:all=>4, + :shown=>4, + :notshown=>0, + :percentage=>100.0}}, statistics.stats) +end def test_variable_potname l = create_l10n l.init_repo_url("file://#{Dir.pwd}/#{@svnTemplateDir}") FileUtils.rm_rf(@dir) FileUtils.cp_r(data('variable-pot'), @dir) assert_raises RuntimeError do l.get(@dir) end end def test_space_and_declared_multi_pot l = create_l10n l.init_repo_url("file://#{Dir.pwd}/#{@svnTemplateDir}") FileUtils.rm_rf(@dir) FileUtils.cp_r(data('space-and-declared-multi-pot'), @dir) l.get(@dir) assert_path_exist("#{@dir}/po/de/amarok.po") assert_path_exist("#{@dir}/po/de/amarokcollectionscanner_qt.po") end def test_find_templates_bogus l = create_l10n templates = l.find_templates(data("bogus-pot")) assert_equal(templates, []) end def test_diff_output_some_not_found_all_not_found # When no translations were found we expect different output versus when # only some were not found. l = create_l10n l.init_repo_url("file://#{Dir.pwd}/#{@svnTemplateDir}") FileUtils.rm_rf(@dir) FileUtils.cp_r(data("multi-pot"), @dir) l.get(@dir) ENV.delete('RELEASEME_SHUTUP') # Reset by testme setup some_missing_stdout = StringIO.open do |io| $stdout = io l.instance_variable_set(:@__logger, nil) # Reset l.print_missing_languages([l.languages.pop]) io.string.strip end all_missing_stdout = StringIO.open do |io| $stdout = io l.instance_variable_set(:@__logger, nil) # Reset l.print_missing_languages(l.languages) io.string.strip end $stdout = STDOUT assert_not_empty(some_missing_stdout) assert_not_empty(all_missing_stdout) assert_not_equal(some_missing_stdout, all_missing_stdout) ensure $stdout = STDOUT end def test_script # https://techbase.kde.org/Localization/Concepts/Transcript l = ReleaseMe::L10n.new(ReleaseMe::L10n::TRUNK, 'ki18n', 'frameworks') l.target = "#{@dir}/l10n" l.init_repo_url("file://#{Dir.pwd}/#{@svnTemplateDir}") FileUtils.rm_rf(@dir) FileUtils.cp_r(data('multi-pot-script'), @dir) l.get(@dir) assert_path_exist("#{@dir}/po/sr/scripts") assert_path_exist("#{@dir}/po/sr/scripts/ki18n5") assert_path_exist("#{@dir}/po/sr/scripts/libplasma5") assert_path_not_exist("#{@dir}/po/sr/scripts/proto") assert_path_exist("#{@dir}/po/sr/scripts/ki18n5/ki18n5.js") assert_path_exist("#{@dir}/po/sr/scripts/ki18n5/trapnakron.pmap") assert_path_exist("#{@dir}/po/sr/scripts/libplasma5/libplasma5.js") assert_path_exist("#{@dir}/po/sr/scripts/libplasma5/plasmoid.js") end end