diff --git a/fetchpo.rb b/fetchpo.rb index a6e021c..d4bf1a3 100755 --- a/fetchpo.rb +++ b/fetchpo.rb @@ -1,68 +1,68 @@ #!/usr/bin/env ruby #-- # Copyright (C) 2017 Aleix Pol Gonzalez # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License or (at your option) version 3 or any later version # accepted by the membership of KDE e.V. (or its successor approved # by the membership of KDE e.V.), which shall act as a proxy # defined in Section 14 of version 3 of the license. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #++ require_relative 'lib/releaseme' require 'ostruct' require 'optparse' options = OpenStruct.new OptionParser.new do |opts| opts.banner = 'Usage: fetchpo.rb --origin ORIGIN SOURCE_DIR OUTPUT_PO_DIR' opts.separator '' opts.on('--origin ORIGIN', ReleaseMe::Origin::ALL, "Origin (#{ReleaseMe::Origin::ALL.join(' | ')}).", ' Used to deduce release branch and localization branches.') do |v| options[:origin] = v end opts.on('--project NAME', 'ProjectName.', ' Repository name in git.kde.org') do |v| options[:project] = v end end.parse! unless options.origin && options.project && ARGV.count==2 warn 'error, you need to set an origin' exit 1 end output_dir = File.expand_path(ARGV.pop) source_dir = File.expand_path(ARGV.pop) elements = ReleaseMe::Project.from_repo_url("git://anongit.kde.org/#{options.project}") unless elements.count == 1 warn "Found #{elements.count} elements for #{options.project}" exit 2 end if File.exist?(output_dir) warn "#{output_dir} should be created by the script, please remove first" exit 3 end # ./fetchpo.rb --origin stable --project kalgebra ~/devel/frameworks/kalgebra/ /tmp/foo/po project_information = elements[0] l10n = ReleaseMe::L10n.new(options.origin, options.project, project_information.i18n_path) -l10n.get(source_dir, output_dir, false) +l10n.get(source_dir, output_dir, allow_edit: false) diff --git a/lib/releaseme/l10n.rb b/lib/releaseme/l10n.rb index b20ac0a..0865df3 100644 --- a/lib/releaseme/l10n.rb +++ b/lib/releaseme/l10n.rb @@ -1,312 +1,312 @@ #-- # Copyright (C) 2007-2015 Harald Sitter # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License or (at your option) version 3 or any later version # accepted by the membership of KDE e.V. (or its successor approved # by the membership of KDE e.V.), which shall act as a proxy # defined in Section 14 of version 3 of the license. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #++ require 'English' require 'fileutils' require 'thwait' require 'tmpdir' require_relative 'cmakeeditor' require_relative 'logable' require_relative 'source' require_relative 'svn' require_relative 'translationunit' module ReleaseMe # https://techbase.kde.org/Localization/Concepts/Transcript # Downloads scripted l10n helpers. class L10nScriptDownloader attr_reader :artifacts attr_reader :lang attr_reader :tmpdir # Caches available scripts for template (i.e. po file). # For every template in every language we'd have to do vcs.get the cache # does a vcs.list for each language exactly once. It records the directories # available so that we later can do a fast lookup and skip vcs.get # altogether. With each svn request taking ~1 second that is a huge # time saver. class TemplateCache def initialize(l10n) @data = {} @l10n = l10n queue = l10n.languages_queue threads = each_thread do loop_queue(queue) end ThreadsWait.all_waits(threads) end def [](*args) @data[*args] end private attr_reader :l10n def loop_queue(queue) loop do lang = begin queue.pop(true) rescue break # loop empty if an exception was raised end @data[lang] = list(lang) # GIL secures this. end end def each_thread threads = [] l10n.class::THREAD_COUNT.times do threads << Thread.new do Thread.current.abort_on_exception = true yield end end threads end def list(lang) list = l10n.vcs.list(script_file_dir(lang, l10n.i18n_path)) list.split($INPUT_RECORD_SEPARATOR).collect do |x| x.delete('/') end end def script_file_dir(lang, i18n_path) "#{lang}/scripts/#{i18n_path}" end end def initialize(lang, tmpdir, cache, l10n) @lang = lang @tmpdir = tmpdir @scripts_dir = "#{tmpdir}/scripts" @l10n = l10n @artifacts = [] @cache = cache end def download templates.each do |template| name = File.basename(template, '.po') next unless @cache[lang].include?(name) target_dir = "#{@scripts_dir}/#{name}" @l10n.vcs.get(target_dir, "#{script_file_dir}/#{name}") unless Dir.glob("#{target_dir}/*").select { |f| File.file?(f) }.empty? @artifacts = [@scripts_dir] end end @artifacts end private def templates @l10n.templates end def script_file_dir "#{lang}/scripts/#{@l10n.i18n_path}" end end # FIXME: doesn't write master cmake right now... class L10n < TranslationUnit prepend Logable def verify_pot(potname) return unless potname.include?('$') raise "l10n pot appears to be a variable. cannot resolve #{potname}" end def find_templates(directory, pos = []) Dir.glob("#{directory}/**/**/Messages.sh").each do |file| File.readlines(file).each do |line| line.match(%r{[^/\s=]+\.pot}).to_a.each do |match| verify_pot(match) pos << match.sub('.pot', '.po') end end end # Templates must be unique as multiple lines can contribute to the same # template, as such it can happen that a.pot appears twice which can # have unintended consequences by an outside user of the Array. pos.uniq end # FIXME: this has no test backing right now def strip_comments(file) # Strip #~ lines, which once were sensible translations, but then the # strings got removed, so they now stick around in case the strings # return, poor souls, waiting for a comeback, reminds me of Sunset Blvd :( # Problem is that msgfmt adds those to the binary! file = File.new(file, File::RDWR) str = file.read file.rewind file.truncate(0) # Sometimes a fuzzy marker can precede an obsolete translation block, so # first remove any fuzzy obsoletion in the file and then remove any # additional obsoleted lines. # This prevents the fuzzy markers from getting left over. str.gsub!(/^#, fuzzy\n#~.*/, '') str.gsub!(/^#~.*/, '') str = str.strip file << str file.close end def po_file_dir(lang) "#{lang}/messages/#{@i18n_path}" end def get_single(lang, tmpdir) # TODO: maybe class this po_file_name = templates[0] vcs_file_path = "#{po_file_dir(lang)}/#{po_file_name}" po_file_path = "#{tmpdir}/#{po_file_name}" vcs.export(po_file_path, vcs_file_path) files = [] if File.exist?(po_file_path) files << po_file_path strip_comments(po_file_path) end files.uniq end def get_multiple(lang, tmpdir) vcs_path = po_file_dir(lang) return [] if @vcs.list(vcs_path).empty? @vcs.get(tmpdir, vcs_path) files = [] templates.each do |po| po_file_path = tmpdir.dup.concat("/#{po}") next unless File.exist?(po_file_path) files << po_file_path strip_comments(po_file_path) end files.uniq end def get(srcdir, target = File.expand_path("#{Dir.getwd}/#{srcdir}/po"), - allow_edit = true) + allow_edit: true) Dir.mkdir(target) @templates = find_templates(srcdir) log_info "Downloading translations for #{srcdir}" languages_without_translation = [] has_translation = false # FIXME: due to threading we do explicit pathing, so this probably can go Dir.chdir(srcdir) do queue = languages_queue threads = [] script_cache = L10nScriptDownloader::TemplateCache.new(self) THREAD_COUNT.times do threads << Thread.new do Thread.current.abort_on_exception = true until queue.empty? begin lang = queue.pop(true) rescue # When pop runs into an empty queue with non_block=true it raises # an exception. We'll simply continue with it as our loop should # naturally end anyway. continue end Dir.mktmpdir(self.class.to_s) do |tmpdir| log_debug "#{srcdir} - downloading #{lang}" if templates.count > 1 files = get_multiple(lang, tmpdir) elsif templates.count == 1 files = get_single(lang, tmpdir) else # FIXME: needs testcase # TODO: this previously aborted entirely, not sure that makes # sense with threading next # No translations need fetching end files += L10nScriptDownloader.new(lang, tmpdir, script_cache, self).download # No files obtained :( if files.empty? # FIXME: not thread safe without GIL languages_without_translation << lang next end # FIXME: not thread safe without GIL has_translation = true # TODO: path confusing with target destination = "#{target}/#{lang}" Dir.mkdir(destination) FileUtils.mv(files, destination) end # FIXME: this is not thread safe without a GIL @languages += [lang] end end end ThreadsWait.all_waits(threads) if completion_requirement = ENV.fetch('RELEASEME_L10N_REQUIREMENT', nil).to_i require_relative 'l10nstatistics' stats = L10nStatistics.new.tap { |l| l.gather!(target) }.stats drop = stats.delete_if { |_, s| s[:percentage] >= completion_requirement } drop.each { |lang, _| FileUtils.rm_r("#{target}/#{lang}", verbose: true) } has_translation = false if Dir.glob("#{target}/*").empty? end if has_translation && allow_edit # Update CMakeLists.txt CMakeEditor.append_po_install_instructions!(Dir.pwd, 'po') elsif !has_translation # Remove the empty translations directory Dir.delete('po') end end return if languages_without_translation.empty? print_missing_languages(languages_without_translation) end def print_missing_languages(missing) if (languages - missing).empty? path = po_file_dir('$lang') log_warn "!!! No translations found at SVN path #{path} !!!" log_warn "Looked for templates: #{@templates}" else log_info "No translations for: #{missing.join(', ')}" end end end end