1#--
2# Copyright (C) 2007-2015 Harald Sitter <apachelogger@ubuntu.com>
3#
4# This program is free software; you can redistribute it and/or
5# modify it under the terms of the GNU General Public License as
6# published by the Free Software Foundation; either version 2 of
7# the License or (at your option) version 3 or any later version
8# accepted by the membership of KDE e.V. (or its successor approved
9# by the membership of KDE e.V.), which shall act as a proxy
10# defined in Section 14 of version 3 of the license.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License
18# along with this program. If not, see <http://www.gnu.org/licenses/>.
19#++
20
21require 'English'
22require 'fileutils'
23require 'thwait'
24require 'tmpdir'
25
26require_relative 'cmakeeditor'
27require_relative 'logable'
28require_relative 'source'
29require_relative 'svn'
30require_relative 'translationunit'
31
32module ReleaseMe
33# https://techbase.kde.org/Localization/Concepts/Transcript
34# Downloads scripted l10n helpers.
35class L10nScriptDownloader
36attr_reader :artifacts
37
38attr_reader :lang
39attr_reader :tmpdir
40
41# Caches available scripts for template (i.e. po file).
42# For every template in every language we'd have to do vcs.get the cache
43# does a vcs.list for each language exactly once. It records the directories
44# available so that we later can do a fast lookup and skip vcs.get
45# altogether. With each svn request taking ~1 second that is a huge
46# time saver.
47class TemplateCache
48def initialize(l10n)
49@data = {}
50@l10n = l10n
51
52queue = l10n.languages_queue
53threads = each_thread do
54loop_queue(queue)
55end
56ThreadsWait.all_waits(threads)
57end
58
59def [](*args)
60@data[*args]
61end
62
63private
64
65attr_reader :l10n
66
67def loop_queue(queue)
68loop do
69lang = begin
70queue.pop(true)
71rescue
72break # loop empty if an exception was raised
73end
74@data[lang] = list(lang) # GIL secures this.
75end
76end
77
78def each_thread
79threads = []
80l10n.class::THREAD_COUNT.times do
81threads << Thread.new do
82Thread.current.abort_on_exception = true
83yield
84end
85end
86threads
87end
88
89def list(lang)
90list = l10n.vcs.list(script_file_dir(lang, l10n.i18n_path))
91list.split($INPUT_RECORD_SEPARATOR).collect do |x|
92x.delete('/')
93end
94end
95
96def script_file_dir(lang, i18n_path)
97"#{lang}/scripts/#{i18n_path}"
98end
99end
100
101def initialize(lang, tmpdir, cache, l10n)
102@lang = lang
103@tmpdir = tmpdir
104@scripts_dir = "#{tmpdir}/scripts"
105@l10n = l10n
106@artifacts = []
107@cache = cache
108end
109
110def download
111templates.each do |template|
112name = File.basename(template, '.po')
113next unless @cache[lang].include?(name)
114target_dir = "#{@scripts_dir}/#{name}"
115@l10n.vcs.get(target_dir, "#{script_file_dir}/#{name}")
116unless Dir.glob("#{target_dir}/*").select { |f| File.file?(f) }.empty?
117@artifacts = [@scripts_dir]
118end
119end
120
121@artifacts
122end
123
124private
125
126def templates
127@l10n.templates
128end
129
130def script_file_dir
131"#{lang}/scripts/#{@l10n.i18n_path}"
132end
133end
134
135# FIXME: doesn't write master cmake right now...
136class L10n < TranslationUnit
137prepend Logable
138
139RELEASEME_TEST_DIR = File.absolute_path("#{__dir__}/../../test").freeze
140
141def verify_pot(potname)
142return unless potname.include?('$')
143raise "l10n pot appears to be a variable. cannot resolve #{potname}"
144end
145
146def find_templates(directory, pos = [], skip_dir: RELEASEME_TEST_DIR)
147Dir.glob("#{directory}/**/**/Messages.sh").each do |file|
148next if skip_dir && File.absolute_path(file).start_with?(skip_dir)
149File.readlines(file).each do |line|
150line.match(%r{[^/\s=]+\.pot}).to_a.each do |match|
151verify_pot(match)
152pos << match.sub('.pot', '.po')
153end
154end
155end
156# Templates must be unique as multiple lines can contribute to the same
157# template, as such it can happen that a.pot appears twice which can
158# have unintended consequences by an outside user of the Array.
159pos.uniq
160end
161
162# FIXME: this has no test backing right now
163def strip_comments(file)
164# Strip #~ lines, which once were sensible translations, but then the
165# strings got removed, so they now stick around in case the strings
166# return, poor souls, waiting for a comeback, reminds me of Sunset Blvd :(
167# Problem is that msgfmt adds those to the binary!
168file = File.new(file, File::RDWR)
169str = file.read
170file.rewind
171file.truncate(0)
172# Sometimes a fuzzy marker can precede an obsolete translation block, so
173# first remove any fuzzy obsoletion in the file and then remove any
174# additional obsoleted lines.
175# This prevents the fuzzy markers from getting left over.
176str.gsub!(/^#, fuzzy\n#~.*/, '')
177str.gsub!(/^#~.*/, '')
178str = str.strip
179file << str
180file.close
181end
182
183def po_file_dir(lang)
184"#{lang}/messages/#{@i18n_path}"
185end
186
187def get_single(lang, tmpdir)
188# TODO: maybe class this
189po_file_name = templates[0]
190vcs_file_path = "#{po_file_dir(lang)}/#{po_file_name}"
191po_file_path = "#{tmpdir}/#{po_file_name}"
192
193vcs.export(po_file_path, vcs_file_path)
194
195files = []
196if File.exist?(po_file_path)
197files << po_file_path
198strip_comments(po_file_path)
199end
200files.uniq
201end
202
203def get_multiple(lang, tmpdir)
204vcs_path = po_file_dir(lang)
205
206return [] if @vcs.list(vcs_path).empty?
207@vcs.get(tmpdir, vcs_path)
208
209files = []
210templates.each do |po|
211po_file_path = tmpdir.dup.concat("/#{po}")
212next unless File.exist?(po_file_path)
213files << po_file_path
214strip_comments(po_file_path)
215end
216
217files.uniq
218end
219
220def get(srcdir, target = File.expand_path("#{srcdir}/po"), edit_cmake: true)
221Dir.mkdir(target)
222
223@templates = find_templates(srcdir)
224log_info "Downloading translations for #{srcdir}"
225
226languages_without_translation = []
227has_translation = false
228# FIXME: due to threading we do explicit pathing, so this probably can go
229Dir.chdir(srcdir) do
230queue = languages_queue
231threads = []
232script_cache = L10nScriptDownloader::TemplateCache.new(self)
233THREAD_COUNT.times do
234threads << Thread.new do
235Thread.current.abort_on_exception = true
236until queue.empty?
237begin
238lang = queue.pop(true)
239rescue
240# When pop runs into an empty queue with non_block=true it raises
241# an exception. We'll simply continue with it as our loop should
242# naturally end anyway.
243continue
244end
245Dir.mktmpdir(self.class.to_s) do |tmpdir|
246log_debug "#{srcdir} - downloading #{lang}"
247if templates.count > 1
248files = get_multiple(lang, tmpdir)
249elsif templates.count == 1
250files = get_single(lang, tmpdir)
251else
252# FIXME: needs testcase
253# TODO: this previously aborted entirely, not sure that makes
254# sense with threading
255next # No translations need fetching
256end
257
258files += L10nScriptDownloader.new(lang, tmpdir, script_cache,
259self).download
260
261# No files obtained :(
262if files.empty?
263# FIXME: not thread safe without GIL
264languages_without_translation << lang
265next
266end
267# FIXME: not thread safe without GIL
268has_translation = true
269
270# TODO: path confusing with target
271destination = "#{target}/#{lang}"
272Dir.mkdir(destination)
273FileUtils.mv(files, destination)
274end
275
276# FIXME: this is not thread safe without a GIL
277@languages += [lang]
278end
279end
280end
281ThreadsWait.all_waits(threads)
282
283if completion_requirement = ENV.fetch('RELEASEME_L10N_REQUIREMENT', nil).to_i
284require_relative 'l10nstatistics'
285stats = L10nStatistics.new.tap { |l| l.gather!(target) }.stats
286drop = stats.delete_if { |_, s| s[:percentage] >= completion_requirement }
287drop.each { |lang, _| FileUtils.rm_r("#{target}/#{lang}", verbose: true) }
288has_translation = false if Dir.glob("#{target}/*").empty?
289end
290
291if has_translation && edit_cmake
292# Update CMakeLists.txt
293CMakeEditor.append_po_install_instructions!(Dir.pwd, 'po')
294elsif !has_translation
295# Remove the empty translations directory
296Dir.delete('po')
297end
298end
299
300return if languages_without_translation.empty?
301print_missing_languages(languages_without_translation)
302end
303
304def print_missing_languages(missing)
305if (languages - missing).empty?
306path = po_file_dir('$lang')
307log_warn "!!! No translations found at SVN path #{path} !!!"
308log_warn "Looked for templates: #{@templates}"
309else
310log_info "No translations for: #{missing.join(', ')}"
311end
312end
313end
314end