diff --git a/kdedocgen.py b/kdedocgen.py index 979664a..d239d55 100755 --- a/kdedocgen.py +++ b/kdedocgen.py @@ -1,119 +1,119 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright 2015 Luigi Toscano # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) version 3, or any # later version accepted by the membership of KDE e.V. (or its # successor approved by the membership of KDE e.V.), which shall # act as a proxy defined in Section 6 of version 3 of the license. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . import argparse import errno import logging import logging.config import os import shutil from datetime import datetime import kdedocs.configuration import kdedocs.environments from kdedocs.docretriever import RetrieveDocumentationBranch from kdedocs.docgenerator import DocGenerator from kdedocs.utils import mkdirp LOGGER = logging.getLogger(__name__) def copy_files(file_list, dest_dir, resource_dir): if not file_list: return LOGGER.info('Copying files %s to %s' % (file_list, dest_dir)) mkdirp(dest_dir) for element in file_list: source_element = os.path.join(resource_dir, element) source_dir_name = os.path.basename(os.path.abspath(source_element)) dest_path = os.path.join(dest_dir, os.path.join(dest_dir, source_dir_name)) if os.path.isdir(source_element): try: shutil.rmtree(dest_path) except OSError as exc: if exc.errno != errno.ENOENT: raise shutil.copytree(source_element, dest_path) else: shutil.copy(source_element, dest_dir) def main(): parser = argparse.ArgumentParser() parser.add_argument('-v', '--verbose', action='store_true', default=False, dest='verbose', help='verbose output (when a logging ' 'file is not specified)') parser.add_argument('-l', '--log-config', default='', dest='logconfigfile', help='logging configuration file') parser.add_argument('-c', '--config-file', default='docgen_conf.ini', dest='configfile', help='configuration file') parser.add_argument('-r', '--retrieve', action='store_true', default=False, dest='retrieve', help='retrieve files') parser.add_argument('-g', '--generate', action='store_true', default=False, dest='generate', help='generate documentation') parser.add_argument('-f', '--force-generate', action='store_true', default=False, dest='forcegenerate', help='force generation of documentation') parser.add_argument('-s', '--static-files', action='store_true', default=False, dest='copyfiles', help='copy static files to website') parser.add_argument('--noenv', action='store_true', default=False, dest='noenv', help='do not regenerate the environments') args = parser.parse_args() logging_level = logging.INFO if args.verbose: logging_level = logging.DEBUG if args.logconfigfile: curr_timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') log_defaults = {'logfilename': 'doc_%s' % (curr_timestamp)} logging.config.fileConfig(args.logconfigfile, defaults=log_defaults, disable_existing_loggers=False) else: logging.basicConfig(level=logging_level, format='%(asctime)s:%(levelname)s:%(name)s:' '%(message)s') conf = kdedocs.configuration.DocConfiguration(args.configfile) env_manager = kdedocs.environments.DocEnvironmentsManager(conf) if not args.noenv: env_manager.setup_all() if args.retrieve: for branch in conf.enabled_branches: doc_getter = RetrieveDocumentationBranch(branch, conf, env_manager) doc_getter.checkout_doc() if args.generate or args.forcegenerate: docgen = DocGenerator(conf, env_manager) docgen.generate_documentation(forced=args.forcegenerate) php_data = os.path.join(conf.websitedir, 'generated_used.inc.php') docgen.write_generated_file(php_data) if args.copyfiles: copy_files(conf.static_files, conf.websitedir, conf.resource_dir) if __name__ == '__main__': main() diff --git a/kdedocs/configuration.py b/kdedocs/configuration.py index 0cf1dd5..5f9ea80 100644 --- a/kdedocs/configuration.py +++ b/kdedocs/configuration.py @@ -1,264 +1,264 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright 2015 Luigi Toscano # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) version 3, or any # later version accepted by the membership of KDE e.V. (or its # successor approved by the membership of KDE e.V.), which shall # act as a proxy defined in Section 6 of version 3 of the license. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . import codecs -import ConfigParser +import configparser import logging import os import sys LOGGER = logging.getLogger(__name__) class InvalidBranchException(Exception): pass class DocConfiguration(object): BASEDIR = os.path.dirname(os.path.realpath(__file__)) BRANCH_CONF_PREFIX = 'branch.' ENV_CONF_PREFIX = 'env.' def __init__(self, conffile): - self._conf = ConfigParser.ConfigParser() + self._conf = configparser.ConfigParser() with codecs.open(conffile, 'r', encoding='utf-8') as cf: self._conf.readfp(cf) self._website_dir = None self._work_dir = None self._all_languages = None self._all_packages = None self._all_branches = None self._static_files = None self._enabled_branches = None self._configured_envs = None self._resource_dir = None @property def enabled_branches(self): if self._enabled_branches is None: # branches requested explicitly try: requested_branches = self._conf.get('common', 'branches') - except (ConfigParser.NoSectionError, - ConfigParser.NoOptionError) as e: + except (configparser.NoSectionError, + configparser.NoOptionError) as e: requested_branches = None # branches with a configuration section configured_branches = [br_sec[len(self.BRANCH_CONF_PREFIX):] for br_sec in self._conf.sections() if br_sec.startswith( self.BRANCH_CONF_PREFIX)] if requested_branches is None: # no requested branches? Then all configured branches are # available self._enabled_branches = configured_branches else: self._enabled_branches = [branch.strip() for branch in requested_branches.split(',') if branch in configured_branches] return self._enabled_branches @property def all_packages(self): if self._all_packages is None: self._all_packages = [] try: with open('packages', 'r') as pfile: self._all_packages = [pkg.strip() for pkg in pfile] except: pass return self._all_packages @property def all_languages(self): if self._all_languages is None: self._all_languages = [] try: with open('languages', 'r') as lfile: self._all_languages = [lang.strip() for lang in lfile] except: pass # always add 'en' as first element self._all_languages.insert(0, 'en') return self._all_languages @property def configured_packages(self): try: pkgs = [pkg.strip() for pkg in self._conf.get('common', 'packages').split(',')] - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError) as e: + except (configparser.NoSectionError, configparser.NoOptionError) as e: pkgs = self.all_packages return pkgs @property def configured_languages(self): try: langs = [lang.strip() for lang in self._conf.get('common', 'languages').split(',')] - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError) as e: + except (configparser.NoSectionError, configparser.NoOptionError) as e: langs = self.all_languages return langs @property def configured_environments(self): if self._configured_envs is None: # branches with a configuration section self._configured_envs = [env_sec[len(self.ENV_CONF_PREFIX):] for env_sec in self._conf.sections() if env_sec.startswith( self.ENV_CONF_PREFIX)] return self._configured_envs @property def static_files(self): if self._static_files is None: value_static_files = self.get_value('static_files') if value_static_files: self._static_files = [sf.strip() for sf in value_static_files.split(',')] else: self._static_files = [] return self._static_files def _get_section_value(self, section, key): try: value = self._conf.get(section, key) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError) as e: + except (configparser.NoSectionError, configparser.NoOptionError) as e: value = None return value def get_branch_value(self, branch, key): """Get the value of a specific key for a certain branch.""" if branch is None or key is None: return None return self._get_section_value(self.BRANCH_CONF_PREFIX + branch, key) def get_env_value(self, env, key): """Get the value of a specific key for a certain environment.""" if env is None or key is None: return None return self._get_section_value(self.ENV_CONF_PREFIX + env, key) def get_value(self, key): """Get the value of a specific key for the common configuration.""" if key is None: return None return self._get_section_value('common', key) @property def websitedir(self): """Directory of the generated website where the documentation files (original and localized) are fetched. If old is specified, the old directory is used. """ if self._website_dir is None: self._website_dir = self.get_value('website_dir') if self._website_dir is None: self._website_dir = os.path.join(self.BASEDIR, 'website') return self._website_dir @property def nextwebsitedir(self): """Previous website.""" return self.websitedir + '.next' @property def workdir(self): """Work directory.""" if self._work_dir is None: self._work_dir = self.get_value('work_dir') if self._work_dir is None: self._work_dir = os.path.join(self.BASEDIR, 'work') return self._work_dir @property def resource_dir(self): """Directory for resources (scripts, php files).""" if self._resource_dir is None: self._resource_dir = os.path.dirname(os.path.abspath(sys.argv[0])) return self._resource_dir def website_branch_dir(self, branch, next=False): """Operation directory for the website.""" if branch not in self.enabled_branches: raise InvalidBranchException('%s is not a configured branch' % (branch)) if not next: return os.path.join(self.websitedir, branch) else: return os.path.join(self.nextwebsitedir, branch) def website_lang_dir(self, branch, lang, next=False): return os.path.join(self.website_branch_dir(branch, next), lang) def website_pkg_dir(self, branch, lang, pkg, next=False): return os.path.join(self.website_lang_dir(branch, lang, next), pkg) def op_branch_dir(self, branch): """Operation directory with scratch data (repositories, build directories, etc). """ if branch not in self.enabled_branches: raise InvalidBranchException('%s is not a configured branch' % (branch)) return os.path.join(self.workdir, branch) def op_lang_dir(self, branch, lang): return os.path.join(self.op_branch_dir(branch), lang) def op_pkg_dir(self, branch, lang, pkg): return os.path.join(self.op_lang_dir(branch, lang), pkg) def get_svn_dir(self, branch): """svn-specific directory for the requested branch.""" if branch not in self.enabled_branches: raise InvalidBranchException('%s is not a configured branch' % (branch)) try: svn_branch = self._conf.get(self.BRANCH_CONF_PREFIX + branch, 'l10n_path') - except (ConfigParser.NoOptionError) as e: + except (configparser.NoOptionError) as e: svn_branch = branch return svn_branch def work_dir_walker(self, branches=None, languages=None, packages=None): """Walks through the work directory (branches, languages, packages) """ if not languages: languages = self.configured_languages if not branches: branches = self.enabled_branches if not packages: packages = self.configured_packages LOGGER.debug('%s, %s, %s' % (branches, languages, packages)) for branch in branches: for lang in languages: for pkg in packages: LOGGER.debug('%s, %s, %s' % (branch, lang, pkg)) curr_path = self.op_pkg_dir(branch, lang, pkg) yield (branch, lang, pkg, curr_path) diff --git a/kdedocs/docgenerator.py b/kdedocs/docgenerator.py index bfb7437..6b45228 100644 --- a/kdedocs/docgenerator.py +++ b/kdedocs/docgenerator.py @@ -1,328 +1,328 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # # Copyright 2015 Luigi Toscano # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) version 3, or any # later version accepted by the membership of KDE e.V. (or its # successor approved by the membership of KDE e.V.), which shall # act as a proxy defined in Section 6 of version 3 of the license. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . from __future__ import absolute_import from datetime import datetime import glob import json import logging import os import shutil from collections import Counter from collections import namedtuple from collections import OrderedDict from .utils import compare_timestamp_file, find_files_pattern, mkdirp LOGGER = logging.getLogger(__name__) class ProgramDocData(object): """ Information about translated documentation for a specific program (for example, kdegraphics|okular or kde-runtime|kioslave/gzip). """ __slots__ = ['_languages', '_namespaces'] def __init__(self): self._languages = OrderedDict() self._namespaces = [] @property def namespaces(self): return self._namespaces @property def languages(self): return self._languages.keys() def get_branches(self, lang, pkg=None): all_branches_pkgs = self._languages.get(lang, None) if not all_branches_pkgs: return [] if not pkg: return [branchpkg[0] for branchpkg in all_branches_pkgs] else: return [branchpkg[0] for branchpkg in all_branches_pkgs if branchpkg[1] == pkg] def add_namespace(self, namespace): if namespace not in self._namespaces: self._namespaces.append(namespace) def add_language(self, lang): if lang not in self._languages.keys(): self._languages[lang] = [] def add_branch(self, lang, branch, pkg): self.add_language(lang) self.add_namespace(pkg) if (branch, pkg) not in self._languages[lang]: self._languages[lang].append((branch, pkg)) class DocGenerator(object): JSON_PREFIX = 'SITEDATA' def __init__(self, config, env_manager): self.config = config self.env_manager = env_manager self.programs_doc_data = [] excluded_lang_pdf_values = self.config.get_value('excluded_lang_pdf') self.excluded_lang_pdf = [lang.strip() for lang in excluded_lang_pdf_values.split(',')] LOGGER.debug('Excluding PDF generation for languages: %s' % (self.excluded_lang_pdf)) def _init_new_website(self): """Create the directory where the new website is built.""" LOGGER.debug('Initializing new website directory') if os.path.isdir(self.config.nextwebsitedir): shutil.rmtree(self.config.nextwebsitedir) mkdirp(self.config.nextwebsitedir) def _init_branches(self, branches): generated_branches = branches if not generated_branches: generated_branches = self.config.enabled_branches for branch in generated_branches: LOGGER.info('Init generation for branch "%s"' % (branch)) # FIXME: guard from invalid branch curr_env = self.env_manager.get_env(branch) curr_env.init_documentation_branch(branch) def _finalize_new_website(self): """Last steps of the generation of the new website: replace the old website with the new one.""" LOGGER.debug('Finalizing the new website') currtimestamp = datetime.now().strftime("%Y%m%d%H%M") if os.path.isdir(self.config.websitedir): shutil.move(self.config.websitedir, self.config.websitedir + currtimestamp) if os.path.isdir(self.config.nextwebsitedir): shutil.move(self.config.nextwebsitedir, self.config.websitedir) @staticmethod def are_docbooks_newer(base_dir, reference_file): """Returns true if at least one docbook in base_dir is newer than the referenced file.""" try: # get each docbook and compare the timestamp for db_name in os.listdir(base_dir): if not db_name.endswith('.docbook'): continue docbook_full_path = os.path.join(base_dir, db_name) docbook_time = os.path.getmtime(docbook_full_path) if compare_timestamp_file(docbook_time, reference_file) > 0: return True except: return True return False @staticmethod def are_images_changed(old_img_dir, new_img_dir): """Returns true if images in new_img_dir are different from the images in old_img_dir (at least one changed/added/removed). """ def get_pngs_set(base_img_dir): return set([img_name for img_name in os.listdir(base_img_dir) if img_name.endswith('.png')]) try: old_img_set = get_pngs_set(old_img_dir) new_img_set = get_pngs_set(new_img_dir) if old_img_set != new_img_set: # set changed, an image was added or removed return True for img in new_img_set: # size changed if (os.path.getsize(os.path.join(old_img_dir, img)) != os.path.getsize(os.path.join(new_img_dir, img))): return True # TODO? at this point the content could be checked as well, # but it is really likely to be the same for images. except: return True return False def generate_documentation(self, forced=False, branches=None, languages=None, packages=None): """ Generate the documentation """ self._init_new_website() self._init_branches(branches) LOGGER.info('Starting generation (forced: %s)' % (forced)) self.programs_doc_data = OrderedDict() for branch, lang, pkg, base_path in self.config.work_dir_walker( branches, languages, packages): # FIXME: guard from invalid branch curr_env = self.env_manager.get_env(branch) docbook_found = find_files_pattern(base_path, 'index.docbook') for ddir, _ in docbook_found: docbook_dir = os.path.join(base_path, ddir) currweb_dir = os.path.join(self.config.website_pkg_dir( branch, lang, pkg), ddir) nextweb_dir = os.path.join(self.config.website_pkg_dir( branch, lang, pkg, next=True), ddir) if lang != 'en': original_en_docbook = os.path.join(self.config.op_pkg_dir( branch, 'en', pkg), ddir, 'index.docbook') if not os.path.isfile(original_en_docbook): docbook_dir_rel = os.path.relpath(docbook_dir, self.config.workdir) LOGGER.debug('Skipping %s, no English version' % (docbook_dir_rel)) continue mkdirp(nextweb_dir) curr_env.copy_image_files(ddir, branch, lang, pkg) # FIXME: better handling of special case kcontrol/kio/etc program_name = ddir # check if the existing generated files (for HTML and PDF) # are really up-to-date with the source files. An additional # check is performed for PDF (the set of images) index_file = os.path.join(currweb_dir, 'index.html') if forced or self.are_docbooks_newer(docbook_dir, index_file): curr_env.generate_doc_html(ddir, branch, lang, pkg) else: # copy the existing HTMLs from the current site html_files = [hf for hf in os.listdir(currweb_dir) if hf.endswith('.html')] for hf in html_files: shutil.copy2(os.path.join(currweb_dir, hf), os.path.join(nextweb_dir, hf)) LOGGER.debug('%s/%s/%s - html: up-to-date' % (lang, branch, program_name)) if lang not in self.excluded_lang_pdf: pdf_name = ddir.split('/')[-1] + '.pdf' pdf_file = os.path.join(currweb_dir, pdf_name) if (forced or self.are_docbooks_newer(docbook_dir, pdf_file) or self.are_images_changed(currweb_dir, nextweb_dir)): curr_env.generate_doc_pdf(ddir, branch, lang, pkg) else: # copy the existing PDF from the current site shutil.copy2(os.path.join(currweb_dir, pdf_name), os.path.join(nextweb_dir, pdf_name)) LOGGER.debug('%s/%s/%s - pdf: up-to-date' % (lang, branch, program_name)) if program_name not in self.programs_doc_data: self.programs_doc_data[program_name] = ProgramDocData() self.programs_doc_data[program_name].add_branch(lang, branch, pkg) self._finalize_new_website() def write_generated_file(self, filename='generated_used.inc.php'): """ Write the files containing the list of available doc for each branch/language/program in json format (which will imported as PHP array and used accordingly). """ LOGGER.info('Writing knowledge files (JSON)') modules_programs = {} programs_multiplemodules = {} programs_docs = {} - for prog, progdocdata in self.programs_doc_data.iteritems(): + for prog, progdocdata in self.programs_doc_data.items(): # an application can have more than one namespace, even if # it is a rare occurence, but track it nevertheless. for namespace in progdocdata.namespaces: # prepare the list of programs for each module if namespace not in modules_programs: modules_programs[namespace] = [] modules_programs[namespace].append(prog) # save the exceptions (programs with branches in different # namespaces) if len(progdocdata.namespaces) > 1: LOGGER.debug('Found multiple namespaces (%s) for %s' % (progdocdata.namespaces, prog)) programs_multiplemodules[prog] = {} namespace_counter = {} try: for namespace in progdocdata.namespaces: for lang in self.config.configured_languages: found_branches = progdocdata.get_branches(lang, namespace ) # count where a certain namespace is defined for # each branch for found_branch in found_branches: if found_branch not in namespace_counter: namespace_counter[found_branch] = Counter() namespace_counter[found_branch].update( [namespace] ) # mark a certain branch of the program analysed as # belonging to a specific namespace; consider the namespace # most used amongst all languages (there should really be # just one namespace regardless of the language, but there # could be some leftovers in some languages...) - for abranch, ns_count in namespace_counter.iteritems(): + for abranch, ns_count in namespace_counter.items(): programs_multiplemodules[prog][abranch] = ( ns_count.most_common(1)[0][0] ) except Exception as e: LOGGER.info('Exception on multinamespace for %s: %s' % (prog, str(e)), exc_info=True) # main data: languages and branch for each program programs_docs[prog] = dict([(lang, progdocdata.get_branches(lang)) for lang in progdocdata.languages]) branch_description = {} for branch in self.config.enabled_branches: branch_desc = self.config.get_branch_value(branch, 'description') if not branch_desc: branch_desc = branch branch_description[branch] = branch_desc output_jsons = OrderedDict([ ('programs_docs', programs_docs), ('modules_programs', modules_programs), ('programs_multiplemodules', programs_multiplemodules), ('languagelist', self.config.configured_languages), ('branch_description', branch_description) ]) # remove old files json_pattern = os.path.join(self.config.workdir, '%s*.json' % (DocGenerator.JSON_PREFIX)) for oldjsons in glob.glob(json_pattern): os.remove(oldjsons) # generate the new json files count = 0 - for data_name, data_content in output_jsons.iteritems(): + for data_name, data_content in output_jsons.items(): filename = os.path.join(self.config.workdir, '%s%02d_%s.json' % (DocGenerator.JSON_PREFIX, count, data_name)) count += 1 with open(filename, 'w') as genfile: json.dump(data_content, genfile, indent=2) LOGGER.debug('Wrote knowledge file %s' % (filename)) diff --git a/kdedocs/environments.py b/kdedocs/environments.py index a4c898a..797d929 100644 --- a/kdedocs/environments.py +++ b/kdedocs/environments.py @@ -1,538 +1,537 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright 2015 Luigi Toscano # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) version 3, or any # later version accepted by the membership of KDE e.V. (or its # successor approved by the membership of KDE e.V.), which shall # act as a proxy defined in Section 6 of version 3 of the license. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . from __future__ import absolute_import import codecs import logging import os import re import shutil import subprocess import git from .utils import find_files_pattern, file_replace_re, mkdirp LOGGER = logging.getLogger(__name__) class DocEnvironment(object): ENV_ID = '' def __init__(self, config): self.config = config mkdirp(self.config.workdir) def get_config(self, key): """Return the value for the specified key in the current environment. """ return self.config.get_env_value(self.ENV_ID, key) def setup(self): LOGGER.info('Setting up environment %s' % (self.ENV_ID)) def local_catalog_path(self, branch): """Additional local DocBook catalogs, if needed.""" return None def customization_dir(self, branch): """Local working customization directory for the specified branch.""" return os.path.join(self.config.op_branch_dir(branch), 'customization') def xslt_stylesheet_path(self, branch): """Path to the main XSL stylesheet.""" raise NotImplementedError('This method must be redefined') def checkout_doc_init(self, branch): # copy prepared kdoctools files to the branch pass def checkout_doc_post(self, branch, languages=None, packages=None): pass def init_documentation_branch(self, branch): """Initialize the website directory of the specified branch.""" pass def get_extended_env(self, branch): """Return the current environment extended with the environment variables requested by the XSL processor. """ extra_catalog_path = self.local_catalog_path(branch) xslt_env = dict(os.environ) # FIXME: extend, not override, the variables if extra_catalog_path: xslt_env.update({'XML_CATALOG_FILES': extra_catalog_path, 'SGML_CATALOG_FILES': extra_catalog_path}) return xslt_env def get_common_path(self, branch, lang): """Return the path to the common documentation resources.""" raise NotImplementedError('This method must be redefined') def copy_image_files(self, docbookdir, branch, lang, pkg): """ Found available PNGs for the current language and - if missing - from English, and copy them.""" webnext_dir = os.path.join(self.config.website_pkg_dir(branch, lang, pkg, next=True), docbookdir) docdir_rel = os.path.relpath(webnext_dir, self.config.nextwebsitedir) png_all_languages = [lang] if 'en' not in png_all_languages: png_all_languages.append('en') for png_lang in png_all_languages: src_img_dir = os.path.join(self.config.op_pkg_dir(branch, png_lang, pkg), docbookdir) try: for png_file in os.listdir(src_img_dir): if not png_file.endswith('.png'): continue if not os.path.isfile(os.path.join(webnext_dir, png_file)): LOGGER.debug('copying %s from "%s" into %s' % (png_file, png_lang, docdir_rel)) shutil.copy2(os.path.join(src_img_dir, png_file), webnext_dir) except Exception as exc: LOGGER.warn('error while trying to access %s (%s)' % (src_img_dir, str(exc))) def generate_doc_html(self, docbookdir, branch, lang, pkg): docdir = os.path.join(self.config.op_pkg_dir(branch, lang, pkg), docbookdir) xslt_env = self.get_extended_env(branch) cmd_html = ['nice', '-n', '19', 'xsltproc', '--stringparam', 'kde.common', self.get_common_path(branch, lang), self.xslt_stylesheet_path(branch), os.path.join(docdir, 'index.docbook')] docdir_rel = os.path.relpath(docdir, self.config.workdir) LOGGER.debug('HTML generation (%s)' % (docdir_rel)) webnext_dir = os.path.join(self.config.website_pkg_dir(branch, lang, pkg, next=True), docbookdir) mkdirp(webnext_dir) proc = subprocess.Popen(cmd_html, cwd=webnext_dir, env=xslt_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc_out, proc_err = proc.communicate() proc_rc = proc.returncode if proc_rc != 0: LOGGER.warning('Error running "%s" from "%s", catalog: %s:\n' % (' '.join(cmd_html), webnext_dir, self.local_catalog_path(branch))) proc_data = [('stdout', proc_out), ('stderr', proc_err)] for p_msg in proc_data: try: LOGGER.warning('%s:\n%s' % (p_msg[0], p_msg[1])) except UnicodeDecodeError: LOGGER.warning('%s:\n%s' % (p_msg[0], p_msg[1].decode( 'latin-1'))) def generate_doc_pdf(self, docbookdir, branch, lang, pkg): docdir = os.path.join(self.config.op_pkg_dir(branch, lang, pkg), docbookdir) webnext_dir = os.path.join(self.config.website_pkg_dir(branch, lang, pkg, next=True), docbookdir) xslt_env = self.get_extended_env(branch) additional_env = {} # the following variables are needed by buildpdf.sh # DBLATEX_BASE_DIR must point to the base directory containing # dblatex-cvs-install additional_env['DBLATEX_BASE_DIR'] = self.config.resource_dir # TEXINPUTS is extended with the next website directory, which # contains the images referenced by the document texinputs = xslt_env.get('TEXINPUTS', '') if texinputs: texinputs = ':%s' % (texinputs) additional_env['TEXINPUTS'] = '%s%s' % (webnext_dir, texinputs) xslt_env.update(additional_env) cmd_pdf = ['nice', '-n', '19', os.path.join(self.config.resource_dir, 'buildpdf.sh'), os.path.join(docdir, 'index.docbook')] docdir_rel = os.path.relpath(docdir, self.config.workdir) LOGGER.debug('PDF generation (%s)' % (docdir_rel)) mkdirp(webnext_dir) proc = subprocess.Popen(cmd_pdf, cwd=webnext_dir, env=xslt_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc_out, proc_err = proc.communicate() proc_rc = proc.returncode if proc_rc != 0: # FIXME: investigate why the output from the command above (hence # pdftex) bails out here without using decode('latin-1') LOGGER.warning('Error running "%s" from "%s", catalog: %s, ' 'additional env: %s' % (' '.join(cmd_pdf), webnext_dir, self.local_catalog_path(branch), additional_env)) proc_data = [('stdout', proc_out), ('stderr', proc_err)] for p_msg in proc_data: try: LOGGER.warning('%s:\n%s' % (p_msg[0], p_msg[1])) except UnicodeDecodeError: LOGGER.warning('%s:\n%s' % (p_msg[0], p_msg[1].decode( 'latin-1'))) class KDELibs4Environment(DocEnvironment): ENV_ID = 'kdelibs4' def __init__(self, config): super(KDELibs4Environment, self).__init__(config) - self._help_re = re.compile( - ur'help:/((kioslave/|kcontrol/)?[A-Za-z0-9_\-]+)/*') + self._help_re = re.compile(r'help:/((kioslave/|kcontrol/)?[A-Za-z0-9_\-]+)/*') self.kdelibs_dir = os.path.join(self.config.workdir, 'kdelibs_repo') # FIXME: better check if it exists self.kdelibs_make_dir = self.get_config('compile_dir') def setup(self): super(KDELibs4Environment, self).setup() if not os.path.isdir(self.kdelibs_dir): git.Repo.clone_from('git://anongit.kde.org/kdelibs', self.kdelibs_dir, branch='KDE/4.14') else: kdelibs_repo = git.Repo(self.kdelibs_dir) b414 = kdelibs_repo.heads['KDE/4.14'].checkout() pull_res = kdelibs_repo.remotes.origin.pull() # Pass the location of the kdelibs4 repository to the shipped # simplified Makefile cmd_make = ['make', 'PATH_KDELIBS=%s' % (self.kdelibs_dir)] proc = subprocess.Popen(cmd_make, cwd=self.kdelibs_make_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc_out, proc_err = proc.communicate() proc_rc = proc.returncode if proc_rc != 0: LOGGER.warning('%s\n%s' % (proc_out, proc_err)) def local_catalog_path(self, branch): """Additional local DocBook catalogs""" catalog_path = os.path.join(self.customization_dir(branch), 'catalog.xml') return catalog_path def xslt_stylesheet_path(self, branch): return os.path.join(self.customization_dir(branch), 'kde-chunk-online.xsl') def checkout_doc_init(self, branch): super(KDELibs4Environment, self).checkout_doc_init(branch) # copy prepared customization files to the branch LOGGER.info('Initializing doc checkout for %s' % (branch)) # rm customization, copy the directory from kdelibs LOGGER.debug('doc_init, %s: copying fresh customization dir' % (branch)) if os.path.isdir(self.customization_dir(branch)): shutil.rmtree(self.customization_dir(branch)) bin_customization_dir = os.path.join(self.kdelibs_dir, 'kdoctools/customization') shutil.copytree(bin_customization_dir, self.customization_dir(branch), symlinks=True) # copy kde-*-online.xsl LOGGER.debug('doc_init, %s: copying kde-*-online.xsl' % (branch)) for xsl_templ in ['kde-chunk-online.xsl', 'kde-navig-online.xsl']: shutil.copy(xsl_templ, os.path.join(self.customization_dir(branch), xsl_templ)) # generate kdex.dtd LOGGER.debug('doc_init, %s: generating kdex.dtd' % (branch)) with codecs.open(os.path.join(self.customization_dir(branch), 'dtd/kdex.dtd.cmake'), 'r', encoding='utf-8') as sf, \ codecs.open(os.path.join(self.customization_dir(branch), 'dtd/kdex.dtd'), 'w', encoding='utf-8') as df: for line in sf: df.write(line.replace('@DOCBOOKXML_CURRENTDTD_DIR@', self.get_config('docbookxml_path'))) # generate kde-include-common.xsl LOGGER.debug('doc_init, %s: generating kde-include-common.xsl' % (branch)) docbookxsl_path = self.get_config('docbookxsl_path') with codecs.open(os.path.join(self.customization_dir(branch), 'kde-include-common.xsl.cmake'), 'r', encoding='utf-8') as sf, \ codecs.open(os.path.join(self.customization_dir(branch), 'kde-include-common.xsl'), 'w', encoding='utf-8') as df: for line in sf: df.write(line.replace('@DOCBOOKXSL_DIR@', docbookxsl_path)) # execute docbookl10nhelper LOGGER.debug('doc_init, %s: running docbookl10nhelper' % (branch)) xsl_doc_dir = os.path.join(self.customization_dir(branch), 'xsl') docbookl10nhelper_cmd = [os.path.join(self.kdelibs_make_dir, 'docbookl10nhelper'), docbookxsl_path, xsl_doc_dir, xsl_doc_dir] proc = subprocess.Popen(docbookl10nhelper_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc_out, proc_err = proc.communicate() proc_rc = proc.returncode if proc_rc != 0: LOGGER.debug('%s\n%s' % (proc_out, proc_err)) def checkout_doc_post(self, branch, languages=None, packages=None): pass def init_documentation_branch(self, branch): """Initialize the website directory of the specified branch.""" # rm common, copy it from kdelibs LOGGER.debug('doc_init, %s: copying common dir' % (branch)) common_en = os.path.join(self.config.website_branch_dir(branch, next=True), 'common') if os.path.isdir(common_en): shutil.rmtree(common_en) shutil.copytree(os.path.join(self.kdelibs_dir, 'doc/common'), common_en, symlinks=True) def get_common_path(self, branch, lang): """Return the path to the common documentation resources.""" return '/%s/common/' % (branch) def generate_doc_html(self, docbookdir, branch, lang, pkg): super(KDELibs4Environment, self).generate_doc_html(docbookdir, branch, lang, pkg) nextweb_docdir = os.path.join(self.config.website_pkg_dir( branch, lang, pkg, next=True), docbookdir) # symlinks to "common" are needed only in KDELibs4Environment # relative path to the global common directory common_symlink_path = os.path.join(nextweb_docdir, 'common') common_relpath = os.path.relpath(os.path.join( self.config.website_branch_dir(branch, next=True), 'common'), nextweb_docdir) if os.path.exists(common_symlink_path): if not os.path.islink(common_symlink_path): LOGGER.warning('%s exists and it is not a symlink' % (common_symlink_path)) else: dbg_location = os.path.relpath(nextweb_docdir, self.config.website_branch_dir( branch, next=True)) LOGGER.debug('adding symlink to "%s" common into: %s' % (lang, dbg_location)) os.symlink(common_relpath, common_symlink_path) # replace the internal help:/... strings. This step should be done # by XSLT - replaced_string = (ur'/?branch=%s&language=%s&application=\1&path=' % + replaced_string = (r'/?branch=%s&language=%s&application=\1&path=' % (branch, lang)) for docdir_file in os.listdir(nextweb_docdir): if docdir_file.endswith('.html'): html_file_path = os.path.join(nextweb_docdir, docdir_file) try: file_replace_re(html_file_path, self._help_re, replaced_string) except Exception as exc: LOGGER.warning('Error replacing %s: %s' % (html_file_path, str(exc))) class KF5Environment(DocEnvironment): ENV_ID = 'kf5' def __init__(self, config): super(KF5Environment, self).__init__(config) self._help_re = re.compile( - ur'help:/((kioslave5?/|kcontrol5?/)?[A-Za-z0-9_\-]+)/*') + r'help:/((kioslave5?/|kcontrol5?/)?[A-Za-z0-9_\-]+)/*') lib_tmp_dir = os.path.join(self.config.workdir, 'kdoctools') self.kdoctools_src_dir = os.path.join(lib_tmp_dir, 'repo') self.kdoctools_build_dir = os.path.join(lib_tmp_dir, 'build') self.kdoctools_compiled_dir = os.path.join(lib_tmp_dir, 'kdoctools5bin') def setup(self): super(KF5Environment, self).setup() try: if not os.path.isdir(self.kdoctools_src_dir): kdoctools_repo = git.Repo.clone_from( 'git://anongit.kde.org/kdoctools', self.kdoctools_src_dir, branch='master') else: kdoctools_repo = git.Repo(self.kdoctools_src_dir) master_branch = kdoctools_repo.heads['master'].checkout() pull_res = kdoctools_repo.remotes.origin.pull(*['--tags']) requested_ref = self.get_config('lib_ref') if requested_ref: # FIXME: error handling kdoctools_repo.head.reference = \ kdoctools_repo.refs[requested_ref] kdoctools_repo.head.reset(index=True, working_tree=True) except git.exc.GitCommandError as e: LOGGER.warn('Error while checking out the KF5 environment: %s' % (str(e))) force_cleanup = False force_cleanup_str = self.get_config('clean_build') if force_cleanup_str and force_cleanup_str.lower() == 'yes': force_cleanup = True if force_cleanup: if os.path.isdir(self.kdoctools_build_dir): shutil.rmtree(self.kdoctools_build_dir) if not os.path.isdir(self.kdoctools_build_dir): mkdirp(self.kdoctools_build_dir) force_cleanup = True cmd_cmake = (['cmake', '-DCMAKE_INSTALL_PREFIX=%s ' % (self.kdoctools_compiled_dir), '-DCMAKE_BUILD_TYPE=debugfull', '-DMEINPROC_NO_KARCHIVE=ON', '%s' % (self.kdoctools_src_dir)]) cmd_make = 'make' cmds = [cmd_make] if force_cleanup: # rerun cmake only for cleanup/newly created build directory cmds.insert(0, cmd_cmake) for cmd in cmds: proc = subprocess.Popen(cmd, cwd=self.kdoctools_build_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc_out, proc_err = proc.communicate() proc_rc = proc.returncode if proc_rc != 0: LOGGER.warning('%s\n%s\n%s' % (' '.join(cmd), proc_out, proc_err)) if not os.path.isdir(self.kdoctools_compiled_dir): # bail out, no previously working kdoctools raise Exception('Setup error') # otherwise return, a new kdoctools can not be installed; # the old one is reused return # cmake && make worked, so we can (safely) remove the old compiled # directory and replace with the up-to-date content if os.path.isdir(self.kdoctools_compiled_dir): shutil.rmtree(self.kdoctools_compiled_dir) cmd_makeinstall = ['make', 'install'] proc = subprocess.Popen(cmd_makeinstall, cwd=self.kdoctools_build_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc_out, proc_err = proc.communicate() proc_rc = proc.returncode if proc_rc != 0: LOGGER.warning('%s\n%s' % (proc_out, proc_err)) def local_catalog_path(self, branch): """Additional local DocBook catalogs""" return os.path.join(self.customization_dir(branch), 'catalog.xml') def xslt_stylesheet_path(self, branch): return os.path.join(self.customization_dir(branch), 'kde-chunk-online.xsl') def checkout_doc_init(self, branch): super(KF5Environment, self).checkout_doc_init(branch) # copy prepared customization files to the branch LOGGER.info('Initializing doc checkout for %s' % (branch)) # rm customization, copy the directory from kdelibs LOGGER.debug('doc_init, %s: copying fresh customization dir' % (branch)) if os.path.isdir(self.customization_dir(branch)): shutil.rmtree(self.customization_dir(branch)) bin_customization_dir = os.path.join(self.kdoctools_compiled_dir, 'share/kf5/kdoctools/' 'customization') shutil.copytree(bin_customization_dir, self.customization_dir(branch), symlinks=True) # copy kde-*-online.xsl LOGGER.debug('doc_init, %s: copying kde-*-online.xsl' % (branch)) for xsl_templ in ['kde-chunk-online.xsl', 'kde-navig-online.xsl']: shutil.copy(xsl_templ, os.path.join(self.customization_dir(branch), xsl_templ)) def init_documentation_branch(self, branch): """Initialize the website directory of the specified branch.""" # rm kdoctools5-common, copy it from kdoctools # Warning: only English for now LOGGER.debug('doc_init, %s: copying kdoctools5-common dir' % (branch)) common_en = os.path.join(self.config.website_lang_dir(branch, 'en', next=True), 'kdoctools5-common') if os.path.isdir(common_en): shutil.rmtree(common_en) shutil.copytree(os.path.join(self.kdoctools_compiled_dir, 'share/doc/HTML/en/kdoctools5-common/'), common_en, symlinks=True) def get_common_path(self, branch, lang): """Return the path to the common documentation resources. Warning: the language is not used, only English resources are used for now.""" return '/%s/en/kdoctools5-common/' % (branch) def generate_doc_html(self, docbookdir, branch, lang, pkg): super(KF5Environment, self).generate_doc_html(docbookdir, branch, lang, pkg) nextweb_docdir = os.path.join(self.config.website_pkg_dir(branch, lang, pkg, next=True), docbookdir) # replace the internal help:/... strings. This step should be done # by XSLT replaced_string = (r'/?branch=%s&language=%s&application=\1&path=' % (branch, lang)) for docdir_file in os.listdir(nextweb_docdir): if docdir_file.endswith('.html'): html_file_path = os.path.join(nextweb_docdir, docdir_file) try: file_replace_re(html_file_path, self._help_re, replaced_string) except Exception as exc: LOGGER.warning('Error replacing %s: %s' % (html_file_path, str(exc))) class DocEnvironmentsManager(object): """Read the list of requested environments from the configuration files and generate them.""" def __init__(self, config): self.config = config self._envs = {} self._init_envs() def _init_envs(self): """Initialize all the environment in configuration.""" for configured_env in self.config.configured_environments: """Find the class which implements the requested env and dynamically creates it. """ env_class = next((e_c for e_c in DocEnvironment.__subclasses__() if e_c.ENV_ID == configured_env), None) self._envs[configured_env] = env_class(self.config) def setup_all(self): for env in self._envs.values(): env.setup() def get_env(self, branch): """Return the environment object for the specified branch, based on the branch configuration.""" if branch not in self.config.enabled_branches: return None env_name = self.config.get_branch_value(branch, 'env') if not env_name: return None return self._envs[env_name]