diff --git a/README b/README index dc17217..960b736 100644 --- a/README +++ b/README @@ -1,57 +1,57 @@ What does each file in this folder do? ====================================== - * add-appstream-versions/add-versions.py + * add_appstream_versions.sh Run this first to add the new release version to the appstream files in your repos * add_bugzilla_versions.sh Run this to add the new release version to the bugs.kde.org versions * config Holds some general configuration * create_log.py Creates some html with log of what went into the release * create_sources_inc Creates the contents of the files that end up in trunk/www/sites/www/info/source-4.9.0.inc Should be run by the packager and uploaded to www * increase_repos_version.sh Increases the KDE_APPLICATIONS_VERSION_ variables in all repositories * language_list Holds the list of languages that need to be released * list_doc_subdirs.awk Retrieve the directories of the available handbooks by parsing the cmake files * modules.git List of git repositories + branches * pack.sh Creates the tarball of a given module * pack_all.sh Creates the tarball of all modules * README This file * RELEASE_PROMOTION Some hints on how to promote a release * remote-gpg Creates a connection that allows using a local gpg to sign packages on a remote server * tag_all.sh Creates tags of a release * UPDATING_VERSIONS A list of things to do check/do when doing a release * utils.sh Some common utilities * version Defines the version to package diff --git a/add-appstream-versions/.gitignore b/add-appstream-versions/.gitignore deleted file mode 100644 index 9f21b54..0000000 --- a/add-appstream-versions/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/venv/ diff --git a/add-appstream-versions/README.rst b/add-appstream-versions/README.rst deleted file mode 100644 index 18d30e7..0000000 --- a/add-appstream-versions/README.rst +++ /dev/null @@ -1,30 +0,0 @@ -Adds new project versions to the appstream metainfo files - -The script does the following: - -#. For each project listed in ``../modules.git`` - - #. Use CMake to determine the project version, as defined by CMake’s project_ - command. - - .. _project: https://cmake.org/cmake/help/latest/command/project.html#command:project - - If the variable is not found, the project is skipped. - - #. Add the version to the Appstream metainfo file for the target project. - -To run the script: - -#. Create, enter and prepare a Python virtual environment:: - - python3 -m venv venv - . venv/bin/activate - pip install -r requirements.txt - -#. Run the script:: - - python3 add-versions.py \ - -s - - .. note:: If you skip a parameter, the script prompts you for it on the - command line. diff --git a/add-appstream-versions/add-versions.py b/add-appstream-versions/add-versions.py deleted file mode 100755 index e7615b8..0000000 --- a/add-appstream-versions/add-versions.py +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2015 Jonathan Riddell -# Copyright 2017 Adrian Chaves -# Copyright 2019 Jonathan Riddell -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of -# the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import sys -sys.path.append("..") - -import os -from subprocess import run, DEVNULL, PIPE -import glob -import add_versions_lib - -from bs4 import BeautifulSoup -from click import command, echo, option, prompt, DateTime -from git import GitCommandError, Repo -from git.exc import InvalidGitRepositoryError, NoSuchPathError -from subprocess import run, DEVNULL, PIPE - -import configparser - -CONFIG_PATH='../config' -with open(CONFIG_PATH, 'r') as f: - config_string = '[dummy_section]\n' + f.read() -config = configparser.ConfigParser() -config.read_string(config_string) - -APPSTREAM_UPDATER=config['dummy_section']['APPSTREAM_UPDATER'] - - -class AddVersionError(RuntimeError): - - def __init__(self, cause=None, *args, **kwargs): - super().__init__(*args, **kwargs) - self.cause = cause - - -def add_version_to_appstream(source_folder, version, date): - try: - appstream_files = glob.glob(source_folder + '/**/*appdata.xml', recursive=True) - appstream_files += glob.glob(source_folder + '/**/*metainfo.xml', recursive=True) - for appstream_file in appstream_files: - date_string = date.strftime('%A, %-d %B %Y') - result = run([APPSTREAM_UPDATER, "--version", version, "--datestring", date_string, "--releases-to-show" , "4", appstream_file]) - result.check_returncode() - if len(appstream_files) > 0: - os.chdir(source_folder) - result = run(['git', 'commit', '-a', '-m', 'Update Appstream for new release']) - result.check_returncode() - result = run(['git', 'push']) - result.check_returncode() - except: - raise AddVersionError() - - -@command() -@option('-s', '--srcdir', prompt='Source folder', - help='Folder containing local clones of the Git repositories from ' - '../modules.git') -@option('--date', prompt='Release date', type=DateTime(), - help='Planned release date to record in AppStream files') -@option('-v', '--verbose', is_flag=True, - help='Provide additional details about errors') -@option('-d', '--dry', is_flag=True, - help='Do not submit anything to Bugzilla. Note: Local Git clones may ' - 'be modified.') -@option('-c', '--clone', is_flag=True, - help='Clone missing Git folders.') -@option('--hide-skipped', is_flag=True, - help='Do not print lines for skipped products') -def main(srcdir, date, verbose, dry, clone, hide_skipped): - version_file = open("../version", "r") - version = version_file.readline().rstrip() - bugfix_version = int(version.split('.')[2]) - if bugfix_version > 50: - echo('Version number indicates this is a testing release so not adding versions to Appstream files') - exit(1) - - srcdir = os.path.abspath(srcdir.rstrip('/')) - for product, branch in add_versions_lib.products_and_branches(): - source_folder = srcdir + '/' + product - try: - repository = Repo(source_folder) - except NoSuchPathError: - if clone: - try: - git_url = 'git://anongit.kde.org/{}'.format(product) - repository = Repo.clone_from(git_url, source_folder) - except KeyboardInterrupt: - if os.path.exists(source_folder): - rmtree(source_folder, ignore_errors=True) - exit(0) - except: - raise - echo('{} (error: could not clone repository)'.format(product)) - if verbose: - echo('\tGit URL: {}'.format(git_url)) - echo('\tTarget folder: {}'.format(source_folder)) - exit(1) - else: - echo('{} (error: source folder not found)'.format(product)) - if verbose: - echo('\tSource folder: {}'.format(source_folder)) - exit(1) - except InvalidGitRepositoryError: - echo('{} (error: source folder is not a Git clone)'.format(product)) - if verbose: - echo('\tSource folder: {}'.format(source_folder)) - exit(1) - if not os.path.isfile(APPSTREAM_UPDATER): - echo('Can not find appstream-metainfo-release-update at ' + APPSTREAM_UPDATER) - echo('Get it from git@invent.kde.org:jriddell/appstream-metainfo-release-update.git') - exit(1) - try: - repository.heads[branch].checkout() - except GitCommandError: - echo('{} (error: cannot checkout branch)'.format(product)) - if verbose: - echo('\tSource folder: {}'.format(source_folder)) - echo('\tCurrent branch: {}'.format(repository.head.name)) - echo('\tBranch to check out: {}'.format(branch)) - exit(1) - except IndexError: - echo('{} (error: cannot checkout branch: branch does not exist)' - .format(product)) - if verbose: - echo('\tSource folder: {}'.format(source_folder)) - echo('\tCurrent branch: {}'.format(repository.head.name)) - echo('\tBranch to check out: {}'.format(branch)) - exit(1) - try: - repository.remote().pull("--rebase") - except GitCommandError: - echo('{} (error: cannot pull branch)'.format(product)) - if verbose: - echo('\tSource folder: {}'.format(source_folder)) - echo('\tBranch: {}'.format(branch)) - exit(1) - try: - version = add_versions_lib.project_version(source_folder) - if version is None: - if hide_skipped: - continue - echo('{} \n (no project version found)'.format(product)) - if verbose: - echo('\tSource folder: {}'.format(source_folder)) - echo('\t{}/CMakeLists.txt does not define a project version' - .format(source_folder)) - echo('\tSee https://community.kde.org/' - 'Guidelines_and_HOWTOs/' - 'Application_Versioning#Bugzilla_versions') - continue - if not dry: - try: - add_version_to_appstream(source_folder, version, date) - echo('{}\n (added {})'.format( - product, version)) - except AddVersionError as error: - if error.cause: - echo('{}\n (would have added {})\n (error: {})'.format(product, version, error.cause)) - else: - echo('{}\n (would have added {})\n (error: {})'.format(product, version, error)) - except Exception as e: - echo('{}\n (would have added {})\n error: {})'.format(product, version, e)) - else: - echo('{}\n (would have added {})'.format( - product, version)) - except RuntimeError as error: - echo('ERROR: {}'.format(error)) - exit(1) - - -if __name__ == '__main__': - main() diff --git a/add-appstream-versions/requirements.txt b/add-appstream-versions/requirements.txt deleted file mode 100644 index 9e403f1..0000000 --- a/add-appstream-versions/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -beautifulsoup4==4.6.0 -click==7.* -GitPython==2.1.14 -requests==2.18.2 diff --git a/add_appstream_versions.sh b/add_appstream_versions.sh new file mode 100755 index 0000000..2df5a79 --- /dev/null +++ b/add_appstream_versions.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +"$(dirname "$(readlink -f $0)")/tools/python-kde-release/run.sh" add-appstream-versions $* diff --git a/tools/python-kde-release/kde_release/appstream.py b/tools/python-kde-release/kde_release/appstream.py new file mode 100644 index 0000000..f6a00bc --- /dev/null +++ b/tools/python-kde-release/kde_release/appstream.py @@ -0,0 +1,79 @@ +# Copyright 2015, 2019 Jonathan Riddell +# Copyright 2017, 2019 Adrian Chaves +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from glob import glob +from os import chdir, path +from shutil import rmtree +from subprocess import run + +from click import echo, ClickException +from git import GitCommandError, Repo +from git.exc import InvalidGitRepositoryError, NoSuchPathError + +from kde_release.cmake import project_version +from kde_release.configuration import CONFIGURATION +from kde_release.environment import KDE_RELEASE_DIR +from kde_release.modules import update_modules + + +APPSTREAM_UPDATER = CONFIGURATION['DEFAULT']['APPSTREAM_UPDATER'] + + +def add_version_to_appstream(source_folder, version, date): + appstream_files = glob(source_folder + '/**/*appdata.xml', recursive=True) + appstream_files += glob(source_folder + '/**/*metainfo.xml', recursive=True) + for appstream_file in appstream_files: + date_string = date.strftime('%A, %-d %B %Y') + result = run([APPSTREAM_UPDATER, "--version", version, "--datestring", date_string, "--releases-to-show" , "4", appstream_file]) + result.check_returncode() + if len(appstream_files) > 0: + chdir(source_folder) + result = run(['git', 'commit', '-a', '-m', 'Update Appstream for new release']) + result.check_returncode() + result = run(['git', 'push']) + result.check_returncode() + + +def add_versions(srcdir, date, verbose, dry, clone, hide_skipped): + with (KDE_RELEASE_DIR / 'version').open() as version_file: + global_version = version_file.readline().rstrip() + if int(global_version.split('.')[2]) > 50: + raise ClickException( + f'Version number {global_version} indicates this is a testing ' + f'release so not adding versions to Appstream files') + + if not path.isfile(APPSTREAM_UPDATER): + raise ClickException( + f'Can not find appstream-metainfo-release-update executable file ' + f'at {APPSTREAM_UPDATER}.\nGet it from git@invent.kde.org:jriddell' + f'/appstream-metainfo-release-update.git') + + kwargs = {'clone': clone, + 'log_missing_versions': not hide_skipped} + for directory, product, version in update_modules(srcdir, **kwargs): + if dry: + echo(f'{product}\n' + f'\t(would have added {version})') + continue + try: + add_version_to_appstream(directory, version, date) + except Exception as error: + echo(f'{product}\n' + f'\t(would have added {version})\n' + f'\t\t(error: {repr(error)})') + else: + echo(f'{product}\n' + f'\t(added {version})') diff --git a/tools/python-kde-release/kde_release/bugzilla.py b/tools/python-kde-release/kde_release/bugzilla.py new file mode 100644 index 0000000..2540092 --- /dev/null +++ b/tools/python-kde-release/kde_release/bugzilla.py @@ -0,0 +1,96 @@ +# Copyright 2015, 2019 Jonathan Riddell +# Copyright 2017, 2019 Adrian Chaves +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import re +from os import path + +from bs4 import BeautifulSoup +from click import echo +from requests import RequestException, Session + +from kde_release.modules import update_modules + + +URL = 'https://bugs.kde.org' +EDIT_VERSION_URL = '{}/editversions.cgi'.format(URL) + + +def response_error(response): + html = response.text + soup = BeautifulSoup(html, 'html.parser') + error_message = soup.find('div', {'id': 'error_msg'}) + if not error_message: + return None + return re.sub(r'\s+', ' ', error_message.get_text()).strip() + + +def log_in(session, email, password): + data = {'Bugzilla_login': email, 'Bugzilla_password': password} + headers = {'Referer': 'https://bugs.kde.org/'} + try: + response = session.post(URL, data=data, headers=headers) + error = response_error(response) + if error: + echo(error) + exit(1) + except RequestException: + echo('Unexpected login error. Please, contact the maintainers of this ' + 'script.') + exit(1) + + +def bugzilla_csrf_token(session, product): + params = {'action': 'add', 'product': product} + response = session.get(EDIT_VERSION_URL, params=params) + soup = BeautifulSoup(response.text, 'html.parser') + try: + return soup.find('input', {'name': 'token'})['value'] + except Exception: + raise RuntimeError('Could not parse token from \'{}\''.format( + response.url)) + + +def add_version_to_bugzilla_project(session, product, version): + params = {'version': version, 'action': 'new', 'product': product, + 'token': bugzilla_csrf_token(session, product)} + response = session.get(EDIT_VERSION_URL, params=params) + response.raise_for_status() + error = response_error(response) + if error: + raise RuntimeError(error) + + +def add_versions(srcdir, email, password, dry, clone, hide_skipped): + session = Session() + if not dry: + log_in(session, email, password) + + kwargs = {'clone': clone, + 'log_missing_versions': not hide_skipped} + for directory, product, version in update_modules(srcdir, **kwargs): + if dry: + echo(f'{product}\n' + f'\t(would have added {version})') + continue + try: + add_version_to_bugzilla_project(session, product, version) + except Exception as error: + echo(f'{product}\n' + f'\t(would have added {version})\n' + f'\t\t(error: {repr(error)})') + else: + echo(f'{product}\n' + f'\t(added {version})') diff --git a/tools/python-kde-release/kde_release/cli.py b/tools/python-kde-release/kde_release/cli.py index 67b9bf3..64347eb 100644 --- a/tools/python-kde-release/kde_release/cli.py +++ b/tools/python-kde-release/kde_release/cli.py @@ -1,244 +1,98 @@ -# Copyright 2015 Jonathan Riddell +# Copyright 2015, 2019 Jonathan Riddell # Copyright 2017, 2019 Adrian Chaves # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import os -import re -from os import environ -from pathlib import Path -from shutil import rmtree -from subprocess import run, DEVNULL, PIPE -from tempfile import mkdtemp +from click import command, option, prompt, ClickException, DateTime -from bs4 import BeautifulSoup -from click import command, echo, option, prompt -from git import GitCommandError, Repo -from git.exc import InvalidGitRepositoryError, NoSuchPathError -from requests import RequestException, Session +from kde_release.appstream import add_versions as _add_appstream_versions +from kde_release.bugzilla import add_versions as _add_bugzilla_versions -URL = 'https://bugs.kde.org' -EDIT_VERSION_URL = '{}/editversions.cgi'.format(URL) -VERSION_RE = re.compile(r'(?i):\s+project\s*\(\s*\S+\s+' - r'VERSION\s+(\d+(?:\.\d+(?:\.\d+(?:\.\d+)?)?)?)') - - -def response_error(response): - html = response.text - soup = BeautifulSoup(html, 'html.parser') - error_message = soup.find('div', {'id': 'error_msg'}) - if not error_message: - return None - return re.sub(r'\s+', ' ', error_message.get_text()).strip() - - -def log_in(session, email, password): - data = {'Bugzilla_login': email, 'Bugzilla_password': password} - headers = {'Referer': 'https://bugs.kde.org/'} - try: - response = session.post(URL, data=data, headers=headers) - error = response_error(response) - if error: - echo(error) - exit(1) - except RequestException: - echo('Unexpected login error. Please, contact the maintainers of this ' - 'script.') - exit(1) - - -def products_and_branches(): - modules_path = Path(environ['KDE_RELEASE_DIR']) / 'modules.git' - with modules_path.open() as modules_file: - for line in modules_file: - match = re.match(r'^(\S+)\s+(\S+)$', line) - if match: - yield match.group(1), match.group(2) - - -def cmake_trace(folder): - temp_folder = mkdtemp() - try: - cmake_command = ['cmake', '--trace-expand', folder] - execution = run( - cmake_command, cwd=temp_folder, stdout=DEVNULL, stderr=PIPE) - return execution.stderr.decode(errors='ignore') - finally: - rmtree(temp_folder) - - -def project_version(source_folder): - match = None - for match in VERSION_RE.finditer(cmake_trace(source_folder)): - pass - if not match: - return None - return match.group(1) - - -def bugzilla_csrf_token(session, product): - params = {'action': 'add', 'product': product} - response = session.get(EDIT_VERSION_URL, params=params) - soup = BeautifulSoup(response.text, 'html.parser') - try: - return soup.find('input', {'name': 'token'})['value'] - except: - raise RuntimeError('Could not parse token from \'{}\''.format( - response.url)) +def handle_all_exceptions(function): + def wrapper(*args, **kwargs): + try: + function(*args, **kwargs) + except ClickException: + raise + except Exception as error: + raise ClickException(repr(error)) + return wrapper -class AddVersionError(RuntimeError): +@command() +@option('-s', '--srcdir', prompt='Source folder', + help='Folder containing local clones of the Git repositories from ' + '../modules.git') +@option('--date', prompt='Release date', type=DateTime(), + help='Planned release date to record in AppStream files') +@option('-v', '--verbose', is_flag=True, + help='Provide additional details about errors') +@option('-d', '--dry', is_flag=True, + help='Do not submit anything to Bugzilla. Note: Local Git clones may ' + 'be modified.') +@option('-c', '--clone', is_flag=True, + help='Clone missing Git folders.') +@option('--hide-skipped', is_flag=True, + help='Do not print lines for skipped products') +@handle_all_exceptions +def add_appstream_versions(srcdir, date, verbose, dry, clone, hide_skipped): + """Adds new project versions to AppStream metainfo files. - def __init__(self, cause=None, *args, **kwargs): - super().__init__(*args, **kwargs) - self.cause = cause + Speficically, for each project listed in modules.git: + 1. CMake is used to determine the project version, as defined by CMake’s + project command. If the version is not found, the project is skipped. -def add_version_to_bugzilla_project(session, product, version): - params = {'version': version, 'action': 'new', 'product': product, - 'token': bugzilla_csrf_token(session, product)} - try: - response = session.get(EDIT_VERSION_URL, params=params) - response.raise_for_status() - except RequestException: - raise AddVersionError() - error = response_error(response) - if error: - raise AddVersionError(error) + 2. A version is added to the AppStream metainfo file of the target + project. + """ + _add_appstream_versions(srcdir, date, verbose, dry, clone, hide_skipped) @command() @option('-s', '--srcdir', prompt='Source folder', help='Folder containing local clones of the Git repositories from ' 'modules.git') @option('-e', '--email', help='Email of your Bugzilla account') @option('-p', '--password', help='Password of your Bugzilla account') -@option('-v', '--verbose', is_flag=True, - help='Provide additional details about errors') +@option('-v', '--verbose', is_flag=True, hidden=True) @option('-d', '--dry', is_flag=True, help='Do not submit anything to Bugzilla. Note: Local Git clones may ' 'be modified.') @option('-c', '--clone', is_flag=True, help='Clone missing Git folders.') @option('--hide-skipped', is_flag=True, help='Do not print lines for skipped products') +@handle_all_exceptions def add_bugzilla_versions(srcdir, email, password, verbose, dry, clone, hide_skipped): """Adds new project versions to the KDE Bugtracking System. Speficically, for each project listed in modules.git: 1. CMake is used to determine the project version, as defined by CMake’s project command. If the version is not found, the project is skipped. 2. A version is added to the target project in the KDE Bugtracking System. """ if not dry: if email is None: email = prompt('Email') if password is None: password = prompt('Password', hide_input=True) - srcdir = os.path.abspath(srcdir.rstrip('/')) - session = Session() - if not dry: - log_in(session, email, password) - for product, branch in products_and_branches(): - source_folder = srcdir + '/' + product - try: - repository = Repo(source_folder) - except NoSuchPathError: - if clone: - try: - git_url = 'git://anongit.kde.org/{}'.format(product) - repository = Repo.clone_from(git_url, source_folder) - except KeyboardInterrupt: - if os.path.exists(source_folder): - rmtree(source_folder, ignore_errors=True) - exit(0) - except: - raise - echo('{} (error: could not clone repository)'.format(product)) - if verbose: - echo('\tGit URL: {}'.format(git_url)) - echo('\tTarget folder: {}'.format(source_folder)) - exit(1) - else: - echo('{} (error: source folder not found)'.format(product)) - if verbose: - echo('\tSource folder: {}'.format(source_folder)) - exit(1) - except InvalidGitRepositoryError: - echo('{} (error: source folder is not a Git clone)'.format(product)) - if verbose: - echo('\tSource folder: {}'.format(source_folder)) - exit(1) - try: - repository.heads[branch].checkout() - except GitCommandError: - echo('{} (error: cannot checkout branch)'.format(product)) - if verbose: - echo('\tSource folder: {}'.format(source_folder)) - echo('\tCurrent branch: {}'.format(repository.head.name)) - echo('\tBranch to check out: {}'.format(branch)) - exit(1) - except IndexError: - echo('{} (error: cannot checkout branch: branch does not exist)' - .format(product)) - if verbose: - echo('\tSource folder: {}'.format(source_folder)) - echo('\tCurrent branch: {}'.format(repository.head.name)) - echo('\tBranch to check out: {}'.format(branch)) - exit(1) - try: - repository.remote().pull("--rebase") - except GitCommandError: - echo('{} (error: cannot pull branch)'.format(product)) - if verbose: - echo('\tSource folder: {}'.format(source_folder)) - echo('\tBranch: {}'.format(branch)) - exit(1) - try: - version = project_version(source_folder) - if version is None: - if hide_skipped: - continue - echo('{} \n (no project version found)'.format(product)) - if verbose: - echo('\tSource folder: {}'.format(source_folder)) - echo('\t{}/CMakeLists.txt does not define a project version' - .format(source_folder)) - echo('\tSee https://community.kde.org/' - 'Guidelines_and_HOWTOs/' - 'Application_Versioning#Bugzilla_versions') - continue - if not dry: - try: - add_version_to_bugzilla_project(session, product, version) - echo('{}\n (added {})'.format( - product, version)) - except AddVersionError as error: - if error.cause: - echo('{}\n (would have added {})\n (error: {})'.format(product, version, error.cause)) - else: - echo('{}\n (would have added {})\n (error: {})'.format(product, version, error)) - except Exception as e: - echo('{}\n (would have added {})\n error: {})'.format(product, version, e)) - else: - echo('{}\n (would have added {})'.format( - product, version)) - except RuntimeError as error: - echo('ERROR: {}'.format(error)) - exit(1) + if verbose: + echo('WARNING: The -v/--verbose option is deprecated') + _add_bugzilla_versions(srcdir, email, password, dry, clone, hide_skipped) diff --git a/add_versions_lib.py b/tools/python-kde-release/kde_release/cmake.py similarity index 68% rename from add_versions_lib.py rename to tools/python-kde-release/kde_release/cmake.py index 2a35338..1a827b8 100644 --- a/add_versions_lib.py +++ b/tools/python-kde-release/kde_release/cmake.py @@ -1,61 +1,44 @@ -# Copyright 2015 Jonathan Riddell -# Copyright 2017 Adrian Chaves -# Copyright 2019 Jonathan Riddell +# Copyright 2015, 2019 Jonathan Riddell +# Copyright 2017, 2019 Adrian Chaves # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import re -from pathlib import Path -from tempfile import mkdtemp from shutil import rmtree from subprocess import run, DEVNULL, PIPE +from tempfile import mkdtemp + VERSION_RE = re.compile(r'(?i):\s+project\s*\(\s*\S+\s+' r'VERSION\s+(\d+(?:\.\d+(?:\.\d+(?:\.\d+)?)?)?)') -class AddVersionError(RuntimeError): - - def __init__(self, cause=None, *args, **kwargs): - super().__init__(*args, **kwargs) - self.cause = cause - - -def products_and_branches(): - modules_path = Path(__file__).resolve().parent / 'modules.git' - with modules_path.open() as modules_file: - for line in modules_file: - match = re.match(r'^(\S+)\s+(\S+)$', line) - if match: - yield match.group(1), match.group(2) - - def cmake_trace(folder): temp_folder = mkdtemp() try: cmake_command = ['cmake', '--trace-expand', folder] execution = run( cmake_command, cwd=temp_folder, stdout=DEVNULL, stderr=PIPE) return execution.stderr.decode(errors='ignore') finally: rmtree(temp_folder) def project_version(source_folder): match = None for match in VERSION_RE.finditer(cmake_trace(source_folder)): pass if not match: return None return match.group(1) diff --git a/tools/python-kde-release/kde_release/configuration.py b/tools/python-kde-release/kde_release/configuration.py new file mode 100644 index 0000000..8c12780 --- /dev/null +++ b/tools/python-kde-release/kde_release/configuration.py @@ -0,0 +1,26 @@ +# Copyright 2015, 2019 Jonathan Riddell +# Copyright 2017, 2019 Adrian Chaves +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from configparser import ConfigParser + +from kde_release.environment import KDE_RELEASE_DIR + + +CONFIGURATION = ConfigParser() +_path = KDE_RELEASE_DIR / 'config' +with _path.open() as config_file: + _configuration_string = '[DEFAULT]\n' + config_file.read() +CONFIGURATION.read_string(_configuration_string, _path) diff --git a/tools/python-kde-release/kde_release/environment.py b/tools/python-kde-release/kde_release/environment.py new file mode 100644 index 0000000..0e7217c --- /dev/null +++ b/tools/python-kde-release/kde_release/environment.py @@ -0,0 +1,20 @@ +# Copyright 2015, 2019 Jonathan Riddell +# Copyright 2017, 2019 Adrian Chaves +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from os import environ +from pathlib import Path + +KDE_RELEASE_DIR = Path(environ['KDE_RELEASE_DIR']) diff --git a/tools/python-kde-release/kde_release/modules.py b/tools/python-kde-release/kde_release/modules.py new file mode 100644 index 0000000..eff904a --- /dev/null +++ b/tools/python-kde-release/kde_release/modules.py @@ -0,0 +1,115 @@ +# Copyright 2015, 2019 Jonathan Riddell +# Copyright 2017, 2019 Adrian Chaves +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import re +from os import path +from shutil import rmtree + +from click import echo +from git import GitCommandError, Repo +from git.exc import InvalidGitRepositoryError, NoSuchPathError + +from kde_release.cmake import project_version +from kde_release.environment import KDE_RELEASE_DIR + + +def products_and_branches(): + modules_path = KDE_RELEASE_DIR / 'modules.git' + with modules_path.open() as modules_file: + for line in modules_file: + match = re.match(r'^(\S+)\s+(\S+)$', line) + if match: + yield match.group(1), match.group(2) + + +class ProductRuntimeError(RuntimeError): + + def __init__(self, product, error, context): + context = '\n'.join(f'\t{key}: {value}' for key, value in context) + message = f'{product} (error: {error})\n{context}' + super().__init__(message) + + +def update_modules(srcdir, clone=True, log_missing_versions=True): + """Iterates the products and branches in the modules.git file, updates them + locally, and yields the path of their local clone.""" + srcdir = path.abspath(srcdir.rstrip('/')) + for product, branch in products_and_branches(): + directory = srcdir + '/' + product + + try: + repository = Repo(directory) + except NoSuchPathError: + if clone: + try: + git_url = 'git://anongit.kde.org/{}'.format(product) + repository = Repo.clone_from(git_url, directory) + except KeyboardInterrupt: + if path.exists(directory): + rmtree(directory, ignore_errors=True) + raise + except Exception: + context = (('Git URL', git_url), + ('Target folder', directory)) + raise ProductRuntimeError( + product, 'could not clone repository', context) + else: + context = (('Source folder', directory),) + raise ProductRuntimeError( + product, 'source folder not found', context) + except InvalidGitRepositoryError: + context = (('Source folder', directory),) + raise ProductRuntimeError( + product, 'source folder is not a Git clone', context) + + try: + repository.heads[branch].checkout() + except GitCommandError: + context = (('Source folder', directory), + ('Current branch', repository.head.name), + ('Branch to check out', branch)) + raise ProductRuntimeError( + product, 'cannot checkout branch', context) + except IndexError: + context = (('Source folder', directory), + ('Current branch', repository.head.name), + ('Branch to check out', branch)) + raise ProductRuntimeError( + product, 'branch does not exist', context) + + try: + repository.remote().pull("--rebase") + except GitCommandError: + context = (('Source folder', directory), + ('Branch', branch)) + raise ProductRuntimeError(product, 'cannot pull branch', context) + + product = path.basename(directory) + + version = project_version(directory) + if version is None: + if log_missing_versions: + echo(f'{product}\n' + f'\t(no project version found)' + f'\t\tSource folder: {directory}' + f'\t\t{directory}/CMakeLists.txt does not define a ' + f'project version' + f'\t\tSee https://community.kde.org/' + f'Guidelines_and_HOWTOs/' + f'Application_Versioning#Bugzilla_versions') + continue + + yield directory, product, version diff --git a/tools/python-kde-release/run.sh b/tools/python-kde-release/run.sh index 6b2fdd5..20ae1ba 100755 --- a/tools/python-kde-release/run.sh +++ b/tools/python-kde-release/run.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash function finish { deactivate &> /dev/null } trap finish EXIT deactivate &> /dev/null dir="$(dirname "$(readlink -f $0)")" -[ ! -d "$DIR" ] && python3 -m venv "$dir/venv" +[ ! -d "$dir" ] && python3 -m venv "$dir/venv" . "$dir/venv/bin/activate" pip install --upgrade "$dir" &> /dev/null KDE_RELEASE_DIR="$(dirname "$(dirname "$dir")")" "$1" "${@:2}" diff --git a/tools/python-kde-release/setup.py b/tools/python-kde-release/setup.py index 16d252f..2c4e105 100644 --- a/tools/python-kde-release/setup.py +++ b/tools/python-kde-release/setup.py @@ -1,23 +1,24 @@ #!/usr/bin/env python3 from setuptools import find_packages, setup setup( name='kde-release', version='1.0', description='Utilities of the KDE Release team', packages=find_packages(), python_requires='>=3.6', install_requires=[ 'beautifulsoup4 >=4.6.0, <5', - 'click >=6.7, <7', + 'click >=7, <8', 'GitPython >=2.1.14, <3', 'requests >=2.18.2, <3', ], entry_points={ 'console_scripts': [ 'add-bugzilla-versions = kde_release.cli:add_bugzilla_versions', + 'add-appstream-versions = kde_release.cli:add_appstream_versions', ], }, )