diff --git a/README b/README index 26d3a87..dc17217 100644 --- a/README +++ b/README @@ -1,57 +1,57 @@ What does each file in this folder do? ====================================== * add-appstream-versions/add-versions.py Run this first to add the new release version to the appstream files in your repos - * add-bugzilla-versions/add-versions.py + * add_bugzilla_versions.sh Run this to add the new release version to the bugs.kde.org versions * config Holds some general configuration * create_log.py Creates some html with log of what went into the release * create_sources_inc Creates the contents of the files that end up in trunk/www/sites/www/info/source-4.9.0.inc Should be run by the packager and uploaded to www * increase_repos_version.sh Increases the KDE_APPLICATIONS_VERSION_ variables in all repositories * language_list Holds the list of languages that need to be released * list_doc_subdirs.awk Retrieve the directories of the available handbooks by parsing the cmake files * modules.git List of git repositories + branches * pack.sh Creates the tarball of a given module * pack_all.sh Creates the tarball of all modules * README This file * RELEASE_PROMOTION Some hints on how to promote a release * remote-gpg Creates a connection that allows using a local gpg to sign packages on a remote server * tag_all.sh Creates tags of a release * UPDATING_VERSIONS A list of things to do check/do when doing a release * utils.sh Some common utilities * version Defines the version to package diff --git a/add-bugzilla-versions/README.rst b/add-bugzilla-versions/README.rst deleted file mode 100644 index 7d7ff3e..0000000 --- a/add-bugzilla-versions/README.rst +++ /dev/null @@ -1,32 +0,0 @@ -Adds new project versions to the KDE Bugtracking System. - -The script does the following: - -#. For each project listed in :code:`add-bugzilla-versions/../modules.git`. - - #. Use CMake to determine the project version, as defined by CMake’s project_ - command. - - .. _project: https://cmake.org/cmake/help/latest/command/project.html#command:project - - If the variable is not found, the project is skipped. - - #. Add the variable to the target project in the KDE Bugtracking System. - -To run the script: - -#. Create, enter and prepare a Python virtual environment:: - - python3 -m venv venv - . venv/bin/activate - pip install -r requirements.txt - -#. Run the script:: - - python3 add-versions.py \ - -s \ - -e \ - -p - - .. note:: If you skip a parameter, the script prompts you for it on the - command line. diff --git a/add-bugzilla-versions/requirements.txt b/add-bugzilla-versions/requirements.txt deleted file mode 100644 index 8ca9abc..0000000 --- a/add-bugzilla-versions/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -beautifulsoup4==4.6.0 -click==6.7 -GitPython==2.1.14 -requests==2.18.2 diff --git a/add_bugzilla_versions.sh b/add_bugzilla_versions.sh new file mode 100755 index 0000000..8dbcb45 --- /dev/null +++ b/add_bugzilla_versions.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +"$(dirname "$(readlink -f $0)")/tools/python-kde-release/run.sh" add-bugzilla-versions $* diff --git a/add-bugzilla-versions/.gitignore b/tools/python-kde-release/.gitignore similarity index 100% rename from add-bugzilla-versions/.gitignore rename to tools/python-kde-release/.gitignore diff --git a/tools/python-kde-release/kde_release/__init__.py b/tools/python-kde-release/kde_release/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/add-bugzilla-versions/add-versions.py b/tools/python-kde-release/kde_release/cli.py old mode 100755 new mode 100644 similarity index 83% rename from add-bugzilla-versions/add-versions.py rename to tools/python-kde-release/kde_release/cli.py index 1f07201..67b9bf3 --- a/add-bugzilla-versions/add-versions.py +++ b/tools/python-kde-release/kde_release/cli.py @@ -1,213 +1,244 @@ -#!/usr/bin/env python3 - -# Copyright 2015 Jonathan Riddell -# Copyright 2017 Adrian Chaves +# Copyright 2015 Jonathan Riddell +# Copyright 2017, 2019 Adrian Chaves # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import sys -sys.path.append("..") - import os -from pathlib import Path import re +from os import environ +from pathlib import Path from shutil import rmtree from subprocess import run, DEVNULL, PIPE from tempfile import mkdtemp from bs4 import BeautifulSoup from click import command, echo, option, prompt from git import GitCommandError, Repo from git.exc import InvalidGitRepositoryError, NoSuchPathError from requests import RequestException, Session -import add_versions_lib - URL = 'https://bugs.kde.org' EDIT_VERSION_URL = '{}/editversions.cgi'.format(URL) +VERSION_RE = re.compile(r'(?i):\s+project\s*\(\s*\S+\s+' + r'VERSION\s+(\d+(?:\.\d+(?:\.\d+(?:\.\d+)?)?)?)') def response_error(response): html = response.text soup = BeautifulSoup(html, 'html.parser') error_message = soup.find('div', {'id': 'error_msg'}) if not error_message: return None return re.sub(r'\s+', ' ', error_message.get_text()).strip() def log_in(session, email, password): data = {'Bugzilla_login': email, 'Bugzilla_password': password} headers = {'Referer': 'https://bugs.kde.org/'} try: response = session.post(URL, data=data, headers=headers) error = response_error(response) if error: echo(error) exit(1) except RequestException: echo('Unexpected login error. Please, contact the maintainers of this ' 'script.') exit(1) +def products_and_branches(): + modules_path = Path(environ['KDE_RELEASE_DIR']) / 'modules.git' + with modules_path.open() as modules_file: + for line in modules_file: + match = re.match(r'^(\S+)\s+(\S+)$', line) + if match: + yield match.group(1), match.group(2) + + +def cmake_trace(folder): + temp_folder = mkdtemp() + try: + cmake_command = ['cmake', '--trace-expand', folder] + execution = run( + cmake_command, cwd=temp_folder, stdout=DEVNULL, stderr=PIPE) + return execution.stderr.decode(errors='ignore') + finally: + rmtree(temp_folder) + + +def project_version(source_folder): + match = None + for match in VERSION_RE.finditer(cmake_trace(source_folder)): + pass + if not match: + return None + return match.group(1) + + def bugzilla_csrf_token(session, product): params = {'action': 'add', 'product': product} response = session.get(EDIT_VERSION_URL, params=params) soup = BeautifulSoup(response.text, 'html.parser') try: return soup.find('input', {'name': 'token'})['value'] except: raise RuntimeError('Could not parse token from \'{}\''.format( response.url)) class AddVersionError(RuntimeError): def __init__(self, cause=None, *args, **kwargs): super().__init__(*args, **kwargs) self.cause = cause def add_version_to_bugzilla_project(session, product, version): params = {'version': version, 'action': 'new', 'product': product, 'token': bugzilla_csrf_token(session, product)} try: response = session.get(EDIT_VERSION_URL, params=params) response.raise_for_status() except RequestException: raise AddVersionError() error = response_error(response) if error: raise AddVersionError(error) @command() @option('-s', '--srcdir', prompt='Source folder', help='Folder containing local clones of the Git repositories from ' - '../modules.git') + 'modules.git') @option('-e', '--email', help='Email of your Bugzilla account') @option('-p', '--password', help='Password of your Bugzilla account') @option('-v', '--verbose', is_flag=True, help='Provide additional details about errors') @option('-d', '--dry', is_flag=True, help='Do not submit anything to Bugzilla. Note: Local Git clones may ' 'be modified.') @option('-c', '--clone', is_flag=True, help='Clone missing Git folders.') @option('--hide-skipped', is_flag=True, help='Do not print lines for skipped products') -def main(srcdir, email, password, verbose, dry, clone, hide_skipped): +def add_bugzilla_versions(srcdir, email, password, verbose, dry, clone, + hide_skipped): + """Adds new project versions to the KDE Bugtracking System. + + Speficically, for each project listed in modules.git: + + 1. CMake is used to determine the project version, as defined by CMake’s + project command. If the version is not found, the project is skipped. + + 2. A version is added to the target project in the KDE Bugtracking System. + """ if not dry: if email is None: email = prompt('Email') if password is None: password = prompt('Password', hide_input=True) srcdir = os.path.abspath(srcdir.rstrip('/')) session = Session() if not dry: log_in(session, email, password) for product, branch in products_and_branches(): source_folder = srcdir + '/' + product try: repository = Repo(source_folder) except NoSuchPathError: if clone: try: git_url = 'git://anongit.kde.org/{}'.format(product) repository = Repo.clone_from(git_url, source_folder) except KeyboardInterrupt: if os.path.exists(source_folder): rmtree(source_folder, ignore_errors=True) exit(0) except: raise echo('{} (error: could not clone repository)'.format(product)) if verbose: echo('\tGit URL: {}'.format(git_url)) echo('\tTarget folder: {}'.format(source_folder)) exit(1) else: echo('{} (error: source folder not found)'.format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) exit(1) except InvalidGitRepositoryError: echo('{} (error: source folder is not a Git clone)'.format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) exit(1) try: repository.heads[branch].checkout() except GitCommandError: echo('{} (error: cannot checkout branch)'.format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) echo('\tCurrent branch: {}'.format(repository.head.name)) echo('\tBranch to check out: {}'.format(branch)) exit(1) except IndexError: echo('{} (error: cannot checkout branch: branch does not exist)' .format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) echo('\tCurrent branch: {}'.format(repository.head.name)) echo('\tBranch to check out: {}'.format(branch)) exit(1) try: repository.remote().pull("--rebase") except GitCommandError: echo('{} (error: cannot pull branch)'.format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) echo('\tBranch: {}'.format(branch)) exit(1) try: version = project_version(source_folder) if version is None: if hide_skipped: continue echo('{} \n (no project version found)'.format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) echo('\t{}/CMakeLists.txt does not define a project version' .format(source_folder)) echo('\tSee https://community.kde.org/' 'Guidelines_and_HOWTOs/' 'Application_Versioning#Bugzilla_versions') continue if not dry: try: add_version_to_bugzilla_project(session, product, version) echo('{}\n (added {})'.format( product, version)) except AddVersionError as error: if error.cause: echo('{}\n (would have added {})\n (error: {})'.format(product, version, error.cause)) else: echo('{}\n (would have added {})\n (error: {})'.format(product, version, error)) except Exception as e: echo('{}\n (would have added {})\n error: {})'.format(product, version, e)) else: echo('{}\n (would have added {})'.format( product, version)) except RuntimeError as error: echo('ERROR: {}'.format(error)) exit(1) - - -if __name__ == '__main__': - main() diff --git a/tools/python-kde-release/run.sh b/tools/python-kde-release/run.sh new file mode 100755 index 0000000..6b2fdd5 --- /dev/null +++ b/tools/python-kde-release/run.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +function finish { + deactivate &> /dev/null +} +trap finish EXIT + +deactivate &> /dev/null +dir="$(dirname "$(readlink -f $0)")" +[ ! -d "$DIR" ] && python3 -m venv "$dir/venv" +. "$dir/venv/bin/activate" +pip install --upgrade "$dir" &> /dev/null + +KDE_RELEASE_DIR="$(dirname "$(dirname "$dir")")" "$1" "${@:2}" diff --git a/tools/python-kde-release/setup.py b/tools/python-kde-release/setup.py new file mode 100644 index 0000000..16d252f --- /dev/null +++ b/tools/python-kde-release/setup.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +from setuptools import find_packages, setup + + +setup( + name='kde-release', + version='1.0', + description='Utilities of the KDE Release team', + packages=find_packages(), + python_requires='>=3.6', + install_requires=[ + 'beautifulsoup4 >=4.6.0, <5', + 'click >=6.7, <7', + 'GitPython >=2.1.14, <3', + 'requests >=2.18.2, <3', + ], + entry_points={ + 'console_scripts': [ + 'add-bugzilla-versions = kde_release.cli:add_bugzilla_versions', + ], + }, +)