diff --git a/add-appstream-versions/add-versions.py b/add-appstream-versions/add-versions.py index 5784b0c..853a807 100755 --- a/add-appstream-versions/add-versions.py +++ b/add-appstream-versions/add-versions.py @@ -1,205 +1,170 @@ #!/usr/bin/env python3 # Copyright 2015 Jonathan Riddell # Copyright 2017 Adrian Chaves # Copyright 2019 Jonathan Riddell # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import sys +sys.path.append("..") + import os from pathlib import Path -import re from shutil import rmtree from subprocess import run, DEVNULL, PIPE from tempfile import mkdtemp import glob +import add_versions_lib from bs4 import BeautifulSoup from click import command, echo, option, prompt from git import GitCommandError, Repo from git.exc import InvalidGitRepositoryError, NoSuchPathError APPSTREAM_UPDATER="/home/jr/src/appstream-metainfo-release-update/appstream-metainfo-release-update/appstream-metainfo-release-update" RELEASEDATE="Tuesday, 9 July 2019" -VERSION_RE = re.compile(r'(?i):\s+project\s*\(\s*\S+\s+' - r'VERSION\s+(\d+(?:\.\d+(?:\.\d+(?:\.\d+)?)?)?)') - - -def products_and_branches(): - modules_path = Path(__file__).resolve().parent.parent / 'modules.git' - with modules_path.open() as modules_file: - for line in modules_file: - match = re.match(r'^(\S+)\s+(\S+)$', line) - if match: - yield match.group(1), match.group(2) - - -def cmake_trace(folder): - temp_folder = mkdtemp() - try: - cmake_command = ['cmake', '--trace-expand', folder] - execution = run( - cmake_command, cwd=temp_folder, stdout=DEVNULL, stderr=PIPE) - return execution.stderr.decode(errors='ignore') - finally: - rmtree(temp_folder) - - -def project_version(source_folder): - match = None - for match in VERSION_RE.finditer(cmake_trace(source_folder)): - pass - if not match: - return None - return match.group(1) - - -class AddVersionError(RuntimeError): - - def __init__(self, cause=None, *args, **kwargs): - super().__init__(*args, **kwargs) - self.cause = cause def add_version_to_appstream(source_folder, version): try: appstream_files = glob.glob(source_folder + '/**/*appdata.xml', recursive=True) appstream_files += glob.glob(source_folder + '/**/*metainfo.xml', recursive=True) for appstream_file in appstream_files: run([APPSTREAM_UPDATER, "--version", version, "--datestring", RELEASEDATE, "--releases-to-show" , "4", appstream_file]) if len(appstream_files) > 0: os.chdir(source_folder) run(['git', 'commit', '-a', '-m', 'Update Appstream for new release']) run(['git', 'push']) except: raise AddVersionError() @command() @option('-s', '--srcdir', prompt='Source folder', help='Folder containing local clones of the Git repositories from ' '../modules.git') @option('-v', '--verbose', is_flag=True, help='Provide additional details about errors') @option('-d', '--dry', is_flag=True, help='Do not submit anything to Bugzilla. Note: Local Git clones may ' 'be modified.') @option('-c', '--clone', is_flag=True, help='Clone missing Git folders.') @option('--hide-skipped', is_flag=True, help='Do not print lines for skipped products') def main(srcdir, verbose, dry, clone, hide_skipped): version_file = open("../version", "r") version = version_file.readline().rstrip() bugfix_version = int(version.split('.')[2]) if bugfix_version > 50: echo('Version number indicates this is not a testing release so not adding versions to Appstream files') exit(1) srcdir = os.path.abspath(srcdir.rstrip('/')) for product, branch in products_and_branches(): source_folder = srcdir + '/' + product try: repository = Repo(source_folder) except NoSuchPathError: if clone: try: git_url = 'git://anongit.kde.org/{}'.format(product) repository = Repo.clone_from(git_url, source_folder) except KeyboardInterrupt: if os.path.exists(source_folder): rmtree(source_folder, ignore_errors=True) exit(0) except: raise echo('{} (error: could not clone repository)'.format(product)) if verbose: echo('\tGit URL: {}'.format(git_url)) echo('\tTarget folder: {}'.format(source_folder)) exit(1) else: echo('{} (error: source folder not found)'.format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) exit(1) except InvalidGitRepositoryError: echo('{} (error: source folder is not a Git clone)'.format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) exit(1) if not os.path.isfile(APPSTREAM_UPDATER): echo('Can not find appstream-updater at ' + APPSTREAM_UPDATER) echo('Get it from git@invent.kde.org:jriddell/appstream-metainfo-release-update.git') exit(1) try: repository.heads[branch].checkout() except GitCommandError: echo('{} (error: cannot checkout branch)'.format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) echo('\tCurrent branch: {}'.format(repository.head.name)) echo('\tBranch to check out: {}'.format(branch)) exit(1) except IndexError: echo('{} (error: cannot checkout branch: branch does not exist)' .format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) echo('\tCurrent branch: {}'.format(repository.head.name)) echo('\tBranch to check out: {}'.format(branch)) exit(1) try: repository.remote().pull("--rebase") except GitCommandError: echo('{} (error: cannot pull branch)'.format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) echo('\tBranch: {}'.format(branch)) exit(1) try: version = project_version(source_folder) if version is None: if hide_skipped: continue echo('{} \n (no project version found)'.format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) echo('\t{}/CMakeLists.txt does not define a project version' .format(source_folder)) echo('\tSee https://community.kde.org/' 'Guidelines_and_HOWTOs/' 'Application_Versioning#Bugzilla_versions') continue if not dry: try: add_version_to_appstream(source_folder, version) echo('{}\n (added {})'.format( product, version)) except AddVersionError as error: if error.cause: echo('{}\n (would have added {})\n (error: {})'.format(product, version, error.cause)) else: echo('{}\n (would have added {})\n (error: {})'.format(product, version, error)) except Exception as e: echo('{}\n (would have added {})\n error: {})'.format(product, version, e)) else: echo('{}\n (would have added {})'.format( product, version)) except RuntimeError as error: echo('ERROR: {}'.format(error)) exit(1) if __name__ == '__main__': main() diff --git a/add-bugzilla-versions/add-versions.py b/add-bugzilla-versions/add-versions.py index 5b40889..1f07201 100755 --- a/add-bugzilla-versions/add-versions.py +++ b/add-bugzilla-versions/add-versions.py @@ -1,239 +1,213 @@ #!/usr/bin/env python3 # Copyright 2015 Jonathan Riddell # Copyright 2017 Adrian Chaves # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import sys +sys.path.append("..") + import os from pathlib import Path import re from shutil import rmtree from subprocess import run, DEVNULL, PIPE from tempfile import mkdtemp from bs4 import BeautifulSoup from click import command, echo, option, prompt from git import GitCommandError, Repo from git.exc import InvalidGitRepositoryError, NoSuchPathError from requests import RequestException, Session +import add_versions_lib + URL = 'https://bugs.kde.org' EDIT_VERSION_URL = '{}/editversions.cgi'.format(URL) -VERSION_RE = re.compile(r'(?i):\s+project\s*\(\s*\S+\s+' - r'VERSION\s+(\d+(?:\.\d+(?:\.\d+(?:\.\d+)?)?)?)') def response_error(response): html = response.text soup = BeautifulSoup(html, 'html.parser') error_message = soup.find('div', {'id': 'error_msg'}) if not error_message: return None return re.sub(r'\s+', ' ', error_message.get_text()).strip() def log_in(session, email, password): data = {'Bugzilla_login': email, 'Bugzilla_password': password} headers = {'Referer': 'https://bugs.kde.org/'} try: response = session.post(URL, data=data, headers=headers) error = response_error(response) if error: echo(error) exit(1) except RequestException: echo('Unexpected login error. Please, contact the maintainers of this ' 'script.') exit(1) -def products_and_branches(): - modules_path = Path(__file__).resolve().parent.parent / 'modules.git' - with modules_path.open() as modules_file: - for line in modules_file: - match = re.match(r'^(\S+)\s+(\S+)$', line) - if match: - yield match.group(1), match.group(2) - - -def cmake_trace(folder): - temp_folder = mkdtemp() - try: - cmake_command = ['cmake', '--trace-expand', folder] - execution = run( - cmake_command, cwd=temp_folder, stdout=DEVNULL, stderr=PIPE) - return execution.stderr.decode(errors='ignore') - finally: - rmtree(temp_folder) - - -def project_version(source_folder): - match = None - for match in VERSION_RE.finditer(cmake_trace(source_folder)): - pass - if not match: - return None - return match.group(1) - - def bugzilla_csrf_token(session, product): params = {'action': 'add', 'product': product} response = session.get(EDIT_VERSION_URL, params=params) soup = BeautifulSoup(response.text, 'html.parser') try: return soup.find('input', {'name': 'token'})['value'] except: raise RuntimeError('Could not parse token from \'{}\''.format( response.url)) class AddVersionError(RuntimeError): def __init__(self, cause=None, *args, **kwargs): super().__init__(*args, **kwargs) self.cause = cause def add_version_to_bugzilla_project(session, product, version): params = {'version': version, 'action': 'new', 'product': product, 'token': bugzilla_csrf_token(session, product)} try: response = session.get(EDIT_VERSION_URL, params=params) response.raise_for_status() except RequestException: raise AddVersionError() error = response_error(response) if error: raise AddVersionError(error) @command() @option('-s', '--srcdir', prompt='Source folder', help='Folder containing local clones of the Git repositories from ' '../modules.git') @option('-e', '--email', help='Email of your Bugzilla account') @option('-p', '--password', help='Password of your Bugzilla account') @option('-v', '--verbose', is_flag=True, help='Provide additional details about errors') @option('-d', '--dry', is_flag=True, help='Do not submit anything to Bugzilla. Note: Local Git clones may ' 'be modified.') @option('-c', '--clone', is_flag=True, help='Clone missing Git folders.') @option('--hide-skipped', is_flag=True, help='Do not print lines for skipped products') def main(srcdir, email, password, verbose, dry, clone, hide_skipped): if not dry: if email is None: email = prompt('Email') if password is None: password = prompt('Password', hide_input=True) srcdir = os.path.abspath(srcdir.rstrip('/')) session = Session() if not dry: log_in(session, email, password) for product, branch in products_and_branches(): source_folder = srcdir + '/' + product try: repository = Repo(source_folder) except NoSuchPathError: if clone: try: git_url = 'git://anongit.kde.org/{}'.format(product) repository = Repo.clone_from(git_url, source_folder) except KeyboardInterrupt: if os.path.exists(source_folder): rmtree(source_folder, ignore_errors=True) exit(0) except: raise echo('{} (error: could not clone repository)'.format(product)) if verbose: echo('\tGit URL: {}'.format(git_url)) echo('\tTarget folder: {}'.format(source_folder)) exit(1) else: echo('{} (error: source folder not found)'.format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) exit(1) except InvalidGitRepositoryError: echo('{} (error: source folder is not a Git clone)'.format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) exit(1) try: repository.heads[branch].checkout() except GitCommandError: echo('{} (error: cannot checkout branch)'.format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) echo('\tCurrent branch: {}'.format(repository.head.name)) echo('\tBranch to check out: {}'.format(branch)) exit(1) except IndexError: echo('{} (error: cannot checkout branch: branch does not exist)' .format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) echo('\tCurrent branch: {}'.format(repository.head.name)) echo('\tBranch to check out: {}'.format(branch)) exit(1) try: repository.remote().pull("--rebase") except GitCommandError: echo('{} (error: cannot pull branch)'.format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) echo('\tBranch: {}'.format(branch)) exit(1) try: version = project_version(source_folder) if version is None: if hide_skipped: continue echo('{} \n (no project version found)'.format(product)) if verbose: echo('\tSource folder: {}'.format(source_folder)) echo('\t{}/CMakeLists.txt does not define a project version' .format(source_folder)) echo('\tSee https://community.kde.org/' 'Guidelines_and_HOWTOs/' 'Application_Versioning#Bugzilla_versions') continue if not dry: try: add_version_to_bugzilla_project(session, product, version) echo('{}\n (added {})'.format( product, version)) except AddVersionError as error: if error.cause: echo('{}\n (would have added {})\n (error: {})'.format(product, version, error.cause)) else: echo('{}\n (would have added {})\n (error: {})'.format(product, version, error)) except Exception as e: echo('{}\n (would have added {})\n error: {})'.format(product, version, e)) else: echo('{}\n (would have added {})'.format( product, version)) except RuntimeError as error: echo('ERROR: {}'.format(error)) exit(1) if __name__ == '__main__': main() diff --git a/add_versions_lib.py b/add_versions_lib.py new file mode 100644 index 0000000..65e26a9 --- /dev/null +++ b/add_versions_lib.py @@ -0,0 +1,57 @@ +# Copyright 2015 Jonathan Riddell +# Copyright 2017 Adrian Chaves +# Copyright 2019 Jonathan Riddell +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import re + +VERSION_RE = re.compile(r'(?i):\s+project\s*\(\s*\S+\s+' + r'VERSION\s+(\d+(?:\.\d+(?:\.\d+(?:\.\d+)?)?)?)') + + +class AddVersionError(RuntimeError): + + def __init__(self, cause=None, *args, **kwargs): + super().__init__(*args, **kwargs) + self.cause = cause + + +def products_and_branches(): + modules_path = Path(__file__).resolve().parent.parent / 'modules.git' + with modules_path.open() as modules_file: + for line in modules_file: + match = re.match(r'^(\S+)\s+(\S+)$', line) + if match: + yield match.group(1), match.group(2) + + +def cmake_trace(folder): + temp_folder = mkdtemp() + try: + cmake_command = ['cmake', '--trace-expand', folder] + execution = run( + cmake_command, cwd=temp_folder, stdout=DEVNULL, stderr=PIPE) + return execution.stderr.decode(errors='ignore') + finally: + rmtree(temp_folder) + + +def project_version(source_folder): + match = None + for match in VERSION_RE.finditer(cmake_trace(source_folder)): + pass + if not match: + return None + return match.group(1)