diff --git a/README b/README --- a/README +++ b/README @@ -1,6 +1,12 @@ What does each file in this folder do? ====================================== + * add-appstream-versions/add-versions.py + Run this first to add the new release version to the appstream files in your repos + + * add-bugzilla-versions/add-versions.py + Run this to add the new release version to the bugs.kde.org versions + * config Holds some general configuration diff --git a/add-appstream-versions/.gitignore b/add-appstream-versions/.gitignore new file mode 100644 --- /dev/null +++ b/add-appstream-versions/.gitignore @@ -0,0 +1 @@ +/venv/ diff --git a/add-appstream-versions/README.rst b/add-appstream-versions/README.rst new file mode 100644 --- /dev/null +++ b/add-appstream-versions/README.rst @@ -0,0 +1,30 @@ +Adds new project versions to the appstream metainfo files + +The script does the following: + +#. For each project listed in ``../modules.git`` + + #. Use CMake to determine the project version, as defined by CMake’s project_ + command. + + .. _project: https://cmake.org/cmake/help/latest/command/project.html#command:project + + If the variable is not found, the project is skipped. + + #. Add the version to the Appstream metainfo file for the target project. + +To run the script: + +#. Create, enter and prepare a Python virtual environment:: + + python3 -m venv venv + . venv/bin/activate + pip install -r requirements.txt + +#. Run the script:: + + python3 add-versions.py \ + -s + + .. note:: If you skip a parameter, the script prompts you for it on the + command line. diff --git a/add-bugzilla-versions/add-versions.py b/add-appstream-versions/add-versions.py copy from add-bugzilla-versions/add-versions.py copy to add-appstream-versions/add-versions.py --- a/add-bugzilla-versions/add-versions.py +++ b/add-appstream-versions/add-versions.py @@ -2,6 +2,7 @@ # Copyright 2015 Jonathan Riddell # Copyright 2017 Adrian Chaves +# Copyright 2019 Jonathan Riddell # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -16,117 +17,50 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import sys +sys.path.append("..") + import os -from pathlib import Path -import re -from shutil import rmtree from subprocess import run, DEVNULL, PIPE -from tempfile import mkdtemp +import glob +import add_versions_lib from bs4 import BeautifulSoup from click import command, echo, option, prompt from git import GitCommandError, Repo from git.exc import InvalidGitRepositoryError, NoSuchPathError -from requests import RequestException, Session - - -URL = 'https://bugs.kde.org' -EDIT_VERSION_URL = '{}/editversions.cgi'.format(URL) -VERSION_RE = re.compile(r'(?i):\s+project\s*\(\s*\S+\s+' - r'VERSION\s+(\d+(?:\.\d+(?:\.\d+(?:\.\d+)?)?)?)') - - -def response_error(response): - html = response.text - soup = BeautifulSoup(html, 'html.parser') - error_message = soup.find('div', {'id': 'error_msg'}) - if not error_message: - return None - return re.sub(r'\s+', ' ', error_message.get_text()).strip() - - -def log_in(session, email, password): - data = {'Bugzilla_login': email, 'Bugzilla_password': password} - headers = {'Referer': 'https://bugs.kde.org/'} - try: - response = session.post(URL, data=data, headers=headers) - error = response_error(response) - if error: - echo(error) - exit(1) - except RequestException: - echo('Unexpected login error. Please, contact the maintainers of this ' - 'script.') - exit(1) - - -def products_and_branches(): - modules_path = Path(__file__).resolve().parent.parent / 'modules.git' - with modules_path.open() as modules_file: - for line in modules_file: - match = re.match(r'^(\S+)\s+(\S+)$', line) - if match: - yield match.group(1), match.group(2) - +from subprocess import run, DEVNULL, PIPE -def cmake_trace(folder): - temp_folder = mkdtemp() - try: - cmake_command = ['cmake', '--trace-expand', folder] - execution = run( - cmake_command, cwd=temp_folder, stdout=DEVNULL, stderr=PIPE) - return execution.stderr.decode(errors='ignore') - finally: - rmtree(temp_folder) +import configparser +CONFIG_PATH='../config' +with open(CONFIG_PATH, 'r') as f: + config_string = '[dummy_section]\n' + f.read() +config = configparser.ConfigParser() +config.read_string(config_string) -def project_version(source_folder): - match = None - for match in VERSION_RE.finditer(cmake_trace(source_folder)): - pass - if not match: - return None - return match.group(1) +APPSTREAM_UPDATER=config['dummy_section']['APPSTREAM_UPDATER'] +RELEASEDATE=config['dummy_section']['RELEASEDATE'] -def bugzilla_csrf_token(session, product): - params = {'action': 'add', 'product': product} - response = session.get(EDIT_VERSION_URL, params=params) - soup = BeautifulSoup(response.text, 'html.parser') +def add_version_to_appstream(source_folder, version): try: - return soup.find('input', {'name': 'token'})['value'] + appstream_files = glob.glob(source_folder + '/**/*appdata.xml', recursive=True) + appstream_files += glob.glob(source_folder + '/**/*metainfo.xml', recursive=True) + for appstream_file in appstream_files: + run([APPSTREAM_UPDATER, "--version", version, "--datestring", RELEASEDATE, "--releases-to-show" , "4", appstream_file]) + if len(appstream_files) > 0: + os.chdir(source_folder) + run(['git', 'commit', '-a', '-m', 'Update Appstream for new release']) + run(['git', 'push']) except: - raise RuntimeError('Could not parse token from \'{}\''.format( - response.url)) - - -class AddVersionError(RuntimeError): - - def __init__(self, cause=None, *args, **kwargs): - super().__init__(*args, **kwargs) - self.cause = cause - - -def add_version_to_bugzilla_project(session, product, version): - params = {'version': version, 'action': 'new', 'product': product, - 'token': bugzilla_csrf_token(session, product)} - try: - response = session.get(EDIT_VERSION_URL, params=params) - response.raise_for_status() - except RequestException: raise AddVersionError() - error = response_error(response) - if error: - raise AddVersionError(error) @command() @option('-s', '--srcdir', prompt='Source folder', help='Folder containing local clones of the Git repositories from ' '../modules.git') -@option('-e', '--email', help='Email of your Bugzilla account') -@option('-p', '--password', - help='Password of your Bugzilla account') @option('-v', '--verbose', is_flag=True, help='Provide additional details about errors') @option('-d', '--dry', is_flag=True, @@ -136,17 +70,16 @@ help='Clone missing Git folders.') @option('--hide-skipped', is_flag=True, help='Do not print lines for skipped products') -def main(srcdir, email, password, verbose, dry, clone, hide_skipped): - if not dry: - if email is None: - email = prompt('Email') - if password is None: - password = prompt('Password', hide_input=True) +def main(srcdir, verbose, dry, clone, hide_skipped): + version_file = open("../version", "r") + version = version_file.readline().rstrip() + bugfix_version = int(version.split('.')[2]) + if bugfix_version > 50: + echo('Version number indicates this is a testing release so not adding versions to Appstream files') + exit(1) + srcdir = os.path.abspath(srcdir.rstrip('/')) - session = Session() - if not dry: - log_in(session, email, password) - for product, branch in products_and_branches(): + for product, branch in add_versions_lib.products_and_branches(): source_folder = srcdir + '/' + product try: repository = Repo(source_folder) @@ -176,6 +109,10 @@ if verbose: echo('\tSource folder: {}'.format(source_folder)) exit(1) + if not os.path.isfile(APPSTREAM_UPDATER): + echo('Can not find appstream-metainfo-release-update at ' + APPSTREAM_UPDATER) + echo('Get it from git@invent.kde.org:jriddell/appstream-metainfo-release-update.git') + exit(1) try: repository.heads[branch].checkout() except GitCommandError: @@ -202,7 +139,7 @@ echo('\tBranch: {}'.format(branch)) exit(1) try: - version = project_version(source_folder) + version = add_versions_lib.project_version(source_folder) if version is None: if hide_skipped: continue @@ -217,7 +154,7 @@ continue if not dry: try: - add_version_to_bugzilla_project(session, product, version) + add_version_to_appstream(source_folder, version) echo('{}\n (added {})'.format( product, version)) except AddVersionError as error: diff --git a/add-appstream-versions/requirements.txt b/add-appstream-versions/requirements.txt new file mode 100644 --- /dev/null +++ b/add-appstream-versions/requirements.txt @@ -0,0 +1,4 @@ +beautifulsoup4==4.6.0 +click==6.7 +GitPython==2.1.5 +requests==2.18.2 diff --git a/add-bugzilla-versions/add-versions.py b/add-bugzilla-versions/add-versions.py --- a/add-bugzilla-versions/add-versions.py +++ b/add-bugzilla-versions/add-versions.py @@ -16,6 +16,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import sys +sys.path.append("..") + import os from pathlib import Path import re @@ -29,11 +32,11 @@ from git.exc import InvalidGitRepositoryError, NoSuchPathError from requests import RequestException, Session +import add_versions_lib + URL = 'https://bugs.kde.org' EDIT_VERSION_URL = '{}/editversions.cgi'.format(URL) -VERSION_RE = re.compile(r'(?i):\s+project\s*\(\s*\S+\s+' - r'VERSION\s+(\d+(?:\.\d+(?:\.\d+(?:\.\d+)?)?)?)') def response_error(response): @@ -60,35 +63,6 @@ exit(1) -def products_and_branches(): - modules_path = Path(__file__).resolve().parent.parent / 'modules.git' - with modules_path.open() as modules_file: - for line in modules_file: - match = re.match(r'^(\S+)\s+(\S+)$', line) - if match: - yield match.group(1), match.group(2) - - -def cmake_trace(folder): - temp_folder = mkdtemp() - try: - cmake_command = ['cmake', '--trace-expand', folder] - execution = run( - cmake_command, cwd=temp_folder, stdout=DEVNULL, stderr=PIPE) - return execution.stderr.decode(errors='ignore') - finally: - rmtree(temp_folder) - - -def project_version(source_folder): - match = None - for match in VERSION_RE.finditer(cmake_trace(source_folder)): - pass - if not match: - return None - return match.group(1) - - def bugzilla_csrf_token(session, product): params = {'action': 'add', 'product': product} response = session.get(EDIT_VERSION_URL, params=params) diff --git a/add_versions_lib.py b/add_versions_lib.py new file mode 100644 --- /dev/null +++ b/add_versions_lib.py @@ -0,0 +1,61 @@ +# Copyright 2015 Jonathan Riddell +# Copyright 2017 Adrian Chaves +# Copyright 2019 Jonathan Riddell +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import re +from pathlib import Path +from tempfile import mkdtemp +from shutil import rmtree +from subprocess import run, DEVNULL, PIPE + +VERSION_RE = re.compile(r'(?i):\s+project\s*\(\s*\S+\s+' + r'VERSION\s+(\d+(?:\.\d+(?:\.\d+(?:\.\d+)?)?)?)') + + +class AddVersionError(RuntimeError): + + def __init__(self, cause=None, *args, **kwargs): + super().__init__(*args, **kwargs) + self.cause = cause + + +def products_and_branches(): + modules_path = Path(__file__).resolve().parent / 'modules.git' + with modules_path.open() as modules_file: + for line in modules_file: + match = re.match(r'^(\S+)\s+(\S+)$', line) + if match: + yield match.group(1), match.group(2) + + +def cmake_trace(folder): + temp_folder = mkdtemp() + try: + cmake_command = ['cmake', '--trace-expand', folder] + execution = run( + cmake_command, cwd=temp_folder, stdout=DEVNULL, stderr=PIPE) + return execution.stderr.decode(errors='ignore') + finally: + rmtree(temp_folder) + + +def project_version(source_folder): + match = None + for match in VERSION_RE.finditer(cmake_trace(source_folder)): + pass + if not match: + return None + return match.group(1) diff --git a/config b/config --- a/config +++ b/config @@ -10,3 +10,9 @@ # name of the subdir of $l10n_repo/$lang/messages # only used if release_l10n_separately=0 l10n_module=dummy + +# link to script from git@invent.kde.org:jriddell/appstream-metainfo-release-update.git +APPSTREAM_UPDATER=/home/jr/src/appstream-metainfo-release-update/appstream-metainfo-release-update/appstream-metainfo-release-update + +# date to add into appstream files for next release, use `date --date='next tuesday' '+%A, %e %B %Y'` +RELEASEDATE=Tuesday, 9 July 2019