Differential D22489 Diff 65477 add-appstream-versions/add-versions.py

Changeset View

Standalone View

View Options

add-appstream-versions/add-versions.py

  • This file was copied from add-bugzilla-versions/add-versions.py.
1#!/usr/bin/env python3 1​#!/usr/bin/env python3
2 2​
3# Copyright 2015 Jonathan Riddell <jr@jriddell.org> 3​# Copyright 2015 Jonathan Riddell <jr@jriddell.org>
4# Copyright 2017 Adrian Chaves <adrian@chaves.io> 4​# Copyright 2017 Adrian Chaves <adrian@chaves.io>
5​# Copyright 2019 Jonathan Riddell <jr@jriddell.org>
5# 6​#
6# This program is free software; you can redistribute it and/or 7​# This program is free software; you can redistribute it and/or
7# modify it under the terms of the GNU General Public License as 8​# modify it under the terms of the GNU General Public License as
8# published by the Free Software Foundation; either version 2 of 9​# published by the Free Software Foundation; either version 2 of
9# the License, or (at your option) any later version. 10​# the License, or (at your option) any later version.
10# 11​#
11# This program is distributed in the hope that it will be useful, 12​# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of 13​# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14​# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU General Public License for more details. 15​# GNU General Public License for more details.
15# 16​#
16# You should have received a copy of the GNU General Public License 17​# You should have received a copy of the GNU General Public License
17# along with this program. If not, see <http://www.gnu.org/licenses/>. 18​# along with this program. If not, see <http://www.gnu.org/licenses/>.
18 19​
20​import sys
21​sys.path.append("..")
22​
19import os 23​import os
20from pathlib import Path ​
21import re ​
22from shutil import rmtree ​
23from subprocess import run, DEVNULL, PIPE 24​from subprocess import run, DEVNULL, PIPE
24from tempfile import mkdtemp 25​import glob
26​import add_versions_lib
25 27​
26from bs4 import BeautifulSoup 28​from bs4 import BeautifulSoup
27from click import command, echo, option, prompt 29​from click import command, echo, option, prompt
28from git import GitCommandError, Repo 30​from git import GitCommandError, Repo
29from git.exc import InvalidGitRepositoryError, NoSuchPathError 31​from git.exc import InvalidGitRepositoryError, NoSuchPathError
30from requests import RequestException, Session 32​from subprocess import run, DEVNULL, PIPE
31 ​
32 ​
33URL = 'https://bugs.kde.org' ​
34EDIT_VERSION_URL = '{}/editversions.cgi'.format(URL) ​
35VERSION_RE = re.compile(r'(?i):\s+project\s*\(\s*\S+\s+' ​
36 r'VERSION\s+(\d+(?:\.\d+(?:\.\d+(?:\.\d+)?)?)?)') ​
37 ​
38 ​
39def response_error(response): ​
40 html = response.text ​
41 soup = BeautifulSoup(html, 'html.parser') ​
42 error_message = soup.find('div', {'id': 'error_msg'}) ​
43 if not error_message: ​
44 return None ​
45 return re.sub(r'\s+', ' ', error_message.get_text()).strip() ​
46 ​
47 ​
48def log_in(session, email, password): ​
49 data = {'Bugzilla_login': email, 'Bugzilla_password': password} ​
50 headers = {'Referer': 'https://bugs.kde.org/'} ​
51 try: ​
52 response = session.post(URL, data=data, headers=headers) ​
53 error = response_error(response) ​
54 if error: ​
55 echo(error) ​
56 exit(1) ​
57 except RequestException: ​
58 echo('Unexpected login error. Please, contact the maintainers of this ' ​
59 'script.') ​
60 exit(1) ​
61 ​
62 ​
63def products_and_branches(): ​
64 modules_path = Path(__file__).resolve().parent.parent / 'modules.git' ​
65 with modules_path.open() as modules_file: ​
66 for line in modules_file: ​
67 match = re.match(r'^(\S+)\s+(\S+)$', line) ​
68 if match: ​
69 yield match.group(1), match.group(2) ​
70 ​
71 33​
72def cmake_trace(folder): 34​import configparser
adrianchavesfernandezUnsubmitted
Done

Why are these values hardcoded here?

adrianchavesfernandez: Why are these values hardcoded here?
73 temp_folder = mkdtemp() ​
74 try: ​
75 cmake_command = ['cmake', '--trace-expand', folder] ​
76 execution = run( ​
77 cmake_command, cwd=temp_folder, stdout=DEVNULL, stderr=PIPE) ​
78 return execution.stderr.decode(errors='ignore') ​
79 finally: ​
80 rmtree(temp_folder) ​
81 35​
36​CONFIG_PATH='../config'
37​with open(CONFIG_PATH, 'r') as f:
38​ config_string = '[dummy_section]\n' + f.read()
39​config = configparser.ConfigParser()
40​config.read_string(config_string)
82 41​
83def project_version(source_folder): 42​APPSTREAM_UPDATER=config['dummy_section']['APPSTREAM_UPDATER']
84 match = None 43​RELEASEDATE=config['dummy_section']['RELEASEDATE']
85 for match in VERSION_RE.finditer(cmake_trace(source_folder)): ​
86 pass ​
87 if not match: ​
88 return None ​
89 return match.group(1) ​
90 44​
91 45​
92def bugzilla_csrf_token(session, product): 46​def add_version_to_appstream(source_folder, version):
93 params = {'action': 'add', 'product': product} ​
94 response = session.get(EDIT_VERSION_URL, params=params) ​
95 soup = BeautifulSoup(response.text, 'html.parser') ​
96 try: 47​ try:
97 return soup.find('input', {'name': 'token'})['value'] 48​ appstream_files = glob.glob(source_folder + '/**/*appdata.xml', recursive=True)
49​ appstream_files += glob.glob(source_folder + '/**/*metainfo.xml', recursive=True)
50​ for appstream_file in appstream_files:
51​ run([APPSTREAM_UPDATER, "--version", version, "--datestring", RELEASEDATE, "--releases-to-show" , "4", appstream_file])
52​ if len(appstream_files) > 0:
53​ os.chdir(source_folder)
54​ run(['git', 'commit', '-a', '-m', 'Update Appstream for new release'])
55​ run(['git', 'push'])
98 except: 56​ except:
99 raise RuntimeError('Could not parse token from \'{}\''.format( ​
100 response.url)) ​
101 ​
102 ​
103class AddVersionError(RuntimeError): ​
104 ​
105 def __init__(self, cause=None, *args, **kwargs): ​
106 super().__init__(*args, **kwargs) ​
107 self.cause = cause ​
108 ​
109 ​
110def add_version_to_bugzilla_project(session, product, version): ​
111 params = {'version': version, 'action': 'new', 'product': product, ​
112 'token': bugzilla_csrf_token(session, product)} ​
113 try: ​
114 response = session.get(EDIT_VERSION_URL, params=params) ​
115 response.raise_for_status() ​
116 except RequestException: ​
117 raise AddVersionError() 57​ raise AddVersionError()
118 error = response_error(response) ​
119 if error: ​
120 raise AddVersionError(error) ​
121 58​
122 59​
123@command() 60​@command()
124@option('-s', '--srcdir', prompt='Source folder', 61​@option('-s', '--srcdir', prompt='Source folder',
125 help='Folder containing local clones of the Git repositories from ' 62​ help='Folder containing local clones of the Git repositories from '
126 '../modules.git') 63​ '../modules.git')
127@option('-e', '--email', help='Email of your Bugzilla account') ​
128@option('-p', '--password', ​
129 help='Password of your Bugzilla account') ​
130@option('-v', '--verbose', is_flag=True, 64​@option('-v', '--verbose', is_flag=True,
131 help='Provide additional details about errors') 65​ help='Provide additional details about errors')
132@option('-d', '--dry', is_flag=True, 66​@option('-d', '--dry', is_flag=True,
133 help='Do not submit anything to Bugzilla. Note: Local Git clones may ' 67​ help='Do not submit anything to Bugzilla. Note: Local Git clones may '
134 'be modified.') 68​ 'be modified.')
135@option('-c', '--clone', is_flag=True, 69​@option('-c', '--clone', is_flag=True,
136 help='Clone missing Git folders.') 70​ help='Clone missing Git folders.')
137@option('--hide-skipped', is_flag=True, 71​@option('--hide-skipped', is_flag=True,
138 help='Do not print lines for skipped products') 72​ help='Do not print lines for skipped products')
139def main(srcdir, email, password, verbose, dry, clone, hide_skipped): 73​def main(srcdir, verbose, dry, clone, hide_skipped):
140 if not dry: 74​ version_file = open("../version", "r")
141 if email is None: 75​ version = version_file.readline().rstrip()
142 email = prompt('Email') 76​ bugfix_version = int(version.split('.')[2])
143 if password is None: 77​ if bugfix_version > 50:
144 password = prompt('Password', hide_input=True) 78​ echo('Version number indicates this is a testing release so not adding versions to Appstream files')
79​ exit(1)
80​
145 srcdir = os.path.abspath(srcdir.rstrip('/')) 81​ srcdir = os.path.abspath(srcdir.rstrip('/'))
146 session = Session() 82​ for product, branch in add_versions_lib.products_and_branches():
147 if not dry: ​
148 log_in(session, email, password) ​
149 for product, branch in products_and_branches(): ​
150 source_folder = srcdir + '/' + product 83​ source_folder = srcdir + '/' + product
151 try: 84​ try:
152 repository = Repo(source_folder) 85​ repository = Repo(source_folder)
153 except NoSuchPathError: 86​ except NoSuchPathError:
154 if clone: 87​ if clone:
155 try: 88​ try:
156 git_url = 'git://anongit.kde.org/{}'.format(product) 89​ git_url = 'git://anongit.kde.org/{}'.format(product)
157 repository = Repo.clone_from(git_url, source_folder) 90​ repository = Repo.clone_from(git_url, source_folder)
Show All 13 Lines102 else:
171 if verbose: 104​ if verbose:
172 echo('\tSource folder: {}'.format(source_folder)) 105​ echo('\tSource folder: {}'.format(source_folder))
173 exit(1) 106​ exit(1)
174 except InvalidGitRepositoryError: 107​ except InvalidGitRepositoryError:
175 echo('{} (error: source folder is not a Git clone)'.format(product)) 108​ echo('{} (error: source folder is not a Git clone)'.format(product))
176 if verbose: 109​ if verbose:
177 echo('\tSource folder: {}'.format(source_folder)) 110​ echo('\tSource folder: {}'.format(source_folder))
178 exit(1) 111​ exit(1)
112​ if not os.path.isfile(APPSTREAM_UPDATER):
113​ echo('Can not find appstream-metainfo-release-update at ' + APPSTREAM_UPDATER)
114​ echo('Get it from git@invent.kde.org:jriddell/appstream-metainfo-release-update.git')
115​ exit(1)
179 try: 116​ try:
180 repository.heads[branch].checkout() 117​ repository.heads[branch].checkout()
181 except GitCommandError: 118​ except GitCommandError:
182 echo('{} (error: cannot checkout branch)'.format(product)) 119​ echo('{} (error: cannot checkout branch)'.format(product))
183 if verbose: 120​ if verbose:
184 echo('\tSource folder: {}'.format(source_folder)) 121​ echo('\tSource folder: {}'.format(source_folder))
185 echo('\tCurrent branch: {}'.format(repository.head.name)) 122​ echo('\tCurrent branch: {}'.format(repository.head.name))
186 echo('\tBranch to check out: {}'.format(branch)) 123​ echo('\tBranch to check out: {}'.format(branch))
Show All 10 Lines133 try:
197 repository.remote().pull("--rebase") 134​ repository.remote().pull("--rebase")
198 except GitCommandError: 135​ except GitCommandError:
199 echo('{} (error: cannot pull branch)'.format(product)) 136​ echo('{} (error: cannot pull branch)'.format(product))
200 if verbose: 137​ if verbose:
201 echo('\tSource folder: {}'.format(source_folder)) 138​ echo('\tSource folder: {}'.format(source_folder))
202 echo('\tBranch: {}'.format(branch)) 139​ echo('\tBranch: {}'.format(branch))
203 exit(1) 140​ exit(1)
204 try: 141​ try:
205 version = project_version(source_folder) 142​ version = add_versions_lib.project_version(source_folder)
206 if version is None: 143​ if version is None:
207 if hide_skipped: 144​ if hide_skipped:
208 continue 145​ continue
209 echo('{} \n (no project version found)'.format(product)) 146​ echo('{} \n (no project version found)'.format(product))
210 if verbose: 147​ if verbose:
211 echo('\tSource folder: {}'.format(source_folder)) 148​ echo('\tSource folder: {}'.format(source_folder))
212 echo('\t{}/CMakeLists.txt does not define a project version' 149​ echo('\t{}/CMakeLists.txt does not define a project version'
213 .format(source_folder)) 150​ .format(source_folder))
214 echo('\tSee https://community.kde.org/' 151​ echo('\tSee https://community.kde.org/'
215 'Guidelines_and_HOWTOs/' 152​ 'Guidelines_and_HOWTOs/'
216 'Application_Versioning#Bugzilla_versions') 153​ 'Application_Versioning#Bugzilla_versions')
217 continue 154​ continue
218 if not dry: 155​ if not dry:
219 try: 156​ try:
220 add_version_to_bugzilla_project(session, product, version) 157​ add_version_to_appstream(source_folder, version)
221 echo('{}\n (added {})'.format( 158​ echo('{}\n (added {})'.format(
222 product, version)) 159​ product, version))
223 except AddVersionError as error: 160​ except AddVersionError as error:
224 if error.cause: 161​ if error.cause:
225 echo('{}\n (would have added {})\n (error: {})'.format(product, version, error.cause)) 162​ echo('{}\n (would have added {})\n (error: {})'.format(product, version, error.cause))
226 else: 163​ else:
227 echo('{}\n (would have added {})\n (error: {})'.format(product, version, error)) 164​ echo('{}\n (would have added {})\n (error: {})'.format(product, version, error))
228 except Exception as e: 165​ except Exception as e:
Show All 11 Lines