Changeset View
Changeset View
Standalone View
Standalone View
helpers/create-abi-dump.py
| 1 | #!/usr/bin/python3 | 1 | #!/usr/bin/python3 | ||
|---|---|---|---|---|---|
| 2 | import os | 2 | import argparse | ||
| 3 | import re | | |||
| 4 | import logging | 3 | import logging | ||
| 4 | import os | ||||
| 5 | import pathlib | 5 | import pathlib | ||
| 6 | import argparse | 6 | import re | ||
| 7 | import tempfile | | |||
| 8 | import subprocess | 7 | import subprocess | ||
| 9 | import sys | 8 | import sys | ||
| 9 | import tempfile | ||||
| 10 | import yaml | ||||
| 11 | | ||||
| 10 | from collections import defaultdict | 12 | from collections import defaultdict | ||
| 11 | from typing import Dict, List, Union, Set | 13 | from typing import Dict, List, Union, Set | ||
| 12 | 14 | | |||
| 15 | from helperslib import CommonUtils, ToolingSettings | ||||
| 13 | from helperslib import Packages, EnvironmentHandler | 16 | from helperslib import Packages, EnvironmentHandler | ||
| 14 | 17 | | |||
| 15 | # Make sure logging is ready to go | 18 | # Make sure logging is ready to go | ||
| 16 | #logging.basicConfig(level=logging.DEBUG) | 19 | #logging.basicConfig(level=logging.DEBUG) | ||
| 17 | 20 | | |||
| 21 | ACCXMLTMPL = """<version>{version}</version> | ||||
| 22 | <headers> | ||||
| 23 | {headers} | ||||
| 24 | </headers> | ||||
| 25 | <libs> | ||||
| 26 | {libs} | ||||
| 27 | </libs> | ||||
| 28 | <skip_include_paths> | ||||
| 29 | {skipIncludePaths} | ||||
| 30 | </skip_include_paths> | ||||
| 31 | <add_include_paths> | ||||
| 32 | {additionalIncludes} | ||||
| 33 | </add_include_paths> | ||||
| 34 | <gcc_options> | ||||
| 35 | {gccOptions} | ||||
| 36 | </gcc_options> | ||||
| 37 | """ | ||||
| 38 | | ||||
| 18 | def cmake_parser(lines: List) -> Dict: | 39 | def cmake_parser(lines: List) -> Dict: | ||
| 19 | """A small cmake parser, if you search for a better solution think about using | 40 | """A small cmake parser, if you search for a better solution think about using | ||
| 20 | a proper one based on ply. | 41 | a proper one based on ply. | ||
| 21 | see https://salsa.debian.org/qt-kde-team/pkg-kde-jenkins/blob/master/hooks/prepare/cmake_update_deps | 42 | see https://salsa.debian.org/qt-kde-team/pkg-kde-jenkins/blob/master/hooks/prepare/cmake_update_deps | ||
| 22 | 43 | | |||
| 23 | But in our case we are only interested in two keywords and do not need many features. | 44 | But in our case we are only interested in two keywords and do not need many features. | ||
| 24 | we return a dictonary with keywords and targets. | 45 | we return a dictonary with keywords and targets. | ||
| 25 | set(VAR "123") | 46 | set(VAR "123") | ||
| ▲ Show 20 Lines • Show All 71 Lines • ▼ Show 20 Line(s) | 117 | if m and m.group('keyword') in keywords: | |||
| 97 | keywords[m.group('keyword')](m.group('args')) | 118 | keywords[m.group('keyword')](m.group('args')) | ||
| 98 | 119 | | |||
| 99 | return ret | 120 | return ret | ||
| 100 | 121 | | |||
| 101 | # Wrapper class to represent a library we have found | 122 | # Wrapper class to represent a library we have found | ||
| 102 | # This class stores information on the library in question and assists in extracting information concerning it | 123 | # This class stores information on the library in question and assists in extracting information concerning it | ||
| 103 | class Library: | 124 | class Library: | ||
| 104 | # Make sure we initialize everything we are going to need | 125 | # Make sure we initialize everything we are going to need | ||
| 105 | def __init__(self, name: str) -> None: | 126 | def __init__(self, name: str, accSettings: Dict) -> None: | ||
| 106 | # name of the library | 127 | # name of the library | ||
| 107 | self.name = name # type: str | 128 | self.name = name # type: str | ||
| 129 | self.accSettings = accSettings # type: Dict | ||||
| 108 | 130 | | |||
| 109 | # The raw cmake Parser output, available for debugging purposes | 131 | # The raw cmake Parser output, available for debugging purposes | ||
| 110 | # see cmake_parser function for the return value | 132 | # see cmake_parser function for the return value | ||
| 111 | self.__parser_output = None # type: Union[Dict, None] | 133 | self.__parser_output = None # type: Union[Dict, None] | ||
| 112 | 134 | | |||
| 113 | # Provide a helpful description of the object (to ease debugging) | 135 | # Provide a helpful description of the object (to ease debugging) | ||
| 114 | def __repr__(self) -> str: | 136 | def __repr__(self) -> str: | ||
| 115 | return "<Library \"{self.name}\">".format(self=self) # replace with f-String in python 3.6 | 137 | return "<Library \"{self.name}\">".format(self=self) # replace with f-String in python 3.6 | ||
| ▲ Show 20 Lines • Show All 87 Lines • ▼ Show 20 Line(s) | 212 | def createABIDump(self, runtimeEnvironment=None) -> None: | |||
| 203 | if not self.__parser_output: | 225 | if not self.__parser_output: | ||
| 204 | self.runCMake(runtimeEnvironment) | 226 | self.runCMake(runtimeEnvironment) | ||
| 205 | 227 | | |||
| 206 | # Start preparations to run abi-compliance-checker | 228 | # Start preparations to run abi-compliance-checker | ||
| 207 | # Gather the information we'll need to write the XML configuration file it uses | 229 | # Gather the information we'll need to write the XML configuration file it uses | ||
| 208 | version = self.version | 230 | version = self.version | ||
| 209 | headers = set() # type: Set[str] | 231 | headers = set() # type: Set[str] | ||
| 210 | libs = set() # type: Set[str] | 232 | libs = set() # type: Set[str] | ||
| 211 | additionalIncludes = set() # type: Set[str] | 233 | skipIncludePaths = set(self.accSettings['skip_include_paths']) # type: Set[str] | ||
| 234 | additionalIncludes = set(self.accSettings['add_include_paths']) # type: Set[str] | ||||
| 235 | gccOptions = set(self.accSettings['gcc_options']) # type: Set[str] | ||||
| 236 | | ||||
bcooksley: We should probably document all the supported options somewhere central.
I'd suggest this is… | |||||
| 212 | 237 | | |||
| 213 | # list of general include directories | 238 | # list of general include directories | ||
| 214 | prefixHeaders = [os.path.abspath(i) for i in buildEnvironment.get('CMAKE_PREFIX_PATH').split(":")] | 239 | prefixHeaders = [os.path.abspath(i) for i in buildEnvironment.get('CMAKE_PREFIX_PATH').split(":")] | ||
| 215 | noHeaders = set(prefixHeaders) | 240 | noHeaders = set(prefixHeaders) | ||
| 216 | noHeaders |= set([os.path.join(i,"include") for i in prefixHeaders]) | 241 | noHeaders |= set([os.path.join(i,"include") for i in prefixHeaders]) | ||
| 217 | 242 | | |||
| 218 | # From the target information we previously collected... | 243 | # From the target information we previously collected... | ||
| 219 | # Grab the list of libraries and include headers for abi-compliance-checker | 244 | # Grab the list of libraries and include headers for abi-compliance-checker | ||
| 220 | for target in self.targets.values(): | 245 | for target in self.targets.values(): | ||
| 221 | # Check each include directory to see if we need to add it.... | 246 | # Check each include directory to see if we need to add it.... | ||
| 222 | for i in target['include_dirs']: | 247 | for i in target['include_dirs']: | ||
| 223 | abspath = os.path.abspath(i) | 248 | abspath = os.path.abspath(i) | ||
| 224 | # ignore general folders, as there are no lib specific headers are placed | 249 | # ignore general folders, as there are no lib specific headers are placed | ||
| 225 | if abspath in noHeaders or abspath.endswith("/KF5"): | 250 | if abspath in noHeaders or abspath.endswith("/KF5"): | ||
| 226 | additionalIncludes.add(abspath) | 251 | additionalIncludes.add(abspath) | ||
| 227 | continue | 252 | continue | ||
| 228 | 253 | | |||
| 229 | # Otherwise, if we don't already have it - add it to the list! | 254 | # Otherwise, if we don't already have it - add it to the list! | ||
| 230 | headers.add(abspath) | 255 | headers.add(abspath) | ||
| 231 | 256 | | |||
| 232 | # If the library path isn't in the list, then we should add it to the list | 257 | # If the library path isn't in the list, then we should add it to the list | ||
| 233 | libs.add(target['path']) | 258 | libs.add(target['path']) | ||
| 234 | 259 | | |||
| 235 | # Now we can go ahead and generate the XML file for abi-compliance-checker | 260 | # Now we can go ahead and generate the XML file for abi-compliance-checker | ||
| 236 | xml = """ | 261 | def _(l): | ||
| 237 | <version>{version}</version> | 262 | return "\n ".join(l) | ||
| 238 | <headers> | 263 | | ||
| 239 | {headers} | 264 | xml = ACCXMLTMPL.format(version=version, | ||
| 240 | </headers> | 265 | headers=_(headers), | ||
| 241 | <libs> | 266 | libs=_(libs), | ||
| 242 | {libs} | 267 | additionalIncludes=_(additionalIncludes), | ||
| 243 | </libs> | 268 | skipIncludePaths=_(skipIncludePaths), | ||
| 244 | <skip_include_paths> | 269 | gccOptions=_(gccOptions), | ||
| 245 | /usr/lib/python3.6/site-packages/utils/fake_libc_include | 270 | ) | ||
| 246 | /usr/include/clang/AST | 271 | | ||
| 247 | /usr/lib64/clang/6.0.1/include | | |||
| 248 | </skip_include_paths> | | |||
| 249 | <add_include_paths> | | |||
| 250 | {additionalIncludes} | | |||
| 251 | /usr/lib64/qt5/mkspecs/linux-g++ | | |||
| 252 | </add_include_paths> | | |||
| 253 | """.format(version=version, headers="\n".join(headers), libs="\n".join(libs), additionalIncludes="\n".join(additionalIncludes)) # replace with f-String in Python 3.6 | | |||
| 254 | | ||||
| 255 | # Write the generated XML out to a file to pass to abi-compliance-checker | 272 | # Write the generated XML out to a file to pass to abi-compliance-checker | ||
| 256 | # We will give this to abi-compliance-checker using it's --dump parameter | 273 | # We will give this to abi-compliance-checker using it's --dump parameter | ||
| 257 | with open("{version}.xml".format(version=version),"w") as f: # replace with f-String in python 3.6 | 274 | with open("{version}.xml".format(version=version),"w") as f: # replace with f-String in python 3.6 | ||
| 258 | f.write(xml) | 275 | f.write(xml) | ||
| 259 | 276 | | |||
| 260 | # acc is compatible for C/C++ as Qt using C++11 and -fPic we need to set the gcc settings explitly | 277 | # acc is compatible for C/C++ (but --lang C++ doesn't remove the warning about C++ only settings). | ||
Shouldn't this comment be updated to reflect the new arguments being passed to abi-compliance-checker? bcooksley: Shouldn't this comment be updated to reflect the new arguments being passed to abi-compliance… | |||||
| 261 | subprocess.check_call(["abi-compliance-checker", "-gcc-options", "-std=c++11 -fPIC", "-l", self.name, "--dump", f.name], env=runtimeEnvironment) | 278 | subprocess.check_call(["abi-compliance-checker", "--lang", "C++", "-l", self.name, "--dump", f.name], env=runtimeEnvironment) | ||
| 262 | 279 | | |||
| 263 | # Parse the command line arguments we've been given | 280 | # Parse the command line arguments we've been given | ||
| 264 | parser = argparse.ArgumentParser(description='Utility to create abi checker tarballs.') | 281 | parser = argparse.ArgumentParser(description='Utility to create abi checker tarballs.') | ||
| 265 | parser.add_argument('--project', type=str, required=True) | 282 | parser.add_argument('--project', type=str, required=True) | ||
| 266 | parser.add_argument('--branchGroup', type=str, required=True) | 283 | parser.add_argument('--branchGroup', type=str, required=True) | ||
| 267 | parser.add_argument('--buildLog', type=str, required=True) | 284 | parser.add_argument('--buildLog', type=str, required=True) | ||
| 268 | parser.add_argument('--environment', type=str, required=True) | 285 | parser.add_argument('--environment', type=str, required=True) | ||
| 269 | parser.add_argument('--platform', type=str, required=True) | 286 | parser.add_argument('--platform', type=str, required=True) | ||
| 270 | parser.add_argument('--usingInstall', type=str, required=True) | 287 | parser.add_argument('--usingInstall', type=str, required=True) | ||
| 271 | arguments = parser.parse_args() | 288 | arguments = parser.parse_args() | ||
| 272 | 289 | | |||
| 273 | # Make sure we have an environment ready for executing commands | 290 | # Make sure we have an environment ready for executing commands | ||
| 274 | buildEnvironment = EnvironmentHandler.generateFor( installPrefix=arguments.usingInstall ) | 291 | buildEnvironment = EnvironmentHandler.generateFor( installPrefix=arguments.usingInstall ) | ||
| 275 | 292 | | |||
| 293 | | ||||
| 294 | # get acc settings | ||||
| 295 | localMetadataPath = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'abi-compliance-checker.yaml' ) | ||||
| 296 | | ||||
| 297 | accSettings = ToolingSettings.Loader( arguments.project, arguments.platform ) | ||||
| 298 | accSettings.loadSpecificationFile( localMetadataPath ) | ||||
| 299 | | ||||
| 276 | # Get ready to start searching for libraries | 300 | # Get ready to start searching for libraries | ||
| 277 | foundLibraries = [] | 301 | foundLibraries = [] | ||
| 278 | 302 | | |||
| 279 | # Search in the build log for the Installing/Up-to-date lines where we install the <name>Config.cmake files. | 303 | # Search in the build log for the Installing/Up-to-date lines where we install the <name>Config.cmake files. | ||
bcooksley: Spelling: abi-complience-checker -> abi-compliance-checker | |||||
| 280 | # From this we get a complete List of installed libraries. | 304 | # From this we get a complete List of installed libraries. | ||
bcooksley: Code style: `yaml.load( open(localMetadataPath) )` | |||||
| 281 | cmakeConfig = re.compile("^-- (Installing|Up-to-date): .*/([^/]*)Config\.cmake$") | 305 | cmakeConfig = re.compile("^-- (Installing|Up-to-date): .*/([^/]*)Config\.cmake$") | ||
| 282 | with open(arguments.buildLog, encoding='utf-8') as log: | 306 | with open(arguments.buildLog, encoding='utf-8') as log: | ||
| 283 | for line in log.readlines(): | 307 | for line in log.readlines(): | ||
| 284 | match = cmakeConfig.match(line) | 308 | match = cmakeConfig.match(line) | ||
| 285 | if match: | 309 | if match: | ||
| 286 | foundLibrary = Library( match.group(2) ) | 310 | foundLibrary = Library( match.group(2), accSettings ) | ||
| 287 | foundLibraries.append(foundLibrary) | 311 | foundLibraries.append(foundLibrary) | ||
| 288 | 312 | | |||
| 289 | # Initialize the archive manager | 313 | # Initialize the archive manager | ||
| 290 | ourArchive = Packages.Archive(arguments.environment, 'ABIReference', usingCache = False, contentsSuffix = ".abidump") | 314 | ourArchive = Packages.Archive(arguments.environment, 'ABIReference', usingCache = False, contentsSuffix = ".abidump") | ||
| 291 | 315 | | |||
| 292 | # Determine which SCM revision we are storing | 316 | # Determine which SCM revision we are storing | ||
| 293 | # This will be embedded into the package metadata which might help someone doing some debugging | 317 | # This will be embedded into the package metadata which might help someone doing some debugging | ||
| 294 | # GIT_COMMIT is set by Jenkins Git plugin, so we can rely on that for most of our builds | 318 | # GIT_COMMIT is set by Jenkins Git plugin, so we can rely on that for most of our builds | ||
| Show All 33 Lines | 347 | extraMetadata = { | |||
| 328 | "project": arguments.project, | 352 | "project": arguments.project, | ||
| 329 | "branchGroup": arguments.branchGroup, | 353 | "branchGroup": arguments.branchGroup, | ||
| 330 | "platform": arguments.platform, | 354 | "platform": arguments.platform, | ||
| 331 | } | 355 | } | ||
| 332 | packageName = "{name}_{scmRevision}_{platform}".format(name=library.name, scmRevision=scmRevision, platform=arguments.platform) | 356 | packageName = "{name}_{scmRevision}_{platform}".format(name=library.name, scmRevision=scmRevision, platform=arguments.platform) | ||
| 333 | ourArchive.storePackage(packageName, fileName, scmRevision, extraMetadata) | 357 | ourArchive.storePackage(packageName, fileName, scmRevision, extraMetadata) | ||
| 334 | except subprocess.CalledProcessError as e: | 358 | except subprocess.CalledProcessError as e: | ||
| 335 | retval = e.returncode | 359 | retval = e.returncode | ||
| 336 | logging.error("abi-complience-checker exited with {retval}".format(retval=retval)) | 360 | logging.error("abi-compliance-checker exited with {retval}".format(retval=retval)) | ||
| 337 | 361 | | |||
| 338 | # We had an issue with one of the ABIs | 362 | # We had an issue with one of the ABIs | ||
| 339 | if retval != 0: | 363 | if retval != 0: | ||
| 340 | sys.exit(retval) | 364 | sys.exit(retval) | ||
We should probably document all the supported options somewhere central.
I'd suggest this is done as comments at the top of the config file.