diff --git a/helpers/create-abi-dump.py b/helpers/create-abi-dump.py index cc9830b..fd390e3 100755 --- a/helpers/create-abi-dump.py +++ b/helpers/create-abi-dump.py @@ -1,323 +1,324 @@ #!/usr/bin/python3 import os import re import logging import pathlib import argparse import tempfile import subprocess import sys from collections import defaultdict from typing import Dict, List, Union from helperslib import Packages, EnvironmentHandler # Make sure logging is ready to go #logging.basicConfig(level=logging.DEBUG) def cmake_parser(lines: List) -> Dict: """A small cmake parser, if you search for a better solution think about using a proper one based on ply. see https://salsa.debian.org/qt-kde-team/pkg-kde-jenkins/blob/master/hooks/prepare/cmake_update_deps But in our case we are only interested in two keywords and do not need many features. we return a dictonary with keywords and targets. set(VAR "123") -> variables["VAR"]="123" set_target_properties(TARGET PROPERTIES PROP1 A B PROP2 C D) -> targets = { "PROP1":["A","B"], "PROP2":["C","D"], } """ variables = {} # type: Dict[str,str] targets = defaultdict(lambda:defaultdict(list)) # type: Dict[str, Dict[str, List[str]]] ret = { "variables": variables, "targets": targets, } def parse_set(args: str) -> None: """process set lines and updates the variables directory: set(VAR 1.2.3) -> args = ["VAR", "1.2.3"] and we set variable["VAR"] = "1.2.3" """ _args = args.split() if len(_args) == 2: name, value = _args variables[name] = value def parse_set_target_properties(args: str) -> None: """process set_target_properties cmake lines and update the targets directory all argiments of set_target_properties are given in the args parameter as list. as cmake using keyword val1 val2 we need to save the keyword so long we detect a next keyword. args[0] is the target we want to update args[1] must be PROPERTIES """ name, properties, *values = args.split() target = targets[name] if not properties == "PROPERTIES": logging.warning("unknown line: %s"%(args)) # Known set_target_properties keywords keywords = [ "IMPORTED_LINK_DEPENDENT_LIBRARIES_DEBUG", "IMPORTED_LOCATION_DEBUG", "IMPORTED_SONAME_DEBUG", "INTERFACE_INCLUDE_DIRECTORIES", "INTERFACE_LINK_LIBRARIES", "INTERFACE_COMPILE_OPTIONS", "INTERFACE_COMPILE_DEFINITIONS", ] tmpKeyword = None for arg in values: if arg in keywords: tmpKeyword = target[arg] continue tmpKeyword.append(arg) #Keywords we want to react on keywords = { "set": parse_set, "set_target_properties": parse_set_target_properties, } RELINE = re.compile("^\s*(?P[^(]+)\s*\(\s*(?P.*)\s*\)\s*$") for line in lines: m = RELINE.match(line) if m and m.group('keyword') in keywords: keywords[m.group('keyword')](m.group('args')) return ret # Wrapper class to represent a library we have found # This class stores information on the library in question and assists in extracting information concerning it class Library: # Make sure we initialize everything we are going to need def __init__(self, name: str) -> None: # name of the library self.name = name # type: str # The raw cmake Parser output, available for debugging purposes # see cmake_parser function for the return value self.__parser_output = None # type: Union[Dict, None] # Provide a helpful description of the object (to ease debugging) def __repr__(self) -> str: return "".format(self=self) # replace with f-String in python 3.6 # Execute CMake to gather the information we need def runCMake(self, runtimeEnvironment) -> None: """Create a CMakeLists.txt to detect the headers, version and library path""" # Prepare to gather the information we need self.__mlines = [] # type: List[str] # To avoid contaminating the directory we are being run in, make sure we are in a temporary directory # This will also allow us to succeed if we are run from the build direcotry with tempfile.TemporaryDirectory() as d: # Create an appropriate CMakeLists.txt which searches for the library in question cmakeFile = (pathlib.Path(d)/"CMakeLists.txt") cmakeFile.write_text("find_package({self.name} CONFIG REQUIRED)\n".format(self=self)) # replace with f-String in python 3.6 # Now run CMake and ask it to process the CMakeLists.txt file we just generated # We want it to run in trace mode so we can examine the log to extract the information we need proc = subprocess.Popen(['cmake', '.', '--trace-expand'], cwd=d, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, env=runtimeEnvironment) # cmake prefixes outout with the name of the file, filter only lines with interessting files retarget = re.compile( '.*/{self.name}(Targets[^/]*|Config[^/]*)\.cmake\(\d+\):\s*(.*)$'.format(self=self) ) # replace with f-String in python 3.6 # Start processing the output of CMake, one line at a time for line in proc.stderr: # Make sure it is UTF-8 formatted theLine = line.decode("utf-8") # Did we find a CMake line we were interested in? m = retarget.match(theLine) if m: # Extract the information from that and store it for further processing mline = m.group(2) self.__mlines.append(mline) # Process the information we've now gathered self.__parser_output = cmake_parser(self.__mlines) self.parser_output = self.__parser_output self.mlines = self.__mlines # Extract the version number of the library for easier use self.version = self.__parser_output["variables"]["PACKAGE_VERSION"] # type: str # targets the targets of the libary ( existing so files) # a dict with keys, SONAME = the SONAME of the lib # path = path of the library # include_dirs = the header files for the library self.targets = {} # type: Dict # Helper Function to parse CMake formatted include directory lists def parseIncludeDirs(args: List[str]) -> List[str]: """ cmake using ";" to seperate different paths split the paths and make a unique list of all paths (do not add paths multiple times) """ d = [] # type: List[str] for arg in args: d += arg.split(";") return d # Process the various targets our parser found for t,value in self.__parser_output["targets"].items(): # Particularly, we want to extract: # Library names (sonames) # The path to the CMake library package # Any include directories specified by the CMake library package target = { "SONAME": re.search("\.([\d]*)$",value["IMPORTED_SONAME_DEBUG"][0]).group(1), "path": value["IMPORTED_LOCATION_DEBUG"][0], "include_dirs": parseIncludeDirs(value["INTERFACE_INCLUDE_DIRECTORIES"]), } self.targets[t]=target def createABIDump(self, runtimeEnvironment=None) -> None: """run abi-compliance-checker (acc) to create a ABIDump tar gz First we need to construct a input file for acc, see xml variable. After that we can run acc with the constructed file. """ # Make sure we have a valid runtime environment for CMake and abi-compliance-checker if runtimeEnvironment is None: runtimeEnvironment = os.environ # If we haven't yet run CMake, do so # Otherwise we won't have anything to give to abi-compliance-checker if not self.__parser_output: self.runCMake(runtimeEnvironment) # Start preparations to run abi-compliance-checker # Gather the information we'll need to write the XML configuration file it uses version = self.version headers = [] # type: List[str] libs = [] # type: List[str] additionalIncludes = [] # type: List[str] # From the target information we previously collected... # Grab the list of libraries and include headers for abi-compliance-checker for target in self.targets.values(): # Check each include directory to see if we need to add it.... for i in target['include_dirs']: # ignore general folders, as there are no lib specific headers are placed if i == '/usr/include' or i.endswith("/KF5"): if not i in additionalIncludes: additionalIncludes.append(i) continue # Otherwise, if we don't already have it - add it to the list! if not i in headers: headers.append(i) # If the library path isn't in the list, then we should add it to the list if not target['path'] in libs: libs.append(target['path']) # Now we can go ahead and generate the XML file for abi-compliance-checker xml = """ {version} {headers} {libs} /usr/lib/python3.6/site-packages/utils/fake_libc_include /usr/include/clang/AST /usr/lib64/clang/6.0.1/include {additionalIncludes} + /usr/lib64/qt5/mkspecs/linux-g++ """.format(version=version, headers="\n".join(headers), libs="\n".join(libs), additionalIncludes="\n".join(additionalIncludes)) # replace with f-String in Python 3.6 # Write the generated XML out to a file to pass to abi-compliance-checker # We will give this to abi-compliance-checker using it's --dump parameter with open("{version}.xml".format(version=version),"w") as f: # replace with f-String in python 3.6 f.write(xml) # acc is compatible for C/C++ as Qt using C++11 and -fPic we need to set the gcc settings explitly subprocess.check_call(["abi-compliance-checker", "-gcc-options", "-std=c++11 -fPIC", "-l", self.name, "--dump", f.name], env=runtimeEnvironment) # Parse the command line arguments we've been given parser = argparse.ArgumentParser(description='Utility to create abi checker tarballs.') parser.add_argument('--project', type=str, required=True) parser.add_argument('--branchGroup', type=str, required=True) parser.add_argument('--buildLog', type=str, required=True) parser.add_argument('--environment', type=str, required=True) parser.add_argument('--platform', type=str, required=True) parser.add_argument('--usingInstall', type=str, required=True) arguments = parser.parse_args() # Make sure we have an environment ready for executing commands buildEnvironment = EnvironmentHandler.generateFor( installPrefix=arguments.usingInstall ) # Get ready to start searching for libraries foundLibraries = [] # Search in the build log for the Installing/Up-to-date lines where we install the Config.cmake files. # From this we get a complete List of installed libraries. cmakeConfig = re.compile("^-- (Installing|Up-to-date): .*/([^/]*)Config\.cmake$") with open(arguments.buildLog, encoding='utf-8') as log: for line in log.readlines(): match = cmakeConfig.match(line) if match: foundLibrary = Library( match.group(2) ) foundLibraries.append(foundLibrary) # Initialize the archive manager ourArchive = Packages.Archive(arguments.environment, 'ABIReference', usingCache = False, contentsSuffix = ".abidump") # Determine which SCM revision we are storing # This will be embedded into the package metadata which might help someone doing some debugging # GIT_COMMIT is set by Jenkins Git plugin, so we can rely on that for most of our builds scmRevision = '' if os.getenv('GIT_COMMIT') != '': scmRevision = os.getenv('GIT_COMMIT') if not scmRevision: scmRevision = subprocess.check_output(["git", "log", "--format=%H", "-n 1", "HEAD"]).strip().decode() # Check every libraries ABI and do not fail, if one is not fine. # Safe the overall retval state retval = 0 # Now we generate the ABI dumps for every library we have found for library in foundLibraries: try: # Create the ABI Dump for this library library.createABIDump( runtimeEnvironment=buildEnvironment ) # Determine where the ABI Dump archive is located # This location is controlled by abi-compliance-checker, but follows a predictable pattern fileName = "abi_dumps/{name}/{version}/ABI.dump".format(name=library.name,version=library.version) extraMetadata = { "SONAME": max([t['SONAME'] for t in library.targets.values()]), # use max because there may be more than one lib inside "version": library.version, "libname": library.name, "targets": list(library.targets), "project": arguments.project, "branchGroup": arguments.branchGroup, "platform": arguments.platform, } packageName = "{name}_{scmRevision}_{platform}".format(name=library.name, scmRevision=scmRevision, platform=arguments.platform) ourArchive.storePackage(packageName, fileName, scmRevision, extraMetadata) except subprocess.CalledProcessError as e: retval = e.returncode logging.error("abi-complience-checker exited with {retval}".format(retval=retval)) # We had an issue with one of the ABIs if retval != 0: sys.exit(retval)