diff --git a/helpers/create-abi-dump.py b/helpers/create-abi-dump.py --- a/helpers/create-abi-dump.py +++ b/helpers/create-abi-dump.py @@ -20,262 +20,262 @@ ACCXMLTMPL = """{version} - {headers} + {headers} - {libs} + {libs} - {skipIncludePaths} + {skipIncludePaths} - {additionalIncludes} + {additionalIncludes} - {gccOptions} + {gccOptions} """ def cmake_parser(lines: List) -> Dict: - """A small cmake parser, if you search for a better solution think about using - a proper one based on ply. - see https://salsa.debian.org/qt-kde-team/pkg-kde-jenkins/blob/master/hooks/prepare/cmake_update_deps - - But in our case we are only interested in two keywords and do not need many features. - we return a dictonary with keywords and targets. - set(VAR "123") - -> variables["VAR"]="123" - set_target_properties(TARGET PROPERTIES PROP1 A B PROP2 C D) - -> targets = { - "PROP1":["A","B"], - "PROP2":["C","D"], - } - """ - - variables = {} # type: Dict[str,str] - targets = defaultdict(lambda:defaultdict(list)) # type: Dict[str, Dict[str, List[str]]] - - ret = { - "variables": variables, - "targets": targets, - } - - def parse_set(args: str) -> None: - """process set lines and updates the variables directory: - set(VAR 1.2.3) -> args = ["VAR", "1.2.3"] - and we set variable["VAR"] = "1.2.3" - """ - _args = args.split() - if len(_args) == 2: - name, value = _args - variables[name] = value - - def parse_set_target_properties(args: str) -> None: - """process set_target_properties cmake lines and update the targets directory - all argiments of set_target_properties are given in the args parameter as list. - as cmake using keyword val1 val2 we need to save the keyword so long we detect - a next keyword. - - args[0] is the target we want to update - args[1] must be PROPERTIES - """ - name, properties, *values = args.split() - target = targets[name] - if not properties == "PROPERTIES": - logging.warning("unknown line: %s"%(args)) - - # Known set_target_properties keywords - keywords = [ - "IMPORTED_LINK_DEPENDENT_LIBRARIES_DEBUG", - "IMPORTED_LOCATION_DEBUG", - "IMPORTED_SONAME_DEBUG", - "INTERFACE_INCLUDE_DIRECTORIES", - "INTERFACE_LINK_LIBRARIES", - "INTERFACE_COMPILE_OPTIONS", - "INTERFACE_COMPILE_DEFINITIONS", - "IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG", - "IMPORTED_CONFIGURATIONS", - "DOXYGEN_TAGFILE", - ] - - tmpKeyword = None - for arg in values: - if arg in keywords: - tmpKeyword = target[arg] - continue - tmpKeyword.append(arg) - - #Keywords we want to react on - keywords = { - "set": parse_set, - "set_target_properties": parse_set_target_properties, - } - - RELINE = re.compile("^\s*(?P[^(]+)\s*\(\s*(?P.*)\s*\)\s*$") - for line in lines: - m = RELINE.match(line) - if m and m.group('keyword') in keywords: - keywords[m.group('keyword')](m.group('args')) - - return ret + """A small cmake parser, if you search for a better solution think about using + a proper one based on ply. + see https://salsa.debian.org/qt-kde-team/pkg-kde-jenkins/blob/master/hooks/prepare/cmake_update_deps + + But in our case we are only interested in two keywords and do not need many features. + we return a dictonary with keywords and targets. + set(VAR "123") + -> variables["VAR"]="123" + set_target_properties(TARGET PROPERTIES PROP1 A B PROP2 C D) + -> targets = { + "PROP1":["A","B"], + "PROP2":["C","D"], + } + """ + + variables = {} # type: Dict[str,str] + targets = defaultdict(lambda:defaultdict(list)) # type: Dict[str, Dict[str, List[str]]] + + ret = { + "variables": variables, + "targets": targets, + } + + def parse_set(args: str) -> None: + """process set lines and updates the variables directory: + set(VAR 1.2.3) -> args = ["VAR", "1.2.3"] + and we set variable["VAR"] = "1.2.3" + """ + _args = args.split() + if len(_args) == 2: + name, value = _args + variables[name] = value + + def parse_set_target_properties(args: str) -> None: + """process set_target_properties cmake lines and update the targets directory + all argiments of set_target_properties are given in the args parameter as list. + as cmake using keyword val1 val2 we need to save the keyword so long we detect + a next keyword. + + args[0] is the target we want to update + args[1] must be PROPERTIES + """ + name, properties, *values = args.split() + target = targets[name] + if not properties == "PROPERTIES": + logging.warning("unknown line: %s"%(args)) + + # Known set_target_properties keywords + keywords = [ + "IMPORTED_LINK_DEPENDENT_LIBRARIES_DEBUG", + "IMPORTED_LOCATION_DEBUG", + "IMPORTED_SONAME_DEBUG", + "INTERFACE_INCLUDE_DIRECTORIES", + "INTERFACE_LINK_LIBRARIES", + "INTERFACE_COMPILE_OPTIONS", + "INTERFACE_COMPILE_DEFINITIONS", + "IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG", + "IMPORTED_CONFIGURATIONS", + "DOXYGEN_TAGFILE", + ] + + tmpKeyword = None + for arg in values: + if arg in keywords: + tmpKeyword = target[arg] + continue + tmpKeyword.append(arg) + + #Keywords we want to react on + keywords = { + "set": parse_set, + "set_target_properties": parse_set_target_properties, + } + + RELINE = re.compile("^\s*(?P[^(]+)\s*\(\s*(?P.*)\s*\)\s*$") + for line in lines: + m = RELINE.match(line) + if m and m.group('keyword') in keywords: + keywords[m.group('keyword')](m.group('args')) + + return ret # Wrapper class to represent a library we have found # This class stores information on the library in question and assists in extracting information concerning it class Library: - # Make sure we initialize everything we are going to need - def __init__(self, name: str, accSettings: Dict) -> None: - # name of the library - self.name = name # type: str - self.accSettings = accSettings # type: Dict - - # The raw cmake Parser output, available for debugging purposes - # see cmake_parser function for the return value - self.__parser_output = None # type: Union[Dict, None] - - # Provide a helpful description of the object (to ease debugging) - def __repr__(self) -> str: - return "".format(self=self) # replace with f-String in python 3.6 - - # Execute CMake to gather the information we need - def runCMake(self, runtimeEnvironment) -> None: - """Create a CMakeLists.txt to detect the headers, version and library path""" - - # Prepare to gather the information we need - self.__mlines = [] # type: List[str] - - # To avoid contaminating the directory we are being run in, make sure we are in a temporary directory - # This will also allow us to succeed if we are run from the build direcotry - with tempfile.TemporaryDirectory() as d: - # Create an appropriate CMakeLists.txt which searches for the library in question - cmakeFile = (pathlib.Path(d)/"CMakeLists.txt") - cmakeFile.write_text("find_package({self.name} CONFIG REQUIRED)\n".format(self=self)) # replace with f-String in python 3.6 - - # Now run CMake and ask it to process the CMakeLists.txt file we just generated - # We want it to run in trace mode so we can examine the log to extract the information we need - proc = subprocess.Popen(['cmake', '.', '--trace-expand'], cwd=d, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, env=runtimeEnvironment) - - # cmake prefixes outout with the name of the file, filter only lines with interessting files - retarget = re.compile( '.*/{self.name}/[^/]*(Targets[^/]*|Config[^/]*)\.cmake\(\d+\):\s*(.*)$'.format(self=self) ) # replace with f-String in python 3.6 - - # Start processing the output of CMake, one line at a time - for line in proc.stderr: - # Make sure it is UTF-8 formatted - theLine = line.decode("utf-8") - - # Did we find a CMake line we were interested in? - m = retarget.match(theLine) - if m: - # Extract the information from that and store it for further processing - mline = m.group(2) - self.__mlines.append(mline) - - # Process the information we've now gathered - self.__parser_output = cmake_parser(self.__mlines) - self.parser_output = self.__parser_output - self.mlines = self.__mlines - - # Extract the version number of the library for easier use - self.version = self.__parser_output["variables"].get("PACKAGE_VERSION") # type: str - - # targets the targets of the libary ( existing so files) - # a dict with keys, SONAME = the SONAME of the lib - # path = path of the library - # include_dirs = the header files for the library - self.targets = {} # type: Dict - - # Helper Function to parse CMake formatted include directory lists - def parseIncludeDirs(args: List[str]) -> List[str]: - """ cmake using ";" to seperate different paths - split the paths and make a unique list of all paths (do not add paths multiple times) - """ - d = [] # type: List[str] - for arg in args: - d += arg.split(";") - return d - - # Process the various targets our parser found - for t,value in self.__parser_output["targets"].items(): - # Particularly, we want to extract: - # Library names (sonames) - # The path to the CMake library package - # Any include directories specified by the CMake library package - try: - target = { - "SONAME": re.search("\.([\d]*)$",value["IMPORTED_SONAME_DEBUG"][0]).group(1), - "path": value["IMPORTED_LOCATION_DEBUG"][0], - "include_dirs": parseIncludeDirs(value["INTERFACE_INCLUDE_DIRECTORIES"]), - } - self.targets[t]=target - except IndexError: - pass - - def createABIDump(self, runtimeEnvironment=None) -> None: - """run abi-compliance-checker (acc) to create a ABIDump tar gz - - First we need to construct a input file for acc, see xml variable. - After that we can run acc with the constructed file. - """ - - # Make sure we have a valid runtime environment for CMake and abi-compliance-checker - if runtimeEnvironment is None: - runtimeEnvironment = os.environ - - # If we haven't yet run CMake, do so - # Otherwise we won't have anything to give to abi-compliance-checker - if not self.__parser_output: - self.runCMake(runtimeEnvironment) - - # Start preparations to run abi-compliance-checker - # Gather the information we'll need to write the XML configuration file it uses - version = self.version - headers = set() # type: Set[str] - libs = set() # type: Set[str] - skipIncludePaths = set(self.accSettings['skip_include_paths']) # type: Set[str] - additionalIncludes = set(self.accSettings['add_include_paths']) # type: Set[str] - gccOptions = set(self.accSettings['gcc_options']) # type: Set[str] - - - # list of general include directories - prefixHeaders = [os.path.abspath(i) for i in buildEnvironment.get('CMAKE_PREFIX_PATH').split(":")] - noHeaders = set(prefixHeaders) - noHeaders |= set([os.path.join(i,"include") for i in prefixHeaders]) - - # From the target information we previously collected... - # Grab the list of libraries and include headers for abi-compliance-checker - for target in self.targets.values(): - # Check each include directory to see if we need to add it.... - for i in target['include_dirs']: - abspath = os.path.abspath(i) - # ignore general folders, as there are no lib specific headers are placed - if abspath in noHeaders or abspath.endswith("/KF5"): - additionalIncludes.add(abspath) - continue - - # Otherwise, if we don't already have it - add it to the list! - headers.add(abspath) - - # If the library path isn't in the list, then we should add it to the list - libs.add(target['path']) - - # Now we can go ahead and generate the XML file for abi-compliance-checker - def _(l): - return "\n ".join(l) - - xml = ACCXMLTMPL.format(version=version, - headers=_(headers), - libs=_(libs), - additionalIncludes=_(additionalIncludes), - skipIncludePaths=_(skipIncludePaths), - gccOptions=_(gccOptions), - ) - - # Write the generated XML out to a file to pass to abi-compliance-checker - # We will give this to abi-compliance-checker using it's --dump parameter - with open("{version}.xml".format(version=version),"w") as f: # replace with f-String in python 3.6 - f.write(xml) - - # acc is compatible for C/C++ (but --lang C++ doesn't remove the warning about C++ only settings). - subprocess.check_call(["abi-compliance-checker", "--lang", "C++", "-l", self.name, "--dump", f.name], env=runtimeEnvironment) + # Make sure we initialize everything we are going to need + def __init__(self, name: str, accSettings: Dict) -> None: + # name of the library + self.name = name # type: str + self.accSettings = accSettings # type: Dict + + # The raw cmake Parser output, available for debugging purposes + # see cmake_parser function for the return value + self.__parser_output = None # type: Union[Dict, None] + + # Provide a helpful description of the object (to ease debugging) + def __repr__(self) -> str: + return "".format(self=self) # replace with f-String in python 3.6 + + # Execute CMake to gather the information we need + def runCMake(self, runtimeEnvironment) -> None: + """Create a CMakeLists.txt to detect the headers, version and library path""" + + # Prepare to gather the information we need + self.__mlines = [] # type: List[str] + + # To avoid contaminating the directory we are being run in, make sure we are in a temporary directory + # This will also allow us to succeed if we are run from the build direcotry + with tempfile.TemporaryDirectory() as d: + # Create an appropriate CMakeLists.txt which searches for the library in question + cmakeFile = (pathlib.Path(d)/"CMakeLists.txt") + cmakeFile.write_text("find_package({self.name} CONFIG REQUIRED)\n".format(self=self)) # replace with f-String in python 3.6 + + # Now run CMake and ask it to process the CMakeLists.txt file we just generated + # We want it to run in trace mode so we can examine the log to extract the information we need + proc = subprocess.Popen(['cmake', '.', '--trace-expand'], cwd=d, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, env=runtimeEnvironment) + + # cmake prefixes outout with the name of the file, filter only lines with interessting files + retarget = re.compile( '.*/{self.name}/[^/]*(Targets[^/]*|Config[^/]*)\.cmake\(\d+\):\s*(.*)$'.format(self=self) ) # replace with f-String in python 3.6 + + # Start processing the output of CMake, one line at a time + for line in proc.stderr: + # Make sure it is UTF-8 formatted + theLine = line.decode("utf-8") + + # Did we find a CMake line we were interested in? + m = retarget.match(theLine) + if m: + # Extract the information from that and store it for further processing + mline = m.group(2) + self.__mlines.append(mline) + + # Process the information we've now gathered + self.__parser_output = cmake_parser(self.__mlines) + self.parser_output = self.__parser_output + self.mlines = self.__mlines + + # Extract the version number of the library for easier use + self.version = self.__parser_output["variables"].get("PACKAGE_VERSION") # type: str + + # targets the targets of the libary ( existing so files) + # a dict with keys, SONAME = the SONAME of the lib + # path = path of the library + # include_dirs = the header files for the library + self.targets = {} # type: Dict + + # Helper Function to parse CMake formatted include directory lists + def parseIncludeDirs(args: List[str]) -> List[str]: + """ cmake using ";" to seperate different paths + split the paths and make a unique list of all paths (do not add paths multiple times) + """ + d = [] # type: List[str] + for arg in args: + d += arg.split(";") + return d + + # Process the various targets our parser found + for t,value in self.__parser_output["targets"].items(): + # Particularly, we want to extract: + # Library names (sonames) + # The path to the CMake library package + # Any include directories specified by the CMake library package + try: + target = { + "SONAME": re.search("\.([\d]*)$",value["IMPORTED_SONAME_DEBUG"][0]).group(1), + "path": value["IMPORTED_LOCATION_DEBUG"][0], + "include_dirs": parseIncludeDirs(value["INTERFACE_INCLUDE_DIRECTORIES"]), + } + self.targets[t]=target + except IndexError: + pass + + def createABIDump(self, runtimeEnvironment=None) -> None: + """run abi-compliance-checker (acc) to create a ABIDump tar gz + + First we need to construct a input file for acc, see xml variable. + After that we can run acc with the constructed file. + """ + + # Make sure we have a valid runtime environment for CMake and abi-compliance-checker + if runtimeEnvironment is None: + runtimeEnvironment = os.environ + + # If we haven't yet run CMake, do so + # Otherwise we won't have anything to give to abi-compliance-checker + if not self.__parser_output: + self.runCMake(runtimeEnvironment) + + # Start preparations to run abi-compliance-checker + # Gather the information we'll need to write the XML configuration file it uses + version = self.version + headers = set() # type: Set[str] + libs = set() # type: Set[str] + skipIncludePaths = set(self.accSettings['skip_include_paths']) # type: Set[str] + additionalIncludes = set(self.accSettings['add_include_paths']) # type: Set[str] + gccOptions = set(self.accSettings['gcc_options']) # type: Set[str] + + + # list of general include directories + prefixHeaders = [os.path.abspath(i) for i in buildEnvironment.get('CMAKE_PREFIX_PATH').split(":")] + noHeaders = set(prefixHeaders) + noHeaders |= set([os.path.join(i,"include") for i in prefixHeaders]) + + # From the target information we previously collected... + # Grab the list of libraries and include headers for abi-compliance-checker + for target in self.targets.values(): + # Check each include directory to see if we need to add it.... + for i in target['include_dirs']: + abspath = os.path.abspath(i) + # ignore general folders, as there are no lib specific headers are placed + if abspath in noHeaders or abspath.endswith("/KF5"): + additionalIncludes.add(abspath) + continue + + # Otherwise, if we don't already have it - add it to the list! + headers.add(abspath) + + # If the library path isn't in the list, then we should add it to the list + libs.add(target['path']) + + # Now we can go ahead and generate the XML file for abi-compliance-checker + def _(l): + return "\n ".join(l) + + xml = ACCXMLTMPL.format(version=version, + headers=_(headers), + libs=_(libs), + additionalIncludes=_(additionalIncludes), + skipIncludePaths=_(skipIncludePaths), + gccOptions=_(gccOptions), + ) + + # Write the generated XML out to a file to pass to abi-compliance-checker + # We will give this to abi-compliance-checker using it's --dump parameter + with open("{version}.xml".format(version=version),"w") as f: # replace with f-String in python 3.6 + f.write(xml) + + # acc is compatible for C/C++ (but --lang C++ doesn't remove the warning about C++ only settings). + subprocess.check_call(["abi-compliance-checker", "--lang", "C++", "-l", self.name, "--dump", f.name], env=runtimeEnvironment) # Parse the command line arguments we've been given parser = argparse.ArgumentParser(description='Utility to create abi checker tarballs.') @@ -304,11 +304,11 @@ # From this we get a complete List of installed libraries. cmakeConfig = re.compile("^-- (Installing|Up-to-date): .*/([^/]*)Config\.cmake$") with open(arguments.buildLog, encoding='utf-8') as log: - for line in log.readlines(): - match = cmakeConfig.match(line) - if match: - foundLibrary = Library( match.group(2), accSettings ) - foundLibraries.append(foundLibrary) + for line in log.readlines(): + match = cmakeConfig.match(line) + if match: + foundLibrary = Library( match.group(2), accSettings ) + foundLibraries.append(foundLibrary) # Initialize the archive manager ourArchive = Packages.Archive(arguments.environment, 'ABIReference', usingCache = False, contentsSuffix = ".abidump") @@ -318,47 +318,47 @@ # GIT_COMMIT is set by Jenkins Git plugin, so we can rely on that for most of our builds scmRevision = '' if os.getenv('GIT_COMMIT') != '': - scmRevision = os.getenv('GIT_COMMIT') + scmRevision = os.getenv('GIT_COMMIT') if not scmRevision: - scmRevision = subprocess.check_output(["git", "log", "--format=%H", "-n 1", "HEAD"]).strip().decode() + scmRevision = subprocess.check_output(["git", "log", "--format=%H", "-n 1", "HEAD"]).strip().decode() # Check every libraries ABI and do not fail, if one is not fine. # Safe the overall retval state retval = 0 # Now we generate the ABI dumps for every library we have found for library in foundLibraries: - try: - #run CMake for library - library.runCMake( runtimeEnvironment=buildEnvironment ) - - if not library.version: - logging.warning("{name} has no version: skipping.".format(name=library.name)) - continue - - # Create the ABI Dump for this library - library.createABIDump( runtimeEnvironment=buildEnvironment ) - - # Determine where the ABI Dump archive is located - # This location is controlled by abi-compliance-checker, but follows a predictable pattern - fileName = "abi_dumps/{name}/{version}/ABI.dump".format(name=library.name,version=library.version) - - extraMetadata = { - "SONAME": max([t['SONAME'] for t in library.targets.values()]), # use max because there may be more than one lib inside - "version": library.version, - "libname": library.name, - "targets": list(library.targets), - "project": arguments.project, - "branchGroup": arguments.branchGroup, - "platform": arguments.platform, - } - packageName = "{name}_{scmRevision}_{platform}".format(name=library.name, scmRevision=scmRevision, platform=arguments.platform) - ourArchive.storePackage(packageName, fileName, scmRevision, extraMetadata) - except subprocess.CalledProcessError as e: - retval = e.returncode - logging.error("abi-compliance-checker exited with {retval}".format(retval=retval)) + try: + #run CMake for library + library.runCMake( runtimeEnvironment=buildEnvironment ) + + if not library.version: + logging.warning("{name} has no version: skipping.".format(name=library.name)) + continue + + # Create the ABI Dump for this library + library.createABIDump( runtimeEnvironment=buildEnvironment ) + + # Determine where the ABI Dump archive is located + # This location is controlled by abi-compliance-checker, but follows a predictable pattern + fileName = "abi_dumps/{name}/{version}/ABI.dump".format(name=library.name,version=library.version) + + extraMetadata = { + "SONAME": max([t['SONAME'] for t in library.targets.values()]), # use max because there may be more than one lib inside + "version": library.version, + "libname": library.name, + "targets": list(library.targets), + "project": arguments.project, + "branchGroup": arguments.branchGroup, + "platform": arguments.platform, + } + packageName = "{name}_{scmRevision}_{platform}".format(name=library.name, scmRevision=scmRevision, platform=arguments.platform) + ourArchive.storePackage(packageName, fileName, scmRevision, extraMetadata) + except subprocess.CalledProcessError as e: + retval = e.returncode + logging.error("abi-compliance-checker exited with {retval}".format(retval=retval)) # We had an issue with one of the ABIs if retval != 0: - sys.exit(retval) + sys.exit(retval)