diff --git a/helpers/create-abi-bump.py b/helpers/create-abi-bump.py --- a/helpers/create-abi-bump.py +++ b/helpers/create-abi-bump.py @@ -3,9 +3,11 @@ import argparse from collections import defaultdict import os +import pathlib import re import subprocess import tempfile +from typing import Dict, List, Union from helperslib import Packages @@ -18,45 +20,76 @@ parser.add_argument('--environment', type=str, required=True) arguments = parser.parse_args() -def cmake_parser(lines): +def cmake_parser(lines: List) -> Dict: + """A small cmake parser, if you search for a better solution think about using + a propper one based on ply. + see https://salsa.debian.org/qt-kde-team/pkg-kde-jenkins/blob/master/hooks/prepare/cmake_update_deps + + But in our case we are only interessed in two keywords and do not need many features. + we return a dictonary with keywords and targets. + set(VAR "123") + -> variables["VAR"]="123" + set_target_properties(TARGET PROPERTIES PROP1 A B PROP2 C D) + -> targets = { + "PROP1":["A","B"], + "PROP2":["C","D"], + } + """ + + variables = {} # type: Dict[str,str] + targets = defaultdict(lambda:defaultdict(list)) # type: Dict[str, Dict[str, List[str]]] + ret = { - "variables":{}, - "targets":defaultdict(lambda:defaultdict(list)) - } - variables = ret['variables'] - targets = ret['targets'] - - def parse_set(args): - args = args.split() - if len(args) == 2: - variables[args[0]] = args[1] - - def parse_set_target_properties(args): - args = args.split() - target = targets[args[0]] - if not args[1] == "PROPERTIES": + "variable": variables, + "targets": targets, + } + + def parse_set(args: str) -> None: + """process set lines and updates the variables directory: + set(VAR 1.2.3) -> args = ["VAR", "1.2.3"] + and we set variable["VAR"] = "1.2.3" + """ + _args = args.split() + if len(_args) == 2: + variables[_args[0]] = _args[1] + + def parse_set_target_properties(args: str) -> None: + """process set_target_properties cmake lines and update the targets directory + all argiments of set_target_properties are given in the args parameter as list. + as cmake using keyword val1 val2 we need to save the keyword so long we detect + a next keyword. + + args[0] is the target we want to update + args[1] must be PROPERTIES + """ + _args = args.split() + target = targets[_args[0]] + if not _args[1] == "PROPERTIES": logging.warning("unknown line: %s"%(args)) - - keywords=["IMPORTED_LINK_DEPENDENT_LIBRARIES_DEBUG", - "IMPORTED_LOCATION_DEBUG", - "IMPORTED_SONAME_DEBUG", - "INTERFACE_INCLUDE_DIRECTORIES", - "INTERFACE_LINK_LIBRARIES", - "INTERFACE_COMPILE_OPTIONS", - ] - - t = None - for arg in args[2:]: + # Known set_target_properties keywords + keywords = [ + "IMPORTED_LINK_DEPENDENT_LIBRARIES_DEBUG", + "IMPORTED_LOCATION_DEBUG", + "IMPORTED_SONAME_DEBUG", + "INTERFACE_INCLUDE_DIRECTORIES", + "INTERFACE_LINK_LIBRARIES", + "INTERFACE_COMPILE_OPTIONS", + ] + + tmpKeyword = None + for arg in _args[2:]: if arg in keywords: - t=target[arg] + tmpKeyword = target[arg] continue - t.append(arg) + tmpKeyword.append(arg) - keywords={ + #Keywords we want to react on + keywords = { "set": parse_set, "set_target_properties": parse_set_target_properties, - } + } + RELINE = re.compile("^\s*(?P[^(]+)\s*\(\s*(?P.*)\s*\)\s*$") for line in lines: m = RELINE.match(line) @@ -67,54 +100,90 @@ class Library: - def __init__(self, name): - self.name = name - self.p = None + def __init__(self, name: str) -> None: + + # name of the library + self.name = name # type: str + + # The raw cmake Parser output, available for debug porpuse + # see cmake_parser function for the return value + self.__parser_output = None # type: Union[Dict, None] - def __repr__(self): - return "".format(self=self) + # version of the library + # created/documented within runCMake function - def runCMake(self): + # targets the targets of the libary ( existing so files) + # created/documented within runCMake function + + + def __repr__(self) -> str: + return "".format(self=self) # replace with f-String in python 3.6 + + def runCMake(self) -> None: + """Create a CMakeLists.txt to detect the headers, version and library path""" with tempfile.TemporaryDirectory() as d: - with open(d+"/CMakeLists.txt","w") as f: - f.write("find_package({self.name} CONFIG REQUIRED)\n".format(self=self)) + + # Create a CMakeLists.txt that depends on the requested library + with (pathlib.Path(d)/"CMakeLists.txt").open("w") as f: + f.write("find_package({self.name} CONFIG REQUIRED)\n".format(self=self)) # replace with f-String in python 3.6 + proc = subprocess.Popen(['cmake', '.', '--trace-expand'], cwd=d, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) - self.mlines = [] - self.libs=defaultdict(dict) + + + # search only for lines that are the output of the specifc cmake files + self.__mlines = [] # type: List[str] + # cmake prefixes outout with the name of the file, filter only lines with interessting files + retarget=re.compile('.*/{self.name}(Targets[^/]*|Config[^/]*)\.cmake\(\d+\):\s*(.*)$'.format(self=self)) # replace with f-String in python 3.6 for line in proc.stderr: theLine = line.decode("utf-8") - m = re.match('.*/{self.name}(Targets[^/]*|Config[^/]*)\.cmake\(\d+\):\s*(.*)$'.format(self=self), theLine) + m = retarget.match(theLine) if m: mline = m.group(2) - self.mlines.append(mline) + self.__mlines.append(mline) + + self.__parser_output = cmake_parser(self.__mlines) - self.p = cmake_parser(self.mlines) + # version of the library + self.version = self.__parser_output["variables"]["PACKAGE_VERSION"] # type: str - self.version = self.p["variables"]["PACKAGE_VERSION"] - self.targets = {} + # targets the targets of the libary ( existing so files) + # a dict with keys, SONAME = the SONAME of the lib + # path = path of the library + # include_dirs = the header files for the library + self.targets = {} # type: Dict - def inclDirs(args): - d = [] + def inclDirs(args: List[str]) -> List[str]: + """ cmake using ";" to seperate different paths + split the paths and make a unique list of all paths (do not add paths multiple times) + """ + d = [] # type: List[str] for arg in args: d += arg.split(";") return d - for t,value in self.p["targets"].items(): - target={"SONAME": re.search("\.([\d]*)$",value["IMPORTED_SONAME_DEBUG"][0]).group(1), + for t,value in self.__parser_output["targets"].items(): + target = { + "SONAME": re.search("\.([\d]*)$",value["IMPORTED_SONAME_DEBUG"][0]).group(1), "path": value["IMPORTED_LOCATION_DEBUG"][0], "include_dirs": inclDirs(value["INTERFACE_INCLUDE_DIRECTORIES"]), - } + } self.targets[t]=target - def createABIDump(self): - if not self.p: + def createABIDump(self) -> None: + """run abi-compliance-checker (acc) to create a ABIDump tar gz + + First we need to construct a input file for acc, see xml variable. + After that we can run acc with the constructed file. + """ + if not self.__parser_output: self.runCMake() version = self.version - headers = [] - libs = [] + headers = [] # type: List[str] + libs = [] # type: List[str] for target in self.targets.values(): for i in target['include_dirs']: + # ignore general folders, as there are no lib specific headers are placed if i == '/usr/include' or i.endswith("/KF5"): continue if not i in headers: @@ -130,26 +199,33 @@ {libs} -""".format(version=version, headers="\n".join(headers), libs="\n".join(libs)) - with open("{version}.xml".format(version=version),"w") as f: +""".format(version=version, headers="\n".join(headers), libs="\n".join(libs)) # replace with f-String in Python 3.6 + with open("{version}.xml".format(version=version),"w") as f: # replace with f-String in python 3.6 f.write(xml) + + # acc is compatible for C/C++ as Qt using C++11 and -fPic we need to set the gcc settings explitly subprocess.call(["abi-compliance-checker", "-gcc-options", "-std=c++11 -fPIC", "-l", self.name, "--dump",f.name]) -libs = [] -RELINE = re.compile("^-- (Installing|Up-to-date): .*/([^/]*)Config\.cmake$") +# search in buildlog for the Installing/Up-to-date lines where we installnig the Config.cmake files. +# with this we get a complete List of installed libraries. + +#List of all libraries +libs = [] +reline = re.compile("^-- (Installing|Up-to-date): .*/([^/]*)Config\.cmake$") with open(arguments.buildlog) as f: for line in f.readlines(): - m = RELINE.match(line) + m = reline.match(line) if m: lib = Library(m.group(2)) libs.append(lib) - - # Initialize the archive manager ourArchive = Packages.Archive(arguments.environment, 'ABIReference', usingCache = False) for lib in libs: lib.createABIDump() - ourArchive.storePackage(lib.name, "abi_dumps/{name}/{name}_{version}.abi.tar.gz".format(name=lib.name,version=lib.version), max([t['SONAME'] for t in lib.targets.values()])) + + fileName = "abi_dumps/{name}/{name}_{version}.abi.tar.gz".format(name=lib.name,version=lib.version) # can replaced with f-String in python 3.6 + srcRevision = max([t['SONAME'] for t in lib.targets.values()]) # a more hackish way, to save the SONAME in the metadata + ourArchive.storePackage(lib.name, fileName, srcRevision)