diff --git a/helpers/create-abi-dump.py b/helpers/create-abi-dump.py
index 4fa7108..214e743 100755
--- a/helpers/create-abi-dump.py
+++ b/helpers/create-abi-dump.py
@@ -1,373 +1,372 @@
#!/usr/bin/python3
import argparse
import logging
import os
import pathlib
import re
import subprocess
import sys
import tempfile
-import yaml
from collections import defaultdict
from typing import Dict, List, Union, Set
from helperslib import CommonUtils, ToolingSettings
from helperslib import Packages, EnvironmentHandler
# Make sure logging is ready to go
logging.basicConfig(level=logging.DEBUG)
ACCXMLTMPL = """{version}
{headers}
{libs}
{skipIncludePaths}
{additionalIncludes}
{gccOptions}
"""
def cmake_parser(lines: List) -> Dict:
"""A small cmake parser, if you search for a better solution think about using
a proper one based on ply.
see https://salsa.debian.org/qt-kde-team/pkg-kde-jenkins/blob/master/hooks/prepare/cmake_update_deps
But in our case we are only interested in two keywords and do not need many features.
we return a dictonary with keywords and targets.
set(VAR "123")
-> variables["VAR"]="123"
set_target_properties(TARGET PROPERTIES PROP1 A B PROP2 C D)
-> targets = {
"PROP1":["A","B"],
"PROP2":["C","D"],
}
"""
variables = {} # type: Dict[str,str]
targets = defaultdict(lambda:defaultdict(list)) # type: Dict[str, Dict[str, List[str]]]
ret = {
"variables": variables,
"targets": targets,
}
def parse_set(args: str) -> None:
"""process set lines and updates the variables directory:
set(VAR 1.2.3) -> args = ["VAR", "1.2.3"]
and we set variable["VAR"] = "1.2.3"
"""
_args = args.split()
if len(_args) == 2:
name, value = _args
variables[name] = value
def parse_set_target_properties(args: str) -> None:
"""process set_target_properties cmake lines and update the targets directory
all argiments of set_target_properties are given in the args parameter as list.
as cmake using keyword val1 val2 we need to save the keyword so long we detect
a next keyword.
args[0] is the target we want to update
args[1] must be PROPERTIES
"""
name, properties, *values = args.split()
target = targets[name]
if not properties == "PROPERTIES":
logging.warning("unknown line: %s"%(args))
# Known set_target_properties keywords
keywords = [
"IMPORTED_LINK_DEPENDENT_LIBRARIES_DEBUG",
"IMPORTED_LOCATION_DEBUG",
"IMPORTED_SONAME_DEBUG",
"INTERFACE_INCLUDE_DIRECTORIES",
"INTERFACE_LINK_LIBRARIES",
"INTERFACE_COMPILE_OPTIONS",
"INTERFACE_COMPILE_DEFINITIONS",
"IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG",
"IMPORTED_CONFIGURATIONS",
"DOXYGEN_TAGFILE",
]
tmpKeyword = None
for arg in values:
if arg in keywords:
tmpKeyword = target[arg]
continue
tmpKeyword.append(arg)
#Keywords we want to react on
keywords = {
"set": parse_set,
"set_target_properties": parse_set_target_properties,
}
RELINE = re.compile("^\s*(?P[^(]+)\s*\(\s*(?P.*)\s*\)\s*$")
for line in lines:
m = RELINE.match(line)
if m and m.group('keyword') in keywords:
keywords[m.group('keyword')](m.group('args'))
return ret
# Wrapper class to represent a library we have found
# This class stores information on the library in question and assists in extracting information concerning it
class Library:
# Make sure we initialize everything we are going to need
def __init__(self, name: str, accSettings: Dict) -> None:
# name of the library
self.name = name # type: str
self.accSettings = accSettings # type: Dict
# The raw cmake Parser output, available for debugging purposes
# see cmake_parser function for the return value
self.__parser_output = None # type: Union[Dict, None]
# Provide a helpful description of the object (to ease debugging)
def __repr__(self) -> str:
return "".format(self=self) # replace with f-String in python 3.6
# Execute CMake to gather the information we need
def runCMake(self, runtimeEnvironment) -> None:
"""Create a CMakeLists.txt to detect the headers, version and library path"""
# Prepare to gather the information we need
self.__mlines = [] # type: List[str]
# To avoid contaminating the directory we are being run in, make sure we are in a temporary directory
# This will also allow us to succeed if we are run from the build direcotry
with tempfile.TemporaryDirectory() as d:
# Create an appropriate CMakeLists.txt which searches for the library in question
cmakeFile = (pathlib.Path(d)/"CMakeLists.txt")
cmakeFile.write_text("find_package({self.name} CONFIG REQUIRED)\n".format(self=self)) # replace with f-String in python 3.6
# Now run CMake and ask it to process the CMakeLists.txt file we just generated
# We want it to run in trace mode so we can examine the log to extract the information we need
proc = subprocess.Popen(['cmake', '.', '--trace-expand'], cwd=d, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, env=runtimeEnvironment)
# cmake prefixes outout with the name of the file, filter only lines with interessting files
retarget = re.compile( '.*/{self.name}/[^/]*(Targets[^/]*|Config[^/]*)\.cmake\(\d+\):\s*(.*)$'.format(self=self) ) # replace with f-String in python 3.6
# Start processing the output of CMake, one line at a time
for line in proc.stderr:
# Make sure it is UTF-8 formatted
theLine = line.decode("utf-8")
# Did we find a CMake line we were interested in?
m = retarget.match(theLine)
if m:
# Extract the information from that and store it for further processing
mline = m.group(2)
self.__mlines.append(mline)
# Process the information we've now gathered
self.__parser_output = cmake_parser(self.__mlines)
self.parser_output = self.__parser_output
self.mlines = self.__mlines
# Extract the version number of the library for easier use
self.version = self.__parser_output["variables"].get("PACKAGE_VERSION") # type: str
# targets the targets of the libary ( existing so files)
# a dict with keys, SONAME = the SONAME of the lib
# path = path of the library
# include_dirs = the header files for the library
self.targets = {} # type: Dict
# Helper Function to parse CMake formatted include directory lists
def parseIncludeDirs(args: List[str]) -> List[str]:
""" cmake using ";" to seperate different paths
split the paths and make a unique list of all paths (do not add paths multiple times)
"""
d = [] # type: List[str]
for arg in args:
d += arg.split(";")
return d
# Process the various targets our parser found
for t,value in self.__parser_output["targets"].items():
# Particularly, we want to extract:
# Library names (sonames)
# The path to the CMake library package
# Any include directories specified by the CMake library package
try:
target = {
"SONAME": re.search("\.([\d]*)$",value["IMPORTED_SONAME_DEBUG"][0]).group(1),
"path": value["IMPORTED_LOCATION_DEBUG"][0],
"include_dirs": parseIncludeDirs(value["INTERFACE_INCLUDE_DIRECTORIES"]),
}
self.targets[t]=target
except IndexError:
pass
def createABIDump(self, runtimeEnvironment=None) -> None:
"""run abi-compliance-checker (acc) to create a ABIDump tar gz
First we need to construct a input file for acc, see xml variable.
After that we can run acc with the constructed file.
"""
# Make sure we have a valid runtime environment for CMake and abi-compliance-checker
if runtimeEnvironment is None:
runtimeEnvironment = os.environ
# If we haven't yet run CMake, do so
# Otherwise we won't have anything to give to abi-compliance-checker
if not self.__parser_output:
self.runCMake(runtimeEnvironment)
# Start preparations to run abi-compliance-checker
# Gather the information we'll need to write the XML configuration file it uses
version = self.version
headers = set() # type: Set[str]
libs = set() # type: Set[str]
skipIncludePaths = set(self.accSettings['skip_include_paths']) # type: Set[str]
additionalIncludes = set(self.accSettings['add_include_paths']) # type: Set[str]
gccOptions = set(self.accSettings['gcc_options']) # type: Set[str]
# list of general include directories
prefixHeaders = [os.path.abspath(i) for i in buildEnvironment.get('CMAKE_PREFIX_PATH').split(":")]
noHeaders = set(prefixHeaders)
noHeaders |= set([os.path.join(i,"include") for i in prefixHeaders])
# From the target information we previously collected...
# Grab the list of libraries and include headers for abi-compliance-checker
for target in self.targets.values():
# Check each include directory to see if we need to add it....
for i in target['include_dirs']:
abspath = os.path.abspath(i)
# ignore general folders, as there are no lib specific headers are placed
if abspath in noHeaders or abspath.endswith("/KF5"):
additionalIncludes.add(abspath)
continue
# Otherwise, if we don't already have it - add it to the list!
headers.add(abspath)
# If the library path isn't in the list, then we should add it to the list
libs.add(target['path'])
# Now we can go ahead and generate the XML file for abi-compliance-checker
def _(l):
return "\n ".join(l)
xml = ACCXMLTMPL.format(version=version,
headers=_(headers),
libs=_(libs),
additionalIncludes=_(additionalIncludes),
skipIncludePaths=_(skipIncludePaths),
gccOptions=_(gccOptions),
)
# Write the generated XML out to a file to pass to abi-compliance-checker
# We will give this to abi-compliance-checker using it's --dump parameter
with open("acc/{libname}-{version}.xml".format(libname=self.name, version=version),"w") as f: # replace with f-String in python 3.6
f.write(xml)
# acc is compatible for C/C++ (but --lang C++ doesn't remove the warning about C++ only settings).
cmd = ["abi-compliance-checker", "--lang", "C++", "-l", self.name, "--dump", f.name]
logging.debug(" ".join(cmd))
subprocess.check_call(cmd, env=runtimeEnvironment)
# Parse the command line arguments we've been given
parser = argparse.ArgumentParser(description='Utility to create abi checker tarballs.')
parser.add_argument('--project', type=str, required=True)
parser.add_argument('--branchGroup', type=str, required=True)
parser.add_argument('--buildLog', type=str, required=True)
parser.add_argument('--environment', type=str, required=True)
parser.add_argument('--platform', type=str, required=True)
parser.add_argument('--usingInstall', type=str, required=True)
arguments = parser.parse_args()
# Make sure we have an environment ready for executing commands
buildEnvironment = EnvironmentHandler.generateFor( installPrefix=arguments.usingInstall )
# get acc settings
localMetadataPath = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'abi-compliance-checker.yaml' )
accSettings = ToolingSettings.Loader( arguments.project, arguments.platform )
accSettings.loadSpecificationFile( localMetadataPath )
# Get ready to start searching for libraries
foundLibraries = []
# Search in the build log for the Installing/Up-to-date lines where we install the Config.cmake files.
# From this we get a complete List of installed libraries.
cmakeConfig = re.compile("-- (Installing|Up-to-date): .*/([^/]*)Config\.cmake$")
with open(arguments.buildLog, encoding='utf-8') as log:
for line in log.readlines():
match = cmakeConfig.search(line)
if match:
foundLibrary = Library( match.group(2), accSettings )
foundLibraries.append(foundLibrary)
# Initialize the archive manager
ourArchive = Packages.Archive(arguments.environment, 'ABIReference', usingCache = False, contentsSuffix = ".abidump")
# Determine which SCM revision we are storing
# This will be embedded into the package metadata which might help someone doing some debugging
# GIT_COMMIT is set by Jenkins Git plugin, so we can rely on that for most of our builds
scmRevision = ''
if os.getenv('GIT_COMMIT') != '':
scmRevision = os.getenv('GIT_COMMIT')
if not scmRevision:
scmRevision = subprocess.check_output(["git", "log", "--format=%H", "-n 1", "HEAD"]).strip().decode()
# Create diretory for string acc input xml
try:
os.mkdir("acc")
except FileExistsError:
pass
# Check every libraries ABI and do not fail, if one is not fine.
# Safe the overall retval state
retval = 0
# Now we generate the ABI dumps for every library we have found
for library in foundLibraries:
logging.info("Start building ABI dump for {name}".format(name=library.name))
try:
#run CMake for library
library.runCMake( runtimeEnvironment=buildEnvironment )
if not library.version:
logging.warning("{name} has no version: skipping.".format(name=library.name))
continue
# Create the ABI Dump for this library
library.createABIDump( runtimeEnvironment=buildEnvironment )
# Determine where the ABI Dump archive is located
# This location is controlled by abi-compliance-checker, but follows a predictable pattern
fileName = "abi_dumps/{name}/{version}/ABI.dump".format(name=library.name,version=library.version)
extraMetadata = {
"SONAME": max([t['SONAME'] for t in library.targets.values()]), # use max because there may be more than one lib inside
"version": library.version,
"libname": library.name,
"targets": list(library.targets),
"project": arguments.project,
"branchGroup": arguments.branchGroup,
"platform": arguments.platform,
}
packageName = "{name}_{scmRevision}_{platform}".format(name=library.name, scmRevision=scmRevision, platform=arguments.platform)
ourArchive.storePackage(packageName, fileName, scmRevision, extraMetadata)
except subprocess.CalledProcessError as e:
retval = e.returncode
logging.error("abi-compliance-checker exited with {retval}".format(retval=retval))
# We had an issue with one of the ABIs
if retval != 0 and accSettings['createABIDumpFailHard']:
sys.exit("Errors detected and createABIDumpFailHard is set, so we fail hard.")