diff --git a/helpers/create-abi-dump.py b/helpers/create-abi-dump.py
index a405742..24c5446 100755
--- a/helpers/create-abi-dump.py
+++ b/helpers/create-abi-dump.py
@@ -1,373 +1,373 @@
#!/usr/bin/python3
import argparse
import logging
import os
import pathlib
import re
import subprocess
import sys
import tempfile
from collections import defaultdict
from typing import Dict, List, Union, Set
from helperslib import CommonUtils, ToolingSettings
from helperslib import Packages, EnvironmentHandler
# Make sure logging is ready to go
logging.basicConfig(level=logging.DEBUG)
+logging.getLogger("paramiko.transport").setLevel(logging.WARNING)
ACCXMLTMPL = """{version}
{headers}
{libs}
{skipIncludePaths}
{additionalIncludes}
{gccOptions}
"""
def cmake_parser(lines: List) -> Dict:
"""A small cmake parser, if you search for a better solution think about using
a proper one based on ply.
see https://salsa.debian.org/qt-kde-team/pkg-kde-jenkins/blob/master/hooks/prepare/cmake_update_deps
But in our case we are only interested in two keywords and do not need many features.
we return a dictonary with keywords and targets.
set(VAR "123")
-> variables["VAR"]="123"
set_target_properties(TARGET PROPERTIES PROP1 A B PROP2 C D)
-> targets = {
"PROP1":["A","B"],
"PROP2":["C","D"],
}
"""
variables = {} # type: Dict[str,str]
targets = defaultdict(lambda:defaultdict(list)) # type: Dict[str, Dict[str, List[str]]]
ret = {
"variables": variables,
"targets": targets,
}
def parse_set(args: str) -> None:
"""process set lines and updates the variables directory:
set(VAR 1.2.3) -> args = ["VAR", "1.2.3"]
and we set variable["VAR"] = "1.2.3"
"""
_args = args.split()
if len(_args) == 2:
name, value = _args
variables[name] = value
def parse_set_target_properties(args: str) -> None:
"""process set_target_properties cmake lines and update the targets directory
all argiments of set_target_properties are given in the args parameter as list.
as cmake using keyword val1 val2 we need to save the keyword so long we detect
a next keyword.
args[0] is the target we want to update
args[1] must be PROPERTIES
"""
name, properties, *values = args.split()
target = targets[name]
if not properties == "PROPERTIES":
logging.warning("unknown line: %s"%(args))
# Known set_target_properties keywords
keywords = [
"IMPORTED_LINK_DEPENDENT_LIBRARIES_DEBUG",
"IMPORTED_LOCATION_DEBUG",
"IMPORTED_SONAME_DEBUG",
"INTERFACE_INCLUDE_DIRECTORIES",
"INTERFACE_LINK_LIBRARIES",
"INTERFACE_COMPILE_OPTIONS",
"INTERFACE_COMPILE_DEFINITIONS",
"IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG",
"IMPORTED_CONFIGURATIONS",
"DOXYGEN_TAGFILE",
]
tmpKeyword = None
for arg in values:
if arg in keywords:
tmpKeyword = target[arg]
continue
tmpKeyword.append(arg)
#Keywords we want to react on
keywords = {
"set": parse_set,
"set_target_properties": parse_set_target_properties,
}
RELINE = re.compile("^\s*(?P[^(]+)\s*\(\s*(?P.*)\s*\)\s*$")
for line in lines:
m = RELINE.match(line)
if m and m.group('keyword') in keywords:
keywords[m.group('keyword')](m.group('args'))
return ret
# Wrapper class to represent a library we have found
# This class stores information on the library in question and assists in extracting information concerning it
class Library:
# Make sure we initialize everything we are going to need
def __init__(self, name: str, accSettings: Dict) -> None:
# name of the library
self.name = name # type: str
self.accSettings = accSettings # type: Dict
# The raw cmake Parser output, available for debugging purposes
# see cmake_parser function for the return value
self.__parser_output = None # type: Union[Dict, None]
# Provide a helpful description of the object (to ease debugging)
def __repr__(self) -> str:
return "".format(self=self) # replace with f-String in python 3.6
# Execute CMake to gather the information we need
def runCMake(self, runtimeEnvironment) -> None:
"""Create a CMakeLists.txt to detect the headers, version and library path"""
# Prepare to gather the information we need
self.__mlines = [] # type: List[str]
# To avoid contaminating the directory we are being run in, make sure we are in a temporary directory
# This will also allow us to succeed if we are run from the build direcotry
with tempfile.TemporaryDirectory() as d:
# Create an appropriate CMakeLists.txt which searches for the library in question
cmakeFile = (pathlib.Path(d)/"CMakeLists.txt")
cmakeFile.write_text("find_package({self.name} CONFIG REQUIRED)\n".format(self=self)) # replace with f-String in python 3.6
# Now run CMake and ask it to process the CMakeLists.txt file we just generated
# We want it to run in trace mode so we can examine the log to extract the information we need
proc = subprocess.Popen(['cmake', '.', '--trace-expand'], cwd=d, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, env=runtimeEnvironment)
# cmake prefixes outout with the name of the file, filter only lines with interessting files
retarget = re.compile( '.*/{self.name}/[^/]*(Targets[^/]*|Config[^/]*)\.cmake\(\d+\):\s*(.*)$'.format(self=self) ) # replace with f-String in python 3.6
# Start processing the output of CMake, one line at a time
for line in proc.stderr:
# Make sure it is UTF-8 formatted
theLine = line.decode("utf-8")
# Did we find a CMake line we were interested in?
m = retarget.match(theLine)
if m:
# Extract the information from that and store it for further processing
mline = m.group(2)
self.__mlines.append(mline)
# Process the information we've now gathered
self.__parser_output = cmake_parser(self.__mlines)
self.parser_output = self.__parser_output
self.mlines = self.__mlines
# Extract the version number of the library for easier use
self.version = self.__parser_output["variables"].get("PACKAGE_VERSION") # type: str
# targets the targets of the libary ( existing so files)
# a dict with keys, SONAME = the SONAME of the lib
# path = path of the library
# include_dirs = the header files for the library
self.targets = {} # type: Dict
# Helper Function to parse CMake formatted include directory lists
def parseIncludeDirs(args: List[str]) -> List[str]:
""" cmake using ";" to seperate different paths
split the paths and make a unique list of all paths (do not add paths multiple times)
"""
d = [] # type: List[str]
for arg in args:
d += arg.split(";")
return d
# Process the various targets our parser found
for t, value in self.__parser_output["targets"].items():
# Particularly, we want to extract:
# Library names (sonames)
# The path to the CMake library package
# Any include directories specified by the CMake library package
try:
relib= re.match("^(?P.*)\.so\.(?P.*)$", value["IMPORTED_SONAME_DEBUG"][0])
target = {
"SONAME": relib.group("SONAME"),
"path": value["IMPORTED_LOCATION_DEBUG"][0],
"include_dirs": parseIncludeDirs(value["INTERFACE_INCLUDE_DIRECTORIES"]),
}
self.targets[relib.group("name")]=target
except IndexError:
pass
def createABIDump(self, runtimeEnvironment=None) -> None:
"""run abi-compliance-checker (acc) to create a ABIDump tar gz
First we need to construct a input file for acc, see xml variable.
After that we can run acc with the constructed file.
"""
# Make sure we have a valid runtime environment for CMake and abi-compliance-checker
if runtimeEnvironment is None:
runtimeEnvironment = os.environ
# If we haven't yet run CMake, do so
# Otherwise we won't have anything to give to abi-compliance-checker
if not self.__parser_output:
self.runCMake(runtimeEnvironment)
# Start preparations to run abi-compliance-checker
# Gather the information we'll need to write the XML configuration file it uses
version = self.version
headers = set() # type: Set[str]
libs = set() # type: Set[str]
skipIncludePaths = set(self.accSettings['skip_include_paths']) # type: Set[str]
additionalIncludes = set(self.accSettings['add_include_paths']) # type: Set[str]
gccOptions = set(self.accSettings['gcc_options']) # type: Set[str]
# list of general include directories
prefixHeaders = [os.path.abspath(i) for i in buildEnvironment.get('CMAKE_PREFIX_PATH').split(":")]
noHeaders = set(prefixHeaders)
noHeaders |= set([os.path.join(i,"include") for i in prefixHeaders])
# From the target information we previously collected...
# Grab the list of libraries and include headers for abi-compliance-checker
for target in self.targets.values():
# Check each include directory to see if we need to add it....
for i in target['include_dirs']:
abspath = os.path.abspath(i)
# ignore general folders, as there are no lib specific headers are placed
if abspath in noHeaders or abspath.endswith("/KF5"):
additionalIncludes.add(abspath)
continue
# Otherwise, if we don't already have it - add it to the list!
headers.add(abspath)
# If the library path isn't in the list, then we should add it to the list
libs.add(target['path'])
# Now we can go ahead and generate the XML file for abi-compliance-checker
def _(l):
return "\n ".join(l)
xml = ACCXMLTMPL.format(version=version,
headers=_(headers),
libs=_(libs),
additionalIncludes=_(additionalIncludes),
skipIncludePaths=_(skipIncludePaths),
gccOptions=_(gccOptions),
)
# Write the generated XML out to a file to pass to abi-compliance-checker
# We will give this to abi-compliance-checker using it's --dump parameter
with open("acc/{libname}-{version}.xml".format(libname=self.name, version=version),"w") as f: # replace with f-String in python 3.6
f.write(xml)
# acc is compatible for C/C++ (but --lang C++ doesn't remove the warning about C++ only settings).
cmd = ["abi-compliance-checker", "--lang", "C++", "-l", self.name, "--dump", f.name]
logging.debug(" ".join(cmd))
subprocess.check_call(cmd, env=runtimeEnvironment)
# Parse the command line arguments we've been given
parser = argparse.ArgumentParser(description='Utility to create abi checker tarballs.')
parser.add_argument('--project', type=str, required=True)
parser.add_argument('--branchGroup', type=str, required=True)
parser.add_argument('--buildLog', type=str, required=True)
parser.add_argument('--environment', type=str, required=True)
parser.add_argument('--platform', type=str, required=True)
parser.add_argument('--usingInstall', type=str, required=True)
arguments = parser.parse_args()
# Make sure we have an environment ready for executing commands
buildEnvironment = EnvironmentHandler.generateFor( installPrefix=arguments.usingInstall )
-
# get acc settings
localMetadataPath = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'abi-compliance-checker.yaml' )
accSettings = ToolingSettings.Loader( arguments.project, arguments.platform )
accSettings.loadSpecificationFile( localMetadataPath )
# Get ready to start searching for libraries
foundLibraries = []
# Search in the build log for the Installing/Up-to-date lines where we install the Config.cmake files.
# From this we get a complete List of installed libraries.
cmakeConfig = re.compile("-- (Installing|Up-to-date): .*/([^/]*)Config\.cmake$")
with open(arguments.buildLog, encoding='utf-8') as log:
for line in log.readlines():
match = cmakeConfig.search(line)
if match:
foundLibrary = Library( match.group(2), accSettings )
foundLibraries.append(foundLibrary)
# Initialize the archive manager
ourArchive = Packages.Archive(arguments.environment, 'ABIReference', usingCache = False, contentsSuffix = ".abidump")
# Determine which SCM revision we are storing
# This will be embedded into the package metadata which might help someone doing some debugging
# GIT_COMMIT is set by Jenkins Git plugin, so we can rely on that for most of our builds
scmRevision = ''
if os.getenv('GIT_COMMIT') != '':
scmRevision = os.getenv('GIT_COMMIT')
if not scmRevision:
scmRevision = subprocess.check_output(["git", "log", "--format=%H", "-n 1", "HEAD"]).strip().decode()
# Create diretory for string acc input xml
try:
os.mkdir("acc")
except FileExistsError:
pass
# Check every libraries ABI and do not fail, if one is not fine.
# Safe the overall retval state
retval = 0
# Now we generate the ABI dumps for every library we have found
for library in foundLibraries:
logging.info("Start building ABI dump for {name}".format(name=library.name))
try:
#run CMake for library
library.runCMake( runtimeEnvironment=buildEnvironment )
if not library.version:
logging.warning("{name} has no version: skipping.".format(name=library.name))
continue
# Create the ABI Dump for this library
library.createABIDump( runtimeEnvironment=buildEnvironment )
# Determine where the ABI Dump archive is located
# This location is controlled by abi-compliance-checker, but follows a predictable pattern
fileName = "abi_dumps/{name}/{version}/ABI.dump".format(name=library.name,version=library.version)
extraMetadata = {
"accMetadataVersion": 2,
"version": library.version,
"cmakePackage": library.name,
"targets": {n:l["SONAME"] for n,l in library.targets.items()},
"project": arguments.project,
"branchGroup": arguments.branchGroup,
"platform": arguments.platform,
}
packageName = "{name}_{scmRevision}_{platform}".format(name=library.name, scmRevision=scmRevision, platform=arguments.platform)
ourArchive.storePackage(packageName, fileName, scmRevision, extraMetadata)
except subprocess.CalledProcessError as e:
retval = e.returncode
logging.error("abi-compliance-checker exited with {retval}".format(retval=retval))
# We had an issue with one of the ABIs
if retval != 0 and accSettings['createABIDumpFailHard']:
sys.exit("Errors detected and createABIDumpFailHard is set, so we fail hard.")