Differential D11914 Diff 31273 helpers/create-abi-bump.py

Changeset View

Standalone View

View Options

helpers/create-abi-bump.py

1#!/usr/bin/python3 1​#!/usr/bin/python3
2 2​
3import argparse 3​import argparse
4from collections import defaultdict 4​from collections import defaultdict
5import os 5​import os
6​import pathlib
6import re 7​import re
7import subprocess 8​import subprocess
8import tempfile 9​import tempfile
10​from typing import Dict, List, Union
9 11​
10from helperslib import Packages 12​from helperslib import Packages
11 13​
12import logging 14​import logging
13logging.basicConfig(level=logging.DEBUG) 15​logging.basicConfig(level=logging.DEBUG)
14 16​
15# Parse the command line arguments we've been given 17​# Parse the command line arguments we've been given
16parser = argparse.ArgumentParser(description='Utility to create abi checker tarballs.') 18​parser = argparse.ArgumentParser(description='Utility to create abi checker tarballs.')
17parser.add_argument('--buildlog', type=str, required=True) 19​parser.add_argument('--buildlog', type=str, required=True)
18parser.add_argument('--environment', type=str, required=True) 20​parser.add_argument('--environment', type=str, required=True)
19arguments = parser.parse_args() 21​arguments = parser.parse_args()
20 22​
21def cmake_parser(lines): 23​def cmake_parser(lines: List) -> Dict:
24​ """A small cmake parser, if you search for a better solution think about using
25​ a propper one based on ply.
26​ see https://salsa.debian.org/qt-kde-team/pkg-kde-jenkins/blob/master/hooks/prepare/cmake_update_deps
27​
28​ But in our case we are only interessed in two keywords and do not need many features.
29​ we return a dictonary with keywords and targets.
30​ set(VAR "123")
31​ -> variables["VAR"]="123"
32​ set_target_properties(TARGET PROPERTIES PROP1 A B PROP2 C D)
33​ -> targets = {
34​ "PROP1":["A","B"],
35​ "PROP2":["C","D"],
36​ }
37​ """
38​
39​ variables = {} # type: Dict[str,str]
40​ targets = defaultdict(lambda:defaultdict(list)) # type: Dict[str, Dict[str, List[str]]]
41​
22 ret = { 42​ ret = {
23 "variables":{}, 43​ "variable": variables,
24 "targets":defaultdict(lambda:defaultdict(list)) 44​ "targets": targets,
25 } 45​ }
26 variables = ret['variables'] 46​
27 targets = ret['targets'] 47​ def parse_set(args: str) -> None:
28 48​ """process set lines and updates the variables directory:
29 def parse_set(args): 49​ set(VAR 1.2.3) -> args = ["VAR", "1.2.3"]
30 args = args.split() 50​ and we set variable["VAR"] = "1.2.3"
31 if len(args) == 2: 51​ """
32 variables[args[0]] = args[1] 52​ _args = args.split()
33 53​ if len(_args) == 2:
34 def parse_set_target_properties(args): 54​ variables[_args[0]] = _args[1]
35 args = args.split() 55​
36 target = targets[args[0]] 56​ def parse_set_target_properties(args: str) -> None:
37 if not args[1] == "PROPERTIES": 57​ """process set_target_properties cmake lines and update the targets directory
58​ all argiments of set_target_properties are given in the args parameter as list.
59​ as cmake using keyword val1 val2 we need to save the keyword so long we detect
60​ a next keyword.
61​
62​ args[0] is the target we want to update
63​ args[1] must be PROPERTIES
64​ """
65​ _args = args.split()
66​ target = targets[_args[0]]
67​ if not _args[1] == "PROPERTIES":
38 logging.warning("unknown line: %s"%(args)) 68​ logging.warning("unknown line: %s"%(args))
39 69​
40 70​ # Known set_target_properties keywords
41 keywords=["IMPORTED_LINK_DEPENDENT_LIBRARIES_DEBUG", 71​ keywords = [
42 "IMPORTED_LOCATION_DEBUG", 72​ "IMPORTED_LINK_DEPENDENT_LIBRARIES_DEBUG",
43 "IMPORTED_SONAME_DEBUG", 73​ "IMPORTED_LOCATION_DEBUG",
44 "INTERFACE_INCLUDE_DIRECTORIES", 74​ "IMPORTED_SONAME_DEBUG",
45 "INTERFACE_LINK_LIBRARIES", 75​ "INTERFACE_INCLUDE_DIRECTORIES",
46 "INTERFACE_COMPILE_OPTIONS", 76​ "INTERFACE_LINK_LIBRARIES",
47 ] 77​ "INTERFACE_COMPILE_OPTIONS",
48 78​ ]
49 t = None 79​
50 for arg in args[2:]: 80​ tmpKeyword = None
81​ for arg in _args[2:]:
51 if arg in keywords: 82​ if arg in keywords:
52 t=target[arg] 83​ tmpKeyword = target[arg]
53 continue 84​ continue
54 t.append(arg) 85​ tmpKeyword.append(arg)
55 86​
56 keywords={ 87​ #Keywords we want to react on
88​ keywords = {
57 "set": parse_set, 89​ "set": parse_set,
58 "set_target_properties": parse_set_target_properties, 90​ "set_target_properties": parse_set_target_properties,
59 } 91​ }
92​
60 RELINE = re.compile("^\s*(?P<keyword>[^(]+)\s*\(\s*(?P<args>.*)\s*\)\s*$") 93​ RELINE = re.compile("^\s*(?P<keyword>[^(]+)\s*\(\s*(?P<args>.*)\s*\)\s*$")
61 for line in lines: 94​ for line in lines:
62 m = RELINE.match(line) 95​ m = RELINE.match(line)
63 if m and m.group('keyword') in keywords: 96​ if m and m.group('keyword') in keywords:
64 keywords[m.group('keyword')](m.group('args')) 97​ keywords[m.group('keyword')](m.group('args'))
65 98​
66 return ret 99​ return ret
67 100​
68 101​
69class Library: 102​class Library:
70 def __init__(self, name): 103​ def __init__(self, name: str) -> None:
71 self.name = name 104​
72 self.p = None 105​ # name of the library
106​ self.name = name # type: str
107​
108​ # The raw cmake Parser output, available for debug porpuse
109​ # see cmake_parser function for the return value
110​ self.__parser_output = None # type: Union[Dict, None]
73 111​
74 def __repr__(self): 112​ # version of the library
75 return "<Library \"{self.name}\">".format(self=self) 113​ # created/documented within runCMake function
76 114​
77 def runCMake(self): 115​ # targets the targets of the libary ( existing so files)
116​ # created/documented within runCMake function
117​
118​
119​ def __repr__(self) -> str:
120​ return "<Library \"{self.name}\">".format(self=self) # replace with f-String in python 3.6
121​
122​ def runCMake(self) -> None:
123​ """Create a CMakeLists.txt to detect the headers, version and library path"""
78 with tempfile.TemporaryDirectory() as d: 124​ with tempfile.TemporaryDirectory() as d:
79 with open(d+"/CMakeLists.txt","w") as f: 125​
80 f.write("find_package({self.name} CONFIG REQUIRED)\n".format(self=self)) 126​ # Create a CMakeLists.txt that depends on the requested library
127​ with (pathlib.Path(d)/"CMakeLists.txt").open("w") as f:
128​ f.write("find_package({self.name} CONFIG REQUIRED)\n".format(self=self)) # replace with f-String in python 3.6
129​
81 proc = subprocess.Popen(['cmake', '.', '--trace-expand'], cwd=d, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) 130​ proc = subprocess.Popen(['cmake', '.', '--trace-expand'], cwd=d, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
82 self.mlines = [] 131​
83 self.libs=defaultdict(dict) 132​
133​ # search only for lines that are the output of the specifc cmake files
134​ self.__mlines = [] # type: List[str]
135​ # cmake prefixes outout with the name of the file, filter only lines with interessting files
136​ retarget=re.compile('.*/{self.name}(Targets[^/]*|Config[^/]*)\.cmake\(\d+\):\s*(.*)$'.format(self=self)) # replace with f-String in python 3.6
84 for line in proc.stderr: 137​ for line in proc.stderr:
85 theLine = line.decode("utf-8") 138​ theLine = line.decode("utf-8")
86 m = re.match('.*/{self.name}(Targets[^/]*|Config[^/]*)\.cmake\(\d+\):\s*(.*)$'.format(self=self), theLine) 139​ m = retarget.match(theLine)
87 if m: 140​ if m:
88 mline = m.group(2) 141​ mline = m.group(2)
89 self.mlines.append(mline) 142​ self.__mlines.append(mline)
143​
144​ self.__parser_output = cmake_parser(self.__mlines)
90 145​
91 self.p = cmake_parser(self.mlines) 146​ # version of the library
147​ self.version = self.__parser_output["variables"]["PACKAGE_VERSION"] # type: str
92 148​
93 self.version = self.p["variables"]["PACKAGE_VERSION"] 149​ # targets the targets of the libary ( existing so files)
94 self.targets = {} 150​ # a dict with keys, SONAME = the SONAME of the lib
151​ # path = path of the library
152​ # include_dirs = the header files for the library
153​ self.targets = {} # type: Dict
95 154​
96 def inclDirs(args): 155​ def inclDirs(args: List[str]) -> List[str]:
97 d = [] 156​ """ cmake using ";" to seperate different paths
157​ split the paths and make a unique list of all paths (do not add paths multiple times)
158​ """
159​ d = [] # type: List[str]
98 for arg in args: 160​ for arg in args:
99 d += arg.split(";") 161​ d += arg.split(";")
100 return d 162​ return d
101 163​
102 for t,value in self.p["targets"].items(): 164​ for t,value in self.__parser_output["targets"].items():
103 target={"SONAME": re.search("\.([\d]*)$",value["IMPORTED_SONAME_DEBUG"][0]).group(1), 165​ target = {
166​ "SONAME": re.search("\.([\d]*)$",value["IMPORTED_SONAME_DEBUG"][0]).group(1),
104 "path": value["IMPORTED_LOCATION_DEBUG"][0], 167​ "path": value["IMPORTED_LOCATION_DEBUG"][0],
105 "include_dirs": inclDirs(value["INTERFACE_INCLUDE_DIRECTORIES"]), 168​ "include_dirs": inclDirs(value["INTERFACE_INCLUDE_DIRECTORIES"]),
106 } 169​ }
107 self.targets[t]=target 170​ self.targets[t]=target
108 171​
109 def createABIDump(self): 172​ def createABIDump(self) -> None:
110 if not self.p: 173​ """run abi-compliance-checker (acc) to create a ABIDump tar gz
174​
175​ First we need to construct a input file for acc, see xml variable.
176​ After that we can run acc with the constructed file.
177​ """
178​ if not self.__parser_output:
111 self.runCMake() 179​ self.runCMake()
112 180​
113 version = self.version 181​ version = self.version
114 headers = [] 182​ headers = [] # type: List[str]
115 libs = [] 183​ libs = [] # type: List[str]
116 for target in self.targets.values(): 184​ for target in self.targets.values():
117 for i in target['include_dirs']: 185​ for i in target['include_dirs']:
186​ # ignore general folders, as there are no lib specific headers are placed
118 if i == '/usr/include' or i.endswith("/KF5"): 187​ if i == '/usr/include' or i.endswith("/KF5"):
119 continue 188​ continue
120 if not i in headers: 189​ if not i in headers:
121 headers.append(i) 190​ headers.append(i)
122 if not target['path'] in libs: 191​ if not target['path'] in libs:
123 libs.append(target['path']) 192​ libs.append(target['path'])
124 193​
125 xml = """ 194​ xml = """
126<version>{version}</version> 195​<version>{version}</version>
127<headers> 196​<headers>
128{headers} 197​{headers}
129</headers> 198​</headers>
130<libs> 199​<libs>
131{libs} 200​{libs}
132</libs> 201​</libs>
133""".format(version=version, headers="\n".join(headers), libs="\n".join(libs)) 202​""".format(version=version, headers="\n".join(headers), libs="\n".join(libs)) # replace with f-String in Python 3.6
134 with open("{version}.xml".format(version=version),"w") as f: 203​ with open("{version}.xml".format(version=version),"w") as f: # replace with f-String in python 3.6
135 f.write(xml) 204​ f.write(xml)
205​
206​ # acc is compatible for C/C++ as Qt using C++11 and -fPic we need to set the gcc settings explitly
136 subprocess.call(["abi-compliance-checker", "-gcc-options", "-std=c++11 -fPIC", "-l", self.name, "--dump",f.name]) 207​ subprocess.call(["abi-compliance-checker", "-gcc-options", "-std=c++11 -fPIC", "-l", self.name, "--dump",f.name])
137 208​
138libs = [] ​
139 209​
140RELINE = re.compile("^-- (Installing|Up-to-date): .*/([^/]*)Config\.cmake$") 210​# search in buildlog for the Installing/Up-to-date lines where we installnig the <name>Config.cmake files.
211​# with this we get a complete List of installed libraries.
212​
213​#List of all libraries
214​libs = []
215​reline = re.compile("^-- (Installing|Up-to-date): .*/([^/]*)Config\.cmake$")
141with open(arguments.buildlog) as f: 216​with open(arguments.buildlog) as f:
142 for line in f.readlines(): 217​ for line in f.readlines():
143 m = RELINE.match(line) 218​ m = reline.match(line)
144 if m: 219​ if m:
145 lib = Library(m.group(2)) 220​ lib = Library(m.group(2))
146 libs.append(lib) 221​ libs.append(lib)
147 222​
148 ​
149 ​
150# Initialize the archive manager 223​# Initialize the archive manager
151ourArchive = Packages.Archive(arguments.environment, 'ABIReference', usingCache = False) 224​ourArchive = Packages.Archive(arguments.environment, 'ABIReference', usingCache = False)
152 225​
153for lib in libs: 226​for lib in libs:
154 lib.createABIDump() 227​ lib.createABIDump()
155 ourArchive.storePackage(lib.name, "abi_dumps/{name}/{name}_{version}.abi.tar.gz".format(name=lib.name,version=lib.version), max([t['SONAME'] for t in lib.targets.values()])) 228​
229​ fileName = "abi_dumps/{name}/{name}_{version}.abi.tar.gz".format(name=lib.name,version=lib.version) # can replaced with f-String in python 3.6
230​ srcRevision = max([t['SONAME'] for t in lib.targets.values()]) # a more hackish way, to save the SONAME in the metadata
231​ ourArchive.storePackage(lib.name, fileName, srcRevision)