Changeset View
Standalone View
helpers/check-abi.py
- This file was added.
Property | Old Value | New Value |
---|---|---|
File Mode | null | 100755 |
1 | #!/usr/bin/python3 | ||||
---|---|---|---|---|---|
2 | | ||||
3 | # check the ABIs against a earlier state. | ||||
4 | # It is designed to run after create-abi-dump has created the abidump already. | ||||
5 | | ||||
6 | # abi-compliance-checker creates a html file with the report. | ||||
7 | # it can be multiple libraries in one repository (e.g messagelib) | ||||
8 | # so we have multiple html files for one repository | ||||
9 | # in order to store them as artifacts in Jenkins add: | ||||
10 | # | ||||
11 | # archiveArtifacts artifacts: 'compat_reports/*_compat_reports.html', onlyIfSuccessful: false | ||||
12 | | ||||
13 | import os | ||||
14 | import logging | ||||
15 | import argparse | ||||
16 | import subprocess | ||||
17 | import sys | ||||
18 | | ||||
19 | from helperslib import Packages | ||||
20 | from helperslib.Version import Version | ||||
21 | | ||||
22 | class Library: | ||||
23 | def __init__(self, packageName, library): | ||||
24 | self.packgeName = packageName | ||||
25 | self.library = library | ||||
26 | self.candidates = [] | ||||
27 | | ||||
28 | def addCandidate(self, key, entry): | ||||
29 | entry['packageName'] = key | ||||
30 | self.candidates.append(entry) | ||||
31 | | ||||
32 | def candidate(self): | ||||
33 | """Find the best candidate to check the ABI against.""" | ||||
34 | candidate = None | ||||
35 | timestamp = self.library["timestamp"] | ||||
36 | | ||||
37 | if not self.candidates: | ||||
38 | return None | ||||
39 | | ||||
40 | # get a list of tagged candidates | ||||
41 | released = list(filter(lambda i: i['scmRevision'] in HASH2TAG, self.candidates)) | ||||
42 | if released: | ||||
43 | # get the first released version, that is available | ||||
44 | candidate = min(released, key=lambda i: HASH2TAG[i['scmRevision']]) | ||||
45 | else: | ||||
46 | #TODO: we may want to return None, as the library was never released so far. | ||||
47 | | ||||
48 | # get oldest candidate. | ||||
49 | candidate = min(self.candidates, key=lambda e:e['timestamp']) | ||||
50 | logging.warning("No released version was found, just use the oldest commit.") | ||||
51 | | ||||
52 | # the candidate needs to be older than the current build. | ||||
bcooksley: This tool is designed to run after the ABI information capture script I assume? | |||||
knauss: yes added this to description. | |||||
53 | if timestamp > candidate['timestamp']: | ||||
54 | return None | ||||
55 | | ||||
56 | return candidate | ||||
57 | | ||||
58 | | ||||
Any ideas on how to solve this issue? Wouldn't one possibility be to only capture ABI reference data from a special job that focuses on building the released version of software being tracked by this? bcooksley: Any ideas on how to solve this issue?
Wouldn't one possibility be to only capture ABI… | |||||
59 | # Make sure logging is ready to go | ||||
60 | logging.basicConfig(level=logging.DEBUG) | ||||
61 | | ||||
62 | # Parse the command line arguments we've been given | ||||
63 | parser = argparse.ArgumentParser(description='Utility to check ABI.') | ||||
64 | parser.add_argument('--project', type=str, required=True) | ||||
65 | parser.add_argument('--environment', type=str, required=True) | ||||
bcooksley: whant -> want | |||||
66 | arguments = parser.parse_args() | ||||
67 | | ||||
bcooksley: interessed -> interested | |||||
68 | # Initialize the archive manager | ||||
69 | ourArchive = Packages.Archive(arguments.environment, 'ABIReference', usingCache = True, contentsSuffix = ".abidump") | ||||
70 | | ||||
Won't this cause a KeyError because while referenceLibraries[libname] will exist (due to the code on line 59, assuming I understand that correctly), the key underneath it doesn't seem to have been populated? (unless i'm missing something of course) bcooksley: Won't this cause a KeyError because while referenceLibraries[libname] will exist (due to the… | |||||
Okay I made it hard to understand, because I use the name "key" but it is actually not the key of the dict. Let me rephrase it. knauss: Okay I made it hard to understand, because I use the name "key" but it is actually not the key… | |||||
71 | # Determine which SCM revision we are storing | ||||
72 | # This will be embedded into the package metadata which might help someone doing some debugging | ||||
73 | # GIT_COMMIT is set by Jenkins Git plugin, so we can rely on that for most of our builds | ||||
74 | scmRevision = '' | ||||
75 | if os.getenv('GIT_COMMIT') != '': | ||||
76 | scmRevision = os.getenv('GIT_COMMIT') | ||||
77 | | ||||
78 | if not scmRevision: | ||||
79 | scmRevision = subprocess.check_output(["git", "log", "--format=%H", "-n 1", "HEAD"]).strip().decode() | ||||
80 | | ||||
81 | scmBranch = subprocess.check_output(["git", "branch"]).strip().decode() | ||||
82 | | ||||
Sorry, missed this on my earlier review - you can't rely on the output of git branch to tell you which branch we've checked out. You'll want to read the same branchGroup parameter the other scripts rely on. bcooksley: Sorry, missed this on my earlier review - you can't rely on the output of `git branch` to tell… | |||||
83 | # get all tags that are in the current commit | ||||
84 | tags = subprocess.check_output(["git", "tag", "--contains", scmRevision]).strip().decode().splitlines() | ||||
D16581 should make it possiible to get rid of the creation/removal symlinks. knauss: D16581 should make it possiible to get rid of the creation/removal symlinks. | |||||
bcooksley: That change has now been landed. | |||||
85 | | ||||
86 | # resolve tags -> git hashes | ||||
87 | taghashes = subprocess.check_output(["git", "rev-parse", *tags]).strip().decode().splitlines() | ||||
88 | | ||||
89 | HASH2TAG = {taghashes[pos]:Version(tag) for pos, tag in enumerate(tags)} | ||||
90 | | ||||
91 | # check if we have a stable build | ||||
92 | # TODO: Maybe we should get the branch information from | ||||
93 | # e.g repo-metadata/projects/kde/applications/dolphin/i18n.json | ||||
94 | stableBranch = False | ||||
https://build-artifacts.kde.org/production/ABIReference/ still has .tar files (not cleaned up), so I need still to create the the symlinks) knauss: https://build-artifacts.kde.org/production/ABIReference/ still has .tar files (not cleaned up)… | |||||
95 | if scmBranch != "master": | ||||
96 | stableBranch = True | ||||
97 | | ||||
98 | # Find all libraries, that are build with the same git commit | ||||
99 | libraries = [] | ||||
100 | | ||||
101 | for key, entry in ourArchive.serverManifest.items(): | ||||
102 | try: | ||||
103 | if entry["project"] == arguments.project and entry["scmRevision"] == scmRevision: | ||||
104 | libraries.append(Library(key,entry)) | ||||
105 | except KeyError: | ||||
106 | continue | ||||
107 | | ||||
108 | # Find all availabe reference dumps | ||||
109 | # * same libname | ||||
110 | # * same SONAME otherwise we have a ABI bump and than it is safe to break ABI | ||||
111 | | ||||
112 | for l in libraries: | ||||
113 | libname = l.library["libname"] | ||||
114 | soname = l.library["SONAME"] | ||||
115 | for key, entry in ourArchive.serverManifest.items(): | ||||
116 | if key == l.packageName: | ||||
117 | continue | ||||
118 | # We want to so search for the library | ||||
119 | if "libname" in entry and entry["libname"] == libname: | ||||
120 | # only interested, for builds with the same SONAME | ||||
121 | if entry['SONAME'] == soname: | ||||
122 | l.addCandidate(key, entry) | ||||
123 | elif entry['SONAME'] > soname: | ||||
124 | if stableBranch and entry["branch"] == "master": | ||||
I would suggest simply comparing arguments.branchGroup against the entry['branchGroup'] here in case we end up adding a third branch group at some point. bcooksley: I would suggest simply comparing arguments.branchGroup against the entry['branchGroup'] here in… | |||||
You mean: if stableBuild and entry["branchGroup"] != arguments.branchGroup : knauss: You mean:
if stableBuild and entry["branchGroup"] != arguments.branchGroup : | |||||
125 | continue | ||||
126 | logging.warning("We searched for SONAME = {}, but found a newer SONAME = {} in the builds, that should not happen, as SONAMEs should only rise and never go lower!".format(soname, entry['SONAME'])) | ||||
127 | | ||||
128 | # Check every libraries ABI and do not fail, if one is not fine. | ||||
129 | # Safe the overall retval state | ||||
130 | retval = 0 | ||||
131 | | ||||
132 | for l in libraries: | ||||
133 | library = l.library | ||||
134 | libname = library['libname'] | ||||
135 | candidate = l.candidate() | ||||
136 | if not candidate: | ||||
137 | logging.info("Did not found any older build for {}, nothing to check ABI against.",libname) | ||||
138 | continue | ||||
139 | | ||||
140 | # get the packages, we want to test against each other | ||||
141 | newLibraryPath, _ = ourArchive.retrievePackage(l.packageName) | ||||
142 | oldLibraryPath, _ = ourArchive.retrievePackage(candidate['packageName']) | ||||
143 | | ||||
144 | logging.info("Let's do a ABI check {} against {}", library['scmRevision'], candidate['scmRevision']) | ||||
145 | | ||||
146 | # check ABI and write compat reports | ||||
147 | cmd = ["abi-compliance-checker", | ||||
148 | "-report-path", "$WORKSPACE/compat_reports/{libname}_compat_report.html".format(libname=libname), | ||||
149 | "-l", libname, | ||||
150 | "--old", oldLibraryPath, | ||||
151 | "--new", newLibraryPath] | ||||
152 | ret = subprocess.call(cmd) | ||||
153 | | ||||
154 | if ret != 0: | ||||
155 | logging.error("abi-compliance-checker exited with {ret}", ret=ret) | ||||
156 | retval = ret | ||||
157 | | ||||
158 | # We had an issue with one of the ABIs | ||||
159 | if retval != 0: | ||||
160 | sys.exit(retval) |
This tool is designed to run after the ABI information capture script I assume?