diff --git a/archive-configs/production.yaml b/archive-configs/production.yaml --- a/archive-configs/production.yaml +++ b/archive-configs/production.yaml @@ -16,3 +16,4 @@ SUSEQt5.11: "/srv/archives/production/SUSEQt5.11/" FreeBSDQt5.11: "/usr/home/jenkins/archives/production/" AndroidQt5.11: "/srv/archives/production/AndroidQt5.11/" + ABIReference: "/srv/archives/production/ABIReference" diff --git a/archive-configs/sandbox.yaml b/archive-configs/sandbox.yaml --- a/archive-configs/sandbox.yaml +++ b/archive-configs/sandbox.yaml @@ -16,3 +16,4 @@ SUSEQt5.11: "/srv/archives/sandbox/SUSEQt5.11/" FreeBSDQt5.11: "/usr/home/jenkins/archives/sandbox/" AndroidQt5.11: "/srv/archives/sandbox/AndroidQt5.11/" + ABIReference: "/srv/archives/sandbox/ABIReference/" diff --git a/helpers/check-abi.py b/helpers/check-abi.py new file mode 100755 --- /dev/null +++ b/helpers/check-abi.py @@ -0,0 +1,160 @@ +#!/usr/bin/python3 + +# check the ABIs against a earlier state. +# It is designed to run after create-abi-dump has created the abidump already. + +# abi-compliance-checker creates a html file with the report. +# it can be multiple libraries in one repository (e.g messagelib) +# so we have multiple html files for one repository +# in order to store them as artifacts in Jenkins add: +# +# archiveArtifacts artifacts: 'compat_reports/*_compat_reports.html', onlyIfSuccessful: false + +import os +import logging +import argparse +import subprocess +import sys + +from helperslib import Packages +from helperslib.Version import Version + +class Library: + def __init__(self, packageName, library): + self.packgeName = packageName + self.library = library + self.candidates = [] + + def addCandidate(self, key, entry): + entry['packageName'] = key + self.candidates.append(entry) + + def candidate(self): + """Find the best candidate to check the ABI against.""" + candidate = None + timestamp = self.library["timestamp"] + + if not self.candidates: + return None + + # get a list of tagged candidates + released = list(filter(lambda i: i['scmRevision'] in HASH2TAG, self.candidates)) + if released: + # get the first released version, that is available + candidate = min(released, key=lambda i: HASH2TAG[i['scmRevision']]) + else: + #TODO: we may want to return None, as the library was never released so far. + + # get oldest candidate. + candidate = min(self.candidates, key=lambda e:e['timestamp']) + logging.warning("No released version was found, just use the oldest commit.") + + # the candidate needs to be older than the current build. + if timestamp > candidate['timestamp']: + return None + + return candidate + + +# Make sure logging is ready to go +logging.basicConfig(level=logging.DEBUG) + +# Parse the command line arguments we've been given +parser = argparse.ArgumentParser(description='Utility to check ABI.') +parser.add_argument('--project', type=str, required=True) +parser.add_argument('--branchGroup', type=str, required=True) +parser.add_argument('--platform', type=str, required=True) +parser.add_argument('--environment', type=str, required=True) +arguments = parser.parse_args() + +# Initialize the archive manager +ourArchive = Packages.Archive(arguments.environment, 'ABIReference', usingCache = True, contentsSuffix = ".abidump") + +# Determine which SCM revision we are storing +# This will be embedded into the package metadata which might help someone doing some debugging +# GIT_COMMIT is set by Jenkins Git plugin, so we can rely on that for most of our builds +scmRevision = '' +if os.getenv('GIT_COMMIT') != '': + scmRevision = os.getenv('GIT_COMMIT') + +if not scmRevision: + scmRevision = subprocess.check_output(["git", "log", "--format=%H", "-n 1", "HEAD"]).strip().decode() + +# get all tags that are in the current commit +tags = subprocess.check_output(["git", "tag", "--contains", scmRevision]).strip().decode().splitlines() + +# resolve tags -> git hashes +taghashes = subprocess.check_output(["git", "rev-parse", *tags]).strip().decode().splitlines() +HASH2TAG = {taghashes[pos]:Version(tag) for pos, tag in enumerate(tags)} + +# Do we want to check for newer SONAMEs on other buildGroups +keepBuildGroup = False +if arguments.branchGroup != "kf5-qt5": + keepBuildGroup = True + +# Find all libraries, that are build with the same git commit +libraries = [] + +for key, entry in ourArchive.serverManifest.items(): + try: + if entry["project"] == arguments.project and entry["scmRevision"] == scmRevision: + libraries.append(Library(key,entry)) + except KeyError: + continue + +# Find all availabe reference dumps +# * same libname +# * same SONAME otherwise we have a ABI bump and than it is safe to break ABI + +for l in libraries: + libname = l.library["libname"] + soname = l.library["SONAME"] + for key, entry in ourArchive.serverManifest.items(): + if key == l.packageName: + continue + if entry['platform'] != arguments.platform: + continue + # We want to search for the library + if entry["libname"] == libname: + # only interested, for builds with the same SONAME + if entry['SONAME'] == soname: + l.addCandidate(key, entry) + elif entry['SONAME'] > soname: + # Ignore new SONAMEs on other branchGroups. + if keepBuildGroup and entry["branchGroup"] != arguments.branchGroup: + continue + logging.warning("We searched for SONAME = {}, but found a newer SONAME = {} in the builds, that should not happen, as SONAMEs should only rise and never go lower!".format(soname, entry['SONAME'])) + +# Check every libraries ABI and do not fail, if one is not fine. +# Safe the overall retval state +retval = 0 + +for l in libraries: + library = l.library + libname = library['libname'] + candidate = l.candidate() + if not candidate: + logging.info("Did not found any older build for {}, nothing to check ABI against.",libname) + continue + + # get the packages, we want to test against each other + newLibraryPath, _ = ourArchive.retrievePackage(l.packageName) + oldLibraryPath, _ = ourArchive.retrievePackage(candidate['packageName']) + + logging.info("Let's do a ABI check {} against {}", library['scmRevision'], candidate['scmRevision']) + + # check ABI and write compat reports + cmd = ["abi-compliance-checker", + "-report-path", "$WORKSPACE/compat_reports/{libname}_compat_report.html".format(libname=libname), + "-l", libname, + "--old", oldLibraryPath, + "--new", newLibraryPath] + ret = subprocess.call(cmd) + + if ret != 0: + logging.error("abi-compliance-checker exited with {ret}", ret=ret) + retval = ret + +# We had an issue with one of the ABIs +if retval != 0: + sys.exit(retval) diff --git a/helpers/helperslib/Version.py b/helpers/helperslib/Version.py new file mode 100644 --- /dev/null +++ b/helpers/helperslib/Version.py @@ -0,0 +1,147 @@ +import functools +from typing import List + +''' To run doctests for this module. + +python3 -m doctest -v helpers/helperslib/Version.py +''' + +SUFFIX2NUMBER = { + "alpha": 70, + "a": 70, + "beta": 80, + "b": 80, + "rc": 90 +} + + +@functools.total_ordering +class Version: + '''class for Version and correct ordering. ''' + + def __init__(self, version: str) -> None: + if version.startswith("v"): + self.version = version[1:] + else: + self.version = version + self._parts = None + self._suffix = None + + def parts(self) -> List: + '''returns a list with different parts of version. + + >>> Version("1.2.3-rc1").parts() + [1, 2, 3] + ''' + if self._parts: + return self._parts + + ps = self.version.split(".") + + self._parts = [] + + for p in ps: + try: + self._parts.append(int(p)) + except ValueError: + v, suffix = p.split("-") + self._parts.append(int(v)) + self._suffix = suffix + + return self._parts + + def suffix(self) -> int: + '''transforms suffix into integer using SUFFIX2NUMBER. + >>> Version("1.2-alpha").suffix() + 70 + >>> Version("1.2-beta").suffix() + 80 + >>> Version("1.2-rc").suffix() + 90 + >>> Version("1.2-rc1").suffix() + 91 + ''' + + if not self._parts: + self.parts() + + #no suffix so we are higher than any suffix + if not self._suffix: + return 100 + + for key, base in SUFFIX2NUMBER.items(): + if self._suffix.startswith(key): + addition = 0 + # handle case with rc1 or beta2 etc. + if len(self._suffix) > len(key): + addition = int(self._suffix[len(key):]) + return base + addition + + def __eq__(self, other): + '''returns True if versions are the same. + + >>> Version("1.2") == "1.2" + True + + The KDE logic that 5.3.90 == "5.4-rc" is not implemented! + + >>> Version("5.4-rc") == Version("5.3.90") + False + ''' + if hasattr(other, "version"): + return self.version == other.version + elif type(other) == str: + return self == Version(other) + else: + NotImplemented + + def __lt__(self, other): + '''returns True if self < other. + + >>> Version("1.2.3-rc1") < Version("1.2.3") + True + >>> Version("1.2.3") < Version("1.2.3-rc1") + False + >>> Version("1.2.3") < Version("1.2.4") + True + >>> Version("1.2.3") < Version("1.2.3") + False + >>> Version("1.2") < Version("1.2.3") + True + >>> Version("1.2.3") < Version("1.2") + False + + The KDE logic that 5.3.89 < "5.4-rc" < 5.3.91 is not implemented! + Currently: + >>> Version("5.3.89") < "5.4-rc" + True + >>> Version("5.4-rc") < "5.3.91" + False + >>> Version("5.3.89") < "5.3.91" + True + ''' + if type(other) == str: + return self < Version(other) + if hasattr(other, "parts"): + parts = self.parts() + otherParts = other.parts() + + for i, part in enumerate(parts): + try: + if part == otherParts[i]: + continue + return part < otherParts[i] + except IndexError: + # @other has less parts than @self. + return False + if len(parts) < len(otherParts): + return True + return self.suffix() < other.suffix() + + def __repr__(self): + parts = self.parts() + version = ".".join([str(i) for i in parts]) + if self._suffix: + version += "-{}".format(self._suffix) + + return "<{}>".format(version)