diff --git a/helpers/create-abi-dump.py b/helpers/create-abi-dump.py index 6f7acda..3b67b46 100755 --- a/helpers/create-abi-dump.py +++ b/helpers/create-abi-dump.py @@ -1,364 +1,373 @@ #!/usr/bin/python3 import argparse import logging import os import pathlib import re import subprocess import sys import tempfile import yaml from collections import defaultdict from typing import Dict, List, Union, Set from helperslib import CommonUtils, ToolingSettings from helperslib import Packages, EnvironmentHandler # Make sure logging is ready to go -#logging.basicConfig(level=logging.DEBUG) +logging.basicConfig(level=logging.DEBUG) ACCXMLTMPL = """{version} {headers} {libs} {skipIncludePaths} {additionalIncludes} {gccOptions} """ def cmake_parser(lines: List) -> Dict: """A small cmake parser, if you search for a better solution think about using a proper one based on ply. see https://salsa.debian.org/qt-kde-team/pkg-kde-jenkins/blob/master/hooks/prepare/cmake_update_deps But in our case we are only interested in two keywords and do not need many features. we return a dictonary with keywords and targets. set(VAR "123") -> variables["VAR"]="123" set_target_properties(TARGET PROPERTIES PROP1 A B PROP2 C D) -> targets = { "PROP1":["A","B"], "PROP2":["C","D"], } """ variables = {} # type: Dict[str,str] targets = defaultdict(lambda:defaultdict(list)) # type: Dict[str, Dict[str, List[str]]] ret = { "variables": variables, "targets": targets, } def parse_set(args: str) -> None: """process set lines and updates the variables directory: set(VAR 1.2.3) -> args = ["VAR", "1.2.3"] and we set variable["VAR"] = "1.2.3" """ _args = args.split() if len(_args) == 2: name, value = _args variables[name] = value def parse_set_target_properties(args: str) -> None: """process set_target_properties cmake lines and update the targets directory all argiments of set_target_properties are given in the args parameter as list. as cmake using keyword val1 val2 we need to save the keyword so long we detect a next keyword. args[0] is the target we want to update args[1] must be PROPERTIES """ name, properties, *values = args.split() target = targets[name] if not properties == "PROPERTIES": logging.warning("unknown line: %s"%(args)) # Known set_target_properties keywords keywords = [ "IMPORTED_LINK_DEPENDENT_LIBRARIES_DEBUG", "IMPORTED_LOCATION_DEBUG", "IMPORTED_SONAME_DEBUG", "INTERFACE_INCLUDE_DIRECTORIES", "INTERFACE_LINK_LIBRARIES", "INTERFACE_COMPILE_OPTIONS", "INTERFACE_COMPILE_DEFINITIONS", "IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG", "IMPORTED_CONFIGURATIONS", "DOXYGEN_TAGFILE", ] tmpKeyword = None for arg in values: if arg in keywords: tmpKeyword = target[arg] continue tmpKeyword.append(arg) #Keywords we want to react on keywords = { "set": parse_set, "set_target_properties": parse_set_target_properties, } RELINE = re.compile("^\s*(?P[^(]+)\s*\(\s*(?P.*)\s*\)\s*$") for line in lines: m = RELINE.match(line) if m and m.group('keyword') in keywords: keywords[m.group('keyword')](m.group('args')) return ret # Wrapper class to represent a library we have found # This class stores information on the library in question and assists in extracting information concerning it class Library: # Make sure we initialize everything we are going to need def __init__(self, name: str, accSettings: Dict) -> None: # name of the library self.name = name # type: str self.accSettings = accSettings # type: Dict # The raw cmake Parser output, available for debugging purposes # see cmake_parser function for the return value self.__parser_output = None # type: Union[Dict, None] # Provide a helpful description of the object (to ease debugging) def __repr__(self) -> str: return "".format(self=self) # replace with f-String in python 3.6 # Execute CMake to gather the information we need def runCMake(self, runtimeEnvironment) -> None: """Create a CMakeLists.txt to detect the headers, version and library path""" # Prepare to gather the information we need self.__mlines = [] # type: List[str] # To avoid contaminating the directory we are being run in, make sure we are in a temporary directory # This will also allow us to succeed if we are run from the build direcotry with tempfile.TemporaryDirectory() as d: # Create an appropriate CMakeLists.txt which searches for the library in question cmakeFile = (pathlib.Path(d)/"CMakeLists.txt") cmakeFile.write_text("find_package({self.name} CONFIG REQUIRED)\n".format(self=self)) # replace with f-String in python 3.6 # Now run CMake and ask it to process the CMakeLists.txt file we just generated # We want it to run in trace mode so we can examine the log to extract the information we need proc = subprocess.Popen(['cmake', '.', '--trace-expand'], cwd=d, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, env=runtimeEnvironment) # cmake prefixes outout with the name of the file, filter only lines with interessting files retarget = re.compile( '.*/{self.name}/[^/]*(Targets[^/]*|Config[^/]*)\.cmake\(\d+\):\s*(.*)$'.format(self=self) ) # replace with f-String in python 3.6 # Start processing the output of CMake, one line at a time for line in proc.stderr: # Make sure it is UTF-8 formatted theLine = line.decode("utf-8") # Did we find a CMake line we were interested in? m = retarget.match(theLine) if m: # Extract the information from that and store it for further processing mline = m.group(2) self.__mlines.append(mline) # Process the information we've now gathered self.__parser_output = cmake_parser(self.__mlines) self.parser_output = self.__parser_output self.mlines = self.__mlines # Extract the version number of the library for easier use self.version = self.__parser_output["variables"].get("PACKAGE_VERSION") # type: str # targets the targets of the libary ( existing so files) # a dict with keys, SONAME = the SONAME of the lib # path = path of the library # include_dirs = the header files for the library self.targets = {} # type: Dict # Helper Function to parse CMake formatted include directory lists def parseIncludeDirs(args: List[str]) -> List[str]: """ cmake using ";" to seperate different paths split the paths and make a unique list of all paths (do not add paths multiple times) """ d = [] # type: List[str] for arg in args: d += arg.split(";") return d # Process the various targets our parser found for t,value in self.__parser_output["targets"].items(): # Particularly, we want to extract: # Library names (sonames) # The path to the CMake library package # Any include directories specified by the CMake library package try: target = { "SONAME": re.search("\.([\d]*)$",value["IMPORTED_SONAME_DEBUG"][0]).group(1), "path": value["IMPORTED_LOCATION_DEBUG"][0], "include_dirs": parseIncludeDirs(value["INTERFACE_INCLUDE_DIRECTORIES"]), } self.targets[t]=target except IndexError: pass def createABIDump(self, runtimeEnvironment=None) -> None: """run abi-compliance-checker (acc) to create a ABIDump tar gz First we need to construct a input file for acc, see xml variable. After that we can run acc with the constructed file. """ # Make sure we have a valid runtime environment for CMake and abi-compliance-checker if runtimeEnvironment is None: runtimeEnvironment = os.environ # If we haven't yet run CMake, do so # Otherwise we won't have anything to give to abi-compliance-checker if not self.__parser_output: self.runCMake(runtimeEnvironment) # Start preparations to run abi-compliance-checker # Gather the information we'll need to write the XML configuration file it uses version = self.version headers = set() # type: Set[str] libs = set() # type: Set[str] skipIncludePaths = set(self.accSettings['skip_include_paths']) # type: Set[str] additionalIncludes = set(self.accSettings['add_include_paths']) # type: Set[str] gccOptions = set(self.accSettings['gcc_options']) # type: Set[str] # list of general include directories prefixHeaders = [os.path.abspath(i) for i in buildEnvironment.get('CMAKE_PREFIX_PATH').split(":")] noHeaders = set(prefixHeaders) noHeaders |= set([os.path.join(i,"include") for i in prefixHeaders]) # From the target information we previously collected... # Grab the list of libraries and include headers for abi-compliance-checker for target in self.targets.values(): # Check each include directory to see if we need to add it.... for i in target['include_dirs']: abspath = os.path.abspath(i) # ignore general folders, as there are no lib specific headers are placed if abspath in noHeaders or abspath.endswith("/KF5"): additionalIncludes.add(abspath) continue # Otherwise, if we don't already have it - add it to the list! headers.add(abspath) # If the library path isn't in the list, then we should add it to the list libs.add(target['path']) # Now we can go ahead and generate the XML file for abi-compliance-checker def _(l): return "\n ".join(l) xml = ACCXMLTMPL.format(version=version, headers=_(headers), libs=_(libs), additionalIncludes=_(additionalIncludes), skipIncludePaths=_(skipIncludePaths), gccOptions=_(gccOptions), ) # Write the generated XML out to a file to pass to abi-compliance-checker # We will give this to abi-compliance-checker using it's --dump parameter - with open("{version}.xml".format(version=version),"w") as f: # replace with f-String in python 3.6 + with open("acc/{libname}-{version}.xml".format(libname=self.name, version=version),"w") as f: # replace with f-String in python 3.6 f.write(xml) # acc is compatible for C/C++ (but --lang C++ doesn't remove the warning about C++ only settings). - subprocess.check_call(["abi-compliance-checker", "--lang", "C++", "-l", self.name, "--dump", f.name], env=runtimeEnvironment) + cmd = ["abi-compliance-checker", "--lang", "C++", "-l", self.name, "--dump", f.name] + logging.debug(" ".join(cmd)) + subprocess.check_call(cmd, env=runtimeEnvironment) # Parse the command line arguments we've been given parser = argparse.ArgumentParser(description='Utility to create abi checker tarballs.') parser.add_argument('--project', type=str, required=True) parser.add_argument('--branchGroup', type=str, required=True) parser.add_argument('--buildLog', type=str, required=True) parser.add_argument('--environment', type=str, required=True) parser.add_argument('--platform', type=str, required=True) parser.add_argument('--usingInstall', type=str, required=True) arguments = parser.parse_args() # Make sure we have an environment ready for executing commands buildEnvironment = EnvironmentHandler.generateFor( installPrefix=arguments.usingInstall ) # get acc settings localMetadataPath = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'abi-compliance-checker.yaml' ) accSettings = ToolingSettings.Loader( arguments.project, arguments.platform ) accSettings.loadSpecificationFile( localMetadataPath ) # Get ready to start searching for libraries foundLibraries = [] # Search in the build log for the Installing/Up-to-date lines where we install the Config.cmake files. # From this we get a complete List of installed libraries. cmakeConfig = re.compile("^-- (Installing|Up-to-date): .*/([^/]*)Config\.cmake$") with open(arguments.buildLog, encoding='utf-8') as log: for line in log.readlines(): match = cmakeConfig.match(line) if match: foundLibrary = Library( match.group(2), accSettings ) foundLibraries.append(foundLibrary) # Initialize the archive manager ourArchive = Packages.Archive(arguments.environment, 'ABIReference', usingCache = False, contentsSuffix = ".abidump") # Determine which SCM revision we are storing # This will be embedded into the package metadata which might help someone doing some debugging # GIT_COMMIT is set by Jenkins Git plugin, so we can rely on that for most of our builds scmRevision = '' if os.getenv('GIT_COMMIT') != '': scmRevision = os.getenv('GIT_COMMIT') if not scmRevision: scmRevision = subprocess.check_output(["git", "log", "--format=%H", "-n 1", "HEAD"]).strip().decode() +# Create diretory for string acc input xml +try: + os.mkdir("acc") +except FileExistsError: + pass + # Check every libraries ABI and do not fail, if one is not fine. # Safe the overall retval state retval = 0 # Now we generate the ABI dumps for every library we have found for library in foundLibraries: + logging.info("Start building ABI dump for {name}".format(name=library.name)) try: #run CMake for library library.runCMake( runtimeEnvironment=buildEnvironment ) if not library.version: logging.warning("{name} has no version: skipping.".format(name=library.name)) continue # Create the ABI Dump for this library library.createABIDump( runtimeEnvironment=buildEnvironment ) # Determine where the ABI Dump archive is located # This location is controlled by abi-compliance-checker, but follows a predictable pattern fileName = "abi_dumps/{name}/{version}/ABI.dump".format(name=library.name,version=library.version) extraMetadata = { "SONAME": max([t['SONAME'] for t in library.targets.values()]), # use max because there may be more than one lib inside "version": library.version, "libname": library.name, "targets": list(library.targets), "project": arguments.project, "branchGroup": arguments.branchGroup, "platform": arguments.platform, } packageName = "{name}_{scmRevision}_{platform}".format(name=library.name, scmRevision=scmRevision, platform=arguments.platform) ourArchive.storePackage(packageName, fileName, scmRevision, extraMetadata) except subprocess.CalledProcessError as e: retval = e.returncode logging.error("abi-compliance-checker exited with {retval}".format(retval=retval)) # We had an issue with one of the ABIs if retval != 0: sys.exit(retval) diff --git a/pipeline-templates/Frameworks/SUSEQt5.11.template b/pipeline-templates/Frameworks/SUSEQt5.11.template index 0f3e8e5..60364a0 100644 --- a/pipeline-templates/Frameworks/SUSEQt5.11.template +++ b/pipeline-templates/Frameworks/SUSEQt5.11.template @@ -1,224 +1,225 @@ // Provisionally mark the build as successful currentBuild.result = 'SUCCESS' // Request a node to be allocated to us node( currentPlatform ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First Thing: Checkout Sources stage('Checkout Sources') { // Actual Application Sources checkout changelog: true, poll: true, scm: [ $class: 'GitSCM', branches: [[name: branchToBuild]], browser: [$class: 'CGit', repoUrl: browserUrl], extensions: [[$class: 'CloneOption', timeout: 120]], userRemoteConfigs: [[url: repositoryUrl]] ] // Our CI scripts checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] ] // Projects metadata and next generation dependency metadata checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/repo-metadata/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/repo-metadata']] ] // Dependency Metadata checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/kde-build-metadata/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/kde-build-metadata']] ] // KApiDox: For api.kde.org metadata extraction checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/kapidox/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/kapidox']] ] // kde-dev-scripts: For packager metadata extraction checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/kde-dev-scripts/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/kde-dev-scripts']] ] } // Now Prepare to Build: Get the dependencies ready stage('Setup Dependencies') { // Now we can determine what our dependencies are // Then update to the latest version of the dependencies available from the master server // Finally extract all of those dependencies in turn into the given 'installTo' directory sh "python3 -u ci-tooling/helpers/prepare-dependencies.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --environment ${ciEnvironment} --platform ${currentPlatform} --installTo '$HOME/install-prefix/'" } // Now we can configure our build stage('Configuring Build') { // This is delegated through a helper script to handle minor special cases like inSourceBuilds, non-CMake build systems, etc sh "python3 -u ci-tooling/helpers/configure-build.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --installTo '$HOME/install-prefix/'" } // Finally we can build it! (Once again, through a helper) stage('Compiling') { // We use a helper here so we can determine the appropriate number of CPUs (-j) to build with sh "python3 -u ci-tooling/helpers/compile-build.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --usingInstall '$HOME/install-prefix/'" } // Now we can run our tests stage('Running Tests') { // Run the unit tests for this project // Tests are run in a basic environment (X, DBus) sh "python3 -u ci-tooling/helpers/run-tests.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --usingInstall '$HOME/install-prefix/'" // Collect our results junit allowEmptyResults: true, testResults: 'JUnitTestResults.xml' } // Now ensure that it installs.... stage('Installing') { // The helper ensures that DESTDIR and INSTALL_ROOT are set to 'divertTo' // This allows us to capture the install at the next stage for later reuse in the Setup Dependencies step sh "python3 -u ci-tooling/helpers/install-build.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --installTo '$HOME/install-prefix/' --divertTo '$WORKSPACE/install-divert/'" } // Looks like it built okay - let's capture this for later use // We'll also take the opportunity to extract metadata from CMake used by packagers and api.kde.org stage('Capturing Installation') { // First we create a tar archive of the installation which was diverted // Then we upload a copy of that to the master server and have it publish the new archive // Finally to save bandwidth our copy of the tar archive is moved to our local cache for reuse on later builds on this node sh "python3 -u ci-tooling/helpers/capture-install.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --environment ${ciEnvironment} --platform ${currentPlatform} --divertedTo '$WORKSPACE/install-divert/' --installedTo '$HOME/install-prefix/'" // Now we extract the CMake metadata and upload that to the appropriate hosts sh "python3 -u ci-tooling/helpers/extract-cmake-dependency-metadata.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --usingInstall '$HOME/install-prefix/'" sh "python3 -u ci-tooling/helpers/generate-dependency-diagram-data.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --usingInstall '$HOME/install-prefix/'" } // Final thing to do: some code quality checks stage('Checking Code Quality') { // Perform Appstream Compliance Checks sh "python3 -u ci-tooling/helpers/check-appstream-compliance.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --usingInstall '$HOME/install-prefix/' --withDiverted '$WORKSPACE/install-divert/'" // Gather ABI Reference information for later checking sh """ curl '$BUILD_URL/consoleText' -o currentBuildLog.txt python3 -u ci-tooling/helpers/create-abi-dump.py --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --buildLog currentBuildLog.txt --environment production --usingInstall '$HOME/install-prefix/' || true """ // Save the ABI build logs to review if necessary archiveArtifacts artifacts: 'logs/*/*/log.txt', onlyIfSuccessful: false, allowEmptyArchive: true - + // Save the input for ACC for building abi dumps locally + archiveArtifacts artifacts: 'acc/*.xml', onlyIfSuccessful: false, allowEmptyArchive: true // Now perform the ABI Compatibility checks // This tool will produce reports stored at compat_reports/ which we will also need to capture sh """ python3 -u ci-tooling/helpers/check-abi.py --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --environment production || true """ // Save the ABI Compatibility reports for developers to review if necessary archiveArtifacts artifacts: 'compat_reports/*_compat_report.html', onlyIfSuccessful: false, allowEmptyArchive: true // Save the ABI Compatibility results yaml file archiveArtifacts artifacts: 'abi-compatibility-results.yaml', onlyIfSuccessful: false, allowEmptyArchive: true // Platform Enablement Checks // Frameworks have a metadata file which specifies the platforms it supports and should be built on // This check compares that metadata file against the records held by the CI system sh """ touch PlatformCheckOutput.txt if [[ -e metainfo.yaml ]]; then python3 ci-tooling/helpers/check-platform.py '$WORKSPACE/metainfo.yaml' &> PlatformCheckOutput.txt; fi """ // If the platform check indicates there are missing platforms then we should flag the build as unstable // We start this process by reading the output of the check command def platformCheckResult = readFile "${env.WORKSPACE}/PlatformCheckOutput.txt" // Then we check to see if it had anything in it - it will be empty if everything is okay if( platformCheckResult != '' ) { // If it does, then mark the build as unstable currentBuild.result = 'UNSTABLE' // We also print the check results so it can be examined easily echo platformCheckResult } // cppcheck is not supported by Pipeline at the moment, so we don't run that for now // See https://issues.jenkins-ci.org/browse/JENKINS-35096 // Perform Cobertura Processing // First, run the LCov extraction sh "python3 -u ci-tooling/helpers/extract-lcov-results.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform}" // Collect the results from the LCov extraction step([ $class: 'CoberturaPublisher', autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: 'CoberturaLcovResults.xml', failNoReports: false, failUnhealthy: false, failUnstable: false, maxNumberOfBuilds: 0, onlyStable: false, zoomCoverageChart: false ]) // Scan the logs and publish a warnings report warnings consoleParsers: [[parserName: 'GNU Make + GNU C Compiler (gcc)'], [parserName: 'Appstreamercli']], excludePattern: "/tmp/**|/home/jenkins/workspace/**/build/**|/usr/include/**" } } // Let's determine if we need to send out notifications // What happened in our previous build? def previousResult = currentBuild.previousBuild?.result // If our condition has changed, is FAILURE or UNSTABLE then we want to send an email if( previousResult != currentBuild.result || currentBuild.result == 'FAILURE' || currentBuild.result == 'UNSTABLE' ) { // Start constructing the list of our recipients // At this point we can only be either a failure or an unstable build, so notify those who have requested unstable build notifications def mailTo = [ unstableBuildEmails ] // If the build was a solid failure (either now or previously) then notify those who want to know about failures only if( previousResult == 'FAILURE' || currentBuild.result == 'FAILURE' ) { // Add them to the list mailTo << buildFailureEmails } // If someone kicked this job off, they're presumably interested as well mailTo << emailextrecipients( [[$class: 'RequesterRecipientProvider']] ) // We always want to notify our dashboard as well mailTo << "kde-dashboard@kde.org" // Finalise the list of recipients mailTo = mailTo.join(',') // Send the email now emailext( to: mailTo, body: '${JELLY_SCRIPT,template="html_gmail"}', mimeType: 'text/html', subject: 'KDE CI: ${PROJECT_NAME} - Build # ${BUILD_NUMBER} - ${BUILD_STATUS}!', attachLog: false ) } // IRC Notifications are currently not supported by Pipeline // See https://issues.jenkins-ci.org/browse/JENKINS-33922 // We can probably workaround this using Pursuivant and the emails Jenkins sends out // This would allow subscribing to build notifications for IRC channels in much the same way one subscribes for Commits and Bugzilla changes } } diff --git a/pipeline-templates/SUSEQt5.10.template b/pipeline-templates/SUSEQt5.10.template index 2c289ad..cc4a754 100644 --- a/pipeline-templates/SUSEQt5.10.template +++ b/pipeline-templates/SUSEQt5.10.template @@ -1,203 +1,205 @@ // Provisionally mark the build as successful currentBuild.result = 'SUCCESS' // Request a node to be allocated to us node( currentPlatform ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First Thing: Checkout Sources stage('Checkout Sources') { // Actual Application Sources checkout changelog: true, poll: true, scm: [ $class: 'GitSCM', branches: [[name: branchToBuild]], browser: [$class: 'CGit', repoUrl: browserUrl], extensions: [[$class: 'CloneOption', timeout: 120]], userRemoteConfigs: [[url: repositoryUrl]] ] // Our CI scripts checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] ] // Projects metadata and next generation dependency metadata checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/repo-metadata/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/repo-metadata']] ] // Dependency Metadata checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/kde-build-metadata/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/kde-build-metadata']] ] // KApiDox: For api.kde.org metadata extraction checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/kapidox/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/kapidox']] ] // kde-dev-scripts: For packager metadata extraction checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/kde-dev-scripts/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/kde-dev-scripts']] ] } // Now Prepare to Build: Get the dependencies ready stage('Setup Dependencies') { // Now we can determine what our dependencies are // Then update to the latest version of the dependencies available from the master server // Finally extract all of those dependencies in turn into the given 'installTo' directory sh "python3 -u ci-tooling/helpers/prepare-dependencies.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --environment ${ciEnvironment} --platform ${currentPlatform} --installTo '$HOME/install-prefix/'" } // Now we can configure our build stage('Configuring Build') { // This is delegated through a helper script to handle minor special cases like inSourceBuilds, non-CMake build systems, etc sh "python3 -u ci-tooling/helpers/configure-build.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --installTo '$HOME/install-prefix/'" } // Finally we can build it! (Once again, through a helper) stage('Compiling') { // We use a helper here so we can determine the appropriate number of CPUs (-j) to build with sh "python3 -u ci-tooling/helpers/compile-build.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --usingInstall '$HOME/install-prefix/'" } // Now ensure that it installs.... stage('Installing') { // The helper ensures that DESTDIR and INSTALL_ROOT are set to 'divertTo' // This allows us to capture the install at the next stage for later reuse in the Setup Dependencies step sh "python3 -u ci-tooling/helpers/install-build.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --installTo '$HOME/install-prefix/' --divertTo '$WORKSPACE/install-divert/'" } // Looks like it built okay - let's capture this for later use // We'll also take the opportunity to extract metadata from CMake used by packagers and api.kde.org stage('Capturing Installation') { // First we create a tar archive of the installation which was diverted // Then we upload a copy of that to the master server and have it publish the new archive // Finally to save bandwidth our copy of the tar archive is moved to our local cache for reuse on later builds on this node sh "python3 -u ci-tooling/helpers/capture-install.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --environment ${ciEnvironment} --platform ${currentPlatform} --divertedTo '$WORKSPACE/install-divert/' --installedTo '$HOME/install-prefix/'" // Now we extract the CMake metadata and upload that to the appropriate hosts sh "python3 -u ci-tooling/helpers/extract-cmake-dependency-metadata.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --usingInstall '$HOME/install-prefix/'" sh "python3 -u ci-tooling/helpers/generate-dependency-diagram-data.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --usingInstall '$HOME/install-prefix/'" } // Now we can run our tests stage('Running Tests') { // Run the unit tests for this project // Tests are run in a basic environment (X, DBus) sh "python3 -u ci-tooling/helpers/run-tests.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --usingInstall '$HOME/install-prefix/'" // Collect our results junit allowEmptyResults: true, testResults: 'JUnitTestResults.xml' } // Final thing to do: some code quality checks stage('Checking Code Quality') { // Perform Appstream Compliance Checks sh "python3 -u ci-tooling/helpers/check-appstream-compliance.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --usingInstall '$HOME/install-prefix/' --withDiverted '$WORKSPACE/install-divert/'" // Gather ABI Reference information for later checking sh """ curl '$BUILD_URL/consoleText' -o currentBuildLog.txt python3 -u ci-tooling/helpers/create-abi-dump.py --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --buildLog currentBuildLog.txt --environment production --usingInstall '$HOME/install-prefix/' || true """ // Save the ABI build logs to review if necessary archiveArtifacts artifacts: 'logs/*/*/log.txt', onlyIfSuccessful: false, allowEmptyArchive: true + // Save the input for ACC for building abi dumps locally + archiveArtifacts artifacts: 'acc/*.xml', onlyIfSuccessful: false, allowEmptyArchive: true // Now perform the ABI Compatibility checks // This tool will produce reports stored at compat_reports/ which we will also need to capture sh """ python3 -u ci-tooling/helpers/check-abi.py --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform} --environment production || true """ // Save the ABI Compatibility reports for developers to review if necessary archiveArtifacts artifacts: 'compat_reports/*_compat_report.html', onlyIfSuccessful: false, allowEmptyArchive: true // Save the ABI Compatibility results yaml file archiveArtifacts artifacts: 'abi-compatibility-results.yaml', onlyIfSuccessful: false, allowEmptyArchive: true // cppcheck is not supported by Pipeline at the moment, so we don't run that for now // See https://issues.jenkins-ci.org/browse/JENKINS-35096 // Perform Cobertura Processing // First, run the LCov extraction sh "python3 -u ci-tooling/helpers/extract-lcov-results.py --product ${productName} --project ${projectName} --branchGroup ${branchGroup} --platform ${currentPlatform}" // Collect the results from the LCov extraction step([ $class: 'CoberturaPublisher', autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: 'CoberturaLcovResults.xml', failNoReports: false, failUnhealthy: false, failUnstable: false, maxNumberOfBuilds: 0, onlyStable: false, zoomCoverageChart: false ]) // Scan the logs and publish a warnings report warnings consoleParsers: [[parserName: 'GNU Make + GNU C Compiler (gcc)'], [parserName: 'Appstreamercli']], excludePattern: "/tmp/**|/home/jenkins/workspace/**/build/**|/usr/include/**" } } // Let's determine if we need to send out notifications // What happened in our previous build? def previousResult = currentBuild.previousBuild?.result // If our condition has changed, is FAILURE or UNSTABLE then we want to send an email if( previousResult != currentBuild.result || currentBuild.result == 'FAILURE' || currentBuild.result == 'UNSTABLE' ) { // Start constructing the list of our recipients // At this point we can only be either a failure or an unstable build, so notify those who have requested unstable build notifications def mailTo = [ unstableBuildEmails ] // If the build was a solid failure (either now or previously) then notify those who want to know about failures only if( previousResult == 'FAILURE' || currentBuild.result == 'FAILURE' ) { // Add them to the list mailTo << buildFailureEmails } // If someone kicked this job off, they're presumably interested as well mailTo << emailextrecipients( [[$class: 'RequesterRecipientProvider']] ) // We always want to notify our dashboard as well mailTo << "kde-dashboard@kde.org" // Finalise the list of recipients mailTo = mailTo.join(',') // Send the email now emailext( to: mailTo, body: '${JELLY_SCRIPT,template="html_gmail"}', mimeType: 'text/html', subject: 'KDE CI: ${PROJECT_NAME} - Build # ${BUILD_NUMBER} - ${BUILD_STATUS}!', attachLog: false ) } // IRC Notifications are currently not supported by Pipeline // See https://issues.jenkins-ci.org/browse/JENKINS-33922 // We can probably workaround this using Pursuivant and the emails Jenkins sends out // This would allow subscribing to build notifications for IRC channels in much the same way one subscribes for Commits and Bugzilla changes } }