diff --git a/craft/enabled-projects.yaml b/craft/enabled-projects.yaml index 654f074..7838ac2 100644 --- a/craft/enabled-projects.yaml +++ b/craft/enabled-projects.yaml @@ -1,251 +1,252 @@ 'Kate': buildBlueprint: "kate" versions: - name: 'Nightly' target: 'master' - name: 'Release' target: '' - platforms: + platforms: - 'macos' - 'win32' - 'win64' 'Okular': buildBlueprint: "okular" versions: - name: 'Nightly' target: 'master' platforms: - 'macos' - 'win32' 'AtCore': buildBlueprint: "atcore" versions: - name: 'Nightly' target: 'master' platforms: - 'macos' - 'win32' 'Calligra': buildBlueprint: "calligra" versions: - name: 'Nightly' target: 'master' platforms: - 'macos' - 'win32' - 'win64' 'KDevelop': buildBlueprint: kdevelop rebuildBlueprint: - kdev-python - kdev-php versions: - name: Release target: 5.2 + options: extragear/kdevelop.version=5.2 platforms: - macos - win32 - win64 'RKWard': buildBlueprint: "rkward" versions: - name: 'Nightly' target: 'master' platforms: - 'macos' - 'mingw64' 'Elisa': buildBlueprint: "elisa" versions: - name: 'Nightly' target: 'master' platforms: - 'macos' - 'win32' 'Filelight': buildBlueprint: "filelight" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' 'KAlgebra': buildBlueprint: "kalgebra" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' 'KBruch': buildBlueprint: "kbruch" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' 'kdeconnect-kde': buildBlueprint: "kdeconnect-kde" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' 'KGeography': buildBlueprint: "kgeography" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' 'Kig': buildBlueprint: "kig" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' 'Kile': buildBlueprint: "kile" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' - 'win64' 'KMPlot': buildBlueprint: "kmplot" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' 'Konversation': buildBlueprint: "konversation" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' 'Kolourpaint': buildBlueprint: "kolourpaint" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' 'KTurtle': buildBlueprint: "kturtle" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' 'KWordQuiz': buildBlueprint: "kwordquiz" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' 'Labplot': buildBlueprint: "labplot" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' 'Okteta': buildBlueprint: "okteta" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' 'Parley': buildBlueprint: "parley" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' 'Peruse': buildBlueprint: "peruse" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' 'Ruqola': buildBlueprint: "ruqola" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' 'Skrooge': buildBlueprint: "skrooge" versions: - name: 'Nightly' target: '' platforms: - 'macos' - 'win32' 'KStars': buildBlueprint: "kstars" versions: - name: 'Release' target: '' - name: 'Nightly' target: 'master' platforms: - 'macos' - 'win32' 'Minuet': buildBlueprint: "minuet" versions: - name: Nightly target: 'master' platforms: - 'macos' - 'win32' 'Dolphin': buildBlueprint: "dolphin" versions: - name: 'Release' target: '' platforms: - 'macos' - 'win32' 'KMyMoney': buildBlueprint: "kmymoney" versions: - name: 'Nightly' target: 'master' platforms: - 'win32' - 'win64' - 'macos' 'Umbrello': buildBlueprint: "umbrello" versions: - name: 'Nightly' target: 'master' platforms: - 'win64' diff --git a/craft/gather-jobs.py b/craft/gather-jobs.py index 11bed8e..d60d0cd 100644 --- a/craft/gather-jobs.py +++ b/craft/gather-jobs.py @@ -1,83 +1,90 @@ #!/usr/bin/python3 import os import sys import json import yaml import argparse # Parse the command line arguments we've been given parser = argparse.ArgumentParser(description='Utility to determine which jobs need to be registered in Jenkins.') parser.add_argument('--projects', type=str, required=True) arguments = parser.parse_args() # Grab our list of projects and their configuration with open(arguments.projects, 'r') as dataFile: # Parse the YAML file projectsToCreate = yaml.load( dataFile ) # Our output will be a list of Dictionaries, containing several keys: # 1) The name of the job # 2) The Craft Blueprints to be built as part of the job # 3) The Craft Blueprints the job should package # 4) The description for the resulting job jobsGathered = [] # We need to let Craft know which target it should be using platformForCraft = { 'win32': 'windows-msvc2017_32-cl', 'win64': 'windows-msvc2017_64-cl', 'mingw64': 'windows-mingw_64-gcc', 'macos': 'macos-64-clang', 'appimage': '' } # Let's get started! for project in projectsToCreate.keys(): # Grab the configuration for the project out to make it easier to access projectConfig = projectsToCreate[ project ] # For each project, it will have a series of platforms which it is enabled for # So we now go over each of those in turn for platform in projectConfig['platforms']: # Each version will be built on all of the platforms for version in projectConfig['versions']: # Create a nice little description for this job jobDescription = "{0} build for {1} on {2}".format( version['name'], project, platform ) # Make sure we have a valid system target if platform not in platformForCraft: continue # Determine what blueprints we need to build and make sure what we've been given is valid buildBlueprint = projectConfig['buildBlueprint'] if not isinstance(buildBlueprint, str): raise TypeException() # Now determine what Blueprints we have to always rebuild rebuildBlueprint = projectConfig.get('rebuildBlueprint', []) if isinstance(rebuildBlueprint, str): rebuildBlueprint = [rebuildBlueprint] + # setup the options string + options = projectConfig.get('options', []) + if options: + if isinstance(options, str): + options = [options] + # At this point we are good to go! jobEntry = { 'project': project, 'platform': platform, 'craftPlatform': platformForCraft[platform], 'target': version['target'], 'targetName': version['name'], 'description': jobDescription, 'buildBlueprint': buildBlueprint, - 'rebuildBlueprint': " ".join( [buildBlueprint] + rebuildBlueprint) + 'rebuildBlueprint': ' '.join( [buildBlueprint] + rebuildBlueprint), + 'options': '--options {0}'.format(' '.join(options)) if options else '' } # Make sure we add it to the list jobsGathered.append( jobEntry ) # Now output the jobs we've gathered in JSON to disk # This will subsequently be read in by a Jenkins DSL script and turned into Jenkins Jobs filePath = os.path.join( os.getcwd(), 'gathered-jobs.json') with open(filePath, 'w') as jobsFile: json.dump( jobsGathered, jobsFile, sort_keys=True, indent=2 ) # All done! sys.exit(0) diff --git a/craft/pipeline-templates/macos.pipeline b/craft/pipeline-templates/macos.pipeline index ec07793..36f92ba 100644 --- a/craft/pipeline-templates/macos.pipeline +++ b/craft/pipeline-templates/macos.pipeline @@ -1,103 +1,103 @@ // Sometimes we need to include additional parameters to Craft which are specific to the project def craftProjectParams = "" // Determine if we need to build a specific version of this project if( craftTarget != '' ) { // If we have a specified version (Craft target) then we need to pass this along to Craft craftProjectParams = "--target ${craftTarget}" } // Request a node to be allocated to us node( "macOS" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Make sure we start with a clean slate deleteDir() // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // Make sure the Git checkouts are up to date sh """ /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository git://anongit.kde.org/craftmaster --into ~/Craft/BinaryFactory/ """ // Update Craft itself sh """ cd ~/Craft/BinaryFactory/ - /usr/local/bin/python3 craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c -i craft + /usr/local/bin/python3 craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c ${craftOptions} -i craft """ } stage('Cleaning Up Prior Builds') { // Cleanup our workspace deleteDir() // Ensure we have nothing left behind in the packaging workspace used by Craft sh """ cd ~/Craft/BinaryFactory/ - packageDir=\$(/usr/local/bin/python3 "craftmaster/Craftmaster.py" --config "craftmaster/config/CraftBinaryCache.ini" --target ${craftPlatform} -c -q --get "packageDestinationDir()" virtual/base) + packageDir=\$(/usr/local/bin/python3 "craftmaster/Craftmaster.py" --config "craftmaster/config/CraftBinaryCache.ini" --target ${craftPlatform} -c ${craftOptions} -q --get "packageDestinationDir()" virtual/base) rm -rf "\$packageDir" """ // Make sure the build environment for this application is clean sh """ cd ~/Craft/BinaryFactory/ - /usr/local/bin/python3 craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c ${craftProjectParams} --unmerge ${craftRebuildBlueprint} + /usr/local/bin/python3 craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c ${craftOptions} ${craftProjectParams} --unmerge ${craftRebuildBlueprint} """ } stage('Installing Dependencies') { // Ask Craftmaster to ensure all the dependencies are installed for this application we are going to be building sh """ cd ~/Craft/BinaryFactory/ - /usr/local/bin/python3 craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c ${craftProjectParams} --install-deps ${craftBuildBlueprint} + /usr/local/bin/python3 craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c ${craftOptions} ${craftProjectParams} --install-deps ${craftBuildBlueprint} """ } stage('Building') { // Actually build the application now sh """ cd ~/Craft/BinaryFactory/ - /usr/local/bin/python3 craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c --no-cache ${craftProjectParams} ${craftRebuildBlueprint} + /usr/local/bin/python3 craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c ${craftOptions} --no-cache ${craftProjectParams} ${craftRebuildBlueprint} """ } stage('Packaging') { // Now generate an installer for it sh """ cd ~/Craft/BinaryFactory/ - /usr/local/bin/python3 craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c ${craftProjectParams} --package ${craftBuildBlueprint} + /usr/local/bin/python3 craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c ${craftOptions} ${craftProjectParams} --package ${craftBuildBlueprint} """ // Now copy it to the Jenkins workspace so it can be grabbed from there sh """ cd ~/Craft/BinaryFactory/ - packageDir=\$(/usr/local/bin/python3 "craftmaster/Craftmaster.py" --config "craftmaster/config/CraftBinaryCache.ini" --target ${craftPlatform} -c -q --get "packageDestinationDir()" virtual/base) + packageDir=\$(/usr/local/bin/python3 "craftmaster/Craftmaster.py" --config "craftmaster/config/CraftBinaryCache.ini" --target ${craftPlatform} -c ${craftOptions} -q --get "packageDestinationDir()" virtual/base) cp -vf \$packageDir/* \$WORKSPACE/ rm -rf "\$packageDir" """ } stage('Capturing Package') { // Then we ask Jenkins to capture the generated installers as an artifact archiveArtifacts artifacts: '*.dmg, *.sha256', onlyIfSuccessful: true } } // As the Mac Slaves are permanent ones, we erase the Workspace as the last thing we do deleteDir() } } diff --git a/craft/pipeline-templates/win64.pipeline b/craft/pipeline-templates/win64.pipeline index 6e7e508..29c16ec 100644 --- a/craft/pipeline-templates/win64.pipeline +++ b/craft/pipeline-templates/win64.pipeline @@ -1,108 +1,108 @@ // Sometimes we need to include additional parameters to Craft which are specific to the project def craftProjectParams = "" // Determine if we need to build a specific version of this project if( craftTarget != '' ) { // If we have a specified version (Craft target) then we need to pass this along to Craft craftProjectParams = "--target ${craftTarget}" } // Request a node to be allocated to us node( "WindowsMSVC" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Make sure we start with a clean slate deleteDir() // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // Make sure that Craftmaster is up to date bat """ python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository git://anongit.kde.org/craftmaster --into "C:/Craft/BinaryFactory/" """ // Update Craft itself and make sure NSIS is installed bat """ cd "C:\\Craft\\BinaryFactory\\" - python craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c -i craft + python craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c ${craftOptions} -i craft if errorlevel 1 exit /b %errorlevel% - python craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c nsis + python craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c ${craftOptions} nsis if errorlevel 1 exit /b %errorlevel% """ } stage('Cleaning Up Prior Builds') { // Ensure we have nothing left behind in the packaging workspace used by Craft bat """ cd "C:\\Craft\\BinaryFactory\\" - for /f %%i in ('python "craftmaster/Craftmaster.py" --config "craftmaster/config/CraftBinaryCache.ini" --target ${craftPlatform} -c -q --get "packageDestinationDir()" virtual/base') do set CRAFT_TMPDIR=%%i + for /f %%i in ('python "craftmaster/Craftmaster.py" --config "craftmaster/config/CraftBinaryCache.ini" --target ${craftPlatform} -c ${craftOptions} -q --get "packageDestinationDir()" virtual/base') do set CRAFT_TMPDIR=%%i del /Q %CRAFT_TMPDIR%\\* """ // Make sure the build environment for this application is clean bat """ cd "C:\\Craft\\BinaryFactory\\" - python craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c ${craftProjectParams} --unmerge ${craftRebuildBlueprint} + python craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c ${craftOptions} ${craftProjectParams} --unmerge ${craftRebuildBlueprint} if errorlevel 1 exit /b %errorlevel% """ } stage('Installing Dependencies') { // Ask Craftmaster to ensure all the dependencies are installed for this application we are going to be building bat """ cd "C:\\Craft\\BinaryFactory\\" - python craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c ${craftProjectParams} --install-deps ${craftBuildBlueprint} + python craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c ${craftOptions} ${craftProjectParams} --install-deps ${craftBuildBlueprint} if errorlevel 1 exit /b %errorlevel% """ } stage('Building') { // Actually build the application now bat """ cd "C:\\Craft\\BinaryFactory\\" - python craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c --no-cache ${craftProjectParams} ${craftRebuildBlueprint} + python craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c ${craftOptions} --no-cache ${craftProjectParams} ${craftRebuildBlueprint} if errorlevel 1 exit /b %errorlevel% """ } stage('Packaging') { // Now generate an installer for it bat """ cd "C:\\Craft\\BinaryFactory\\" - python craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c ${craftProjectParams} --package ${craftBuildBlueprint} + python craftmaster/Craftmaster.py --config craftmaster/config/CraftBinaryCache.ini --target ${craftPlatform} -c ${craftOptions} ${craftProjectParams} --package ${craftBuildBlueprint} if errorlevel 1 exit /b %errorlevel% """ // Copy it to the Jenkins workspace so it can be grabbed from there bat """ cd "C:\\Craft\\BinaryFactory\\" - for /f %%i in ('python "craftmaster/Craftmaster.py" --config "craftmaster/config/CraftBinaryCache.ini" --target ${craftPlatform} -c -q --get "packageDestinationDir()" virtual/base') do set CRAFT_TMPDIR=%%i + for /f %%i in ('python "craftmaster/Craftmaster.py" --config "craftmaster/config/CraftBinaryCache.ini" --target ${craftPlatform} -c ${craftOptions} -q --get "packageDestinationDir()" virtual/base') do set CRAFT_TMPDIR=%%i xcopy /y %CRAFT_TMPDIR%\\* %WORKSPACE% del /Q %CRAFT_TMPDIR%\\* """ } stage('Capturing Package') { // Then we ask Jenkins to capture the generated installers as an artifact archiveArtifacts artifacts: '*.exe, *.7z, *.sha256', onlyIfSuccessful: true } } // As the Windows Slaves are permanent ones, we erase the Workspace as the last thing we do deleteDir() } } diff --git a/dsl/craft_jobs.groovy b/dsl/craft_jobs.groovy index 27accc9..48f3f14 100644 --- a/dsl/craft_jobs.groovy +++ b/dsl/craft_jobs.groovy @@ -1,56 +1,57 @@ // Read the contents of the gathered-jobs.json file a step created for us previously def jobsToParse = readFileFromWorkspace('craft/gathered-jobs.json') def knownJobs = new groovy.json.JsonSlurper().parseText( jobsToParse ) // Iterate over all of the known jobs and create them! knownJobs.each { // Create our job name def jobName = "${it.project}_${it.targetName}_${it.platform}" // Read in the necessary Pipeline template def pipelineTemplate = readFileFromWorkspace("craft/pipeline-templates/${it.platform}.pipeline") // Now we can construct our Pipeline script // We append a series of variables to the top of it to provide a variety of useful information to the otherwise templated script // These appended variables are what makes one build different to the next, aside from the template which was used def pipelineScript = """ |def project = "${it.project}" |def craftTarget = "${it.target}" |def craftPlatform = "${it.craftPlatform}" |def craftBuildBlueprint = "${it.buildBlueprint}" |def craftRebuildBlueprint = "${it.rebuildBlueprint}" + |def craftOptions = "${it.options}" |${pipelineTemplate}""".stripMargin() // Actually create the job now pipelineJob( jobName ) { properties { // We don't want to keep build results forever // We'll set it to keep the last 10 builds and discard everything else buildDiscarder { strategy { logRotator { numToKeepStr("10") daysToKeepStr('') artifactDaysToKeepStr('') artifactNumToKeepStr('') } } } // We don't want to be building the same project more than once // This is to prevent one project hogging resources // And also has a practical component as otherwise an older build could finish afterwards and upload old build results disableConcurrentBuilds() } triggers { // We want to automatically rebuild once a day cron("@daily") } // This is where the Pipeline script actually happens :) definition { cps { script( pipelineScript ) sandbox() } } } }