diff --git a/craft-cache/devel-pipelines/linux.pipeline b/craft-cache/experimental-pipelines/linux.pipeline similarity index 100% rename from craft-cache/devel-pipelines/linux.pipeline rename to craft-cache/experimental-pipelines/linux.pipeline diff --git a/craft-cache/devel-pipelines/macos.pipeline b/craft-cache/experimental-pipelines/macos.pipeline similarity index 100% copy from craft-cache/devel-pipelines/macos.pipeline copy to craft-cache/experimental-pipelines/macos.pipeline diff --git a/craft-cache/devel-pipelines/windows.pipeline b/craft-cache/experimental-pipelines/windows.pipeline similarity index 100% copy from craft-cache/devel-pipelines/windows.pipeline copy to craft-cache/experimental-pipelines/windows.pipeline diff --git a/craft-cache/stable-pipelines/linux.pipeline b/craft-cache/master-pipelines/linux.pipeline similarity index 90% rename from craft-cache/stable-pipelines/linux.pipeline rename to craft-cache/master-pipelines/linux.pipeline index 2adeecc..c86959f 100644 --- a/craft-cache/stable-pipelines/linux.pipeline +++ b/craft-cache/master-pipelines/linux.pipeline @@ -1,55 +1,56 @@ // Where will the craftmaster configuration be found (relative to the working directory that is)? -def craftmasterConfigDir = "binary-factory-tooling/craft/configs/stable/" +def craftmasterConfigDir = "binary-factory-tooling/craft/configs/master/" // Request a node to be allocated to us node( "Ubuntu1604Craft" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // We are also reliant on the CI System Tooling for parts of this, so grab it too checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] ] // Make sure that Craftmaster is ready to go sh """ python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BinaryCache/ python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BinaryCache/ """ } // Now that is done, it's time to rebuild the Craft Cache! stage('Build Craft Cache') { sh """ cd ~/Craft/BinaryCache/ python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft + python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackagesBase.list python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackages.list """ } } // And finally, we'll upload it to the master server stage('Publish Craft Cache') { sh """ cd "$WORKSPACE/ci-tooling/" python3 helpers/sftp-directory-mirror.py --source ~/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft """ } } } diff --git a/craft-cache/devel-pipelines/macos.pipeline b/craft-cache/master-pipelines/macos.pipeline similarity index 100% rename from craft-cache/devel-pipelines/macos.pipeline rename to craft-cache/master-pipelines/macos.pipeline diff --git a/craft-cache/devel-pipelines/windows.pipeline b/craft-cache/master-pipelines/windows.pipeline similarity index 100% rename from craft-cache/devel-pipelines/windows.pipeline rename to craft-cache/master-pipelines/windows.pipeline diff --git a/craft-cache/stable-pipelines/macos.pipeline b/craft-cache/stable-pipelines/macos.pipeline deleted file mode 100644 index 0f5539b..0000000 --- a/craft-cache/stable-pipelines/macos.pipeline +++ /dev/null @@ -1,72 +0,0 @@ -// Where will the craftmaster configuration be found (relative to the working directory that is)? -def craftmasterConfigDir = "binary-factory-tooling/craft/configs/stable/" - -// Request a node to be allocated to us -node( "macOS" ) { -// We want Timestamps on everything -timestamps { - // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed - catchError { - - // First we want to make sure Craft is ready to go - stage('Preparing Craft') { - // Make sure we start with a clean slate - deleteDir() - - // Grab our tooling which we will need in a few moments - checkout changelog: false, poll: false, scm: [ - $class: 'GitSCM', - branches: [[name: 'master']], - extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], - userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] - ] - - // We are also reliant on the CI System Tooling for parts of this, so grab it too - checkout changelog: false, poll: false, scm: [ - $class: 'GitSCM', - branches: [[name: 'master']], - extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], - userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] - ] - - // Make sure that Craftmaster is ready to go - sh """ - /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BinaryCache/ - /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BinaryCache/ - """ - } - - // Now that is done, it's time to rebuild the Craft Cache! - stage('Build Craft Cache') { - sh """ - cd ~/Craft/BinaryCache/ - /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft - /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackages.list - """ - } - } - - // We need to catch errors here to ensure that the cleanup always runs, even if the upload fails - // While this is a bit wasteful it at least determines that errors in the publishing process leave a builder with a very full disk! - catchError { - // And finally, we'll upload it to the master server - stage('Publish Craft Cache') { - sh """ - cd "$WORKSPACE/ci-tooling/" - /usr/local/bin/python3 helpers/sftp-directory-mirror.py --source ~/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft - """ - } - } - - - // Then we can cleanup everything we did - stage('Cleaning Up') { - sh """ - rm -rf ~/Craft/BinaryCache/ - """ - } - - // As the macOS Slaves are permanent ones, we erase the Workspace as the last thing we do - deleteDir() -} -} diff --git a/craft-cache/stable-pipelines/windows.pipeline b/craft-cache/stable-pipelines/windows.pipeline deleted file mode 100644 index 10b09e2..0000000 --- a/craft-cache/stable-pipelines/windows.pipeline +++ /dev/null @@ -1,72 +0,0 @@ -// Where will the craftmaster configuration be found (relative to the working directory that is)? -def craftmasterConfigDir = "binary-factory-tooling/craft/configs/stable/" - -// Request a node to be allocated to us -node( "WindowsMSVC" ) { -// We want Timestamps on everything -timestamps { - // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed - catchError { - - // First we want to make sure Craft is ready to go - stage('Preparing Craft') { - // Make sure we start with a clean slate - deleteDir() - - // Grab our tooling which we will need in a few moments - checkout changelog: false, poll: false, scm: [ - $class: 'GitSCM', - branches: [[name: 'master']], - extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], - userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] - ] - - // We are also reliant on the CI System Tooling for parts of this, so grab it too - checkout changelog: false, poll: false, scm: [ - $class: 'GitSCM', - branches: [[name: 'master']], - extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], - userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] - ] - - // Make sure that Craftmaster is ready to go - bat """ - python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into "C:/Craft/BinaryCache/" - python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into "C:/Craft/BinaryCache/" - """ - } - - // Now that is done, it's time to rebuild the Craft Cache! - stage('Build Craft Cache') { - bat """ - cd C:/Craft/BinaryCache/ - python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft - python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackages.list - """ - } - } - - // We need to catch errors here to ensure that the cleanup always runs, even if the upload fails - // While this is a bit wasteful it at least determines that errors in the publishing process leave a builder with a very full disk! - catchError { - // Upload it to the master server - stage('Publish Craft Cache') { - bat """ - cd "%WORKSPACE%\\ci-tooling" - python helpers/sftp-directory-mirror.py --source C:/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft - """ - } - } - - // Then we can cleanup everything we did - // It is critical this always happens, otherwise builders can run out of disk space! - stage('Cleaning Up') { - bat """ - rmdir /s /q C:\\Craft\\BinaryCache\\ - """ - } - - // As the Windows Slaves are permanent ones, we erase the Workspace as the last thing we do - deleteDir() -} -} diff --git a/craft/configs/stable/BinaryCachePackages.list b/craft/configs/experimental/BinaryCachePackages.list similarity index 87% rename from craft/configs/stable/BinaryCachePackages.list rename to craft/configs/experimental/BinaryCachePackages.list index cb01311..3c05ffd 100644 --- a/craft/configs/stable/BinaryCachePackages.list +++ b/craft/configs/experimental/BinaryCachePackages.list @@ -1,88 +1,78 @@ -[Packages] -dev-utils/qtbinpatcher -dev-utils/nsis - -[Qt] -libs/qt5 - -[Frameworks] -kde/frameworks - [kde/applications] kde/applications/kate [Notifications] qt-libs/snorenotify [Applications] # Various Applications kdesupport/qca # PIM Libraries libs/libical # libkexiv2 libs/exiv2 # KStars libs/eigen3 # Minuet libs/glib libs/fluidsynth [Extragear] # KDevelop libs/llvm-meta/llvm kdesupport/grantlee # Labplot libs/gsl # Skrooge dev-utils/pkg-config # akonadi libs/boost/boost-graph # Alkimia libs/libgmp libs/mpir # Amarok libs/taglib # qtcreator qt-apps/qtcreator/qtcreator # kdenlive libs/liblame libs/libopus libs/libsamplerate libs/libsdl2 libs/libogg libs/libvorbis libs/libvpx libs/x264 libs/x265 -#libs/ffmpeg +libs/ffmpeg libs/libfftw libs/dlfcn-win32 -#libs/frei0r-plugins +libs/frei0r-plugins libs/mlt kde/plasma/kdecoration kde/plasma/breeze # ocular qt-libs/poppler libs/ghostscript libs/libspectre # calzy ## llvm doesn't install some import libs so we need to cache it to be able to ## get it with a cached llvm dev-utils/clazy # other binary/vlc diff --git a/craft/configs/experimental/BinaryCachePackagesBase.list b/craft/configs/experimental/BinaryCachePackagesBase.list new file mode 100644 index 0000000..d0807b8 --- /dev/null +++ b/craft/configs/experimental/BinaryCachePackagesBase.list @@ -0,0 +1,9 @@ +[Packages] +dev-utils/qtbinpatcher +dev-utils/nsis + +[Qt] +libs/qt5 + +[Frameworks] +kde/frameworks diff --git a/craft/configs/master/CraftBinaryCache.ini b/craft/configs/experimental/CraftBinaryCache.ini similarity index 100% copy from craft/configs/master/CraftBinaryCache.ini copy to craft/configs/experimental/CraftBinaryCache.ini diff --git a/craft/configs/master/CraftBinaryCache.ini b/craft/configs/master/CraftBinaryCache.ini index 3278c5d..4d0112a 100644 --- a/craft/configs/master/CraftBinaryCache.ini +++ b/craft/configs/master/CraftBinaryCache.ini @@ -1,84 +1,82 @@ [General] Branch = master ShallowClone = True [Variables] Msys = C:/Craft/Msys DownloadDir = C:/Craft/Downloads #Values need to be overwritten to create a cache CreateCache = False UseCache = True -CacheVersion = Qt_5.12.3_1 - # Settings applicable for all Crafts matrices # Settings are Category/key=value # Category is case sensitive [GeneralSettings] Paths/Python = C:/Program Files/Python36-32/ Paths/Python27 = C:/Program Files (x86)/Python27-13 Paths/DownloadDir = ${Variables:DownloadDir} Paths/Msys = ${Variables:Msys} ShortPath/Enabled = False ShortPath/EnableJunctions = True ShortPath/JunctionDir = C:/_ Packager/CacheDir = ${Variables:Root}/cache Packager/UseCache = ${Variables:UseCache} Packager/CreateCache = ${Variables:CreateCache} -Packager/CacheVersion = ${Variables:CacheVersion} Blueprints/BlueprintRoot = ${Variables:Root}/blueprints ContinuousIntegration/Enabled = True ContinuousIntegration/ClearBuildFolder = True ContinuousIntegration/UpdateRepository = True CodeSigning/Enabled = True CodeSigning/CommonName = K Desktop Environment e.V. CodeSigning/Organization = K Desktop Environment e.V. CodeSigning/Locality = Berlin CodeSigning/Country = DE Packager/AppxPublisherId = CN=98B52D9A-DF7C-493E-BADC-37004A92EFC8 [BlueprintSettings] extragear/kdevelop/kdevelop.fullKDevelop = True # don't try to pip install on the ci python-modules.ignored = True # keeping all build dirs around takes too much space libs/qt5/qtdoc.ignored = True [windows-msvc2017_64-cl-debug] General/ABI = windows-msvc2017_64-cl Compile/BuildType = Debug [windows-msvc2017_64-cl] General/ABI = windows-msvc2017_64-cl Compile/BuildType = RelWithDebInfo [windows-msvc2017_32-cl] +Paths/Python = C:/Program Files (x86)/Python36-32 General/ABI = windows-msvc2017_32-cl Compile/BuildType = RelWithDebInfo [windows-msvc2019_64-cl] General/ABI = windows-msvc2019_64-cl Compile/BuildType = RelWithDebInfo [windows-mingw_64-gcc] General/ABI = windows-mingw_64-gcc Compile/MakeProgram = mingw32-make Compile/BuildType = RelWithDebInfo [linux-64-gcc] Paths/Python = General/ABI = linux-64-gcc Compile/MakeProgram = make Compile/BuildType = RelWithDebInfo Paths/DownloadDir = ${Variables:Root}/downloads [macos-64-clang] Paths/Python = General/ABI = macos-64-clang Compile/MakeProgram = make Compile/BuildType = RelWithDebInfo Packager/PackageType = MacDMGPackager Paths/DownloadDir = ${Variables:Root}/downloads diff --git a/craft/configs/stable/CraftBinaryCache.ini b/craft/configs/stable/CraftBinaryCache.ini deleted file mode 100644 index 6642af2..0000000 --- a/craft/configs/stable/CraftBinaryCache.ini +++ /dev/null @@ -1,82 +0,0 @@ -[General] -Branch = master -ShallowClone = True - -[Variables] -Msys = C:/Craft/Msys -DownloadDir = C:/Craft/Downloads - -#Values need to be overwritten to create a cache -CreateCache = False -UseCache = True - -# Settings applicable for all Crafts matrices -# Settings are Category/key=value -# Category is case sensitive -[GeneralSettings] -Paths/Python = C:/Program Files/Python36-32/ -Paths/Python27 = C:/Program Files (x86)/Python27-13 -Paths/DownloadDir = ${Variables:DownloadDir} -Paths/Msys = ${Variables:Msys} -ShortPath/Enabled = False -ShortPath/EnableJunctions = True -ShortPath/JunctionDir = C:/_ -Packager/CacheDir = ${Variables:Root}/cache -Packager/UseCache = ${Variables:UseCache} -Packager/CreateCache = ${Variables:CreateCache} -Blueprints/BlueprintRoot = ${Variables:Root}/blueprints -ContinuousIntegration/Enabled = True -ContinuousIntegration/ClearBuildFolder = True -ContinuousIntegration/UpdateRepository = True -CodeSigning/Enabled = True -CodeSigning/CommonName = K Desktop Environment e.V. -CodeSigning/Organization = K Desktop Environment e.V. -CodeSigning/Locality = Berlin -CodeSigning/Country = DE -Packager/AppxPublisherId = CN=98B52D9A-DF7C-493E-BADC-37004A92EFC8 - -[BlueprintSettings] -extragear/kdevelop/kdevelop.fullKDevelop = True - -# don't try to pip install on the ci -python-modules.ignored = True - -# keeping all build dirs around takes too much space -libs/qt5/qtdoc.ignored = True - -[windows-msvc2017_64-cl-debug] -General/ABI = windows-msvc2017_64-cl -Compile/BuildType = Debug - -[windows-msvc2017_64-cl] -General/ABI = windows-msvc2017_64-cl -Compile/BuildType = RelWithDebInfo - -[windows-msvc2015_64-cl] -General/ABI = windows-msvc2015_64-cl -Compile/BuildType = RelWithDebInfo - -[windows-msvc2017_32-cl] -Paths/Python = C:/Program Files (x86)/Python36-32 -General/ABI = windows-msvc2017_32-cl -Compile/BuildType = RelWithDebInfo - -[windows-mingw_64-gcc] -General/ABI = windows-mingw_64-gcc -Compile/MakeProgram = mingw32-make -Compile/BuildType = RelWithDebInfo - -[linux-64-gcc] -Paths/Python = -General/ABI = linux-64-gcc -Compile/MakeProgram = make -Compile/BuildType = RelWithDebInfo -Paths/DownloadDir = ${Variables:Root}/downloads - -[macos-64-clang] -Paths/Python = -General/ABI = macos-64-clang -Compile/MakeProgram = make -Compile/BuildType = RelWithDebInfo -Packager/PackageType = MacDMGPackager -Paths/DownloadDir = ${Variables:Root}/downloads diff --git a/craft/pipeline-templates/appimage.pipeline b/craft/pipeline-templates/appimage.pipeline index 18dd399..c87c74c 100644 --- a/craft/pipeline-templates/appimage.pipeline +++ b/craft/pipeline-templates/appimage.pipeline @@ -1,84 +1,84 @@ // Sometimes we need to include additional parameters to Craft which are specific to the project def craftProjectParams = "" // Where will the craftmaster configuration be found (relative to the working directory that is)? -def craftmasterConfig = "binary-factory-tooling/craft/configs/stable/CraftBinaryCache.ini" +def craftmasterConfig = "binary-factory-tooling/craft/configs/master/CraftBinaryCache.ini" // Determine if we need to build a specific version of this project if( craftTarget != '' ) { // If we have a specified version (Craft target) then we need to pass this along to Craft craftProjectParams = "--target ${craftTarget}" } // Request a node to be allocated to us node( "Appimage" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // Make sure the Git checkouts are up to date sh """ python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository git://anongit.kde.org/craftmaster --into ~/Craft/BinaryFactory/ python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository git://anongit.kde.org/sysadmin/binary-factory-tooling --into ~/Craft/BinaryFactory/ """ // Update Craft itself sh """ cd ~/Craft/BinaryFactory/ python3 craftmaster/Craftmaster.py --config "${craftmasterConfig}" --target ${craftPlatform} -c -i craft """ } stage('Installing Dependencies') { // Ask Craftmaster to ensure all the dependencies are installed for this application we are going to be building sh """ cd ~/Craft/BinaryFactory/ python3 craftmaster/Craftmaster.py --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} ${craftProjectParams} --install-deps ${craftBuildBlueprint} """ } stage('Building') { // Actually build the application now sh """ cd ~/Craft/BinaryFactory/ python3 craftmaster/Craftmaster.py --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} --no-cache ${craftProjectParams} ${craftRebuildBlueprint} """ } stage('Packaging') { // Now generate the appimage for it sh """ cd ~/Craft/BinaryFactory/ // If Craft knew how to generate Appimages we could just run the below... //python3 craftmaster/Craftmaster.py --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} ${craftProjectParams} --package ${craftBuildBlueprint} // Instead we'll have to ask Craft to stage all the resources like it normally would, then we can run the necessary external tools to generate the appimage """ // Now copy it to the Jenkins workspace so it can be grabbed from there sh """ cd ~/Craft/BinaryFactory/ packageDir=\$(python3 "craftmaster/Craftmaster.py" --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} -q --get "packageDestinationDir()" virtual/base) cp -vf \$packageDir/* \$WORKSPACE/ """ } stage('Capturing Package') { // Then we ask Jenkins to capture the generated installers as an artifact archiveArtifacts artifacts: '*.appimage, *.sha256', onlyIfSuccessful: true } } } } diff --git a/craft/pipeline-templates/macos.pipeline b/craft/pipeline-templates/macos.pipeline index aabad25..adf2a45 100644 --- a/craft/pipeline-templates/macos.pipeline +++ b/craft/pipeline-templates/macos.pipeline @@ -1,107 +1,107 @@ // Sometimes we need to include additional parameters to Craft which are specific to the project def craftProjectParams = "" // Where will the craftmaster configuration be found (relative to the working directory that is)? -def craftmasterConfig = "binary-factory-tooling/craft/configs/stable/CraftBinaryCache.ini" +def craftmasterConfig = "binary-factory-tooling/craft/configs/master/CraftBinaryCache.ini" // Determine if we need to build a specific version of this project if( craftTarget != '' ) { // If we have a specified version (Craft target) then we need to pass this along to Craft craftProjectParams = "--target ${craftTarget}" } // Request a node to be allocated to us node( "macOS" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Make sure we start with a clean slate deleteDir() // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // Make sure the Git checkouts are up to date sh """ /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository git://anongit.kde.org/craftmaster --into ~/Craft/BinaryFactory/ /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository git://anongit.kde.org/sysadmin/binary-factory-tooling --into ~/Craft/BinaryFactory/ """ // Update Craft itself sh """ cd ~/Craft/BinaryFactory/ /usr/local/bin/python3 craftmaster/Craftmaster.py --config "${craftmasterConfig}" --target ${craftPlatform} -c -i craft """ } stage('Cleaning Up Prior Builds') { // Cleanup our workspace deleteDir() // Ensure we have nothing left behind in the packaging workspace used by Craft sh """ cd ~/Craft/BinaryFactory/ packageDir=\$(/usr/local/bin/python3 "craftmaster/Craftmaster.py" --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} -q --get "packageDestinationDir()" virtual/base) rm -rf "\$packageDir" """ // Make sure the build environment for this application is clean sh """ cd ~/Craft/BinaryFactory/ /usr/local/bin/python3 craftmaster/Craftmaster.py --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} ${craftProjectParams} --unmerge ${craftRebuildBlueprint} """ } stage('Installing Dependencies') { // Ask Craftmaster to ensure all the dependencies are installed for this application we are going to be building sh """ cd ~/Craft/BinaryFactory/ /usr/local/bin/python3 craftmaster/Craftmaster.py --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} ${craftProjectParams} --install-deps ${craftBuildBlueprint} """ } stage('Building') { // Actually build the application now sh """ cd ~/Craft/BinaryFactory/ /usr/local/bin/python3 craftmaster/Craftmaster.py --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} --no-cache ${craftProjectParams} ${craftRebuildBlueprint} """ } stage('Packaging') { // Now generate an installer for it sh """ cd ~/Craft/BinaryFactory/ /usr/local/bin/python3 craftmaster/Craftmaster.py --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} ${craftProjectParams} --package ${craftBuildBlueprint} """ // Now copy it to the Jenkins workspace so it can be grabbed from there sh """ cd ~/Craft/BinaryFactory/ packageDir=\$(/usr/local/bin/python3 "craftmaster/Craftmaster.py" --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} -q --get "packageDestinationDir()" virtual/base) cp -vf \$packageDir/* \$WORKSPACE/ rm -rf "\$packageDir" """ } stage('Capturing Package') { // Then we ask Jenkins to capture the generated installers as an artifact archiveArtifacts artifacts: '*.dmg, *.sha256', onlyIfSuccessful: true } } // As the Mac Slaves are permanent ones, we erase the Workspace as the last thing we do deleteDir() } } diff --git a/craft/pipeline-templates/win64.pipeline b/craft/pipeline-templates/win64.pipeline index b22fc3a..924c267 100644 --- a/craft/pipeline-templates/win64.pipeline +++ b/craft/pipeline-templates/win64.pipeline @@ -1,116 +1,116 @@ // Sometimes we need to include additional parameters to Craft which are specific to the project def craftProjectParams = "" // Where will the craftmaster configuration be found (relative to the working directory that is)? -def craftmasterConfig = "binary-factory-tooling/craft/configs/stable/CraftBinaryCache.ini" +def craftmasterConfig = "binary-factory-tooling/craft/configs/master/CraftBinaryCache.ini" // Determine if we need to build a specific version of this project if( craftTarget != '' ) { // If we have a specified version (Craft target) then we need to pass this along to Craft craftProjectParams = "--target ${craftTarget}" } // Request a node to be allocated to us node( "WindowsMSVC" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Make sure we start with a clean slate deleteDir() // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // Make sure that Craftmaster is up to date bat """ python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository git://anongit.kde.org/craftmaster --into "C:/Craft/BinaryFactory/" python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository git://anongit.kde.org/sysadmin/binary-factory-tooling --into "C:/Craft/BinaryFactory/" """ // Update Craft itself and make sure NSIS is installed bat """ cd "C:\\Craft\\BinaryFactory\\" python craftmaster/Craftmaster.py --config "${craftmasterConfig}" --target ${craftPlatform} -c -i craft if errorlevel 1 exit /b %errorlevel% python craftmaster/Craftmaster.py --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} nsis if errorlevel 1 exit /b %errorlevel% """ } stage('Cleaning Up Prior Builds') { // Ensure we have nothing left behind in the packaging workspace used by Craft powershell """ cd "C:/Craft/BinaryFactory/" \$CRAFT_TMPDIR = python "craftmaster/Craftmaster.py" --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} -q --get "packageDestinationDir()" virtual/base rm "\$CRAFT_TMPDIR/*" """ // Make sure the build environment for this application is clean bat """ cd "C:\\Craft\\BinaryFactory\\" python craftmaster/Craftmaster.py --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} ${craftProjectParams} --unmerge ${craftRebuildBlueprint} if errorlevel 1 exit /b %errorlevel% """ } stage('Installing Dependencies') { // Ask Craftmaster to ensure all the dependencies are installed for this application we are going to be building bat """ cd "C:\\Craft\\BinaryFactory\\" python craftmaster/Craftmaster.py --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} ${craftProjectParams} --install-deps ${craftBuildBlueprint} if errorlevel 1 exit /b %errorlevel% """ } stage('Building') { // Actually build the application now bat """ cd "C:\\Craft\\BinaryFactory\\" python craftmaster/Craftmaster.py --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} --no-cache ${craftProjectParams} ${craftRebuildBlueprint} if errorlevel 1 exit /b %errorlevel% """ } stage('Packaging') { // Now generate an installer for it powershell """ cd "C:/Craft/BinaryFactory/" python craftmaster/Craftmaster.py --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} ${craftProjectParams} --package ${craftBuildBlueprint} if(\$LASTEXITCODE -ne 0) {exit \$LASTEXITCODE} if (\$${craftPackageAppx}) { python craftmaster/Craftmaster.py --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} --options "[Packager]PackageType=AppxPackager" ${craftProjectParams} --package ${craftBuildBlueprint} } if(\$LASTEXITCODE -ne 0) {exit \$LASTEXITCODE} """ // Copy it to the Jenkins workspace so it can be grabbed from there powershell """ cd "C:/Craft/BinaryFactory/" \$CRAFT_TMPDIR = python "craftmaster/Craftmaster.py" --config "${craftmasterConfig}" --target ${craftPlatform} -c ${craftOptions} -q --get "packageDestinationDir()" virtual/base copy "\$CRAFT_TMPDIR/*" "\$env:WORKSPACE" rm "\$CRAFT_TMPDIR/*" """ } stage('Capturing Package') { // Then we ask Jenkins to capture the generated installers as an artifact archiveArtifacts artifacts: '*.appx, *.exe, *.7z, *.sha256', onlyIfSuccessful: true } } // As the Windows Slaves are permanent ones, we erase the Workspace as the last thing we do deleteDir() } } diff --git a/dsl/craft_cache_jobs.groovy b/dsl/craft_cache_jobs.groovy index 599a5eb..7552e09 100644 --- a/dsl/craft_cache_jobs.groovy +++ b/dsl/craft_cache_jobs.groovy @@ -1,162 +1,162 @@ // Read the contents of the gathered-jobs.json file a step created for us previously -def masterJobsToParse = readFileFromWorkspace('craft-cache/devel-pipelines/gathered-jobs.json') +def masterJobsToParse = readFileFromWorkspace('craft-cache/master-pipelines/gathered-jobs.json') def knownMasterJobs = new groovy.json.JsonSlurper().parseText( masterJobsToParse ) // First do the unstable/master branch... knownMasterJobs.each { // Create our job name - def jobName = "Craft_Build_Cache_${it.craftPlatform}" + def jobName = "Craft_Build_Master_Cache_${it.craftPlatform}" // Read in the necessary Pipeline template - def pipelineTemplate = readFileFromWorkspace("craft-cache/devel-pipelines/${it.buildPipeline}.pipeline") + def pipelineTemplate = readFileFromWorkspace("craft-cache/master-pipelines/${it.buildPipeline}.pipeline") // Now we can construct our Pipeline script // We append a series of variables to the top of it to provide a variety of useful information to the otherwise templated script // These appended variables are what makes one build different to the next, aside from the template which was used def pipelineScript = """ |def craftPlatform = "${it.craftPlatform}" |${pipelineTemplate}""".stripMargin() // Actually create the job now pipelineJob( jobName ) { properties { // We don't want to keep build results forever // We'll set it to keep the last 10 builds and discard everything else buildDiscarder { strategy { logRotator { numToKeepStr("5") daysToKeepStr('') artifactDaysToKeepStr('') artifactNumToKeepStr('') } } } // We don't want to be building the same project more than once // This is to prevent one project hogging resources // And also has a practical component as otherwise an older build could finish afterwards and upload old build results disableConcurrentBuilds() } // This is where the Pipeline script actually happens :) definition { cps { script( pipelineScript ) sandbox() } } } } -// We also want to create similar jobs for the stable branch +// We also want to create similar jobs for the experimental branch // Read the contents of the gathered-jobs.json file a step created for us previously -def stableJobsToParse = readFileFromWorkspace('craft-cache/stable-pipelines/gathered-jobs.json') -def knownStableJobs = new groovy.json.JsonSlurper().parseText( stableJobsToParse ) +def experimentalJobsToParse = readFileFromWorkspace('craft-cache/experimental-pipelines/gathered-jobs.json') +def knownExperimentalJobs = new groovy.json.JsonSlurper().parseText( experimentalJobsToParse ) -knownStableJobs.each { +experimentalStableJobs.each { // Create our job name - def jobName = "Craft_Build_Stable_Cache_${it.craftPlatform}" + def jobName = "Craft_Build_Experimental_Cache_${it.craftPlatform}" // Read in the necessary Pipeline template - def pipelineTemplate = readFileFromWorkspace("craft-cache/stable-pipelines/${it.buildPipeline}.pipeline") + def pipelineTemplate = readFileFromWorkspace("craft-cache/experimental-pipelines/${it.buildPipeline}.pipeline") // Now we can construct our Pipeline script // We append a series of variables to the top of it to provide a variety of useful information to the otherwise templated script // These appended variables are what makes one build different to the next, aside from the template which was used def pipelineScript = """ |def craftPlatform = "${it.craftPlatform}" |${pipelineTemplate}""".stripMargin() // Actually create the job now pipelineJob( jobName ) { properties { // We don't want to keep build results forever // We'll set it to keep the last 10 builds and discard everything else buildDiscarder { strategy { logRotator { numToKeepStr("5") daysToKeepStr('') artifactDaysToKeepStr('') artifactNumToKeepStr('') } } } // We don't want to be building the same project more than once // This is to prevent one project hogging resources // And also has a practical component as otherwise an older build could finish afterwards and upload old build results disableConcurrentBuilds() } // This is where the Pipeline script actually happens :) definition { cps { script( pipelineScript ) sandbox() } } } } // We also want to ensure a cleanup job for the nodes is created // Read in the necessary Pipeline template def pipelineScript = readFileFromWorkspace("craft-cache/cleanup-nodes.pipeline") // Actually create the job now pipelineJob( "Craft_Builder_Cleanup" ) { properties { // We don't want to keep build results forever // We'll set it to keep the last 10 builds and discard everything else buildDiscarder { strategy { logRotator { numToKeepStr("5") daysToKeepStr('') artifactDaysToKeepStr('') artifactNumToKeepStr('') } } } // We don't want to be building the same project more than once // This is to prevent one project hogging resources // And also has a practical component as otherwise an older build could finish afterwards and upload old build results disableConcurrentBuilds() } // This is where the Pipeline script actually happens :) definition { cps { script( pipelineScript ) sandbox() } } } // We also want to ensure a MingW for MSVC build exists pipelineScript = readFileFromWorkspace("craft-cache/mingw-for-msvc.pipeline") // Actually create the job now pipelineJob( "MinGW-w64_binaries_for_MSVC" ) { properties { // We don't want to keep build results forever // We'll set it to keep the last 10 builds and discard everything else buildDiscarder { strategy { logRotator { numToKeepStr("5") daysToKeepStr('') artifactDaysToKeepStr('') artifactNumToKeepStr('') } } } // We don't want to be building the same project more than once // This is to prevent one project hogging resources // And also has a practical component as otherwise an older build could finish afterwards and upload old build results disableConcurrentBuilds() } // This is where the Pipeline script actually happens :) definition { cps { script( pipelineScript ) sandbox() } } }