diff --git a/craft-cache/devel-pipelines/linux.pipeline b/craft-cache/devel-pipelines/linux.pipeline index 5322bbd..c2165b3 100644 --- a/craft-cache/devel-pipelines/linux.pipeline +++ b/craft-cache/devel-pipelines/linux.pipeline @@ -1,55 +1,56 @@ // Where will the craftmaster configuration be found (relative to the working directory that is)? def craftmasterConfigDir = "binary-factory-tooling/craft/configs/master/" // Request a node to be allocated to us node( "Ubuntu1604Craft" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // We are also reliant on the CI System Tooling for parts of this, so grab it too checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] ] // Make sure that Craftmaster is ready to go sh """ python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BinaryCache/ python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BinaryCache/ """ } // Now that is done, it's time to rebuild the Craft Cache! stage('Build Craft Cache') { sh """ cd ~/Craft/BinaryCache/ python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft - python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --option "[ContinuousIntegration]ClearBuildFolder=False" --list-file ${craftmasterConfigDir}/BinaryCachePackages.list + python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackagesBase.list + python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackages.list """ } } // And finally, we'll upload it to the master server stage('Publish Craft Cache') { sh """ cd "$WORKSPACE/ci-tooling/" python3 helpers/sftp-directory-mirror.py --source ~/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft """ } } } diff --git a/craft-cache/devel-pipelines/macos.pipeline b/craft-cache/devel-pipelines/macos.pipeline index 774b440..4d812c9 100644 --- a/craft-cache/devel-pipelines/macos.pipeline +++ b/craft-cache/devel-pipelines/macos.pipeline @@ -1,72 +1,73 @@ // Where will the craftmaster configuration be found (relative to the working directory that is)? def craftmasterConfigDir = "binary-factory-tooling/craft/configs/master/" // Request a node to be allocated to us node( "macOS" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Make sure we start with a clean slate deleteDir() // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // We are also reliant on the CI System Tooling for parts of this, so grab it too checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] ] // Make sure that Craftmaster is ready to go sh """ /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BinaryCache/ /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BinaryCache/ """ } // Now that is done, it's time to rebuild the Craft Cache! stage('Build Craft Cache') { sh """ cd ~/Craft/BinaryCache/ /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft - /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --option "[ContinuousIntegration]ClearBuildFolder=False" --list-file ${craftmasterConfigDir}/BinaryCachePackages.list + /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackagesBase.list + /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackages.list """ } } // We need to catch errors here to ensure that the cleanup always runs, even if the upload fails // While this is a bit wasteful it at least determines that errors in the publishing process leave a builder with a very full disk! catchError { // And finally, we'll upload it to the master server stage('Publish Craft Cache') { sh """ cd "$WORKSPACE/ci-tooling/" /usr/local/bin/python3 helpers/sftp-directory-mirror.py --source ~/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft """ } } // Then we can cleanup everything we did stage('Cleaning Up') { sh """ rm -rf ~/Craft/BinaryCache/ """ } // As the macOS Slaves are permanent ones, we erase the Workspace as the last thing we do deleteDir() } } diff --git a/craft-cache/devel-pipelines/windows.pipeline b/craft-cache/devel-pipelines/windows.pipeline index 53782ff..059aaa4 100644 --- a/craft-cache/devel-pipelines/windows.pipeline +++ b/craft-cache/devel-pipelines/windows.pipeline @@ -1,72 +1,75 @@ // Where will the craftmaster configuration be found (relative to the working directory that is)? def craftmasterConfigDir = "binary-factory-tooling/craft/configs/master/" // Request a node to be allocated to us node( "WindowsMSVC" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Make sure we start with a clean slate deleteDir() // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // We are also reliant on the CI System Tooling for parts of this, so grab it too checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] ] // Make sure that Craftmaster is ready to go bat """ python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into "C:/Craft/BinaryCache/" python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into "C:/Craft/BinaryCache/" """ } // Now that is done, it's time to rebuild the Craft Cache! stage('Build Craft Cache') { bat """ cd C:/Craft/BinaryCache/ python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft - python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --option "[ContinuousIntegration]ClearBuildFolder=False" --list-file ${craftmasterConfigDir}/BinaryCachePackages.list + python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackagesBase.list + if not "${craftPlatform}" == "windows-msvc2017_32-cl" ( + python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackages.list + ) """ } } // We need to catch errors here to ensure that the cleanup always runs, even if the upload fails // While this is a bit wasteful it at least determines that errors in the publishing process leave a builder with a very full disk! catchError { // Upload it to the master server stage('Publish Craft Cache') { bat """ cd "%WORKSPACE%\\ci-tooling" python helpers/sftp-directory-mirror.py --source C:/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft """ } } // Then we can cleanup everything we did // It is critical this always happens, otherwise builders can run out of disk space! stage('Cleaning Up') { bat """ rmdir /s /q C:\\Craft\\BinaryCache\\ """ } // As the Windows Slaves are permanent ones, we erase the Workspace as the last thing we do deleteDir() } } diff --git a/craft/configs/master/BinaryCachePackages.list b/craft/configs/master/BinaryCachePackages.list index cb01311..a603c6d 100644 --- a/craft/configs/master/BinaryCachePackages.list +++ b/craft/configs/master/BinaryCachePackages.list @@ -1,88 +1,78 @@ -[Packages] -dev-utils/qtbinpatcher -dev-utils/nsis - -[Qt] -libs/qt5 - -[Frameworks] -kde/frameworks - [kde/applications] kde/applications/kate [Notifications] qt-libs/snorenotify [Applications] # Various Applications kdesupport/qca # PIM Libraries libs/libical # libkexiv2 libs/exiv2 # KStars libs/eigen3 # Minuet libs/glib libs/fluidsynth [Extragear] # KDevelop libs/llvm-meta/llvm kdesupport/grantlee # Labplot libs/gsl # Skrooge dev-utils/pkg-config # akonadi libs/boost/boost-graph # Alkimia libs/libgmp libs/mpir # Amarok libs/taglib # qtcreator qt-apps/qtcreator/qtcreator # kdenlive libs/liblame libs/libopus libs/libsamplerate libs/libsdl2 libs/libogg libs/libvorbis libs/libvpx libs/x264 libs/x265 #libs/ffmpeg libs/libfftw libs/dlfcn-win32 #libs/frei0r-plugins libs/mlt kde/plasma/kdecoration kde/plasma/breeze # ocular qt-libs/poppler libs/ghostscript libs/libspectre # calzy ## llvm doesn't install some import libs so we need to cache it to be able to ## get it with a cached llvm dev-utils/clazy # other binary/vlc diff --git a/craft/configs/master/BinaryCachePackagesBase.list b/craft/configs/master/BinaryCachePackagesBase.list new file mode 100644 index 0000000..d0807b8 --- /dev/null +++ b/craft/configs/master/BinaryCachePackagesBase.list @@ -0,0 +1,9 @@ +[Packages] +dev-utils/qtbinpatcher +dev-utils/nsis + +[Qt] +libs/qt5 + +[Frameworks] +kde/frameworks diff --git a/craft/configs/master/CraftBinaryCache.ini b/craft/configs/master/CraftBinaryCache.ini index 605520a..f8edfe9 100644 --- a/craft/configs/master/CraftBinaryCache.ini +++ b/craft/configs/master/CraftBinaryCache.ini @@ -1,83 +1,87 @@ [General] Branch = master ShallowClone = True [Variables] Msys = C:/Craft/Msys DownloadDir = C:/Craft/Downloads #Values need to be overwritten to create a cache CreateCache = False UseCache = True CacheVersion = Qt_5.12.3 # Settings applicable for all Crafts matrices # Settings are Category/key=value # Category is case sensitive [GeneralSettings] Paths/Python = C:/Program Files/Python36-32/ Paths/Python27 = C:/Program Files (x86)/Python27-13 Paths/DownloadDir = ${Variables:DownloadDir} Paths/Msys = ${Variables:Msys} ShortPath/Enabled = False ShortPath/EnableJunctions = True ShortPath/JunctionDir = C:/_ Packager/CacheDir = ${Variables:Root}/cache Packager/UseCache = ${Variables:UseCache} Packager/CreateCache = ${Variables:CreateCache} Packager/CacheVersion = ${Variables:CacheVersion} Blueprints/BlueprintRoot = ${Variables:Root}/blueprints ContinuousIntegration/Enabled = True ContinuousIntegration/ClearBuildFolder = True ContinuousIntegration/UpdateRepository = True CodeSigning/Enabled = True CodeSigning/CommonName = K Desktop Environment e.V. CodeSigning/Organization = K Desktop Environment e.V. CodeSigning/Locality = Berlin CodeSigning/Country = DE Packager/AppxPublisherId = CN=98B52D9A-DF7C-493E-BADC-37004A92EFC8 [BlueprintSettings] libs/qt5.version = 5.12.3 libs/llvm-meta.version = 8.0.0 extragear/kdevelop/kdevelop.fullKDevelop = True # don't try to pip install on the ci python-modules.ignored = True # keeping all build dirs around takes too much space libs/qt5/qtdoc.ignored = True [windows-msvc2017_64-cl-debug] General/ABI = windows-msvc2017_64-cl Compile/BuildType = Debug [windows-msvc2017_64-cl] General/ABI = windows-msvc2017_64-cl Compile/BuildType = RelWithDebInfo +[windows-msvc2017_32-cl] +General/ABI = windows-msvc2017_32-cl +Compile/BuildType = RelWithDebInfo + [windows-msvc2019_64-cl] General/ABI = windows-msvc2019_64-cl Compile/BuildType = RelWithDebInfo [windows-mingw_64-gcc] General/ABI = windows-mingw_64-gcc Compile/MakeProgram = mingw32-make Compile/BuildType = RelWithDebInfo [linux-64-gcc] Paths/Python = General/ABI = linux-64-gcc Compile/MakeProgram = make Compile/BuildType = RelWithDebInfo Paths/DownloadDir = ${Variables:Root}/downloads [macos-64-clang] Paths/Python = General/ABI = macos-64-clang Compile/MakeProgram = make Compile/BuildType = RelWithDebInfo Packager/PackageType = MacDMGPackager Paths/DownloadDir = ${Variables:Root}/downloads