diff --git a/craft-cache/prebuild/linux.pipeline b/craft-cache/prebuild/linux.pipeline new file mode 100644 index 0000000..00fdf0f --- /dev/null +++ b/craft-cache/prebuild/linux.pipeline @@ -0,0 +1,56 @@ +// Where will the craftmaster configuration be found (relative to the working directory that is)? +def craftmasterConfigDir = "binary-factory-tooling/craft/configs/prebuilt/" + +// Request a node to be allocated to us +node( "Ubuntu1604Craft" ) { +// We want Timestamps on everything +timestamps { + // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed + catchError { + + // First we want to make sure Craft is ready to go + stage('Preparing Craft') { + // Grab our tooling which we will need in a few moments + checkout changelog: false, poll: false, scm: [ + $class: 'GitSCM', + branches: [[name: 'master']], + extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], + userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] + ] + + // We are also reliant on the CI System Tooling for parts of this, so grab it too + checkout changelog: false, poll: false, scm: [ + $class: 'GitSCM', + branches: [[name: 'master']], + extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], + userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] + ] + + // Make sure that Craftmaster is ready to go + sh """ + python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BinaryCache/ + python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BinaryCache/ + """ + } + + // Now that is done, it's time to rebuild the Craft Cache! + stage('Build Craft Cache') { + sh """ + cd ~/Craft/BinaryCache/ + python3.6 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft + python3.6 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --add-blueprint-repository git@invent.kde.org:vonreth/craft-blueprints-prebuilt.git + python3.6 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c prebuilt + """ + } + + } + + // And finally, we'll upload it to the master server + stage('Publish Craft Cache') { + sh """ + cd "$WORKSPACE/ci-tooling/" + python3 helpers/sftp-directory-mirror.py --source ~/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/prebuilt/ --server milonia.kde.org --username craft + """ + } +} +} diff --git a/craft-cache/prebuild/macos.pipeline b/craft-cache/prebuild/macos.pipeline new file mode 100644 index 0000000..1ba0622 --- /dev/null +++ b/craft-cache/prebuild/macos.pipeline @@ -0,0 +1,73 @@ +// Where will the craftmaster configuration be found (relative to the working directory that is)? +def craftmasterConfigDir = "binary-factory-tooling/craft/configs/prebuilt/" + +// Request a node to be allocated to us +node( "macOS" ) { +// We want Timestamps on everything +timestamps { + // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed + catchError { + + // First we want to make sure Craft is ready to go + stage('Preparing Craft') { + // Make sure we start with a clean slate + deleteDir() + + // Grab our tooling which we will need in a few moments + checkout changelog: false, poll: false, scm: [ + $class: 'GitSCM', + branches: [[name: 'master']], + extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], + userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] + ] + + // We are also reliant on the CI System Tooling for parts of this, so grab it too + checkout changelog: false, poll: false, scm: [ + $class: 'GitSCM', + branches: [[name: 'master']], + extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], + userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] + ] + + // Make sure that Craftmaster is ready to go + sh """ + /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BinaryCache/ + /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BinaryCache/ + """ + } + + // Now that is done, it's time to rebuild the Craft Cache! + stage('Build Craft Cache') { + sh """ + cd ~/Craft/BinaryCache/ + /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft + /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --add-blueprint-repository git@invent.kde.org:vonreth/craft-blueprints-prebuilt.git + /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c prebuilt + """ + } + } + + // We need to catch errors here to ensure that the cleanup always runs, even if the upload fails + // While this is a bit wasteful it at least determines that errors in the publishing process leave a builder with a very full disk! + catchError { + // And finally, we'll upload it to the master server + stage('Publish Craft Cache') { + sh """ + cd "$WORKSPACE/ci-tooling/" + /usr/local/bin/python3 helpers/sftp-directory-mirror.py --source ~/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/prebuilt/ --server milonia.kde.org --username craft + """ + } + } + + + // Then we can cleanup everything we did + stage('Cleaning Up') { + sh """ + rm -rf ~/Craft/BinaryCache/ + """ + } + + // As the macOS Slaves are permanent ones, we erase the Workspace as the last thing we do + deleteDir() +} +} diff --git a/craft-cache/prebuild/windows.pipeline b/craft-cache/prebuild/windows.pipeline new file mode 100644 index 0000000..72231cf --- /dev/null +++ b/craft-cache/prebuild/windows.pipeline @@ -0,0 +1,88 @@ +// Where will the craftmaster configuration be found (relative to the working directory that is)? +def craftmasterConfigDir = "binary-factory-tooling/craft/configs/prebuilt/" + +// Request a node to be allocated to us +node( "WindowsMSVC" ) { +// We want Timestamps on everything +timestamps { + // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed + catchError { + + // First things first, make sure everything that could be using up disk space is removed + // This ensures that WebEngine builds don't get tripped up... + stage('Cleaning Up Craft') { + bat """ + rmdir /s /q C:\\Craft\\BinaryCache\\ + rmdir /s /q C:\\Craft\\BinaryFactory\\ + rmdir /s /q C:\\Craft\\Downloads\\ + rmdir /s /q C:\\_\\ + exit /b 0 + """ + } + + // First we want to make sure Craft is ready to go + stage('Preparing Craft') { + // Make sure we start with a clean slate + deleteDir() + + // Grab our tooling which we will need in a few moments + checkout changelog: false, poll: false, scm: [ + $class: 'GitSCM', + branches: [[name: 'master']], + extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], + userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] + ] + + // We are also reliant on the CI System Tooling for parts of this, so grab it too + checkout changelog: false, poll: false, scm: [ + $class: 'GitSCM', + branches: [[name: 'master']], + extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], + userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] + ] + + // Make sure that Craftmaster is ready to go + bat """ + python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into "C:/Craft/BinaryCache/" + python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into "C:/Craft/BinaryCache/" + """ + } + + // Now that is done, it's time to rebuild the Craft Cache! + stage('Build Craft Cache') { + bat """ + setlocal enabledelayedexpansion + + cd C:/Craft/BinaryCache/ + + python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft + python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --add-blueprint-repository git@invent.kde.org:vonreth/craft-blueprints-prebuilt.git + python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c prebuilt + """ + } + } + + // We need to catch errors here to ensure that the cleanup always runs, even if the upload fails + // While this is a bit wasteful it at least determines that errors in the publishing process leave a builder with a very full disk! + catchError { + // Upload it to the master server + stage('Publish Craft Cache') { + bat """ + cd "%WORKSPACE%\\ci-tooling" + python helpers/sftp-directory-mirror.py --source C:/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/prebuilt/ --server milonia.kde.org --username craft + """ + } + } + + // Then we can cleanup everything we did + // It is critical this always happens, otherwise builders can run out of disk space! + stage('Cleaning Up') { + bat """ + rmdir /s /q C:\\Craft\\BinaryCache\\ + """ + } + + // As the Windows Slaves are permanent ones, we erase the Workspace as the last thing we do + deleteDir() +} +} diff --git a/craft/configs/prebuilt/CraftBinaryCache.ini b/craft/configs/prebuilt/CraftBinaryCache.ini new file mode 100644 index 0000000..d0f41f6 --- /dev/null +++ b/craft/configs/prebuilt/CraftBinaryCache.ini @@ -0,0 +1,70 @@ +[General] +Branch = master +ShallowClone = True + +[Variables] +Msys = C:/Craft/Msys +DownloadDir = C:/Craft/Downloads + +#Values need to be overwritten to create a cache +CreateCache = False +UseCache = True + +# Settings applicable for all Crafts matrices +# Settings are Category/key=value +# Category is case sensitive +[GeneralSettings] +Paths/Python = C:/Program Files/Python36-32/ +Paths/Python27 = C:/Program Files (x86)/Python27-13 +Paths/DownloadDir = ${Variables:DownloadDir} +Paths/Msys = ${Variables:Msys} +ShortPath/Enabled = False +ShortPath/EnableJunctions = True +ShortPath/JunctionDir = C:/_ +Packager/CacheDir = ${Variables:Root}/cache +Packager/UseCache = ${Variables:UseCache} +Packager/CreateCache = ${Variables:CreateCache} +Blueprints/BlueprintRoot = ${Variables:Root}/blueprints +ContinuousIntegration/Enabled = True +ContinuousIntegration/ClearBuildFolder = True +ContinuousIntegration/UpdateRepository = True +CodeSigning/Enabled = True +CodeSigning/CommonName = K Desktop Environment e.V. +CodeSigning/Organization = K Desktop Environment e.V. +CodeSigning/Locality = Berlin +CodeSigning/Country = DE +Packager/AppxPublisherId = CN=98B52D9A-DF7C-493E-BADC-37004A92EFC8 + + +[windows-msvc2019_64-cl] +General/ABI = windows-msvc2019_64-cl +Compile/BuildType = RelWithDebInfo + +[windows-msvc2019_32-cl] +General/ABI = windows-msvc2019_62-cl +Compile/BuildType = RelWithDebInfo + +[windows-mingw_62-gcc] +General/ABI = windows-mingw_62-gcc +Compile/MakeProgram = mingw32-make +Compile/BuildType = RelWithDebInfo + +[windows-mingw_64-gcc] +General/ABI = windows-mingw_64-gcc +Compile/MakeProgram = mingw32-make +Compile/BuildType = RelWithDebInfo + +[linux-64-gcc] +Paths/Python = +General/ABI = linux-64-gcc +Compile/MakeProgram = make +Compile/BuildType = RelWithDebInfo +Paths/DownloadDir = ${Variables:Root}/downloads + +[macos-64-clang] +Paths/Python = +General/ABI = macos-64-clang +Compile/MakeProgram = make +Compile/BuildType = RelWithDebInfo +Packager/PackageType = MacDMGPackager +Paths/DownloadDir = ${Variables:Root}/downloads