diff --git a/craft-cache/src-pipelines/linux.pipeline b/craft-cache/src-pipelines/linux.pipeline new file mode 100644 index 0000000..ea99c1b --- /dev/null +++ b/craft-cache/src-pipelines/linux.pipeline @@ -0,0 +1,55 @@ +// Where will the craftmaster configuration be found (relative to the working directory that is)? +def craftmasterConfigDir = "binary-factory-tooling/craft/configs/experimental/" + +// Request a node to be allocated to us +node( "Ubuntu1804Craft" ) { +// We want Timestamps on everything +timestamps { + // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed + catchError { + + // First we want to make sure Craft is ready to go + stage('Preparing Craft') { + // Grab our tooling which we will need in a few moments + checkout changelog: false, poll: false, scm: [ + $class: 'GitSCM', + branches: [[name: 'master']], + extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], + userRemoteConfigs: [[url: 'https://invent.kde.org/sysadmin/binary-factory-tooling.git']] + ] + + // We are also reliant on the CI System Tooling for parts of this, so grab it too + checkout changelog: false, poll: false, scm: [ + $class: 'GitSCM', + branches: [[name: 'master']], + extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], + userRemoteConfigs: [[url: 'https://invent.kde.org/sysadmin/ci-tooling.git']] + ] + + // Make sure that Craftmaster is ready to go + sh """ + python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "https://invent.kde.org/packaging/craftmaster.git" --into ~/Craft/BC/ + python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "https://invent.kde.org/sysadmin/binary-factory-tooling.git" --into ~/Craft/BC/ + """ + } + + // Now that is done, it's time to rebuild the Craft Cache! + stage('Build Craft Cache') { + sh """ + cd ~/Craft/BC/ + python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft + python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables "DownloadDir=~/Craft/BC/src" -c --create-download-cache --list-file ${craftmasterConfigDir}/BinaryCachePackages.list + """ + } + + } + + // And finally, we'll upload it to the master server + stage('Publish Craft Cache') { + sh """ + cd "$WORKSPACE/ci-tooling/" + python3 helpers/sftp-directory-mirror.py --source ~/Craft/BC/src/ --destination /srv/archives/files/craft/src/ --server milonia.kde.org --username craft + """ + } +} +} diff --git a/craft-cache/src-pipelines/macos.pipeline b/craft-cache/src-pipelines/macos.pipeline new file mode 100644 index 0000000..6303929 --- /dev/null +++ b/craft-cache/src-pipelines/macos.pipeline @@ -0,0 +1,72 @@ +// Where will the craftmaster configuration be found (relative to the working directory that is)? +def craftmasterConfigDir = "binary-factory-tooling/craft/configs/experimental/" + +// Request a node to be allocated to us +node( "macOS" ) { +// We want Timestamps on everything +timestamps { + // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed + catchError { + + // First we want to make sure Craft is ready to go + stage('Preparing Craft') { + // Make sure we start with a clean slate + deleteDir() + + // Grab our tooling which we will need in a few moments + checkout changelog: false, poll: false, scm: [ + $class: 'GitSCM', + branches: [[name: 'master']], + extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], + userRemoteConfigs: [[url: 'https://invent.kde.org/sysadmin/binary-factory-tooling.git']] + ] + + // We are also reliant on the CI System Tooling for parts of this, so grab it too + checkout changelog: false, poll: false, scm: [ + $class: 'GitSCM', + branches: [[name: 'master']], + extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], + userRemoteConfigs: [[url: 'https://invent.kde.org/sysadmin/ci-tooling.git']] + ] + + // Make sure that Craftmaster is ready to go + sh """ + /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "https://invent.kde.org/packaging/craftmaster.git" --into ~/Craft/BC/ + /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "https://invent.kde.org/sysadmin/binary-factory-tooling.git" --into ~/Craft/BC/ + """ + } + + // Now that is done, it's time to rebuild the Craft Cache! + stage('Build Craft Cache') { + sh """ + cd ~/Craft/BC/ + /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft + /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables "DownloadDir=~/Craft/BC/src" -c --create-download-cache --list-file ${craftmasterConfigDir}/BinaryCachePackages.list + """ + } + } + + // We need to catch errors here to ensure that the cleanup always runs, even if the upload fails + // While this is a bit wasteful it at least determines that errors in the publishing process leave a builder with a very full disk! + catchError { + // And finally, we'll upload it to the master server + stage('Publish Craft Cache') { + sh """ + cd "$WORKSPACE/ci-tooling/" + /usr/local/bin/python3 helpers/sftp-directory-mirror.py --source ~/Craft/BC/src/ --destination /srv/archives/files/craft/src/ --server milonia.kde.org --username craft + """ + } + } + + + // Then we can cleanup everything we did + stage('Cleaning Up') { + sh """ + rm -rf ~/Craft/BC/ + """ + } + + // As the macOS Slaves are permanent ones, we erase the Workspace as the last thing we do + deleteDir() +} +} diff --git a/craft-cache/src-pipelines/windows.pipeline b/craft-cache/src-pipelines/windows.pipeline new file mode 100644 index 0000000..85b0ea8 --- /dev/null +++ b/craft-cache/src-pipelines/windows.pipeline @@ -0,0 +1,90 @@ +// Where will the craftmaster configuration be found (relative to the working directory that is)? +def craftmasterConfigDir = "binary-factory-tooling/craft/configs/experimental/" + +// Request a node to be allocated to us +node( "WindowsMSVC" ) { +// We want Timestamps on everything +timestamps { + // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed + catchError { + + // First things first, make sure everything that could be using up disk space is removed + // This ensures that WebEngine builds don't get tripped up... + stage('Cleaning Up Craft') { + bat """ + rmdir /s /q C:\\Craft\\BC\\ + rmdir /s /q C:\\Craft\\BinaryFactory\\ + rmdir /s /q C:\\Craft\\Downloads\\ + rmdir /s /q C:\\_\\ + exit /b 0 + """ + } + + // First we want to make sure Craft is ready to go + stage('Preparing Craft') { + // Make sure we start with a clean slate + deleteDir() + + // Grab our tooling which we will need in a few moments + checkout changelog: false, poll: false, scm: [ + $class: 'GitSCM', + branches: [[name: 'master']], + extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], + userRemoteConfigs: [[url: 'https://invent.kde.org/sysadmin/binary-factory-tooling.git']] + ] + + // We are also reliant on the CI System Tooling for parts of this, so grab it too + checkout changelog: false, poll: false, scm: [ + $class: 'GitSCM', + branches: [[name: 'master']], + extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], + userRemoteConfigs: [[url: 'https://invent.kde.org/sysadmin/ci-tooling.git']] + ] + + // Make sure that Craftmaster is ready to go + bat """ + python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "https://invent.kde.org/packaging/craftmaster.git" --into "C:/Craft/BC/" + python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "https://invent.kde.org/sysadmin/binary-factory-tooling.git" --into "C:/Craft/BC/" + """ + } + + // Now that is done, it's time to rebuild the Craft Cache! + stage('Build Craft Cache') { + bat """ + setlocal enabledelayedexpansion + + cd C:/Craft/BC/ + + python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft + if errorlevel 1 exit /b %errorlevel% + + python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables "DownloadDir=C:/Craft/BC/src" -c --create-download-cache --list-file ${craftmasterConfigDir}/BinaryCachePackages.list + if errorlevel 1 exit /b %errorlevel% + """ + } + } + + // We need to catch errors here to ensure that the cleanup always runs, even if the upload fails + // While this is a bit wasteful it at least determines that errors in the publishing process leave a builder with a very full disk! + catchError { + // Upload it to the master server + stage('Publish Craft Cache') { + bat """ + cd "%WORKSPACE%\\ci-tooling" + python helpers/sftp-directory-mirror.py --source C:/Craft/BC/src/ --destination /srv/archives/files/craft/src/ --server milonia.kde.org --username craft + """ + } + } + + // Then we can cleanup everything we did + // It is critical this always happens, otherwise builders can run out of disk space! + stage('Cleaning Up') { + bat """ + rmdir /s /q C:\\Craft\\BC\\ + """ + } + + // As the Windows Slaves are permanent ones, we erase the Workspace as the last thing we do + deleteDir() +} +}