diff --git a/craft-cache/cleanup-nodes.pipeline b/craft-cache/cleanup-nodes.pipeline index d5b0e3d..841a6fb 100644 --- a/craft-cache/cleanup-nodes.pipeline +++ b/craft-cache/cleanup-nodes.pipeline @@ -1,46 +1,46 @@ // First we'll cleanup Builder Iota.... node( "Windows Builder Iota" ) { // We want Timestamps on everything timestamps { // Now that is done, build out the two targets we need stage('Cleaning Up Craft') { bat """ - rmdir /s /q C:\\Craft\\BinaryCache\\ + rmdir /s /q C:\\Craft\\BC\\ rmdir /s /q C:\\Craft\\BinaryFactory\\ rmdir /s /q C:\\Craft\\Downloads\\ rmdir /s /q C:\\_\\ exit /b 0 """ } } } // Then Windows Builder Kappa.... node( "Windows Builder Kappa" ) { // We want Timestamps on everything timestamps { // Now that is done, build out the two targets we need stage('Cleaning Up Craft') { bat """ - rmdir /s /q C:\\Craft\\BinaryCache\\ + rmdir /s /q C:\\Craft\\BC\\ rmdir /s /q C:\\Craft\\BinaryFactory\\ rmdir /s /q C:\\Craft\\Downloads\\ rmdir /s /q C:\\_\\ exit /b 0 """ } } } // Finally the Mac can be cleaned up node( "macOS Builder 1" ) { // We want Timestamps on everything timestamps { // Now that is done, build out the two targets we need stage('Cleaning Up Craft') { sh """ - rm -rf ~/Craft/BinaryCache/ ~/Craft/BinaryFactory/ + rm -rf ~/Craft/BC/ ~/Craft/BinaryFactory/ """ } } } diff --git a/craft-cache/experimental-pipelines/linux.pipeline b/craft-cache/experimental-pipelines/linux.pipeline index 21ef6df..561151c 100644 --- a/craft-cache/experimental-pipelines/linux.pipeline +++ b/craft-cache/experimental-pipelines/linux.pipeline @@ -1,56 +1,56 @@ // Where will the craftmaster configuration be found (relative to the working directory that is)? def craftmasterConfigDir = "binary-factory-tooling/craft/configs/experimental/" // Request a node to be allocated to us node( "Ubuntu1604Craft" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // We are also reliant on the CI System Tooling for parts of this, so grab it too checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] ] // Make sure that Craftmaster is ready to go sh """ - python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BinaryCache/ - python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BinaryCache/ + python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BC/ + python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BC/ """ } // Now that is done, it's time to rebuild the Craft Cache! stage('Build Craft Cache') { sh """ - cd ~/Craft/BinaryCache/ + cd ~/Craft/BC/ python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackagesBase.list python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackages.list """ } } // And finally, we'll upload it to the master server stage('Publish Craft Cache') { sh """ cd "$WORKSPACE/ci-tooling/" - python3 helpers/sftp-directory-mirror.py --source ~/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft + python3 helpers/sftp-directory-mirror.py --source ~/Craft/BC/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft """ } } } diff --git a/craft-cache/experimental-pipelines/macos.pipeline b/craft-cache/experimental-pipelines/macos.pipeline index 5396610..fa387a8 100644 --- a/craft-cache/experimental-pipelines/macos.pipeline +++ b/craft-cache/experimental-pipelines/macos.pipeline @@ -1,73 +1,73 @@ // Where will the craftmaster configuration be found (relative to the working directory that is)? def craftmasterConfigDir = "binary-factory-tooling/craft/configs/experimental/" // Request a node to be allocated to us node( "macOS" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Make sure we start with a clean slate deleteDir() // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // We are also reliant on the CI System Tooling for parts of this, so grab it too checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] ] // Make sure that Craftmaster is ready to go sh """ - /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BinaryCache/ - /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BinaryCache/ + /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BC/ + /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BC/ """ } // Now that is done, it's time to rebuild the Craft Cache! stage('Build Craft Cache') { sh """ - cd ~/Craft/BinaryCache/ + cd ~/Craft/BC/ /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackagesBase.list /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackages.list """ } } // We need to catch errors here to ensure that the cleanup always runs, even if the upload fails // While this is a bit wasteful it at least determines that errors in the publishing process leave a builder with a very full disk! catchError { // And finally, we'll upload it to the master server stage('Publish Craft Cache') { sh """ cd "$WORKSPACE/ci-tooling/" - /usr/local/bin/python3 helpers/sftp-directory-mirror.py --source ~/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft + /usr/local/bin/python3 helpers/sftp-directory-mirror.py --source ~/Craft/BC/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft """ } } // Then we can cleanup everything we did stage('Cleaning Up') { sh """ - rm -rf ~/Craft/BinaryCache/ + rm -rf ~/Craft/BC/ """ } // As the macOS Slaves are permanent ones, we erase the Workspace as the last thing we do deleteDir() } } diff --git a/craft-cache/experimental-pipelines/windows.pipeline b/craft-cache/experimental-pipelines/windows.pipeline index 9912339..d14c703 100644 --- a/craft-cache/experimental-pipelines/windows.pipeline +++ b/craft-cache/experimental-pipelines/windows.pipeline @@ -1,95 +1,95 @@ // Where will the craftmaster configuration be found (relative to the working directory that is)? def craftmasterConfigDir = "binary-factory-tooling/craft/configs/experimental/" // Request a node to be allocated to us node( "WindowsMSVC" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First things first, make sure everything that could be using up disk space is removed // This ensures that WebEngine builds don't get tripped up... stage('Cleaning Up Craft') { bat """ - rmdir /s /q C:\\Craft\\BinaryCache\\ + rmdir /s /q C:\\Craft\\BC\\ rmdir /s /q C:\\Craft\\BinaryFactory\\ rmdir /s /q C:\\Craft\\Downloads\\ rmdir /s /q C:\\_\\ exit /b 0 """ } // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Make sure we start with a clean slate deleteDir() // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // We are also reliant on the CI System Tooling for parts of this, so grab it too checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] ] // Make sure that Craftmaster is ready to go bat """ - python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into "C:/Craft/BinaryCache/" - python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into "C:/Craft/BinaryCache/" + python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into "C:/Craft/BC/" + python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into "C:/Craft/BC/" """ } // Now that is done, it's time to rebuild the Craft Cache! stage('Build Craft Cache') { bat """ setlocal enabledelayedexpansion - cd C:/Craft/BinaryCache/ + cd C:/Craft/BC/ python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft if errorlevel 1 exit /b %errorlevel% python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackagesBase.list if errorlevel 1 exit /b %errorlevel% if not "${craftPlatform}" == "windows-msvc2017_32-cl" ( python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackages.list if !errorlevel! neq 0 exit /b !errorlevel! ) """ } } // We need to catch errors here to ensure that the cleanup always runs, even if the upload fails // While this is a bit wasteful it at least determines that errors in the publishing process leave a builder with a very full disk! catchError { // Upload it to the master server stage('Publish Craft Cache') { bat """ cd "%WORKSPACE%\\ci-tooling" - python helpers/sftp-directory-mirror.py --source C:/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft + python helpers/sftp-directory-mirror.py --source C:/Craft/BC/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft """ } } // Then we can cleanup everything we did // It is critical this always happens, otherwise builders can run out of disk space! stage('Cleaning Up') { bat """ - rmdir /s /q C:\\Craft\\BinaryCache\\ + rmdir /s /q C:\\Craft\\BC\\ """ } // As the Windows Slaves are permanent ones, we erase the Workspace as the last thing we do deleteDir() } } diff --git a/craft-cache/master-pipelines/linux.pipeline b/craft-cache/master-pipelines/linux.pipeline index c86959f..51ad12f 100644 --- a/craft-cache/master-pipelines/linux.pipeline +++ b/craft-cache/master-pipelines/linux.pipeline @@ -1,56 +1,56 @@ // Where will the craftmaster configuration be found (relative to the working directory that is)? def craftmasterConfigDir = "binary-factory-tooling/craft/configs/master/" // Request a node to be allocated to us node( "Ubuntu1604Craft" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // We are also reliant on the CI System Tooling for parts of this, so grab it too checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] ] // Make sure that Craftmaster is ready to go sh """ - python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BinaryCache/ - python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BinaryCache/ + python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BC/ + python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BC/ """ } // Now that is done, it's time to rebuild the Craft Cache! stage('Build Craft Cache') { sh """ - cd ~/Craft/BinaryCache/ + cd ~/Craft/BC/ python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackagesBase.list python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackages.list """ } } // And finally, we'll upload it to the master server stage('Publish Craft Cache') { sh """ cd "$WORKSPACE/ci-tooling/" - python3 helpers/sftp-directory-mirror.py --source ~/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft + python3 helpers/sftp-directory-mirror.py --source ~/Craft/BC/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft """ } } } diff --git a/craft-cache/master-pipelines/macos.pipeline b/craft-cache/master-pipelines/macos.pipeline index 4d812c9..f6d4864 100644 --- a/craft-cache/master-pipelines/macos.pipeline +++ b/craft-cache/master-pipelines/macos.pipeline @@ -1,73 +1,73 @@ // Where will the craftmaster configuration be found (relative to the working directory that is)? def craftmasterConfigDir = "binary-factory-tooling/craft/configs/master/" // Request a node to be allocated to us node( "macOS" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Make sure we start with a clean slate deleteDir() // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // We are also reliant on the CI System Tooling for parts of this, so grab it too checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] ] // Make sure that Craftmaster is ready to go sh """ - /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BinaryCache/ - /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BinaryCache/ + /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BC/ + /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BC/ """ } // Now that is done, it's time to rebuild the Craft Cache! stage('Build Craft Cache') { sh """ - cd ~/Craft/BinaryCache/ + cd ~/Craft/BC/ /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackagesBase.list /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackages.list """ } } // We need to catch errors here to ensure that the cleanup always runs, even if the upload fails // While this is a bit wasteful it at least determines that errors in the publishing process leave a builder with a very full disk! catchError { // And finally, we'll upload it to the master server stage('Publish Craft Cache') { sh """ cd "$WORKSPACE/ci-tooling/" - /usr/local/bin/python3 helpers/sftp-directory-mirror.py --source ~/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft + /usr/local/bin/python3 helpers/sftp-directory-mirror.py --source ~/Craft/BC/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft """ } } // Then we can cleanup everything we did stage('Cleaning Up') { sh """ - rm -rf ~/Craft/BinaryCache/ + rm -rf ~/Craft/BC/ """ } // As the macOS Slaves are permanent ones, we erase the Workspace as the last thing we do deleteDir() } } diff --git a/craft-cache/master-pipelines/windows.pipeline b/craft-cache/master-pipelines/windows.pipeline index 67159c0..52ee7e0 100644 --- a/craft-cache/master-pipelines/windows.pipeline +++ b/craft-cache/master-pipelines/windows.pipeline @@ -1,95 +1,95 @@ // Where will the craftmaster configuration be found (relative to the working directory that is)? def craftmasterConfigDir = "binary-factory-tooling/craft/configs/master/" // Request a node to be allocated to us node( "WindowsMSVC" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First things first, make sure everything that could be using up disk space is removed // This ensures that WebEngine builds don't get tripped up... stage('Cleaning Up Craft') { bat """ - rmdir /s /q C:\\Craft\\BinaryCache\\ + rmdir /s /q C:\\Craft\\BC\\ rmdir /s /q C:\\Craft\\BinaryFactory\\ rmdir /s /q C:\\Craft\\Downloads\\ rmdir /s /q C:\\_\\ exit /b 0 """ } // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Make sure we start with a clean slate deleteDir() // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // We are also reliant on the CI System Tooling for parts of this, so grab it too checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] ] // Make sure that Craftmaster is ready to go bat """ - python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into "C:/Craft/BinaryCache/" - python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into "C:/Craft/BinaryCache/" + python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into "C:/Craft/BC/" + python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into "C:/Craft/BC/" """ } // Now that is done, it's time to rebuild the Craft Cache! stage('Build Craft Cache') { bat """ setlocal enabledelayedexpansion - cd C:/Craft/BinaryCache/ + cd C:/Craft/BC/ python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft if errorlevel 1 exit /b %errorlevel% python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackagesBase.list if errorlevel 1 exit /b %errorlevel% if not "${craftPlatform}" == "windows-msvc2017_32-cl" ( python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --option "/.buildTests=False" --list-file ${craftmasterConfigDir}/BinaryCachePackages.list if !errorlevel! neq 0 exit /b !errorlevel! ) """ } } // We need to catch errors here to ensure that the cleanup always runs, even if the upload fails // While this is a bit wasteful it at least determines that errors in the publishing process leave a builder with a very full disk! catchError { // Upload it to the master server stage('Publish Craft Cache') { bat """ cd "%WORKSPACE%\\ci-tooling" - python helpers/sftp-directory-mirror.py --source C:/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft + python helpers/sftp-directory-mirror.py --source C:/Craft/BC/cache/ --destination /srv/archives/files/craft/master/ --server milonia.kde.org --username craft """ } } // Then we can cleanup everything we did // It is critical this always happens, otherwise builders can run out of disk space! stage('Cleaning Up') { bat """ - rmdir /s /q C:\\Craft\\BinaryCache\\ + rmdir /s /q C:\\Craft\\BC\\ """ } // As the Windows Slaves are permanent ones, we erase the Workspace as the last thing we do deleteDir() } } diff --git a/craft-cache/prebuilt-pipelines/linux.pipeline b/craft-cache/prebuilt-pipelines/linux.pipeline index 369bc09..ca437b6 100644 --- a/craft-cache/prebuilt-pipelines/linux.pipeline +++ b/craft-cache/prebuilt-pipelines/linux.pipeline @@ -1,56 +1,56 @@ // Where will the craftmaster configuration be found (relative to the working directory that is)? def craftmasterConfigDir = "binary-factory-tooling/craft/configs/prebuilt/" // Request a node to be allocated to us node( "Ubuntu1604Craft" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // We are also reliant on the CI System Tooling for parts of this, so grab it too checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] ] // Make sure that Craftmaster is ready to go sh """ - python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BinaryCache/ - python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BinaryCache/ + python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BC/ + python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BC/ """ } // Now that is done, it's time to rebuild the Craft Cache! stage('Build Craft Cache') { sh """ - cd ~/Craft/BinaryCache/ + cd ~/Craft/BC/ python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --add-blueprint-repository https://invent.kde.org/vonreth/craft-blueprints-prebuilt.git python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --list-file ${craftmasterConfigDir}/BinaryCachePackages.list """ } } // And finally, we'll upload it to the master server stage('Publish Craft Cache') { sh """ cd "$WORKSPACE/ci-tooling/" - python3 helpers/sftp-directory-mirror.py --source ~/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/prebuilt/ --server milonia.kde.org --username craft + python3 helpers/sftp-directory-mirror.py --source ~/Craft/BC/cache/ --destination /srv/archives/files/craft/prebuilt/ --server milonia.kde.org --username craft """ } } } diff --git a/craft-cache/prebuilt-pipelines/macos.pipeline b/craft-cache/prebuilt-pipelines/macos.pipeline index 320f7d9..0f54d11 100644 --- a/craft-cache/prebuilt-pipelines/macos.pipeline +++ b/craft-cache/prebuilt-pipelines/macos.pipeline @@ -1,73 +1,73 @@ // Where will the craftmaster configuration be found (relative to the working directory that is)? def craftmasterConfigDir = "binary-factory-tooling/craft/configs/prebuilt/" // Request a node to be allocated to us node( "macOS" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Make sure we start with a clean slate deleteDir() // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // We are also reliant on the CI System Tooling for parts of this, so grab it too checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] ] // Make sure that Craftmaster is ready to go sh """ - /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BinaryCache/ - /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BinaryCache/ + /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into ~/Craft/BC/ + /usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into ~/Craft/BC/ """ } // Now that is done, it's time to rebuild the Craft Cache! stage('Build Craft Cache') { sh """ - cd ~/Craft/BinaryCache/ + cd ~/Craft/BC/ /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --add-blueprint-repository https://invent.kde.org/vonreth/craft-blueprints-prebuilt.git /usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --list-file ${craftmasterConfigDir}/BinaryCachePackages.list """ } } // We need to catch errors here to ensure that the cleanup always runs, even if the upload fails // While this is a bit wasteful it at least determines that errors in the publishing process leave a builder with a very full disk! catchError { // And finally, we'll upload it to the master server stage('Publish Craft Cache') { sh """ cd "$WORKSPACE/ci-tooling/" - /usr/local/bin/python3 helpers/sftp-directory-mirror.py --source ~/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/prebuilt/ --server milonia.kde.org --username craft + /usr/local/bin/python3 helpers/sftp-directory-mirror.py --source ~/Craft/BC/cache/ --destination /srv/archives/files/craft/prebuilt/ --server milonia.kde.org --username craft """ } } // Then we can cleanup everything we did stage('Cleaning Up') { sh """ - rm -rf ~/Craft/BinaryCache/ + rm -rf ~/Craft/BC/ """ } // As the macOS Slaves are permanent ones, we erase the Workspace as the last thing we do deleteDir() } } diff --git a/craft-cache/prebuilt-pipelines/windows.pipeline b/craft-cache/prebuilt-pipelines/windows.pipeline index 2714c30..6b893a1 100644 --- a/craft-cache/prebuilt-pipelines/windows.pipeline +++ b/craft-cache/prebuilt-pipelines/windows.pipeline @@ -1,99 +1,99 @@ // Where will the craftmaster configuration be found (relative to the working directory that is)? def craftmasterConfigDir = "binary-factory-tooling/craft/configs/prebuilt/" // Request a node to be allocated to us node( "WindowsMSVC" ) { // We want Timestamps on everything timestamps { // We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed catchError { // First things first, make sure everything that could be using up disk space is removed // This ensures that WebEngine builds don't get tripped up... stage('Cleaning Up Craft') { bat """ - rmdir /s /q C:\\Craft\\BinaryCache\\ + rmdir /s /q C:\\Craft\\BC\\ rmdir /s /q C:\\Craft\\BinaryFactory\\ rmdir /s /q C:\\Craft\\Downloads\\ rmdir /s /q C:\\_\\ exit /b 0 """ } // First we want to make sure Craft is ready to go stage('Preparing Craft') { // Make sure we start with a clean slate deleteDir() // Grab our tooling which we will need in a few moments checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/binary-factory-tooling']] ] // We are also reliant on the CI System Tooling for parts of this, so grab it too checkout changelog: false, poll: false, scm: [ $class: 'GitSCM', branches: [[name: 'master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']], userRemoteConfigs: [[url: 'https://anongit.kde.org/sysadmin/ci-tooling']] ] // Make sure that Craftmaster is ready to go bat """ - python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into "C:/Craft/BinaryCache/" - python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into "C:/Craft/BinaryCache/" + python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/craftmaster" --into "C:/Craft/BC/" + python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "git://anongit.kde.org/sysadmin/binary-factory-tooling" --into "C:/Craft/BC/" """ } // Now that is done, it's time to rebuild the Craft Cache! stage('Build Craft Cache') { bat """ setlocal enabledelayedexpansion - cd C:/Craft/BinaryCache/ + cd C:/Craft/BC/ python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --add-blueprint-repository https://invent.kde.org/vonreth/craft-blueprints-prebuilt.git if !errorlevel! neq 0 exit /b !errorlevel! python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --list-file ${craftmasterConfigDir}/BinaryCachePackages.list if !errorlevel! neq 0 exit /b !errorlevel! if "${craftPlatform}" == "windows-mingw_64-gcc" GOTO autotools if "${craftPlatform}" == "windows-mingw_32-gcc" GOTO autotools GOTO end :autotools python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --list-file ${craftmasterConfigDir}/autotools.list if !errorlevel! neq 0 exit /b !errorlevel! python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c --list-file ${craftmasterConfigDir}/autotools_package.list --package --option [Packager]PackageType=PortablePackager :end if !errorlevel! neq 0 exit /b !errorlevel! """ } } // We need to catch errors here to ensure that the cleanup always runs, even if the upload fails // While this is a bit wasteful it at least determines that errors in the publishing process leave a builder with a very full disk! catchError { // Upload it to the master server stage('Publish Craft Cache') { bat """ cd "%WORKSPACE%\\ci-tooling" - python helpers/sftp-directory-mirror.py --source C:/Craft/BinaryCache/cache/ --destination /srv/archives/files/craft/prebuilt/ --server milonia.kde.org --username craft + python helpers/sftp-directory-mirror.py --source C:/Craft/BC/cache/ --destination /srv/archives/files/craft/prebuilt/ --server milonia.kde.org --username craft """ } } // Then we can cleanup everything we did // It is critical this always happens, otherwise builders can run out of disk space! stage('Cleaning Up') { bat """ - rmdir /s /q C:\\Craft\\BinaryCache\\ + rmdir /s /q C:\\Craft\\BC\\ """ } // As the Windows Slaves are permanent ones, we erase the Workspace as the last thing we do deleteDir() } }