diff --git a/roles/kde-backup/templates/backup-code.sh b/roles/kde-backup/templates/backup-code.sh index 99aea3c..b1c95d9 100755 --- a/roles/kde-backup/templates/backup-code.sh +++ b/roles/kde-backup/templates/backup-code.sh @@ -1,49 +1,49 @@ #!/bin/bash LOCATION=/root/{{backup_directory}} # Cleanup old backups find $LOCATION -name '*z' -mtime +21 | xargs rm -f # Grab trashed Git repositories tar -cJf $LOCATION/git-trash.`date +%w`.tar.xz -C / srv/git/trash/ # Grab the private sysadmin and board repos tar -cJf $LOCATION/evboard-repo.`date +%w`.tar.xz -C / srv/evboard/ tar -cJf $LOCATION/sysadmin-repo.`date +%w`.tar.xz -C / srv/sysadmin/ # Grab the gitolite-admin repositories, which are private, for both git.kde.org and git.neon.kde.org tar -cJf $LOCATION/gitolite-admin-main.`date +%w`.tar.xz -C / srv/git/repositories/gitolite-admin.git/ tar -cJf $LOCATION/gitolite-admin-neon.`date +%w`.tar.xz -C / srv/neon/gitolite-admin.git/ # Grab the system config, package listing and cronjobs tar -czf $LOCATION/crontabs.`date +%w`.tgz -C / var/spool/cron/ tar -czf $LOCATION/etc.`date +%w`.tgz -C / etc/ dpkg -l > $LOCATION/dpkg.`date +%w` # Secure our backups chmod -R 700 $LOCATION # Transfer them to the backup server cd $LOCATION/.. lftp -f ~/bin/backup-options # Prepare to run Borg backups export BORG_PASSPHRASE='{{backup_borg_passphrase}}' {% set backup_creds = hetzner_backup_creds[hetzner_backup_host] %} -BORG_SERVER="ssh://{{backup_creds.username}}@{{backup_creds.username}}.your-backup.de:23" +BORG_SERVER="ssh://{{backup_creds.username}}@{{backup_creds.hostname}}:23" # Backup Homes export BORG_REPO="$BORG_SERVER/./borg-backups/code-homes" borg create --compression zlib,5 --exclude-caches ::'{hostname}-phabricator-{now}' /home/ 2>&1 | grep -v "Compacting segments" borg prune --prefix '{hostname}-' --keep-daily 7 --keep-weekly 4 --keep-monthly 6 2>&1 | grep -v "Compacting segments" # Backup Databases export BORG_REPO="$BORG_SERVER/./borg-backups/db-backups" su -l phabricator -c "cd /srv/phabricator/phabricator && ./bin/storage dump --no-indexes" | borg create --compression lzma,3 ::'{hostname}-db-backups-{now}' - 2>&1 | grep -v "Compacting segments" borg prune --prefix '{hostname}-' --keep-daily 7 --keep-weekly 4 --keep-monthly 6 2>&1 | grep -v "Compacting segments" # Backup Phabricator export BORG_REPO="$BORG_SERVER/./borg-backups/phabricator" borg create --compression zlib,5 --exclude-caches --exclude /srv/phabricator/logs/ --exclude /srv/phabricator/repos/ ::'{hostname}-phabricator-{now}' /srv/phabricator/ 2>&1 | grep -v "Compacting segments" borg prune --prefix '{hostname}-' --keep-daily 7 --keep-weekly 4 --keep-monthly 6 2>&1 | grep -v "Compacting segments" diff --git a/roles/kde-backup/templates/backup-edulis.sh b/roles/kde-backup/templates/backup-edulis.sh index e3eb294..40515ee 100644 --- a/roles/kde-backup/templates/backup-edulis.sh +++ b/roles/kde-backup/templates/backup-edulis.sh @@ -1,32 +1,32 @@ LOCATION=/root/{{backup_directory}} # Backup configuration and list of installed packages tar -czf $LOCATION/etc.`date +%w`.tgz -C / etc/ tar -czf $LOCATION/crontabs.`date +%w`.tgz -C / var/spool/cron/ dpkg -l > $LOCATION/dpkg.`date +%w` # Transfer the generated backups lftp -f ~/bin/backup-options # Borg backup time! Let's get ready to go... export BORG_PASSPHRASE='{{backup_borg_passphrase}}' {% set backup_creds = hetzner_backup_creds[hetzner_backup_host] %} -BORG_SERVER="ssh://{{backup_creds.username}}@{{backup_creds.username}}.your-backup.de:23" +BORG_SERVER="ssh://{{backup_creds.username}}@{{backup_creds.hostname}}:23" # Backup databases export BORG_REPO="$BORG_SERVER/./borg-backups/edulis-databases" for DATABASE in `mysqlshow | grep -v "information_schema" | grep -v "performance_schema" | grep -v "mysql" | grep -v "sys" | tail -n+4 | awk '{print $2}'`; do mysqldump --opt --quick --single-transaction --skip-extended-insert --events --create-options --set-charset $DATABASE | borg create --compression lzma,3 "::{hostname}-$DATABASE-{now}" - 2>&1 | grep -v "Compacting segments" borg prune --prefix "{hostname}-$DATABASE" --keep-daily 7 --keep-weekly 4 --keep-monthly 6 2>&1 | grep -v "Compacting segments" done # Backup /srv export BORG_REPO="$BORG_SERVER/./borg-backups/edulis-app-files" borg create --compression zlib,5 --exclude-caches --exclude /srv/www/paste.kde.org/app/storage/sessions/ ::'{hostname}-srv-{now}' /srv/ 2>&1 | grep -v "Compacting segments" borg prune --prefix '{hostname}-' --keep-daily 7 --keep-weekly 4 --keep-monthly 6 2>&1 | grep -v "Compacting segments" # Backup /home export BORG_REPO="$BORG_SERVER/./borg-backups/edulis-homes" borg create --compression zlib,5 --exclude-caches ::'{hostname}-homes-{now}' /home/ 2>&1 | grep -v "Compacting segments" borg prune --prefix '{hostname}-' --keep-daily 7 --keep-weekly 4 --keep-monthly 6 2>&1 | grep -v "Compacting segments" diff --git a/roles/kde-backup/templates/backup-komaci.sh b/roles/kde-backup/templates/backup-komaci.sh index f859b2a..02ace1e 100755 --- a/roles/kde-backup/templates/backup-komaci.sh +++ b/roles/kde-backup/templates/backup-komaci.sh @@ -1,35 +1,35 @@ #!/bin/bash LOCATION=/root/{{backup_directory}} # Cleanup old backups find $LOCATION -mtime +3 | xargs rm -f # Backup the installed software tar -cJf $LOCATION/srv-bugs.`date +%w`.tar.xz -C / srv/ # Backup all our home directories tar -cJf $LOCATION/homes.`date +%w`.tar.xz -C / home # Grab the system config, package listing and cronjobs dpkg -l > $LOCATION/dpkg.`date +%w` tar -czf $LOCATION/crontabs.`date +%w`.tgz -C / var/spool/cron/ tar -czf $LOCATION/etc.`date +%w`.tar.gz -C / etc chmod -R 700 $LOCATION # Transfer them to the backup server cd $LOCATION/.. lftp -f ~/bin/backup-options # Prepare to run Borg backups export BORG_PASSPHRASE='{{backup_borg_passphrase}}' {% set backup_creds = hetzner_backup_creds[hetzner_backup_host] %} -BORG_SERVER="ssh://{{backup_creds.username}}@{{backup_creds.username}}.your-backup.de:23" +BORG_SERVER="ssh://{{backup_creds.username}}@{{backup_creds.hostname}}:23" # Backup databases into Borg export BORG_REPO="$BORG_SERVER/./borg-backups/bugs-database" mysqldump --opt --quick --single-transaction --skip-extended-insert --events --create-options --set-charset bugs | borg create --compression lzma,3 ::'{hostname}-db-backups-{now}' - 2>&1 | grep -v "Compacting segments" borg prune --prefix '{hostname}-' --keep-daily 7 --keep-weekly 4 --keep-monthly 6 2>&1 | grep -v "Compacting segments" diff --git a/roles/kde-backup/templates/backup-mimi.sh b/roles/kde-backup/templates/backup-mimi.sh index ab81f2d..b7ebd32 100755 --- a/roles/kde-backup/templates/backup-mimi.sh +++ b/roles/kde-backup/templates/backup-mimi.sh @@ -1,39 +1,39 @@ #!/bin/bash LOCATION=/root/{{backup_directory}} # Backup configuration and list of installed packages tar -czf $LOCATION/etc.`date +%w`.tgz -C / etc/ tar -czf $LOCATION/crontabs.`date +%w`.tgz -C / var/spool/cron/ dpkg -l > $LOCATION/dpkg.`date +%w` lftp -f ~/bin/backup-options # Borg backup time! Let's get ready to go... export BORG_PASSPHRASE='{{backup_borg_passphrase}}' {% set backup_creds = hetzner_backup_creds[hetzner_backup_host] %} -BORG_SERVER="ssh://{{backup_creds.username}}@{{backup_creds.username}}.your-backup.de:23" +BORG_SERVER="ssh://{{backup_creds.username}}@{{backup_creds.hostname}}:23" # Backup databases export BORG_REPO="$BORG_SERVER/./borg-backups/mimi-databases" for DATABASE in `mysqlshow | grep -v "information_schema" | grep -v "performance_schema" | grep -v "mysql" | tail -n+4 | awk '{print $2}'`; do mysqldump --opt --quick --single-transaction --skip-extended-insert --events --create-options --set-charset $DATABASE | borg create --compression lzma,3 "::{hostname}-$DATABASE-{now}" - 2>&1 | grep -v "Compacting segments" borg prune --prefix "{hostname}-$DATABASE" --keep-daily 7 --keep-weekly 4 --keep-monthly 6 2>&1 | grep -v "Compacting segments" done # Backup Mongo export BORG_REPO="$BORG_SERVER/./borg-backups/mimi-mongodb" mongodump --out $LOCATION/mongodb/ &> /dev/null borg create --compression zlib,5 --exclude-caches ::'{hostname}-mongodb-{now}' $LOCATION/mongodb/ 2>&1 | grep -v "Compacting segments" borg prune --prefix '{hostname}-' --keep-daily 7 --keep-weekly 4 --keep-monthly 6 2>&1 | grep -v "Compacting segments" rm -rf $LOCATION/mongodb/ # Backup home directories export BORG_REPO="$BORG_SERVER/./borg-backups/mimi-homes" borg create --compression zlib,5 --exclude-caches ::'{hostname}-homes-{now}' /home/ 2>&1 | grep -v "Compacting segments" borg prune --prefix '{hostname}-' --keep-daily 7 --keep-weekly 4 --keep-monthly 6 2>&1 | grep -v "Compacting segments" # Backup website contents export BORG_REPO="$BORG_SERVER/./borg-backups/mimi-app-files" borg create --compression zlib,5 --exclude-caches --exclude /srv/www/reviewboard/svn.reviewboard.kde.org/logs --exclude /srv/www/reviewboard/git.reviewboard.kde.org/logs --exclude /srv/www/reports.kde.org/log/production.log --exclude /srv/www/reviewboard/svn.reviewboard.kde.org/search-index --exclude /srv/www/reviewboard/git.reviewboard.kde.org/search-index --exclude /srv/www/conf.kde.org/log ::'{hostname}-srv-{now}' /srv/ 2>&1 | grep -v "Compacting segments" borg prune --prefix '{hostname}-' --keep-daily 7 --keep-weekly 4 --keep-monthly 6 2>&1 | grep -v "Compacting segments" diff --git a/roles/kde-backup/templates/backup-options.j2 b/roles/kde-backup/templates/backup-options.j2 index b3f0ca7..feb02c8 100644 --- a/roles/kde-backup/templates/backup-options.j2 +++ b/roles/kde-backup/templates/backup-options.j2 @@ -1,3 +1,3 @@ {% set backup_creds = hetzner_backup_creds[hetzner_backup_host] %} -connect sftp://{{backup_creds.username}}.your-backup.de/ -u {{backup_creds.username}},{{backup_creds.password}} +connect sftp://{{backup_creds.hostname}}/ -u {{backup_creds.username}},{{backup_creds.password}} mirror --reverse --delete --only-newer /root/{{backup_directory}}/ /{{ansible_hostname}}-backups diff --git a/vars/backups-vault.yml b/vars/backups-vault.yml index 33386e6..dbb1477 100644 --- a/vars/backups-vault.yml +++ b/vars/backups-vault.yml @@ -1,25 +1,34 @@ $ANSIBLE_VAULT;1.1;AES256 -66326239373833346435396635653235376434396263636131373033343765633036643062336633 -3061323238373861376439653637666531376437623766330a653132633865613233373831623163 -62346133643162376133306465663134326134323833363838326266393239373335623466336639 -3631643436303739350a373465656463373631663335393336613332393533326338336532343661 -61366136343539376231643937383639306137393832306637383463636331393337363165653237 -63303462353135303661393738623336313334633462353866396238633962306465626338396364 -62363362656463383163613136336231303631623531663133366438616538663832316435663039 -36653734313934386232346564386432636330663930393037653131353763303464663035613037 -31653066363838353164313563663763633663326230313064343766363362336638353030396564 -35663839306139353035393563643232323930306238323938633361383831383635343762633666 -32346661636361306266663961326433363138303161356363613333636437376264373539356639 -66373065626631346336613237623761383534386365393136623831636438306636343339396461 -31376266633735643434623235393263643430386461393936303264356164393965623563343562 -36666236363639333439616335343063376131343933346165373665646361346534333965316663 -33636132323839623065313937636162616530616161666630316137656230356136653739613437 -62613265393766393339326431333562623362376163306431653632313134656563636133633037 -64666636613336636562393966626130326463386165386263343332633337386633373165653135 -35613033616639376130353561646262376435393039373030366564373730363932643661626130 -37646561313036393366303337646461356138326439306132623164616336646562373333303035 -35303731353264623162646336303732646664333961353531656630333636326664356531633033 -37313864336630326631363637316561356232346136353865306439333230623030376461646638 -38643130333535346161626434386330653663396461383234643164326336373535636164653034 -30383732323465316133336666343562656462326430366235326566316239323662366633323832 -30356130353561653630 +31373737616566353235396661643633666632336430326631393563363830363030396634376164 +3031376638343739626538653162643932613863343432390a613334626630393236393333366639 +35636365323231313935646535333030653336333031636365373134333537336663366232333333 +6335666339306338330a363236653831306436333133356135353439336431356562383637653734 +61396630313836333339303930616335643532643066653066633163366463373731333630353536 +35353366643430383538333534383566356334663536326631396337313938663564326266623930 +63626366613233656630316132636339313137386134366364336465366435613839366135343232 +37623537646262323062653138333334633037643566646339383862626336323039326231643865 +63343335663961643761306262346264303836373537373062393332386637616238376538303132 +61386638323536643232626436356336333231363533373939623032386535353132623330353135 +61613839386336333066356137633638643334643762326462333339323033633762646235373635 +65633432663635393666363336653964623936373961303231386639356533633137323433303863 +65323834336266636632653736353536356539666132623265376261623834613533663363333130 +66353966373336306361653332303638393433623062316138336531346333346531383763643537 +30356634323539646538623832623437653836383732373461383363646362343466646432653836 +62633665343761313835393730663162656634353731643935336235323437666337336332336337 +33653665393038363766326536646535393531653866616133643364656431646437643263633435 +30643438623031346332336561356238623662653536656435363566396637313235333564363766 +37313130333863386533323562316565646433656663383762386332346366316266356662366362 +34396364663936316161633530303732373834376439386664326430393430613562396666396365 +65376234363738336363323534646439653362643231346463643366383938633030633665636462 +36643062383338356435633931623361613438313261613730613934366165396535626261633134 +39643832623964386332336431373663333436323536626339303961363339656162656666336331 +65333764333832323033386361643430393932646435613766353063653239646632623438666364 +35303862356664393237646539333634666335626130386235386137356236366434663032326233 +33646136666165356438383030626232393562653064373639303038643135666339643461396262 +37313539353134316538626630363864306435393731323834386238313730363761383136393535 +36336235333330323135366134613538366365326166353836626366383265353633613932323366 +66633931333361303562666265396530343431656334396330333065663733393464346464656663 +62623364363765393961313562373534333461353934313166366232363635643938343436383135 +31646131646366626133356333383165656561613034336163636163323063623461616336353064 +30386539653965396537343832636363313266383039653234303835613461393963326238666234 +39636431633265313565353734633335363834643737653666393833363433643831 diff --git a/vars/backups.yml b/vars/backups.yml index 1c24324..7319b9b 100644 --- a/vars/backups.yml +++ b/vars/backups.yml @@ -1,10 +1,11 @@ --- # this dictionary has the form: # host1: +# hostname: u12345.example.com # username: u12345 # password: Qw3RtYuIoP # host2: # ... hetzner_backup_creds: "{{vault_hetzner_backup_creds}}" backup_borg_passphrase: "{{vault_backup_borg_passphrase}}"