From 8026b6c87449e4675d1d46f408da26f8e19fe62d Mon Sep 17 00:00:00 2001 From: DerLinkman Date: Wed, 19 Oct 2022 11:15:12 +0200 Subject: [PATCH 1/7] Swapped PIGZ instead of gzip --- data/Dockerfiles/backup/Dockerfile | 3 +++ helper-scripts/backup_and_restore.sh | 23 +++++++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 data/Dockerfiles/backup/Dockerfile diff --git a/data/Dockerfiles/backup/Dockerfile b/data/Dockerfiles/backup/Dockerfile new file mode 100644 index 00000000..f9d849b1 --- /dev/null +++ b/data/Dockerfiles/backup/Dockerfile @@ -0,0 +1,3 @@ +FROM debian:bullseye-slim + +RUN apt update && apt install pigz \ No newline at end of file diff --git a/helper-scripts/backup_and_restore.sh b/helper-scripts/backup_and_restore.sh index 6781600f..bf173a5f 100755 --- a/helper-scripts/backup_and_restore.sh +++ b/helper-scripts/backup_and_restore.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -DEBIAN_DOCKER_IMAGE="debian:bullseye-slim" +DEBIAN_DOCKER_IMAGE="mailcow/backup:1.0" if [[ ! -z ${MAILCOW_BACKUP_LOCATION} ]]; then BACKUP_LOCATION="${MAILCOW_BACKUP_LOCATION}" @@ -48,10 +48,21 @@ else fi fi +if [[ -z ${THREADS} ]]; then + read -ep "How many threads do you want to use? (Default $(echo "CPU threads: $(grep -c processor /proc/cpuinfo)" | grep -E -o "[0-9]+")):" THREADS + THREADS=${THREADS:-$(echo "CPU threads: $(grep -c processor /proc/cpuinfo)" | grep -E -o "[0-9]+")} +fi + +if ! [[ "${THREADS}" =~ ^[0-9]+$ ]] ; then + echo "Thread input is not a number!" + exit 1 +fi + BACKUP_LOCATION=$(echo ${BACKUP_LOCATION} | sed 's#/$##') SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" COMPOSE_FILE=${SCRIPT_DIR}/../docker-compose.yml ENV_FILE=${SCRIPT_DIR}/../.env +THREADS=$(echo ${THREADS}) if [ ! -f ${COMPOSE_FILE} ]; then echo "Compose file not found" @@ -99,32 +110,32 @@ function backup() { docker run --name mailcow-backup --rm \ -v ${BACKUP_LOCATION}/mailcow-${DATE}:/backup:z \ -v $(docker volume ls -qf name=^${CMPS_PRJ}_vmail-vol-1$):/vmail:ro,z \ - ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="gzip --rsyncable" -Pcvpf /backup/backup_vmail.tar.gz /vmail + ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="pigz --rsyncable -p ${THREADS}" -Pcvpf /backup/backup_vmail.tar.gz /vmail ;;& crypt|all) docker run --name mailcow-backup --rm \ -v ${BACKUP_LOCATION}/mailcow-${DATE}:/backup:z \ -v $(docker volume ls -qf name=^${CMPS_PRJ}_crypt-vol-1$):/crypt:ro,z \ - ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="gzip --rsyncable" -Pcvpf /backup/backup_crypt.tar.gz /crypt + ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="pigz --rsyncable -p ${THREADS}" -Pcvpf /backup/backup_crypt.tar.gz /crypt ;;& redis|all) docker exec $(docker ps -qf name=redis-mailcow) redis-cli save docker run --name mailcow-backup --rm \ -v ${BACKUP_LOCATION}/mailcow-${DATE}:/backup:z \ -v $(docker volume ls -qf name=^${CMPS_PRJ}_redis-vol-1$):/redis:ro,z \ - ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="gzip --rsyncable" -Pcvpf /backup/backup_redis.tar.gz /redis + ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="pigz --rsyncable -p ${THREADS}" -Pcvpf /backup/backup_redis.tar.gz /redis ;;& rspamd|all) docker run --name mailcow-backup --rm \ -v ${BACKUP_LOCATION}/mailcow-${DATE}:/backup:z \ -v $(docker volume ls -qf name=^${CMPS_PRJ}_rspamd-vol-1$):/rspamd:ro,z \ - ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="gzip --rsyncable" -Pcvpf /backup/backup_rspamd.tar.gz /rspamd + ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="pigz --rsyncable -p ${THREADS}" -Pcvpf /backup/backup_rspamd.tar.gz /rspamd ;;& postfix|all) docker run --name mailcow-backup --rm \ -v ${BACKUP_LOCATION}/mailcow-${DATE}:/backup:z \ -v $(docker volume ls -qf name=^${CMPS_PRJ}_postfix-vol-1$):/postfix:ro,z \ - ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="gzip --rsyncable" -Pcvpf /backup/backup_postfix.tar.gz /postfix + ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="pigz --rsyncable -p ${THREADS}" -Pcvpf /backup/backup_postfix.tar.gz /postfix ;;& mysql|all) SQLIMAGE=$(grep -iEo '(mysql|mariadb)\:.+' ${COMPOSE_FILE}) From 6306c4555c7221493fac0ff19e9aed2f73459d64 Mon Sep 17 00:00:00 2001 From: DerLinkman Date: Fri, 21 Oct 2022 11:48:29 +0200 Subject: [PATCH 2/7] Removed Thread Prompt and set default value to 1 Thread --- helper-scripts/backup_and_restore.sh | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/helper-scripts/backup_and_restore.sh b/helper-scripts/backup_and_restore.sh index bf173a5f..372e9ccc 100755 --- a/helper-scripts/backup_and_restore.sh +++ b/helper-scripts/backup_and_restore.sh @@ -48,21 +48,13 @@ else fi fi -if [[ -z ${THREADS} ]]; then - read -ep "How many threads do you want to use? (Default $(echo "CPU threads: $(grep -c processor /proc/cpuinfo)" | grep -E -o "[0-9]+")):" THREADS - THREADS=${THREADS:-$(echo "CPU threads: $(grep -c processor /proc/cpuinfo)" | grep -E -o "[0-9]+")} -fi - -if ! [[ "${THREADS}" =~ ^[0-9]+$ ]] ; then - echo "Thread input is not a number!" - exit 1 -fi - BACKUP_LOCATION=$(echo ${BACKUP_LOCATION} | sed 's#/$##') SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" COMPOSE_FILE=${SCRIPT_DIR}/../docker-compose.yml ENV_FILE=${SCRIPT_DIR}/../.env -THREADS=$(echo ${THREADS}) +THREADS=$(echo ${THREADS:-1}) + +echo $THREADS if [ ! -f ${COMPOSE_FILE} ]; then echo "Compose file not found" From 58527857d992e6febc089001122e80bf19ba5f95 Mon Sep 17 00:00:00 2001 From: DerLinkman Date: Fri, 21 Oct 2022 11:54:23 +0200 Subject: [PATCH 3/7] Removed debug message --- helper-scripts/backup_and_restore.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/helper-scripts/backup_and_restore.sh b/helper-scripts/backup_and_restore.sh index 372e9ccc..6e46c228 100755 --- a/helper-scripts/backup_and_restore.sh +++ b/helper-scripts/backup_and_restore.sh @@ -54,8 +54,6 @@ COMPOSE_FILE=${SCRIPT_DIR}/../docker-compose.yml ENV_FILE=${SCRIPT_DIR}/../.env THREADS=$(echo ${THREADS:-1}) -echo $THREADS - if [ ! -f ${COMPOSE_FILE} ]; then echo "Compose file not found" exit 1 From b22ff59f7ba7a26917496b252b78efc213e0980b Mon Sep 17 00:00:00 2001 From: DerLinkman Date: Mon, 24 Oct 2022 12:28:41 +0200 Subject: [PATCH 4/7] Added PIGZ for Restoring as well. --- helper-scripts/backup_and_restore.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/helper-scripts/backup_and_restore.sh b/helper-scripts/backup_and_restore.sh index 6e46c228..705bc99c 100755 --- a/helper-scripts/backup_and_restore.sh +++ b/helper-scripts/backup_and_restore.sh @@ -192,7 +192,7 @@ function restore() { docker run -it --name mailcow-backup --rm \ -v ${RESTORE_LOCATION}:/backup:z \ -v $(docker volume ls -qf name=^${CMPS_PRJ}_vmail-vol-1$):/vmail:z \ - ${DEBIAN_DOCKER_IMAGE} /bin/tar -Pxvzf /backup/backup_vmail.tar.gz + ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -xvf /backup/backup_vmail.tar.gz docker start $(docker ps -aqf name=dovecot-mailcow) echo echo "In most cases it is not required to run a full resync, you can run the command printed below at any time after testing wether the restore process broke a mailbox:" @@ -211,7 +211,7 @@ function restore() { docker run -it --name mailcow-backup --rm \ -v ${RESTORE_LOCATION}:/backup:z \ -v $(docker volume ls -qf name=^${CMPS_PRJ}_redis-vol-1$):/redis:z \ - ${DEBIAN_DOCKER_IMAGE} /bin/tar -Pxvzf /backup/backup_redis.tar.gz + ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -xvf /backup/backup_redis.tar.gz docker start $(docker ps -aqf name=redis-mailcow) ;; crypt) @@ -219,7 +219,7 @@ function restore() { docker run -it --name mailcow-backup --rm \ -v ${RESTORE_LOCATION}:/backup:z \ -v $(docker volume ls -qf name=^${CMPS_PRJ}_crypt-vol-1$):/crypt:z \ - ${DEBIAN_DOCKER_IMAGE} /bin/tar -Pxvzf /backup/backup_crypt.tar.gz + ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -xvf /backup/backup_crypt.tar.gz docker start $(docker ps -aqf name=dovecot-mailcow) ;; rspamd) @@ -227,7 +227,7 @@ function restore() { docker run -it --name mailcow-backup --rm \ -v ${RESTORE_LOCATION}:/backup:z \ -v $(docker volume ls -qf name=^${CMPS_PRJ}_rspamd-vol-1$):/rspamd:z \ - ${DEBIAN_DOCKER_IMAGE} /bin/tar -Pxvzf /backup/backup_rspamd.tar.gz + ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -xvf /backup/backup_rspamd.tar.gz docker start $(docker ps -aqf name=rspamd-mailcow) ;; postfix) @@ -235,7 +235,7 @@ function restore() { docker run -it --name mailcow-backup --rm \ -v ${RESTORE_LOCATION}:/backup:z \ -v $(docker volume ls -qf name=^${CMPS_PRJ}_postfix-vol-1$):/postfix:z \ - ${DEBIAN_DOCKER_IMAGE} /bin/tar -Pxvzf /backup/backup_postfix.tar.gz + ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -xvf /backup/backup_postfix.tar.gz docker start $(docker ps -aqf name=postfix-mailcow) ;; mysql|mariadb) From 65ca42ca421133bca64733ca474ffd59f5c96add Mon Sep 17 00:00:00 2001 From: DerLinkman Date: Mon, 24 Oct 2022 15:10:15 +0200 Subject: [PATCH 5/7] Restored Thread Prompt due to implementation in restore --- helper-scripts/backup_and_restore.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/helper-scripts/backup_and_restore.sh b/helper-scripts/backup_and_restore.sh index 705bc99c..035120e1 100755 --- a/helper-scripts/backup_and_restore.sh +++ b/helper-scripts/backup_and_restore.sh @@ -48,6 +48,16 @@ else fi fi +if [[ -z ${THREADS} ]]; then + read -ep "How many threads do you want to use? (Default $(echo "CPU threads: $(grep -c processor /proc/cpuinfo)" | grep -E -o "[0-9]+")):" THREADS + THREADS=${THREADS:-$(echo "CPU threads: $(grep -c processor /proc/cpuinfo)" | grep -E -o "[0-9]+")} +fi + +if ! [[ "${THREADS}" =~ ^[0-9]+$ ]] ; then + echo "Thread input is not a number!" + exit 1 +fi + BACKUP_LOCATION=$(echo ${BACKUP_LOCATION} | sed 's#/$##') SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" COMPOSE_FILE=${SCRIPT_DIR}/../docker-compose.yml From b73d879f3c681593f763cbf1a0ed286e9edb17e8 Mon Sep 17 00:00:00 2001 From: DerLinkman Date: Tue, 25 Oct 2022 09:55:29 +0200 Subject: [PATCH 6/7] Removed thread prompt again. Added notice message --- helper-scripts/backup_and_restore.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/helper-scripts/backup_and_restore.sh b/helper-scripts/backup_and_restore.sh index 035120e1..e212bd77 100755 --- a/helper-scripts/backup_and_restore.sh +++ b/helper-scripts/backup_and_restore.sh @@ -48,22 +48,20 @@ else fi fi -if [[ -z ${THREADS} ]]; then - read -ep "How many threads do you want to use? (Default $(echo "CPU threads: $(grep -c processor /proc/cpuinfo)" | grep -E -o "[0-9]+")):" THREADS - THREADS=${THREADS:-$(echo "CPU threads: $(grep -c processor /proc/cpuinfo)" | grep -E -o "[0-9]+")} -fi - -if ! [[ "${THREADS}" =~ ^[0-9]+$ ]] ; then - echo "Thread input is not a number!" - exit 1 -fi - BACKUP_LOCATION=$(echo ${BACKUP_LOCATION} | sed 's#/$##') SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" COMPOSE_FILE=${SCRIPT_DIR}/../docker-compose.yml ENV_FILE=${SCRIPT_DIR}/../.env THREADS=$(echo ${THREADS:-1}) +if ! [[ "${THREADS}" =~ ^[1-9]+$ ]] ; then + echo "Thread input is not a number!" + exit 1 +elif [[ "${THREADS}" =~ ^[1-9]+$ ]] ; then + echo "Using ${THREADS} Thread(s) for this run." + echo "Notice: You can set the Thread count with the THREADS Variable before you run this script." +fi + if [ ! -f ${COMPOSE_FILE} ]; then echo "Compose file not found" exit 1 From 50d2671d7511877fae9647c3d82bd3f247a26fab Mon Sep 17 00:00:00 2001 From: DerLinkman Date: Tue, 25 Oct 2022 10:06:53 +0200 Subject: [PATCH 7/7] Fixed leading / warning removal --- helper-scripts/backup_and_restore.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/helper-scripts/backup_and_restore.sh b/helper-scripts/backup_and_restore.sh index e212bd77..1853f501 100755 --- a/helper-scripts/backup_and_restore.sh +++ b/helper-scripts/backup_and_restore.sh @@ -200,7 +200,7 @@ function restore() { docker run -it --name mailcow-backup --rm \ -v ${RESTORE_LOCATION}:/backup:z \ -v $(docker volume ls -qf name=^${CMPS_PRJ}_vmail-vol-1$):/vmail:z \ - ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -xvf /backup/backup_vmail.tar.gz + ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_vmail.tar.gz docker start $(docker ps -aqf name=dovecot-mailcow) echo echo "In most cases it is not required to run a full resync, you can run the command printed below at any time after testing wether the restore process broke a mailbox:" @@ -219,7 +219,7 @@ function restore() { docker run -it --name mailcow-backup --rm \ -v ${RESTORE_LOCATION}:/backup:z \ -v $(docker volume ls -qf name=^${CMPS_PRJ}_redis-vol-1$):/redis:z \ - ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -xvf /backup/backup_redis.tar.gz + ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_redis.tar.gz docker start $(docker ps -aqf name=redis-mailcow) ;; crypt) @@ -227,7 +227,7 @@ function restore() { docker run -it --name mailcow-backup --rm \ -v ${RESTORE_LOCATION}:/backup:z \ -v $(docker volume ls -qf name=^${CMPS_PRJ}_crypt-vol-1$):/crypt:z \ - ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -xvf /backup/backup_crypt.tar.gz + ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_crypt.tar.gz docker start $(docker ps -aqf name=dovecot-mailcow) ;; rspamd) @@ -235,7 +235,7 @@ function restore() { docker run -it --name mailcow-backup --rm \ -v ${RESTORE_LOCATION}:/backup:z \ -v $(docker volume ls -qf name=^${CMPS_PRJ}_rspamd-vol-1$):/rspamd:z \ - ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -xvf /backup/backup_rspamd.tar.gz + ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_rspamd.tar.gz docker start $(docker ps -aqf name=rspamd-mailcow) ;; postfix) @@ -243,7 +243,7 @@ function restore() { docker run -it --name mailcow-backup --rm \ -v ${RESTORE_LOCATION}:/backup:z \ -v $(docker volume ls -qf name=^${CMPS_PRJ}_postfix-vol-1$):/postfix:z \ - ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -xvf /backup/backup_postfix.tar.gz + ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_postfix.tar.gz docker start $(docker ps -aqf name=postfix-mailcow) ;; mysql|mariadb)