Merge pull request #11 from xcompass/db-backup

Enable database backup through Wal-E

README.md

@@ -55,6 +55,31 @@ Dockerfiles for Mattermost in production
 
     sudo rm -rf volumes
 
+## Database Backup
+
+When AWS S3 environment variables are specified on db docker container, it enables [Wel-E](https://github.com/wal-e/wal-e) backup to S3.
+
+```bash
+docker run -d --name mattermost-db \
+    -e AWS_ACCESS_KEY_ID=XXXX \
+    -e AWS_SECRET_ACCESS_KEY=XXXX \
+    -e WALE_S3_PREFIX=s3://BUCKET_NAME/PATH \
+    -e AWS_REGION=us-east-1
+    -v ./volumes/db/var/lib/postgresql/data:/var/lib/postgresql/data
+    -v /etc/localtime:/etc/localtime:ro
+    db
+```
+
+All four environment variables are required. It will enable completed WAL segments sent to archive storage (S3). The base backup and clean up can be done through the following command:
+
+```bash
+# base backup
+docker exec mattermost-db su - postgres sh -c "/usr/bin/envdir /etc/wal-e.d/env /usr/local/bin/wal-e backup-push /var/lib/postgresql/data"
+# keep the most recent 7 base backups and remove the old ones
+docker exec mattermost-db su - postgres sh -c "/usr/bin/envdir /etc/wal-e.d/env /usr/local/bin/wal-e delete --confirm retain 7"
+```
+Those tasks can be executed through a cron job or systemd timer.
+
 ## Known Issues
 
 * Do not modify the Listen Address in Service Settings.

db/Dockerfile

@@ -1,3 +1,18 @@
 FROM postgres:9.4
 
+RUN apt-get update \
+    && apt-get install -y python-dev lzop pv daemontools curl build-essential \
+    && curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py | python \
+    && pip install wal-e \
+    && apt-get remove -y build-essential python-dev \
+    && apt-get autoremove -y \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
 ADD make_db.sh /docker-entrypoint-initdb.d/
+ADD setup-wale.sh /docker-entrypoint-initdb.d/
+COPY docker-entrypoint1.sh /
+
+ENTRYPOINT ["/docker-entrypoint1.sh"]
+
+CMD ["postgres"]

db/docker-entrypoint1.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+if [ "${1:0:1}" = '-'  ]; then
+    set -- postgres "$@"
+fi
+
+if [ "$1" = 'postgres'  ]; then
+    VARS=(AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY WALE_S3_PREFIX AWS_REGION)
+
+    for v in ${VARS[@]}; do
+        if [ "${!v}" = "" ]; then
+            echo "$v is required for Wal-E but not set. Skipping Wal-E setup."
+            . /docker-entrypoint.sh
+            exit
+        fi
+    done
+
+    umask u=rwx,g=rx,o=
+    mkdir -p /etc/wal-e.d/env
+
+    for v in ${VARS[@]}; do
+        echo "${!v}" > /etc/wal-e.d/env/$v
+    done
+    chown -R root:postgres /etc/wal-e.d
+
+    . /docker-entrypoint.sh
+fi
+
+exec "$@"

db/setup-wale.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# wal-e specific
+echo "wal_level = archive" >> /var/lib/postgresql/data/postgresql.conf
+echo "archive_mode = on" >> /var/lib/postgresql/data/postgresql.conf
+echo "archive_command = 'envdir /etc/wal-e.d/env /usr/local/bin/wal-e wal-push %p'" >> /var/lib/postgresql/data/postgresql.conf
+echo "archive_timeout = 60" >> /var/lib/postgresql/data/postgresql.conf
+
+# no cron in the image, use systemd timer on host instead
+#su - postgres -c "crontab -l | { cat; echo \"0 3 * * * /usr/bin/envdir /etc/wal-e.d/env /usr/local/bin/wal-e backup-push /var/lib/postgresql/data\"; } | crontab -"
+#su - postgres -c "crontab -l | { cat; echo \"0 4 * * * /usr/bin/envdir /etc/wal-e.d/env /usr/local/bin/wal-e delete --confirm retain 7\"; } | crontab -"

docker-compose-nossl.yml

@@ -3,6 +3,12 @@ db:
   volumes:
     - ./volumes/db/var/lib/postgresql/data:/var/lib/postgresql/data
     - /etc/localtime:/etc/localtime:ro
+  # uncomment the following to enable backup
+  #environment:
+  #  - AWS_ACCESS_KEY_ID=XXXX
+  #  - AWS_SECRET_ACCESS_KEY=XXXX
+  #  - WALE_S3_PREFIX=s3://BUCKET_NAME/PATH
+  #  - AWS_REGION=us-east-1
 app:
   build: app
   links:

docker-compose-ssl.yml

@@ -3,6 +3,12 @@ db:
   volumes:
     - ./volumes/db/var/lib/postgresql/data:/var/lib/postgresql/data
     - /etc/localtime:/etc/localtime:ro
+  # uncomment the following to enable backup
+  #environment:
+  #  - AWS_ACCESS_KEY_ID=XXXX
+  #  - AWS_SECRET_ACCESS_KEY=XXXX
+  #  - WALE_S3_PREFIX=s3://BUCKET_NAME/PATH
+  #  - AWS_REGION=us-east-1
 app:
   build: app
   links: