Backup and restore process (#641)

* Make db dump more modular

* Add basic backup and restore scripts

* Exit on error

* Exit with error code in case of missing file

* Determine volumes location programatically

A bit hacky, maybe there's a better way - open for suggestions!

* gzip database dump

* Restore database

* Fail on pipe errors

* Encrypt backup using gpg; improve log messages

* Ensure database container is up during dump and import

* Restore volumes

backup.sh

@@ -52,7 +52,7 @@ esac
 volumes_dir=/var/lib/docker/volumes
 backup_dir="$volumes_dir/backup_datadir"
 filename="backup.tar.gz"
-dumpname="postgres.sql"
+dumpname="postgres.sql.gz"
 
 if [ "$BACKUP_TIMESTAMP" == true ]; then
   timestamp=$(date "+%Y%m%d-%H%M%S")
@@ -66,9 +66,14 @@ dbdump_path="$backup_dir/_data/${dumpname}"
 cd "$BTCPAY_BASE_DIRECTORY/btcpayserver-docker"
 . helpers.sh
 
+# ensure backup dir exists
+if [ ! -d "$backup_dir" ]; then
+    docker volume create backup_datadir
+fi
+
 # dump database
 echo "Dumping database …"
-btcpay_dump_db $dumpname
+btcpay_dump_db $dbdump_path
 
 if [[ "$1" == "--only-db" ]]; then
     tar -cvzf $backup_path $dbdump_path

btcpay-backup.sh

@@ -0,0 +1,109 @@
+#!/bin/bash -e
+
+set -o pipefail -o errexit
+
+# Please be aware of these important issues:
+#
+# - Old channel state is toxic and you can loose all your funds, if you or someone
+#   else closes a channel based on the backup with old state - and the state changes
+#   often! If you publish an old state (say from yesterday's backup) on chain, you
+#   WILL LOSE ALL YOUR FUNDS IN A CHANNEL, because the counterparty will publish a
+#   revocation key!
+
+if [ "$(id -u)" != "0" ]; then
+  printf "\n🚨 This script must be run as root.\n"
+  printf "➡️  Use the command 'sudo su -' (include the trailing hypen) and try again.\n\n"
+  exit 1
+fi
+
+# preparation
+docker_dir=$(docker volume inspect generated_btcpay_datadir --format="{{.Mountpoint}}" | sed -e "s%/volumes/.*%%g")
+dbdump_name=postgres.sql.gz
+btcpay_dir="$BTCPAY_BASE_DIRECTORY/btcpayserver-docker"
+backup_dir="$docker_dir/volumes/backup_datadir/_data"
+dbdump_path="$docker_dir/$dbdump_name"
+backup_path="$backup_dir/backup.tar.gz"
+
+# ensure backup dir exists
+if [ ! -d "$backup_dir" ]; then
+  mkdir -p $backup_dir
+fi
+
+cd $btcpay_dir
+. helpers.sh
+
+dbcontainer=$(docker ps -a -q -f "name=postgres_1")
+if [ -z "$dbcontainer" ]; then
+  printf "\n"
+  echo "ℹ️  Database container is not up and running. Starting BTCPay Server …"
+  docker volume create generated_postgres_datadir
+  docker-compose -f $BTCPAY_DOCKER_COMPOSE up -d postgres
+
+  printf "\n"
+  dbcontainer=$(docker ps -a -q -f "name=postgres_1")
+  if [ -z "$dbcontainer" ]; then
+    echo "🚨 Database container could not be started or found."
+    exit 1
+  fi
+fi
+
+printf "\n"
+echo "ℹ️  Dumping database …"
+{
+  docker exec $dbcontainer pg_dumpall -c -U postgres | gzip > $dbdump_path
+  echo "✅ Database dump done."
+} || {
+  echo "🚨 Dumping failed. Please check the error message above."
+  exit 1
+}
+
+printf "\nℹ️  Stopping BTCPay Server …\n\n"
+btcpay_down
+
+printf "\n"
+cd $docker_dir
+echo "ℹ️  Archiving files in $(pwd)…"
+
+{
+  tar \
+    --exclude="volumes/backup_datadir" \
+    --exclude="volumes/generated_bitcoin_datadir" \
+    --exclude="volumes/generated_litecoin_datadir" \
+    --exclude="volumes/generated_postgres_datadir" \
+    --exclude="volumes/generated_clightning_bitcoin_datadir/_data/lightning-rpc" \
+    --exclude="**/logs/*" \
+    -cvzf $backup_path $dbdump_name volumes/generated_*
+  echo "✅ Archive done."
+
+  if [ ! -z "$BTCPAY_BACKUP_PASSPHRASE" ]; then
+    printf "\n"
+    echo "🔐 BTCPAY_BACKUP_PASSPHRASE is set, the backup will be encrypted."
+    {
+      gpg -o "$backup_path.gpg" --batch --yes -c --passphrase "$BTCPAY_BACKUP_PASSPHRASE" $backup_path
+      rm $backup_path
+      backup_path="$backup_path.gpg"
+      echo "✅ Encryption done."
+    } || {
+      echo "🚨  Encrypting failed. Please check the error message above."
+      printf "\nℹ️  Restarting BTCPay Server …\n\n"
+      cd $btcpay_dir
+      btcpay_up
+      exit 1
+    }
+  fi
+} || {
+  echo "🚨  Archiving failed. Please check the error message above."
+  printf "\nℹ️  Restarting BTCPay Server …\n\n"
+  cd $btcpay_dir
+  btcpay_up
+  exit 1
+}
+
+printf "\nℹ️  Restarting BTCPay Server …\n\n"
+cd $btcpay_dir
+btcpay_up
+
+printf "\nℹ️  Cleaning up …\n\n"
+rm $dbdump_path
+
+printf "✅ Backup done => $backup_path\n\n"

btcpay-restore.sh

@@ -0,0 +1,138 @@
+#!/bin/bash -e
+
+set -o pipefail -o errexit
+
+if [ "$(id -u)" != "0" ]; then
+  printf "\n🚨 This script must be run as root.\n"
+  printf "➡️  Use the command 'sudo su -' (include the trailing hypen) and try again.\n\n"
+  exit 1
+fi
+
+backup_path=$1
+if [ -z "$backup_path" ]; then
+  printf "\nℹ️  Usage: btcpay-restore.sh /path/to/backup.tar.gz\n\n"
+  exit 1
+fi
+
+if [ ! -f "$backup_path" ]; then
+  printf "\n🚨 $backup_path does not exist.\n\n"
+  exit 1
+fi
+
+if [[ "$backup_path" == *.gpg && -z "$BTCPAY_BACKUP_PASSPHRASE" ]]; then
+  printf "\n🔐 $backup_path is encrypted. Please provide the passphrase to decrypt it."
+  printf "\nℹ️  Usage: BTCPAY_BACKUP_PASSPHRASE=t0pSeCrEt btcpay-restore.sh /path/to/backup.tar.gz.gpg\n\n"
+  exit 1
+fi
+
+# preparation
+docker_dir=$(docker volume inspect generated_btcpay_datadir --format="{{.Mountpoint}}" | sed -e "s%/volumes/.*%%g")
+restore_dir="$docker_dir/volumes/backup_datadir/_data/restore"
+dbdump_name=postgres.sql.gz
+btcpay_dir="$BTCPAY_BASE_DIRECTORY/btcpayserver-docker"
+
+# ensure clean restore dir
+printf "\nℹ️  Cleaning restore directory $restore_dir …\n\n"
+rm -rf $restore_dir
+mkdir -p $restore_dir
+
+if [[ "$backup_path" == *.gpg ]]; then
+  echo "🔐 Decrypting backup file …"
+  {
+    gpg -o "${backup_path%.*}" --batch --yes --passphrase "$BTCPAY_BACKUP_PASSPHRASE" -d $backup_path
+    backup_path="${backup_path%.*}"
+    printf "✅ Decryption done.\n\n"
+  } || {
+    echo "🚨  Decryption failed. Please check the error message above."
+    exit 1
+  }
+fi
+
+cd $restore_dir
+
+echo "ℹ️  Extracting files in $(pwd) …"
+tar -xvf $backup_path -C $restore_dir
+
+# basic control checks
+if [ ! -f "$dbdump_name" ]; then
+  printf "\n🚨 $dbdump_name does not exist.\n\n"
+  exit 1
+fi
+
+if [ ! -d "volumes" ]; then
+  printf "\n🚨 volumes directory does not exist.\n\n"
+  exit 1
+fi
+
+cd $btcpay_dir
+. helpers.sh
+
+printf "\nℹ️  Stopping BTCPay Server …\n\n"
+btcpay_down
+
+cd $restore_dir
+
+{
+  printf "\nℹ️  Restoring volumes …\n"
+  # ensure volumes dir exists
+  if [ ! -d "$docker_dir/volumes" ]; then
+    mkdir -p $docker_dir/volumes
+  fi
+  # copy volume directories over
+  cp -r volumes/* $docker_dir/volumes/
+  # ensure datadirs excluded in backup exist
+  mkdir -p $docker_dir/volumes/generated_bitcoin_datadir/_data
+  mkdir -p $docker_dir/volumes/generated_litecoin_datadir/_data
+  mkdir -p $docker_dir/volumes/generated_postgres_datadir/_data
+  echo "✅ Volume restore done."
+} || {
+  echo "🚨  Restoring volumes failed. Please check the error message above."
+  printf "\nℹ️  Restarting BTCPay Server …\n\n"
+  cd $btcpay_dir
+  btcpay_up
+  exit 1
+}
+
+{
+  printf "\nℹ️  Starting database container …\n"
+  docker-compose -f $BTCPAY_DOCKER_COMPOSE up -d postgres
+  sleep 10
+  dbcontainer=$(docker ps -a -q -f "name=postgres")
+  if [ -z "$dbcontainer" ]; then
+    echo "🚨 Database container could not be started or found."
+    printf "\nℹ️  Restarting BTCPay Server …\n\n"
+    cd $btcpay_dir
+    btcpay_up
+    exit 1
+  fi
+} || {
+  echo "🚨  Starting database container failed. Please check the error message above."
+  printf "\nℹ️  Restarting BTCPay Server …\n\n"
+  cd $btcpay_dir
+  btcpay_up
+  exit 1
+}
+
+cd $restore_dir
+
+{
+  printf "\nℹ️  Restoring database …"
+  gunzip -c $dbdump_name | docker exec -i $dbcontainer psql -U postgres postgres -a
+  echo "✅ Database restore done."
+} || {
+  echo "🚨  Restoring database failed. Please check the error message above."
+  printf "\nℹ️  Restarting BTCPay Server …\n\n"
+  cd $btcpay_dir
+  btcpay_up
+  exit 1
+}
+
+printf "\nℹ️  Restarting BTCPay Server …\n\n"
+cd $btcpay_dir
+btcpay_up
+
+printf "\nℹ️  Cleaning up …\n\n"
+# rm -rf $restore_dir
+# rm -rf $backup_path $backup_path.gpg
+
+printf "✅ Restore done\n\n"

helpers.sh

@@ -175,11 +175,7 @@ btcpay_restart() {
 btcpay_dump_db() {
     pushd . > /dev/null
     cd "$(dirname "$BTCPAY_ENV_FILE")"
-    backup_dir="/var/lib/docker/volumes/backup_datadir/_data"
+    local file_path=${1:-"postgres-$(date "+%Y%m%d-%H%M%S").sql.gz"}
-    if [ ! -d "$backup_dir" ]; then
+    docker exec $(docker ps -a -q -f "name=postgres_1") pg_dumpall -c -U postgres | gzip > "$file_path"
-        docker volume create backup_datadir
-    fi
-    local filename=${1:-"postgres-$(date "+%Y%m%d-%H%M%S").sql"}
-    docker exec $(docker ps -a -q -f "name=postgres_1") pg_dumpall -c -U postgres > "$backup_dir/$filename"
     popd > /dev/null
 }