This script is used on Linux hosts to complete additional offsite file backups as part of our larger DR plan. It has several features:

  • File backup and prune jobs
  • Integrated monitoring on Healthchecks.io, including start, success and failure messages.
  • Multiple attempts to complete both backups and prunes in case of failure.
#!/bin/bash
eval `ssh-agent -s`
ssh-add /root/.ssh/rem-bkups
export RESTIC_REPOSITORY="sftp:username@host:/data/directory"
export RESTIC_PASSWORD="12345"
HC="https://hc-ping.com/12345"

MAX_ATTEMPTS=3
RETRY_DELAY=60   # seconds between attempts

# Generic retry wrapper: retries the given command up to MAX_ATTEMPTS times
run_with_retries() {
    local label="$1"; shift
    local attempt=1
    while [ "$attempt" -le "$MAX_ATTEMPTS" ]; do
        echo "$label: attempt $attempt of $MAX_ATTEMPTS at $(date)"
        if "$@"; then
            echo "$label succeeded on attempt $attempt"
            return 0
        fi
        echo "$label failed on attempt $attempt"
        attempt=$((attempt + 1))
        [ "$attempt" -le "$MAX_ATTEMPTS" ] && sleep "$RETRY_DELAY"
    done
    echo "$label failed after $MAX_ATTEMPTS attempts"
    return 1
}

do_backup() {
    restic backup \
      /home \
      /etc \
      /opt \
      /var/www \
      --verbose
}

do_prune() {
    restic forget \
      --keep-last 1 \
      --keep-daily 2 \
      --keep-weekly 3 \
      --keep-monthly 4   \
      --prune
}

echo "Starting system backup at $(date)"
curl -fsS "$HC/start" >/dev/null 2>&1

# Only signal HC success if BOTH backup and prune succeed
if run_with_retries "Backup" do_backup && run_with_retries "Prune" do_prune; then
    echo "System backup completed at $(date)"
    curl -fsS "$HC" >/dev/null 2>&1
else
    echo "System backup or prune failed at $(date)"
    curl -fsS "$HC/fail" >/dev/null 2>&1
    exit 1
fi