zorruno/zorruno-backupdockerscript
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

script tidyups

Browse Source
This commit is contained in:
zorruno
2026-04-20 14:12:35 +12:00
parent a94d723bdf
commit 5c7f9b0c9d
3 changed files with 213 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+66 -3
View File
@@ -9,9 +9,72 @@ Simple shell scripts for backing up Docker-related data and pruning old backups.
- `containers.txt` - list of containers or backup targets
- `.env.example` - example environment file
## Setup
## Reqirements
Copy the example env file and edit it for your system:
docker-compose
```bash
## BACKUPS: Example use docker-backup.sh
Configure the backup script, docker-backup.sh
Set up your list of containers in container.txt . The script assumes each container is in a subdirectory of one main docker directory
Edit the location of where you want docker backups stored in docker-backup.sh (default is "/dockerbackups") or use
./docker-backup.sh -s /home/me/mycontainers
It will recurse though the directories if they are listed in containers.txt .
It will run docker-compose to take down the container, pull a new version if there is one, then bring the container back up.
Then it will create a timestamped arcive of each container in the backup directory, under a directory with that container name.
Usage: ./docker-backup.sh [-f containers_list_file] [-s source_base_dir] [-t] [-v] [-h]
Flags:
-t Test (dry run; no backups performed)
-v Verbose (debug logging to stderr)
-f FILE File containing list of container names
Default: '$containersfile'
-s DIR Base directory containing container/project folders
Default: '$SOURCE_BASE_DIR'
-h Show this help
## PRUNE and REMOTE BACKUP dockerbackup_prune.sh
Set up the env file using the example.
cp .env.example .env
You don't need to worry about the rclone info (backups to google drive) it you won't be using that.
It will just backup up the file locally. Feel free to adjust for other remote backup systems.
If you run .dockerbackup_prune.sh it will prune the number of backup archives that are already there (based on your prune numbers), make a .zst archive of all container backups
then finally uploades to google drive and deletes the large archive (if that's what you set).
Then it will prune the number of archive files for each
# local archive only (no pruning)
./dockerbackup_prune.sh
# local archive + real prune, keep new archive locally. Default prune is 5 archives)
./dockerbackup_prune.sh --apply
# local archive + upload + prune dry-run (no actual prune), keep new archive locally (default location is /tmp)
./dockerbackup_prune.sh --rclone
# full remote archive, google drive upload + real prune (default 5) + delete new local archive
./dockerbackup_prune.sh --rclone --apply
# Full remote workflow with actual prune leaving 3 archives of each container and final full archive delete.
# Specify location of docker archives, final archive dir and remote google drive folder.
./dockerbackup_prune.sh --root /mydockerarchives --/mybigbackupdolder --keep 3 --rclone /dockergooglebackup --apply
Usage: $0 [--apply] [--quiet] [--dry-run] [--keep N] [--root PATH] [--source PATH] [--outdir PATH] [--remote PATH]
Flags:
--rclone Enable rclone upload flow
--apply Actually delete pruned files; with --rclone also delete the new local archive
--quiet Minimal console output (no live progress)
--dry-run Simulate everything (no backup, no upload, no deletes)
--keep N Keep newest N .zst files per subfolder (default ${KEEP_COUNT})
--root PATH Root folder to prune (default ${PRUNE_ROOT})
--source PATH Folder to archive (default ${SOURCE_DIR})
--outdir PATH Where to write the .zst (default ${OUT_DIR})
--remote PATH rclone destination (default ${RCLONE_REMOTE})
docker-backup.sh
Executable → Regular
+14 -5
View File
@@ -1,6 +1,7 @@
#!/usr/bin/env bash
# AUTHOR: Karl Mowatt-Wilson (updates by zorruno)
# ORIGINAL AUTHOR: kmw (updates by zorruno)
# HISTORY:
# 2026-04-20 - add configurable SOURCE_BASE_DIR and -s option for source path fallback.
# 2025-08-22 - add start wait with one retry; fix find_working_dir typo; tidy.
# 2025-08-22 - tidy, stricter bash, compose detection, better checks, clearer logging.
# 2022-03-25 - add -t option for dry-run (no backups performed).
@@ -27,6 +28,9 @@ dryrun=
# base directory for backups
BACKUP_DIR="/dockerbackups"
# base directory for source docker project folders
SOURCE_BASE_DIR="${SOURCE_BASE_DIR:-/dockervolumes}"
# start/health wait tuning
START_TIMEOUT=90 # seconds to wait for service to be up
WAIT_INTERVAL=3 # seconds between polls
@@ -99,23 +103,27 @@ tidy_up() {
usage() {
cat 1>&2 <<EOF
Usage: ${0##*/} [-f containers_list_file] [-t] [-v] [-h]
Usage: ${0##*/} [-f containers_list_file] [-s source_base_dir] [-t] [-v] [-h]
-t Test (dry run; no backups performed)
-v Verbose (debug logging to stderr)
-f FILE File containing list of container names
Default: '$containersfile'
-s DIR Base directory containing container/project folders
Default: '$SOURCE_BASE_DIR'
-h Show this help
Example:
${0##*/} -f containers.servername.txt
${0##*/} -f containers.servername.txt -s /dockervolumes
EOF
}
parse_cmdline() {
while getopts ":f:htv" opt; do
while getopts ":f:hs:tv" opt; do
case "$opt" in
f) containersfile="$OPTARG" ;;
s) SOURCE_BASE_DIR="$OPTARG" ;;
h) usage; exit 0 ;;
t) dryrun=1 ;;
v) dbg=VERBOSE ;;
@@ -267,8 +275,8 @@ find_working_dir() {
wd="$(docker inspect "$name" 2>/dev/null | jq -r '.[0].Config.Labels["com.docker.compose.project.working_dir"] // empty')"
fi
if [ -z "$wd" ]; then
if [ -d "/dockervolumes/$name" ]; then
wd="/dockervolumes/$name"
if [ -d "$SOURCE_BASE_DIR/$name" ]; then
wd="$SOURCE_BASE_DIR/$name"
elif [ -d "$HOME/prj/docker/$name" ]; then
wd="$HOME/prj/docker/$name"
else
@@ -335,6 +343,7 @@ parse_cmdline "$@"
debug "-----"
debug "logfile = '$logfile'"
debug "source_base_dir = '$SOURCE_BASE_DIR'"
toolcheck docker jq sed tar
dockerbackup_prune.sh
Executable → Regular
+133 -98
View File
@@ -4,12 +4,25 @@ set -Eeuo pipefail
###############################################################################
# dockerbackup_prune.sh
#
# A) Create a dated .zst backup of /dockerbackups (tar stream | zstd, output .zst)
# - Clean single-line progress with percent and current dir/file (requires pv)
# - Fallback: single-line [N/TOTAL] without percent if pv is unavailable
# B) Upload via rclone
# C) Prune *.zst under PRUNE_ROOT subfolders, keeping newest N per subfolder
# D) Delete the local .zst (only with --apply)
# Default:
# A) Create a dated .zst backup locally and keep it
#
# With --apply:
# A) Create a dated .zst backup
# C) Prune *.zst under PRUNE_ROOT subfolders, keeping newest N per subfolder
# D) Keep the new local archive
#
# With --rclone:
# A) Create a dated .zst backup
# B) Upload via rclone
# C) Show what prune would do (unless --apply is also set)
# D) Keep the new local archive unless --apply is also set
#
# With --rclone --apply:
# A) Create a dated .zst backup
# B) Upload via rclone
# C) Prune for real
# D) Delete the new local archive
#
# Restore:
# zstd -d -c /path/backup.zst | tar -x -C /restore/target
@@ -34,7 +47,6 @@ ZSTD_EXTRA_OPTS=""
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
ENV_FILE="${DOCKERBACKUP_ENV:-$SCRIPT_DIR/.env}"
if [[ -f "$ENV_FILE" ]]; then
# Warn if perms are too open; still load to keep it convenient
perms="$(stat -c '%a' "$ENV_FILE" 2>/dev/null || stat -f '%Lp' "$ENV_FILE" 2>/dev/null || echo '')"
if [[ -n "$perms" && ! "$perms" =~ ^(600|400)$ ]]; then
echo "$(date '+%F %T') [WARN] $ENV_FILE permissions are $perms; recommend 600." >&2
@@ -49,15 +61,30 @@ fi
APPLY=0
QUIET=0
DRYRUN=0
USE_RCLONE=0
usage() {
cat <<EOF
Usage: $0 [--apply] [--quiet] [--dry-run] [--keep N] [--root PATH] [--source PATH] [--outdir PATH] [--remote PATH]
Usage: $0 [--rclone] [--apply] [--quiet] [--dry-run] [--keep N] [--root PATH] [--source PATH] [--outdir PATH] [--remote PATH]
Default: creates and uploads backup, shows what WOULD be deleted (no deletes).
Default:
Creates the local .zst archive and keeps it locally.
No rclone upload. No prune.
With --apply:
Creates the local .zst archive, prunes old .zst files for real, and keeps the new archive locally.
With --rclone:
Uploads the archive via rclone.
Prune also runs, but only as a dry-run unless --apply is also set.
The new local archive is kept unless --apply is also set.
With --rclone --apply:
Uploads the archive, prunes for real, and deletes the new local archive.
Flags:
--apply Actually delete pruned files and the local .zst
--rclone Enable rclone upload flow
--apply Actually delete pruned files; with --rclone also delete the new local archive
--quiet Minimal console output (no live progress)
--dry-run Simulate everything (no backup, no upload, no deletes)
--keep N Keep newest N .zst files per subfolder (default ${KEEP_COUNT})
@@ -74,6 +101,7 @@ EOF
while [[ $# -gt 0 ]]; do
case "$1" in
--rclone) USE_RCLONE=1 ;;
--apply) APPLY=1 ;;
--quiet) QUIET=1 ;;
--dry-run) DRYRUN=1 ;;
@@ -94,7 +122,6 @@ log() { [[ $QUIET -eq 0 ]] && echo "$(ts) [$1] $2"; }
log_err() { echo "$(ts) [ERROR] $1" >&2; }
run() { if [[ $DRYRUN -eq 1 ]]; then log "DRY" "$*"; else "$@"; fi; }
# Truncate to terminal width and erase tail so old text is cleared
truncate_and_print() {
local s="$1" cols
if [[ -t 2 ]]; then
@@ -102,8 +129,8 @@ truncate_and_print() {
else
cols=120
fi
(( cols<20 )) && cols=120
if (( ${#s} > cols-1 )); then
(( cols < 20 )) && cols=120
if (( ${#s} > cols - 1 )); then
s="${s:0:cols-1}"
fi
printf '\r%s\033[K' "$s" >&2
@@ -118,7 +145,7 @@ status_line_shown=0
cleanup() {
local why="${1:-EXIT}"
# Kill the whole pipeline (tar|pv|zstd)
if [[ -n "$pipe_pid" ]]; then
if command -v ps >/dev/null 2>&1; then
pipe_pgid=$(ps -o pgid= "$pipe_pid" 2>/dev/null | tr -d ' ' || true)
@@ -134,19 +161,19 @@ cleanup() {
fi
wait "$pipe_pid" 2>/dev/null || true
fi
# Close FDs
if (( pct_fd_open == 1 )); then exec 4>&- 4<&-; pct_fd_open=0; fi
if (( path_fd_open == 1 )); then exec 3>&- 3<&-; path_fd_open=0; fi
# Remove fifos/dir
if [[ -n "$cleanup_tmp" ]]; then
rm -f "$cleanup_tmp/pct" "$cleanup_tmp/path" 2>/dev/null || true
rmdir "$cleanup_tmp" 2>/dev/null || true
fi
# End the progress line cleanly
if (( status_line_shown == 1 )) && [[ $QUIET -eq 0 ]]; then
printf '\n' >&2
fi
# Exit with 130 on INT/TERM
case "$why" in
INT|TERM) exit 130 ;;
esac
@@ -162,10 +189,12 @@ trap 'cleanup EXIT' EXIT
[[ -d "$PRUNE_ROOT" ]] || { log_err "Prune root not found: $PRUNE_ROOT"; exit 1; }
command -v tar >/dev/null 2>&1 || { log_err "tar not found"; exit 1; }
command -v zstd >/dev/null 2>&1 || { log_err "zstd not found"; exit 1; }
command -v rclone >/dev/null 2>&1 || { log_err "rclone not found"; exit 1; }
command -v find >/dev/null 2>&1 || { log_err "find not found"; exit 1; }
# Timestamp in name: YYYY-MM-DD_HH-MM (system timezone or override externally)
if (( USE_RCLONE == 1 )) && (( DRYRUN == 0 )); then
command -v rclone >/dev/null 2>&1 || { log_err "rclone not found"; exit 1; }
fi
STAMP="${STAMP_OVERRIDE:-$(date +%F_%H-%M)}"
BACKUP_NAME="${BACKUP_PREFIX}-${STAMP}.zst"
BACKUP_PATH="${OUT_DIR%/}/${BACKUP_NAME}"
@@ -173,15 +202,17 @@ BACKUP_PATH="${OUT_DIR%/}/${BACKUP_NAME}"
RCLONE_FLAGS=()
[[ $QUIET -eq 1 ]] && RCLONE_FLAGS+=("-q") || RCLONE_FLAGS+=("--progress")
# Silence zstd "Read: ... ==> 100%"
ZSTD_FLAGS=("-q")
[[ -n "$ZSTD_EXTRA_OPTS" ]] && ZSTD_FLAGS+=($ZSTD_EXTRA_OPTS)
log "INFO" "Mode: $( [[ $DRYRUN -eq 1 ]] && echo 'DRY-RUN ALL' || ([[ $APPLY -eq 1 ]] && echo 'APPLY DELETES' || echo 'NO DELETES') )"
log "INFO" "Rclone mode: $( [[ $USE_RCLONE -eq 1 ]] && echo 'ENABLED' || echo 'DISABLED' )"
log "INFO" "Creating archive: $BACKUP_PATH (.zst)"
log "INFO" "Keeping newest $KEEP_COUNT file(s) per subfolder matching: $PRUNE_PATTERN"
log "INFO" "Source: $SOURCE_DIR"
log "INFO" "Remote: $RCLONE_REMOTE"
if (( USE_RCLONE == 1 )); then
log "INFO" "Remote: $RCLONE_REMOTE"
fi
log "INFO" "Prune root (root itself will be skipped): $PRUNE_ROOT"
# --------------------------- Step A: create .zst with one-liner --------------
@@ -204,11 +235,9 @@ else
mkfifo "$pct_fifo"; have_pv=1
fi
# Open FIFOs RDWR so writers/readers never block
exec 3<> "$path_fifo"; path_fd_open=1
if (( have_pv == 1 )); then exec 4<> "$pct_fifo"; pct_fd_open=1; fi
# Run pipeline in background; send tar -v names into the path FIFO
if (( have_pv == 1 )); then
(
set -o pipefail
@@ -227,7 +256,6 @@ else
fi
pipe_pid=$!
# Status loop: single clean line on stderr
files_seen=0
last_path=""
last_pct="0.0"
@@ -235,23 +263,20 @@ else
last_print_time=0
while kill -0 "$pipe_pid" 2>/dev/null; do
# read percent from pv (if any)
if (( have_pv == 1 )); then
if IFS= read -r -t 0.03 pct_line <&4; then last_pct="$pct_line"; fi
fi
# read latest path from tar
if IFS= read -r -t 0.03 path_line <&3; then
last_path="$path_line"
[[ -n "$last_path" ]] && (( files_seen++ )) || true
fi
# Split into dir/file for display (already relative)
display_rel="$last_path"
dir_part="$(dirname -- "$display_rel" 2>/dev/null || echo .)"
base_part="$(basename -- "$display_rel" 2>/dev/null || echo .)"
[[ "$dir_part" == "." ]] && dir_part="."
# Refresh line and erase tail (\033[K)
if [[ $QUIET -eq 0 ]]; then
now_ns=$(date +%s%N)
if (( now_ns - last_print_time > 50000000 )); then
@@ -267,7 +292,6 @@ else
fi
done
# Drain, finish, cleanup
while IFS= read -r -t 0.03 path_line <&3; do
last_path="$path_line"; (( files_seen++ )) || true
[[ $QUIET -eq 0 ]] && truncate_and_print "[${files_seen}/${TOTAL_FILES}] archiving: $(dirname -- "$last_path")/ $(basename -- "$last_path")"
@@ -280,7 +304,6 @@ else
status_line_shown=0
fi
# Close and clean up
exec 3>&- 3<&-; path_fd_open=0
if (( have_pv == 1 )); then exec 4>&- 4<&-; pct_fd_open=0; fi
rm -f "$path_fifo" "$pct_fifo" 2>/dev/null || true
@@ -289,76 +312,88 @@ else
fi
log "OK" "Archive step completed"
# --------------------------- Step B: rclone copy -----------------------------
log "STEP" "B) rclone copy to remote"
if [[ $DRYRUN -eq 1 ]]; then
log "DRY" "rclone copy \"$BACKUP_PATH\" \"$RCLONE_REMOTE\" ${RCLONE_FLAGS[*]}"
else
run rclone copy "$BACKUP_PATH" "$RCLONE_REMOTE" "${RCLONE_FLAGS[@]}"
fi
log "OK" "Upload step completed"
# --------------------------- Step C: prune .zst ------------------------------
log "STEP" "C) Pruning $PRUNE_PATTERN under $PRUNE_ROOT (keep $KEEP_COUNT)"
export LC_ALL=C
prune_ok=1
while IFS= read -r -d '' dir; do
mapfile -d '' -t files < <(
find "$dir" -maxdepth 1 -type f -name "$PRUNE_PATTERN" -printf '%T@ %p\0' |
sort -z -n |
cut -z -d' ' -f2-
)
count=${#files[@]}
if (( count == 0 )); then
log "SKIP" "No matches in: $dir"
continue
fi
keep_from=$(( count - KEEP_COUNT ))
(( keep_from < 0 )) && keep_from=0
if (( count <= KEEP_COUNT )); then
log "KEEP" "At or below limit in: $dir (count=$count)"
continue
fi
keep_list=("${files[@]:keep_from:KEEP_COUNT}")
del_list=("${files[@]:0:keep_from}")
for k in "${keep_list[@]}"; do
log "KEEP" "Keep: $k"
done
for f in "${del_list[@]}"; do
if [[ $DRYRUN -eq 1 || $APPLY -eq 0 ]]; then
log "WOULD" "Remove: $f"
else
if rm -f -- "$f"; then
log "DELETE" "Removed: $f"
else
log_err "Failed to remove: $f"; prune_ok=0
fi
fi
done
done < <(find "$PRUNE_ROOT" -mindepth 1 -type d -print0)
if [[ $prune_ok -ne 1 ]]; then
log_err "Prune step encountered errors; aborting final delete"
exit 1
fi
log "OK" "Prune step completed"
# --------------------------- Step D: remove local .zst -----------------------
log "STEP" "D) Removing local backup"
if [[ $DRYRUN -eq 1 || $APPLY -eq 0 ]]; then
log "WOULD" "Remove local backup: $BACKUP_PATH"
else
if rm -f -- "$BACKUP_PATH"; then
log "DELETE" "Removed local backup: $BACKUP_PATH"
# --------------------------- Step B: optional rclone copy --------------------
if (( USE_RCLONE == 1 )); then
log "STEP" "B) rclone copy to remote"
if [[ $DRYRUN -eq 1 ]]; then
log "DRY" "rclone copy \"$BACKUP_PATH\" \"$RCLONE_REMOTE\" ${RCLONE_FLAGS[*]}"
else
log_err "Failed to remove local backup: $BACKUP_PATH"
run rclone copy "$BACKUP_PATH" "$RCLONE_REMOTE" "${RCLONE_FLAGS[@]}"
fi
log "OK" "Upload step completed"
else
log "SKIP" "--rclone not set; skipping upload"
fi
# --------------------------- Step C: prune if apply or rclone ---------------
if (( APPLY == 1 || USE_RCLONE == 1 )); then
log "STEP" "C) Pruning $PRUNE_PATTERN under $PRUNE_ROOT (keep $KEEP_COUNT)"
export LC_ALL=C
prune_ok=1
while IFS= read -r -d '' dir; do
mapfile -d '' -t files < <(
find "$dir" -maxdepth 1 -type f -name "$PRUNE_PATTERN" -printf '%T@ %p\0' |
sort -z -n |
cut -z -d' ' -f2-
)
count=${#files[@]}
if (( count == 0 )); then
log "SKIP" "No matches in: $dir"
continue
fi
keep_from=$(( count - KEEP_COUNT ))
(( keep_from < 0 )) && keep_from=0
if (( count <= KEEP_COUNT )); then
log "KEEP" "At or below limit in: $dir (count=$count)"
continue
fi
keep_list=("${files[@]:keep_from:KEEP_COUNT}")
del_list=("${files[@]:0:keep_from}")
for k in "${keep_list[@]}"; do
log "KEEP" "Keep: $k"
done
for f in "${del_list[@]}"; do
if [[ $DRYRUN -eq 1 || $APPLY -eq 0 ]]; then
log "WOULD" "Remove: $f"
else
if rm -f -- "$f"; then
log "DELETE" "Removed: $f"
else
log_err "Failed to remove: $f"; prune_ok=0
fi
fi
done
done < <(find "$PRUNE_ROOT" -mindepth 1 -type d -print0)
if [[ $prune_ok -ne 1 ]]; then
log_err "Prune step encountered errors; aborting final delete"
exit 1
fi
log "OK" "Prune step completed"
else
log "SKIP" "--apply and --rclone not set; skipping prune"
fi
# --------------------------- Step D: remove local .zst -----------------------
if (( USE_RCLONE == 1 )); then
log "STEP" "D) Removing local backup"
if [[ $DRYRUN -eq 1 || $APPLY -eq 0 ]]; then
log "WOULD" "Remove local backup: $BACKUP_PATH"
else
if rm -f -- "$BACKUP_PATH"; then
log "DELETE" "Removed local backup: $BACKUP_PATH"
else
log_err "Failed to remove local backup: $BACKUP_PATH"
exit 1
fi
fi
else
log "KEEP" "Local archive retained: $BACKUP_PATH"
fi
log "DONE" "All steps completed successfully"