script tidyups

README.md

@@ -9,9 +9,72 @@ Simple shell scripts for backing up Docker-related data and pruning old backups.
 - `containers.txt` - list of containers or backup targets
 - `.env.example` - example environment file
 
-## Setup
+## Reqirements
 
-Copy the example env file and edit it for your system:
+docker-compose
 
-```bash
+## BACKUPS: Example use docker-backup.sh
+
+Configure the backup script, docker-backup.sh
+Set up your list of containers in container.txt . The script assumes each container is in a subdirectory of one main docker directory
+
+Edit the location of where you want docker backups stored in docker-backup.sh (default is "/dockerbackups") or use
+
+./docker-backup.sh -s /home/me/mycontainers
+
+It will recurse though the directories if they are listed in containers.txt .
+It will run docker-compose to take down the container, pull a new version if there is one, then bring the container back up.
+Then it will create a timestamped arcive of each container in the backup directory, under a directory with that container name.
+
+Usage:  ./docker-backup.sh [-f containers_list_file] [-s source_base_dir] [-t] [-v] [-h]
+
+Flags:
+ -t          Test (dry run; no backups performed)
+ -v          Verbose (debug logging to stderr)
+ -f FILE     File containing list of container names
+             Default: '$containersfile'
+ -s DIR      Base directory containing container/project folders
+             Default: '$SOURCE_BASE_DIR'
+ -h          Show this help
+
+
+## PRUNE and REMOTE BACKUP dockerbackup_prune.sh
+
+Set up the env file using the example. 
 cp .env.example .env
+
+You don't need to worry about the rclone info (backups to google drive) it you won't be using that.
+It will just backup up the file locally.  Feel free to adjust for other remote backup systems.
+
+If you run .dockerbackup_prune.sh it will prune the number of backup archives that are already there (based on your prune numbers), make a .zst archive of all container backups
+then finally uploades to google drive and deletes the large archive (if that's what you set).
+Then it will prune the number of archive files for each 
+
+# local archive only (no pruning)
+./dockerbackup_prune.sh
+
+# local archive + real prune, keep new archive locally. Default prune is 5 archives)
+./dockerbackup_prune.sh --apply
+
+# local archive + upload + prune dry-run (no actual prune), keep new archive locally (default location is /tmp)
+./dockerbackup_prune.sh --rclone
+
+# full remote archive, google drive upload + real prune (default 5) + delete new local archive
+./dockerbackup_prune.sh --rclone --apply
+
+# Full remote workflow with actual prune leaving 3 archives of each container and final full archive delete.
+# Specify location of docker archives, final archive dir and remote google drive folder.
+./dockerbackup_prune.sh --root /mydockerarchives --/mybigbackupdolder --keep 3 --rclone /dockergooglebackup --apply
+
+Usage: $0 [--apply] [--quiet] [--dry-run] [--keep N] [--root PATH] [--source PATH] [--outdir PATH] [--remote PATH]
+
+Flags:
+  --rclone       Enable rclone upload flow
+  --apply        Actually delete pruned files; with --rclone also delete the new local archive
+  --quiet        Minimal console output (no live progress)
+  --dry-run      Simulate everything (no backup, no upload, no deletes)
+  --keep N       Keep newest N .zst files per subfolder (default ${KEEP_COUNT})
+  --root PATH    Root folder to prune (default ${PRUNE_ROOT})
+  --source PATH  Folder to archive (default ${SOURCE_DIR})
+  --outdir PATH  Where to write the .zst (default ${OUT_DIR})
+  --remote PATH  rclone destination (default ${RCLONE_REMOTE})

docker-backup.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
-# AUTHOR: Karl Mowatt-Wilson (updates by zorruno)
+# ORIGINAL AUTHOR: kmw (updates by zorruno)
 # HISTORY:
+#   2026-04-20 - add configurable SOURCE_BASE_DIR and -s option for source path fallback.
 #   2025-08-22 - add start wait with one retry; fix find_working_dir typo; tidy.
 #   2025-08-22 - tidy, stricter bash, compose detection, better checks, clearer logging.
 #   2022-03-25 - add -t option for dry-run (no backups performed).
@@ -27,6 +28,9 @@ dryrun=
 # base directory for backups
 BACKUP_DIR="/dockerbackups"
 
+# base directory for source docker project folders
+SOURCE_BASE_DIR="${SOURCE_BASE_DIR:-/dockervolumes}"
+
 # start/health wait tuning
 START_TIMEOUT=90     # seconds to wait for service to be up
 WAIT_INTERVAL=3      # seconds between polls
@@ -99,23 +103,27 @@ tidy_up() {
 
 usage() {
   cat 1>&2 <<EOF
-Usage:  ${0##*/} [-f containers_list_file] [-t] [-v] [-h]
+Usage:  ${0##*/} [-f containers_list_file] [-s source_base_dir] [-t] [-v] [-h]
 
  -t          Test (dry run; no backups performed)
  -v          Verbose (debug logging to stderr)
  -f FILE     File containing list of container names
              Default: '$containersfile'
+ -s DIR      Base directory containing container/project folders
+             Default: '$SOURCE_BASE_DIR'
  -h          Show this help
 
 Example:
   ${0##*/} -f containers.servername.txt
+  ${0##*/} -f containers.servername.txt -s /dockervolumes
 EOF
 }
 
 parse_cmdline() {
-  while getopts ":f:htv" opt; do
+  while getopts ":f:hs:tv" opt; do
     case "$opt" in
       f) containersfile="$OPTARG" ;;
+      s) SOURCE_BASE_DIR="$OPTARG" ;;
       h) usage; exit 0 ;;
       t) dryrun=1 ;;
       v) dbg=VERBOSE ;;
@@ -267,8 +275,8 @@ find_working_dir() {
     wd="$(docker inspect "$name" 2>/dev/null | jq -r '.[0].Config.Labels["com.docker.compose.project.working_dir"] // empty')"
   fi
   if [ -z "$wd" ]; then
-    if [ -d "/dockervolumes/$name" ]; then
+    if [ -d "$SOURCE_BASE_DIR/$name" ]; then
-      wd="/dockervolumes/$name"
+      wd="$SOURCE_BASE_DIR/$name"
     elif [ -d "$HOME/prj/docker/$name" ]; then
       wd="$HOME/prj/docker/$name"
     else
@@ -335,6 +343,7 @@ parse_cmdline "$@"
 
 debug "-----"
 debug "logfile = '$logfile'"
+debug "source_base_dir = '$SOURCE_BASE_DIR'"
 
 toolcheck docker jq sed tar
 

dockerbackup_prune.sh

@@ -4,12 +4,25 @@ set -Eeuo pipefail
 ###############################################################################
 # dockerbackup_prune.sh
 #
-# A) Create a dated .zst backup of /dockerbackups (tar stream | zstd, output .zst)
+# Default:
-#    - Clean single-line progress with percent and current dir/file (requires pv)
+#   A) Create a dated .zst backup locally and keep it
-#    - Fallback: single-line [N/TOTAL] without percent if pv is unavailable
+#
-# B) Upload via rclone
+# With --apply:
+#   A) Create a dated .zst backup
 #   C) Prune *.zst under PRUNE_ROOT subfolders, keeping newest N per subfolder
-# D) Delete the local .zst (only with --apply)
+#   D) Keep the new local archive
+#
+# With --rclone:
+#   A) Create a dated .zst backup
+#   B) Upload via rclone
+#   C) Show what prune would do (unless --apply is also set)
+#   D) Keep the new local archive unless --apply is also set
+#
+# With --rclone --apply:
+#   A) Create a dated .zst backup
+#   B) Upload via rclone
+#   C) Prune for real
+#   D) Delete the new local archive
 #
 # Restore:
 #   zstd -d -c /path/backup.zst | tar -x -C /restore/target
@@ -34,7 +47,6 @@ ZSTD_EXTRA_OPTS=""
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
 ENV_FILE="${DOCKERBACKUP_ENV:-$SCRIPT_DIR/.env}"
 if [[ -f "$ENV_FILE" ]]; then
-  # Warn if perms are too open; still load to keep it convenient
   perms="$(stat -c '%a' "$ENV_FILE" 2>/dev/null || stat -f '%Lp' "$ENV_FILE" 2>/dev/null || echo '')"
   if [[ -n "$perms" && ! "$perms" =~ ^(600|400)$ ]]; then
     echo "$(date '+%F %T') [WARN] $ENV_FILE permissions are $perms; recommend 600." >&2
@@ -49,15 +61,30 @@ fi
 APPLY=0
 QUIET=0
 DRYRUN=0
+USE_RCLONE=0
 
 usage() {
   cat <<EOF
-Usage: $0 [--apply] [--quiet] [--dry-run] [--keep N] [--root PATH] [--source PATH] [--outdir PATH] [--remote PATH]
+Usage: $0 [--rclone] [--apply] [--quiet] [--dry-run] [--keep N] [--root PATH] [--source PATH] [--outdir PATH] [--remote PATH]
 
-Default: creates and uploads backup, shows what WOULD be deleted (no deletes).
+Default:
+  Creates the local .zst archive and keeps it locally.
+  No rclone upload. No prune.
+
+With --apply:
+  Creates the local .zst archive, prunes old .zst files for real, and keeps the new archive locally.
+
+With --rclone:
+  Uploads the archive via rclone.
+  Prune also runs, but only as a dry-run unless --apply is also set.
+  The new local archive is kept unless --apply is also set.
+
+With --rclone --apply:
+  Uploads the archive, prunes for real, and deletes the new local archive.
 
 Flags:
-  --apply        Actually delete pruned files and the local .zst
+  --rclone       Enable rclone upload flow
+  --apply        Actually delete pruned files; with --rclone also delete the new local archive
   --quiet        Minimal console output (no live progress)
   --dry-run      Simulate everything (no backup, no upload, no deletes)
   --keep N       Keep newest N .zst files per subfolder (default ${KEEP_COUNT})
@@ -74,6 +101,7 @@ EOF
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
+    --rclone) USE_RCLONE=1 ;;
     --apply) APPLY=1 ;;
     --quiet) QUIET=1 ;;
     --dry-run) DRYRUN=1 ;;
@@ -94,7 +122,6 @@ log() { [[ $QUIET -eq 0 ]] && echo "$(ts) [$1] $2"; }
 log_err() { echo "$(ts) [ERROR] $1" >&2; }
 run() { if [[ $DRYRUN -eq 1 ]]; then log "DRY" "$*"; else "$@"; fi; }
 
-# Truncate to terminal width and erase tail so old text is cleared
 truncate_and_print() {
   local s="$1" cols
   if [[ -t 2 ]]; then
@@ -118,7 +145,7 @@ status_line_shown=0
 
 cleanup() {
   local why="${1:-EXIT}"
-  # Kill the whole pipeline (tar|pv|zstd)
+
   if [[ -n "$pipe_pid" ]]; then
     if command -v ps >/dev/null 2>&1; then
       pipe_pgid=$(ps -o pgid= "$pipe_pid" 2>/dev/null | tr -d ' ' || true)
@@ -134,19 +161,19 @@ cleanup() {
     fi
     wait "$pipe_pid" 2>/dev/null || true
   fi
-  # Close FDs
+
   if (( pct_fd_open == 1 )); then exec 4>&- 4<&-; pct_fd_open=0; fi
   if (( path_fd_open == 1 )); then exec 3>&- 3<&-; path_fd_open=0; fi
-  # Remove fifos/dir
+
   if [[ -n "$cleanup_tmp" ]]; then
     rm -f "$cleanup_tmp/pct" "$cleanup_tmp/path" 2>/dev/null || true
     rmdir "$cleanup_tmp" 2>/dev/null || true
   fi
-  # End the progress line cleanly
+
   if (( status_line_shown == 1 )) && [[ $QUIET -eq 0 ]]; then
     printf '\n' >&2
   fi
-  # Exit with 130 on INT/TERM
+
   case "$why" in
     INT|TERM) exit 130 ;;
   esac
@@ -162,10 +189,12 @@ trap 'cleanup EXIT' EXIT
 [[ -d "$PRUNE_ROOT" ]] || { log_err "Prune root not found: $PRUNE_ROOT"; exit 1; }
 command -v tar >/dev/null 2>&1 || { log_err "tar not found"; exit 1; }
 command -v zstd >/dev/null 2>&1 || { log_err "zstd not found"; exit 1; }
-command -v rclone >/dev/null 2>&1 || { log_err "rclone not found"; exit 1; }
 command -v find >/dev/null 2>&1 || { log_err "find not found"; exit 1; }
 
-# Timestamp in name: YYYY-MM-DD_HH-MM (system timezone or override externally)
+if (( USE_RCLONE == 1 )) && (( DRYRUN == 0 )); then
+  command -v rclone >/dev/null 2>&1 || { log_err "rclone not found"; exit 1; }
+fi
+
 STAMP="${STAMP_OVERRIDE:-$(date +%F_%H-%M)}"
 BACKUP_NAME="${BACKUP_PREFIX}-${STAMP}.zst"
 BACKUP_PATH="${OUT_DIR%/}/${BACKUP_NAME}"
@@ -173,15 +202,17 @@ BACKUP_PATH="${OUT_DIR%/}/${BACKUP_NAME}"
 RCLONE_FLAGS=()
 [[ $QUIET -eq 1 ]] && RCLONE_FLAGS+=("-q") || RCLONE_FLAGS+=("--progress")
 
-# Silence zstd "Read: ... ==> 100%"
 ZSTD_FLAGS=("-q")
 [[ -n "$ZSTD_EXTRA_OPTS" ]] && ZSTD_FLAGS+=($ZSTD_EXTRA_OPTS)
 
 log "INFO" "Mode: $( [[ $DRYRUN -eq 1 ]] && echo 'DRY-RUN ALL' || ([[ $APPLY -eq 1 ]] && echo 'APPLY DELETES' || echo 'NO DELETES') )"
+log "INFO" "Rclone mode: $( [[ $USE_RCLONE -eq 1 ]] && echo 'ENABLED' || echo 'DISABLED' )"
 log "INFO" "Creating archive: $BACKUP_PATH (.zst)"
 log "INFO" "Keeping newest $KEEP_COUNT file(s) per subfolder matching: $PRUNE_PATTERN"
 log "INFO" "Source: $SOURCE_DIR"
+if (( USE_RCLONE == 1 )); then
   log "INFO" "Remote: $RCLONE_REMOTE"
+fi
 log "INFO" "Prune root (root itself will be skipped): $PRUNE_ROOT"
 
 # --------------------------- Step A: create .zst with one-liner --------------
@@ -204,11 +235,9 @@ else
     mkfifo "$pct_fifo"; have_pv=1
   fi
 
-  # Open FIFOs RDWR so writers/readers never block
   exec 3<> "$path_fifo"; path_fd_open=1
   if (( have_pv == 1 )); then exec 4<> "$pct_fifo"; pct_fd_open=1; fi
 
-  # Run pipeline in background; send tar -v names into the path FIFO
   if (( have_pv == 1 )); then
     (
       set -o pipefail
@@ -227,7 +256,6 @@ else
   fi
   pipe_pid=$!
 
-  # Status loop: single clean line on stderr
   files_seen=0
   last_path=""
   last_pct="0.0"
@@ -235,23 +263,20 @@ else
   last_print_time=0
 
   while kill -0 "$pipe_pid" 2>/dev/null; do
-    # read percent from pv (if any)
     if (( have_pv == 1 )); then
       if IFS= read -r -t 0.03 pct_line <&4; then last_pct="$pct_line"; fi
     fi
-    # read latest path from tar
+
     if IFS= read -r -t 0.03 path_line <&3; then
       last_path="$path_line"
       [[ -n "$last_path" ]] && (( files_seen++ )) || true
     fi
 
-    # Split into dir/file for display (already relative)
     display_rel="$last_path"
     dir_part="$(dirname -- "$display_rel" 2>/dev/null || echo .)"
     base_part="$(basename -- "$display_rel" 2>/dev/null || echo .)"
     [[ "$dir_part" == "." ]] && dir_part="."
 
-    # Refresh line and erase tail (\033[K)
     if [[ $QUIET -eq 0 ]]; then
       now_ns=$(date +%s%N)
       if (( now_ns - last_print_time > 50000000 )); then
@@ -267,7 +292,6 @@ else
     fi
   done
 
-  # Drain, finish, cleanup
   while IFS= read -r -t 0.03 path_line <&3; do
     last_path="$path_line"; (( files_seen++ )) || true
     [[ $QUIET -eq 0 ]] && truncate_and_print "[${files_seen}/${TOTAL_FILES}] archiving: $(dirname -- "$last_path")/ $(basename -- "$last_path")"
@@ -280,7 +304,6 @@ else
     status_line_shown=0
   fi
 
-  # Close and clean up
   exec 3>&- 3<&-; path_fd_open=0
   if (( have_pv == 1 )); then exec 4>&- 4<&-; pct_fd_open=0; fi
   rm -f "$path_fifo" "$pct_fifo" 2>/dev/null || true
@@ -289,7 +312,8 @@ else
 fi
 log "OK" "Archive step completed"
 
-# --------------------------- Step B: rclone copy -----------------------------
+# --------------------------- Step B: optional rclone copy --------------------
+if (( USE_RCLONE == 1 )); then
   log "STEP" "B) rclone copy to remote"
   if [[ $DRYRUN -eq 1 ]]; then
     log "DRY" "rclone copy \"$BACKUP_PATH\" \"$RCLONE_REMOTE\" ${RCLONE_FLAGS[*]}"
@@ -297,8 +321,12 @@ else
     run rclone copy "$BACKUP_PATH" "$RCLONE_REMOTE" "${RCLONE_FLAGS[@]}"
   fi
   log "OK" "Upload step completed"
+else
+  log "SKIP" "--rclone not set; skipping upload"
+fi
 
-# --------------------------- Step C: prune .zst ------------------------------
+# --------------------------- Step C: prune if apply or rclone ---------------
+if (( APPLY == 1 || USE_RCLONE == 1 )); then
   log "STEP" "C) Pruning $PRUNE_PATTERN under $PRUNE_ROOT (keep $KEEP_COUNT)"
   export LC_ALL=C
   prune_ok=1
@@ -347,8 +375,12 @@ if [[ $prune_ok -ne 1 ]]; then
     exit 1
   fi
   log "OK" "Prune step completed"
+else
+  log "SKIP" "--apply and --rclone not set; skipping prune"
+fi
 
 # --------------------------- Step D: remove local .zst -----------------------
+if (( USE_RCLONE == 1 )); then
   log "STEP" "D) Removing local backup"
   if [[ $DRYRUN -eq 1 || $APPLY -eq 0 ]]; then
     log "WOULD" "Remove local backup: $BACKUP_PATH"
@@ -360,5 +392,8 @@ else
       exit 1
     fi
   fi
+else
+  log "KEEP" "Local archive retained: $BACKUP_PATH"
+fi
 
 log "DONE" "All steps completed successfully"