diff --git a/README.md b/README.md index ffec089..38d7c20 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,72 @@ Simple shell scripts for backing up Docker-related data and pruning old backups. - `containers.txt` - list of containers or backup targets - `.env.example` - example environment file -## Setup +## Reqirements -Copy the example env file and edit it for your system: +docker-compose -```bash +## BACKUPS: Example use docker-backup.sh + +Configure the backup script, docker-backup.sh +Set up your list of containers in container.txt . The script assumes each container is in a subdirectory of one main docker directory + +Edit the location of where you want docker backups stored in docker-backup.sh (default is "/dockerbackups") or use + +./docker-backup.sh -s /home/me/mycontainers + +It will recurse though the directories if they are listed in containers.txt . +It will run docker-compose to take down the container, pull a new version if there is one, then bring the container back up. +Then it will create a timestamped arcive of each container in the backup directory, under a directory with that container name. + +Usage: ./docker-backup.sh [-f containers_list_file] [-s source_base_dir] [-t] [-v] [-h] + +Flags: + -t Test (dry run; no backups performed) + -v Verbose (debug logging to stderr) + -f FILE File containing list of container names + Default: '$containersfile' + -s DIR Base directory containing container/project folders + Default: '$SOURCE_BASE_DIR' + -h Show this help + + +## PRUNE and REMOTE BACKUP dockerbackup_prune.sh + +Set up the env file using the example. cp .env.example .env + +You don't need to worry about the rclone info (backups to google drive) it you won't be using that. +It will just backup up the file locally. Feel free to adjust for other remote backup systems. + +If you run .dockerbackup_prune.sh it will prune the number of backup archives that are already there (based on your prune numbers), make a .zst archive of all container backups +then finally uploades to google drive and deletes the large archive (if that's what you set). +Then it will prune the number of archive files for each + +# local archive only (no pruning) +./dockerbackup_prune.sh + +# local archive + real prune, keep new archive locally. Default prune is 5 archives) +./dockerbackup_prune.sh --apply + +# local archive + upload + prune dry-run (no actual prune), keep new archive locally (default location is /tmp) +./dockerbackup_prune.sh --rclone + +# full remote archive, google drive upload + real prune (default 5) + delete new local archive +./dockerbackup_prune.sh --rclone --apply + +# Full remote workflow with actual prune leaving 3 archives of each container and final full archive delete. +# Specify location of docker archives, final archive dir and remote google drive folder. +./dockerbackup_prune.sh --root /mydockerarchives --/mybigbackupdolder --keep 3 --rclone /dockergooglebackup --apply + +Usage: $0 [--apply] [--quiet] [--dry-run] [--keep N] [--root PATH] [--source PATH] [--outdir PATH] [--remote PATH] + +Flags: + --rclone Enable rclone upload flow + --apply Actually delete pruned files; with --rclone also delete the new local archive + --quiet Minimal console output (no live progress) + --dry-run Simulate everything (no backup, no upload, no deletes) + --keep N Keep newest N .zst files per subfolder (default ${KEEP_COUNT}) + --root PATH Root folder to prune (default ${PRUNE_ROOT}) + --source PATH Folder to archive (default ${SOURCE_DIR}) + --outdir PATH Where to write the .zst (default ${OUT_DIR}) + --remote PATH rclone destination (default ${RCLONE_REMOTE}) diff --git a/docker-backup.sh b/docker-backup.sh old mode 100755 new mode 100644 index 971781e..f20a730 --- a/docker-backup.sh +++ b/docker-backup.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -# AUTHOR: Karl Mowatt-Wilson (updates by zorruno) +# ORIGINAL AUTHOR: kmw (updates by zorruno) # HISTORY: +# 2026-04-20 - add configurable SOURCE_BASE_DIR and -s option for source path fallback. # 2025-08-22 - add start wait with one retry; fix find_working_dir typo; tidy. # 2025-08-22 - tidy, stricter bash, compose detection, better checks, clearer logging. # 2022-03-25 - add -t option for dry-run (no backups performed). @@ -27,6 +28,9 @@ dryrun= # base directory for backups BACKUP_DIR="/dockerbackups" +# base directory for source docker project folders +SOURCE_BASE_DIR="${SOURCE_BASE_DIR:-/dockervolumes}" + # start/health wait tuning START_TIMEOUT=90 # seconds to wait for service to be up WAIT_INTERVAL=3 # seconds between polls @@ -99,23 +103,27 @@ tidy_up() { usage() { cat 1>&2 </dev/null | jq -r '.[0].Config.Labels["com.docker.compose.project.working_dir"] // empty')" fi if [ -z "$wd" ]; then - if [ -d "/dockervolumes/$name" ]; then - wd="/dockervolumes/$name" + if [ -d "$SOURCE_BASE_DIR/$name" ]; then + wd="$SOURCE_BASE_DIR/$name" elif [ -d "$HOME/prj/docker/$name" ]; then wd="$HOME/prj/docker/$name" else @@ -335,6 +343,7 @@ parse_cmdline "$@" debug "-----" debug "logfile = '$logfile'" +debug "source_base_dir = '$SOURCE_BASE_DIR'" toolcheck docker jq sed tar diff --git a/dockerbackup_prune.sh b/dockerbackup_prune.sh old mode 100755 new mode 100644 index 4d1ae32..4eab700 --- a/dockerbackup_prune.sh +++ b/dockerbackup_prune.sh @@ -4,12 +4,25 @@ set -Eeuo pipefail ############################################################################### # dockerbackup_prune.sh # -# A) Create a dated .zst backup of /dockerbackups (tar stream | zstd, output .zst) -# - Clean single-line progress with percent and current dir/file (requires pv) -# - Fallback: single-line [N/TOTAL] without percent if pv is unavailable -# B) Upload via rclone -# C) Prune *.zst under PRUNE_ROOT subfolders, keeping newest N per subfolder -# D) Delete the local .zst (only with --apply) +# Default: +# A) Create a dated .zst backup locally and keep it +# +# With --apply: +# A) Create a dated .zst backup +# C) Prune *.zst under PRUNE_ROOT subfolders, keeping newest N per subfolder +# D) Keep the new local archive +# +# With --rclone: +# A) Create a dated .zst backup +# B) Upload via rclone +# C) Show what prune would do (unless --apply is also set) +# D) Keep the new local archive unless --apply is also set +# +# With --rclone --apply: +# A) Create a dated .zst backup +# B) Upload via rclone +# C) Prune for real +# D) Delete the new local archive # # Restore: # zstd -d -c /path/backup.zst | tar -x -C /restore/target @@ -34,7 +47,6 @@ ZSTD_EXTRA_OPTS="" SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" ENV_FILE="${DOCKERBACKUP_ENV:-$SCRIPT_DIR/.env}" if [[ -f "$ENV_FILE" ]]; then - # Warn if perms are too open; still load to keep it convenient perms="$(stat -c '%a' "$ENV_FILE" 2>/dev/null || stat -f '%Lp' "$ENV_FILE" 2>/dev/null || echo '')" if [[ -n "$perms" && ! "$perms" =~ ^(600|400)$ ]]; then echo "$(date '+%F %T') [WARN] $ENV_FILE permissions are $perms; recommend 600." >&2 @@ -49,15 +61,30 @@ fi APPLY=0 QUIET=0 DRYRUN=0 +USE_RCLONE=0 usage() { cat <&2; } run() { if [[ $DRYRUN -eq 1 ]]; then log "DRY" "$*"; else "$@"; fi; } -# Truncate to terminal width and erase tail so old text is cleared truncate_and_print() { local s="$1" cols if [[ -t 2 ]]; then @@ -102,8 +129,8 @@ truncate_and_print() { else cols=120 fi - (( cols<20 )) && cols=120 - if (( ${#s} > cols-1 )); then + (( cols < 20 )) && cols=120 + if (( ${#s} > cols - 1 )); then s="${s:0:cols-1}" fi printf '\r%s\033[K' "$s" >&2 @@ -118,7 +145,7 @@ status_line_shown=0 cleanup() { local why="${1:-EXIT}" - # Kill the whole pipeline (tar|pv|zstd) + if [[ -n "$pipe_pid" ]]; then if command -v ps >/dev/null 2>&1; then pipe_pgid=$(ps -o pgid= "$pipe_pid" 2>/dev/null | tr -d ' ' || true) @@ -134,19 +161,19 @@ cleanup() { fi wait "$pipe_pid" 2>/dev/null || true fi - # Close FDs + if (( pct_fd_open == 1 )); then exec 4>&- 4<&-; pct_fd_open=0; fi if (( path_fd_open == 1 )); then exec 3>&- 3<&-; path_fd_open=0; fi - # Remove fifos/dir + if [[ -n "$cleanup_tmp" ]]; then rm -f "$cleanup_tmp/pct" "$cleanup_tmp/path" 2>/dev/null || true rmdir "$cleanup_tmp" 2>/dev/null || true fi - # End the progress line cleanly + if (( status_line_shown == 1 )) && [[ $QUIET -eq 0 ]]; then printf '\n' >&2 fi - # Exit with 130 on INT/TERM + case "$why" in INT|TERM) exit 130 ;; esac @@ -162,10 +189,12 @@ trap 'cleanup EXIT' EXIT [[ -d "$PRUNE_ROOT" ]] || { log_err "Prune root not found: $PRUNE_ROOT"; exit 1; } command -v tar >/dev/null 2>&1 || { log_err "tar not found"; exit 1; } command -v zstd >/dev/null 2>&1 || { log_err "zstd not found"; exit 1; } -command -v rclone >/dev/null 2>&1 || { log_err "rclone not found"; exit 1; } command -v find >/dev/null 2>&1 || { log_err "find not found"; exit 1; } -# Timestamp in name: YYYY-MM-DD_HH-MM (system timezone or override externally) +if (( USE_RCLONE == 1 )) && (( DRYRUN == 0 )); then + command -v rclone >/dev/null 2>&1 || { log_err "rclone not found"; exit 1; } +fi + STAMP="${STAMP_OVERRIDE:-$(date +%F_%H-%M)}" BACKUP_NAME="${BACKUP_PREFIX}-${STAMP}.zst" BACKUP_PATH="${OUT_DIR%/}/${BACKUP_NAME}" @@ -173,15 +202,17 @@ BACKUP_PATH="${OUT_DIR%/}/${BACKUP_NAME}" RCLONE_FLAGS=() [[ $QUIET -eq 1 ]] && RCLONE_FLAGS+=("-q") || RCLONE_FLAGS+=("--progress") -# Silence zstd "Read: ... ==> 100%" ZSTD_FLAGS=("-q") [[ -n "$ZSTD_EXTRA_OPTS" ]] && ZSTD_FLAGS+=($ZSTD_EXTRA_OPTS) log "INFO" "Mode: $( [[ $DRYRUN -eq 1 ]] && echo 'DRY-RUN ALL' || ([[ $APPLY -eq 1 ]] && echo 'APPLY DELETES' || echo 'NO DELETES') )" +log "INFO" "Rclone mode: $( [[ $USE_RCLONE -eq 1 ]] && echo 'ENABLED' || echo 'DISABLED' )" log "INFO" "Creating archive: $BACKUP_PATH (.zst)" log "INFO" "Keeping newest $KEEP_COUNT file(s) per subfolder matching: $PRUNE_PATTERN" log "INFO" "Source: $SOURCE_DIR" -log "INFO" "Remote: $RCLONE_REMOTE" +if (( USE_RCLONE == 1 )); then + log "INFO" "Remote: $RCLONE_REMOTE" +fi log "INFO" "Prune root (root itself will be skipped): $PRUNE_ROOT" # --------------------------- Step A: create .zst with one-liner -------------- @@ -204,11 +235,9 @@ else mkfifo "$pct_fifo"; have_pv=1 fi - # Open FIFOs RDWR so writers/readers never block exec 3<> "$path_fifo"; path_fd_open=1 if (( have_pv == 1 )); then exec 4<> "$pct_fifo"; pct_fd_open=1; fi - # Run pipeline in background; send tar -v names into the path FIFO if (( have_pv == 1 )); then ( set -o pipefail @@ -227,7 +256,6 @@ else fi pipe_pid=$! - # Status loop: single clean line on stderr files_seen=0 last_path="" last_pct="0.0" @@ -235,23 +263,20 @@ else last_print_time=0 while kill -0 "$pipe_pid" 2>/dev/null; do - # read percent from pv (if any) if (( have_pv == 1 )); then if IFS= read -r -t 0.03 pct_line <&4; then last_pct="$pct_line"; fi fi - # read latest path from tar + if IFS= read -r -t 0.03 path_line <&3; then last_path="$path_line" [[ -n "$last_path" ]] && (( files_seen++ )) || true fi - # Split into dir/file for display (already relative) display_rel="$last_path" dir_part="$(dirname -- "$display_rel" 2>/dev/null || echo .)" base_part="$(basename -- "$display_rel" 2>/dev/null || echo .)" [[ "$dir_part" == "." ]] && dir_part="." - # Refresh line and erase tail (\033[K) if [[ $QUIET -eq 0 ]]; then now_ns=$(date +%s%N) if (( now_ns - last_print_time > 50000000 )); then @@ -267,7 +292,6 @@ else fi done - # Drain, finish, cleanup while IFS= read -r -t 0.03 path_line <&3; do last_path="$path_line"; (( files_seen++ )) || true [[ $QUIET -eq 0 ]] && truncate_and_print "[${files_seen}/${TOTAL_FILES}] archiving: $(dirname -- "$last_path")/ $(basename -- "$last_path")" @@ -280,7 +304,6 @@ else status_line_shown=0 fi - # Close and clean up exec 3>&- 3<&-; path_fd_open=0 if (( have_pv == 1 )); then exec 4>&- 4<&-; pct_fd_open=0; fi rm -f "$path_fifo" "$pct_fifo" 2>/dev/null || true @@ -289,76 +312,88 @@ else fi log "OK" "Archive step completed" -# --------------------------- Step B: rclone copy ----------------------------- -log "STEP" "B) rclone copy to remote" -if [[ $DRYRUN -eq 1 ]]; then - log "DRY" "rclone copy \"$BACKUP_PATH\" \"$RCLONE_REMOTE\" ${RCLONE_FLAGS[*]}" -else - run rclone copy "$BACKUP_PATH" "$RCLONE_REMOTE" "${RCLONE_FLAGS[@]}" -fi -log "OK" "Upload step completed" - -# --------------------------- Step C: prune .zst ------------------------------ -log "STEP" "C) Pruning $PRUNE_PATTERN under $PRUNE_ROOT (keep $KEEP_COUNT)" -export LC_ALL=C -prune_ok=1 - -while IFS= read -r -d '' dir; do - mapfile -d '' -t files < <( - find "$dir" -maxdepth 1 -type f -name "$PRUNE_PATTERN" -printf '%T@ %p\0' | - sort -z -n | - cut -z -d' ' -f2- - ) - count=${#files[@]} - if (( count == 0 )); then - log "SKIP" "No matches in: $dir" - continue - fi - - keep_from=$(( count - KEEP_COUNT )) - (( keep_from < 0 )) && keep_from=0 - - if (( count <= KEEP_COUNT )); then - log "KEEP" "At or below limit in: $dir (count=$count)" - continue - fi - - keep_list=("${files[@]:keep_from:KEEP_COUNT}") - del_list=("${files[@]:0:keep_from}") - - for k in "${keep_list[@]}"; do - log "KEEP" "Keep: $k" - done - for f in "${del_list[@]}"; do - if [[ $DRYRUN -eq 1 || $APPLY -eq 0 ]]; then - log "WOULD" "Remove: $f" - else - if rm -f -- "$f"; then - log "DELETE" "Removed: $f" - else - log_err "Failed to remove: $f"; prune_ok=0 - fi - fi - done -done < <(find "$PRUNE_ROOT" -mindepth 1 -type d -print0) - -if [[ $prune_ok -ne 1 ]]; then - log_err "Prune step encountered errors; aborting final delete" - exit 1 -fi -log "OK" "Prune step completed" - -# --------------------------- Step D: remove local .zst ----------------------- -log "STEP" "D) Removing local backup" -if [[ $DRYRUN -eq 1 || $APPLY -eq 0 ]]; then - log "WOULD" "Remove local backup: $BACKUP_PATH" -else - if rm -f -- "$BACKUP_PATH"; then - log "DELETE" "Removed local backup: $BACKUP_PATH" +# --------------------------- Step B: optional rclone copy -------------------- +if (( USE_RCLONE == 1 )); then + log "STEP" "B) rclone copy to remote" + if [[ $DRYRUN -eq 1 ]]; then + log "DRY" "rclone copy \"$BACKUP_PATH\" \"$RCLONE_REMOTE\" ${RCLONE_FLAGS[*]}" else - log_err "Failed to remove local backup: $BACKUP_PATH" + run rclone copy "$BACKUP_PATH" "$RCLONE_REMOTE" "${RCLONE_FLAGS[@]}" + fi + log "OK" "Upload step completed" +else + log "SKIP" "--rclone not set; skipping upload" +fi + +# --------------------------- Step C: prune if apply or rclone --------------- +if (( APPLY == 1 || USE_RCLONE == 1 )); then + log "STEP" "C) Pruning $PRUNE_PATTERN under $PRUNE_ROOT (keep $KEEP_COUNT)" + export LC_ALL=C + prune_ok=1 + + while IFS= read -r -d '' dir; do + mapfile -d '' -t files < <( + find "$dir" -maxdepth 1 -type f -name "$PRUNE_PATTERN" -printf '%T@ %p\0' | + sort -z -n | + cut -z -d' ' -f2- + ) + count=${#files[@]} + if (( count == 0 )); then + log "SKIP" "No matches in: $dir" + continue + fi + + keep_from=$(( count - KEEP_COUNT )) + (( keep_from < 0 )) && keep_from=0 + + if (( count <= KEEP_COUNT )); then + log "KEEP" "At or below limit in: $dir (count=$count)" + continue + fi + + keep_list=("${files[@]:keep_from:KEEP_COUNT}") + del_list=("${files[@]:0:keep_from}") + + for k in "${keep_list[@]}"; do + log "KEEP" "Keep: $k" + done + for f in "${del_list[@]}"; do + if [[ $DRYRUN -eq 1 || $APPLY -eq 0 ]]; then + log "WOULD" "Remove: $f" + else + if rm -f -- "$f"; then + log "DELETE" "Removed: $f" + else + log_err "Failed to remove: $f"; prune_ok=0 + fi + fi + done + done < <(find "$PRUNE_ROOT" -mindepth 1 -type d -print0) + + if [[ $prune_ok -ne 1 ]]; then + log_err "Prune step encountered errors; aborting final delete" exit 1 fi + log "OK" "Prune step completed" +else + log "SKIP" "--apply and --rclone not set; skipping prune" +fi + +# --------------------------- Step D: remove local .zst ----------------------- +if (( USE_RCLONE == 1 )); then + log "STEP" "D) Removing local backup" + if [[ $DRYRUN -eq 1 || $APPLY -eq 0 ]]; then + log "WOULD" "Remove local backup: $BACKUP_PATH" + else + if rm -f -- "$BACKUP_PATH"; then + log "DELETE" "Removed local backup: $BACKUP_PATH" + else + log_err "Failed to remove local backup: $BACKUP_PATH" + exit 1 + fi + fi +else + log "KEEP" "Local archive retained: $BACKUP_PATH" fi log "DONE" "All steps completed successfully"