script tidyups
This commit is contained in:
Executable → Regular
+133
-98
@@ -4,12 +4,25 @@ set -Eeuo pipefail
|
||||
###############################################################################
|
||||
# dockerbackup_prune.sh
|
||||
#
|
||||
# A) Create a dated .zst backup of /dockerbackups (tar stream | zstd, output .zst)
|
||||
# - Clean single-line progress with percent and current dir/file (requires pv)
|
||||
# - Fallback: single-line [N/TOTAL] without percent if pv is unavailable
|
||||
# B) Upload via rclone
|
||||
# C) Prune *.zst under PRUNE_ROOT subfolders, keeping newest N per subfolder
|
||||
# D) Delete the local .zst (only with --apply)
|
||||
# Default:
|
||||
# A) Create a dated .zst backup locally and keep it
|
||||
#
|
||||
# With --apply:
|
||||
# A) Create a dated .zst backup
|
||||
# C) Prune *.zst under PRUNE_ROOT subfolders, keeping newest N per subfolder
|
||||
# D) Keep the new local archive
|
||||
#
|
||||
# With --rclone:
|
||||
# A) Create a dated .zst backup
|
||||
# B) Upload via rclone
|
||||
# C) Show what prune would do (unless --apply is also set)
|
||||
# D) Keep the new local archive unless --apply is also set
|
||||
#
|
||||
# With --rclone --apply:
|
||||
# A) Create a dated .zst backup
|
||||
# B) Upload via rclone
|
||||
# C) Prune for real
|
||||
# D) Delete the new local archive
|
||||
#
|
||||
# Restore:
|
||||
# zstd -d -c /path/backup.zst | tar -x -C /restore/target
|
||||
@@ -34,7 +47,6 @@ ZSTD_EXTRA_OPTS=""
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
ENV_FILE="${DOCKERBACKUP_ENV:-$SCRIPT_DIR/.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
# Warn if perms are too open; still load to keep it convenient
|
||||
perms="$(stat -c '%a' "$ENV_FILE" 2>/dev/null || stat -f '%Lp' "$ENV_FILE" 2>/dev/null || echo '')"
|
||||
if [[ -n "$perms" && ! "$perms" =~ ^(600|400)$ ]]; then
|
||||
echo "$(date '+%F %T') [WARN] $ENV_FILE permissions are $perms; recommend 600." >&2
|
||||
@@ -49,15 +61,30 @@ fi
|
||||
APPLY=0
|
||||
QUIET=0
|
||||
DRYRUN=0
|
||||
USE_RCLONE=0
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $0 [--apply] [--quiet] [--dry-run] [--keep N] [--root PATH] [--source PATH] [--outdir PATH] [--remote PATH]
|
||||
Usage: $0 [--rclone] [--apply] [--quiet] [--dry-run] [--keep N] [--root PATH] [--source PATH] [--outdir PATH] [--remote PATH]
|
||||
|
||||
Default: creates and uploads backup, shows what WOULD be deleted (no deletes).
|
||||
Default:
|
||||
Creates the local .zst archive and keeps it locally.
|
||||
No rclone upload. No prune.
|
||||
|
||||
With --apply:
|
||||
Creates the local .zst archive, prunes old .zst files for real, and keeps the new archive locally.
|
||||
|
||||
With --rclone:
|
||||
Uploads the archive via rclone.
|
||||
Prune also runs, but only as a dry-run unless --apply is also set.
|
||||
The new local archive is kept unless --apply is also set.
|
||||
|
||||
With --rclone --apply:
|
||||
Uploads the archive, prunes for real, and deletes the new local archive.
|
||||
|
||||
Flags:
|
||||
--apply Actually delete pruned files and the local .zst
|
||||
--rclone Enable rclone upload flow
|
||||
--apply Actually delete pruned files; with --rclone also delete the new local archive
|
||||
--quiet Minimal console output (no live progress)
|
||||
--dry-run Simulate everything (no backup, no upload, no deletes)
|
||||
--keep N Keep newest N .zst files per subfolder (default ${KEEP_COUNT})
|
||||
@@ -74,6 +101,7 @@ EOF
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--rclone) USE_RCLONE=1 ;;
|
||||
--apply) APPLY=1 ;;
|
||||
--quiet) QUIET=1 ;;
|
||||
--dry-run) DRYRUN=1 ;;
|
||||
@@ -94,7 +122,6 @@ log() { [[ $QUIET -eq 0 ]] && echo "$(ts) [$1] $2"; }
|
||||
log_err() { echo "$(ts) [ERROR] $1" >&2; }
|
||||
run() { if [[ $DRYRUN -eq 1 ]]; then log "DRY" "$*"; else "$@"; fi; }
|
||||
|
||||
# Truncate to terminal width and erase tail so old text is cleared
|
||||
truncate_and_print() {
|
||||
local s="$1" cols
|
||||
if [[ -t 2 ]]; then
|
||||
@@ -102,8 +129,8 @@ truncate_and_print() {
|
||||
else
|
||||
cols=120
|
||||
fi
|
||||
(( cols<20 )) && cols=120
|
||||
if (( ${#s} > cols-1 )); then
|
||||
(( cols < 20 )) && cols=120
|
||||
if (( ${#s} > cols - 1 )); then
|
||||
s="${s:0:cols-1}"
|
||||
fi
|
||||
printf '\r%s\033[K' "$s" >&2
|
||||
@@ -118,7 +145,7 @@ status_line_shown=0
|
||||
|
||||
cleanup() {
|
||||
local why="${1:-EXIT}"
|
||||
# Kill the whole pipeline (tar|pv|zstd)
|
||||
|
||||
if [[ -n "$pipe_pid" ]]; then
|
||||
if command -v ps >/dev/null 2>&1; then
|
||||
pipe_pgid=$(ps -o pgid= "$pipe_pid" 2>/dev/null | tr -d ' ' || true)
|
||||
@@ -134,19 +161,19 @@ cleanup() {
|
||||
fi
|
||||
wait "$pipe_pid" 2>/dev/null || true
|
||||
fi
|
||||
# Close FDs
|
||||
|
||||
if (( pct_fd_open == 1 )); then exec 4>&- 4<&-; pct_fd_open=0; fi
|
||||
if (( path_fd_open == 1 )); then exec 3>&- 3<&-; path_fd_open=0; fi
|
||||
# Remove fifos/dir
|
||||
|
||||
if [[ -n "$cleanup_tmp" ]]; then
|
||||
rm -f "$cleanup_tmp/pct" "$cleanup_tmp/path" 2>/dev/null || true
|
||||
rmdir "$cleanup_tmp" 2>/dev/null || true
|
||||
fi
|
||||
# End the progress line cleanly
|
||||
|
||||
if (( status_line_shown == 1 )) && [[ $QUIET -eq 0 ]]; then
|
||||
printf '\n' >&2
|
||||
fi
|
||||
# Exit with 130 on INT/TERM
|
||||
|
||||
case "$why" in
|
||||
INT|TERM) exit 130 ;;
|
||||
esac
|
||||
@@ -162,10 +189,12 @@ trap 'cleanup EXIT' EXIT
|
||||
[[ -d "$PRUNE_ROOT" ]] || { log_err "Prune root not found: $PRUNE_ROOT"; exit 1; }
|
||||
command -v tar >/dev/null 2>&1 || { log_err "tar not found"; exit 1; }
|
||||
command -v zstd >/dev/null 2>&1 || { log_err "zstd not found"; exit 1; }
|
||||
command -v rclone >/dev/null 2>&1 || { log_err "rclone not found"; exit 1; }
|
||||
command -v find >/dev/null 2>&1 || { log_err "find not found"; exit 1; }
|
||||
|
||||
# Timestamp in name: YYYY-MM-DD_HH-MM (system timezone or override externally)
|
||||
if (( USE_RCLONE == 1 )) && (( DRYRUN == 0 )); then
|
||||
command -v rclone >/dev/null 2>&1 || { log_err "rclone not found"; exit 1; }
|
||||
fi
|
||||
|
||||
STAMP="${STAMP_OVERRIDE:-$(date +%F_%H-%M)}"
|
||||
BACKUP_NAME="${BACKUP_PREFIX}-${STAMP}.zst"
|
||||
BACKUP_PATH="${OUT_DIR%/}/${BACKUP_NAME}"
|
||||
@@ -173,15 +202,17 @@ BACKUP_PATH="${OUT_DIR%/}/${BACKUP_NAME}"
|
||||
RCLONE_FLAGS=()
|
||||
[[ $QUIET -eq 1 ]] && RCLONE_FLAGS+=("-q") || RCLONE_FLAGS+=("--progress")
|
||||
|
||||
# Silence zstd "Read: ... ==> 100%"
|
||||
ZSTD_FLAGS=("-q")
|
||||
[[ -n "$ZSTD_EXTRA_OPTS" ]] && ZSTD_FLAGS+=($ZSTD_EXTRA_OPTS)
|
||||
|
||||
log "INFO" "Mode: $( [[ $DRYRUN -eq 1 ]] && echo 'DRY-RUN ALL' || ([[ $APPLY -eq 1 ]] && echo 'APPLY DELETES' || echo 'NO DELETES') )"
|
||||
log "INFO" "Rclone mode: $( [[ $USE_RCLONE -eq 1 ]] && echo 'ENABLED' || echo 'DISABLED' )"
|
||||
log "INFO" "Creating archive: $BACKUP_PATH (.zst)"
|
||||
log "INFO" "Keeping newest $KEEP_COUNT file(s) per subfolder matching: $PRUNE_PATTERN"
|
||||
log "INFO" "Source: $SOURCE_DIR"
|
||||
log "INFO" "Remote: $RCLONE_REMOTE"
|
||||
if (( USE_RCLONE == 1 )); then
|
||||
log "INFO" "Remote: $RCLONE_REMOTE"
|
||||
fi
|
||||
log "INFO" "Prune root (root itself will be skipped): $PRUNE_ROOT"
|
||||
|
||||
# --------------------------- Step A: create .zst with one-liner --------------
|
||||
@@ -204,11 +235,9 @@ else
|
||||
mkfifo "$pct_fifo"; have_pv=1
|
||||
fi
|
||||
|
||||
# Open FIFOs RDWR so writers/readers never block
|
||||
exec 3<> "$path_fifo"; path_fd_open=1
|
||||
if (( have_pv == 1 )); then exec 4<> "$pct_fifo"; pct_fd_open=1; fi
|
||||
|
||||
# Run pipeline in background; send tar -v names into the path FIFO
|
||||
if (( have_pv == 1 )); then
|
||||
(
|
||||
set -o pipefail
|
||||
@@ -227,7 +256,6 @@ else
|
||||
fi
|
||||
pipe_pid=$!
|
||||
|
||||
# Status loop: single clean line on stderr
|
||||
files_seen=0
|
||||
last_path=""
|
||||
last_pct="0.0"
|
||||
@@ -235,23 +263,20 @@ else
|
||||
last_print_time=0
|
||||
|
||||
while kill -0 "$pipe_pid" 2>/dev/null; do
|
||||
# read percent from pv (if any)
|
||||
if (( have_pv == 1 )); then
|
||||
if IFS= read -r -t 0.03 pct_line <&4; then last_pct="$pct_line"; fi
|
||||
fi
|
||||
# read latest path from tar
|
||||
|
||||
if IFS= read -r -t 0.03 path_line <&3; then
|
||||
last_path="$path_line"
|
||||
[[ -n "$last_path" ]] && (( files_seen++ )) || true
|
||||
fi
|
||||
|
||||
# Split into dir/file for display (already relative)
|
||||
display_rel="$last_path"
|
||||
dir_part="$(dirname -- "$display_rel" 2>/dev/null || echo .)"
|
||||
base_part="$(basename -- "$display_rel" 2>/dev/null || echo .)"
|
||||
[[ "$dir_part" == "." ]] && dir_part="."
|
||||
|
||||
# Refresh line and erase tail (\033[K)
|
||||
if [[ $QUIET -eq 0 ]]; then
|
||||
now_ns=$(date +%s%N)
|
||||
if (( now_ns - last_print_time > 50000000 )); then
|
||||
@@ -267,7 +292,6 @@ else
|
||||
fi
|
||||
done
|
||||
|
||||
# Drain, finish, cleanup
|
||||
while IFS= read -r -t 0.03 path_line <&3; do
|
||||
last_path="$path_line"; (( files_seen++ )) || true
|
||||
[[ $QUIET -eq 0 ]] && truncate_and_print "[${files_seen}/${TOTAL_FILES}] archiving: $(dirname -- "$last_path")/ $(basename -- "$last_path")"
|
||||
@@ -280,7 +304,6 @@ else
|
||||
status_line_shown=0
|
||||
fi
|
||||
|
||||
# Close and clean up
|
||||
exec 3>&- 3<&-; path_fd_open=0
|
||||
if (( have_pv == 1 )); then exec 4>&- 4<&-; pct_fd_open=0; fi
|
||||
rm -f "$path_fifo" "$pct_fifo" 2>/dev/null || true
|
||||
@@ -289,76 +312,88 @@ else
|
||||
fi
|
||||
log "OK" "Archive step completed"
|
||||
|
||||
# --------------------------- Step B: rclone copy -----------------------------
|
||||
log "STEP" "B) rclone copy to remote"
|
||||
if [[ $DRYRUN -eq 1 ]]; then
|
||||
log "DRY" "rclone copy \"$BACKUP_PATH\" \"$RCLONE_REMOTE\" ${RCLONE_FLAGS[*]}"
|
||||
else
|
||||
run rclone copy "$BACKUP_PATH" "$RCLONE_REMOTE" "${RCLONE_FLAGS[@]}"
|
||||
fi
|
||||
log "OK" "Upload step completed"
|
||||
|
||||
# --------------------------- Step C: prune .zst ------------------------------
|
||||
log "STEP" "C) Pruning $PRUNE_PATTERN under $PRUNE_ROOT (keep $KEEP_COUNT)"
|
||||
export LC_ALL=C
|
||||
prune_ok=1
|
||||
|
||||
while IFS= read -r -d '' dir; do
|
||||
mapfile -d '' -t files < <(
|
||||
find "$dir" -maxdepth 1 -type f -name "$PRUNE_PATTERN" -printf '%T@ %p\0' |
|
||||
sort -z -n |
|
||||
cut -z -d' ' -f2-
|
||||
)
|
||||
count=${#files[@]}
|
||||
if (( count == 0 )); then
|
||||
log "SKIP" "No matches in: $dir"
|
||||
continue
|
||||
fi
|
||||
|
||||
keep_from=$(( count - KEEP_COUNT ))
|
||||
(( keep_from < 0 )) && keep_from=0
|
||||
|
||||
if (( count <= KEEP_COUNT )); then
|
||||
log "KEEP" "At or below limit in: $dir (count=$count)"
|
||||
continue
|
||||
fi
|
||||
|
||||
keep_list=("${files[@]:keep_from:KEEP_COUNT}")
|
||||
del_list=("${files[@]:0:keep_from}")
|
||||
|
||||
for k in "${keep_list[@]}"; do
|
||||
log "KEEP" "Keep: $k"
|
||||
done
|
||||
for f in "${del_list[@]}"; do
|
||||
if [[ $DRYRUN -eq 1 || $APPLY -eq 0 ]]; then
|
||||
log "WOULD" "Remove: $f"
|
||||
else
|
||||
if rm -f -- "$f"; then
|
||||
log "DELETE" "Removed: $f"
|
||||
else
|
||||
log_err "Failed to remove: $f"; prune_ok=0
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done < <(find "$PRUNE_ROOT" -mindepth 1 -type d -print0)
|
||||
|
||||
if [[ $prune_ok -ne 1 ]]; then
|
||||
log_err "Prune step encountered errors; aborting final delete"
|
||||
exit 1
|
||||
fi
|
||||
log "OK" "Prune step completed"
|
||||
|
||||
# --------------------------- Step D: remove local .zst -----------------------
|
||||
log "STEP" "D) Removing local backup"
|
||||
if [[ $DRYRUN -eq 1 || $APPLY -eq 0 ]]; then
|
||||
log "WOULD" "Remove local backup: $BACKUP_PATH"
|
||||
else
|
||||
if rm -f -- "$BACKUP_PATH"; then
|
||||
log "DELETE" "Removed local backup: $BACKUP_PATH"
|
||||
# --------------------------- Step B: optional rclone copy --------------------
|
||||
if (( USE_RCLONE == 1 )); then
|
||||
log "STEP" "B) rclone copy to remote"
|
||||
if [[ $DRYRUN -eq 1 ]]; then
|
||||
log "DRY" "rclone copy \"$BACKUP_PATH\" \"$RCLONE_REMOTE\" ${RCLONE_FLAGS[*]}"
|
||||
else
|
||||
log_err "Failed to remove local backup: $BACKUP_PATH"
|
||||
run rclone copy "$BACKUP_PATH" "$RCLONE_REMOTE" "${RCLONE_FLAGS[@]}"
|
||||
fi
|
||||
log "OK" "Upload step completed"
|
||||
else
|
||||
log "SKIP" "--rclone not set; skipping upload"
|
||||
fi
|
||||
|
||||
# --------------------------- Step C: prune if apply or rclone ---------------
|
||||
if (( APPLY == 1 || USE_RCLONE == 1 )); then
|
||||
log "STEP" "C) Pruning $PRUNE_PATTERN under $PRUNE_ROOT (keep $KEEP_COUNT)"
|
||||
export LC_ALL=C
|
||||
prune_ok=1
|
||||
|
||||
while IFS= read -r -d '' dir; do
|
||||
mapfile -d '' -t files < <(
|
||||
find "$dir" -maxdepth 1 -type f -name "$PRUNE_PATTERN" -printf '%T@ %p\0' |
|
||||
sort -z -n |
|
||||
cut -z -d' ' -f2-
|
||||
)
|
||||
count=${#files[@]}
|
||||
if (( count == 0 )); then
|
||||
log "SKIP" "No matches in: $dir"
|
||||
continue
|
||||
fi
|
||||
|
||||
keep_from=$(( count - KEEP_COUNT ))
|
||||
(( keep_from < 0 )) && keep_from=0
|
||||
|
||||
if (( count <= KEEP_COUNT )); then
|
||||
log "KEEP" "At or below limit in: $dir (count=$count)"
|
||||
continue
|
||||
fi
|
||||
|
||||
keep_list=("${files[@]:keep_from:KEEP_COUNT}")
|
||||
del_list=("${files[@]:0:keep_from}")
|
||||
|
||||
for k in "${keep_list[@]}"; do
|
||||
log "KEEP" "Keep: $k"
|
||||
done
|
||||
for f in "${del_list[@]}"; do
|
||||
if [[ $DRYRUN -eq 1 || $APPLY -eq 0 ]]; then
|
||||
log "WOULD" "Remove: $f"
|
||||
else
|
||||
if rm -f -- "$f"; then
|
||||
log "DELETE" "Removed: $f"
|
||||
else
|
||||
log_err "Failed to remove: $f"; prune_ok=0
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done < <(find "$PRUNE_ROOT" -mindepth 1 -type d -print0)
|
||||
|
||||
if [[ $prune_ok -ne 1 ]]; then
|
||||
log_err "Prune step encountered errors; aborting final delete"
|
||||
exit 1
|
||||
fi
|
||||
log "OK" "Prune step completed"
|
||||
else
|
||||
log "SKIP" "--apply and --rclone not set; skipping prune"
|
||||
fi
|
||||
|
||||
# --------------------------- Step D: remove local .zst -----------------------
|
||||
if (( USE_RCLONE == 1 )); then
|
||||
log "STEP" "D) Removing local backup"
|
||||
if [[ $DRYRUN -eq 1 || $APPLY -eq 0 ]]; then
|
||||
log "WOULD" "Remove local backup: $BACKUP_PATH"
|
||||
else
|
||||
if rm -f -- "$BACKUP_PATH"; then
|
||||
log "DELETE" "Removed local backup: $BACKUP_PATH"
|
||||
else
|
||||
log_err "Failed to remove local backup: $BACKUP_PATH"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
log "KEEP" "Local archive retained: $BACKUP_PATH"
|
||||
fi
|
||||
|
||||
log "DONE" "All steps completed successfully"
|
||||
|
||||
Reference in New Issue
Block a user