From 261e88f5dc04bb9a2f18a8039be87c09d54fd319 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 19 Sep 2026 23:46:36 -0400 Subject: [PATCH] Refactor Docker backups into modular pipeline --- .env.example | 31 +++- .gitignore | 1 + README.md | 209 ++++++++++++++------- backup.sh | 32 ++++ containers.txt | 1 - docker-backup.sh | 379 +------------------------------------- dockerbackup_prune.sh | 398 +--------------------------------------- lib/common.sh | 62 +++++++ methods/20-rclone.sh | 24 +++ methods/30-smb.sh | 29 +++ stages/10-containers.sh | 137 ++++++++++++++ stages/90-prune.sh | 25 +++ 12 files changed, 480 insertions(+), 848 deletions(-) mode change 100755 => 100644 .env.example create mode 100755 backup.sh mode change 100644 => 100755 docker-backup.sh mode change 100644 => 100755 dockerbackup_prune.sh create mode 100644 lib/common.sh create mode 100755 methods/20-rclone.sh create mode 100755 methods/30-smb.sh create mode 100755 stages/10-containers.sh create mode 100755 stages/90-prune.sh diff --git a/.env.example b/.env.example old mode 100755 new mode 100644 index 201e298..e424b4a --- a/.env.example +++ b/.env.example @@ -1,7 +1,26 @@ -# /etc/dockerbackup/.env -RCLONE_REMOTE="gdrive:backuplocation/sublocation" -SOURCE_DIR="/mybackups" -PRUNE_ROOT="/mybackups" -OUT_DIR="/tmp" +# Copy to .env and customise on each Docker server. Keep .env out of Git. +SOURCE_BASE_DIR="/dockervolumes" +BACKUP_ROOT="/dockerbackups" +CONTAINERS_FILE="containers.local.txt" +BACKUP_HOSTNAME="beaver" + +# Main pipeline method order. Add future method names here. +BACKUP_METHODS="rclone smb" KEEP_COUNT=5 -ZSTD_EXTRA_OPTS="-T0 -12" +PULL_IMAGES=1 +START_TIMEOUT=90 +WAIT_INTERVAL=3 +START_RETRIES=1 +LOCK_FILE="/run/lock/zorruno-docker-backup.lock" + +# Rclone destination becomes /beaver/. +RCLONE_ENABLED=0 +RCLONE_REMOTE="gdrive:dockerbackups" +RCLONE_EXTRA_ARGS="" + +# The SMB share must already be mounted by the operating system. +# Destination becomes /beaver/ when SMB_HOST_SUBDIR=1. +SMB_ENABLED=0 +SMB_MOUNT="/mnt/dockerbackups" +SMB_HOST_SUBDIR=1 +SMB_RSYNC_ARGS="-a" diff --git a/.gitignore b/.gitignore index 4c49bd7..e63274a 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .env +containers.local.txt diff --git a/README.md b/README.md index 35894cf..d0fcf8f 100644 --- a/README.md +++ b/README.md @@ -1,83 +1,154 @@ -# Docker Backup Scripts +# Zorruno Docker Backup Scripts -Simple shell scripts for backing up Docker-related data and pruning old backups. +Modular backup scripts for Docker Compose servers such as Panda and Beaver. +Each server has its own untracked `.env` and `containers.txt`, while all script +logic remains common and can be updated through Git. -## Files +## Pipeline -- `docker-backup.sh` - create backups -- `dockerbackup_prune.sh` - prune old backups -- `containers.txt` - list of containers or backup targets -- `.env.example` - example environment file +`backup.sh` runs these stages in order: -## Reqirements +1. `stages/10-containers.sh` — stop, archive, pull, start and verify projects. +2. `methods/20-rclone.sh` — copy archives to an rclone destination when enabled. +3. `methods/30-smb.sh` — copy archives to a mounted SMB share when enabled. +4. `stages/90-prune.sh` — retain the newest configured number locally. -docker-compose +Every stage can also be run directly. The old `docker-backup.sh` name invokes +the container archive stage, while `dockerbackup_prune.sh` invokes the local +prune stage. Use the new `backup.sh` for the complete pipeline. -## BACKUPS: Example use docker-backup.sh +## Requirements -Configure the backup script, docker-backup.sh -Set up your list of containers in container.txt . The script assumes each container is in a subdirectory of one main docker directory +- Bash 4 or later +- Docker with `docker compose`, or legacy `docker-compose` +- GNU `tar` with zstd support +- `flock` +- `rclone` only when the rclone method is enabled +- `rsync` and `mountpoint` only when the SMB method is enabled -Edit the location of where you want docker backups stored in docker-backup.sh (default is "/dockerbackups") or use +The account running the scripts must be allowed to control Docker and read all +files under `/dockervolumes`. -./docker-backup.sh -s /home/me/mycontainers +## Installation on Beaver -**Actions:** -- It will recurse though the directories if they are listed in containers.txt -- It will run docker-compose to take down the container, pull a new version if there is one, then bring the container back up. -- it will create a timestamped archive of each container in the backup directory, under a directory with that container name. - -``` -Usage: ./docker-backup.sh [-f containers_list_file] [-s source_base_dir] [-t] [-v] [-h] - -Flags: - -t Test (dry run; no backups performed) - -v Verbose (debug logging to stderr) - -f FILE File containing list of container names - Default: '$containersfile' - -s DIR Base directory containing container/project folders - Default: '$SOURCE_BASE_DIR' - -h Show this help +```bash +sudo git clone https://home.fox.co.nz/gitea/zorruno/zorruno-backupdockerscript.git /backupdockerscripts +cd /backupdockerscripts +sudo cp .env.example .env +sudo cp containers.txt containers.local.txt +sudo chmod 600 .env +sudo chmod +x backup.sh docker-backup.sh dockerbackup_prune.sh stages/*.sh methods/*.sh ``` -## PRUNE and REMOTE BACKUP dockerbackup_prune.sh +Edit `.env`. Beaver's Docker source is already represented by: -Set up the env file using the example. -```cp .env.example .env``` - -You don't need to worry about the rclone info (backups to google drive) it you won't be using that. -It will just backup up the file locally. Feel free to adjust for other remote backup systems. - -If you run .dockerbackup_prune.sh it will prune the number of backup archives that are already there (based on your prune numbers), make a .zst archive of all container backups -then finally uploades to google drive and deletes the large archive (if that's what you set). -Then it will prune the number of archive files for each - -**local archive only (no pruning)** -```./dockerbackup_prune.sh``` - -**local archive + real prune, keep new archive locally. Default prune is 5 archives)** -```./dockerbackup_prune.sh --apply``` - -**local archive + upload + prune dry-run (no actual prune), keep new archive locally (default location is /tmp)** -```./dockerbackup_prune.sh --rclone``` - -**full remote archive, google drive upload + real prune (default 5) + delete new local archive** -```./dockerbackup_prune.sh --rclone --apply``` - -**Full remote workflow with actual prune leaving 3 archives of each container and final full archive delete.** -**Specify location of docker archives, final archive dir and remote google drive folder.** -```./dockerbackup_prune.sh --root /mydockerarchives --/mybigbackupdolder --keep 3 --rclone /dockergooglebackup --apply - -Usage: $0 [--apply] [--quiet] [--dry-run] [--keep N] [--root PATH] [--source PATH] [--outdir PATH] [--remote PATH] - -Flags: - --rclone Enable rclone upload flow - --apply Actually delete pruned files; with --rclone also delete the new local archive - --quiet Minimal console output (no live progress) - --dry-run Simulate everything (no backup, no upload, no deletes) - --keep N Keep newest N .zst files per subfolder (default ${KEEP_COUNT}) - --root PATH Root folder to prune (default ${PRUNE_ROOT}) - --source PATH Folder to archive (default ${SOURCE_DIR}) - --outdir PATH Where to write the .zst (default ${OUT_DIR}) - --remote PATH rclone destination (default ${RCLONE_REMOTE}) +```bash +SOURCE_BASE_DIR="/dockervolumes" +BACKUP_HOSTNAME="beaver" ``` + +Choose and create a local staging path for `BACKUP_ROOT`. `/dockerbackups` is +the default, but it should be on a filesystem with enough free space to hold +at least `KEEP_COUNT` copies of every listed project. + +Edit Beaver's untracked `containers.local.txt`. Each active line can be either a running +container name or a directory name beneath `/dockervolumes`. Blank lines and +text after `#` are ignored. If several entries resolve to the same Compose +project directory, that project is backed up only once. Keeping this host-local +file out of Git prevents a pull on Panda or Beaver from replacing its list. + +## Configuration + +Copy `.env.example` to `.env`; `.env` is ignored by Git. Important values: + +| Variable | Purpose | +|---|---| +| `SOURCE_BASE_DIR` | Parent of Compose project directories | +| `BACKUP_ROOT` | Local archive staging and retention tree | +| `CONTAINERS_FILE` | Absolute path, or path relative to this repository | +| `BACKUP_HOSTNAME` | Host folder/name embedded in archive filenames | +| `BACKUP_METHODS` | Ordered method names, default `rclone smb` | +| `KEEP_COUNT` | Local archives retained per Compose project | +| `PULL_IMAGES` | Pull images before restarting (`1` or `0`) | + +### Rclone + +```bash +RCLONE_ENABLED=1 +RCLONE_REMOTE="gdrive:dockerbackups" +``` + +Archives are copied beneath `//`. The method uses +`rclone copy`, so it does not delete older destination files. + +### SMB + +Mount the SMB share through `/etc/fstab` or a systemd mount unit; do not put SMB +credentials in this repository. Then configure: + +```bash +SMB_ENABLED=1 +SMB_MOUNT="/mnt/dockerbackups" +SMB_HOST_SUBDIR=1 +``` + +The script refuses to copy unless `SMB_MOUNT` is an actual mount point. This +prevents a failed network mount from silently filling the server's root disk. + +## Running + +Run the complete pipeline: + +```bash +sudo /backupdockerscripts/backup.sh +``` + +Preview all stages without stopping containers, copying, or deleting: + +```bash +sudo /backupdockerscripts/backup.sh --dry-run +``` + +Run individual parts: + +```bash +sudo stages/10-containers.sh --dry-run +sudo methods/20-rclone.sh --dry-run --force +sudo methods/30-smb.sh --dry-run --force +sudo stages/90-prune.sh --dry-run +``` + +`--force` runs a transfer method even when its `*_ENABLED` value is false. It +still requires the method's destination to be configured and available. + +## Adding another backup method + +Add an executable script named `methods/NN-name.sh`, for example +`methods/40-restic.sh`. It should load `lib/common.sh`, accept `--dry-run`, and +return a non-zero status on failure. Add its short name to `.env`: + +```bash +BACKUP_METHODS="rclone smb restic" +``` + +The main pipeline discovers the numbered file from the short name and runs it +after the earlier methods. A failed method stops the pipeline, so pruning will +not run after a failed transfer. + +## Archive and restore format + +Archives are stored as: + +```text +//...tar.zst +``` + +Each archive contains the project's top-level directory rather than absolute +paths. Example restore into an empty temporary directory: + +```bash +mkdir -p /tmp/docker-restore +tar --zstd -xf backup.tar.zst -C /tmp/docker-restore +``` + +Inspect and verify restored content before replacing a live project. diff --git a/backup.sh b/backup.sh new file mode 100755 index 0000000..8778884 --- /dev/null +++ b/backup.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Main pipeline: container archives, configured methods, then local prune. +set -Eeuo pipefail +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +. "$SCRIPT_DIR/lib/common.sh" +DRY_RUN=0; SKIP_CONTAINERS=0; SKIP_PRUNE=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run|-t) DRY_RUN=1 ;; --skip-containers) SKIP_CONTAINERS=1 ;; --skip-prune) SKIP_PRUNE=1 ;; + -h|--help) + echo "Usage: ${0##*/} [--dry-run] [--skip-containers] [--skip-prune]" + echo "Methods run in BACKUP_METHODS order. Add methods/NN-name.sh and list name in BACKUP_METHODS." + exit 0 ;; + *) die "Unknown option: $1" 2 ;; + esac + shift +done +require_command flock +mkdir -p "$(dirname "$LOCK_FILE")" +exec 9>"$LOCK_FILE" +flock -n 9 || die "Another backup run holds lock: $LOCK_FILE" +stage_args=(); (( DRY_RUN )) && stage_args+=(--dry-run) +log START "Backup pipeline for $BACKUP_HOSTNAME" +(( SKIP_CONTAINERS )) || "$SCRIPT_DIR/stages/10-containers.sh" "${stage_args[@]}" +for method in $BACKUP_METHODS; do + method_script="" + while IFS= read -r candidate; do method_script="$candidate"; break; done < <(find "$SCRIPT_DIR/methods" -maxdepth 1 -type f -name "*-${method}.sh" -print | sort) + [[ -n "$method_script" ]] || die "Configured backup method not found: $method" + "$method_script" "${stage_args[@]}" +done +(( SKIP_PRUNE )) || "$SCRIPT_DIR/stages/90-prune.sh" "${stage_args[@]}" +log DONE "Backup pipeline completed successfully" diff --git a/containers.txt b/containers.txt index 590eb54..2105b35 100644 --- a/containers.txt +++ b/containers.txt @@ -55,4 +55,3 @@ ttyd #weewx whoami wireguard -zigbee2mqtt diff --git a/docker-backup.sh b/docker-backup.sh old mode 100644 new mode 100755 index f20a730..0b49d7c --- a/docker-backup.sh +++ b/docker-backup.sh @@ -1,378 +1,5 @@ #!/usr/bin/env bash -# ORIGINAL AUTHOR: kmw (updates by zorruno) -# HISTORY: -# 2026-04-20 - add configurable SOURCE_BASE_DIR and -s option for source path fallback. -# 2025-08-22 - add start wait with one retry; fix find_working_dir typo; tidy. -# 2025-08-22 - tidy, stricter bash, compose detection, better checks, clearer logging. -# 2022-03-25 - add -t option for dry-run (no backups performed). -# 2022-01-15 - fix error reporting when a container is not found. -# 2022-01-09 - ignore trailing comments -# 2022-01-03 - display help; list from file; -f option -# 2021-10-16 - structure and logging -# 2020-05-ish - first version - +# Compatibility entry point for stage A. Use backup.sh for the full pipeline. set -Eeuo pipefail - -timestamp_format='%Y-%m-%d_%H-%M-%S' - -# verbosity: set non-empty to enable debug -dbg= -# dbg=VERBOSE - -# default containers list file -containersfile="containers.txt" - -# dry run (or pass -t) -dryrun= - -# base directory for backups -BACKUP_DIR="/dockerbackups" - -# base directory for source docker project folders -SOURCE_BASE_DIR="${SOURCE_BASE_DIR:-/dockervolumes}" - -# start/health wait tuning -START_TIMEOUT=90 # seconds to wait for service to be up -WAIT_INTERVAL=3 # seconds between polls -RETRIES=1 # number of restart retries on failure (per container) - -# log file -logdir="$HOME/.local/share/log/${0##*/}" -mkdir -p "$logdir" -ymdhms="$(/bin/date +'%Y-%m-%d_%H:%M:%S')" -ymd="${ymdhms%%_*}" -logfile="$logdir/${0##*/}_${ymd}.log" - -# compose command detection -COMPOSE_CMD=() -if docker compose version >/dev/null 2>&1; then - COMPOSE_CMD=(docker compose) -elif command -v docker-compose >/dev/null 2>&1; then - COMPOSE_CMD=(docker-compose) -else - echo "ERROR: Neither 'docker compose' nor 'docker-compose' found." >&2 - exit 3 -fi - -log() { - local timestamp - timestamp="$(/bin/date +"$timestamp_format")" - while [ "$#" -gt 0 ]; do - echo "$timestamp: ${0##*/}: $1" >>"$logfile" - if [ -n "$dbg" ]; then - echo "$timestamp LOG: ${0##*/}: $1" >&2 - else - echo "$1" >&2 - fi - shift - done -} - -debug() { - local timestamp - timestamp="$(/bin/date +"$timestamp_format")" - while [ "$#" -gt 0 ]; do - echo "$timestamp DBG: ${0##*/}: $1" >>"$logfile" - if [ -n "$dbg" ]; then - echo "$timestamp DBG: ${0##*/}: $1" >&2 - fi - shift - done -} - -fail() { - local exitcode="$1" - shift - log "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" - log "${0##*/}: FATAL ERROR" - while [ "$#" -gt 0 ]; do - log "$1" - shift - done - sleep 1 - tidy_up - exit "$exitcode" -} - -tidy_up() { - debug "tidy_up" - cd || log "Failed to cd in tidy_up" - trap - EXIT - trap - INT -} - -usage() { - cat 1>&2 <&2; usage; exit 1 ;; - :) echo "Option -$OPTARG requires an argument" 1>&2; exit 1 ;; - esac - done - shift $((OPTIND - 1)) -} - -toolcheck() { - local tool - while [ "$#" -gt 0 ]; do - tool="$1" - shift - debug "Checking for tool: '$tool'" - if ! command -v "$tool" >/dev/null 2>&1; then - fail 3 "Failed to find required tool: '$tool'" - fi - done -} - -compose() { - "${COMPOSE_CMD[@]}" "$@" -} - -# Return " " for a container by inspecting labels -get_compose_identity() { - # arg: container_name - local name="$1" - local proj serv - proj="$(docker inspect "$name" 2>/dev/null | jq -r '.[0].Config.Labels["com.docker.compose.project"] // empty')" - serv="$(docker inspect "$name" 2>/dev/null | jq -r '.[0].Config.Labels["com.docker.compose.service"] // empty')" - if [ -z "$proj" ] || [ -z "$serv" ]; then - echo "" - return 1 - fi - echo "$proj $serv" -} - -# Check current state/health for any container matching project+service -# Returns 0 if running and healthy (or no healthcheck), else 1 -service_is_healthy() { - # args: project service - local proj="$1" - local serv="$2" - local ids - # find containers by labels (may be multiple if scaled; any healthy one counts) - ids="$(docker ps -a -q --filter "label=com.docker.compose.project=$proj" --filter "label=com.docker.compose.service=$serv")" || true - if [ -z "$ids" ]; then - debug "No containers found for project=$proj service=$serv" - return 1 - fi - local id - for id in $ids; do - # status: created, restarting, running, removing, paused, exited, dead - local st hlth - st="$(docker inspect -f '{{.State.Status}}' "$id" 2>/dev/null || echo "unknown")" - if [ "$st" != "running" ]; then - continue - fi - # Health may not exist; treat missing as pass if running - hlth="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$id" 2>/dev/null || echo "none")" - if [ "$hlth" = "none" ] || [ "$hlth" = "healthy" ]; then - debug "Container $id is running (health=$hlth)" - return 0 - fi - done - return 1 -} - -# Wait for service to become healthy/running within timeout -wait_for_service() { - # args: project service timeout_seconds - local proj="$1" - local serv="$2" - local timeout="$3" - local start now - start="$(date +%s)" - while true; do - if service_is_healthy "$proj" "$serv"; then - return 0 - fi - now="$(date +%s)" - if [ $(( now - start )) -ge "$timeout" ]; then - return 1 - fi - sleep "$WAIT_INTERVAL" - done -} - -down_backup_up() { - # args: WORKING_DIR PROJECT SERVICE - local WORKING_DIR="$1" - local PROJ="$2" - local SERV="$3" - - local PROJECT_NAME - local BAK_YMDHMS - local BACKUP_CODE_DIR - local BACKUP_CODE_ZSTD - - PROJECT_NAME="$(basename "$WORKING_DIR")" - BAK_YMDHMS="$(date +'%Y-%m-%d_%H-%M-%S')" - BACKUP_CODE_DIR="${BACKUP_DIR}/${PROJECT_NAME}" - BACKUP_CODE_ZSTD="${BACKUP_CODE_DIR}/$(hostname -s).${PROJECT_NAME}.${BAK_YMDHMS}.zst" - - mkdir -p "${BACKUP_CODE_DIR}" - log "Backup target: ${BACKUP_CODE_ZSTD}" - - if [ -n "$dryrun" ]; then - log "DRY-RUN: would run: compose down; tar --zstd; compose pull; compose up -d; wait for ${PROJ}/${SERV}" - return 0 - fi - - # stop, archive, start, then verify - compose down || log "WARNING: non-zero exit from 'compose down' (continuing)" - tar --one-file-system --zstd -cf "${BACKUP_CODE_ZSTD}" "${WORKING_DIR}" - ls -lh "${BACKUP_CODE_DIR}" || true - compose pull || log "WARNING: non-zero exit from 'compose pull' (continuing)" - compose up -d - - if wait_for_service "$PROJ" "$SERV" "$START_TIMEOUT"; then - log "Service ${PROJ}/${SERV} is up." - return 0 - fi - - log "Service ${PROJ}/${SERV} did not become healthy in ${START_TIMEOUT}s. Retrying ($RETRIES)." - local attempt=1 - while [ "$attempt" -le "$RETRIES" ]; do - log "Retry $attempt: restarting ${PROJ}/${SERV}" - compose down || log "WARNING: retry: non-zero from 'compose down'" - compose up -d - if wait_for_service "$PROJ" "$SERV" "$START_TIMEOUT"; then - log "Service ${PROJ}/${SERV} is up after retry." - return 0 - fi - attempt=$(( attempt + 1 )) - done - - fail 1 "Service ${PROJ}/${SERV} failed to start/healthy after ${RETRIES} retry(ies)." -} - -find_working_dir() { - # args: CONTAINER_NAME - local name="$1" - local wd="" - if docker inspect "$name" >/dev/null 2>&1; then - wd="$(docker inspect "$name" 2>/dev/null | jq -r '.[0].Config.Labels["com.docker.compose.project.working_dir"] // empty')" - fi - if [ -z "$wd" ]; then - if [ -d "$SOURCE_BASE_DIR/$name" ]; then - wd="$SOURCE_BASE_DIR/$name" - elif [ -d "$HOME/prj/docker/$name" ]; then - wd="$HOME/prj/docker/$name" - else - wd="" - fi - fi - printf '%s' "$wd" -} - -container_exists() { - docker ps -a --format '{{.Names}}' | grep -Fxq "$1" -} - -backup_container() { - local container_name="$1" - log "Processing container '$container_name'" - - local container_start_time container_finish_time container_duration container_duration_m - container_start_time="$(date +%s)" - - if ! container_exists "$container_name"; then - log "Container not found: '$container_name' (skipping)" - return 0 - fi - - # capture compose identity before we stop anything - local identity proj serv - identity="$(get_compose_identity "$container_name" || true)" - if [ -z "$identity" ]; then - fail 1 "Could not determine compose identity (project/service) for '$container_name'." - fi - proj="$(echo "$identity" | awk '{print $1}')" - serv="$(echo "$identity" | awk '{print $2}')" - debug "Compose identity: project=$proj service=$serv" - - local working_dir - working_dir="$(find_working_dir "$container_name")" - if [ -z "$working_dir" ]; then - fail 1 "Could not determine working_dir for container '$container_name'." - fi - log "Working dir: $working_dir" - if [ ! -d "$working_dir" ]; then - fail 1 "Working dir does not exist: '$working_dir'" - fi - - local original_dir - original_dir="$(pwd)" - cd "$working_dir" || fail 1 "Could not cd to '$working_dir'" - - down_backup_up "$working_dir" "$proj" "$serv" || log "WARNING: non-zero exit from backup function" - - cd "$original_dir" || fail 1 "Could not cd back to '$original_dir'" - - container_finish_time="$(date +%s)" - container_duration=$((container_finish_time - container_start_time)) - container_duration_m=$((container_duration / 60)) - log "Finished '$container_name' after ${container_duration}s (approx ${container_duration_m} min)" -} - -trap tidy_up EXIT -trap 'exit 130' INT - -parse_cmdline "$@" - -debug "-----" -debug "logfile = '$logfile'" -debug "source_base_dir = '$SOURCE_BASE_DIR'" - -toolcheck docker jq sed tar - -if [ ! -f "$containersfile" ]; then - fail 2 "Containers file not found: '$containersfile'" -fi -if [ ! -r "$containersfile" ]; then - fail 2 "Containers file not readable: '$containersfile'" -fi - -containers="$(sed -rn 's/^ +//; s/#.*//; s/ +$//; /^$/b; p' "$containersfile" || true)" - -if [ -z "${containers:-}" ]; then - log "No containers listed in '$containersfile' (nothing to do)." - exit 0 -fi - -script_start_time="$(date +%s)" - -for container in $containers; do - backup_container "$container" -done - -script_finish_time="$(date +%s)" -script_duration=$((script_finish_time - script_start_time)) -script_duration_m=$((script_duration / 60)) - -debug "-----" -log "Script finished after ${script_duration}s (${script_duration_m} min)" - -exit 0 -# EOF +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +exec "$SCRIPT_DIR/stages/10-containers.sh" "$@" diff --git a/dockerbackup_prune.sh b/dockerbackup_prune.sh old mode 100644 new mode 100755 index 4eab700..1b75535 --- a/dockerbackup_prune.sh +++ b/dockerbackup_prune.sh @@ -1,399 +1,5 @@ #!/usr/bin/env bash +# Compatibility entry point. Use stages/90-prune.sh for new installations. set -Eeuo pipefail - -############################################################################### -# dockerbackup_prune.sh -# -# Default: -# A) Create a dated .zst backup locally and keep it -# -# With --apply: -# A) Create a dated .zst backup -# C) Prune *.zst under PRUNE_ROOT subfolders, keeping newest N per subfolder -# D) Keep the new local archive -# -# With --rclone: -# A) Create a dated .zst backup -# B) Upload via rclone -# C) Show what prune would do (unless --apply is also set) -# D) Keep the new local archive unless --apply is also set -# -# With --rclone --apply: -# A) Create a dated .zst backup -# B) Upload via rclone -# C) Prune for real -# D) Delete the new local archive -# -# Restore: -# zstd -d -c /path/backup.zst | tar -x -C /restore/target -############################################################################### - -# --------------------------- Defaults (can be overridden) -------------------- -SOURCE_DIR="/mybackups" # what to archive -PRUNE_ROOT="/mybackups" # where to prune .zst -PRUNE_PATTERN="*.zst" # files to prune -KEEP_COUNT=5 # keep newest N per subfolder - -RCLONE_REMOTE="gdrive:location/sublocation" # rclone destination -OUT_DIR="/tmp" # where to write the .zst -BACKUP_PREFIX="panda-dockerbackup" # backup name prefix - -# Optional zstd tuning, e.g. "-19 -T0" for max level, multithread -ZSTD_EXTRA_OPTS="" - -# --------------------------- Load .env (same dir as script) ----------------- -# Order of precedence: -# defaults < .env (same dir) < CLI flags SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" -ENV_FILE="${DOCKERBACKUP_ENV:-$SCRIPT_DIR/.env}" -if [[ -f "$ENV_FILE" ]]; then - perms="$(stat -c '%a' "$ENV_FILE" 2>/dev/null || stat -f '%Lp' "$ENV_FILE" 2>/dev/null || echo '')" - if [[ -n "$perms" && ! "$perms" =~ ^(600|400)$ ]]; then - echo "$(date '+%F %T') [WARN] $ENV_FILE permissions are $perms; recommend 600." >&2 - fi - set -a - # shellcheck disable=SC1090 - . "$ENV_FILE" - set +a -fi - -# --------------------------- CLI flags -------------------------------------- -APPLY=0 -QUIET=0 -DRYRUN=0 -USE_RCLONE=0 - -usage() { - cat <&2; usage; exit 2 ;; - esac - shift -done - -# --------------------------- Logging helpers -------------------------------- -ts() { date '+%F %T'; } -log() { [[ $QUIET -eq 0 ]] && echo "$(ts) [$1] $2"; } -log_err() { echo "$(ts) [ERROR] $1" >&2; } -run() { if [[ $DRYRUN -eq 1 ]]; then log "DRY" "$*"; else "$@"; fi; } - -truncate_and_print() { - local s="$1" cols - if [[ -t 2 ]]; then - cols=$(tput cols 2>/dev/null || echo 120) - else - cols=120 - fi - (( cols < 20 )) && cols=120 - if (( ${#s} > cols - 1 )); then - s="${s:0:cols-1}" - fi - printf '\r%s\033[K' "$s" >&2 -} - -cleanup_tmp="" -pipe_pid="" -pipe_pgid="" -pct_fd_open=0 -path_fd_open=0 -status_line_shown=0 - -cleanup() { - local why="${1:-EXIT}" - - if [[ -n "$pipe_pid" ]]; then - if command -v ps >/dev/null 2>&1; then - pipe_pgid=$(ps -o pgid= "$pipe_pid" 2>/dev/null | tr -d ' ' || true) - fi - if [[ -n "$pipe_pgid" ]]; then - kill -TERM "-$pipe_pgid" 2>/dev/null || true - sleep 0.2 - kill -KILL "-$pipe_pgid" 2>/dev/null || true - else - kill -TERM "$pipe_pid" 2>/dev/null || true - sleep 0.2 - kill -KILL "$pipe_pid" 2>/dev/null || true - fi - wait "$pipe_pid" 2>/dev/null || true - fi - - if (( pct_fd_open == 1 )); then exec 4>&- 4<&-; pct_fd_open=0; fi - if (( path_fd_open == 1 )); then exec 3>&- 3<&-; path_fd_open=0; fi - - if [[ -n "$cleanup_tmp" ]]; then - rm -f "$cleanup_tmp/pct" "$cleanup_tmp/path" 2>/dev/null || true - rmdir "$cleanup_tmp" 2>/dev/null || true - fi - - if (( status_line_shown == 1 )) && [[ $QUIET -eq 0 ]]; then - printf '\n' >&2 - fi - - case "$why" in - INT|TERM) exit 130 ;; - esac -} - -trap 'log_err "Command failed: $BASH_COMMAND"; cleanup ERR; exit 1' ERR -trap 'cleanup INT' INT -trap 'cleanup TERM' TERM -trap 'cleanup EXIT' EXIT - -# --------------------------- Preconditions ---------------------------------- -[[ -d "$SOURCE_DIR" ]] || { log_err "Source dir not found: $SOURCE_DIR"; exit 1; } -[[ -d "$PRUNE_ROOT" ]] || { log_err "Prune root not found: $PRUNE_ROOT"; exit 1; } -command -v tar >/dev/null 2>&1 || { log_err "tar not found"; exit 1; } -command -v zstd >/dev/null 2>&1 || { log_err "zstd not found"; exit 1; } -command -v find >/dev/null 2>&1 || { log_err "find not found"; exit 1; } - -if (( USE_RCLONE == 1 )) && (( DRYRUN == 0 )); then - command -v rclone >/dev/null 2>&1 || { log_err "rclone not found"; exit 1; } -fi - -STAMP="${STAMP_OVERRIDE:-$(date +%F_%H-%M)}" -BACKUP_NAME="${BACKUP_PREFIX}-${STAMP}.zst" -BACKUP_PATH="${OUT_DIR%/}/${BACKUP_NAME}" - -RCLONE_FLAGS=() -[[ $QUIET -eq 1 ]] && RCLONE_FLAGS+=("-q") || RCLONE_FLAGS+=("--progress") - -ZSTD_FLAGS=("-q") -[[ -n "$ZSTD_EXTRA_OPTS" ]] && ZSTD_FLAGS+=($ZSTD_EXTRA_OPTS) - -log "INFO" "Mode: $( [[ $DRYRUN -eq 1 ]] && echo 'DRY-RUN ALL' || ([[ $APPLY -eq 1 ]] && echo 'APPLY DELETES' || echo 'NO DELETES') )" -log "INFO" "Rclone mode: $( [[ $USE_RCLONE -eq 1 ]] && echo 'ENABLED' || echo 'DISABLED' )" -log "INFO" "Creating archive: $BACKUP_PATH (.zst)" -log "INFO" "Keeping newest $KEEP_COUNT file(s) per subfolder matching: $PRUNE_PATTERN" -log "INFO" "Source: $SOURCE_DIR" -if (( USE_RCLONE == 1 )); then - log "INFO" "Remote: $RCLONE_REMOTE" -fi -log "INFO" "Prune root (root itself will be skipped): $PRUNE_ROOT" - -# --------------------------- Step A: create .zst with one-liner -------------- -log "STEP" "A) Creating archive" - -if [[ $DRYRUN -eq 1 ]]; then - log "DRY" "tar -C \"$SOURCE_DIR\" -cvf - . | pv -n -s \$(du -sb \"$SOURCE_DIR\"|cut -f1) 2>pct.fifo | zstd ${ZSTD_FLAGS[*]} -o \"$BACKUP_PATH\"" -else - mkdir -p "$OUT_DIR" - - export LC_ALL=C - TOTAL_FILES=$(find "$SOURCE_DIR" -type f | wc -l | awk '{print $1}') - TOTAL_BYTES=$(du -sb "$SOURCE_DIR" | cut -f1) - - tmpdir="$(mktemp -d)"; cleanup_tmp="$tmpdir" - path_fifo="$tmpdir/path"; pct_fifo="$tmpdir/pct" - mkfifo "$path_fifo" - have_pv=0 - if command -v pv >/dev/null 2>&1 && [[ $QUIET -eq 0 ]] && [[ -t 2 ]]; then - mkfifo "$pct_fifo"; have_pv=1 - fi - - exec 3<> "$path_fifo"; path_fd_open=1 - if (( have_pv == 1 )); then exec 4<> "$pct_fifo"; pct_fd_open=1; fi - - if (( have_pv == 1 )); then - ( - set -o pipefail - tar -C "$SOURCE_DIR" -cvf - . \ - 2> >(sed -u 's#^\./##' > "$path_fifo" || true) \ - | pv -n -s "$TOTAL_BYTES" 2> "$pct_fifo" \ - | zstd "${ZSTD_FLAGS[@]}" -o "$BACKUP_PATH" - ) & - else - ( - set -o pipefail - tar -C "$SOURCE_DIR" -cvf - . \ - 2> >(sed -u 's#^\./##' > "$path_fifo" || true) \ - | zstd "${ZSTD_FLAGS[@]}" -o "$BACKUP_PATH" - ) & - fi - pipe_pid=$! - - files_seen=0 - last_path="" - last_pct="0.0" - status_line_shown=0 - last_print_time=0 - - while kill -0 "$pipe_pid" 2>/dev/null; do - if (( have_pv == 1 )); then - if IFS= read -r -t 0.03 pct_line <&4; then last_pct="$pct_line"; fi - fi - - if IFS= read -r -t 0.03 path_line <&3; then - last_path="$path_line" - [[ -n "$last_path" ]] && (( files_seen++ )) || true - fi - - display_rel="$last_path" - dir_part="$(dirname -- "$display_rel" 2>/dev/null || echo .)" - base_part="$(basename -- "$display_rel" 2>/dev/null || echo .)" - [[ "$dir_part" == "." ]] && dir_part="." - - if [[ $QUIET -eq 0 ]]; then - now_ns=$(date +%s%N) - if (( now_ns - last_print_time > 50000000 )); then - if (( have_pv == 1 )); then - line="[$(printf '%.1f' "${last_pct:-0}")%] archiving: ${dir_part}/ ${base_part}" - else - line="[${files_seen}/${TOTAL_FILES}] archiving: ${dir_part}/ ${base_part}" - fi - truncate_and_print "$line" - status_line_shown=1 - last_print_time=$now_ns - fi - fi - done - - while IFS= read -r -t 0.03 path_line <&3; do - last_path="$path_line"; (( files_seen++ )) || true - [[ $QUIET -eq 0 ]] && truncate_and_print "[${files_seen}/${TOTAL_FILES}] archiving: $(dirname -- "$last_path")/ $(basename -- "$last_path")" - status_line_shown=1 - done || true - - wait "$pipe_pid" - if [[ $QUIET -eq 0 ]]; then - printf '\n' >&2 - status_line_shown=0 - fi - - exec 3>&- 3<&-; path_fd_open=0 - if (( have_pv == 1 )); then exec 4>&- 4<&-; pct_fd_open=0; fi - rm -f "$path_fifo" "$pct_fifo" 2>/dev/null || true - rmdir "$tmpdir" 2>/dev/null || true - cleanup_tmp="" -fi -log "OK" "Archive step completed" - -# --------------------------- Step B: optional rclone copy -------------------- -if (( USE_RCLONE == 1 )); then - log "STEP" "B) rclone copy to remote" - if [[ $DRYRUN -eq 1 ]]; then - log "DRY" "rclone copy \"$BACKUP_PATH\" \"$RCLONE_REMOTE\" ${RCLONE_FLAGS[*]}" - else - run rclone copy "$BACKUP_PATH" "$RCLONE_REMOTE" "${RCLONE_FLAGS[@]}" - fi - log "OK" "Upload step completed" -else - log "SKIP" "--rclone not set; skipping upload" -fi - -# --------------------------- Step C: prune if apply or rclone --------------- -if (( APPLY == 1 || USE_RCLONE == 1 )); then - log "STEP" "C) Pruning $PRUNE_PATTERN under $PRUNE_ROOT (keep $KEEP_COUNT)" - export LC_ALL=C - prune_ok=1 - - while IFS= read -r -d '' dir; do - mapfile -d '' -t files < <( - find "$dir" -maxdepth 1 -type f -name "$PRUNE_PATTERN" -printf '%T@ %p\0' | - sort -z -n | - cut -z -d' ' -f2- - ) - count=${#files[@]} - if (( count == 0 )); then - log "SKIP" "No matches in: $dir" - continue - fi - - keep_from=$(( count - KEEP_COUNT )) - (( keep_from < 0 )) && keep_from=0 - - if (( count <= KEEP_COUNT )); then - log "KEEP" "At or below limit in: $dir (count=$count)" - continue - fi - - keep_list=("${files[@]:keep_from:KEEP_COUNT}") - del_list=("${files[@]:0:keep_from}") - - for k in "${keep_list[@]}"; do - log "KEEP" "Keep: $k" - done - for f in "${del_list[@]}"; do - if [[ $DRYRUN -eq 1 || $APPLY -eq 0 ]]; then - log "WOULD" "Remove: $f" - else - if rm -f -- "$f"; then - log "DELETE" "Removed: $f" - else - log_err "Failed to remove: $f"; prune_ok=0 - fi - fi - done - done < <(find "$PRUNE_ROOT" -mindepth 1 -type d -print0) - - if [[ $prune_ok -ne 1 ]]; then - log_err "Prune step encountered errors; aborting final delete" - exit 1 - fi - log "OK" "Prune step completed" -else - log "SKIP" "--apply and --rclone not set; skipping prune" -fi - -# --------------------------- Step D: remove local .zst ----------------------- -if (( USE_RCLONE == 1 )); then - log "STEP" "D) Removing local backup" - if [[ $DRYRUN -eq 1 || $APPLY -eq 0 ]]; then - log "WOULD" "Remove local backup: $BACKUP_PATH" - else - if rm -f -- "$BACKUP_PATH"; then - log "DELETE" "Removed local backup: $BACKUP_PATH" - else - log_err "Failed to remove local backup: $BACKUP_PATH" - exit 1 - fi - fi -else - log "KEEP" "Local archive retained: $BACKUP_PATH" -fi - -log "DONE" "All steps completed successfully" +exec "$SCRIPT_DIR/stages/90-prune.sh" "$@" diff --git a/lib/common.sh b/lib/common.sh new file mode 100644 index 0000000..085aca6 --- /dev/null +++ b/lib/common.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)" +ENV_FILE="${DOCKERBACKUP_ENV:-$SCRIPT_ROOT/.env}" + +SOURCE_BASE_DIR="/dockervolumes" +BACKUP_ROOT="/dockerbackups" +CONTAINERS_FILE="$SCRIPT_ROOT/containers.txt" +BACKUP_HOSTNAME="$(hostname -s)" +BACKUP_METHODS="rclone smb" +KEEP_COUNT=5 +PULL_IMAGES=1 +START_TIMEOUT=90 +WAIT_INTERVAL=3 +START_RETRIES=1 +LOCK_FILE="/run/lock/zorruno-docker-backup.lock" + +RCLONE_ENABLED=0 +RCLONE_REMOTE="" +RCLONE_EXTRA_ARGS="" +SMB_ENABLED=0 +SMB_MOUNT="" +SMB_HOST_SUBDIR=1 +SMB_RSYNC_ARGS="-a" + +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + . "$ENV_FILE" + set +a +fi + +# Ease migration from the original prune script's .env names. +if [[ -n "${SOURCE_DIR:-}" && "$BACKUP_ROOT" == "/dockerbackups" ]]; then + BACKUP_ROOT="$SOURCE_DIR" +fi + +timestamp() { date '+%F %T'; } +log() { printf '%s [%s] %s\n' "$(timestamp)" "$1" "$2" >&2; } +die() { log ERROR "$1"; exit "${2:-1}"; } + +is_true() { + case "${1,,}" in + 1|true|yes|on|enabled) return 0 ;; + *) return 1 ;; + esac +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "Required command not found: $1" 3 +} + +require_positive_integer() { + [[ "$2" =~ ^[1-9][0-9]*$ ]] || die "$1 must be a positive integer (received: $2)" 2 +} + +resolve_path() { + if [[ "$1" = /* ]]; then printf '%s\n' "$1"; else printf '%s/%s\n' "$SCRIPT_ROOT" "$1"; fi +} + +run_id() { date '+%Y-%m-%d_%H-%M-%S'; } diff --git a/methods/20-rclone.sh b/methods/20-rclone.sh new file mode 100755 index 0000000..e1a84f4 --- /dev/null +++ b/methods/20-rclone.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Copy the local backup tree to an rclone destination. +set -Eeuo pipefail +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +. "$SCRIPT_DIR/../lib/common.sh" +DRY_RUN=0; FORCE=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) DRY_RUN=1 ;; --force) FORCE=1 ;; + -h|--help) echo "Usage: ${0##*/} [--dry-run] [--force]"; exit 0 ;; + *) die "Unknown option: $1" 2 ;; + esac + shift +done +if ! is_true "$RCLONE_ENABLED" && (( FORCE == 0 )); then log SKIP "rclone method is disabled"; exit 0; fi +[[ -n "$RCLONE_REMOTE" ]] || die "RCLONE_REMOTE is not configured" +require_command rclone +[[ -d "$BACKUP_ROOT" ]] || die "Backup root not found: $BACKUP_ROOT" +read -r -a extra_args <<<"$RCLONE_EXTRA_ARGS" +args=(copy "$BACKUP_ROOT" "${RCLONE_REMOTE%/}/$BACKUP_HOSTNAME" "${extra_args[@]}") +(( DRY_RUN )) && args+=(--dry-run) +log INFO "Copying backups to ${RCLONE_REMOTE%/}/$BACKUP_HOSTNAME" +rclone "${args[@]}" +log DONE "rclone backup method completed" diff --git a/methods/30-smb.sh b/methods/30-smb.sh new file mode 100755 index 0000000..0ff49ae --- /dev/null +++ b/methods/30-smb.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Copy the local backup tree to an already-mounted SMB share. +set -Eeuo pipefail +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +. "$SCRIPT_DIR/../lib/common.sh" +DRY_RUN=0; FORCE=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) DRY_RUN=1 ;; --force) FORCE=1 ;; + -h|--help) echo "Usage: ${0##*/} [--dry-run] [--force]"; exit 0 ;; + *) die "Unknown option: $1" 2 ;; + esac + shift +done +if ! is_true "$SMB_ENABLED" && (( FORCE == 0 )); then log SKIP "SMB method is disabled"; exit 0; fi +[[ -n "$SMB_MOUNT" ]] || die "SMB_MOUNT is not configured" +require_command rsync +require_command mountpoint +[[ -d "$BACKUP_ROOT" ]] || die "Backup root not found: $BACKUP_ROOT" +[[ -d "$SMB_MOUNT" ]] || die "SMB mount does not exist: $SMB_MOUNT" +mountpoint -q "$SMB_MOUNT" || die "SMB path is not a mounted filesystem: $SMB_MOUNT" +destination="${SMB_MOUNT%/}" +is_true "$SMB_HOST_SUBDIR" && destination="$destination/$BACKUP_HOSTNAME" +(( DRY_RUN )) || mkdir -p "$destination" +read -r -a rsync_args <<<"$SMB_RSYNC_ARGS" +(( DRY_RUN )) && rsync_args+=(--dry-run) +log INFO "Copying backups to $destination" +rsync "${rsync_args[@]}" "$BACKUP_ROOT/" "$destination/" +log DONE "SMB backup method completed" diff --git a/stages/10-containers.sh b/stages/10-containers.sh new file mode 100755 index 0000000..8e254ad --- /dev/null +++ b/stages/10-containers.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# Stop, archive, update, start, and verify each listed Compose project. +set -Eeuo pipefail +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +# shellcheck source=../lib/common.sh +. "$SCRIPT_DIR/../lib/common.sh" + +DRY_RUN=0 +usage() { + cat </dev/null 2>&1; then + COMPOSE=(docker compose) +elif command -v docker-compose >/dev/null 2>&1; then + COMPOSE=(docker-compose) +else + die "Neither 'docker compose' nor 'docker-compose' is available" 3 +fi +compose() { "${COMPOSE[@]}" "$@"; } + +resolve_project() { + local entry="$1" working_dir="" project="" + if docker inspect "$entry" >/dev/null 2>&1; then + working_dir="$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project.working_dir"}}' "$entry" 2>/dev/null || true)" + project="$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}' "$entry" 2>/dev/null || true)" + fi + [[ -n "$working_dir" ]] || { [[ -d "$SOURCE_BASE_DIR/$entry" ]] && working_dir="$SOURCE_BASE_DIR/$entry"; } + [[ -n "$working_dir" ]] || return 1 + [[ -n "$project" ]] || project="$(basename "$working_dir")" + printf '%s\t%s\n' "$project" "$working_dir" +} + +project_is_ready() { + local project="$1" ids id status health + ids="$(docker ps -a -q --filter "label=com.docker.compose.project=$project" --filter "label=com.docker.compose.oneoff=False")" + [[ -n "$ids" ]] || return 1 + for id in $ids; do + status="$(docker inspect -f '{{.State.Status}}' "$id")" + health="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$id")" + [[ "$status" == running ]] || return 1 + [[ "$health" == none || "$health" == healthy ]] || return 1 + done +} + +wait_for_project() { + local project="$1" started now + started="$(date +%s)" + while ! project_is_ready "$project"; do + now="$(date +%s)" + (( now - started < START_TIMEOUT )) || return 1 + sleep "$WAIT_INTERVAL" + done +} + +backup_project() { + local project="$1" working_dir="$2" parent base output_dir output attempt + parent="$(dirname "$working_dir")" + base="$(basename "$working_dir")" + output_dir="$BACKUP_ROOT/$project" + output="$output_dir/${BACKUP_HOSTNAME}.${project}.$(run_id).tar.zst" + log INFO "Project: $project ($working_dir)" + log INFO "Archive: $output" + if (( DRY_RUN )); then + log DRY "Would stop, archive, optionally pull, start, and verify $project" + return 0 + fi + + mkdir -p "$output_dir" + cd "$working_dir" + compose down + if ! tar --one-file-system --zstd -cf "$output" -C "$parent" "$base"; then + log ERROR "Archive failed; attempting to restart $project" + compose up -d || true + return 1 + fi + if is_true "$PULL_IMAGES"; then + compose pull || log WARN "Image pull failed for $project; starting with available images" + fi + compose up -d + + if wait_for_project "$project"; then log OK "$project is running and healthy"; return 0; fi + for (( attempt=1; attempt<=START_RETRIES; attempt++ )); do + log WARN "$project did not become ready; restart attempt $attempt/$START_RETRIES" + compose down || true + compose up -d + if wait_for_project "$project"; then log OK "$project is running after retry"; return 0; fi + done + log ERROR "$project did not become ready" + return 1 +} + +declare -A SEEN=() +failures=0 +while IFS= read -r entry; do + entry="${entry%%#*}" + entry="${entry#"${entry%%[![:space:]]*}"}" + entry="${entry%"${entry##*[![:space:]]}"}" + [[ -n "$entry" ]] || continue + if ! resolved="$(resolve_project "$entry")"; then + log ERROR "Could not resolve container or project: $entry" + (( failures++ )) || true + continue + fi + IFS=$'\t' read -r project working_dir <<<"$resolved" + if [[ -n "${SEEN[$working_dir]:-}" ]]; then + log SKIP "Project already processed: $project ($working_dir)" + continue + fi + SEEN[$working_dir]=1 + backup_project "$project" "$working_dir" || { (( failures++ )) || true; } +done < "$CONTAINERS_FILE" + +(( failures == 0 )) || die "$failures project backup(s) failed" +log DONE "Container backup stage completed" diff --git a/stages/90-prune.sh b/stages/90-prune.sh new file mode 100755 index 0000000..6c1d5f9 --- /dev/null +++ b/stages/90-prune.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Prune local per-project archives, retaining the newest N in each. +set -Eeuo pipefail +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +. "$SCRIPT_DIR/../lib/common.sh" +DRY_RUN=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) DRY_RUN=1 ;; --keep) KEEP_COUNT="${2:?Missing keep count}"; shift ;; + --root) BACKUP_ROOT="${2:?Missing backup root}"; shift ;; + -h|--help) echo "Usage: ${0##*/} [--dry-run] [--keep N] [--root DIR]"; exit 0 ;; + *) die "Unknown option: $1" 2 ;; + esac + shift +done +require_positive_integer KEEP_COUNT "$KEEP_COUNT" +[[ -d "$BACKUP_ROOT" ]] || { log SKIP "Backup root does not exist: $BACKUP_ROOT"; exit 0; } +while IFS= read -r -d '' dir; do + mapfile -d '' -t files < <(find "$dir" -maxdepth 1 -type f -name '*.tar.zst' -printf '%T@ %p\0' | sort -z -nr | cut -z -d' ' -f2-) + (( ${#files[@]} > KEEP_COUNT )) || continue + for file in "${files[@]:KEEP_COUNT}"; do + if (( DRY_RUN )); then log WOULD "Remove $file"; else rm -f -- "$file"; log DELETE "Removed $file"; fi + done +done < <(find "$BACKUP_ROOT" -mindepth 1 -maxdepth 1 -type d -print0) +log DONE "Local prune stage completed; kept newest $KEEP_COUNT per project"