From 5d07cc4f54639acf122cabec2f121dea079775aa Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 26 Sep 2026 15:33:34 +1000 Subject: [PATCH] Document backup pipeline behaviour --- backup.sh | 20 +++++++++++++++++++- bin/mqtt-trigger-listener.sh | 6 +++++- lib/common.sh | 13 +++++++++++++ lib/mqtt.sh | 4 ++++ stages/10-containers.sh | 18 +++++++++++++++++- stages/20-rclone.sh | 5 ++++- stages/30-smb.sh | 5 ++++- stages/90-prune.sh | 7 +++++-- stages/99-mqtt-report.sh | 8 +++++++- systemd/zorruno-docker-backup-mqtt.service | 2 ++ 10 files changed, 80 insertions(+), 8 deletions(-) diff --git a/backup.sh b/backup.sh index 3f58b17..a2b6fa0 100755 --- a/backup.sh +++ b/backup.sh @@ -1,5 +1,14 @@ #!/usr/bin/env bash -# Main pipeline: container archives, configured methods, then local prune. +# Main pipeline orchestrator. +# +# Execution order: +# 1. Create cold archives for the selected Docker Compose projects. +# 2. Run each enabled transfer stage listed in BACKUP_METHODS. +# 3. Prune old local archives only after every earlier stage succeeds. +# +# An EXIT trap publishes the final MQTT result even when a stage fails. The +# original exit code is preserved so cron, systemd and shell callers can detect +# failure normally. set -Eeuo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" . "$SCRIPT_DIR/lib/common.sh" @@ -19,12 +28,16 @@ done BACKUP_RUN_ID="$(run_id)" BACKUP_STARTED_EPOCH="$(date +%s)" +# Every script sourced through this process appends structured log lines here. +# The MQTT reporting stage parses this temporary log to build its summary. BACKUP_RUN_LOG="$(mktemp "${TMPDIR:-/tmp}/zorruno-docker-backup.XXXXXX.log")" export BACKUP_RUN_ID BACKUP_STARTED_EPOCH BACKUP_RUN_LOG finish_run() { local exit_code=$? finished status + # Prevent recursion when this function exits with the pipeline's status. trap - EXIT + # Reporting is best-effort and must never replace the real pipeline result. set +e finished="$(date +%s)" if (( exit_code == 0 )); then @@ -44,6 +57,8 @@ trap finish_run EXIT require_command flock mkdir -p "$(dirname "$LOCK_FILE")" +# Keep descriptor 9 open for the whole run. Closing it automatically releases +# the advisory lock, including on errors and signals that terminate the shell. exec 9>"$LOCK_FILE" flock -n 9 || die "Another backup run holds lock: $LOCK_FILE" stage_args=() @@ -51,12 +66,15 @@ stage_args=() [[ -n "$CONTAINERS_OVERRIDE" ]] && stage_args+=(--containers "$CONTAINERS_OVERRIDE") log START "Backup pipeline for $BACKUP_HOSTNAME" if is_true "$MQTT_REPORT_ENABLED"; then + # The retained running state lets Home Assistant show that work is active. "$SCRIPT_DIR/stages/99-mqtt-report.sh" running 0 "$BACKUP_STARTED_EPOCH" \ "$BACKUP_STARTED_EPOCH" "$BACKUP_RUN_ID" "$BACKUP_RUN_LOG" \ || log WARN "Unable to publish MQTT running state; backup will continue" fi (( SKIP_CONTAINERS )) || "$SCRIPT_DIR/stages/10-containers.sh" "${stage_args[@]}" method_args=(); (( DRY_RUN )) && method_args+=(--dry-run) +# BACKUP_METHODS contains short names (for example: "rclone smb"). Numeric +# prefixes control directory ordering without becoming part of configuration. for method in $BACKUP_METHODS; do method_script="" while IFS= read -r candidate; do method_script="$candidate"; break; done < <(find "$SCRIPT_DIR/stages" -maxdepth 1 -type f -name "*-${method}.sh" -print | sort) diff --git a/bin/mqtt-trigger-listener.sh b/bin/mqtt-trigger-listener.sh index 94e8f05..1f81c95 100755 --- a/bin/mqtt-trigger-listener.sh +++ b/bin/mqtt-trigger-listener.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash -# Listen for an authenticated JSON command and run the backup pipeline. +# Long-running MQTT command listener, normally managed by systemd as root. +# Accepts only {"action":"run"}; broker authentication and ACLs should restrict +# who can publish to MQTT_COMMAND_TOPIC. set -Eeuo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)" . "$SCRIPT_DIR/lib/common.sh" @@ -20,5 +22,7 @@ mosquitto_sub "${MQTT_ARGS[@]}" -R -t "$MQTT_COMMAND_TOPIC" | while IFS= read -r continue fi log MQTT "Accepted backup run command" + # backup.sh owns concurrency control. A command received during an active run + # fails safely on the same flock used by cron and manual invocations. "$SCRIPT_DIR/backup.sh" || log ERROR "MQTT-triggered backup failed" done diff --git a/lib/common.sh b/lib/common.sh index cc1060b..503ca89 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -1,9 +1,15 @@ #!/usr/bin/env bash +# Shared configuration, validation and logging helpers. +# +# Every executable script sources this file. Defaults are deliberately safe: +# transfer methods and MQTT reporting are disabled until enabled in the +# host-local, Git-ignored .env file. set -Eeuo pipefail SCRIPT_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)" ENV_FILE="${DOCKERBACKUP_ENV:-$SCRIPT_ROOT/.env}" +# Core archive pipeline defaults. SOURCE_BASE_DIR="/dockervolumes" BACKUP_ROOT="/dockerbackups" CONTAINERS_FILE="$SCRIPT_ROOT/containers.txt" @@ -16,6 +22,7 @@ WAIT_INTERVAL=3 START_RETRIES=1 LOCK_FILE="/run/lock/zorruno-docker-backup.lock" ZSTD_EXTRA_OPTS="-T0 -12" +# Optional MQTT reporting and command-listener defaults. MQTT_REPORT_ENABLED=0 MQTT_HOST="" MQTT_PORT=1883 @@ -24,6 +31,7 @@ MQTT_PASSWORD_FILE="" MQTT_STATUS_TOPIC="viewroad-status/dockerbackup/$BACKUP_HOSTNAME" MQTT_COMMAND_TOPIC="viewroad-commands/dockerbackup/$BACKUP_HOSTNAME/run" +# Transfer-stage defaults. Host-specific destinations belong in .env. RCLONE_ENABLED=0 RCLONE_REMOTE="" RCLONE_HOST_SUBDIR=1 @@ -34,6 +42,7 @@ SMB_HOST_SUBDIR=1 SMB_RSYNC_ARGS="-a" if [[ -f "$ENV_FILE" ]]; then + # Export loaded values so child stage scripts inherit the same configuration. set -a # shellcheck disable=SC1090 . "$ENV_FILE" @@ -50,6 +59,8 @@ log() { local line line="$(timestamp) [$1] $2" printf '%s\n' "$line" >&2 + # Directly executed stages have no BACKUP_RUN_LOG. Always return success so + # an informational log call cannot terminate a script running with set -e. if [[ -n "${BACKUP_RUN_LOG:-}" ]]; then printf '%s\n' "$line" >>"$BACKUP_RUN_LOG" fi @@ -73,6 +84,8 @@ require_positive_integer() { } resolve_path() { + # Host-local filenames are resolved relative to the repository, not the + # caller's current directory. Absolute paths pass through unchanged. if [[ "$1" = /* ]]; then printf '%s\n' "$1"; else printf '%s/%s\n' "$SCRIPT_ROOT" "$1"; fi } diff --git a/lib/mqtt.sh b/lib/mqtt.sh index bd1076c..70fe28d 100644 --- a/lib/mqtt.sh +++ b/lib/mqtt.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# Shared mosquitto_pub/mosquitto_sub argument construction. +# Passwords are read from a protected file rather than placed in .env or Git. mqtt_common_args() { MQTT_ARGS=(-h "$MQTT_HOST" -p "$MQTT_PORT") @@ -14,6 +16,8 @@ mqtt_publish() { [[ -n "$MQTT_HOST" ]] || die "MQTT_HOST is not configured" require_command mosquitto_pub mqtt_common_args + # Backup state is retained so Home Assistant receives the latest result after + # reconnecting. Command messages are handled separately and are never kept. (( retain )) && MQTT_ARGS+=(-r) mosquitto_pub "${MQTT_ARGS[@]}" -t "$topic" -m "$payload" } diff --git a/stages/10-containers.sh b/stages/10-containers.sh index d348d5a..7c467ae 100755 --- a/stages/10-containers.sh +++ b/stages/10-containers.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash -# Stop, archive, update, start, and verify each listed Compose project. +# Stage 10: create a cold, compressed archive of each selected Compose project. +# +# Entries may be Compose project-directory names or live container names. A +# container name is resolved through its Compose labels. Duplicate entries that +# resolve to the same working directory are processed only once. set -Eeuo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" # shellcheck source=../lib/common.sh @@ -44,10 +48,13 @@ compose() { "${COMPOSE[@]}" "$@"; } resolve_project() { local entry="$1" working_dir="" project="" + # Prefer authoritative Compose labels when the entry names a container. if docker inspect "$entry" >/dev/null 2>&1; then working_dir="$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project.working_dir"}}' "$entry" 2>/dev/null || true)" project="$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}' "$entry" 2>/dev/null || true)" fi + # Fall back to a project directory beneath SOURCE_BASE_DIR. This also allows + # useful dry-runs when the caller cannot access the Docker socket. [[ -n "$working_dir" ]] || { [[ -d "$SOURCE_BASE_DIR/$entry" ]] && working_dir="$SOURCE_BASE_DIR/$entry"; } [[ -n "$working_dir" ]] || return 1 [[ -n "$project" ]] || project="$(basename "$working_dir")" @@ -56,6 +63,8 @@ resolve_project() { project_is_ready() { local project="$1" ids id status health + # Ignore one-off Compose jobs; all persistent project containers must be + # running and, where a healthcheck exists, healthy. ids="$(docker ps -a -q --filter "label=com.docker.compose.project=$project" --filter "label=com.docker.compose.oneoff=False")" [[ -n "$ids" ]] || return 1 for id in $ids; do @@ -92,9 +101,12 @@ backup_project() { mkdir -p "$output_dir" cd "$working_dir" + # Stopping the whole project creates a consistent filesystem-level backup. compose down read -r -a zstd_args <<<"$ZSTD_EXTRA_OPTS" if ! tar --one-file-system -cf - -C "$parent" "$base" | zstd "${zstd_args[@]}" -o "$output"; then + # Never leave a partial archive that a transfer stage could mistake for a + # valid backup. Make a best-effort restart before recording the failure. rm -f -- "$output" log ERROR "Archive failed; attempting to restart $project" compose up -d || true @@ -102,6 +114,8 @@ backup_project() { fi log ARCHIVE "Created $output" if is_true "$PULL_IMAGES"; then + # Image updates are useful but not allowed to prevent restart with the + # already-cached image when a registry is temporarily unavailable. compose pull || log WARN "Image pull failed for $project; starting with available images" fi compose up -d @@ -119,6 +133,8 @@ backup_project() { declare -A SEEN=() failures=0 +# Continue through the list so one failed project does not leave later projects +# without a backup. The stage still exits non-zero after reporting all failures. while IFS= read -r entry; do entry="${entry%%#*}" entry="${entry#"${entry%%[![:space:]]*}"}" diff --git a/stages/20-rclone.sh b/stages/20-rclone.sh index 84d3776..41afd86 100755 --- a/stages/20-rclone.sh +++ b/stages/20-rclone.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Copy the local backup tree to an rclone destination. +# Stage 20: copy the local backup tree to an rclone destination. +# Uses `rclone copy`, so files absent locally are not deleted remotely. set -Eeuo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" . "$SCRIPT_DIR/../lib/common.sh" @@ -18,6 +19,8 @@ require_command rclone [[ -d "$BACKUP_ROOT" ]] || die "Backup root not found: $BACKUP_ROOT" read -r -a extra_args <<<"$RCLONE_EXTRA_ARGS" destination="${RCLONE_REMOTE%/}" +# Append the host name for shared destinations, or disable RCLONE_HOST_SUBDIR +# when RCLONE_REMOTE already points at this host's final directory. is_true "$RCLONE_HOST_SUBDIR" && destination="$destination/$BACKUP_HOSTNAME" args=(copy "$BACKUP_ROOT" "$destination" "${extra_args[@]}") (( DRY_RUN )) && args+=(--dry-run) diff --git a/stages/30-smb.sh b/stages/30-smb.sh index 0ff49ae..720d8c5 100755 --- a/stages/30-smb.sh +++ b/stages/30-smb.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Copy the local backup tree to an already-mounted SMB share. +# Stage 30: copy the local backup tree to an already-mounted SMB share. +# The script does not mount the share or store SMB credentials. set -Eeuo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" . "$SCRIPT_DIR/../lib/common.sh" @@ -18,6 +19,8 @@ require_command rsync require_command mountpoint [[ -d "$BACKUP_ROOT" ]] || die "Backup root not found: $BACKUP_ROOT" [[ -d "$SMB_MOUNT" ]] || die "SMB mount does not exist: $SMB_MOUNT" +# Refuse an ordinary directory at the configured path. This prevents a failed +# network mount from silently filling the server's local root filesystem. mountpoint -q "$SMB_MOUNT" || die "SMB path is not a mounted filesystem: $SMB_MOUNT" destination="${SMB_MOUNT%/}" is_true "$SMB_HOST_SUBDIR" && destination="$destination/$BACKUP_HOSTNAME" diff --git a/stages/90-prune.sh b/stages/90-prune.sh index fe1fc5d..fd5d0f4 100755 --- a/stages/90-prune.sh +++ b/stages/90-prune.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Prune local per-project archives, retaining the newest N in each. +# Stage 90: prune local archives after successful archive and transfer stages. +# Retention is applied independently to each immediate project directory. set -Eeuo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" . "$SCRIPT_DIR/../lib/common.sh" @@ -16,7 +17,9 @@ done require_positive_integer KEEP_COUNT "$KEEP_COUNT" [[ -d "$BACKUP_ROOT" ]] || { log SKIP "Backup root does not exist: $BACKUP_ROOT"; exit 0; } while IFS= read -r -d '' dir; do - # Match both current *.tar.zst archives and legacy *.zst archives. + # Match both current *.tar.zst and legacy *.zst names. NUL delimiters keep + # paths containing whitespace safe. Files are sorted by modification time, + # newest first; the timestamp embedded in a filename is not trusted. mapfile -d '' -t files < <(find "$dir" -maxdepth 1 -type f -name '*.zst' -printf '%T@ %p\0' | sort -z -nr | cut -z -d' ' -f2-) (( ${#files[@]} > KEEP_COUNT )) || continue for file in "${files[@]:KEEP_COUNT}"; do diff --git a/stages/99-mqtt-report.sh b/stages/99-mqtt-report.sh index 02738d4..fba03fa 100755 --- a/stages/99-mqtt-report.sh +++ b/stages/99-mqtt-report.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Publish retained backup run state for Home Assistant. +# Stage 99: publish retained backup run state for Home Assistant. +# Called once at pipeline start and again from backup.sh's EXIT trap. set -Eeuo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" . "$SCRIPT_DIR/../lib/common.sh" @@ -14,6 +15,8 @@ LOG_FILE="${6:-/dev/null}" require_command jq total=0; archived=0; dry_run=0; error_count=0 +# Derive counts and error messages from the shared run log. Keeping reporting +# outside the work stages avoids coupling archive/transfer logic to MQTT. if [[ -r "$LOG_FILE" ]]; then total="$(grep -c '\[INFO\] Project:' "$LOG_FILE" || true)" archived="$(grep -c '\[ARCHIVE\] Created ' "$LOG_FILE" || true)" @@ -24,6 +27,8 @@ else errors_json='[]' fi if (( EXIT_CODE != 0 && error_count == 0 )); then + # Some commands can exit non-zero before emitting a structured [ERROR] line. + # Supply a fallback so a failed result never reports an empty error list. errors_json="$(jq -cn --arg message "Pipeline exited with status $EXIT_CODE" '[ $message ]')" error_count=1 fi @@ -39,5 +44,6 @@ payload="$(jq -cn \ duration_seconds:$duration,containers_total:$total,containers_backed_up:$archived, containers_dry_run:$dry_run,error_count:$error_count,errors:$errors}')" +# Retention gives HA the latest known state after restarts or broker reconnects. mqtt_publish "$MQTT_STATUS_TOPIC" "$payload" 1 log MQTT "Published $STATUS result to $MQTT_STATUS_TOPIC" diff --git a/systemd/zorruno-docker-backup-mqtt.service b/systemd/zorruno-docker-backup-mqtt.service index bbbb448..1e02dd0 100644 --- a/systemd/zorruno-docker-backup-mqtt.service +++ b/systemd/zorruno-docker-backup-mqtt.service @@ -5,6 +5,8 @@ Wants=network-online.target [Service] Type=simple +# Root is required because the listener launches a pipeline that stops and +# starts Docker Compose projects and reads all project data. User=root WorkingDirectory=/backupdockerscripts ExecStart=/backupdockerscripts/bin/mqtt-trigger-listener.sh