Document backup pipeline behaviour

backup.sh

@@ -1,5 +1,14 @@
 #!/usr/bin/env bash
-# Main pipeline: container archives, configured methods, then local prune.
+# Main pipeline orchestrator.
+#
+# Execution order:
+#   1. Create cold archives for the selected Docker Compose projects.
+#   2. Run each enabled transfer stage listed in BACKUP_METHODS.
+#   3. Prune old local archives only after every earlier stage succeeds.
+#
+# An EXIT trap publishes the final MQTT result even when a stage fails. The
+# original exit code is preserved so cron, systemd and shell callers can detect
+# failure normally.
 set -Eeuo pipefail
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
 . "$SCRIPT_DIR/lib/common.sh"
@@ -19,12 +28,16 @@ done
 
 BACKUP_RUN_ID="$(run_id)"
 BACKUP_STARTED_EPOCH="$(date +%s)"
+# Every script sourced through this process appends structured log lines here.
+# The MQTT reporting stage parses this temporary log to build its summary.
 BACKUP_RUN_LOG="$(mktemp "${TMPDIR:-/tmp}/zorruno-docker-backup.XXXXXX.log")"
 export BACKUP_RUN_ID BACKUP_STARTED_EPOCH BACKUP_RUN_LOG
 
 finish_run() {
   local exit_code=$? finished status
+  # Prevent recursion when this function exits with the pipeline's status.
   trap - EXIT
+  # Reporting is best-effort and must never replace the real pipeline result.
   set +e
   finished="$(date +%s)"
   if (( exit_code == 0 )); then
@@ -44,6 +57,8 @@ trap finish_run EXIT
 
 require_command flock
 mkdir -p "$(dirname "$LOCK_FILE")"
+# Keep descriptor 9 open for the whole run. Closing it automatically releases
+# the advisory lock, including on errors and signals that terminate the shell.
 exec 9>"$LOCK_FILE"
 flock -n 9 || die "Another backup run holds lock: $LOCK_FILE"
 stage_args=()
@@ -51,12 +66,15 @@ stage_args=()
 [[ -n "$CONTAINERS_OVERRIDE" ]] && stage_args+=(--containers "$CONTAINERS_OVERRIDE")
 log START "Backup pipeline for $BACKUP_HOSTNAME"
 if is_true "$MQTT_REPORT_ENABLED"; then
+  # The retained running state lets Home Assistant show that work is active.
   "$SCRIPT_DIR/stages/99-mqtt-report.sh" running 0 "$BACKUP_STARTED_EPOCH" \
     "$BACKUP_STARTED_EPOCH" "$BACKUP_RUN_ID" "$BACKUP_RUN_LOG" \
     || log WARN "Unable to publish MQTT running state; backup will continue"
 fi
 (( SKIP_CONTAINERS )) || "$SCRIPT_DIR/stages/10-containers.sh" "${stage_args[@]}"
 method_args=(); (( DRY_RUN )) && method_args+=(--dry-run)
+# BACKUP_METHODS contains short names (for example: "rclone smb"). Numeric
+# prefixes control directory ordering without becoming part of configuration.
 for method in $BACKUP_METHODS; do
   method_script=""
   while IFS= read -r candidate; do method_script="$candidate"; break; done < <(find "$SCRIPT_DIR/stages" -maxdepth 1 -type f -name "*-${method}.sh" -print | sort)

bin/mqtt-trigger-listener.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
-# Listen for an authenticated JSON command and run the backup pipeline.
+# Long-running MQTT command listener, normally managed by systemd as root.
+# Accepts only {"action":"run"}; broker authentication and ACLs should restrict
+# who can publish to MQTT_COMMAND_TOPIC.
 set -Eeuo pipefail
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)"
 . "$SCRIPT_DIR/lib/common.sh"
@@ -20,5 +22,7 @@ mosquitto_sub "${MQTT_ARGS[@]}" -R -t "$MQTT_COMMAND_TOPIC" | while IFS= read -r
     continue
   fi
   log MQTT "Accepted backup run command"
+  # backup.sh owns concurrency control. A command received during an active run
+  # fails safely on the same flock used by cron and manual invocations.
   "$SCRIPT_DIR/backup.sh" || log ERROR "MQTT-triggered backup failed"
 done

lib/common.sh

@@ -1,9 +1,15 @@
 #!/usr/bin/env bash
+# Shared configuration, validation and logging helpers.
+#
+# Every executable script sources this file. Defaults are deliberately safe:
+# transfer methods and MQTT reporting are disabled until enabled in the
+# host-local, Git-ignored .env file.
 set -Eeuo pipefail
 
 SCRIPT_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)"
 ENV_FILE="${DOCKERBACKUP_ENV:-$SCRIPT_ROOT/.env}"
 
+# Core archive pipeline defaults.
 SOURCE_BASE_DIR="/dockervolumes"
 BACKUP_ROOT="/dockerbackups"
 CONTAINERS_FILE="$SCRIPT_ROOT/containers.txt"
@@ -16,6 +22,7 @@ WAIT_INTERVAL=3
 START_RETRIES=1
 LOCK_FILE="/run/lock/zorruno-docker-backup.lock"
 ZSTD_EXTRA_OPTS="-T0 -12"
+# Optional MQTT reporting and command-listener defaults.
 MQTT_REPORT_ENABLED=0
 MQTT_HOST=""
 MQTT_PORT=1883
@@ -24,6 +31,7 @@ MQTT_PASSWORD_FILE=""
 MQTT_STATUS_TOPIC="viewroad-status/dockerbackup/$BACKUP_HOSTNAME"
 MQTT_COMMAND_TOPIC="viewroad-commands/dockerbackup/$BACKUP_HOSTNAME/run"
 
+# Transfer-stage defaults. Host-specific destinations belong in .env.
 RCLONE_ENABLED=0
 RCLONE_REMOTE=""
 RCLONE_HOST_SUBDIR=1
@@ -34,6 +42,7 @@ SMB_HOST_SUBDIR=1
 SMB_RSYNC_ARGS="-a"
 
 if [[ -f "$ENV_FILE" ]]; then
+  # Export loaded values so child stage scripts inherit the same configuration.
   set -a
   # shellcheck disable=SC1090
   . "$ENV_FILE"
@@ -50,6 +59,8 @@ log() {
   local line
   line="$(timestamp) [$1] $2"
   printf '%s\n' "$line" >&2
+  # Directly executed stages have no BACKUP_RUN_LOG. Always return success so
+  # an informational log call cannot terminate a script running with set -e.
   if [[ -n "${BACKUP_RUN_LOG:-}" ]]; then
     printf '%s\n' "$line" >>"$BACKUP_RUN_LOG"
   fi
@@ -73,6 +84,8 @@ require_positive_integer() {
 }
 
 resolve_path() {
+  # Host-local filenames are resolved relative to the repository, not the
+  # caller's current directory. Absolute paths pass through unchanged.
   if [[ "$1" = /* ]]; then printf '%s\n' "$1"; else printf '%s/%s\n' "$SCRIPT_ROOT" "$1"; fi
 }
 

lib/mqtt.sh

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Shared mosquitto_pub/mosquitto_sub argument construction.
+# Passwords are read from a protected file rather than placed in .env or Git.
 
 mqtt_common_args() {
   MQTT_ARGS=(-h "$MQTT_HOST" -p "$MQTT_PORT")
@@ -14,6 +16,8 @@ mqtt_publish() {
   [[ -n "$MQTT_HOST" ]] || die "MQTT_HOST is not configured"
   require_command mosquitto_pub
   mqtt_common_args
+  # Backup state is retained so Home Assistant receives the latest result after
+  # reconnecting. Command messages are handled separately and are never kept.
   (( retain )) && MQTT_ARGS+=(-r)
   mosquitto_pub "${MQTT_ARGS[@]}" -t "$topic" -m "$payload"
 }

stages/10-containers.sh

@@ -1,5 +1,9 @@
 #!/usr/bin/env bash
-# Stop, archive, update, start, and verify each listed Compose project.
+# Stage 10: create a cold, compressed archive of each selected Compose project.
+#
+# Entries may be Compose project-directory names or live container names. A
+# container name is resolved through its Compose labels. Duplicate entries that
+# resolve to the same working directory are processed only once.
 set -Eeuo pipefail
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
 # shellcheck source=../lib/common.sh
@@ -44,10 +48,13 @@ compose() { "${COMPOSE[@]}" "$@"; }
 
 resolve_project() {
   local entry="$1" working_dir="" project=""
+  # Prefer authoritative Compose labels when the entry names a container.
   if docker inspect "$entry" >/dev/null 2>&1; then
     working_dir="$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project.working_dir"}}' "$entry" 2>/dev/null || true)"
     project="$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}' "$entry" 2>/dev/null || true)"
   fi
+  # Fall back to a project directory beneath SOURCE_BASE_DIR. This also allows
+  # useful dry-runs when the caller cannot access the Docker socket.
   [[ -n "$working_dir" ]] || { [[ -d "$SOURCE_BASE_DIR/$entry" ]] && working_dir="$SOURCE_BASE_DIR/$entry"; }
   [[ -n "$working_dir" ]] || return 1
   [[ -n "$project" ]] || project="$(basename "$working_dir")"
@@ -56,6 +63,8 @@ resolve_project() {
 
 project_is_ready() {
   local project="$1" ids id status health
+  # Ignore one-off Compose jobs; all persistent project containers must be
+  # running and, where a healthcheck exists, healthy.
   ids="$(docker ps -a -q --filter "label=com.docker.compose.project=$project" --filter "label=com.docker.compose.oneoff=False")"
   [[ -n "$ids" ]] || return 1
   for id in $ids; do
@@ -92,9 +101,12 @@ backup_project() {
 
   mkdir -p "$output_dir"
   cd "$working_dir"
+  # Stopping the whole project creates a consistent filesystem-level backup.
   compose down
   read -r -a zstd_args <<<"$ZSTD_EXTRA_OPTS"
   if ! tar --one-file-system -cf - -C "$parent" "$base" | zstd "${zstd_args[@]}" -o "$output"; then
+    # Never leave a partial archive that a transfer stage could mistake for a
+    # valid backup. Make a best-effort restart before recording the failure.
     rm -f -- "$output"
     log ERROR "Archive failed; attempting to restart $project"
     compose up -d || true
@@ -102,6 +114,8 @@ backup_project() {
   fi
   log ARCHIVE "Created $output"
   if is_true "$PULL_IMAGES"; then
+    # Image updates are useful but not allowed to prevent restart with the
+    # already-cached image when a registry is temporarily unavailable.
     compose pull || log WARN "Image pull failed for $project; starting with available images"
   fi
   compose up -d
@@ -119,6 +133,8 @@ backup_project() {
 
 declare -A SEEN=()
 failures=0
+# Continue through the list so one failed project does not leave later projects
+# without a backup. The stage still exits non-zero after reporting all failures.
 while IFS= read -r entry; do
   entry="${entry%%#*}"
   entry="${entry#"${entry%%[![:space:]]*}"}"

stages/20-rclone.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-# Copy the local backup tree to an rclone destination.
+# Stage 20: copy the local backup tree to an rclone destination.
+# Uses `rclone copy`, so files absent locally are not deleted remotely.
 set -Eeuo pipefail
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
 . "$SCRIPT_DIR/../lib/common.sh"
@@ -18,6 +19,8 @@ require_command rclone
 [[ -d "$BACKUP_ROOT" ]] || die "Backup root not found: $BACKUP_ROOT"
 read -r -a extra_args <<<"$RCLONE_EXTRA_ARGS"
 destination="${RCLONE_REMOTE%/}"
+# Append the host name for shared destinations, or disable RCLONE_HOST_SUBDIR
+# when RCLONE_REMOTE already points at this host's final directory.
 is_true "$RCLONE_HOST_SUBDIR" && destination="$destination/$BACKUP_HOSTNAME"
 args=(copy "$BACKUP_ROOT" "$destination" "${extra_args[@]}")
 (( DRY_RUN )) && args+=(--dry-run)

stages/30-smb.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-# Copy the local backup tree to an already-mounted SMB share.
+# Stage 30: copy the local backup tree to an already-mounted SMB share.
+# The script does not mount the share or store SMB credentials.
 set -Eeuo pipefail
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
 . "$SCRIPT_DIR/../lib/common.sh"
@@ -18,6 +19,8 @@ require_command rsync
 require_command mountpoint
 [[ -d "$BACKUP_ROOT" ]] || die "Backup root not found: $BACKUP_ROOT"
 [[ -d "$SMB_MOUNT" ]] || die "SMB mount does not exist: $SMB_MOUNT"
+# Refuse an ordinary directory at the configured path. This prevents a failed
+# network mount from silently filling the server's local root filesystem.
 mountpoint -q "$SMB_MOUNT" || die "SMB path is not a mounted filesystem: $SMB_MOUNT"
 destination="${SMB_MOUNT%/}"
 is_true "$SMB_HOST_SUBDIR" && destination="$destination/$BACKUP_HOSTNAME"

stages/90-prune.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-# Prune local per-project archives, retaining the newest N in each.
+# Stage 90: prune local archives after successful archive and transfer stages.
+# Retention is applied independently to each immediate project directory.
 set -Eeuo pipefail
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
 . "$SCRIPT_DIR/../lib/common.sh"
@@ -16,7 +17,9 @@ done
 require_positive_integer KEEP_COUNT "$KEEP_COUNT"
 [[ -d "$BACKUP_ROOT" ]] || { log SKIP "Backup root does not exist: $BACKUP_ROOT"; exit 0; }
 while IFS= read -r -d '' dir; do
-  # Match both current *.tar.zst archives and legacy *.zst archives.
+  # Match both current *.tar.zst and legacy *.zst names. NUL delimiters keep
+  # paths containing whitespace safe. Files are sorted by modification time,
+  # newest first; the timestamp embedded in a filename is not trusted.
   mapfile -d '' -t files < <(find "$dir" -maxdepth 1 -type f -name '*.zst' -printf '%T@ %p\0' | sort -z -nr | cut -z -d' ' -f2-)
   (( ${#files[@]} > KEEP_COUNT )) || continue
   for file in "${files[@]:KEEP_COUNT}"; do

stages/99-mqtt-report.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-# Publish retained backup run state for Home Assistant.
+# Stage 99: publish retained backup run state for Home Assistant.
+# Called once at pipeline start and again from backup.sh's EXIT trap.
 set -Eeuo pipefail
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
 . "$SCRIPT_DIR/../lib/common.sh"
@@ -14,6 +15,8 @@ LOG_FILE="${6:-/dev/null}"
 
 require_command jq
 total=0; archived=0; dry_run=0; error_count=0
+# Derive counts and error messages from the shared run log. Keeping reporting
+# outside the work stages avoids coupling archive/transfer logic to MQTT.
 if [[ -r "$LOG_FILE" ]]; then
   total="$(grep -c '\[INFO\] Project:' "$LOG_FILE" || true)"
   archived="$(grep -c '\[ARCHIVE\] Created ' "$LOG_FILE" || true)"
@@ -24,6 +27,8 @@ else
   errors_json='[]'
 fi
 if (( EXIT_CODE != 0 && error_count == 0 )); then
+  # Some commands can exit non-zero before emitting a structured [ERROR] line.
+  # Supply a fallback so a failed result never reports an empty error list.
   errors_json="$(jq -cn --arg message "Pipeline exited with status $EXIT_CODE" '[ $message ]')"
   error_count=1
 fi
@@ -39,5 +44,6 @@ payload="$(jq -cn \
     duration_seconds:$duration,containers_total:$total,containers_backed_up:$archived,
     containers_dry_run:$dry_run,error_count:$error_count,errors:$errors}')"
 
+# Retention gives HA the latest known state after restarts or broker reconnects.
 mqtt_publish "$MQTT_STATUS_TOPIC" "$payload" 1
 log MQTT "Published $STATUS result to $MQTT_STATUS_TOPIC"

systemd/zorruno-docker-backup-mqtt.service

@@ -5,6 +5,8 @@ Wants=network-online.target
 
 [Service]
 Type=simple
+# Root is required because the listener launches a pipeline that stops and
+# starts Docker Compose projects and reads all project data.
 User=root
 WorkingDirectory=/backupdockerscripts
 ExecStart=/backupdockerscripts/bin/mqtt-trigger-listener.sh