Remove personal values from environment example

.env.example

@@ -2,7 +2,7 @@
 SOURCE_BASE_DIR="/dockervolumes"
 BACKUP_ROOT="/dockerbackups"
 CONTAINERS_FILE="containers.local.txt"
-BACKUP_HOSTNAME="beaver"
+BACKUP_HOSTNAME="examplehostname"
 
 # Main pipeline method order. Add future method names here.
 BACKUP_METHODS="rclone smb"
@@ -16,21 +16,21 @@ ZSTD_EXTRA_OPTS="-T0 -12"
 
 # Optional retained MQTT status and command listener settings.
 MQTT_REPORT_ENABLED=0
-MQTT_HOST="beaver"
+MQTT_HOST="examplehostname"
 MQTT_PORT=1883
 MQTT_USERNAME="dockerbackup"
-MQTT_PASSWORD_FILE="/etc/zorruno-docker-backup/mqtt-password"
+MQTT_PASSWORD_FILE="/etc/docker-backup/mqtt-password"
-MQTT_STATUS_TOPIC="viewroad-status/dockerbackup/beaver"
+MQTT_STATUS_TOPIC="backup-status/dockerbackup/examplehostname"
-MQTT_COMMAND_TOPIC="viewroad-commands/dockerbackup/beaver/run"
+MQTT_COMMAND_TOPIC="backup-commands/dockerbackup/examplehostname/run"
 
-# Rclone destination becomes <remote>/beaver/.
+# Rclone destination becomes <remote>/examplehostname/.
 RCLONE_ENABLED=0
 RCLONE_REMOTE="gdrive:dockerbackups"
 RCLONE_HOST_SUBDIR=1
 RCLONE_EXTRA_ARGS=""
 
 # The SMB share must already be mounted by the operating system.
-# Destination becomes <mount>/beaver/ when SMB_HOST_SUBDIR=1.
+# Destination becomes <mount>/examplehostname/ when SMB_HOST_SUBDIR=1.
 SMB_ENABLED=0
 SMB_MOUNT="/mnt/dockerbackups"
 SMB_HOST_SUBDIR=1

README.md

@@ -1,7 +1,7 @@
 # Zorruno Docker Backup Scripts
 
-Modular backup scripts for Docker Compose servers such as Panda and Beaver.
+Modular backup scripts for Docker Compose servers.
-Each server has its own untracked `.env` and `containers.txt`, while all script
+Each server has its own untracked `.env` and `containers.local.txt`, while all script
 logic remains common and can be updated through Git.
 
 ## Pipeline
@@ -9,8 +9,8 @@ logic remains common and can be updated through Git.
 `backup.sh` runs these stages in order:
 
 1. `stages/10-containers.sh` — stop, archive, pull, start and verify projects.
-2. `methods/20-rclone.sh` — copy archives to an rclone destination when enabled.
+2. `stages/20-rclone.sh` — copy archives to an rclone destination when enabled.
-3. `methods/30-smb.sh` — copy archives to a mounted SMB share when enabled.
+3. `stages/30-smb.sh` — copy archives to a mounted SMB share when enabled.
 4. `stages/90-prune.sh` — retain the newest configured number of `.zst`
    archives locally, including legacy `.zst` and current `.tar.zst` names.
 
@@ -30,7 +30,7 @@ prune stage. Use the new `backup.sh` for the complete pipeline.
 The account running the scripts must be allowed to control Docker and read all
 files under `/dockervolumes`.
 
-## Installation on Beaver
+## Installation on a server
 
 ```bash
 sudo git clone https://home.fox.co.nz/gitea/zorruno/zorruno-backupdockerscript.git /backupdockerscripts
@@ -38,25 +38,28 @@ cd /backupdockerscripts
 sudo cp .env.example .env
 sudo cp containers.txt containers.local.txt
 sudo chmod 600 .env
-sudo chmod +x backup.sh docker-backup.sh dockerbackup_prune.sh stages/*.sh methods/*.sh
+sudo chmod +x backup.sh stages/*.sh
 ```
 
-Edit `.env`. Beaver's Docker source is already represented by:
+Edit `.env`. A typical Docker source and example host name are represented by:
 
 ```bash
 SOURCE_BASE_DIR="/dockervolumes"
-BACKUP_HOSTNAME="beaver"
+BACKUP_HOSTNAME="examplehostname"
 ```
 
 Choose and create a local staging path for `BACKUP_ROOT`. `/dockerbackups` is
 the default, but it should be on a filesystem with enough free space to hold
 at least `KEEP_COUNT` copies of every listed project.
 
-Edit Beaver's untracked `containers.local.txt`. Each active line can be either a running
+Edit the server's untracked `containers.local.txt`. Each active line can be either a running
 container name or a directory name beneath `/dockervolumes`. Blank lines and
 text after `#` are ignored. If several entries resolve to the same Compose
 project directory, that project is backed up only once. Keeping this host-local
-file out of Git prevents a pull on Panda or Beaver from replacing its list.
+file out of Git prevents a pull on any server from replacing its local list.
+When a listed directory contains other running Compose projects, the archive
+stage also stops those nested projects before archiving and starts them again
+afterward. Intentionally stopped nested projects are left stopped.
 
 ## Configuration
 
@@ -127,8 +130,8 @@ Run individual parts:
 
 ```bash
 sudo stages/10-containers.sh --dry-run
-sudo methods/20-rclone.sh --dry-run --force
+sudo stages/20-rclone.sh --dry-run --force
-sudo methods/30-smb.sh --dry-run --force
+sudo stages/30-smb.sh --dry-run --force
 sudo stages/90-prune.sh --dry-run
 ```
 
@@ -141,12 +144,12 @@ Enable retained run results in the host's `.env`:
 
 ```bash
 MQTT_REPORT_ENABLED=1
-MQTT_HOST="beaver"
+MQTT_HOST="examplehostname"
 MQTT_PORT=1883
 MQTT_USERNAME="dockerbackup"
 MQTT_PASSWORD_FILE="/etc/zorruno-docker-backup/mqtt-password"
-MQTT_STATUS_TOPIC="viewroad-status/dockerbackup/panda"
+MQTT_STATUS_TOPIC="viewroad-status/dockerbackup/examplehostname"
-MQTT_COMMAND_TOPIC="viewroad-commands/dockerbackup/panda/run"
+MQTT_COMMAND_TOPIC="viewroad-commands/dockerbackup/examplehostname/run"
 ```
 
 The retained status JSON includes overall status, timestamps, duration, exit
@@ -179,8 +182,8 @@ pipeline lock prevents concurrent runs.
 
 ## Adding another backup method
 
-Add an executable script named `methods/NN-name.sh`, for example
+Add an executable script named `stages/NN-name.sh`, for example
-`methods/40-restic.sh`. It should load `lib/common.sh`, accept `--dry-run`, and
+`stages/40-restic.sh`. It should load `lib/common.sh`, accept `--dry-run`, and
 return a non-zero status on failure. Add its short name to `.env`:
 
 ```bash

backup.sh

@@ -1,5 +1,14 @@
 #!/usr/bin/env bash
-# Main pipeline: container archives, configured methods, then local prune.
+# Main pipeline orchestrator.
+#
+# Execution order:
+#   1. Create cold archives for the selected Docker Compose projects.
+#   2. Run each enabled transfer stage listed in BACKUP_METHODS.
+#   3. Prune old local archives only after every earlier stage succeeds.
+#
+# An EXIT trap publishes the final MQTT result even when a stage fails. The
+# original exit code is preserved so cron, systemd and shell callers can detect
+# failure normally.
 set -Eeuo pipefail
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
 . "$SCRIPT_DIR/lib/common.sh"
@@ -10,23 +19,86 @@ while [[ $# -gt 0 ]]; do
     --containers|-f) CONTAINERS_OVERRIDE="${2:?Missing containers file}"; shift ;;
     -h|--help)
       echo "Usage: ${0##*/} [--dry-run] [--containers FILE] [--skip-containers] [--skip-prune]"
-      echo "Methods run in BACKUP_METHODS order. Add methods/NN-name.sh and list name in BACKUP_METHODS."
+      echo "Transfer stages run in BACKUP_METHODS order. Add stages/NN-name.sh and list name in BACKUP_METHODS."
       exit 0 ;;
     *) die "Unknown option: $1" 2 ;;
   esac
   shift
 done
 
+if (( SKIP_CONTAINERS )); then
+  BACKUP_LOCAL_ARCHIVE_STATUS="skipped"
+else
+  BACKUP_LOCAL_ARCHIVE_STATUS="running"
+fi
+BACKUP_RCLONE_STATUS="not_configured"
+BACKUP_SMB_STATUS="not_configured"
+for configured_method in $BACKUP_METHODS; do
+  case "$configured_method" in
+    rclone) if is_true "$RCLONE_ENABLED"; then BACKUP_RCLONE_STATUS="pending"; else BACKUP_RCLONE_STATUS="skipped"; fi ;;
+    smb) if is_true "$SMB_ENABLED"; then BACKUP_SMB_STATUS="pending"; else BACKUP_SMB_STATUS="skipped"; fi ;;
+  esac
+done
+BACKUP_REMOTE_COPY_STATUS="pending"
+if [[ "$BACKUP_RCLONE_STATUS" == "not_configured" || "$BACKUP_RCLONE_STATUS" == "skipped" ]] && [[ "$BACKUP_SMB_STATUS" == "not_configured" || "$BACKUP_SMB_STATUS" == "skipped" ]]; then
+  BACKUP_REMOTE_COPY_STATUS="skipped"
+fi
+BACKUP_REMOTE_COPY_FINISHED_AT=""
+export BACKUP_LOCAL_ARCHIVE_STATUS BACKUP_REMOTE_COPY_STATUS BACKUP_RCLONE_STATUS BACKUP_SMB_STATUS BACKUP_REMOTE_COPY_FINISHED_AT
+
 BACKUP_RUN_ID="$(run_id)"
 BACKUP_STARTED_EPOCH="$(date +%s)"
+# Every script sourced through this process appends structured log lines here.
+# The MQTT reporting stage parses this temporary log to build its summary.
 BACKUP_RUN_LOG="$(mktemp "${TMPDIR:-/tmp}/zorruno-docker-backup.XXXXXX.log")"
 export BACKUP_RUN_ID BACKUP_STARTED_EPOCH BACKUP_RUN_LOG
 
 finish_run() {
-  local exit_code=$? finished status
+  local exit_code=$? finished status archived_count remote_success=0 remote_failure=0 remote_running=0 remote_dry_run=0 transfer_status
+  # Prevent recursion when this function exits with the pipeline's status.
   trap - EXIT
+  # Reporting is best-effort and must never replace the real pipeline result.
   set +e
   finished="$(date +%s)"
+  if [[ "$BACKUP_LOCAL_ARCHIVE_STATUS" == "running" ]]; then
+    if (( DRY_RUN )); then
+      BACKUP_LOCAL_ARCHIVE_STATUS="dry_run"
+    elif (( exit_code == 0 )); then
+      BACKUP_LOCAL_ARCHIVE_STATUS="success"
+    else
+      archived_count="$(grep -c "\[ARCHIVE\] Created " "$BACKUP_RUN_LOG" || true)"
+      if (( archived_count > 0 )); then BACKUP_LOCAL_ARCHIVE_STATUS="partial"; else BACKUP_LOCAL_ARCHIVE_STATUS="failed"; fi
+    fi
+  fi
+  for transfer_status in BACKUP_RCLONE_STATUS BACKUP_SMB_STATUS; do
+    if [[ "${!transfer_status}" == "running" ]]; then
+      if (( DRY_RUN )); then printf -v "$transfer_status" "%s" "dry_run"; else printf -v "$transfer_status" "%s" "failed"; fi
+    elif [[ "${!transfer_status}" == "pending" ]]; then
+      printf -v "$transfer_status" "%s" "not_attempted"
+    fi
+  done
+  for transfer_status in "$BACKUP_RCLONE_STATUS" "$BACKUP_SMB_STATUS"; do
+    case "$transfer_status" in
+      success) (( remote_success++ )) ;;
+      partial) (( remote_success++ )); (( remote_failure++ )) ;;
+      failed) (( remote_failure++ )) ;;
+      running|pending) (( remote_running++ )) ;;
+      dry_run) (( remote_dry_run++ )) ;;
+    esac
+  done
+  if (( remote_running > 0 )); then
+    BACKUP_REMOTE_COPY_STATUS="running"
+  elif (( remote_failure > 0 && remote_success > 0 )); then
+    BACKUP_REMOTE_COPY_STATUS="partial"
+  elif (( remote_failure > 0 )); then
+    BACKUP_REMOTE_COPY_STATUS="failed"
+  elif (( remote_success > 0 )); then
+    BACKUP_REMOTE_COPY_STATUS="success"
+  elif (( remote_dry_run > 0 )); then
+    BACKUP_REMOTE_COPY_STATUS="dry_run"
+  else
+    BACKUP_REMOTE_COPY_STATUS="skipped"
+  fi
   if (( exit_code == 0 )); then
     if (( DRY_RUN )); then status="dry_run_success"; else status="success"; fi
   else
@@ -44,6 +116,8 @@ trap finish_run EXIT
 
 require_command flock
 mkdir -p "$(dirname "$LOCK_FILE")"
+# Keep descriptor 9 open for the whole run. Closing it automatically releases
+# the advisory lock, including on errors and signals that terminate the shell.
 exec 9>"$LOCK_FILE"
 flock -n 9 || die "Another backup run holds lock: $LOCK_FILE"
 stage_args=()
@@ -51,17 +125,40 @@ stage_args=()
 [[ -n "$CONTAINERS_OVERRIDE" ]] && stage_args+=(--containers "$CONTAINERS_OVERRIDE")
 log START "Backup pipeline for $BACKUP_HOSTNAME"
 if is_true "$MQTT_REPORT_ENABLED"; then
+  # The retained running state lets Home Assistant show that work is active.
   "$SCRIPT_DIR/stages/99-mqtt-report.sh" running 0 "$BACKUP_STARTED_EPOCH" \
     "$BACKUP_STARTED_EPOCH" "$BACKUP_RUN_ID" "$BACKUP_RUN_LOG" \
     || log WARN "Unable to publish MQTT running state; backup will continue"
 fi
-(( SKIP_CONTAINERS )) || "$SCRIPT_DIR/stages/10-containers.sh" "${stage_args[@]}"
+if (( SKIP_CONTAINERS )); then
+  BACKUP_LOCAL_ARCHIVE_STATUS="skipped"
+else
+  "$SCRIPT_DIR/stages/10-containers.sh" "${stage_args[@]}"
+  if (( DRY_RUN )); then BACKUP_LOCAL_ARCHIVE_STATUS="dry_run"; else BACKUP_LOCAL_ARCHIVE_STATUS="success"; fi
+fi
 method_args=(); (( DRY_RUN )) && method_args+=(--dry-run)
+# BACKUP_METHODS contains short names (for example: "rclone smb"). Numeric
+# prefixes control directory ordering without becoming part of configuration.
 for method in $BACKUP_METHODS; do
   method_script=""
-  while IFS= read -r candidate; do method_script="$candidate"; break; done < <(find "$SCRIPT_DIR/methods" -maxdepth 1 -type f -name "*-${method}.sh" -print | sort)
+  while IFS= read -r candidate; do method_script="$candidate"; break; done < <(find "$SCRIPT_DIR/stages" -maxdepth 1 -type f -name "*-${method}.sh" -print | sort)
   [[ -n "$method_script" ]] || die "Configured backup method not found: $method"
+  case "$method" in
+    rclone) if is_true "$RCLONE_ENABLED"; then BACKUP_RCLONE_STATUS="running"; fi ;;
+    smb) if is_true "$SMB_ENABLED"; then BACKUP_SMB_STATUS="running"; fi ;;
+  esac
   "$method_script" "${method_args[@]}"
+  if (( DRY_RUN )); then
+    case "$method" in
+      rclone) [[ "$BACKUP_RCLONE_STATUS" == "running" ]] && BACKUP_RCLONE_STATUS="dry_run" ;;
+      smb) [[ "$BACKUP_SMB_STATUS" == "running" ]] && BACKUP_SMB_STATUS="dry_run" ;;
+    esac
+  else
+    case "$method" in
+      rclone) if [[ "$BACKUP_RCLONE_STATUS" == "running" ]]; then BACKUP_RCLONE_STATUS="success"; BACKUP_REMOTE_COPY_FINISHED_AT="$(date -u +%FT%TZ)"; fi ;;
+      smb) if [[ "$BACKUP_SMB_STATUS" == "running" ]]; then BACKUP_SMB_STATUS="success"; BACKUP_REMOTE_COPY_FINISHED_AT="$(date -u +%FT%TZ)"; fi ;;
+    esac
+  fi
 done
 (( SKIP_PRUNE )) || "$SCRIPT_DIR/stages/90-prune.sh" "${method_args[@]}"
 log DONE "Backup pipeline completed successfully"

bin/mqtt-trigger-listener.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
-# Listen for an authenticated JSON command and run the backup pipeline.
+# Long-running MQTT command listener, normally managed by systemd as root.
+# Accepts only {"action":"run"}; broker authentication and ACLs should restrict
+# who can publish to MQTT_COMMAND_TOPIC.
 set -Eeuo pipefail
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)"
 . "$SCRIPT_DIR/lib/common.sh"
@@ -20,5 +22,7 @@ mosquitto_sub "${MQTT_ARGS[@]}" -R -t "$MQTT_COMMAND_TOPIC" | while IFS= read -r
     continue
   fi
   log MQTT "Accepted backup run command"
+  # backup.sh owns concurrency control. A command received during an active run
+  # fails safely on the same flock used by cron and manual invocations.
   "$SCRIPT_DIR/backup.sh" || log ERROR "MQTT-triggered backup failed"
 done

lib/common.sh

@@ -1,9 +1,15 @@
 #!/usr/bin/env bash
+# Shared configuration, validation and logging helpers.
+#
+# Every executable script sources this file. Defaults are deliberately safe:
+# transfer methods and MQTT reporting are disabled until enabled in the
+# host-local, Git-ignored .env file.
 set -Eeuo pipefail
 
 SCRIPT_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)"
 ENV_FILE="${DOCKERBACKUP_ENV:-$SCRIPT_ROOT/.env}"
 
+# Core archive pipeline defaults.
 SOURCE_BASE_DIR="/dockervolumes"
 BACKUP_ROOT="/dockerbackups"
 CONTAINERS_FILE="$SCRIPT_ROOT/containers.txt"
@@ -16,6 +22,7 @@ WAIT_INTERVAL=3
 START_RETRIES=1
 LOCK_FILE="/run/lock/zorruno-docker-backup.lock"
 ZSTD_EXTRA_OPTS="-T0 -12"
+# Optional MQTT reporting and command-listener defaults.
 MQTT_REPORT_ENABLED=0
 MQTT_HOST=""
 MQTT_PORT=1883
@@ -24,6 +31,7 @@ MQTT_PASSWORD_FILE=""
 MQTT_STATUS_TOPIC="viewroad-status/dockerbackup/$BACKUP_HOSTNAME"
 MQTT_COMMAND_TOPIC="viewroad-commands/dockerbackup/$BACKUP_HOSTNAME/run"
 
+# Transfer-stage defaults. Host-specific destinations belong in .env.
 RCLONE_ENABLED=0
 RCLONE_REMOTE=""
 RCLONE_HOST_SUBDIR=1
@@ -34,6 +42,7 @@ SMB_HOST_SUBDIR=1
 SMB_RSYNC_ARGS="-a"
 
 if [[ -f "$ENV_FILE" ]]; then
+  # Export loaded values so child stage scripts inherit the same configuration.
   set -a
   # shellcheck disable=SC1090
   . "$ENV_FILE"
@@ -50,6 +59,8 @@ log() {
   local line
   line="$(timestamp) [$1] $2"
   printf '%s\n' "$line" >&2
+  # Directly executed stages have no BACKUP_RUN_LOG. Always return success so
+  # an informational log call cannot terminate a script running with set -e.
   if [[ -n "${BACKUP_RUN_LOG:-}" ]]; then
     printf '%s\n' "$line" >>"$BACKUP_RUN_LOG"
   fi
@@ -73,6 +84,8 @@ require_positive_integer() {
 }
 
 resolve_path() {
+  # Host-local filenames are resolved relative to the repository, not the
+  # caller's current directory. Absolute paths pass through unchanged.
   if [[ "$1" = /* ]]; then printf '%s\n' "$1"; else printf '%s/%s\n' "$SCRIPT_ROOT" "$1"; fi
 }
 

lib/mqtt.sh

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Shared mosquitto_pub/mosquitto_sub argument construction.
+# Passwords are read from a protected file rather than placed in .env or Git.
 
 mqtt_common_args() {
   MQTT_ARGS=(-h "$MQTT_HOST" -p "$MQTT_PORT")
@@ -14,6 +16,8 @@ mqtt_publish() {
   [[ -n "$MQTT_HOST" ]] || die "MQTT_HOST is not configured"
   require_command mosquitto_pub
   mqtt_common_args
+  # Backup state is retained so Home Assistant receives the latest result after
+  # reconnecting. Command messages are handled separately and are never kept.
   (( retain )) && MQTT_ARGS+=(-r)
   mosquitto_pub "${MQTT_ARGS[@]}" -t "$topic" -m "$payload"
 }

stages/10-containers.sh

@@ -1,5 +1,10 @@
 #!/usr/bin/env bash
-# Stop, archive, update, start, and verify each listed Compose project.
+# Stage 10: create a cold, compressed archive of each selected Compose project
+# directory, including any nested Compose projects.
+#
+# Entries may be Compose project-directory names or live container names. A
+# container name is resolved through its Compose labels. Duplicate entries that
+# resolve to the same working directory are processed only once.
 set -Eeuo pipefail
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
 # shellcheck source=../lib/common.sh
@@ -44,10 +49,13 @@ compose() { "${COMPOSE[@]}" "$@"; }
 
 resolve_project() {
   local entry="$1" working_dir="" project=""
+  # Prefer authoritative Compose labels when the entry names a container.
   if docker inspect "$entry" >/dev/null 2>&1; then
     working_dir="$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project.working_dir"}}' "$entry" 2>/dev/null || true)"
     project="$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}' "$entry" 2>/dev/null || true)"
   fi
+  # Fall back to a project directory beneath SOURCE_BASE_DIR. This also allows
+  # useful dry-runs when the caller cannot access the Docker socket.
   [[ -n "$working_dir" ]] || { [[ -d "$SOURCE_BASE_DIR/$entry" ]] && working_dir="$SOURCE_BASE_DIR/$entry"; }
   [[ -n "$working_dir" ]] || return 1
   [[ -n "$project" ]] || project="$(basename "$working_dir")"
@@ -56,6 +64,8 @@ resolve_project() {
 
 project_is_ready() {
   local project="$1" ids id status health
+  # Ignore one-off Compose jobs; all persistent project containers must be
+  # running and, where a healthcheck exists, healthy.
   ids="$(docker ps -a -q --filter "label=com.docker.compose.project=$project" --filter "label=com.docker.compose.oneoff=False")"
   [[ -n "$ids" ]] || return 1
   for id in $ids; do
@@ -76,9 +86,41 @@ wait_for_project() {
   done
 }
 
+discover_child_projects() {
+  local root="$1" id row project working_dir
+  local -A seen=()
+  # Discover live Compose projects by their container labels. Only running
+  # projects are stopped/restarted, preserving intentionally stopped stacks.
+  while IFS= read -r id; do
+    [[ -n "$id" ]] || continue
+    row="$(docker inspect -f '{{printf "%s\t%s" (index .Config.Labels "com.docker.compose.project") (index .Config.Labels "com.docker.compose.project.working_dir")}}' "$id" 2>/dev/null || true)"
+    IFS=$'\t' read -r project working_dir <<<"$row"
+    [[ -n "$project" && -n "$working_dir" ]] || continue
+    [[ "$working_dir" == "$root"/* ]] || continue
+    [[ -n "${seen[$working_dir]:-}" ]] && continue
+    seen[$working_dir]=1
+    printf '%s\t%s\n' "$project" "$working_dir"
+  done < <(docker ps -q)
+}
+
+restart_children() {
+  local -a projects=("$@")
+  local record child_project child_dir failed=0
+  for record in "${projects[@]}"; do
+    IFS=$'\t' read -r child_project child_dir <<<"$record"
+    log INFO "Restarting nested project: $child_project ($child_dir)"
+    if ! (cd "$child_dir" && compose up -d); then
+      log ERROR "Unable to restart nested project $child_project"
+      failed=1
+    fi
+  done
+  return "$failed"
+}
+
 backup_project() {
-  local project="$1" working_dir="$2" parent base output_dir output attempt
+  local project="$1" working_dir="$2" parent base output_dir output attempt child_project child_dir record
   local -a zstd_args=()
+  local -a child_records=() stopped_children=()
   parent="$(dirname "$working_dir")"
   base="$(basename "$working_dir")"
   output_dir="$BACKUP_ROOT/$project"
@@ -86,25 +128,52 @@ backup_project() {
   log INFO "Project: $project ($working_dir)"
   log INFO "Archive: $output"
   if (( DRY_RUN )); then
-    log DRY "Would stop, archive, optionally pull, start, and verify $project"
+    mapfile -t child_records < <(discover_child_projects "$working_dir")
+    log DRY "Would stop ${#child_records[@]} nested project(s), archive, then restart them with $project"
     return 0
   fi
 
   mkdir -p "$output_dir"
+  mapfile -t child_records < <(discover_child_projects "$working_dir")
+  for record in "${child_records[@]}"; do
+    IFS=$'\t' read -r child_project child_dir <<<"$record"
+    log INFO "Stopping nested project: $child_project ($child_dir)"
+    # Track before down: Compose may partially stop a project and then fail.
+    stopped_children+=("$record")
+    if (cd "$child_dir" && compose down); then
+      :
+    else
+      log ERROR "Unable to stop nested project $child_project"
+      restart_children "${stopped_children[@]}" || true
+      return 1
+    fi
+  done
   cd "$working_dir"
-  compose down
+  # Stopping the whole project creates a consistent filesystem-level backup.
+  if ! compose down; then
+    log ERROR "Unable to stop project $project"
+    compose up -d || log ERROR "Unable to restart project $project"
+    restart_children "${stopped_children[@]}" || true
+    return 1
+  fi
   read -r -a zstd_args <<<"$ZSTD_EXTRA_OPTS"
   if ! tar --one-file-system -cf - -C "$parent" "$base" | zstd "${zstd_args[@]}" -o "$output"; then
+    # Never leave a partial archive that a transfer stage could mistake for a
+    # valid backup. Make a best-effort restart before recording the failure.
     rm -f -- "$output"
     log ERROR "Archive failed; attempting to restart $project"
     compose up -d || true
+    restart_children "${stopped_children[@]}" || true
     return 1
   fi
   log ARCHIVE "Created $output"
   if is_true "$PULL_IMAGES"; then
+    # Image updates are useful but not allowed to prevent restart with the
+    # already-cached image when a registry is temporarily unavailable.
     compose pull || log WARN "Image pull failed for $project; starting with available images"
   fi
   compose up -d
+  restart_children "${stopped_children[@]}" || return 1
 
   if wait_for_project "$project"; then log OK "$project is running and healthy"; return 0; fi
   for (( attempt=1; attempt<=START_RETRIES; attempt++ )); do
@@ -119,6 +188,8 @@ backup_project() {
 
 declare -A SEEN=()
 failures=0
+# Continue through the list so one failed project does not leave later projects
+# without a backup. The stage still exits non-zero after reporting all failures.
 while IFS= read -r entry; do
   entry="${entry%%#*}"
   entry="${entry#"${entry%%[![:space:]]*}"}"

stages/20-rclone.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-# Copy the local backup tree to an rclone destination.
+# Stage 20: copy the local backup tree to an rclone destination.
+# Uses `rclone copy`, so files absent locally are not deleted remotely.
 set -Eeuo pipefail
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
 . "$SCRIPT_DIR/../lib/common.sh"
@@ -18,6 +19,8 @@ require_command rclone
 [[ -d "$BACKUP_ROOT" ]] || die "Backup root not found: $BACKUP_ROOT"
 read -r -a extra_args <<<"$RCLONE_EXTRA_ARGS"
 destination="${RCLONE_REMOTE%/}"
+# Append the host name for shared destinations, or disable RCLONE_HOST_SUBDIR
+# when RCLONE_REMOTE already points at this host's final directory.
 is_true "$RCLONE_HOST_SUBDIR" && destination="$destination/$BACKUP_HOSTNAME"
 args=(copy "$BACKUP_ROOT" "$destination" "${extra_args[@]}")
 (( DRY_RUN )) && args+=(--dry-run)

stages/30-smb.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-# Copy the local backup tree to an already-mounted SMB share.
+# Stage 30: copy the local backup tree to an already-mounted SMB share.
+# The script does not mount the share or store SMB credentials.
 set -Eeuo pipefail
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
 . "$SCRIPT_DIR/../lib/common.sh"
@@ -18,6 +19,8 @@ require_command rsync
 require_command mountpoint
 [[ -d "$BACKUP_ROOT" ]] || die "Backup root not found: $BACKUP_ROOT"
 [[ -d "$SMB_MOUNT" ]] || die "SMB mount does not exist: $SMB_MOUNT"
+# Refuse an ordinary directory at the configured path. This prevents a failed
+# network mount from silently filling the server's local root filesystem.
 mountpoint -q "$SMB_MOUNT" || die "SMB path is not a mounted filesystem: $SMB_MOUNT"
 destination="${SMB_MOUNT%/}"
 is_true "$SMB_HOST_SUBDIR" && destination="$destination/$BACKUP_HOSTNAME"

stages/90-prune.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-# Prune local per-project archives, retaining the newest N in each.
+# Stage 90: prune local archives after successful archive and transfer stages.
+# Retention is applied independently to each immediate project directory.
 set -Eeuo pipefail
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
 . "$SCRIPT_DIR/../lib/common.sh"
@@ -16,7 +17,9 @@ done
 require_positive_integer KEEP_COUNT "$KEEP_COUNT"
 [[ -d "$BACKUP_ROOT" ]] || { log SKIP "Backup root does not exist: $BACKUP_ROOT"; exit 0; }
 while IFS= read -r -d '' dir; do
-  # Match both current *.tar.zst archives and legacy *.zst archives.
+  # Match both current *.tar.zst and legacy *.zst names. NUL delimiters keep
+  # paths containing whitespace safe. Files are sorted by modification time,
+  # newest first; the timestamp embedded in a filename is not trusted.
   mapfile -d '' -t files < <(find "$dir" -maxdepth 1 -type f -name '*.zst' -printf '%T@ %p\0' | sort -z -nr | cut -z -d' ' -f2-)
   (( ${#files[@]} > KEEP_COUNT )) || continue
   for file in "${files[@]:KEEP_COUNT}"; do

stages/99-mqtt-report.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-# Publish retained backup run state for Home Assistant.
+# Stage 99: publish retained backup run state for Home Assistant.
+# Called once at pipeline start and again from backup.sh's EXIT trap.
 set -Eeuo pipefail
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
 . "$SCRIPT_DIR/../lib/common.sh"
@@ -11,9 +12,16 @@ STARTED_EPOCH="${3:-$(date +%s)}"
 FINISHED_EPOCH="${4:-$(date +%s)}"
 RUN_ID="${5:-unknown}"
 LOG_FILE="${6:-/dev/null}"
+LOCAL_ARCHIVE_STATUS="${BACKUP_LOCAL_ARCHIVE_STATUS:-unknown}"
+REMOTE_COPY_STATUS="${BACKUP_REMOTE_COPY_STATUS:-unknown}"
+RCLONE_STATUS="${BACKUP_RCLONE_STATUS:-unknown}"
+SMB_STATUS="${BACKUP_SMB_STATUS:-unknown}"
+REMOTE_COPY_FINISHED_AT="${BACKUP_REMOTE_COPY_FINISHED_AT:-}"
 
 require_command jq
 total=0; archived=0; dry_run=0; error_count=0
+# Derive counts and error messages from the shared run log. Keeping reporting
+# outside the work stages avoids coupling archive/transfer logic to MQTT.
 if [[ -r "$LOG_FILE" ]]; then
   total="$(grep -c '\[INFO\] Project:' "$LOG_FILE" || true)"
   archived="$(grep -c '\[ARCHIVE\] Created ' "$LOG_FILE" || true)"
@@ -24,6 +32,8 @@ else
   errors_json='[]'
 fi
 if (( EXIT_CODE != 0 && error_count == 0 )); then
+  # Some commands can exit non-zero before emitting a structured [ERROR] line.
+  # Supply a fallback so a failed result never reports an empty error list.
   errors_json="$(jq -cn --arg message "Pipeline exited with status $EXIT_CODE" '[ $message ]')"
   error_count=1
 fi
@@ -34,10 +44,14 @@ payload="$(jq -cn \
   --argjson finished "$FINISHED_EPOCH" --argjson duration "$((FINISHED_EPOCH - STARTED_EPOCH))" \
   --argjson total "$total" --argjson archived "$archived" --argjson dry_run "$dry_run" \
   --argjson error_count "$error_count" --argjson errors "$errors_json" \
-  '{host:$host,status:$status,run_id:$run_id,exit_code:$exit_code,
+  --arg local_archive_status "$LOCAL_ARCHIVE_STATUS" --arg remote_copy_status "$REMOTE_COPY_STATUS" \
+  --arg rclone_status "$RCLONE_STATUS" --arg smb_status "$SMB_STATUS" \
+  --arg remote_copy_finished_at "$REMOTE_COPY_FINISHED_AT" \
+  '{host:$host,status:$status,local_archive_status:$local_archive_status,remote_copy_status:$remote_copy_status,rclone_status:$rclone_status,smb_status:$smb_status,remote_copy_finished_at:(if $remote_copy_finished_at == "" then null else $remote_copy_finished_at end),run_id:$run_id,exit_code:$exit_code,
     started_at:($started|todateiso8601),finished_at:($finished|todateiso8601),
     duration_seconds:$duration,containers_total:$total,containers_backed_up:$archived,
     containers_dry_run:$dry_run,error_count:$error_count,errors:$errors}')"
 
+# Retention gives HA the latest known state after restarts or broker reconnects.
 mqtt_publish "$MQTT_STATUS_TOPIC" "$payload" 1
 log MQTT "Published $STATUS result to $MQTT_STATUS_TOPIC"

systemd/zorruno-docker-backup-mqtt.service

@@ -5,6 +5,8 @@ Wants=network-online.target
 
 [Service]
 Type=simple
+# Root is required because the listener launches a pipeline that stops and
+# starts Docker Compose projects and reads all project data.
 User=root
 WorkingDirectory=/backupdockerscripts
 ExecStart=/backupdockerscripts/bin/mqtt-trigger-listener.sh

tatus --short

@@ -1,63 +0,0 @@
-diff --git a/containers.txt b/containers.txt
-index 2105b35..b29a1fd 100644
---- a/containers.txt
-+++ b/containers.txt
-@@ -1,57 +1,9 @@
- # Container names
- 
--
--#bitnami
--#containers.panda.old
--#containers.panda.txt
--#dashbling
--#_disabled-chrome2mqtt
--#_disabled-chromecast-mqtt-connector
--#_disabled-huginn
--#_disabled-ipsec-vpn-server
--#_disabled-isd
--#_disabled-rtl-mqtt
--#_disabled-rtl-sdr
--#_disabled-speedy
--#_disabled-tasmoadmin
--#_disabled-tasmobackup
--zigbee2mqtt
--#_disabled-zigbee2mqttassistant
--#_disabled-zoneminder
- diun
--#docker-backup.sh
--#docker-compose.yml.sample
--docker-ntp
--#down-backup-up
--#evnex-mqtt
--gitea
- grafana
--#guacamole
--#homeassistant
--#influxdb
-+homeassistant
- influxdb2
--mediawiki
- mosquitto
--mqcontrol-scripts
--#mqtt-chromecast
--#mqttview
--mqtt-watchdog
--nodered
--nsupdater
--ombi
--#openvpn-as
- pihole
- portainer
--rtl-433tomqtt
--rustdesk
--#smashing
--speedtest
--#tasmota-device-manager
--tautulli
--telegraf
--traefik
--ttyd
--#tvheadend
--#weewx
--whoami
--wireguard