Track current backup pipeline and nested project handling

README.md

@@ -57,6 +57,9 @@ container name or a directory name beneath `/dockervolumes`. Blank lines and
 text after `#` are ignored. If several entries resolve to the same Compose
 project directory, that project is backed up only once. Keeping this host-local
 file out of Git prevents a pull on any server from replacing its local list.
+When a listed directory contains other running Compose projects, the archive
+stage also stops those nested projects before archiving and starts them again
+afterward. Intentionally stopped nested projects are left stopped.
 
 ## Configuration
 

backup.sh

@@ -26,6 +26,26 @@ while [[ $# -gt 0 ]]; do
   shift
 done
 
+if (( SKIP_CONTAINERS )); then
+  BACKUP_LOCAL_ARCHIVE_STATUS="skipped"
+else
+  BACKUP_LOCAL_ARCHIVE_STATUS="running"
+fi
+BACKUP_RCLONE_STATUS="not_configured"
+BACKUP_SMB_STATUS="not_configured"
+for configured_method in $BACKUP_METHODS; do
+  case "$configured_method" in
+    rclone) if is_true "$RCLONE_ENABLED"; then BACKUP_RCLONE_STATUS="pending"; else BACKUP_RCLONE_STATUS="skipped"; fi ;;
+    smb) if is_true "$SMB_ENABLED"; then BACKUP_SMB_STATUS="pending"; else BACKUP_SMB_STATUS="skipped"; fi ;;
+  esac
+done
+BACKUP_REMOTE_COPY_STATUS="pending"
+if [[ "$BACKUP_RCLONE_STATUS" == "not_configured" || "$BACKUP_RCLONE_STATUS" == "skipped" ]] && [[ "$BACKUP_SMB_STATUS" == "not_configured" || "$BACKUP_SMB_STATUS" == "skipped" ]]; then
+  BACKUP_REMOTE_COPY_STATUS="skipped"
+fi
+BACKUP_REMOTE_COPY_FINISHED_AT=""
+export BACKUP_LOCAL_ARCHIVE_STATUS BACKUP_REMOTE_COPY_STATUS BACKUP_RCLONE_STATUS BACKUP_SMB_STATUS BACKUP_REMOTE_COPY_FINISHED_AT
+
 BACKUP_RUN_ID="$(run_id)"
 BACKUP_STARTED_EPOCH="$(date +%s)"
 # Every script sourced through this process appends structured log lines here.
@@ -34,12 +54,51 @@ BACKUP_RUN_LOG="$(mktemp "${TMPDIR:-/tmp}/zorruno-docker-backup.XXXXXX.log")"
 export BACKUP_RUN_ID BACKUP_STARTED_EPOCH BACKUP_RUN_LOG
 
 finish_run() {
-  local exit_code=$? finished status
+  local exit_code=$? finished status archived_count remote_success=0 remote_failure=0 remote_running=0 remote_dry_run=0 transfer_status
   # Prevent recursion when this function exits with the pipeline's status.
   trap - EXIT
   # Reporting is best-effort and must never replace the real pipeline result.
   set +e
   finished="$(date +%s)"
+  if [[ "$BACKUP_LOCAL_ARCHIVE_STATUS" == "running" ]]; then
+    if (( DRY_RUN )); then
+      BACKUP_LOCAL_ARCHIVE_STATUS="dry_run"
+    elif (( exit_code == 0 )); then
+      BACKUP_LOCAL_ARCHIVE_STATUS="success"
+    else
+      archived_count="$(grep -c "\[ARCHIVE\] Created " "$BACKUP_RUN_LOG" || true)"
+      if (( archived_count > 0 )); then BACKUP_LOCAL_ARCHIVE_STATUS="partial"; else BACKUP_LOCAL_ARCHIVE_STATUS="failed"; fi
+    fi
+  fi
+  for transfer_status in BACKUP_RCLONE_STATUS BACKUP_SMB_STATUS; do
+    if [[ "${!transfer_status}" == "running" ]]; then
+      if (( DRY_RUN )); then printf -v "$transfer_status" "%s" "dry_run"; else printf -v "$transfer_status" "%s" "failed"; fi
+    elif [[ "${!transfer_status}" == "pending" ]]; then
+      printf -v "$transfer_status" "%s" "not_attempted"
+    fi
+  done
+  for transfer_status in "$BACKUP_RCLONE_STATUS" "$BACKUP_SMB_STATUS"; do
+    case "$transfer_status" in
+      success) (( remote_success++ )) ;;
+      partial) (( remote_success++ )); (( remote_failure++ )) ;;
+      failed) (( remote_failure++ )) ;;
+      running|pending) (( remote_running++ )) ;;
+      dry_run) (( remote_dry_run++ )) ;;
+    esac
+  done
+  if (( remote_running > 0 )); then
+    BACKUP_REMOTE_COPY_STATUS="running"
+  elif (( remote_failure > 0 && remote_success > 0 )); then
+    BACKUP_REMOTE_COPY_STATUS="partial"
+  elif (( remote_failure > 0 )); then
+    BACKUP_REMOTE_COPY_STATUS="failed"
+  elif (( remote_success > 0 )); then
+    BACKUP_REMOTE_COPY_STATUS="success"
+  elif (( remote_dry_run > 0 )); then
+    BACKUP_REMOTE_COPY_STATUS="dry_run"
+  else
+    BACKUP_REMOTE_COPY_STATUS="skipped"
+  fi
   if (( exit_code == 0 )); then
     if (( DRY_RUN )); then status="dry_run_success"; else status="success"; fi
   else
@@ -71,7 +130,12 @@ if is_true "$MQTT_REPORT_ENABLED"; then
     "$BACKUP_STARTED_EPOCH" "$BACKUP_RUN_ID" "$BACKUP_RUN_LOG" \
     || log WARN "Unable to publish MQTT running state; backup will continue"
 fi
-(( SKIP_CONTAINERS )) || "$SCRIPT_DIR/stages/10-containers.sh" "${stage_args[@]}"
+if (( SKIP_CONTAINERS )); then
+  BACKUP_LOCAL_ARCHIVE_STATUS="skipped"
+else
+  "$SCRIPT_DIR/stages/10-containers.sh" "${stage_args[@]}"
+  if (( DRY_RUN )); then BACKUP_LOCAL_ARCHIVE_STATUS="dry_run"; else BACKUP_LOCAL_ARCHIVE_STATUS="success"; fi
+fi
 method_args=(); (( DRY_RUN )) && method_args+=(--dry-run)
 # BACKUP_METHODS contains short names (for example: "rclone smb"). Numeric
 # prefixes control directory ordering without becoming part of configuration.
@@ -79,7 +143,22 @@ for method in $BACKUP_METHODS; do
   method_script=""
   while IFS= read -r candidate; do method_script="$candidate"; break; done < <(find "$SCRIPT_DIR/stages" -maxdepth 1 -type f -name "*-${method}.sh" -print | sort)
   [[ -n "$method_script" ]] || die "Configured backup method not found: $method"
+  case "$method" in
+    rclone) if is_true "$RCLONE_ENABLED"; then BACKUP_RCLONE_STATUS="running"; fi ;;
+    smb) if is_true "$SMB_ENABLED"; then BACKUP_SMB_STATUS="running"; fi ;;
+  esac
   "$method_script" "${method_args[@]}"
+  if (( DRY_RUN )); then
+    case "$method" in
+      rclone) [[ "$BACKUP_RCLONE_STATUS" == "running" ]] && BACKUP_RCLONE_STATUS="dry_run" ;;
+      smb) [[ "$BACKUP_SMB_STATUS" == "running" ]] && BACKUP_SMB_STATUS="dry_run" ;;
+    esac
+  else
+    case "$method" in
+      rclone) if [[ "$BACKUP_RCLONE_STATUS" == "running" ]]; then BACKUP_RCLONE_STATUS="success"; BACKUP_REMOTE_COPY_FINISHED_AT="$(date -u +%FT%TZ)"; fi ;;
+      smb) if [[ "$BACKUP_SMB_STATUS" == "running" ]]; then BACKUP_SMB_STATUS="success"; BACKUP_REMOTE_COPY_FINISHED_AT="$(date -u +%FT%TZ)"; fi ;;
+    esac
+  fi
 done
 (( SKIP_PRUNE )) || "$SCRIPT_DIR/stages/90-prune.sh" "${method_args[@]}"
 log DONE "Backup pipeline completed successfully"

stages/10-containers.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-# Stage 10: create a cold, compressed archive of each selected Compose project.
+# Stage 10: create a cold, compressed archive of each selected Compose project
+# directory, including any nested Compose projects.
 #
 # Entries may be Compose project-directory names or live container names. A
 # container name is resolved through its Compose labels. Duplicate entries that
@@ -85,9 +86,41 @@ wait_for_project() {
   done
 }
 
+discover_child_projects() {
+  local root="$1" id row project working_dir
+  local -A seen=()
+  # Discover live Compose projects by their container labels. Only running
+  # projects are stopped/restarted, preserving intentionally stopped stacks.
+  while IFS= read -r id; do
+    [[ -n "$id" ]] || continue
+    row="$(docker inspect -f '{{printf "%s\t%s" (index .Config.Labels "com.docker.compose.project") (index .Config.Labels "com.docker.compose.project.working_dir")}}' "$id" 2>/dev/null || true)"
+    IFS=$'\t' read -r project working_dir <<<"$row"
+    [[ -n "$project" && -n "$working_dir" ]] || continue
+    [[ "$working_dir" == "$root"/* ]] || continue
+    [[ -n "${seen[$working_dir]:-}" ]] && continue
+    seen[$working_dir]=1
+    printf '%s\t%s\n' "$project" "$working_dir"
+  done < <(docker ps -q)
+}
+
+restart_children() {
+  local -a projects=("$@")
+  local record child_project child_dir failed=0
+  for record in "${projects[@]}"; do
+    IFS=$'\t' read -r child_project child_dir <<<"$record"
+    log INFO "Restarting nested project: $child_project ($child_dir)"
+    if ! (cd "$child_dir" && compose up -d); then
+      log ERROR "Unable to restart nested project $child_project"
+      failed=1
+    fi
+  done
+  return "$failed"
+}
+
 backup_project() {
-  local project="$1" working_dir="$2" parent base output_dir output attempt
+  local project="$1" working_dir="$2" parent base output_dir output attempt child_project child_dir record
   local -a zstd_args=()
+  local -a child_records=() stopped_children=()
   parent="$(dirname "$working_dir")"
   base="$(basename "$working_dir")"
   output_dir="$BACKUP_ROOT/$project"
@@ -95,14 +128,34 @@ backup_project() {
   log INFO "Project: $project ($working_dir)"
   log INFO "Archive: $output"
   if (( DRY_RUN )); then
-    log DRY "Would stop, archive, optionally pull, start, and verify $project"
+    mapfile -t child_records < <(discover_child_projects "$working_dir")
+    log DRY "Would stop ${#child_records[@]} nested project(s), archive, then restart them with $project"
     return 0
   fi
 
   mkdir -p "$output_dir"
+  mapfile -t child_records < <(discover_child_projects "$working_dir")
+  for record in "${child_records[@]}"; do
+    IFS=$'\t' read -r child_project child_dir <<<"$record"
+    log INFO "Stopping nested project: $child_project ($child_dir)"
+    # Track before down: Compose may partially stop a project and then fail.
+    stopped_children+=("$record")
+    if (cd "$child_dir" && compose down); then
+      :
+    else
+      log ERROR "Unable to stop nested project $child_project"
+      restart_children "${stopped_children[@]}" || true
+      return 1
+    fi
+  done
   cd "$working_dir"
   # Stopping the whole project creates a consistent filesystem-level backup.
-  compose down
+  if ! compose down; then
+    log ERROR "Unable to stop project $project"
+    compose up -d || log ERROR "Unable to restart project $project"
+    restart_children "${stopped_children[@]}" || true
+    return 1
+  fi
   read -r -a zstd_args <<<"$ZSTD_EXTRA_OPTS"
   if ! tar --one-file-system -cf - -C "$parent" "$base" | zstd "${zstd_args[@]}" -o "$output"; then
     # Never leave a partial archive that a transfer stage could mistake for a
@@ -110,6 +163,7 @@ backup_project() {
     rm -f -- "$output"
     log ERROR "Archive failed; attempting to restart $project"
     compose up -d || true
+    restart_children "${stopped_children[@]}" || true
     return 1
   fi
   log ARCHIVE "Created $output"
@@ -119,6 +173,7 @@ backup_project() {
     compose pull || log WARN "Image pull failed for $project; starting with available images"
   fi
   compose up -d
+  restart_children "${stopped_children[@]}" || return 1
 
   if wait_for_project "$project"; then log OK "$project is running and healthy"; return 0; fi
   for (( attempt=1; attempt<=START_RETRIES; attempt++ )); do

stages/99-mqtt-report.sh

@@ -12,6 +12,11 @@ STARTED_EPOCH="${3:-$(date +%s)}"
 FINISHED_EPOCH="${4:-$(date +%s)}"
 RUN_ID="${5:-unknown}"
 LOG_FILE="${6:-/dev/null}"
+LOCAL_ARCHIVE_STATUS="${BACKUP_LOCAL_ARCHIVE_STATUS:-unknown}"
+REMOTE_COPY_STATUS="${BACKUP_REMOTE_COPY_STATUS:-unknown}"
+RCLONE_STATUS="${BACKUP_RCLONE_STATUS:-unknown}"
+SMB_STATUS="${BACKUP_SMB_STATUS:-unknown}"
+REMOTE_COPY_FINISHED_AT="${BACKUP_REMOTE_COPY_FINISHED_AT:-}"
 
 require_command jq
 total=0; archived=0; dry_run=0; error_count=0
@@ -39,7 +44,10 @@ payload="$(jq -cn \
   --argjson finished "$FINISHED_EPOCH" --argjson duration "$((FINISHED_EPOCH - STARTED_EPOCH))" \
   --argjson total "$total" --argjson archived "$archived" --argjson dry_run "$dry_run" \
   --argjson error_count "$error_count" --argjson errors "$errors_json" \
-  '{host:$host,status:$status,run_id:$run_id,exit_code:$exit_code,
+  --arg local_archive_status "$LOCAL_ARCHIVE_STATUS" --arg remote_copy_status "$REMOTE_COPY_STATUS" \
+  --arg rclone_status "$RCLONE_STATUS" --arg smb_status "$SMB_STATUS" \
+  --arg remote_copy_finished_at "$REMOTE_COPY_FINISHED_AT" \
+  '{host:$host,status:$status,local_archive_status:$local_archive_status,remote_copy_status:$remote_copy_status,rclone_status:$rclone_status,smb_status:$smb_status,remote_copy_finished_at:(if $remote_copy_finished_at == "" then null else $remote_copy_finished_at end),run_id:$run_id,exit_code:$exit_code,
     started_at:($started|todateiso8601),finished_at:($finished|todateiso8601),
     duration_seconds:$duration,containers_total:$total,containers_backed_up:$archived,
     containers_dry_run:$dry_run,error_count:$error_count,errors:$errors}')"